;
+ },
+ valueGetter: (unused: any, row: any) => {
+ return row[c];
+ },
+ renderCell: (params: GridRenderCellParams) => {
+ return (
+
+
+
+ );
+ },
+ };
+ }
+ );
+ cols.push(...scoreColumns);
+ }
+
cols.push({
field: 'wb_user_id',
headerName: 'User',
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
index 4060735cc67a..5f84121f0889 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
@@ -4,6 +4,7 @@ import {
TEAL_500,
TEAL_600,
} from '@wandb/weave/common/css/color.styles';
+import {WeaveObjectRef} from '@wandb/weave/react';
import React from 'react';
import {Link as LinkComp, useHistory} from 'react-router-dom';
import styled, {css} from 'styled-components';
@@ -272,6 +273,47 @@ export const OpVersionLink: React.FC<{
);
};
+export const CallRefLink: React.FC<{
+ callRef: WeaveObjectRef;
+}> = props => {
+ const history = useHistory();
+ const {peekingRouter} = useWeaveflowRouteContext();
+ const callId = props.callRef.artifactName;
+ const to = peekingRouter.callUIUrl(
+ props.callRef.entityName,
+ props.callRef.projectName,
+ '',
+ callId
+ );
+ const onClick = () => {
+ history.push(to);
+ };
+
+ if (props.callRef.weaveKind !== 'call') {
+ return null;
+ }
+
+ return (
+
+
+
+
+
+
+
+
+
+ );
+};
+
export const CallLink: React.FC<{
entityName: string;
projectName: string;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
index e039747042cb..7c89efd44196 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
@@ -165,6 +165,7 @@ export type FeedbackQueryReq = {
export type Feedback = {
id: string;
+ project_id: string;
weave_ref: string;
wb_user_id: string; // authenticated creator username
creator: string | null; // display name
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts
index 6b08bbba26f4..34496ffeed04 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts
@@ -275,7 +275,8 @@ export type WFDataModelHooksInterface = {
useFeedback: (
key: FeedbackKey | null,
sortBy?: traceServerClientTypes.SortBy[]
- ) => LoadableWithError & Refetchable;
+ ) => LoadableWithError &
+ Refetchable;
useTableUpdate: () => (
projectId: string,
baseDigest: string,
diff --git a/weave/flow/eval.py b/weave/flow/eval.py
index 7400fc45ce38..f540dfb1dfe8 100644
--- a/weave/flow/eval.py
+++ b/weave/flow/eval.py
@@ -27,8 +27,6 @@
get_scorer_attributes,
transpose,
)
-from weave.scorers.base_scorer import apply_scorer_async
-from weave.trace.context.weave_client_context import get_weave_client
from weave.trace.env import get_weave_parallelism
from weave.trace.errors import OpCallError
from weave.trace.isinstance import weave_isinstance
@@ -209,22 +207,8 @@ async def predict_and_score(self, model: Union[Op, Model], example: dict) -> dic
scorers = self._post_init_scorers
for scorer in scorers:
- apply_scorer_result = await apply_scorer_async(
- scorer, example, model_output
- )
+ apply_scorer_result = await model_call.apply_scorer(scorer, example)
result = apply_scorer_result.result
- score_call = apply_scorer_result.score_call
-
- wc = get_weave_client()
- if wc:
- scorer_ref_uri = None
- if weave_isinstance(scorer, Scorer):
- # Very important: if the score is generated from a Scorer subclass,
- # then scorer_ref_uri will be None, and we will use the op_name from
- # the score_call instead.
- scorer_ref = get_ref(scorer)
- scorer_ref_uri = scorer_ref.uri() if scorer_ref else None
- wc._send_score_call(model_call, score_call, scorer_ref_uri)
scorer_attributes = get_scorer_attributes(scorer)
scorer_name = scorer_attributes.scorer_name
scores[scorer_name] = result
diff --git a/weave/scorers/base_scorer.py b/weave/scorers/base_scorer.py
index 462246914f5b..9616fb0c4500 100644
--- a/weave/scorers/base_scorer.py
+++ b/weave/scorers/base_scorer.py
@@ -185,7 +185,7 @@ class ApplyScorerSuccess:
async def apply_scorer_async(
- scorer: Union[Op, Scorer], example: dict, model_output: dict
+ scorer: Union[Op, Scorer], example: dict, model_output: Any
) -> ApplyScorerResult:
"""Apply a scoring function to model output and example data asynchronously.
diff --git a/weave/trace/op.py b/weave/trace/op.py
index ea3f0c66dc0c..bf3ee58f7a79 100644
--- a/weave/trace/op.py
+++ b/weave/trace/op.py
@@ -417,15 +417,19 @@ def _do_call(
# Handle all of the possible cases where we would skip tracing.
if settings.should_disable_weave():
res = func(*pargs.args, **pargs.kwargs)
+ call.output = res
return res, call
if weave_client_context.get_weave_client() is None:
res = func(*pargs.args, **pargs.kwargs)
+ call.output = res
return res, call
if not op._tracing_enabled:
res = func(*pargs.args, **pargs.kwargs)
+ call.output = res
return res, call
if not get_tracing_enabled():
res = func(*pargs.args, **pargs.kwargs)
+ call.output = res
return res, call
current_call = call_context.get_current_call()
@@ -435,6 +439,7 @@ def _do_call(
# Disable tracing for this call and all descendants
with tracing_disabled():
res = func(*pargs.args, **pargs.kwargs)
+ call.output = res
return res, call
# Proceed with tracing. Note that we don't check the sample rate here.
@@ -478,15 +483,19 @@ async def _do_call_async(
# Handle all of the possible cases where we would skip tracing.
if settings.should_disable_weave():
res = await func(*args, **kwargs)
+ call.output = res
return res, call
if weave_client_context.get_weave_client() is None:
res = await func(*args, **kwargs)
+ call.output = res
return res, call
if not op._tracing_enabled:
res = await func(*args, **kwargs)
+ call.output = res
return res, call
if not get_tracing_enabled():
res = await func(*args, **kwargs)
+ call.output = res
return res, call
current_call = call_context.get_current_call()
@@ -496,6 +505,7 @@ async def _do_call_async(
# Disable tracing for this call and all descendants
with tracing_disabled():
res = await func(*args, **kwargs)
+ call.output = res
return res, call
# Proceed with tracing
diff --git a/weave/trace/weave_client.py b/weave/trace/weave_client.py
index 6decbc12cffc..0754b9f57eb5 100644
--- a/weave/trace/weave_client.py
+++ b/weave/trace/weave_client.py
@@ -2,7 +2,6 @@
import dataclasses
import datetime
-import inspect
import logging
import platform
import re
@@ -10,7 +9,16 @@
from collections.abc import Iterator, Sequence
from concurrent.futures import Future
from functools import lru_cache
-from typing import Any, Callable, Generic, Protocol, TypeVar, cast, overload
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Callable,
+ Generic,
+ Protocol,
+ TypeVar,
+ cast,
+ overload,
+)
import pydantic
from requests import HTTPError
@@ -22,6 +30,7 @@
from weave.trace.context import weave_client_context as weave_client_context
from weave.trace.exception import exception_to_json_str
from weave.trace.feedback import FeedbackQuery, RefFeedbackQuery
+from weave.trace.isinstance import weave_isinstance
from weave.trace.object_record import (
ObjectRecord,
dataclass_object_record,
@@ -83,6 +92,10 @@
)
from weave.trace_server_bindings.remote_http_trace_server import RemoteHTTPTraceServer
+if TYPE_CHECKING:
+ from weave.scorers.base_scorer import ApplyScorerResult, Scorer
+
+
# Controls if objects can have refs to projects not the WeaveClient project.
# If False, object refs with with mismatching projects will be recreated.
# If True, use existing ref to object in other project.
@@ -445,43 +458,60 @@ def set_display_name(self, name: str | None) -> None:
def remove_display_name(self) -> None:
self.set_display_name(None)
- def _apply_scorer(self, scorer_op: Op) -> None:
+ async def apply_scorer(
+ self, scorer: Op | Scorer, additional_scorer_kwargs: dict | None = None
+ ) -> ApplyScorerResult:
"""
- This is a private method that applies a scorer to a call and records the feedback.
- In the near future, this will be made public, but for now it is only used internally
- for testing.
+ `apply_scorer` is a method that applies a Scorer to a Call. This is useful
+ for guarding application logic with a scorer and/or monitoring the quality
+ of critical ops. Scorers are automatically logged to Weave as Feedback and
+ can be used in queries & analysis.
+
+ Args:
+ scorer: The Scorer to apply.
+ additional_scorer_kwargs: Additional kwargs to pass to the scorer. This is
+ useful for passing in additional context that is not part of the call
+ inputs.useful for passing in additional context that is not part of the call
+ inputs.
- Before making this public, we should refactor such that the `predict_and_score` method
- inside `eval.py` uses this method inside the scorer block.
+ Returns:
+ The result of the scorer application in the form of an `ApplyScorerResult`.
- Current limitations:
- - only works for ops (not Scorer class)
- - no async support
- - no context yet (ie. ground truth)
+ ```python
+ class ApplyScorerSuccess:
+ result: Any
+ score_call: Call
+ ```
+
+ Example usage:
+
+ ```python
+ my_scorer = ... # construct a scorer
+ prediction, prediction_call = my_op.call(input_data)
+ result, score_call = prediction.apply_scorer(my_scorer)
+ ```
"""
- client = weave_client_context.require_weave_client()
- scorer_signature = inspect.signature(scorer_op)
- scorer_arg_names = list(scorer_signature.parameters.keys())
- score_args = {k: v for k, v in self.inputs.items() if k in scorer_arg_names}
- if "output" in scorer_arg_names:
- score_args["output"] = self.output
- _, score_call = scorer_op.call(**score_args)
- scorer_op_ref = get_ref(scorer_op)
- if scorer_op_ref is None:
- raise ValueError("Scorer op has no ref")
- self_ref = get_ref(self)
- if self_ref is None:
- raise ValueError("Call has no ref")
- score_results = score_call.output
- score_call_ref = get_ref(score_call)
- if score_call_ref is None:
- raise ValueError("Score call has no ref")
- client._add_runnable_feedback(
- weave_ref_uri=self_ref.uri(),
- output=score_results,
- call_ref_uri=score_call_ref.uri(),
- runnable_ref_uri=scorer_op_ref.uri(),
- )
+ from weave.scorers.base_scorer import Scorer, apply_scorer_async
+
+ model_inputs = {k: v for k, v in self.inputs.items() if k != "self"}
+ example = {**model_inputs, **(additional_scorer_kwargs or {})}
+ output = self.output
+ if isinstance(output, ObjectRef):
+ output = output.get()
+ apply_scorer_result = await apply_scorer_async(scorer, example, output)
+ score_call = apply_scorer_result.score_call
+
+ wc = weave_client_context.get_weave_client()
+ if wc:
+ scorer_ref_uri = None
+ if weave_isinstance(scorer, Scorer):
+ # Very important: if the score is generated from a Scorer subclass,
+ # then scorer_ref_uri will be None, and we will use the op_name from
+ # the score_call instead.
+ scorer_ref = get_ref(scorer)
+ scorer_ref_uri = scorer_ref.uri() if scorer_ref else None
+ wc._send_score_call(self, score_call, scorer_ref_uri)
+ return apply_scorer_result
def make_client_call(
@@ -807,6 +837,7 @@ def create_call(
trace_id=trace_id,
parent_id=parent_id,
id=call_id,
+ # It feels like this should be inputs_postprocessed, not the refs.
inputs=inputs_with_refs,
attributes=attributes,
)
@@ -875,8 +906,8 @@ def finish_call(
postprocessed_output = _global_postprocess_output(postprocessed_output)
self._save_nested_objects(postprocessed_output)
-
- call.output = map_to_refs(postprocessed_output)
+ output_as_refs = map_to_refs(postprocessed_output)
+ call.output = postprocessed_output
# Summary handling
summary = {}
@@ -931,7 +962,7 @@ def finish_call(
op._on_finish_handler(call, original_output, exception)
def send_end_call() -> None:
- output_json = to_json(call.output, project_id, self, use_dictify=False)
+ output_json = to_json(output_as_refs, project_id, self, use_dictify=False)
self.server.call_end(
CallEndReq(
end=EndedCallSchemaForInsert(