fix(weave): Allow back-compat for "old-style" scorers and evaluations using model_output kwarg

tests/trace/test_evaluate.py

@@ -4,7 +4,6 @@
 
 import weave
 from weave import Dataset, Evaluation, Model
-from weave.scorers import MultiTaskBinaryClassificationF1
 
 dataset_rows = [{"input": "1 + 2", "target": 3}, {"input": "2**4", "target": 15}]
 dataset = Dataset(rows=dataset_rows)
@@ -153,29 +152,3 @@ def score(self, target, output):
             "mean": pytest.approx(0, abs=1),
         },
     }
-
-
-def test_multiclass_f1_score(client):
-    evaluation = Evaluation(
-        dataset=[{"target": {"a": False, "b": True}, "pred": {"a": True, "b": False}}],
-        scorers=[MultiTaskBinaryClassificationF1(class_names=["a", "b"])],
-    )
-
-    @weave.op()
-    def return_pred(pred):
-        return pred
-
-    result = asyncio.run(evaluation.evaluate(return_pred))
-    assert result == {
-        "output": {
-            "a": {"true_count": 1, "true_fraction": 1.0},
-            "b": {"true_count": 0, "true_fraction": 0.0},
-        },
-        "MultiTaskBinaryClassificationF1": {
-            "a": {"f1": 0, "precision": 0.0, "recall": 0},
-            "b": {"f1": 0, "precision": 0, "recall": 0.0},
-        },
-        "model_latency": {
-            "mean": pytest.approx(0, abs=1),
-        },
-    }

tests/trace/test_evaluate_oldstyle.py

@@ -0,0 +1,181 @@
+import asyncio
+
+import pytest
+
+import weave
+from weave import Dataset, Evaluation, Model
+from weave.scorers import MultiTaskBinaryClassificationF1
+
+dataset_rows = [{"input": "1 + 2", "target": 3}, {"input": "2**4", "target": 15}]
+dataset = Dataset(rows=dataset_rows)
+
+
+expected_eval_result = {
+    "model_output": {"mean": 9.5},
+    "score_oldstyle": {"true_count": 1, "true_fraction": 0.5},
+    "model_latency": {"mean": pytest.approx(0, abs=1)},
+}
+
+
+class EvalModel(Model):
+    @weave.op()
+    async def predict(self, input) -> str:
+        return eval(input)
+
+
+@weave.op()
+def score_oldstyle(model_output, target):
+    return model_output == target
+
+
+@weave.op()
+def example_to_model_input(example):
+    return {"input": example["input"]}
+
+
+def test_evaluate_callable_as_model(client):
+    @weave.op()
+    async def model_predict(input) -> str:
+        return eval(input)
+
+    evaluation = Evaluation(
+        dataset=dataset_rows,
+        scorers=[score_oldstyle],
+    )
+    result = asyncio.run(evaluation.evaluate(model_predict))
+    assert result == expected_eval_result
+
+
+def test_predict_can_receive_other_params(client):
+    @weave.op()
+    async def model_predict(input, target) -> str:
+        return eval(input) + target
+
+    evaluation = Evaluation(
+        dataset=dataset_rows,
+        scorers=[score_oldstyle],
+    )
+    result = asyncio.run(evaluation.evaluate(model_predict))
+    assert result == {
+        "model_output": {"mean": 18.5},
+        "score_oldstyle": {"true_count": 0, "true_fraction": 0.0},
+        "model_latency": {
+            "mean": pytest.approx(0, abs=1),
+        },
+    }
+
+
+def test_can_preprocess_model_input(client):
+    @weave.op()
+    async def model_predict(x) -> str:
+        return eval(x)
+
+    @weave.op()
+    def preprocess(example):
+        return {"x": example["input"]}
+
+    evaluation = Evaluation(
+        dataset=dataset_rows,
+        scorers=[score_oldstyle],
+        preprocess_model_input=preprocess,
+    )
+    result = asyncio.run(evaluation.evaluate(model_predict))
+    assert result == expected_eval_result
+
+
+def test_evaluate_rows_only(client):
+    evaluation = Evaluation(
+        dataset=dataset_rows,
+        scorers=[score_oldstyle],
+    )
+    model = EvalModel()
+    result = asyncio.run(evaluation.evaluate(model))
+    assert result == expected_eval_result
+
+
+def test_evaluate_other_model_method_names():
+    class EvalModel(Model):
+        @weave.op()
+        async def infer(self, input) -> str:
+            return eval(input)
+
+    evaluation = Evaluation(
+        dataset=dataset_rows,
+        scorers=[score_oldstyle],
+    )
+    model = EvalModel()
+    result = asyncio.run(evaluation.evaluate(model))
+    assert result == expected_eval_result
+
+
+def test_score_as_class(client):
+    class MyScorerOldstyle(weave.Scorer):
+        @weave.op()
+        def score(self, model_output, target):
+            return model_output == target
+
+    evaluation = Evaluation(
+        dataset=dataset_rows,
+        scorers=[MyScorerOldstyle()],
+    )
+    model = EvalModel()
+    result = asyncio.run(evaluation.evaluate(model))
+    assert result == {
+        "model_output": {"mean": 9.5},
+        "MyScorerOldstyle": {"true_count": 1, "true_fraction": 0.5},
+        "model_latency": {
+            "mean": pytest.approx(0, abs=1),
+        },
+    }
+
+
+def test_score_with_custom_summarize(client):
+    class MyScorerOldstyle(weave.Scorer):
+        @weave.op()
+        def summarize(self, score_rows):
+            assert list(score_rows) == [True, False]
+            return {"awesome": 3}
+
+        @weave.op()
+        def score(self, model_output, target):
+            return model_output == target
+
+    evaluation = Evaluation(
+        dataset=dataset_rows,
+        scorers=[MyScorerOldstyle()],
+    )
+    model = EvalModel()
+    result = asyncio.run(evaluation.evaluate(model))
+    assert result == {
+        "model_output": {"mean": 9.5},
+        "MyScorerOldstyle": {"awesome": 3},
+        "model_latency": {
+            "mean": pytest.approx(0, abs=1),
+        },
+    }
+
+
+def test_multiclass_f1_score(client):
+    evaluation = Evaluation(
+        dataset=[{"target": {"a": False, "b": True}, "pred": {"a": True, "b": False}}],
+        scorers=[MultiTaskBinaryClassificationF1(class_names=["a", "b"])],
+    )
+
+    @weave.op()
+    def return_pred(pred):
+        return pred
+
+    result = asyncio.run(evaluation.evaluate(return_pred))
+    assert result == {
+        "model_output": {
+            "a": {"true_count": 1, "true_fraction": 1.0},
+            "b": {"true_count": 0, "true_fraction": 0.0},
+        },
+        "MultiTaskBinaryClassificationF1": {
+            "a": {"f1": 0, "precision": 0.0, "recall": 0},
+            "b": {"f1": 0, "precision": 0, "recall": 0.0},
+        },
+        "model_latency": {
+            "mean": pytest.approx(0, abs=1),
+        },
+    }

weave/flow/eval.py

@@ -1,10 +1,12 @@
 import asyncio
 import inspect
+import logging
 import textwrap
 import time
 import traceback
 from typing import Any, Callable, Coroutine, Optional, Union, cast
 
+from pydantic import PrivateAttr
 from rich import print
 from rich.console import Console
 
@@ -28,7 +30,7 @@
 from weave.trace.weave_client import Call, get_ref
 
 console = Console()
-
+logger = logging.getLogger(__name__)
 
 INVALID_MODEL_ERROR = (
     "`Evaluation.evaluate` requires a `Model` or `Op` instance as the `model` argument. "
@@ -96,7 +98,7 @@ def function_to_evaluate(question: str):
 
     # Score your examples using scoring functions
     evaluation = Evaluation(
-        dataset=examples, scorers=[match_score1]
+        dataset=examples, scorers=[match_score1], output_key="generated_text"
     )
 
     # Start tracking the evaluation
@@ -111,6 +113,8 @@ def function_to_evaluate(question: str):
     preprocess_model_input: Optional[Callable] = None
     trials: int = 1
 
+    _output_key: str = PrivateAttr("output")
+
     def model_post_init(self, __context: Any) -> None:
         scorers: list[Union[Callable, Scorer, Op]] = []
         for scorer in self.scorers or []:
@@ -339,7 +343,12 @@ async def predict_and_score(
                     raise ValueError(
                         f"{score_fn} expects arguments: {score_arg_names}, provide a preprocess_model_input function that returns a dict with those keys."
                     )
-            score_args["output"] = model_output
+            if "model_output" in score_arg_names:
+                util.warn_once(
+                    logger, "model_output is deprecated, please use output instead"
+                )
+                self._output_key = "model_output"
+            score_args[self._output_key] = model_output
 
             try:
                 if is_op(score_fn) and model_call:
@@ -370,7 +379,7 @@ async def predict_and_score(
                     for param in score_signature.parameters.values()
                     if param.default == inspect.Parameter.empty
                 ]
-                required_arg_names.remove("output")
+                required_arg_names.remove(self._output_key)
 
                 message = textwrap.dedent(
                     f"""
@@ -397,7 +406,7 @@ async def predict_and_score(
             scores[scorer_name] = result
 
         return {
-            "output": model_output,
+            self._output_key: model_output,
             "scores": scores,
             "model_latency": model_latency,
         }
@@ -421,7 +430,6 @@ async def summarize(self, eval_table: EvaluationResults) -> dict:
                 model_output_summary = auto_summarize(vals)
                 if model_output_summary:
                     summary[name] = model_output_summary
-
         return summary
 
     async def get_eval_results(
@@ -441,7 +449,7 @@ async def eval_example(example: dict) -> dict:
             except Exception as e:
                 print("Predict and score failed")
                 traceback.print_exc()
-                return {"output": None, "scores": {}}
+                return {self._output_key: None, "scores": {}}
             return eval_row
 
         n_complete = 0
@@ -458,7 +466,7 @@ async def eval_example(example: dict) -> dict:
             #     f"Evaluating... {duration:.2f}s [{n_complete} / {len(self.dataset.rows)} complete]"  # type:ignore
             # )
             if eval_row is None:
-                eval_row = {"output": None, "scores": {}}
+                eval_row = {self._output_key: None, "scores": {}}
             else:
                 eval_row["scores"] = eval_row.get("scores", {})
             for scorer in self.scorers or []:

weave/flow/util.py

@@ -1,10 +1,13 @@
 import asyncio
+import logging
 import multiprocessing
 from typing import Any, AsyncIterator, Awaitable, Callable, Iterable, Tuple, TypeVar
 
 T = TypeVar("T")
 U = TypeVar("U")
 
+_shown_warnings = set()
+
 
 async def async_foreach(
     sequence: Iterable[T],
@@ -70,3 +73,10 @@ async def run_in_process_with_timeout(
         raise ValueError(
             "Unhandled exception in subprocess. Exitcode: " + str(process.exitcode)
         )
+
+
+def warn_once(logger: logging.Logger, message: str) -> None:
+    """Display a warning message only once. If the message has already been shown, do nothing."""
+    if message not in _shown_warnings:
+        logger.warning(message)
+        _shown_warnings.add(message)

weave/scorers/base_scorer.py

@@ -16,7 +16,7 @@ class Scorer(Object):
         description="A mapping from column names in the dataset to the names expected by the scorer",
     )
 
-    def score(self, input: Any, target: Any, output: Any) -> Any:
+    def score(self, *, output: Any, **kwargs: Any) -> Any:
         raise NotImplementedError
 
     @weave.op()

weave/scorers/classification_scorer.py

@@ -37,12 +37,15 @@ def summarize(self, score_rows: list) -> Optional[dict]:
 
         return result
 
+    # NOTE: This is an old-style scorer that uses `model_output` instead of `output` for
+    # backwards compatibility.  In future, this behaviour may change to use the newer `output` key.
+    # You can still pass a `column_map` to map to the new `output` key if you prefer.
     @weave.op()
-    def score(self, target: dict, output: Optional[dict]) -> dict:
+    def score(self, target: dict, model_output: Optional[dict]) -> dict:
         result = {}
         for class_name in self.class_names:
             class_label = target.get(class_name)
-            class_output = output.get(class_name) if output else None
+            class_output = model_output.get(class_name) if model_output else None
             result[class_name] = {
                 "correct": class_label == class_output,
                 "negative": not class_output,