feat(evaluations): pass the tool trajectory to scorers

packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt

@@ -81,3 +81,13 @@ Scorer  [class, @dataclass(frozen=True)]
   success_direction: SuccessDirection = 'higher_is_better'
   threshold: float | None = 1.0
   to_criteria_wire() -> dict[str, Any]
+ScorerContext  [class, @dataclass(frozen=True)]
+  __init__(tool_calls: tuple[ToolInvocation, ...] = (), tool_calls_omitted: int = 0)
+  tool_calls: tuple[ToolInvocation, ...] = ()
+  tool_calls_omitted: int = 0
+ToolInvocation  [class, @dataclass(frozen=True)]
+  __init__(name: str, arguments: Any = None, result: Any = None, error: str | None = None)
+  arguments: Any = None
+  error: str | None = None
+  name: str
+  result: Any = None

packages/client/src/launchdarkly_ai_server/evaluations/criteria.py

@@ -1,13 +1,19 @@
 from __future__ import annotations
 
+import inspect
 from collections.abc import Awaitable, Callable
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from math import isnan
 from typing import Any, Literal
 
+from ..trajectory import ToolInvocation
 from .types import DatasetRow
 
-type ScorerFn = Callable[[DatasetRow, Any], float | bool | Awaitable[float | bool]]
+type ScorerResult = float | bool | Awaitable[float | bool]
+type ScorerFn = (
+    Callable[[DatasetRow, Any], ScorerResult]
+    | Callable[[DatasetRow, Any, ScorerContext], ScorerResult]
+)
 
 type SuccessDirection = Literal["higher_is_better", "lower_is_better"]
 
@@ -63,15 +69,36 @@ def to_criteria_wire(self) -> dict[str, Any]:
         }
 
 
+@dataclass(frozen=True)
+class ScorerContext:
+    """Extra inputs for a scorer, beyond the row and the generated output.
+
+    A scorer receives this as an optional third argument. New inputs are added
+    as new fields, so the scorer signature does not change.
+
+    ``tool_calls`` lists the tools the handler called while it produced the
+    output, in the order the calls started. It is empty when the handler
+    called no tool. At most 50 calls are recorded. ``tool_calls_omitted``
+    counts the calls made past that limit. Each call records its arguments and
+    result as text, cut to 2000 characters.
+    """
+
+    tool_calls: tuple[ToolInvocation, ...] = ()
+    tool_calls_omitted: int = 0
+
+
 @dataclass(frozen=True)
 class Scorer:
     """Local deterministic scorer run for each generated evaluation row.
 
     ``fn`` may be sync or async and receives ``(row, output)``, where ``row``
     is the :class:`~launchdarkly_ai_server.evaluations.types.DatasetRow` the
-    output was generated from and ``output`` is the generated output. It must
-    return a boolean or a numeric score from 0 to 1. Boolean results are
-    converted to 1.0 or 0.0 before being emitted as evaluation events.
+    output was generated from and ``output`` is the generated output. To
+    receive more inputs, declare a required third positional parameter. It
+    receives a :class:`ScorerContext`. Any other function is called with two
+    arguments. ``fn`` must return a boolean or a numeric score
+    from 0 to 1. Boolean results are converted to 1.0 or 0.0 before being
+    emitted as evaluation events.
 
     ``threshold`` defaults to 1.0: a row passes only on a perfect score, which
     matches the common case of boolean scorers. Pass a lower threshold for
@@ -90,12 +117,14 @@ class Scorer:
     threshold: float | None = 1.0
     pass_rate_threshold: float | None = None
     success_direction: SuccessDirection = "higher_is_better"
+    _accepts_context: bool = field(init=False, repr=False, compare=False)
 
     def __post_init__(self) -> None:
         if not isinstance(self.name, str) or not self.name.strip():
             raise ValueError("scorer name must not be blank")
         if not callable(self.fn):
             raise ValueError("scorer fn must be callable")
+        object.__setattr__(self, "_accepts_context", _takes_context(self.fn))
         _validate_thresholds(
             threshold=self.threshold,
             pass_rate_threshold=self.pass_rate_threshold,
@@ -120,6 +149,32 @@ def to_criteria_wire(self) -> dict[str, Any]:
 type Criterion = Judge | Scorer
 
 
+def _takes_context(fn: Callable[..., Any]) -> bool:
+    """Whether ``fn`` requires a third positional parameter for the context.
+
+    Only required parameters count, so ``(row, output, extra=None)`` is called
+    with two arguments. Raises ``ValueError`` when ``fn`` requires more than
+    three positional arguments or any keyword-only argument, since every call
+    would fail. A function with no inspectable signature is called with two.
+    """
+    try:
+        parameters = inspect.signature(fn).parameters.values()
+    except (TypeError, ValueError):
+        return False
+    required = [p for p in parameters if p.default is p.empty]
+    if any(p.kind is inspect.Parameter.KEYWORD_ONLY for p in required):
+        raise ValueError("scorer fn must not require a keyword-only argument")
+    positional = [
+        p
+        for p in required
+        if p.kind
+        in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+    ]
+    if len(positional) > 3:
+        raise ValueError("scorer fn must accept at most (row, output, context)")
+    return len(positional) == 3
+
+
 def _validate_thresholds(
     *,
     threshold: float | None,

packages/client/src/launchdarkly_ai_server/evaluations/runner.py

@@ -43,7 +43,7 @@
     require_string,
     segment,
 )
-from .criteria import Criterion, Judge, Scorer
+from .criteria import Criterion, Judge, Scorer, ScorerContext
 from .events import (
     CriterionEventPayload,
     CriterionStatus,
@@ -882,7 +882,14 @@ async def _run_scorer_for_result(
             metadata=row.get("metadata"),
         )
         try:
-            score_value = scorer.fn(dataset_row, row.get("output"))
+            if scorer._accepts_context:
+                context = ScorerContext(
+                    tool_calls=tuple(row.get("tool_calls") or ()),
+                    tool_calls_omitted=int(row.get("tool_calls_omitted") or 0),
+                )
+                score_value = scorer.fn(dataset_row, row.get("output"), context)  # type: ignore[call-arg]
+            else:
+                score_value = scorer.fn(dataset_row, row.get("output"))  # type: ignore[call-arg]
             if inspect.isawaitable(score_value):
                 score_value = await score_value
         except Exception as error:

packages/client/src/launchdarkly_ai_server/experimental/evaluations.py

@@ -8,7 +8,7 @@
 """
 
 from ..evaluations.api import EvaluationsError
-from ..evaluations.criteria import Criterion, Judge, Scorer
+from ..evaluations.criteria import Criterion, Judge, Scorer, ScorerContext
 from ..evaluations.module import EvaluationsModule, init_evaluations
 from ..evaluations.tools import EvalTool
 from ..evaluations.types import (
@@ -20,6 +20,7 @@
     GenerationOverrides,
     RunSummary,
 )
+from ..trajectory import ToolInvocation
 
 __all__ = [  # noqa: RUF022
     # entry point
@@ -36,6 +37,8 @@
     "Criterion",
     "Judge",
     "Scorer",
+    "ScorerContext",
+    "ToolInvocation",
     # results and errors
     "EvalRunResult",
     "RunSummary",

packages/client/tests/test_evaluations.py

@@ -608,6 +608,8 @@ def test_missing_project_key_raises_before_network_io(monkeypatch) -> None:
         "Criterion",
         "Judge",
         "Scorer",
+        "ScorerContext",
+        "ToolInvocation",
         "EvalRunResult",
         "RunSummary",
         "EvaluationsError",

packages/client/tests/test_evaluations_run.py

@@ -20,6 +20,8 @@
     EvaluationsError,
     Judge,
     Scorer,
+    ScorerContext,
+    ToolInvocation,
     init_evaluations,
 )
 
@@ -4867,3 +4869,172 @@ async def test_a_completion_or_list_tagged_handler_is_selected(
     )
 
     assert stub.await_count == 1
+
+
+def _scorer_runner() -> Any:
+    from launchdarkly_ai_server.evaluations.api import LDApiClient
+    from launchdarkly_ai_server.evaluations.runner import EvaluationsRunner
+
+    return EvaluationsRunner(LDApiClient(api_key="token", transport=failing_transport))
+
+
+def _generated_row(**overrides: Any) -> dict[str, Any]:
+    row: dict[str, Any] = {
+        "row_index": 0,
+        "input": "Question",
+        "expected_output": None,
+        "variables": {},
+        "metadata": None,
+        "output": "answer",
+        "status": "COMPLETE",
+        "tool_calls": [
+            ToolInvocation(name="lookup_order", arguments='{"id":"A1"}', result="ok"),
+            ToolInvocation(name="refund", arguments='{"id":"A1"}', result="done"),
+        ],
+        "tool_calls_omitted": 3,
+    }
+    row.update(overrides)
+    return row
+
+
+@pytest.mark.asyncio
+async def test_scorer_with_a_third_parameter_receives_the_tool_calls_in_order() -> None:
+    seen: list[ScorerContext] = []
+
+    def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float:
+        seen.append(context)
+        names = [call.name for call in context.tool_calls]
+        return 1.0 if names == ["lookup_order", "refund"] else 0.0
+
+    result = await _scorer_runner()._run_scorer_for_result(
+        _generated_row(), Scorer(name="trajectory", fn=scorer_fn)
+    )
+
+    assert result["status"] == "COMPLETE"
+    assert result["score"] == 1.0
+    assert seen[0].tool_calls_omitted == 3
+
+
+@pytest.mark.asyncio
+async def test_async_scorer_with_a_context_is_awaited() -> None:
+    async def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float:
+        return 1.0 if len(context.tool_calls) == 2 else 0.0
+
+    result = await _scorer_runner()._run_scorer_for_result(
+        _generated_row(), Scorer(name="trajectory", fn=scorer_fn)
+    )
+
+    assert result["score"] == 1.0
+
+
+@pytest.mark.asyncio
+async def test_scorer_context_is_empty_when_no_tool_was_called() -> None:
+    def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float:
+        assert context.tool_calls == ()
+        assert context.tool_calls_omitted == 0
+        return 1.0
+
+    result = await _scorer_runner()._run_scorer_for_result(
+        _generated_row(tool_calls=[], tool_calls_omitted=0),
+        Scorer(name="trajectory", fn=scorer_fn),
+    )
+
+    assert result["status"] == "COMPLETE"
+
+
+@pytest.mark.asyncio
+async def test_two_parameter_scorer_is_called_without_a_context() -> None:
+    calls: list[tuple[Any, ...]] = []
+
+    def scorer_fn(*args: Any) -> float:
+        calls.append(args)
+        return 1.0
+
+    def two_parameter(row: DatasetRow, output: Any) -> float:
+        return scorer_fn(row, output)
+
+    await _scorer_runner()._run_scorer_for_result(
+        _generated_row(), Scorer(name="plain", fn=two_parameter)
+    )
+
+    assert len(calls[0]) == 2
+
+
+def test_scorer_detects_the_context_parameter_once_at_construction() -> None:
+    assert Scorer(name="a", fn=lambda row, output: 1.0)._accepts_context is False
+    assert Scorer(name="b", fn=lambda row, output, ctx: 1.0)._accepts_context is True
+    # Only required parameters count: a defaulted third one is called with two.
+    assert (
+        Scorer(name="c", fn=lambda row, output, ctx=None: 1.0)._accepts_context is False
+    )
+    assert Scorer(name="d", fn=lambda *args: 1.0)._accepts_context is False
+    assert (
+        Scorer(name="e", fn=lambda row, output, ctx, *, opt=1: 1.0)._accepts_context
+        is True
+    )
+
+
+def test_scorer_rejects_a_function_that_needs_more_than_three_arguments() -> None:
+    with pytest.raises(ValueError, match="at most"):
+        Scorer(name="too-many", fn=lambda a, b, c, d: 1.0)
+
+
+def test_scorer_rejects_a_required_keyword_only_argument() -> None:
+    # It would pass validation and then fail on every row.
+    with pytest.raises(ValueError, match="keyword-only"):
+        Scorer(name="kw", fn=lambda row, output, *, ctx: 1.0)
+
+
+@pytest.mark.asyncio
+async def test_run_calls_two_and_three_parameter_scorers(
+    stub_sdk_client: MagicMock,
+) -> None:
+    transport = SequencedTransport(
+        [
+            response(200, {"id": "dataset-id", "name": "golden"}),
+            response(
+                200, dataset_page([{"rowIndex": 0, "input": "Ticket A"}], total=1)
+            ),
+            response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}),
+            response(
+                201,
+                {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"},
+            ),
+            response(
+                200,
+                {"statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}},
+            ),
+        ]
+    )
+    evals = init_evaluations(
+        project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport
+    )
+    seen: list[ScorerContext] = []
+
+    async def handler(*args: Any) -> dict[str, Any]:
+        return {"output": "refunded"}
+
+    def plain(row: DatasetRow, output: Any) -> float:
+        return 1.0 if output == "refunded" else 0.0
+
+    def with_context(row: DatasetRow, output: Any, context: ScorerContext) -> float:
+        seen.append(context)
+        return 1.0
+
+    await evals.run(
+        key="support-qa",
+        dataset="golden",
+        handler=tagged(handler),
+        generation=COMPLETION,
+        criteria=[
+            Scorer(name="plain", fn=plain),
+            Scorer(name="with-context", fn=with_context),
+        ],
+    )
+
+    events = [call.args[2] for call in stub_sdk_client.track.call_args_list]
+    scores = {e["criterionType"]: e for e in events if e.get("kind") == "scorer"}
+    assert scores["plain"]["status"] == "COMPLETE"
+    assert scores["plain"]["score"] == 1.0
+    assert scores["with-context"]["status"] == "COMPLETE"
+    assert seen == [ScorerContext()]