diff --git a/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt b/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt index b3761b86..8494f970 100644 --- a/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt +++ b/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt @@ -81,3 +81,13 @@ Scorer [class, @dataclass(frozen=True)] success_direction: SuccessDirection = 'higher_is_better' threshold: float | None = 1.0 to_criteria_wire() -> dict[str, Any] +ScorerContext [class, @dataclass(frozen=True)] + __init__(tool_calls: tuple[ToolInvocation, ...] = (), tool_calls_omitted: int = 0) + tool_calls: tuple[ToolInvocation, ...] = () + tool_calls_omitted: int = 0 +ToolInvocation [class, @dataclass(frozen=True)] + __init__(name: str, arguments: Any = None, result: Any = None, error: str | None = None) + arguments: Any = None + error: str | None = None + name: str + result: Any = None diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py index ee0962fd..1ab3e2f3 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py @@ -1,13 +1,19 @@ from __future__ import annotations +import inspect from collections.abc import Awaitable, Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from math import isnan from typing import Any, Literal +from ..trajectory import ToolInvocation from .types import DatasetRow -type ScorerFn = Callable[[DatasetRow, Any], float | bool | Awaitable[float | bool]] +type ScorerResult = float | bool | Awaitable[float | bool] +type ScorerFn = ( + Callable[[DatasetRow, Any], ScorerResult] + | Callable[[DatasetRow, Any, ScorerContext], ScorerResult] +) type SuccessDirection = Literal["higher_is_better", "lower_is_better"] @@ -63,15 +69,36 @@ def to_criteria_wire(self) -> dict[str, Any]: } +@dataclass(frozen=True) +class ScorerContext: + """Extra inputs for a scorer, beyond the row and the generated output. + + A scorer receives this as an optional third argument. New inputs are added + as new fields, so the scorer signature does not change. + + ``tool_calls`` lists the tools the handler called while it produced the + output, in the order the calls started. It is empty when the handler + called no tool. At most 50 calls are recorded. ``tool_calls_omitted`` + counts the calls made past that limit. Each call records its arguments and + result as text, cut to 2000 characters. + """ + + tool_calls: tuple[ToolInvocation, ...] = () + tool_calls_omitted: int = 0 + + @dataclass(frozen=True) class Scorer: """Local deterministic scorer run for each generated evaluation row. ``fn`` may be sync or async and receives ``(row, output)``, where ``row`` is the :class:`~launchdarkly_ai_server.evaluations.types.DatasetRow` the - output was generated from and ``output`` is the generated output. It must - return a boolean or a numeric score from 0 to 1. Boolean results are - converted to 1.0 or 0.0 before being emitted as evaluation events. + output was generated from and ``output`` is the generated output. To + receive more inputs, declare a required third positional parameter. It + receives a :class:`ScorerContext`. Any other function is called with two + arguments. ``fn`` must return a boolean or a numeric score + from 0 to 1. Boolean results are converted to 1.0 or 0.0 before being + emitted as evaluation events. ``threshold`` defaults to 1.0: a row passes only on a perfect score, which matches the common case of boolean scorers. Pass a lower threshold for @@ -90,12 +117,14 @@ class Scorer: threshold: float | None = 1.0 pass_rate_threshold: float | None = None success_direction: SuccessDirection = "higher_is_better" + _accepts_context: bool = field(init=False, repr=False, compare=False) def __post_init__(self) -> None: if not isinstance(self.name, str) or not self.name.strip(): raise ValueError("scorer name must not be blank") if not callable(self.fn): raise ValueError("scorer fn must be callable") + object.__setattr__(self, "_accepts_context", _takes_context(self.fn)) _validate_thresholds( threshold=self.threshold, pass_rate_threshold=self.pass_rate_threshold, @@ -120,6 +149,32 @@ def to_criteria_wire(self) -> dict[str, Any]: type Criterion = Judge | Scorer +def _takes_context(fn: Callable[..., Any]) -> bool: + """Whether ``fn`` requires a third positional parameter for the context. + + Only required parameters count, so ``(row, output, extra=None)`` is called + with two arguments. Raises ``ValueError`` when ``fn`` requires more than + three positional arguments or any keyword-only argument, since every call + would fail. A function with no inspectable signature is called with two. + """ + try: + parameters = inspect.signature(fn).parameters.values() + except (TypeError, ValueError): + return False + required = [p for p in parameters if p.default is p.empty] + if any(p.kind is inspect.Parameter.KEYWORD_ONLY for p in required): + raise ValueError("scorer fn must not require a keyword-only argument") + positional = [ + p + for p in required + if p.kind + in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + ] + if len(positional) > 3: + raise ValueError("scorer fn must accept at most (row, output, context)") + return len(positional) == 3 + + def _validate_thresholds( *, threshold: float | None, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index e76bf0ea..646a8f0d 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -43,7 +43,7 @@ require_string, segment, ) -from .criteria import Criterion, Judge, Scorer +from .criteria import Criterion, Judge, Scorer, ScorerContext from .events import ( CriterionEventPayload, CriterionStatus, @@ -882,7 +882,14 @@ async def _run_scorer_for_result( metadata=row.get("metadata"), ) try: - score_value = scorer.fn(dataset_row, row.get("output")) + if scorer._accepts_context: + context = ScorerContext( + tool_calls=tuple(row.get("tool_calls") or ()), + tool_calls_omitted=int(row.get("tool_calls_omitted") or 0), + ) + score_value = scorer.fn(dataset_row, row.get("output"), context) # type: ignore[call-arg] + else: + score_value = scorer.fn(dataset_row, row.get("output")) # type: ignore[call-arg] if inspect.isawaitable(score_value): score_value = await score_value except Exception as error: diff --git a/packages/client/src/launchdarkly_ai_server/experimental/evaluations.py b/packages/client/src/launchdarkly_ai_server/experimental/evaluations.py index 593f90e8..c5bbc90a 100644 --- a/packages/client/src/launchdarkly_ai_server/experimental/evaluations.py +++ b/packages/client/src/launchdarkly_ai_server/experimental/evaluations.py @@ -8,7 +8,7 @@ """ from ..evaluations.api import EvaluationsError -from ..evaluations.criteria import Criterion, Judge, Scorer +from ..evaluations.criteria import Criterion, Judge, Scorer, ScorerContext from ..evaluations.module import EvaluationsModule, init_evaluations from ..evaluations.tools import EvalTool from ..evaluations.types import ( @@ -20,6 +20,7 @@ GenerationOverrides, RunSummary, ) +from ..trajectory import ToolInvocation __all__ = [ # noqa: RUF022 # entry point @@ -36,6 +37,8 @@ "Criterion", "Judge", "Scorer", + "ScorerContext", + "ToolInvocation", # results and errors "EvalRunResult", "RunSummary", diff --git a/packages/client/tests/test_evaluations.py b/packages/client/tests/test_evaluations.py index 069f1fe9..d59f5d47 100644 --- a/packages/client/tests/test_evaluations.py +++ b/packages/client/tests/test_evaluations.py @@ -608,6 +608,8 @@ def test_missing_project_key_raises_before_network_io(monkeypatch) -> None: "Criterion", "Judge", "Scorer", + "ScorerContext", + "ToolInvocation", "EvalRunResult", "RunSummary", "EvaluationsError", diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 435d0f4c..676dba85 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -20,6 +20,8 @@ EvaluationsError, Judge, Scorer, + ScorerContext, + ToolInvocation, init_evaluations, ) @@ -4867,3 +4869,172 @@ async def test_a_completion_or_list_tagged_handler_is_selected( ) assert stub.await_count == 1 + + +def _scorer_runner() -> Any: + from launchdarkly_ai_server.evaluations.api import LDApiClient + from launchdarkly_ai_server.evaluations.runner import EvaluationsRunner + + return EvaluationsRunner(LDApiClient(api_key="token", transport=failing_transport)) + + +def _generated_row(**overrides: Any) -> dict[str, Any]: + row: dict[str, Any] = { + "row_index": 0, + "input": "Question", + "expected_output": None, + "variables": {}, + "metadata": None, + "output": "answer", + "status": "COMPLETE", + "tool_calls": [ + ToolInvocation(name="lookup_order", arguments='{"id":"A1"}', result="ok"), + ToolInvocation(name="refund", arguments='{"id":"A1"}', result="done"), + ], + "tool_calls_omitted": 3, + } + row.update(overrides) + return row + + +@pytest.mark.asyncio +async def test_scorer_with_a_third_parameter_receives_the_tool_calls_in_order() -> None: + seen: list[ScorerContext] = [] + + def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float: + seen.append(context) + names = [call.name for call in context.tool_calls] + return 1.0 if names == ["lookup_order", "refund"] else 0.0 + + result = await _scorer_runner()._run_scorer_for_result( + _generated_row(), Scorer(name="trajectory", fn=scorer_fn) + ) + + assert result["status"] == "COMPLETE" + assert result["score"] == 1.0 + assert seen[0].tool_calls_omitted == 3 + + +@pytest.mark.asyncio +async def test_async_scorer_with_a_context_is_awaited() -> None: + async def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float: + return 1.0 if len(context.tool_calls) == 2 else 0.0 + + result = await _scorer_runner()._run_scorer_for_result( + _generated_row(), Scorer(name="trajectory", fn=scorer_fn) + ) + + assert result["score"] == 1.0 + + +@pytest.mark.asyncio +async def test_scorer_context_is_empty_when_no_tool_was_called() -> None: + def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float: + assert context.tool_calls == () + assert context.tool_calls_omitted == 0 + return 1.0 + + result = await _scorer_runner()._run_scorer_for_result( + _generated_row(tool_calls=[], tool_calls_omitted=0), + Scorer(name="trajectory", fn=scorer_fn), + ) + + assert result["status"] == "COMPLETE" + + +@pytest.mark.asyncio +async def test_two_parameter_scorer_is_called_without_a_context() -> None: + calls: list[tuple[Any, ...]] = [] + + def scorer_fn(*args: Any) -> float: + calls.append(args) + return 1.0 + + def two_parameter(row: DatasetRow, output: Any) -> float: + return scorer_fn(row, output) + + await _scorer_runner()._run_scorer_for_result( + _generated_row(), Scorer(name="plain", fn=two_parameter) + ) + + assert len(calls[0]) == 2 + + +def test_scorer_detects_the_context_parameter_once_at_construction() -> None: + assert Scorer(name="a", fn=lambda row, output: 1.0)._accepts_context is False + assert Scorer(name="b", fn=lambda row, output, ctx: 1.0)._accepts_context is True + # Only required parameters count: a defaulted third one is called with two. + assert ( + Scorer(name="c", fn=lambda row, output, ctx=None: 1.0)._accepts_context is False + ) + assert Scorer(name="d", fn=lambda *args: 1.0)._accepts_context is False + assert ( + Scorer(name="e", fn=lambda row, output, ctx, *, opt=1: 1.0)._accepts_context + is True + ) + + +def test_scorer_rejects_a_function_that_needs_more_than_three_arguments() -> None: + with pytest.raises(ValueError, match="at most"): + Scorer(name="too-many", fn=lambda a, b, c, d: 1.0) + + +def test_scorer_rejects_a_required_keyword_only_argument() -> None: + # It would pass validation and then fail on every row. + with pytest.raises(ValueError, match="keyword-only"): + Scorer(name="kw", fn=lambda row, output, *, ctx: 1.0) + + +@pytest.mark.asyncio +async def test_run_calls_two_and_three_parameter_scorers( + stub_sdk_client: MagicMock, +) -> None: + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, dataset_page([{"rowIndex": 0, "input": "Ticket A"}], total=1) + ), + response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}}, + ), + ] + ) + evals = init_evaluations( + project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport + ) + seen: list[ScorerContext] = [] + + async def handler(*args: Any) -> dict[str, Any]: + return {"output": "refunded"} + + def plain(row: DatasetRow, output: Any) -> float: + return 1.0 if output == "refunded" else 0.0 + + def with_context(row: DatasetRow, output: Any, context: ScorerContext) -> float: + seen.append(context) + return 1.0 + + await evals.run( + key="support-qa", + dataset="golden", + handler=tagged(handler), + generation=COMPLETION, + criteria=[ + Scorer(name="plain", fn=plain), + Scorer(name="with-context", fn=with_context), + ], + ) + + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + scores = {e["criterionType"]: e for e in events if e.get("kind") == "scorer"} + assert scores["plain"]["status"] == "COMPLETE" + assert scores["plain"]["score"] == 1.0 + assert scores["with-context"]["status"] == "COMPLETE" + assert seen == [ScorerContext()]