feat(evaluations): normalize text behaviors

packages/client/src/launchdarkly_ai_server/canonical_json.py

@@ -0,0 +1,98 @@
+"""RFC 8785 JSON Canonicalization Scheme (JCS), for hashes every SDK must agree on.
+
+JCS spells JSON the way ECMAScript's ``JSON.stringify`` does: no whitespace,
+object keys sorted by UTF-16 code unit, non-ASCII written as is, and numbers in
+ECMAScript's shortest round-trip form. That is the native form in JavaScript
+and close to it in Go, so the only language that needs this module is Python,
+whose ``json.dumps`` writes ``1.0`` where JCS writes ``1``.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from decimal import Decimal
+from math import isfinite
+from typing import Any
+
+# Above this an int has no exact IEEE double, so it is written as the double
+# nearest to it, as a JavaScript JSON.parse reads it.
+_MAX_SAFE_INTEGER = 2**53 - 1
+_SURROGATE = re.compile("[\ud800-\udfff]")
+
+
+def canonical_json(value: Any, *, sort_keys: bool = True) -> str:
+    """The JCS text of a JSON-like value.
+
+    With ``sort_keys=False`` object keys keep their insertion order, which is
+    ``JSON.stringify``'s output exactly: the form judge text uses, so a judge
+    reads the same text in every SDK. It is not JCS, so never hash it.
+
+    The value is first normalized through ``json.dumps(..., default=str)``, so
+    a tuple is a list, a non-string key is the string ``json.dumps`` writes for
+    it, and a value JSON cannot hold is its ``str()``. A non-finite number is
+    written ``null``, as ``JSON.stringify`` writes it.
+    """
+    return _encode(json.loads(json.dumps(value, default=str)), sort_keys)
+
+
+def _encode(value: Any, sort_keys: bool = True) -> str:
+    if value is None or value is True or value is False:
+        return json.dumps(value)
+    if isinstance(value, int):
+        if abs(value) <= _MAX_SAFE_INTEGER:
+            return str(value)
+        try:
+            return _number(float(value))
+        except OverflowError:
+            return "null"
+    if isinstance(value, float):
+        return _number(value)
+    if isinstance(value, str):
+        return _string(value)
+    if isinstance(value, list):
+        return "[" + ",".join(_encode(item, sort_keys) for item in value) + "]"
+    keys = sorted(value, key=_utf16) if sort_keys else list(value)
+    return (
+        "{"
+        + ",".join(f"{_string(key)}:{_encode(value[key], sort_keys)}" for key in keys)
+        + "}"
+    )
+
+
+def _number(value: float) -> str:
+    """ECMAScript's Number::toString, from the shortest round-trip digits ``repr`` gives."""
+    if not isfinite(value):
+        return "null"
+    if value == 0:
+        return "0"
+    _, digit_tuple, exponent = Decimal(repr(abs(value))).as_tuple()
+    # The value is 0.<digits> x 10^n, the form Number::toString starts from.
+    n = len(digit_tuple) + int(exponent)
+    digits = "".join(map(str, digit_tuple)).rstrip("0")
+    k = len(digits)
+    if k <= n <= 21:
+        text = digits + "0" * (n - k)
+    elif 0 < n <= 21:
+        text = f"{digits[:n]}.{digits[n:]}"
+    elif -6 < n <= 0:
+        text = f"0.{'0' * -n}{digits}"
+    else:
+        fraction = f".{digits[1:]}" if k > 1 else ""
+        text = f"{digits[0]}{fraction}e{'+' if n - 1 >= 0 else '-'}{abs(n - 1)}"
+    return ("-" if value < 0 else "") + text
+
+
+def _string(text: str) -> str:
+    # A surrogate pair held as two code points is joined into one character, as
+    # it is in a JavaScript string; a lone surrogate is escaped, as
+    # JSON.stringify escapes it, so the text always has a UTF-8 encoding.
+    joined = text.encode("utf-16", "surrogatepass").decode("utf-16", "surrogatepass")
+    return _SURROGATE.sub(
+        lambda match: f"\\u{ord(match.group()):04x}",
+        json.dumps(joined, ensure_ascii=False),
+    )
+
+
+def _utf16(key: str) -> bytes:
+    return key.encode("utf-16-be", "surrogatepass")

packages/client/src/launchdarkly_ai_server/client.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
-import json
 from collections.abc import AsyncGenerator, Callable
 from contextlib import aclosing
 from typing import Any
 
 from .agui import StableProjector, judge_completed_stream
 from .conversation import bind_conversation_id
+from .judge_scoring import judge_text
 from .judges import _build_judge_tasks, run_judges
 from .lifecycle import extract_variation
 from .registry import resolve_handlers, resolve_tools
@@ -120,11 +120,8 @@ async def _invoke(
             config.get("outputFormat") if isinstance(config, dict) else None,
         )
 
-        llm_str = (
-            parsed_response
-            if isinstance(parsed_response, str)
-            else json.dumps(parsed_response)
-        )
+        # The same text an offline judge reads, so a rubric sees one spelling.
+        llm_str = judge_text(parsed_response)
 
         usage_obj = to_usage_dict(usage)
 

packages/client/src/launchdarkly_ai_server/evaluations/runner.py

@@ -10,12 +10,14 @@
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from dataclasses import dataclass
 from datetime import UTC, datetime
+from math import isfinite
 from typing import Any, Literal, cast
 
-from ..content import tool_result_string
+from ..canonical_json import canonical_json
 from ..judge_scoring import (
     FORMATTING_INSTRUCTIONS,
     build_message_history,
+    judge_text,
     numeric_score,
     parse_judge_response,
     typesafe_judge_entries,
@@ -95,6 +97,17 @@ def _provides_for(
     return handler_key(handler)
 
 
+def _describe_value(value: Any) -> str:
+    """A value in an error message, as JSON rather than a Python repr, so it reads the same in every SDK.
+
+    Spelled as ``JSON.stringify`` spells it, except that a non-finite number is
+    ``NaN`` / ``Infinity``, and a value JSON cannot hold is its ``str()``.
+    """
+    if isinstance(value, float) and not isfinite(value):
+        return json.dumps(value)
+    return canonical_json(value, sort_keys=False)
+
+
 def describe_handlers(handlers: Sequence[EvalHandler]) -> str:
     """Return the ``provides_for`` of each handler, for an error message."""
     return ", ".join(repr(_provides_for(handler)) for handler in handlers)
@@ -733,9 +746,7 @@ def _emit_generation_events(
             }
             if dataset.id is None:
                 del identity["datasetId"]
-            event_id = hashlib.sha256(
-                json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()
-            ).hexdigest()
+            event_id = hashlib.sha256(canonical_json(identity).encode()).hexdigest()
             error = result.get("error")
             generated = {
                 "status": result["status"],
@@ -758,9 +769,7 @@ def _emit_generation_events(
                     "outputTokens": normalized_usage["output"],
                 }
             content_hash = hashlib.sha256(
-                json.dumps(
-                    generated, sort_keys=True, separators=(",", ":"), default=str
-                ).encode()
+                canonical_json(generated).encode()
             ).hexdigest()
             emitted_at = datetime.now(UTC).isoformat().replace("+00:00", "Z")
             payload: dict[str, Any] = {
@@ -824,7 +833,7 @@ def _judge_variables(
         variables.update(
             {
                 "input": row_result.get("input") or "",
-                "response_to_evaluate": tool_result_string(output),
+                "response_to_evaluate": judge_text(output),
                 "message_history": build_message_history(
                     user_input=row_result.get("input"),
                     trajectory=trajectory,
@@ -899,15 +908,15 @@ async def _run_scorer_for_result(
                     started_clock,
                     "invalid_score",
                     "scorer fn must return a bool or a finite number, "
-                    f"got {score_value!r}",
+                    f"got {_describe_value(score_value)}",
                 )
             score = maybe_score
         if score < 0 or score > 1:
             return self._criterion_error_result(
                 base,
                 started_clock,
                 "invalid_score",
-                f"scorer fn score must be between 0 and 1, got {score_value!r}",
+                f"scorer fn score must be between 0 and 1, got {_describe_value(score_value)}",
             )
         completed = datetime.now(UTC)
         return {
@@ -990,7 +999,7 @@ async def _run_ld_judge_for_result(
                 base,
                 started_clock,
                 "invalid_score",
-                f"judge score must be a number between 0 and 1, got {raw_score!r}",
+                f"judge score must be a number between 0 and 1, got {_describe_value(raw_score)}",
             )
         completed = datetime.now(UTC)
         # No verdict: the SDK reports the score and LaunchDarkly rules on it. The
@@ -1080,9 +1089,7 @@ def _emit_evaluation_events(
                 }
                 if dataset.id is None:
                     del identity["datasetId"]
-                event_id = hashlib.sha256(
-                    json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()
-                ).hexdigest()
+                event_id = hashlib.sha256(canonical_json(identity).encode()).hexdigest()
                 emitted_at = datetime.now(UTC).isoformat().replace("+00:00", "Z")
                 usage: TokenUsage | None = None
                 if result["kind"] == "judge" and isinstance(

packages/client/src/launchdarkly_ai_server/judge_scoring.py

@@ -16,7 +16,7 @@
 from math import isfinite
 from typing import Any
 
-from .content import tool_result_string
+from .canonical_json import canonical_json
 from .utils import parse_json_with_possible_fences
 
 logger = logging.getLogger(__name__)
@@ -42,25 +42,52 @@ def build_message_history(
     """The conversation a judge is shown, as its ``message_history`` variable.
 
     Ordered as it happened: what was asked, what the agent did, what it
-    answered, then how to format the verdict. Empty parts are skipped, so a
-    run with no tools yields the history it did before trajectories existed.
+    answered, then how to format the verdict. An empty part (see
+    :func:`_is_empty_part`) is skipped, so a run with no tools yields the
+    history it did before trajectories existed.
 
     The formatting block is appended here, not by callers: judges built from
     the AI Library's default templates read the JSON shape from
     ``{{message_history}}``, and one that stopped being told it would return
     prose and fail every result as invalid output.
 
-    A non-string output is shown as JSON, by the rule tool results follow, so a
-    rubric reads the same text in every SDK.
+    Every other part is rendered by the rule tool results follow, so a
+    non-string output is shown as JSON and a rubric reads the same text in
+    every SDK. An output of ``0`` or ``False`` is an answer, and is shown.
     """
-    output_text = tool_result_string(output) if output else None
     return "\n\n".join(
-        str(part)
-        for part in (user_input, trajectory, output_text, FORMATTING_INSTRUCTIONS)
-        if part
+        judge_text(part)
+        for part in (user_input, trajectory, output, FORMATTING_INSTRUCTIONS)
+        if not _is_empty_part(part)
     )
 
 
+def judge_text(value: Any) -> str:
+    """A value as a judge is shown it: a string as it is, ``None`` as ``""``, anything else as JSON.
+
+    The JSON is ``JSON.stringify``'s spelling (no whitespace, keys in insertion
+    order, ``1`` for ``1.0``), not ``json.dumps``'s, so a rubric reads the same
+    text in every SDK.
+    """
+    if isinstance(value, str):
+        return value
+    if value is None:
+        return ""
+    return canonical_json(value, sort_keys=False)
+
+
+def _is_empty_part(value: Any) -> bool:
+    """``None``, ``""``, an empty list or an empty mapping, tested before rendering.
+
+    An empty container is skipped, as it was before structured outputs were
+    rendered as JSON. Python truthiness is not the test: it would also drop
+    ``0`` and ``False``, which are answers.
+    """
+    if value is None or value == "":
+        return True
+    return isinstance(value, (list, tuple, Mapping)) and not value
+
+
 def numeric_score(score: Any) -> float | None:
     """Return ``score`` as a float only when it already is a finite number.
 
@@ -129,6 +156,10 @@ def parse_judge_response(raw: Any) -> tuple[Any, str]:
     already-decoded mapping. The score is returned untouched — callers apply
     their own policy to non-numeric values via :func:`numeric_score`.
 
+    The reasoning is the first of ``reasoning`` and ``reason`` that is not an
+    empty part (:func:`_is_empty_part`), rendered by the tool-result rule: a
+    string as it is, anything else as JSON, never as a Python repr.
+
     Raises ``ValueError`` when the response is not a non-empty JSON object.
     """
     parsed: Any
@@ -140,8 +171,15 @@ def parse_judge_response(raw: Any) -> tuple[Any, str]:
         parsed = None
     if not isinstance(parsed, Mapping) or not parsed:
         raise ValueError("Invalid JSON from judge")
-    reasoning = parsed.get("reasoning") or parsed.get("reason") or ""
-    return parsed.get("score"), str(reasoning)
+    reasoning = next(
+        (
+            judge_text(value)
+            for value in (parsed.get("reasoning"), parsed.get("reason"))
+            if not _is_empty_part(value)
+        ),
+        "",
+    )
+    return parsed.get("score"), reasoning
 
 
 def without_output_format(config: Any, judge_key: str) -> Any: