From 303d20ee63d7499ddddf1883025944e658743097 Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Thu, 30 Jul 2026 22:40:30 +0800 Subject: [PATCH 01/13] feat: add auditable evaluation optimization loop reference --- .../eval_optimize_loop/.gitignore | 4 + .../optimization/eval_optimize_loop/DESIGN.md | 129 +++ .../optimization/eval_optimize_loop/README.md | 59 ++ .../eval_optimize_loop/optimizer.json | 82 ++ .../eval_optimize_loop/pipeline/__init__.py | 1 + .../eval_optimize_loop/pipeline/artifacts.py | 191 +++++ .../pipeline/attribution.py | 209 +++++ .../eval_optimize_loop/pipeline/backends.py | 443 +++++++++++ .../pipeline/candidate_runtime.py | 121 +++ .../pipeline/configuration.py | 144 ++++ .../eval_optimize_loop/pipeline/contracts.py | 42 + .../eval_optimize_loop/pipeline/costing.py | 37 + .../eval_optimize_loop/pipeline/evaluation.py | 490 ++++++++++++ .../pipeline/evaluation_runtime.py | 145 ++++ .../eval_optimize_loop/pipeline/gate.py | 199 +++++ .../eval_optimize_loop/pipeline/models.py | 682 ++++++++++++++++ .../pipeline/offline_evaluation.py | 218 ++++++ .../pipeline/orchestrator.py | 347 ++++++++ .../eval_optimize_loop/pipeline/preflight.py | 176 +++++ .../pipeline/prompt_workspace.py | 127 +++ .../eval_optimize_loop/pipeline/reporting.py | 352 +++++++++ .../eval_optimize_loop/pipeline/schema.py | 166 ++++ .../eval_optimize_loop/prompts/system.md | 3 + .../eval_optimize_loop/run_pipeline.py | 71 ++ .../traces/trace_cases.json | 105 +++ .../eval_optimize_loop/train.evalset.json | 57 ++ .../eval_optimize_loop/val.evalset.json | 57 ++ .../test_attribution_gate.py | 416 ++++++++++ .../eval_optimize_loop/test_backends.py | 741 ++++++++++++++++++ .../test_candidate_runtime.py | 57 ++ .../eval_optimize_loop/test_evaluation.py | 232 ++++++ .../test_models_and_imports.py | 321 ++++++++ .../eval_optimize_loop/test_orchestrator.py | 550 +++++++++++++ .../test_workspace_artifacts.py | 207 +++++ 34 files changed, 7181 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/.gitignore create mode 100644 examples/optimization/eval_optimize_loop/DESIGN.md create mode 100644 examples/optimization/eval_optimize_loop/README.md create mode 100644 examples/optimization/eval_optimize_loop/optimizer.json create mode 100644 examples/optimization/eval_optimize_loop/pipeline/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/artifacts.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/attribution.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/backends.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/configuration.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/contracts.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/costing.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/evaluation.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/evaluation_runtime.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/gate.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/models.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/offline_evaluation.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/orchestrator.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/preflight.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/reporting.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/schema.py create mode 100644 examples/optimization/eval_optimize_loop/prompts/system.md create mode 100644 examples/optimization/eval_optimize_loop/run_pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/traces/trace_cases.json create mode 100644 examples/optimization/eval_optimize_loop/train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/val.evalset.json create mode 100644 tests/examples/eval_optimize_loop/test_attribution_gate.py create mode 100644 tests/examples/eval_optimize_loop/test_backends.py create mode 100644 tests/examples/eval_optimize_loop/test_candidate_runtime.py create mode 100644 tests/examples/eval_optimize_loop/test_evaluation.py create mode 100644 tests/examples/eval_optimize_loop/test_models_and_imports.py create mode 100644 tests/examples/eval_optimize_loop/test_orchestrator.py create mode 100644 tests/examples/eval_optimize_loop/test_workspace_artifacts.py diff --git a/examples/optimization/eval_optimize_loop/.gitignore b/examples/optimization/eval_optimize_loop/.gitignore new file mode 100644 index 000000000..b24bd972d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/.gitignore @@ -0,0 +1,4 @@ +/artifacts/ +/optimization_report.json +/optimization_report.md +__pycache__/ diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 000000000..41008b7ba --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,129 @@ +# Design + +`run_pipeline.py` only parses arguments and assembles an optional live callback. +`pipeline/orchestrator.py` is the sole composition root. Raw SDK +`EvaluateResult` objects are handed directly to `evaluation.normalize_result`; +no other module unfolds them. + +The dependency direction is flat and one-way: + +```text +CLI -> orchestrator -> preflight / evaluation_runtime / candidate_runtime / reporting +configuration -> models / schema +models -> schema +evaluation_runtime -> backend contract / normalizer / cost ledger / artifact sink +candidate_runtime -> generator contract / split policy / prompt workspace / artifact sink +backends -> offline_evaluation / contracts / models +offline_evaluation -> SDK evaluation types +pure policies -> models +reporting -> models / artifacts +``` + +`schema.py` owns strict parsing and model primitives; `configuration.py` owns +validated input and policy configuration; `models.py` owns stage outputs and +report facts. `preflight.py` owns validated inputs and run identity. +`evaluation_runtime.py` owns ordered backend calls, completed-call accounting, +and snapshot persistence. `candidate_runtime.py` owns inner-split persistence, +the OS-temporary optimizer workspace, and sanitized optimizer output import. +`offline_evaluation.py` owns offline rule parsing, evidence extraction, +deterministic evaluators, and the run-local replacement registry. `backends.py` +only adapts fake, trace, and live inputs to SDK calls. These are concrete flat +modules, not additional protocol or service layers. + +Only `EvaluationBackend` and `CandidateGenerator` are protocols because each has +real fake, trace, and live substitutions. Attribution, comparison, normalization, +and gate logic are ordinary pure functions. + +The example does not extend or patch the SDK. `backends.py` composes the exported +`LocalEvalService`, `RemoteEvalService`, and `InMemoryEvalSetsManager` APIs and +rebuilds the exported `EvaluateResult` aggregate at the example boundary. A +run-local `EvaluatorRegistry` lets fake and trace modes replace LLM judges without +mutating global registry state. The SDK remains the owner of remote metric +compatibility: black-box evaluation rejects trajectory and knowledge-recall +metrics because callback results do not expose their required intermediate data. + +Offline substitutions preserve each metric's contract. `llm_final_response` uses +deterministic exact-reference scoring. Rubric metrics require an explicit +machine-readable offline rule in each rubric's `type`; natural-language rubric +text is never guessed or treated as a pass. Response rules can compare with the +reference, compare or search literal rubric content, or require a non-empty +response. Knowledge rules inspect configured knowledge-tool responses and are +trace-only because black-box callbacks do not expose intermediate data. Each +rubric is scored independently. Normalization preserves its ID, score, pass +status, and reason; attribution consumes only failed rubric outcomes. Unsupported +rules, missing operands, or unavailable evidence fail the evaluation instead of +fabricating a score. + +Candidate generators return independent cost sources instead of a lossy aggregate. +The live optimizer source reports only reflection calls and optimizer cost; a +separate unknown judge source records that judge calls and judge cost are not +available from the SDK result. Unknown accounting remains `null` through the cost +ledger, so the total is also unknown. Only the built-in deterministic generator +declares zero cost and zero model calls. Custom generators default to an explicit +`unreported` source; an enabled cost gate rejects it with `COST_UNAVAILABLE`. + +The report schema is `v2`. It intentionally replaces `rubricIds` with structured +`rubrics` and candidate aggregate accounting with source-level accounting; no +compatibility shim reconstructs the discarded `v1` information. + +## Lifecycle + +1. Preflight strictly parses config and datasets, rejects duplicate JSON keys, + validates split isolation, hashes every input, and validates trace pins. +2. The artifact directory is created exclusively; an existing run ID fails. +3. Baseline train and held-out validation are evaluated sequentially. +4. A seeded inner train/selection split is persisted. Only inner-train failure + attribution is passed to the candidate generator. +5. The candidate is generated with source updates disabled, then evaluated on + full train and held-out validation inside a verified temporary prompt context. +6. Pure comparison and gate functions produce the only business decision. +7. A report is persisted before optional apply. Apply occurs only on ACCEPT when + explicitly enabled, and every write is read back and hash verified. +8. A complete terminal write cycle is measured before the final duration is + recorded. The report excludes its own files from its embedded artifact list; + the manifest hashes the final JSON and Markdown files. + +The immutable `artifacts//` directory and its manifest are the authority. +Root-level report files are atomic latest-run snapshots for interactive use. They +are not published as a transaction, so a consumer that needs a coherent JSON and +Markdown pair reads the run ID from the root JSON and then verifies both files in +the corresponding immutable run directory. Unique temporary names prevent +concurrent latest-run publishers from corrupting or deleting each other's writes. + +Any evaluator, optimizer, normalization, comparison, gate, render, apply, write, +cancel, or system-exit failure restores and verifies the baseline prompt. A +restoration failure raises `PromptRestoreError`; it is never downgraded to +REJECT. Reports store no chain-of-thought and recursively redact credentials. +Live optimizer config, prompt sandbox and raw optimizer output stay in an OS +temporary directory; only sanitized known artifact types enter the audit tree. +The import boundary enforces file-count, per-file-byte and total-byte budgets +before reading optimizer output. This boundary protects audit storage; it is not +a sandbox for a programmatic generator, which is trusted in-process code. + +Live cancellation writes the optimizer stop signal and waits for cooperative +shutdown so an optimization thread is not silently abandoned. Python cannot +safely force-kill an in-process thread. Deployments requiring a hard timeout must +place live optimization in a supervised process or container and enforce the +deadline there. + +## Industrial Acceptance + +The executable acceptance matrix covers every orchestration stage, cancellation, +partial backend failures, failed candidate accounting, malicious artifact sizes, +reference-free attribution, fake/trace replay, and live adapter shutdown. Local +policy matrices assert every declared attribution category and gate decision; +they are contract tests, not an independent accuracy benchmark. Issue-level +accuracy thresholds must be measured by a separately owned hidden corpus. Fake +and trace runs must finish within 180 seconds and produce hash-valid immutable +manifests. The dependency test is an allowlist for all flat pipeline modules and +rejects any new pipeline module, reverse import, or cycle until the architecture +contract is updated deliberately. + +The pipeline is roughly 3,700 non-empty lines and its tests roughly 2,150. That +size reflects immutable manifests, verified prompt rollback and apply, strict +SDK-result normalization, partial ERROR reports, reproducibility metadata, and +three execution modes. The implementation stays in one flat package with explicit +ownership and an import allowlist. Line count is tracked as a review signal, not +used as a reason to merge unrelated responsibilities or introduce facade layers. +The largest modules are validated result models, normalization, backend adapters, +report persistence, and the composition root. diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 000000000..d97ade400 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,59 @@ +# Evaluation + Optimization Loop + +This example runs a baseline on train and held-out validation data, attributes +failures, generates a candidate from an inner training split, reruns full +regression, and makes a deterministic gate decision. Every completed run writes +an immutable, authoritative audit directory under `artifacts//`. The +generated `optimization_report.json` and `optimization_report.md` files in this +directory are ignored atomic latest-run convenience snapshots, not a +transactional report pair. Consumers that need a coherent pair must read the JSON +run ID and then use the two immutable files and manifest under +`artifacts//`. + +The default fake mode is deterministic, offline, and does not read API keys: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --run-id local-fake +``` + +Trace mode replays the hash-pinned fixture through the SDK evaluator: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --run-id local-trace +``` + +In fake and trace modes, LLM rubric metrics use explicit deterministic rules in +each rubric's `type`. Response rubrics support `OFFLINE_RESPONSE_EXACT_REFERENCE`, +`OFFLINE_RESPONSE_EQUALS`, `OFFLINE_RESPONSE_CONTAINS`, and +`OFFLINE_RESPONSE_NON_EMPTY`. Trace knowledge rubrics support +`OFFLINE_KNOWLEDGE_CONTAINS` and `OFFLINE_KNOWLEDGE_NON_EMPTY`. Literal operands +come from `content.text`. Natural-language-only rubrics and knowledge recall in +fake black-box mode fail fast because they cannot be scored deterministically. +Reports retain each rubric's ID, score, pass status, and reason. Failure +attribution uses only rubric outcomes that failed their metric threshold. + +Live mode requires an importable async `query -> str` callback. The optimizer is +always called with `update_source=False`; only this pipeline's gate may apply a +candidate: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --mode live --call-agent my_package.agent:call_agent --run-id live-review +``` + +Source prompts remain unchanged by default. Add `--apply-candidate` only when a +gate ACCEPT should be written back. REJECT is a completed audit run and exits 0; +ERROR exits non-zero. Run IDs are immutable and cannot be reused. A reproducible +report command preserves all effective inputs but appends `-replay` to the run ID +so it can execute without overwriting the authoritative original run. + +Cost is reported by source. Any unreported source makes the total cost unknown; +when a cost budget is enabled, the gate fails closed with `COST_UNAVAILABLE`. + +Optimizer artifacts are accepted only through a sanitized allowlist and bounded +by configurable file-count, per-file-byte, and total-byte limits. Live optimizer +cancellation is cooperative. A production deployment that requires a hard +deadline must run the live optimizer in a supervised worker process or container +and terminate that worker at the platform boundary. + +See [DESIGN.md](DESIGN.md) for stage contracts and failure semantics. diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 000000000..f907ae0f8 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,82 @@ +{ + "evaluate": { + "metrics": [ + { + "metricName": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "finalResponse": { + "text": { + "match": "exact", + "caseInsensitive": false + } + } + } + } + ], + "numRuns": 1 + }, + "optimize": { + "evalCaseParallelism": 1, + "stop": { + "requiredMetrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflectionLm": { + "modelName": "${TRPC_AGENT_MODEL_NAME}", + "baseUrl": "${TRPC_AGENT_BASE_URL}", + "apiKey": "${TRPC_AGENT_API_KEY}", + "generationConfig": { + "maxTokens": 1024, + "temperature": 0.2 + } + }, + "reflectionMinibatchSize": 2, + "skipPerfectScore": false, + "maxMetricCalls": 12 + } + }, + "pipeline": { + "mode": "fake", + "seed": 42, + "innerSelectionRatio": 0.34, + "applyCandidate": false, + "artifactRoot": "artifacts", + "traceFixture": "traces/trace_cases.json", + "maxImportFiles": 256, + "maxImportFileBytes": 5242880, + "maxImportTotalBytes": 26214400, + "promptPaths": { + "system": "prompts/system.md" + }, + "criticalCaseIds": [ + "validation_stable" + ], + "hardCaseIds": [ + "validation_regress" + ], + "metricWeights": { + "final_response_avg_score": 1.0 + }, + "attribution": { + "metricCategories": { + "final_response_avg_score": "FINAL_RESPONSE_MISMATCH" + } + }, + "gate": { + "minValidationScoreDelta": 0.05, + "minValidationPassRateDelta": 0.0, + "maxNewHardFailures": 0, + "criticalCaseMinDelta": 0.0, + "metricMaxRegression": { + "final_response_avg_score": 0.0 + }, + "maxCostUsd": null, + "maxDurationSeconds": 180.0, + "epsilon": 1e-09, + "overfitGuard": true + } + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline/__init__.py b/examples/optimization/eval_optimize_loop/pipeline/__init__.py new file mode 100644 index 000000000..80dac5c6b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/__init__.py @@ -0,0 +1 @@ +"""Auditable evaluation and prompt-optimization pipeline example.""" diff --git a/examples/optimization/eval_optimize_loop/pipeline/artifacts.py b/examples/optimization/eval_optimize_loop/pipeline/artifacts.py new file mode 100644 index 000000000..92174c201 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/artifacts.py @@ -0,0 +1,191 @@ +"""Immutable audit directory, recursive sanitization and content manifest.""" + +from __future__ import annotations + +import hashlib +import json +import os +import threading +import time +import uuid +from pathlib import Path +from typing import Any + +from .models import ArtifactRecord +from .schema import parse_strict_json, sanitize, sanitized_text, validate_safe_component + +_REPORT_FILES = {"optimization_report.json", "optimization_report.md"} +_TEXT_ARTIFACT_SUFFIXES = {".log", ".md", ".txt"} +_ATOMIC_REPLACE_LOCK = threading.Lock() +_ATOMIC_REPLACE_ATTEMPTS = 8 + + +def load_strict_json(path: str | Path) -> dict[str, Any]: + return parse_strict_json(Path(path).read_text(encoding="utf-8")) + + +class AuditSink: + """Own safe paths and atomic persistence beneath one immutable run directory.""" + + def __init__( + self, + artifact_root: str | Path, + run_id: str, + *, + max_text_chars: int, + publication_root: str | Path | None = None, + max_import_files: int = 256, + max_import_file_bytes: int = 5 * 1024 * 1024, + max_import_total_bytes: int = 25 * 1024 * 1024, + ) -> None: + validate_safe_component(run_id, name="run ID") + self.root = Path(artifact_root).resolve() + self.publication_root = Path(publication_root).resolve() if publication_root is not None else self.root.parent + self.run_id = run_id + self.run_dir = self.root / run_id + self.max_text_chars = max_text_chars + self.max_import_files = max_import_files + self.max_import_file_bytes = max_import_file_bytes + self.max_import_total_bytes = max_import_total_bytes + + def create(self) -> Path: + self.root.mkdir(parents=True, exist_ok=True) + self.run_dir.mkdir(exist_ok=False) + return self.run_dir + + def phase_dir(self, name: str) -> Path: + validate_safe_component(name, name="stage name") + path = self.run_dir / name + path.mkdir(exist_ok=False) + return path + + def _resolve(self, relative_path: str) -> Path: + raw = Path(relative_path) + if raw.is_absolute() or not raw.parts: + raise ValueError("artifact path must be relative") + for part in raw.parts: + validate_safe_component(part, name="artifact path component") + resolved = (self.run_dir / raw).resolve() + if self.run_dir not in resolved.parents: + raise ValueError("artifact path escapes the run directory") + return resolved + + @staticmethod + def _atomic_write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + temporary.write_text(content, encoding="utf-8", newline="\n") + with _ATOMIC_REPLACE_LOCK: + for attempt in range(_ATOMIC_REPLACE_ATTEMPTS): + try: + os.replace(temporary, path) + break + except PermissionError: + if attempt + 1 == _ATOMIC_REPLACE_ATTEMPTS: + raise + time.sleep(0.005 * (2**attempt)) + finally: + temporary.unlink(missing_ok=True) + + def write_json(self, relative_path: str, payload: Any) -> Path: + path = self._resolve(relative_path) + clean = sanitize(payload, max_text_chars=self.max_text_chars) + content = json.dumps(clean, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False) + "\n" + self._atomic_write(path, content) + return path + + def write_text(self, relative_path: str, content: str) -> Path: + path = self._resolve(relative_path) + clean = sanitized_text(content, max_text_chars=self.max_text_chars) + self._atomic_write(path, clean) + return path + + def write_jsonl(self, relative_path: str, payloads: list[dict[str, Any]]) -> Path: + path = self._resolve(relative_path) + lines = [ + json.dumps( + sanitize(payload, max_text_chars=self.max_text_chars), + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) for payload in payloads + ] + self._atomic_write(path, "\n".join(lines) + ("\n" if lines else "")) + return path + + def import_tree(self, source: str | Path, destination: str) -> None: + """Import known optimizer artifacts through the shared sanitization boundary.""" + + source_root = Path(source).resolve() + if not source_root.exists(): + return + paths = sorted(item for item in source_root.rglob("*") if item.is_file()) + if len(paths) > self.max_import_files: + raise ValueError(f"optimizer artifact count exceeds limit: {len(paths)} > {self.max_import_files}") + total_bytes = 0 + for path in paths: + if path.is_symlink(): + raise ValueError("optimizer artifact symlinks are not allowed") + byte_size = path.stat().st_size + if byte_size > self.max_import_file_bytes: + raise ValueError(f"optimizer artifact exceeds per-file byte limit: " + f"{path.name} ({byte_size} > {self.max_import_file_bytes})") + total_bytes += byte_size + if total_bytes > self.max_import_total_bytes: + raise ValueError(f"optimizer artifacts exceed total byte limit: " + f"{total_bytes} > {self.max_import_total_bytes}") + relative = path.relative_to(source_root) + target = (Path(destination) / relative).as_posix() + suffix = path.suffix.casefold() + if suffix == ".json": + self.write_json(target, parse_strict_json(path.read_text(encoding="utf-8"))) + elif suffix == ".jsonl": + payloads = [ + parse_strict_json(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip() + ] + self.write_jsonl(target, payloads) + elif suffix in _TEXT_ARTIFACT_SUFFIXES: + self.write_text(target, path.read_text(encoding="utf-8")) + else: + raise ValueError(f"unsupported optimizer artifact type: {relative.as_posix()}") + + def records( + self, + *, + include_manifest: bool = False, + include_report_files: bool = True, + ) -> tuple[ArtifactRecord, ...]: + records: list[ArtifactRecord] = [] + for path in sorted(item for item in self.run_dir.rglob("*") if item.is_file()): + relative = path.relative_to(self.run_dir).as_posix() + if not include_manifest and relative == "manifest.json": + continue + if not include_report_files and relative in _REPORT_FILES: + continue + content = path.read_bytes() + records.append( + ArtifactRecord( + path=relative, + sha256=hashlib.sha256(content).hexdigest(), + byte_size=len(content), + )) + return tuple(records) + + def write_manifest(self) -> tuple[ArtifactRecord, ...]: + records = self.records() + self.write_json( + "manifest.json", + { + "schemaVersion": "v1", + "files": [record.model_dump(by_alias=True) for record in records] + }, + ) + return records + + def publish_latest_snapshot(self, source_name: str, destination_name: str) -> Path: + source = self._resolve(source_name) + validate_safe_component(destination_name, name="latest report name") + destination = self.publication_root / destination_name + self._atomic_write(destination, source.read_text(encoding="utf-8")) + return destination diff --git a/examples/optimization/eval_optimize_loop/pipeline/attribution.py b/examples/optimization/eval_optimize_loop/pipeline/attribution.py new file mode 100644 index 000000000..1f85f3283 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/attribution.py @@ -0,0 +1,209 @@ +"""Deterministic, evidence-based failure attribution without I/O or model calls.""" + +from __future__ import annotations + +from typing import Any + +from trpc_agent_sdk.evaluation import Invocation, get_all_tool_calls + +from .configuration import AttributionConfig +from .models import ( + AttributionPair, + AttributionSnapshot, + EvaluationSnapshot, + FailureAttribution, + FailureCategory, + SnapshotPair, +) +from .schema import sanitized_text + +_PRIORITY = tuple(FailureCategory) +_KNOWN_METRICS = { + "tool_trajectory_avg_score": FailureCategory.TOOL_CALL_ERROR, + "llm_rubric_knowledge_recall": FailureCategory.KNOWLEDGE_RECALL_INSUFFICIENT, + "llm_rubric_response": FailureCategory.LLM_RUBRIC_NOT_MET, + "llm_final_response": FailureCategory.LLM_RUBRIC_NOT_MET, + "response_evaluation_score": FailureCategory.LLM_RUBRIC_NOT_MET, + "final_response_avg_score": FailureCategory.FINAL_RESPONSE_MISMATCH, + "response_match_score": FailureCategory.FINAL_RESPONSE_MISMATCH, +} + + +def _safe_text(value: Any, max_chars: int) -> str: + return sanitized_text(value, max_text_chars=max_chars) + + +def _tool_calls(invocation: dict[str, Any]) -> list[tuple[str, Any]]: + parsed = Invocation.model_validate(invocation) + return [(str(call.name), call.args) for call in get_all_tool_calls(parsed.intermediate_data)] + + +def _tool_differences(case: Any, max_chars: int) -> list[tuple[FailureCategory, str]]: + differences: list[tuple[FailureCategory, str]] = [] + for run in case.runs: + for trace in run.trace: + actual = _tool_calls(trace["actual"]) + expected_payload = trace["expected"] + if expected_payload is None: + continue + expected = _tool_calls(expected_payload) + actual_names = [name for name, _ in actual] + expected_names = [name for name, _ in expected] + if actual_names != expected_names: + differences.append(( + FailureCategory.TOOL_CALL_ERROR, + _safe_text({ + "expectedToolNames": expected_names, + "actualToolNames": actual_names + }, max_chars), + )) + continue + argument_diffs = [{ + "tool": actual_item[0], + "expected": expected_item[1], + "actual": actual_item[1] + } for actual_item, expected_item in zip(actual, expected) if actual_item[1] != expected_item[1]] + if argument_diffs: + differences.append((FailureCategory.TOOL_ARGUMENT_ERROR, _safe_text(argument_diffs, max_chars))) + return differences + + +def _fallback_category(reason: str) -> FailureCategory: + lowered = reason.casefold() + if "format" in lowered or "schema" in lowered: + return FailureCategory.FORMAT_VIOLATION + if "argument" in lowered or "parameter" in lowered: + return FailureCategory.TOOL_ARGUMENT_ERROR + if "tool" in lowered or "function" in lowered: + return FailureCategory.TOOL_CALL_ERROR + if "knowledge" in lowered or "recall" in lowered or "retriev" in lowered: + return FailureCategory.KNOWLEDGE_RECALL_INSUFFICIENT + if "rubric" in lowered or "judge" in lowered: + return FailureCategory.LLM_RUBRIC_NOT_MET + if "response" in lowered or "mismatch" in lowered or "answer" in lowered: + return FailureCategory.FINAL_RESPONSE_MISMATCH + return FailureCategory.UNKNOWN + + +def attribute_failures( + snapshot: EvaluationSnapshot, + config: AttributionConfig, + *, + max_text_chars: int, +) -> AttributionSnapshot: + """Attribute each failed case using the documented precedence order.""" + + failures: list[FailureAttribution] = [] + for case in snapshot.cases: + if case.passed: + continue + candidates: list[FailureCategory] = [] + reasons: list[str] = [] + evidence: list[str] = [] + trigger_metrics: list[str] = [] + trigger_rubrics: list[str] = [] + authoritative = False + if case.error: + candidates.append(FailureCategory.EVALUATION_ERROR) + reasons.append("Evaluator reported an execution error.") + evidence.append(_safe_text(case.error, max_text_chars)) + authoritative = True + + tool_differences = _tool_differences(case, max_text_chars) + structured_tool_categories = {category for category, _ in tool_differences} + for category, item_evidence in tool_differences: + candidates.append(category) + reasons.append("Structured expected and actual tool trajectories differ.") + evidence.append(item_evidence) + authoritative = True + + for run in case.runs: + for metric in run.metrics: + if metric.passed: + continue + trigger_metrics.append(metric.metric_name) + reason = metric.reason or ( + f"Metric {metric.metric_name} scored {metric.score:g} below threshold {metric.threshold:g}.") + reasons.append(_safe_text(reason, max_text_chars)) + evidence.append( + _safe_text( + { + "metric": metric.metric_name, + "score": metric.score, + "threshold": metric.threshold, + }, + max_text_chars, + )) + if metric.metric_name in config.metric_categories: + candidates.append(config.metric_categories[metric.metric_name]) + authoritative = True + for rubric in metric.rubrics: + if rubric.passed: + continue + trigger_rubrics.append(rubric.id) + if rubric.id in config.rubric_categories: + candidates.append(config.rubric_categories[rubric.id]) + authoritative = True + if metric.metric_name in _KNOWN_METRICS and not (metric.metric_name == "tool_trajectory_avg_score" + and structured_tool_categories): + candidates.append(_KNOWN_METRICS[metric.metric_name]) + authoritative = True + elif not authoritative: + candidates.append(_fallback_category(reason)) + + categories = sorted(set(candidates or [FailureCategory.UNKNOWN]), key=_PRIORITY.index) + primary = categories[0] + confidence = ("high" if primary in { + FailureCategory.EVALUATION_ERROR, + FailureCategory.TOOL_CALL_ERROR, + FailureCategory.TOOL_ARGUMENT_ERROR, + } else "medium" if authoritative else "low") + failures.append( + FailureAttribution( + case_id=case.case_id, + primary=primary, + secondary=tuple(categories[1:]), + reasons=tuple(dict.fromkeys(reasons or ["The failure has no mapped deterministic cause."])), + trigger_metrics=tuple(dict.fromkeys(trigger_metrics)), + trigger_rubrics=tuple(dict.fromkeys(trigger_rubrics)), + evidence=tuple(dict.fromkeys(evidence or ["No structured evidence was available."])), + confidence=confidence, + )) + return AttributionSnapshot(split=snapshot.split, phase=snapshot.phase, failures=tuple(failures)) + + +def attribute_pair( + snapshots: SnapshotPair, + config: AttributionConfig, + *, + max_text_chars: int, +) -> AttributionPair: + """Attribute a complete train/validation phase without side effects.""" + + if snapshots.train is None or snapshots.validation is None: + raise ValueError("attribution requires complete train and validation snapshots") + return AttributionPair( + train=attribute_failures( + snapshots.train, + config, + max_text_chars=max_text_chars, + ), + validation=attribute_failures( + snapshots.validation, + config, + max_text_chars=max_text_chars, + ), + ) + + +def select_attribution( + snapshot: AttributionSnapshot, + case_ids: set[str] | frozenset[str], +) -> AttributionSnapshot: + """Restrict attribution facts to the cases visible to a downstream stage.""" + + return AttributionSnapshot( + split=snapshot.split, + phase=snapshot.phase, + failures=tuple(failure for failure in snapshot.failures if failure.case_id in case_ids), + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/backends.py b/examples/optimization/eval_optimize_loop/pipeline/backends.py new file mode 100644 index 000000000..c8e880836 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/backends.py @@ -0,0 +1,443 @@ +"""SDK-backed fake, trace and live adapters plus candidate generators.""" + +from __future__ import annotations + +import asyncio +import hashlib +import inspect +from copy import deepcopy +from pathlib import Path +from typing import Awaitable, Callable, Optional + +from trpc_agent_sdk.evaluation import ( + AgentOptimizer, + EvalConfig, + EvalSet, + EvalSetAggregateResult, + EvaluatorRegistry, + EvaluateConfig, + EvaluateRequest, + EvaluateResult, + InMemoryEvalSetsManager, + InferenceConfig, + InferenceRequest, + LocalEvalService, + RemoteEvalService, + TargetPrompt, +) + +from .contracts import CandidateGenerator, EvaluationBackend +from .models import ( + AttributionSnapshot, + CandidateProposal, + CandidateRound, + CostSource, + Phase, + Split, +) +from .offline_evaluation import prepare_offline_evaluation +from .schema import add_exception_note, parse_strict_json + +CallAgent = Callable[[str], Awaitable[str]] +_DEFAULT_APP_NAME = "test_app" + + +async def _evaluate_with_service( + eval_set: EvalSet, + eval_config: EvalConfig, + *, + call_agent: Optional[CallAgent], + runtime_dir: str, + evaluator_registry: Optional[EvaluatorRegistry] = None, +) -> EvaluateResult: + """Compose public eval services and preserve the SDK aggregate contract.""" + + dataset = deepcopy(eval_set) + config = deepcopy(eval_config) + runtime_root = Path(runtime_dir) + if not runtime_root.is_dir(): + raise FileNotFoundError(f"backend audit directory does not exist: {runtime_root}") + if config.num_runs < 1: + raise ValueError("evaluation num_runs must be at least one") + + app_name = dataset.app_name or _DEFAULT_APP_NAME + manager = InMemoryEvalSetsManager() + manager.create_eval_set(app_name=app_name, eval_set_id=dataset.eval_set_id) + for eval_case in dataset.eval_cases: + manager.add_eval_case( + app_name=app_name, + eval_set_id=dataset.eval_set_id, + eval_case=eval_case, + ) + + if call_agent is None: + service = LocalEvalService( + root_agent=None, + eval_sets_manager=manager, + evaluator_registry=evaluator_registry, + ) + else: + service = RemoteEvalService( + call_agent=call_agent, + eval_sets_manager=manager, + evaluator_registry=evaluator_registry, + ) + + inference_results = [] + inference_request = InferenceRequest( + app_name=app_name, + eval_set_id=dataset.eval_set_id, + inference_config=InferenceConfig(), + ) + for run_id in range(1, config.num_runs + 1): + async for inference_result in service.perform_inference(inference_request): + inference_result.run_id = run_id + inference_results.append(inference_result) + + results_by_case = {} + evaluate_request = EvaluateRequest( + inference_results=inference_results, + evaluate_config=EvaluateConfig(eval_metrics=config.get_eval_metrics()), + ) + async for case_result in service.evaluate(evaluate_request): + results_by_case.setdefault(case_result.eval_id, []).append(case_result) + + return EvaluateResult( + results_by_eval_set_id={ + dataset.eval_set_id: EvalSetAggregateResult( + eval_results_by_eval_id=results_by_case, + num_runs=config.num_runs, + ) + }) + + +async def _evaluate_offline( + eval_set: EvalSet, + eval_config: EvalConfig, + *, + call_agent: Optional[CallAgent], + runtime_dir: str, +) -> EvaluateResult: + """Evaluate using a run-local deterministic replacement registry.""" + + offline_config, registry = prepare_offline_evaluation(eval_config) + return await _evaluate_with_service( + eval_set, + offline_config, + call_agent=call_agent, + runtime_dir=runtime_dir, + evaluator_registry=registry, + ) + + +def _prompt_profile(prompts: dict[str, str]) -> bool: + return any("BEHAVIOR_PROFILE=precision-v2" in text for text in prompts.values()) + + +def deterministic_fake_response(query: str, prompts: dict[str, str]) -> str: + """Offline agent used by the public example; behavior is data-driven, not ID-driven.""" + + fields: dict[str, str] = {} + for component in query.split(";"): + if ":" in component: + key, value = component.split(":", 1) + fields[key.strip().upper()] = value.strip() + behavior = fields.get("BEHAVIOR") + answer = fields.get("ANSWER") + if behavior not in {"improve", "regress", "stable"} or answer is None: + return "INVALID_REQUEST" + candidate = _prompt_profile(prompts) + if behavior == "improve": + return answer if candidate else "INCORRECT" + if behavior == "regress": + return "INCORRECT" if candidate else answer + return answer + + +class FakeEvaluationBackend: + """Deterministic evaluator that never reads API credentials or uses the network.""" + + async def evaluate( + self, + *, + eval_set: EvalSet, + eval_config: EvalConfig, + prompts: dict[str, str], + split: Split, + phase: Phase, + audit_dir: str, + ) -> EvaluateResult: + del split, phase + prompt_copy = deepcopy(prompts) + + async def call_agent(query: str) -> str: + return deterministic_fake_response(query, prompt_copy) + + return await _evaluate_offline( + eval_set, + eval_config, + call_agent=call_agent, + runtime_dir=audit_dir, + ) + + +class TraceEvaluationBackend: + """Replay actual conversations from a hash-pinned fixture through LocalEvalService.""" + + def __init__( + self, + fixture_path: str, + dataset_hashes: dict[str, str], + fixture_hash: Optional[str] = None, + ) -> None: + self._fixture_path = Path(fixture_path) + self._dataset_hashes = dict(dataset_hashes) + if not self._fixture_path.is_file(): + raise FileNotFoundError(f"trace fixture does not exist: {self._fixture_path}") + content = self._fixture_path.read_bytes() + actual_hash = hashlib.sha256(content).hexdigest() + if fixture_hash is not None and actual_hash != fixture_hash: + raise ValueError("trace fixture changed after preflight") + self._fixture_hash = actual_hash + self._payload = parse_strict_json(content.decode("utf-8")) + + def validate_fixture(self, train: EvalSet, validation: EvalSet) -> None: + """Fail before run creation when any pinned trace input has drifted.""" + + for phase in Phase: + self._trace_eval_set(train, Split.TRAIN, phase) + self._trace_eval_set(validation, Split.VALIDATION, phase) + + def _trace_eval_set(self, eval_set: EvalSet, split: Split, phase: Phase) -> EvalSet: + payload = self._payload + if payload.get("schemaVersion") != "v1": + raise ValueError("unsupported trace fixture schemaVersion") + recorded_hashes = payload.get("datasetHashes") + if recorded_hashes != self._dataset_hashes: + raise ValueError("trace fixture dataset hashes do not match validated inputs") + try: + cases = payload["phases"][phase.value][split.value] + except (KeyError, TypeError) as error: + raise ValueError(f"trace fixture is missing {phase.value}/{split.value}") from error + expected_ids = [case.eval_id for case in eval_set.eval_cases] + if not isinstance(cases, dict) or set(cases) != set(expected_ids): + raise ValueError("trace fixture case IDs do not match the dataset") + raw = eval_set.model_dump(mode="json", by_alias=True, exclude_none=True) + for case in raw["evalCases"]: + conversation = deepcopy(cases[case["evalId"]]) + if not isinstance(conversation, list) or not conversation: + raise ValueError("each trace fixture conversation must be non-empty") + case["evalMode"] = "trace" + case["actualConversation"] = conversation + return EvalSet.model_validate(raw) + + async def evaluate( + self, + *, + eval_set: EvalSet, + eval_config: EvalConfig, + prompts: dict[str, str], + split: Split, + phase: Phase, + audit_dir: str, + ) -> EvaluateResult: + del prompts + traced = self._trace_eval_set(deepcopy(eval_set), split, phase) + return await _evaluate_offline( + traced, + eval_config, + call_agent=None, + runtime_dir=audit_dir, + ) + + +class LiveEvaluationBackend: + """Run an injected async business callback through RemoteEvalService.""" + + def __init__(self, call_agent: CallAgent) -> None: + if not inspect.iscoroutinefunction(call_agent): + raise TypeError("live call_agent must be an async function") + self._call_agent = call_agent + + async def evaluate( + self, + *, + eval_set: EvalSet, + eval_config: EvalConfig, + prompts: dict[str, str], + split: Split, + phase: Phase, + audit_dir: str, + ) -> EvaluateResult: + del prompts, split, phase + return await _evaluate_with_service( + eval_set, + eval_config, + call_agent=self._call_agent, + runtime_dir=audit_dir, + ) + + +class DeterministicCandidateGenerator: + """Generate one auditable candidate solely from inner-train failure facts.""" + + async def generate( + self, + *, + target_prompt: TargetPrompt, + baseline_prompts: dict[str, str], + train_attribution: AttributionSnapshot, + inner_train_path: str, + inner_selection_path: str, + config_path: str, + output_dir: str, + ) -> CandidateProposal: + del inner_train_path, inner_selection_path, config_path, output_dir + if set(target_prompt.names()) != set(baseline_prompts): + raise ValueError("candidate target fields do not match baseline prompts") + changed = bool(train_attribution.failures) + prompts = dict(baseline_prompts) + rounds: tuple[CandidateRound, ...] = () + if changed: + for name in prompts: + prompts[name] = prompts[name].rstrip() + "\n\nBEHAVIOR_PROFILE=precision-v2\n" + rounds = (CandidateRound( + round=1, + candidate_prompts=prompts, + accepted=True, + score=1.0, + kind="deterministic", + optimized_fields=tuple(prompts), + acceptance_reason="deterministic failure-attribution rewrite", + cost_usd=0, + ), ) + proposal = CandidateProposal( + algorithm="deterministic_failure_rewrite", + baseline_prompts=dict(baseline_prompts), + prompts=prompts, + changed=changed, + stop_reason="completed", + rounds=rounds, + cost_sources=(CostSource(name="deterministic", cost_usd=0, model_calls=0, metric_calls=0), ), + duration_seconds=0, + ) + return CandidateProposal.model_validate(proposal.model_dump(mode="python", by_alias=True)) + + +class LiveCandidateGenerator: + """Adapt AgentOptimizer output into the strict candidate-only contract.""" + + def __init__(self, call_agent: CallAgent, *, verbose: int = 0) -> None: + if not inspect.iscoroutinefunction(call_agent): + raise TypeError("live call_agent must be an async function") + self._call_agent = call_agent + self._verbose = verbose + + async def generate( + self, + *, + target_prompt: TargetPrompt, + baseline_prompts: dict[str, str], + train_attribution: AttributionSnapshot, + inner_train_path: str, + inner_selection_path: str, + config_path: str, + output_dir: str, + ) -> CandidateProposal: + del train_attribution + optimize_task = asyncio.create_task( + AgentOptimizer.optimize( + config_path=config_path, + call_agent=self._call_agent, + target_prompt=target_prompt, + train_dataset_path=inner_train_path, + validation_dataset_path=inner_selection_path, + output_dir=output_dir, + update_source=False, + verbose=self._verbose, + )) + try: + result = await asyncio.shield(optimize_task) + except asyncio.CancelledError as cancellation: + stop_path = Path(output_dir) / "optimize.stop" + try: + stop_path.parent.mkdir(parents=True, exist_ok=True) + stop_path.write_text("cancel requested\n", encoding="utf-8") + except OSError as stop_error: + add_exception_note( + cancellation, + f"could not request optimizer stop: {stop_error}", + ) + try: + await optimize_task + except BaseException as shutdown_error: + add_exception_note( + cancellation, + f"optimizer shutdown completed with {type(shutdown_error).__name__}: " + f"{shutdown_error}", + ) + raise + if result.status == "CANCELED": + raise asyncio.CancelledError(result.error_message or "AgentOptimizer canceled") + if result.status != "SUCCEEDED": + raise RuntimeError(f"AgentOptimizer candidate generation failed: {result.error_message}") + rounds = tuple( + CandidateRound( + round=round_.round, + candidate_prompts=dict(round_.candidate_prompts), + accepted=round_.accepted, + score=(round_.validation_pass_rate if round_.candidate_prompts and not getattr( + round_, "skip_reason", None) and not getattr(round_, "error_message", None) else None), + kind=getattr(round_, "kind", "reflective"), + optimized_fields=tuple(getattr(round_, "optimized_field_names", ())), + metric_scores=dict(getattr(round_, "metric_breakdown", {})), + acceptance_reason=getattr(round_, "acceptance_reason", None) or None, + skip_reason=getattr(round_, "skip_reason", None), + error_message=getattr(round_, "error_message", None), + cost_usd=round_.round_llm_cost, + token_usage=dict(round_.round_token_usage), + duration_seconds=round_.duration_seconds, + ) for round_ in result.rounds) + proposal = CandidateProposal( + algorithm=result.algorithm, + baseline_prompts=dict(result.baseline_prompts), + prompts=dict(result.best_prompts), + changed=result.best_prompts != baseline_prompts, + stop_reason=result.stop_reason, + rounds=rounds, + cost_sources=( + CostSource( + name="optimizer_reported", + cost_usd=result.total_llm_cost, + model_calls=result.total_reflection_lm_calls, + token_usage=dict(result.total_token_usage), + ), + CostSource(name="judge_unreported", cost_usd=None, model_calls=None), + ), + duration_seconds=result.duration_seconds, + ) + return CandidateProposal.model_validate(proposal.model_dump(mode="python", by_alias=True)) + + +def create_backends( + mode: str, + *, + call_agent: Optional[CallAgent] = None, + trace_fixture_path: Optional[str] = None, + trace_fixture_hash: Optional[str] = None, + dataset_hashes: Optional[dict[str, str]] = None, +) -> tuple[EvaluationBackend, CandidateGenerator]: + if mode == "fake": + return FakeEvaluationBackend(), DeterministicCandidateGenerator() + if mode == "trace": + if not trace_fixture_path or dataset_hashes is None: + raise ValueError("trace mode requires a fixture path and dataset hashes") + return ( + TraceEvaluationBackend(trace_fixture_path, dataset_hashes, trace_fixture_hash), + DeterministicCandidateGenerator(), + ) + if mode == "live": + if call_agent is None: + raise ValueError("live mode requires call_agent") + return LiveEvaluationBackend(call_agent), LiveCandidateGenerator(call_agent) + raise ValueError(f"unsupported mode: {mode!r}") diff --git a/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py b/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py new file mode 100644 index 000000000..5d726dc56 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py @@ -0,0 +1,121 @@ +"""Isolated candidate-generation workspace and sanitized artifact ingestion.""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path + +from trpc_agent_sdk.evaluation import EvalConfig, EvalSet, OptimizeConfig + +from .artifacts import AuditSink +from .attribution import select_attribution +from .contracts import CandidateGenerator +from .evaluation import dataset_fingerprint, split_train_dataset +from .models import ( + AttributionSnapshot, + CandidateProposal, + EvaluationSnapshot, + InnerSplit, +) +from .prompt_workspace import PromptWorkspace +from .schema import add_exception_note + + +def prepare_candidate_inputs( + *, + train: EvalSet, + baseline: EvaluationSnapshot, + attribution: AttributionSnapshot, + seed: int, + selection_ratio: float, + sink: AuditSink, +) -> tuple[InnerSplit, AttributionSnapshot, Path, Path]: + """Create and persist the optimizer-only inner split and attribution.""" + + inner_train, inner_selection = split_train_dataset( + train, + baseline, + seed=seed, + selection_ratio=selection_ratio, + ) + train_path = sink.write_json( + "inner_train.evalset.json", + inner_train.model_dump(mode="json", by_alias=True), + ) + selection_path = sink.write_json( + "inner_selection.evalset.json", + inner_selection.model_dump(mode="json", by_alias=True), + ) + inner = InnerSplit( + train_case_ids=tuple(case.eval_id for case in inner_train.eval_cases), + selection_case_ids=tuple(case.eval_id for case in inner_selection.eval_cases), + train_hash=dataset_fingerprint(inner_train), + selection_hash=dataset_fingerprint(inner_selection), + train_path=train_path.relative_to(sink.run_dir).as_posix(), + selection_path=selection_path.relative_to(sink.run_dir).as_posix(), + ) + selected = select_attribution(attribution, frozenset(inner.train_case_ids)) + sink.write_json( + "inner_train.attribution.json", + selected.model_dump(mode="json", by_alias=True), + ) + return inner, selected, train_path, selection_path + + +async def generate_candidate( + *, + generator: CandidateGenerator, + workspace: PromptWorkspace, + sink: AuditSink, + eval_config: EvalConfig, + optimize_config: OptimizeConfig, + train_attribution: AttributionSnapshot, + inner_train_path: Path, + inner_selection_path: Path, +) -> CandidateProposal: + """Run a generator outside the audit tree, then import only sanitized output.""" + + sink.phase_dir("candidate_generation") + with tempfile.TemporaryDirectory(prefix="trpc-agent-eval-optimize-") as runtime_value: + runtime_root = Path(runtime_value) + candidate_target = workspace.create_candidate_target(str(runtime_root / "prompt_sandbox")) + runtime_config = runtime_root / "optimizer.json" + runtime_config.write_text( + json.dumps( + { + "evaluate": eval_config.model_dump(mode="json", by_alias=True), + "optimize": optimize_config.model_dump(mode="json", by_alias=True), + }, + ensure_ascii=False, + allow_nan=False, + ), + encoding="utf-8", + ) + optimizer_output = runtime_root / "optimizer-output" + try: + proposal = await generator.generate( + target_prompt=candidate_target, + baseline_prompts=workspace.baseline, + train_attribution=train_attribution, + inner_train_path=str(inner_train_path), + inner_selection_path=str(inner_selection_path), + config_path=str(runtime_config), + output_dir=str(optimizer_output), + ) + except BaseException as primary_error: + try: + sink.import_tree(optimizer_output, "candidate_generation/optimizer") + except Exception as import_error: + add_exception_note( + primary_error, + f"optimizer artifact import also failed with " + f"{type(import_error).__name__}: {import_error}", + ) + raise + else: + sink.import_tree(optimizer_output, "candidate_generation/optimizer") + validated = CandidateProposal.model_validate(proposal.model_dump(mode="python", by_alias=True)) + if validated.baseline_prompts != workspace.baseline: + raise ValueError("candidate generator baseline prompts differ from the workspace baseline") + return validated diff --git a/examples/optimization/eval_optimize_loop/pipeline/configuration.py b/examples/optimization/eval_optimize_loop/pipeline/configuration.py new file mode 100644 index 000000000..90230c8ab --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/configuration.py @@ -0,0 +1,144 @@ +"""Validated input and policy configuration contracts.""" + +from __future__ import annotations + +from typing import Optional + +from pydantic import ( + Field, + StrictBool, + StrictFloat, + StrictInt, + field_validator, + model_validator, +) + +from trpc_agent_sdk.evaluation import EvalConfig, EvalSet, OptimizeConfig + +from .models import FailureCategory, RunMode +from .schema import ( + StrictModel, + finite_number as _finite, + non_negative_number as _non_negative, +) + + +class AttributionConfig(StrictModel): + metric_categories: dict[str, FailureCategory] = Field(default_factory=dict) + rubric_categories: dict[str, FailureCategory] = Field(default_factory=dict) + + +class GateConfig(StrictModel): + min_validation_score_delta: StrictFloat = 0.0 + min_validation_pass_rate_delta: StrictFloat = 0.0 + max_new_hard_failures: StrictInt = Field(default=0, ge=0) + critical_case_min_delta: StrictFloat = 0.0 + metric_max_regression: dict[str, StrictFloat] = Field(default_factory=dict) + max_cost_usd: Optional[StrictFloat] = None + max_duration_seconds: Optional[StrictFloat] = 180.0 + epsilon: StrictFloat = 1e-9 + overfit_guard: StrictBool = True + + @field_validator( + "min_validation_score_delta", + "min_validation_pass_rate_delta", + "critical_case_min_delta", + ) + @classmethod + def finite_threshold(cls, value: float) -> float: + return _finite(value, "gate threshold") + + @field_validator("epsilon") + @classmethod + def valid_epsilon(cls, value: float) -> float: + return _non_negative(value, "epsilon") + + @field_validator("max_cost_usd", "max_duration_seconds") + @classmethod + def valid_budget(cls, value: Optional[float]) -> Optional[float]: + return None if value is None else _non_negative(value, "budget") + + @field_validator("metric_max_regression") + @classmethod + def valid_metric_regression(cls, value: dict[str, float]) -> dict[str, float]: + for metric, limit in value.items(): + if not metric: + raise ValueError("metric regression keys must be non-empty") + _non_negative(limit, f"metric_max_regression[{metric!r}]") + return value + + +class PipelineSettings(StrictModel): + mode: RunMode = "fake" + seed: StrictInt = 42 + inner_selection_ratio: StrictFloat = 0.34 + apply_candidate: StrictBool = False + artifact_root: str = "artifacts" + trace_fixture: str = "traces/trace_cases.json" + run_id: Optional[str] = None + prompt_paths: dict[str, str] = Field(default_factory=lambda: {"system": "prompts/system.md"}) + critical_case_ids: tuple[str, ...] = () + hard_case_ids: tuple[str, ...] = () + metric_weights: dict[str, StrictFloat] = Field(default_factory=dict) + train_case_weights: dict[str, StrictFloat] = Field(default_factory=dict) + validation_case_weights: dict[str, StrictFloat] = Field(default_factory=dict) + max_text_chars: StrictInt = Field(default=4000, ge=64, le=100_000) + max_import_files: StrictInt = Field(default=256, ge=1, le=10_000) + max_import_file_bytes: StrictInt = Field(default=5 * 1024 * 1024, ge=1, le=100 * 1024 * 1024) + max_import_total_bytes: StrictInt = Field(default=25 * 1024 * 1024, ge=1, le=500 * 1024 * 1024) + attribution: AttributionConfig = Field(default_factory=AttributionConfig) + gate: GateConfig = Field(default_factory=GateConfig) + + @model_validator(mode="after") + def valid_import_budget(self) -> "PipelineSettings": + if self.max_import_total_bytes < self.max_import_file_bytes: + raise ValueError("max_import_total_bytes must be at least max_import_file_bytes") + return self + + @field_validator("inner_selection_ratio") + @classmethod + def valid_ratio(cls, value: float) -> float: + value = _finite(value, "inner_selection_ratio") + if not 0 < value < 1: + raise ValueError("inner_selection_ratio must be in (0, 1)") + return value + + @field_validator("metric_weights", "train_case_weights", "validation_case_weights") + @classmethod + def positive_weights(cls, value: dict[str, float]) -> dict[str, float]: + for key, weight in value.items(): + if not key: + raise ValueError("weight keys must be non-empty") + if _finite(weight, f"weight[{key!r}]") <= 0: + raise ValueError("weights must be positive") + return value + + @field_validator("prompt_paths") + @classmethod + def non_empty_prompts(cls, value: dict[str, str]) -> dict[str, str]: + if not value: + raise ValueError("at least one prompt path is required") + return value + + +class PipelineConfig(StrictModel): + evaluate: EvalConfig + optimize: OptimizeConfig + pipeline: PipelineSettings = Field(default_factory=PipelineSettings) + + +class ValidatedRunConfig(StrictModel): + root_dir: str + config_path: str + train_path: str + validation_path: str + trace_fixture_path: Optional[str] + artifact_root: str + run_id: str + config: PipelineConfig + train: EvalSet + validation: EvalSet + input_hashes: dict[str, str] + prompt_paths: dict[str, str] + prompt_hashes: dict[str, str] + adapter_identity: str diff --git a/examples/optimization/eval_optimize_loop/pipeline/contracts.py b/examples/optimization/eval_optimize_loop/pipeline/contracts.py new file mode 100644 index 000000000..e7677e6d3 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/contracts.py @@ -0,0 +1,42 @@ +"""The two deliberate substitution boundaries used by the pipeline.""" + +from __future__ import annotations + +from typing import Protocol + +from trpc_agent_sdk.evaluation import EvalConfig, EvalSet, EvaluateResult, TargetPrompt + +from .models import AttributionSnapshot, CandidateProposal, Phase, Split + + +class EvaluationBackend(Protocol): + """Evaluate an immutable request and return the SDK's raw result object.""" + + async def evaluate( # noqa: E704 + self, + *, + eval_set: EvalSet, + eval_config: EvalConfig, + prompts: dict[str, str], + split: Split, + phase: Phase, + audit_dir: str, + ) -> EvaluateResult: + ... + + +class CandidateGenerator(Protocol): + """Propose prompts from inner-training evidence without making the gate decision.""" + + async def generate( # noqa: E704 + self, + *, + target_prompt: TargetPrompt, + baseline_prompts: dict[str, str], + train_attribution: AttributionSnapshot, + inner_train_path: str, + inner_selection_path: str, + config_path: str, + output_dir: str, + ) -> CandidateProposal: + ... diff --git a/examples/optimization/eval_optimize_loop/pipeline/costing.py b/examples/optimization/eval_optimize_loop/pipeline/costing.py new file mode 100644 index 000000000..912cc13e3 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/costing.py @@ -0,0 +1,37 @@ +"""Completed-call cost ledger and deterministic source merge.""" + +from __future__ import annotations + +from .models import CostSource, CostSummary + + +class CostLedger: + """Run-local ledger; callers record a source only after its call completes.""" + + def __init__(self) -> None: + self._sources: list[CostSource] = [] + + def record( + self, + name: str, + *, + cost_usd: float | None, + model_calls: int | None = None, + metric_calls: int | None = None, + token_usage: dict[str, int] | None = None, + ) -> None: + if any(source.name == name for source in self._sources): + raise ValueError(f"cost source already recorded: {name!r}") + self._sources.append( + CostSource( + name=name, + cost_usd=cost_usd, + model_calls=model_calls, + metric_calls=metric_calls, + token_usage=dict(token_usage or {}), + )) + + def summary(self) -> CostSummary: + known = all(source.cost_usd is not None for source in self._sources) + total = sum(source.cost_usd or 0 for source in self._sources) if known else None + return CostSummary(sources=tuple(self._sources), total_cost_usd=total) diff --git a/examples/optimization/eval_optimize_loop/pipeline/evaluation.py b/examples/optimization/eval_optimize_loop/pipeline/evaluation.py new file mode 100644 index 000000000..64c59ef14 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/evaluation.py @@ -0,0 +1,490 @@ +"""Pure dataset validation, deterministic splitting, normalization and comparison.""" + +from __future__ import annotations + +import hashlib +import json +import math +from copy import deepcopy +from typing import Any, Iterable, Mapping + +from trpc_agent_sdk.evaluation import EvalConfig, EvalSet, EvalStatus, EvaluateResult + +from .models import ( + AttributionSnapshot, + CaseComparison, + CaseRun, + CaseSnapshot, + ComparisonSnapshot, + EvaluationSnapshot, + FailureAttribution, + MetricDelta, + MetricRun, + MetricSnapshot, + Phase, + RubricOutcome, + Split, + Transition, +) + +_VOLATILE_KEYS = { + "invocationId", + "invocation_id", + "callId", + "call_id", + "toolCallId", + "tool_call_id", + "executionId", + "execution_id", + "eventId", + "event_id", + "creationTimestamp", + "creation_timestamp", +} + + +def canonical_json(value: Any) -> str: + """Return deterministic JSON suitable for content hashing.""" + + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def sha256_json(value: Any) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def _without_execution_ids(value: Any) -> Any: + if isinstance(value, dict): + return {key: _without_execution_ids(item) for key, item in value.items() if key not in _VOLATILE_KEYS} + if isinstance(value, list): + return [_without_execution_ids(item) for item in value] + return value + + +def case_content_fingerprint(case: Any) -> str: + payload = case.model_dump(mode="json", by_alias=True) + payload.pop("evalId", None) + return sha256_json(_without_execution_ids(payload)) + + +def case_input_fingerprint(case: Any) -> str: + """Fingerprint only model-visible inputs so changed labels cannot hide leakage.""" + + invocations = case.conversation or case.actual_conversation or [] + payload = { + "userContents": [invocation.user_content.model_dump(mode="json", by_alias=True) for invocation in invocations], + "conversationScenario": (case.conversation_scenario.model_dump(mode="json", by_alias=True) + if case.conversation_scenario is not None else None), + "sessionInput": + (case.session_input.model_dump(mode="json", by_alias=True) if case.session_input is not None else None), + "contextMessages": [item.model_dump(mode="json", by_alias=True) for item in (case.context_messages or [])], + } + return sha256_json(_without_execution_ids(payload)) + + +def dataset_fingerprint(eval_set: EvalSet) -> str: + payload = eval_set.model_dump(mode="json", by_alias=True) + return sha256_json(_without_execution_ids(payload)) + + +def _validate_case_ids(eval_set: EvalSet, label: str, minimum: int) -> set[str]: + if len(eval_set.eval_cases) < minimum: + raise ValueError(f"{label} must contain at least {minimum} cases") + case_ids = [case.eval_id for case in eval_set.eval_cases] + if any(not case_id or case_id.strip() != case_id for case_id in case_ids): + raise ValueError(f"{label} case IDs must be non-empty and already trimmed") + if len(case_ids) != len(set(case_ids)): + raise ValueError(f"{label} case IDs must be unique") + return set(case_ids) + + +def validate_datasets( + train: EvalSet, + validation: EvalSet, + *, + train_path: str, + validation_path: str, + configured_metrics: Iterable[str], + critical_case_ids: Iterable[str] = (), + hard_case_ids: Iterable[str] = (), + metric_weights: Mapping[str, float] | None = None, + train_case_weights: Mapping[str, float] | None = None, + validation_case_weights: Mapping[str, float] | None = None, +) -> None: + """Validate split isolation and all configured references.""" + + if train_path.casefold() == validation_path.casefold(): + raise ValueError("train and validation paths must differ") + train_ids = _validate_case_ids(train, "train", 3) + validation_ids = _validate_case_ids(validation, "validation", 1) + overlap = train_ids & validation_ids + if overlap: + raise ValueError(f"train and validation share case IDs: {sorted(overlap)}") + + train_fingerprints = {case_content_fingerprint(case) for case in train.eval_cases} + validation_fingerprints = {case_content_fingerprint(case) for case in validation.eval_cases} + if train_fingerprints & validation_fingerprints: + raise ValueError("train and validation contain duplicate case content") + train_inputs = {case_input_fingerprint(case) for case in train.eval_cases} + validation_inputs = {case_input_fingerprint(case) for case in validation.eval_cases} + if train_inputs & validation_inputs: + raise ValueError("train and validation contain duplicate model inputs") + + validation_references = set(critical_case_ids) | set(hard_case_ids) + unknown_validation = validation_references - validation_ids + if unknown_validation: + raise ValueError(f"critical/hard cases are missing from validation: {sorted(unknown_validation)}") + + metric_list = list(configured_metrics) + if not metric_list or any(not name or name != name.strip() for name in metric_list): + raise ValueError("configured metric names must be non-empty and trimmed") + if len(metric_list) != len(set(metric_list)): + raise ValueError("configured metric names must be unique") + metrics = set(metric_list) + unknown_metrics = set(metric_weights or {}) - metrics + if unknown_metrics: + raise ValueError(f"metric weights reference unknown metrics: {sorted(unknown_metrics)}") + unknown_train = set(train_case_weights or {}) - train_ids + if unknown_train: + raise ValueError(f"train weights reference unknown cases: {sorted(unknown_train)}") + unknown_val = set(validation_case_weights or {}) - validation_ids + if unknown_val: + raise ValueError(f"validation weights reference unknown cases: {sorted(unknown_val)}") + + +def split_train_dataset( + train: EvalSet, + baseline: EvaluationSnapshot, + *, + seed: int, + selection_ratio: float, +) -> tuple[EvalSet, EvalSet]: + """Return deterministic inner-train and selection sets, stratified when possible.""" + + case_ids = [case.eval_id for case in train.eval_cases] + if tuple(case_ids) != baseline.case_ids: + raise ValueError("baseline train snapshot does not match train dataset") + if len(case_ids) < 3: + raise ValueError("inner split requires at least three train cases") + selection_count = 1 if len(case_ids) == 3 else round(len(case_ids) * selection_ratio) + selection_count = max(1, min(len(case_ids) - 1, selection_count)) + + passed = {case.case_id: case.passed for case in baseline.cases} + + def rank(case_id: str) -> str: + return hashlib.sha256(f"{seed}:{case_id}".encode("utf-8")).hexdigest() + + groups = { + False: sorted((case_id for case_id in case_ids if not passed[case_id]), key=rank), + True: sorted((case_id for case_id in case_ids if passed[case_id]), key=rank), + } + selection: list[str] = [] + if selection_count >= 2 and groups[False] and groups[True]: + selection.extend((groups[False].pop(0), groups[True].pop(0))) + remaining = sorted(groups[False] + groups[True], key=rank) + selection.extend(remaining[:selection_count - len(selection)]) + selection_ids = set(selection) + + train_cases = [deepcopy(case) for case in train.eval_cases if case.eval_id not in selection_ids] + selection_cases = [deepcopy(case) for case in train.eval_cases if case.eval_id in selection_ids] + inner_train_update = {"eval_set_id": f"{train.eval_set_id}_inner_train", "eval_cases": train_cases} + inner_selection_update = {"eval_set_id": f"{train.eval_set_id}_inner_selection", "eval_cases": selection_cases} + inner_train = train.model_copy(update=inner_train_update, deep=True) + inner_selection = train.model_copy(update=inner_selection_update, deep=True) + return inner_train, inner_selection + + +def _metric_config(eval_config: EvalConfig) -> tuple[list[str], dict[str, float]]: + metrics = eval_config.get_eval_metrics() + names = [metric.metric_name for metric in metrics] + if not names or len(names) != len(set(names)): + raise ValueError("configured metric names must be non-empty and unique") + thresholds = {metric.metric_name: float(metric.threshold) for metric in metrics} + if any(not math.isfinite(value) for value in thresholds.values()): + raise ValueError("configured metric thresholds must be finite") + return names, thresholds + + +def _status_passed(status: EvalStatus) -> bool: + if status == EvalStatus.NOT_EVALUATED: + raise ValueError("metric status cannot be NOT_EVALUATED") + return status == EvalStatus.PASSED + + +def _same_model(left: Any, right: Any) -> bool: + if left is None or right is None: + return left is right + return left.model_dump(mode="json", by_alias=True) == right.model_dump(mode="json", by_alias=True) + + +def _normalize_run( + result: Any, + *, + expected_case: Any, + expected_run_id: int, + eval_set_id: str, + metric_names: list[str], + thresholds: dict[str, float], +) -> CaseRun: + if result.eval_set_id != eval_set_id or result.eval_id != expected_case.eval_id: + raise ValueError("run eval-set/case identity drift") + if result.run_id != expected_run_id: + raise ValueError("run IDs must be contiguous and 1-based") + overall = result.overall_eval_metric_results + if [item.metric_name for item in overall] != metric_names: + raise ValueError("overall metric schema drift") + + has_reference = bool(expected_case.conversation) + expected_invocations = expected_case.conversation or expected_case.actual_conversation or [] + per_invocation = result.eval_metric_result_per_invocation + if len(per_invocation) != len(expected_invocations): + raise ValueError("invocation count drift") + trace: list[dict[str, Any]] = [] + for index, invocation_result in enumerate(per_invocation): + expected = expected_invocations[index] + if has_reference: + if not _same_model(invocation_result.expected_invocation, expected): + raise ValueError("expected invocation drift") + else: + placeholder = invocation_result.expected_invocation + if (placeholder is None or placeholder.final_response is not None + or placeholder.intermediate_data is not None + or not _same_model(placeholder.user_content, expected.user_content)): + raise ValueError("reference-free expected invocation drift") + if not _same_model(invocation_result.actual_invocation.user_content, expected.user_content): + raise ValueError("actual user content drift") + per_metric = invocation_result.eval_metric_results + if [item.metric_name for item in per_metric] != metric_names: + raise ValueError("per-invocation metric schema drift") + for item in per_metric: + if item.score is None or not math.isfinite(float(item.score)): + raise ValueError("per-invocation metric score must be finite") + if float(item.threshold) != thresholds[item.metric_name]: + raise ValueError("per-invocation metric threshold drift") + if _status_passed(item.eval_status) != (float(item.score) >= float(item.threshold)): + raise ValueError("per-invocation metric status drift") + trace.append({ + "actual": invocation_result.actual_invocation.model_dump(mode="json", by_alias=True), + "expected": (expected.model_dump(mode="json", by_alias=True) if has_reference else None), + }) + + metrics: list[MetricRun] = [] + for item in overall: + if item.score is None or not math.isfinite(float(item.score)): + raise ValueError("overall metric score must be finite") + if float(item.threshold) != thresholds[item.metric_name]: + raise ValueError("overall metric threshold drift") + passed = _status_passed(item.eval_status) + if passed != (float(item.score) >= float(item.threshold)): + raise ValueError("overall metric status drift") + details = item.details + rubrics: list[RubricOutcome] = [] + if details and details.rubric_scores: + for rubric in details.rubric_scores: + if isinstance(rubric, Mapping): + rubric_id = rubric.get("rubricId") or rubric.get("rubric_id") or rubric.get("id") + rubric_score = rubric.get("score") + rubric_reason = rubric.get("reason") + else: + rubric_id = getattr(rubric, "rubric_id", None) or getattr(rubric, "id", None) + rubric_score = getattr(rubric, "score", None) + rubric_reason = getattr(rubric, "reason", None) + if not rubric_id or rubric_score is None: + raise ValueError("rubric result requires a non-empty id and score") + score = float(rubric_score) + rubrics.append( + RubricOutcome( + id=str(rubric_id), + score=score, + passed=score >= float(item.threshold), + reason=str(rubric_reason) if rubric_reason is not None else None, + )) + metrics.append( + MetricRun( + metric_name=item.metric_name, + score=float(item.score), + threshold=float(item.threshold), + passed=passed, + reason=details.reason if details else None, + rubrics=tuple(rubrics), + )) + final_passed = _status_passed(result.final_eval_status) + expected_passed = result.error_message is None and all(metric.passed for metric in metrics) + if final_passed != expected_passed: + raise ValueError("final run status drift") + return CaseRun( + run_id=expected_run_id, + passed=final_passed, + metrics=tuple(metrics), + error=result.error_message, + trace=tuple(trace), + ) + + +def _weighted_average(values: Iterable[tuple[float, float]]) -> float: + pairs = list(values) + total_weight = sum(weight for _, weight in pairs) + if not pairs or total_weight <= 0: + raise ValueError("weighted average requires positive total weight") + return sum(value * weight for value, weight in pairs) / total_weight + + +def normalize_result( + raw: EvaluateResult, + eval_set: EvalSet, + eval_config: EvalConfig, + *, + split: Split, + phase: Phase, + metric_weights: Mapping[str, float] | None = None, + case_weights: Mapping[str, float] | None = None, +) -> EvaluationSnapshot: + """Validate and unfold an SDK EvaluateResult exactly once.""" + + if set(raw.results_by_eval_set_id) != {eval_set.eval_set_id}: + raise ValueError("EvaluateResult must contain exactly the expected eval set") + aggregate = raw.results_by_eval_set_id[eval_set.eval_set_id] + if aggregate.num_runs != eval_config.num_runs: + raise ValueError("aggregate run count does not match eval config") + expected_ids = [case.eval_id for case in eval_set.eval_cases] + if set(aggregate.eval_results_by_eval_id) != set(expected_ids): + raise ValueError("EvaluateResult case IDs do not match input dataset") + metric_names, thresholds = _metric_config(eval_config) + metric_weight_map = {name: float((metric_weights or {}).get(name, 1.0)) for name in metric_names} + + cases: list[CaseSnapshot] = [] + for expected_case in eval_set.eval_cases: + raw_runs = aggregate.eval_results_by_eval_id[expected_case.eval_id] + if len(raw_runs) != eval_config.num_runs: + raise ValueError("case run count does not match eval config") + by_run_id = {run.run_id: run for run in raw_runs} + if set(by_run_id) != set(range(1, eval_config.num_runs + 1)): + raise ValueError("case run IDs are missing or duplicated") + runs = tuple( + _normalize_run( + by_run_id[run_id], + expected_case=expected_case, + expected_run_id=run_id, + eval_set_id=eval_set.eval_set_id, + metric_names=metric_names, + thresholds=thresholds, + ) for run_id in range(1, eval_config.num_runs + 1)) + metric_snapshots: list[MetricSnapshot] = [] + for metric_name in metric_names: + run_metrics = [next(metric for metric in run.metrics if metric.metric_name == metric_name) for run in runs] + score = sum(metric.score for metric in run_metrics) / len(run_metrics) + metric_snapshots.append( + MetricSnapshot( + metric_name=metric_name, + score=score, + threshold=thresholds[metric_name], + passed=all(metric.passed for metric in run_metrics), + )) + case_score = _weighted_average( + (metric.score, metric_weight_map[metric.metric_name]) for metric in metric_snapshots) + cases.append( + CaseSnapshot( + case_id=expected_case.eval_id, + passed=all(run.passed for run in runs), + score=case_score, + metrics=tuple(metric_snapshots), + runs=runs, + error=next((run.error for run in runs if run.error), None), + )) + + case_weight_map = {case.case_id: float((case_weights or {}).get(case.case_id, 1.0)) for case in cases} + dataset_score = _weighted_average((case.score, case_weight_map[case.case_id]) for case in cases) + metric_scores = { + metric_name: + _weighted_average(( + next(metric.score for metric in case.metrics if metric.metric_name == metric_name), + case_weight_map[case.case_id], + ) for case in cases) + for metric_name in metric_names + } + return EvaluationSnapshot( + split=split, + phase=phase, + dataset_score=dataset_score, + pass_rate=sum(case.passed for case in cases) / len(cases), + metric_scores=metric_scores, + case_ids=tuple(expected_ids), + cases=tuple(cases), + ) + + +def compare_snapshots( + baseline: EvaluationSnapshot, + candidate: EvaluationSnapshot, + *, + epsilon: float, + critical_case_ids: Iterable[str] = (), + hard_case_ids: Iterable[str] = (), + baseline_attribution: AttributionSnapshot | None = None, + candidate_attribution: AttributionSnapshot | None = None, +) -> ComparisonSnapshot: + """Compare schema-identical snapshots and classify every case transition.""" + + if baseline.split != candidate.split or baseline.case_ids != candidate.case_ids: + raise ValueError("baseline and candidate case schemas differ") + if set(baseline.metric_scores) != set(candidate.metric_scores): + raise ValueError("baseline and candidate metric schemas differ") + baseline_failures = _attribution_map(baseline_attribution) + candidate_failures = _attribution_map(candidate_attribution) + critical = set(critical_case_ids) + hard = set(hard_case_ids) + comparisons: list[CaseComparison] = [] + for base_case, new_case in zip(baseline.cases, candidate.cases): + if base_case.case_id != new_case.case_id: + raise ValueError("baseline and candidate case ordering differs") + base_metrics = {metric.metric_name: metric for metric in base_case.metrics} + new_metrics = {metric.metric_name: metric for metric in new_case.metrics} + if set(base_metrics) != set(new_metrics): + raise ValueError("baseline and candidate per-case metric schemas differ") + delta = new_case.score - base_case.score + if not base_case.passed and new_case.passed: + transition = Transition.NEW_PASS + elif base_case.passed and not new_case.passed: + transition = Transition.NEW_FAIL + elif delta > epsilon: + transition = Transition.IMPROVED + elif delta < -epsilon: + transition = Transition.REGRESSED + else: + transition = Transition.UNCHANGED + comparisons.append( + CaseComparison( + case_id=base_case.case_id, + baseline_passed=base_case.passed, + candidate_passed=new_case.passed, + baseline_score=base_case.score, + candidate_score=new_case.score, + delta=delta, + metrics=tuple( + MetricDelta( + metric_name=name, + baseline=base_metrics[name].score, + candidate=new_metrics[name].score, + delta=new_metrics[name].score - base_metrics[name].score, + ) for name in base_metrics), + transition=transition, + critical=base_case.case_id in critical, + hard=base_case.case_id in hard, + baseline_attribution=baseline_failures.get(base_case.case_id), + candidate_attribution=candidate_failures.get(base_case.case_id), + new_hard_failure=base_case.case_id in hard and base_case.passed and not new_case.passed, + )) + return ComparisonSnapshot( + split=baseline.split, + score_delta=candidate.dataset_score - baseline.dataset_score, + pass_rate_delta=candidate.pass_rate - baseline.pass_rate, + metric_deltas={ + name: candidate.metric_scores[name] - baseline.metric_scores[name] + for name in baseline.metric_scores + }, + cases=tuple(comparisons), + ) + + +def _attribution_map(snapshot: AttributionSnapshot | None) -> dict[str, FailureAttribution]: + return {} if snapshot is None else {failure.case_id: failure for failure in snapshot.failures} diff --git a/examples/optimization/eval_optimize_loop/pipeline/evaluation_runtime.py b/examples/optimization/eval_optimize_loop/pipeline/evaluation_runtime.py new file mode 100644 index 000000000..f7b61ad33 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/evaluation_runtime.py @@ -0,0 +1,145 @@ +"""Side-effecting evaluation stage runner around the pure normalizer.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from typing import Callable, Optional + +from trpc_agent_sdk.evaluation import EvalConfig, EvalSet + +from .artifacts import AuditSink +from .configuration import ValidatedRunConfig +from .contracts import EvaluationBackend +from .costing import CostLedger +from .evaluation import normalize_result +from .models import EvaluationSnapshot, Phase, SnapshotPair, Split + + +def create_evaluation_runtime( + *, + validated: ValidatedRunConfig, + backend: EvaluationBackend, + custom_backend: bool, + sink: AuditSink, + ledger: CostLedger, +) -> "EvaluationRuntime": + """Construct accounting policy once at the composition boundary.""" + + settings = validated.config.pipeline + offline = not custom_backend and settings.mode in {"fake", "trace"} + calls_per_invocation = (1 if not custom_backend and settings.mode == "fake" else + 0 if not custom_backend and settings.mode == "trace" else None) + return EvaluationRuntime( + backend=backend, + sink=sink, + ledger=ledger, + eval_config=validated.config.evaluate, + cost_usd=0 if offline else None, + model_calls_per_invocation=calls_per_invocation, + metric_weights=settings.metric_weights, + train_case_weights=settings.train_case_weights, + validation_case_weights=settings.validation_case_weights, + ) + + +@dataclass +class EvaluationRuntime: + """Own backend invocation, completed-call accounting and snapshot persistence.""" + + backend: EvaluationBackend + sink: AuditSink + ledger: CostLedger + eval_config: EvalConfig + cost_usd: float | None + model_calls_per_invocation: int | None + metric_weights: dict[str, float] + train_case_weights: dict[str, float] + validation_case_weights: dict[str, float] + + async def evaluate( + self, + *, + eval_set: EvalSet, + prompts: dict[str, str], + split: Split, + phase: Phase, + ) -> EvaluationSnapshot: + stage = f"{phase.value}_{split.value}" + phase_dir = self.sink.phase_dir(stage) + metric_count = len(self.eval_config.get_eval_metrics()) + invocations = sum(len(case.conversation or case.actual_conversation or []) for case in eval_set.eval_cases) + metric_calls = invocations * self.eval_config.num_runs * metric_count + try: + raw = await self.backend.evaluate( + eval_set=deepcopy(eval_set), + eval_config=deepcopy(self.eval_config), + prompts=deepcopy(prompts), + split=split, + phase=phase, + audit_dir=str(phase_dir), + ) + except BaseException: + self.ledger.record( + stage, + cost_usd=self.cost_usd, + model_calls=(0 if self.model_calls_per_invocation == 0 else None), + metric_calls=None, + ) + raise + model_calls = None + if self.model_calls_per_invocation is not None: + model_calls = invocations * self.eval_config.num_runs * self.model_calls_per_invocation + self.ledger.record( + stage, + cost_usd=self.cost_usd, + model_calls=model_calls, + metric_calls=metric_calls, + ) + snapshot = normalize_result( + raw, + eval_set, + self.eval_config, + split=split, + phase=phase, + metric_weights=self.metric_weights, + case_weights=(self.train_case_weights if split == Split.TRAIN else self.validation_case_weights), + ) + self.sink.write_json( + f"{stage}/snapshot.json", + snapshot.model_dump(mode="json", by_alias=True), + ) + return snapshot + + async def evaluate_phase( + self, + *, + train: EvalSet, + validation: EvalSet, + prompts: dict[str, str], + phase: Phase, + on_stage: Optional[Callable[[str], None]] = None, + on_progress: Optional[Callable[[Phase, SnapshotPair], None]] = None, + ) -> SnapshotPair: + """Evaluate train then validation for one phase in the fixed order.""" + + snapshots = {} + for split, eval_set in ( + (Split.TRAIN, train), + (Split.VALIDATION, validation), + ): + if on_stage is not None: + on_stage(f"{phase.value}_{split.value}") + snapshots[split] = await self.evaluate( + eval_set=eval_set, + prompts=prompts, + split=split, + phase=phase, + ) + progress = SnapshotPair( + train=snapshots.get(Split.TRAIN), + validation=snapshots.get(Split.VALIDATION), + ) + if on_progress is not None: + on_progress(phase, progress) + return progress diff --git a/examples/optimization/eval_optimize_loop/pipeline/gate.py b/examples/optimization/eval_optimize_loop/pipeline/gate.py new file mode 100644 index 000000000..4c240da85 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/gate.py @@ -0,0 +1,199 @@ +"""Pure deterministic acceptance gate with fixed AND semantics.""" + +from __future__ import annotations + +import math +from typing import Any + +from .configuration import GateConfig +from .models import ( + CandidateProposal, + ComparisonSnapshot, + CostSummary, + Decision, + GateCheck, + GateDecision, +) + + +def _check(code: str, passed: bool, observed: Any, threshold: Any, message: str) -> GateCheck: + return GateCheck(code=code, passed=passed, observed=observed, threshold=threshold, message=message) + + +def _invalid_input(message: str) -> GateDecision: + return GateDecision( + decision=Decision.ERROR, + accepted=False, + checks=(_check("REPORT_COMPLETE", False, "invalid", "complete", message), ), + reasons=("GATE_INPUT_INVALID", ), + ) + + +def evaluate_gate( + *, + train: ComparisonSnapshot, + validation: ComparisonSnapshot, + candidate: CandidateProposal, + cost: CostSummary, + duration_seconds: float, + baseline_prompt_hashes: dict[str, str], + candidate_prompt_hashes: dict[str, str], + config: GateConfig, + report_complete: bool = True, +) -> GateDecision: + """Evaluate all ten checks in their stable order.""" + + try: + if train.split.value != "train" or validation.split.value != "validation": + raise ValueError("comparison splits are misplaced") + if set(baseline_prompt_hashes) != set(candidate_prompt_hashes) or not baseline_prompt_hashes: + raise ValueError("prompt hash schemas are incomplete") + if not math.isfinite(duration_seconds) or duration_seconds < 0: + raise ValueError("duration must be finite and non-negative") + for value in ( + train.score_delta, + train.pass_rate_delta, + validation.score_delta, + validation.pass_rate_delta, + *train.metric_deltas.values(), + *validation.metric_deltas.values(), + ): + if not math.isfinite(value): + raise ValueError("gate inputs must be finite") + missing_metrics = set(config.metric_max_regression) - set(validation.metric_deltas) + if missing_metrics: + raise ValueError(f"configured metrics are missing: {sorted(missing_metrics)}") + except (AttributeError, TypeError, ValueError) as error: + return _invalid_input(str(error)) + + checks: list[GateCheck] = [] + failed_reasons: list[str] = [] + + def add( + code: str, + passed: bool, + observed: Any, + threshold: Any, + message: str, + reason: str, + ) -> None: + checks.append(_check(code, passed, observed, threshold, message)) + if not passed: + failed_reasons.append(reason) + + add( + "REPORT_COMPLETE", + report_complete, + report_complete, + True, + "All required regression snapshots and report inputs are present.", + "REPORT_INCOMPLETE", + ) + hashes_changed = baseline_prompt_hashes != candidate_prompt_hashes + add( + "CANDIDATE_CHANGED", + candidate.changed and hashes_changed, + hashes_changed, + True, + "Candidate prompt hashes differ from baseline.", + "CANDIDATE_UNCHANGED", + ) + score_passed = validation.score_delta + config.epsilon >= config.min_validation_score_delta + add( + "VALIDATION_SCORE_DELTA", + score_passed, + validation.score_delta, + config.min_validation_score_delta, + "Held-out validation score delta meets the configured minimum.", + "VALIDATION_SCORE_DELTA_BELOW_MINIMUM", + ) + pass_rate_passed = validation.pass_rate_delta + config.epsilon >= config.min_validation_pass_rate_delta + add( + "VALIDATION_PASS_RATE_DELTA", + pass_rate_passed, + validation.pass_rate_delta, + config.min_validation_pass_rate_delta, + "Held-out validation pass-rate delta meets the configured minimum.", + "VALIDATION_PASS_RATE_DELTA_BELOW_MINIMUM", + ) + new_hard_failures = sum(case.new_hard_failure for case in validation.cases) + add( + "NO_NEW_HARD_FAIL", + new_hard_failures <= config.max_new_hard_failures, + new_hard_failures, + config.max_new_hard_failures, + "New held-out hard failures stay within budget.", + "NEW_HARD_FAIL_BUDGET_EXCEEDED", + ) + + def is_critical_regression(case) -> bool: + new_failure = case.baseline_passed and not case.candidate_passed + below_delta = case.delta + config.epsilon < config.critical_case_min_delta + return case.critical and (new_failure or below_delta) + + critical_regressions = [case.case_id for case in validation.cases if is_critical_regression(case)] + add( + "CRITICAL_CASE_NON_REGRESSION", + not critical_regressions, + critical_regressions, + config.critical_case_min_delta, + "Critical held-out cases do not regress.", + "CRITICAL_CASE_REGRESSION", + ) + metric_regressions = { + metric: validation.metric_deltas[metric] + for metric, limit in config.metric_max_regression.items() + if validation.metric_deltas[metric] + config.epsilon < -limit + } + add( + "METRIC_NON_REGRESSION", + not metric_regressions, + metric_regressions, + config.metric_max_regression, + "Configured validation metrics stay within regression limits.", + "METRIC_REGRESSION", + ) + if config.max_cost_usd is None: + cost_passed, cost_observed, cost_reason = True, cost.total_cost_usd, "COST_BUDGET_EXCEEDED" + elif cost.total_cost_usd is None or any(source.cost_usd is None for source in cost.sources): + cost_passed, cost_observed, cost_reason = False, None, "COST_UNAVAILABLE" + else: + cost_passed = cost.total_cost_usd <= config.max_cost_usd + config.epsilon + cost_observed, cost_reason = cost.total_cost_usd, "COST_BUDGET_EXCEEDED" + add( + "COST_BUDGET", + cost_passed, + cost_observed, + config.max_cost_usd, + "Known enabled costs stay within budget.", + cost_reason, + ) + duration_passed = (config.max_duration_seconds is None + or duration_seconds <= config.max_duration_seconds + config.epsilon) + add( + "DURATION_BUDGET", + duration_passed, + duration_seconds, + config.max_duration_seconds, + "Gate-observed duration stays within budget.", + "DURATION_BUDGET_EXCEEDED", + ) + overfit = config.overfit_guard and train.score_delta > config.epsilon and not (score_passed and pass_rate_passed) + add( + "OVERFIT_GUARD", + not overfit, + { + "trainScoreDelta": train.score_delta, + "validationScoreDelta": validation.score_delta + }, + "train improvement requires held-out validation thresholds", + "Train-only improvement is rejected as overfitting.", + "OVERFIT_TRAIN_UP_VALIDATION_DOWN", + ) + accepted = all(check.passed for check in checks) + return GateDecision( + decision=Decision.ACCEPT if accepted else Decision.REJECT, + accepted=accepted, + checks=tuple(checks), + reasons=tuple(dict.fromkeys(failed_reasons)), + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py new file mode 100644 index 000000000..58f7a1cc9 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -0,0 +1,682 @@ +"""Strict, immutable stage contracts for the evaluation/optimization loop.""" + +from __future__ import annotations + +import math +from enum import Enum +from typing import Any, Literal, Optional + +from pydantic import Field, field_validator, model_validator + +from .schema import ( + StrictModel, + finite_number as _finite, + non_negative_number as _non_negative, +) + + +class Split(str, Enum): + TRAIN = "train" + VALIDATION = "validation" + + +class Phase(str, Enum): + BASELINE = "baseline" + CANDIDATE = "candidate" + + +class Transition(str, Enum): + NEW_PASS = "NEW_PASS" + NEW_FAIL = "NEW_FAIL" + IMPROVED = "IMPROVED" + REGRESSED = "REGRESSED" + UNCHANGED = "UNCHANGED" + + +class Decision(str, Enum): + ACCEPT = "ACCEPT" + REJECT = "REJECT" + ERROR = "ERROR" + + +class FailureCategory(str, Enum): + EVALUATION_ERROR = "EVALUATION_ERROR" + TOOL_CALL_ERROR = "TOOL_CALL_ERROR" + TOOL_ARGUMENT_ERROR = "TOOL_ARGUMENT_ERROR" + KNOWLEDGE_RECALL_INSUFFICIENT = "KNOWLEDGE_RECALL_INSUFFICIENT" + FORMAT_VIOLATION = "FORMAT_VIOLATION" + LLM_RUBRIC_NOT_MET = "LLM_RUBRIC_NOT_MET" + FINAL_RESPONSE_MISMATCH = "FINAL_RESPONSE_MISMATCH" + UNKNOWN = "UNKNOWN" + + +Confidence = Literal["low", "medium", "high"] +RunMode = Literal["fake", "trace", "live"] + + +class RubricOutcome(StrictModel): + id: str + score: float + passed: bool + reason: Optional[str] = None + + @field_validator("id") + @classmethod + def valid_id(cls, value: str) -> str: + if not value or value != value.strip(): + raise ValueError("rubric outcome id must be non-empty and trimmed") + return value + + @field_validator("score") + @classmethod + def finite_score(cls, value: float) -> float: + return _finite(value, "rubric score") + + +class MetricRun(StrictModel): + metric_name: str + score: float + threshold: float + passed: bool + reason: Optional[str] = None + rubrics: tuple[RubricOutcome, ...] = () + + @field_validator("score", "threshold") + @classmethod + def finite_score(cls, value: float) -> float: + return _finite(value, "metric value") + + @model_validator(mode="after") + def consistent_status(self) -> "MetricRun": + if self.passed != (self.score >= self.threshold): + raise ValueError("metric passed status does not match score >= threshold") + for rubric in self.rubrics: + if rubric.passed != (rubric.score >= self.threshold): + raise ValueError(f"rubric {rubric.id!r} passed status does not match metric threshold") + return self + + +class CaseRun(StrictModel): + run_id: int = Field(ge=1) + passed: bool + metrics: tuple[MetricRun, ...] + error: Optional[str] = None + trace: tuple[dict[str, Any], ...] = () + + @model_validator(mode="after") + def consistent_status(self) -> "CaseRun": + if not self.metrics: + raise ValueError("a case run must contain metrics") + expected = self.error is None and all(metric.passed for metric in self.metrics) + if self.passed != expected: + raise ValueError("case run passed status does not match metric statuses") + return self + + +class MetricSnapshot(StrictModel): + metric_name: str + score: float + threshold: float + passed: bool + + @field_validator("score", "threshold") + @classmethod + def finite_score(cls, value: float) -> float: + return _finite(value, "metric aggregate") + + +class CaseSnapshot(StrictModel): + case_id: str + passed: bool + score: float + metrics: tuple[MetricSnapshot, ...] + runs: tuple[CaseRun, ...] + error: Optional[str] = None + + @field_validator("score") + @classmethod + def finite_score(cls, value: float) -> float: + return _finite(value, "case score") + + @model_validator(mode="after") + def consistent_aggregate(self) -> "CaseSnapshot": + if not self.metrics or not self.runs: + raise ValueError("case snapshot requires metrics and runs") + metric_names = tuple(metric.metric_name for metric in self.metrics) + if len(set(metric_names)) != len(metric_names): + raise ValueError("case snapshot metric names must be unique") + if tuple(run.run_id for run in self.runs) != tuple(range(1, len(self.runs) + 1)): + raise ValueError("case snapshot run IDs must be contiguous and 1-based") + for run in self.runs: + if tuple(metric.metric_name for metric in run.metrics) != metric_names: + raise ValueError("case snapshot run metric schemas must match the aggregate") + if self.passed != all(run.passed for run in self.runs): + raise ValueError("case snapshot passed status does not match run statuses") + expected_error = next((run.error for run in self.runs if run.error), None) + if self.error != expected_error: + raise ValueError("case snapshot error does not match the first run error") + scores = [metric.score for metric in self.metrics] + if self.score < min(scores) - 1e-12 or self.score > max(scores) + 1e-12: + raise ValueError("case score must be a positive-weight aggregate of metric scores") + return self + + +class EvaluationSnapshot(StrictModel): + split: Split + phase: Phase + dataset_score: float + pass_rate: float + metric_scores: dict[str, float] + case_ids: tuple[str, ...] + cases: tuple[CaseSnapshot, ...] + + @field_validator("dataset_score", "pass_rate") + @classmethod + def finite_aggregate(cls, value: float) -> float: + return _finite(value, "evaluation aggregate") + + @field_validator("metric_scores") + @classmethod + def finite_metric_scores(cls, value: dict[str, float]) -> dict[str, float]: + for name, score in value.items(): + _finite(score, f"metric_scores[{name!r}]") + return value + + @model_validator(mode="after") + def complete_cases(self) -> "EvaluationSnapshot": + if not self.cases: + raise ValueError("evaluation snapshot requires at least one case") + if self.case_ids != tuple(case.case_id for case in self.cases): + raise ValueError("case_ids must match cases in order") + if len(set(self.case_ids)) != len(self.case_ids): + raise ValueError("evaluation snapshot case IDs must be unique") + metric_names = set(self.metric_scores) + if not metric_names or any(not name or name != name.strip() for name in metric_names): + raise ValueError("evaluation snapshot metric names must be non-empty and trimmed") + for case in self.cases: + if {metric.metric_name for metric in case.metrics} != metric_names: + raise ValueError("evaluation snapshot case metric schemas must match") + expected_pass_rate = sum(case.passed for case in self.cases) / len(self.cases) + if not math.isclose(self.pass_rate, expected_pass_rate, rel_tol=0, abs_tol=1e-12): + raise ValueError("pass_rate does not match case statuses") + case_scores = [case.score for case in self.cases] + if self.dataset_score < min(case_scores) - 1e-12 or self.dataset_score > max(case_scores) + 1e-12: + raise ValueError("dataset score must be a positive-weight aggregate of case scores") + for name, aggregate in self.metric_scores.items(): + scores = [ + next(metric.score for metric in case.metrics if metric.metric_name == name) for case in self.cases + ] + if aggregate < min(scores) - 1e-12 or aggregate > max(scores) + 1e-12: + raise ValueError(f"metric score {name!r} must be a positive-weight aggregate of case metrics") + return self + + +class FailureAttribution(StrictModel): + case_id: str + primary: FailureCategory + secondary: tuple[FailureCategory, ...] = () + reasons: tuple[str, ...] + trigger_metrics: tuple[str, ...] = () + trigger_rubrics: tuple[str, ...] = () + evidence: tuple[str, ...] + confidence: Confidence + + @model_validator(mode="after") + def explanatory(self) -> "FailureAttribution": + if not self.reasons or not self.evidence: + raise ValueError("failure attribution needs reasons and evidence") + if self.primary in self.secondary: + raise ValueError("primary category cannot also be secondary") + return self + + +class AttributionSnapshot(StrictModel): + split: Split + phase: Phase + failures: tuple[FailureAttribution, ...] + + +class InnerSplit(StrictModel): + train_case_ids: tuple[str, ...] + selection_case_ids: tuple[str, ...] + train_hash: str + selection_hash: str + train_path: str + selection_path: str + + +class CostSource(StrictModel): + name: str + cost_usd: Optional[float] + model_calls: Optional[int] = Field(default=None, ge=0) + metric_calls: Optional[int] = Field(default=None, ge=0) + token_usage: dict[str, int] = Field(default_factory=dict) + + @field_validator("name") + @classmethod + def valid_name(cls, value: str) -> str: + if not value or value != value.strip(): + raise ValueError("cost source name must be non-empty and trimmed") + return value + + @field_validator("cost_usd") + @classmethod + def valid_cost(cls, value: Optional[float]) -> Optional[float]: + return None if value is None else _non_negative(value, "cost") + + @field_validator("token_usage") + @classmethod + def valid_tokens(cls, value: dict[str, int]) -> dict[str, int]: + if any(amount < 0 for amount in value.values()): + raise ValueError("token usage must be non-negative") + return value + + +def _unreported_candidate_cost() -> tuple[CostSource, ...]: + return (CostSource(name="unreported", cost_usd=None), ) + + +class CandidateRound(StrictModel): + round: int = Field(ge=1) + candidate_prompts: dict[str, str] + accepted: bool + score: Optional[float] = None + kind: Literal["deterministic", "reflective", "merge"] = "reflective" + optimized_fields: tuple[str, ...] = () + metric_scores: dict[str, float] = Field(default_factory=dict) + acceptance_reason: Optional[str] = None + skip_reason: Optional[str] = None + error_message: Optional[str] = None + cost_usd: Optional[float] = None + token_usage: dict[str, int] = Field(default_factory=dict) + duration_seconds: float = 0.0 + + @field_validator("score") + @classmethod + def valid_score(cls, value: Optional[float]) -> Optional[float]: + if value is None: + return None + value = _finite(value, "round score") + if not 0 <= value <= 1: + raise ValueError("round score must be in [0, 1]") + return value + + @field_validator("optimized_fields") + @classmethod + def valid_optimized_fields(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if len(set(value)) != len(value) or any(not name or name != name.strip() for name in value): + raise ValueError("optimized fields must be non-empty and unique") + return value + + @field_validator("metric_scores") + @classmethod + def valid_metric_scores(cls, value: dict[str, float]) -> dict[str, float]: + for name, score in value.items(): + if not name or name != name.strip(): + raise ValueError("round metric names must be non-empty and trimmed") + _finite(score, f"round metric score {name!r}") + return value + + @field_validator("cost_usd") + @classmethod + def valid_cost(cls, value: Optional[float]) -> Optional[float]: + return None if value is None else _non_negative(value, "round cost") + + @field_validator("token_usage") + @classmethod + def valid_tokens(cls, value: dict[str, int]) -> dict[str, int]: + if any(amount < 0 for amount in value.values()): + raise ValueError("round token usage must be non-negative") + return value + + @field_validator("duration_seconds") + @classmethod + def valid_duration(cls, value: float) -> float: + return _non_negative(value, "round duration") + + @model_validator(mode="after") + def consistent_outcome(self) -> "CandidateRound": + if self.skip_reason and self.error_message: + raise ValueError("round cannot be both skipped and errored") + if self.accepted and (not self.candidate_prompts or self.skip_reason or self.error_message): + raise ValueError("accepted round must contain a normal candidate") + if self.candidate_prompts and not (self.skip_reason or self.error_message) and self.score is None: + raise ValueError("candidate round requires a validation score") + if not self.candidate_prompts and not (self.skip_reason or self.error_message): + raise ValueError("empty candidate round requires a skip or error reason") + if not self.candidate_prompts and self.optimized_fields: + raise ValueError("empty candidate round cannot report optimized fields") + return self + + +class CandidateProposal(StrictModel): + status: Literal["SUCCEEDED"] = "SUCCEEDED" + algorithm: str + baseline_prompts: dict[str, str] + prompts: dict[str, str] + changed: bool + stop_reason: Optional[str] = None + rounds: tuple[CandidateRound, ...] = () + cost_sources: tuple[CostSource, ...] = Field(default_factory=_unreported_candidate_cost) + duration_seconds: float = 0.0 + + @field_validator("duration_seconds") + @classmethod + def valid_duration(cls, value: float) -> float: + return _non_negative(value, "candidate duration") + + @model_validator(mode="after") + def valid_round_history(self) -> "CandidateProposal": + if set(self.baseline_prompts) != set(self.prompts): + raise ValueError("candidate prompt keys must equal baseline keys") + if self.changed != (self.prompts != self.baseline_prompts): + raise ValueError("candidate changed flag is inconsistent") + if tuple(round_.round for round_ in self.rounds) != tuple(range(1, len(self.rounds) + 1)): + raise ValueError("candidate rounds must be contiguous and 1-based") + source_names = tuple(source.name for source in self.cost_sources) + if not source_names or len(set(source_names)) != len(source_names): + raise ValueError("candidate cost source names must be non-empty and unique") + registered = set(self.prompts) + for round_ in self.rounds: + if round_.candidate_prompts and set(round_.candidate_prompts) != registered: + raise ValueError("candidate round prompt keys must equal registered prompt keys") + if not set(round_.optimized_fields) <= registered: + raise ValueError("round optimized fields must reference registered prompt keys") + final_was_accepted = any(round_.accepted and round_.candidate_prompts == self.prompts for round_ in self.rounds) + if self.changed and not final_was_accepted: + raise ValueError("changed final candidate must equal an accepted round") + return self + + +class MetricDelta(StrictModel): + metric_name: str + baseline: float + candidate: float + delta: float + + @field_validator("baseline", "candidate", "delta") + @classmethod + def finite_value(cls, value: float) -> float: + return _finite(value, "metric delta") + + @model_validator(mode="after") + def consistent_delta(self) -> "MetricDelta": + if not math.isclose( + self.delta, + self.candidate - self.baseline, + rel_tol=0, + abs_tol=1e-12, + ): + raise ValueError("metric delta does not match candidate minus baseline") + return self + + +class CaseComparison(StrictModel): + case_id: str + baseline_passed: bool + candidate_passed: bool + baseline_score: float + candidate_score: float + delta: float + metrics: tuple[MetricDelta, ...] + transition: Transition + critical: bool = False + hard: bool = False + baseline_attribution: Optional[FailureAttribution] = None + candidate_attribution: Optional[FailureAttribution] = None + new_hard_failure: bool = False + + @field_validator("baseline_score", "candidate_score", "delta") + @classmethod + def finite_score(cls, value: float) -> float: + return _finite(value, "case comparison score") + + @model_validator(mode="after") + def consistent_transition(self) -> "CaseComparison": + if not math.isclose( + self.delta, + self.candidate_score - self.baseline_score, + rel_tol=0, + abs_tol=1e-12, + ): + raise ValueError("case delta does not match candidate minus baseline") + metric_names = tuple(metric.metric_name for metric in self.metrics) + if len(set(metric_names)) != len(metric_names): + raise ValueError("case comparison metric names must be unique") + expected_new_pass = not self.baseline_passed and self.candidate_passed + expected_new_fail = self.baseline_passed and not self.candidate_passed + if expected_new_pass != (self.transition == Transition.NEW_PASS): + raise ValueError("NEW_PASS transition does not match case statuses") + if expected_new_fail != (self.transition == Transition.NEW_FAIL): + raise ValueError("NEW_FAIL transition does not match case statuses") + expected_hard_failure = self.hard and expected_new_fail + if self.new_hard_failure != expected_hard_failure: + raise ValueError("new hard failure flag does not match case statuses") + for attribution in (self.baseline_attribution, self.candidate_attribution): + if attribution is not None and attribution.case_id != self.case_id: + raise ValueError("case comparison attribution ID does not match the case") + return self + + +class ComparisonSnapshot(StrictModel): + split: Split + score_delta: float + pass_rate_delta: float + metric_deltas: dict[str, float] + cases: tuple[CaseComparison, ...] + + @field_validator("score_delta", "pass_rate_delta") + @classmethod + def finite_value(cls, value: float) -> float: + return _finite(value, "comparison delta") + + @field_validator("metric_deltas") + @classmethod + def finite_metric_deltas(cls, value: dict[str, float]) -> dict[str, float]: + for name, delta in value.items(): + if not name or name != name.strip(): + raise ValueError("comparison metric names must be non-empty and trimmed") + _finite(delta, f"comparison metric delta {name!r}") + return value + + @model_validator(mode="after") + def consistent_aggregate(self) -> "ComparisonSnapshot": + if not self.cases: + raise ValueError("comparison snapshot requires at least one case") + case_ids = tuple(case.case_id for case in self.cases) + if len(set(case_ids)) != len(case_ids): + raise ValueError("comparison case IDs must be unique") + metric_names = set(self.metric_deltas) + if not metric_names: + raise ValueError("comparison snapshot requires metric deltas") + for case in self.cases: + if {metric.metric_name for metric in case.metrics} != metric_names: + raise ValueError("comparison case metric schemas must match aggregate metrics") + expected_pass_delta = (sum(case.candidate_passed + for case in self.cases) - sum(case.baseline_passed + for case in self.cases)) / len(self.cases) + if not math.isclose( + self.pass_rate_delta, + expected_pass_delta, + rel_tol=0, + abs_tol=1e-12, + ): + raise ValueError("pass-rate delta does not match case statuses") + case_deltas = [case.delta for case in self.cases] + if self.score_delta < min(case_deltas) - 1e-12 or self.score_delta > max(case_deltas) + 1e-12: + raise ValueError("score delta must be a positive-weight aggregate of case deltas") + for name, aggregate in self.metric_deltas.items(): + deltas = [ + next(metric.delta for metric in case.metrics if metric.metric_name == name) for case in self.cases + ] + if aggregate < min(deltas) - 1e-12 or aggregate > max(deltas) + 1e-12: + raise ValueError(f"metric delta {name!r} must be a positive-weight aggregate of case metrics") + return self + + +class CostSummary(StrictModel): + sources: tuple[CostSource, ...] + total_cost_usd: Optional[float] + + @field_validator("total_cost_usd") + @classmethod + def valid_total(cls, value: Optional[float]) -> Optional[float]: + return None if value is None else _non_negative(value, "total cost") + + @model_validator(mode="after") + def consistent_total(self) -> "CostSummary": + names = tuple(source.name for source in self.sources) + if len(set(names)) != len(names): + raise ValueError("cost source names must be unique") + has_unknown = any(source.cost_usd is None for source in self.sources) + if has_unknown and self.total_cost_usd is not None: + raise ValueError("total cost must be unknown when any source cost is unknown") + if not has_unknown and self.total_cost_usd is None: + raise ValueError("total cost must be reported when all source costs are known") + if not has_unknown and not math.isclose( + self.total_cost_usd or 0, + sum(source.cost_usd or 0 for source in self.sources), + rel_tol=0, + abs_tol=1e-12, + ): + raise ValueError("total cost does not equal the sum of cost sources") + return self + + +class GateCheck(StrictModel): + code: str + passed: bool + observed: Any + threshold: Any + message: str + + +class GateDecision(StrictModel): + decision: Decision + accepted: bool + checks: tuple[GateCheck, ...] + reasons: tuple[str, ...] + + @model_validator(mode="after") + def consistent_decision(self) -> "GateDecision": + if self.accepted != (self.decision == Decision.ACCEPT): + raise ValueError("accepted flag must match ACCEPT decision") + return self + + +class SnapshotPair(StrictModel): + train: Optional[EvaluationSnapshot] = None + validation: Optional[EvaluationSnapshot] = None + + @model_validator(mode="after") + def has_completed_snapshot(self) -> "SnapshotPair": + if self.train is None and self.validation is None: + raise ValueError("snapshot pair requires at least one completed split") + if self.train is not None and self.train.split != Split.TRAIN: + raise ValueError("train snapshot must use the train split") + if self.validation is not None and self.validation.split != Split.VALIDATION: + raise ValueError("validation snapshot must use the validation split") + return self + + +class ComparisonPair(StrictModel): + train: Optional[ComparisonSnapshot] = None + validation: Optional[ComparisonSnapshot] = None + + @model_validator(mode="after") + def has_completed_comparison(self) -> "ComparisonPair": + if self.train is None and self.validation is None: + raise ValueError("comparison pair requires at least one completed split") + if self.train is not None and self.train.split != Split.TRAIN: + raise ValueError("train comparison must use the train split") + if self.validation is not None and self.validation.split != Split.VALIDATION: + raise ValueError("validation comparison must use the validation split") + return self + + +class AttributionPair(StrictModel): + train: Optional[AttributionSnapshot] = None + validation: Optional[AttributionSnapshot] = None + + @model_validator(mode="after") + def has_completed_attribution(self) -> "AttributionPair": + if self.train is None and self.validation is None: + raise ValueError("attribution pair requires at least one completed split") + if self.train is not None and self.train.split != Split.TRAIN: + raise ValueError("train attribution must use the train split") + if self.validation is not None and self.validation.split != Split.VALIDATION: + raise ValueError("validation attribution must use the validation split") + return self + + +class Reproducibility(StrictModel): + reproducible: bool + command: Optional[str] + reason: Optional[str] + git_commit: Optional[str] + git_dirty: Optional[bool] + + +class SourceApplication(StrictModel): + requested: bool + applied: bool + baseline_hashes: dict[str, str] + final_hashes: dict[str, str] + + +class ArtifactRecord(StrictModel): + path: str + sha256: str + byte_size: int = Field(ge=0) + + +class RunError(StrictModel): + stage: str + error_type: str + message: str + + +class OptimizationReport(StrictModel): + schema_version: Literal["v2"] = "v2" + run_id: str + status: Decision + mode: RunMode + stage: str + started_at: str + finished_at: str + duration_seconds: float + reproducibility: Reproducibility + inputs: dict[str, Any] + prompts: dict[str, Any] + baseline: Optional[SnapshotPair] = None + candidate: Optional[SnapshotPair] = None + delta: Optional[ComparisonPair] = None + failure_attribution: Optional[AttributionPair] = None + candidate_failure_attribution: Optional[AttributionPair] = None + optimization: Optional[dict[str, Any]] = None + cost: Optional[CostSummary] = None + gate_decision: Optional[GateDecision] = None + source_application: SourceApplication + artifacts: tuple[ArtifactRecord, ...] = () + errors: tuple[RunError, ...] = () + + @field_validator("duration_seconds") + @classmethod + def valid_duration(cls, value: float) -> float: + return _non_negative(value, "duration") + + @model_validator(mode="after") + def terminal_consistency(self) -> "OptimizationReport": + if self.status in (Decision.ACCEPT, Decision.REJECT): + if self.gate_decision is None or self.gate_decision.decision != self.status: + raise ValueError("terminal report status must match gate decision") + for pair, name in ( + (self.baseline, "baseline"), + (self.candidate, "candidate"), + (self.delta, "delta"), + ): + if pair is None or pair.train is None or pair.validation is None: + raise ValueError(f"terminal report requires complete {name} results") + if self.status == Decision.ERROR and not self.errors: + raise ValueError("ERROR report must include at least one error") + return self diff --git a/examples/optimization/eval_optimize_loop/pipeline/offline_evaluation.py b/examples/optimization/eval_optimize_loop/pipeline/offline_evaluation.py new file mode 100644 index 000000000..74a057a59 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/offline_evaluation.py @@ -0,0 +1,218 @@ +"""Deterministic replacements for LLM judge metrics in offline modes.""" + +from __future__ import annotations + +import json +import statistics +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import ClassVar, Optional + +from trpc_agent_sdk.evaluation import ( + EvalConfig, + EvalMetric, + EvalStatus, + EvaluationResult, + Evaluator, + EvaluatorRegistry, + FinalResponseEvaluator, + Invocation, + LLM_METRIC_NAMES, + PerInvocationResult, + RubricScore, + get_all_tool_responses, + get_llm_criterion_from_metric, +) + +_RESPONSE_RULES = frozenset({ + "OFFLINE_RESPONSE_CONTAINS", + "OFFLINE_RESPONSE_EQUALS", + "OFFLINE_RESPONSE_EXACT_REFERENCE", + "OFFLINE_RESPONSE_NON_EMPTY", +}) +_KNOWLEDGE_RULES = frozenset({ + "OFFLINE_KNOWLEDGE_CONTAINS", + "OFFLINE_KNOWLEDGE_NON_EMPTY", +}) + + +@dataclass(frozen=True) +class _OfflineRubric: + id: str + rule: str + operand: str + + +def _response_text(invocation: Optional[Invocation]) -> str: + response = invocation.final_response if invocation is not None else None + if response is None: + return "" + return "".join(part.text or "" for part in response.parts or [] if getattr(part, "text", None) is not None) + + +def _parse_rubrics(eval_metric: EvalMetric, supported: frozenset[str]) -> tuple[_OfflineRubric, ...]: + criterion = get_llm_criterion_from_metric(eval_metric) + if criterion is None or not criterion.rubrics: + raise ValueError(f"{eval_metric.metric_name} requires criterion.llmJudge.rubrics") + parsed = [] + ids = set() + for rubric in criterion.rubrics: + rubric_id = rubric.id + if not rubric_id or rubric_id != rubric_id.strip() or rubric_id in ids: + raise ValueError("offline rubric ids must be non-empty, trimmed, and unique") + ids.add(rubric_id) + rule = rubric.type.strip().upper() + if rule not in supported: + allowed = ", ".join(sorted(supported)) + raise ValueError(f"offline rubric {rubric_id!r} requires an explicit type; " + f"supported types: {allowed}") + content = rubric.content + operand = content.text if content is not None else "" + if rule.endswith("_EQUALS") and content is None: + raise ValueError(f"offline rubric {rubric_id!r} requires content.text") + if rule.endswith("_CONTAINS") and not operand: + raise ValueError(f"offline rubric {rubric_id!r} requires non-empty content.text") + parsed.append(_OfflineRubric(id=rubric_id, rule=rule, operand=operand)) + return tuple(parsed) + + +def _evidence_fragments(value: object) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str): + return (value, ) if value.strip() else () + if isinstance(value, Mapping): + return tuple(fragment for key in sorted(value, key=str) for fragment in _evidence_fragments(value[key])) + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + return tuple(fragment for item in value for fragment in _evidence_fragments(item)) + return (json.dumps(value, ensure_ascii=False, sort_keys=True, allow_nan=False), ) + + +class _OfflineRubricEvaluator(Evaluator): + requires_reference = False + supported_rules: ClassVar[frozenset[str]] = frozenset() + + def __init__(self, eval_metric: Optional[EvalMetric] = None) -> None: + if eval_metric is None: + raise ValueError("eval_metric is required for an offline rubric evaluator") + criterion = get_llm_criterion_from_metric(eval_metric) + if criterion is None: + raise ValueError(f"{eval_metric.metric_name} requires criterion.llmJudge") + self._threshold = eval_metric.threshold + self._rubrics = _parse_rubrics(eval_metric, self.supported_rules) + self._knowledge_tool_names = frozenset(criterion.get_knowledge_tool_names()) + + def _score_rubric( + self, + rubric: _OfflineRubric, + actual: Invocation, + expected: Optional[Invocation], + ) -> tuple[float, str]: + raise NotImplementedError + + def evaluate_invocations( + self, + actual_invocations: list[Invocation], + expected_invocations: Optional[list[Invocation]], + ) -> EvaluationResult: + per_invocation = [] + invocation_scores = [] + for index, actual in enumerate(actual_invocations): + expected = (expected_invocations[index] + if expected_invocations is not None and index < len(expected_invocations) else None) + results = [(rubric, *self._score_rubric(rubric, actual, expected)) for rubric in self._rubrics] + score = statistics.fmean(result[1] for result in results) + reason = "; ".join(result[2] for result in results) + invocation_scores.append(score) + per_invocation.append( + PerInvocationResult( + actual_invocation=actual, + expected_invocation=expected, + score=score, + eval_status=(EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED), + reason=reason, + rubric_scores=[ + RubricScore(id=rubric.id, score=item_score, reason=item_reason) + for rubric, item_score, item_reason in results + ], + )) + if not per_invocation: + return EvaluationResult() + overall_score = statistics.fmean(invocation_scores) + return EvaluationResult( + overall_score=overall_score, + overall_eval_status=(EvalStatus.PASSED if overall_score >= self._threshold else EvalStatus.FAILED), + per_invocation_results=per_invocation, + ) + + +class _OfflineResponseEvaluator(_OfflineRubricEvaluator): + supported_rules = _RESPONSE_RULES + + def _score_rubric( + self, + rubric: _OfflineRubric, + actual: Invocation, + expected: Optional[Invocation], + ) -> tuple[float, str]: + actual_text = _response_text(actual) + if rubric.rule == "OFFLINE_RESPONSE_EXACT_REFERENCE": + if expected is None or expected.final_response is None: + raise ValueError(f"offline rubric {rubric.id!r} requires a reference final response") + matched = actual_text == _response_text(expected) + elif rubric.rule == "OFFLINE_RESPONSE_EQUALS": + matched = actual_text == rubric.operand + elif rubric.rule == "OFFLINE_RESPONSE_CONTAINS": + matched = rubric.operand in actual_text + else: + matched = bool(actual_text.strip()) + return float(matched), f"{rubric.rule} {'matched' if matched else 'did not match'}" + + +class _OfflineKnowledgeEvaluator(_OfflineRubricEvaluator): + supports_remote = False + supported_rules = _KNOWLEDGE_RULES + + def _score_rubric( + self, + rubric: _OfflineRubric, + actual: Invocation, + expected: Optional[Invocation], + ) -> tuple[float, str]: + del expected + fragments = tuple(fragment for response in get_all_tool_responses(actual.intermediate_data) + if response.name in self._knowledge_tool_names + for fragment in _evidence_fragments(response.response)) + matched = (rubric.operand in "\n".join(fragments) + if rubric.rule == "OFFLINE_KNOWLEDGE_CONTAINS" else bool(fragments)) + return float(matched), f"{rubric.rule} {'matched' if matched else 'did not match'}" + + +def prepare_offline_evaluation(eval_config: EvalConfig, ) -> tuple[EvalConfig, Optional[EvaluatorRegistry]]: + """Build a run-local deterministic config and registry for offline modes.""" + + configured = eval_config.get_eval_metrics() + llm_names = {metric.metric_name for metric in configured if metric.metric_name in LLM_METRIC_NAMES} + if not llm_names: + return eval_config, None + metrics = [] + for metric in configured: + payload = metric.model_dump(mode="json", by_alias=True) + if metric.metric_name == "llm_final_response": + payload["criterion"] = {"finalResponse": {"text": {"match": "exact"}}} + metrics.append(payload) + payload = eval_config.model_dump(mode="python", by_alias=False) + payload.update({"criteria": {}, "metrics": metrics}) + offline_config = EvalConfig.model_validate(payload) + registry = EvaluatorRegistry() + replacements = { + "llm_final_response": FinalResponseEvaluator, + "llm_rubric_response": _OfflineResponseEvaluator, + "llm_rubric_knowledge_recall": _OfflineKnowledgeEvaluator, + } + for metric_name in llm_names: + registry.register(metric_name, replacements[metric_name]) + for metric in offline_config.get_eval_metrics(): + if metric.metric_name in llm_names and metric.metric_name != "llm_final_response": + registry.get_evaluator(metric) + return offline_config, registry diff --git a/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py new file mode 100644 index 000000000..7d9b1bf5e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py @@ -0,0 +1,347 @@ +"""Sole composition root and lifecycle coordinator for the loop example.""" + +from __future__ import annotations + +import asyncio +import time +from typing import Callable, Optional + +from trpc_agent_sdk.evaluation import TargetPrompt + +from .artifacts import AuditSink +from .attribution import attribute_failures +from .backends import CallAgent, TraceEvaluationBackend, create_backends +from .candidate_runtime import generate_candidate, prepare_candidate_inputs +from .contracts import CandidateGenerator, EvaluationBackend +from .costing import CostLedger +from .evaluation import compare_snapshots +from .evaluation_runtime import create_evaluation_runtime +from .gate import evaluate_gate +from .models import ( + AttributionPair, + CandidateProposal, + ComparisonPair, + Decision, + InnerSplit, + OptimizationReport, + Phase, + RunError, + SnapshotPair, +) +from .preflight import preflight_run +from .prompt_workspace import PromptRestoreError, PromptWorkspace, prompt_hashes +from .reporting import ( + build_optimization_report, + create_report_context, + persist_report, + persist_terminal_report, + utc_now, +) +from .schema import sanitized_text + +FaultInjector = Callable[[str], None] + + +def _fault(stage: str, injector: Optional[FaultInjector]) -> None: + if injector: + injector(stage) + + +async def run_pipeline( + root_dir: str, + *, + config_path: Optional[str] = None, + train_path: Optional[str] = None, + validation_path: Optional[str] = None, + mode: Optional[str] = None, + run_id: Optional[str] = None, + apply_candidate: Optional[bool] = None, + call_agent: Optional[CallAgent] = None, + callback_spec: Optional[str] = None, + backend: Optional[EvaluationBackend] = None, + candidate_generator: Optional[CandidateGenerator] = None, + fault_injector: Optional[FaultInjector] = None, +) -> OptimizationReport: + """Run the audited lifecycle; REJECT returns normally and ERROR is reported.""" + + started_at = utc_now() + started_clock = time.monotonic() + validated = preflight_run( + root_dir, + config_path=config_path, + train_path=train_path, + validation_path=validation_path, + mode=mode, + run_id=run_id, + apply_candidate=apply_candidate, + call_agent=call_agent, + callback_spec=callback_spec, + backend=backend, + candidate_generator=candidate_generator, + ) + settings = validated.config.pipeline + custom_backend = backend is not None + custom_components = custom_backend or candidate_generator is not None + if backend is None or candidate_generator is None: + default_backend, default_generator = create_backends( + settings.mode, + call_agent=call_agent, + trace_fixture_path=validated.trace_fixture_path, + trace_fixture_hash=validated.input_hashes.get("trace"), + dataset_hashes={ + "train": validated.input_hashes["train"], + "validation": validated.input_hashes["validation"], + }, + ) + backend = backend or default_backend + candidate_generator = candidate_generator or default_generator + if isinstance(backend, TraceEvaluationBackend): + backend.validate_fixture(validated.train, validated.validation) + + target = TargetPrompt() + for name, path in validated.prompt_paths.items(): + target.add_path(name, path) + workspace = PromptWorkspace(target) + await workspace.initialize() + if workspace.baseline_hashes != validated.prompt_hashes: + raise ValueError("prompt changed between preflight and workspace initialization") + sink = AuditSink( + validated.artifact_root, + validated.run_id, + max_text_chars=settings.max_text_chars, + publication_root=validated.root_dir, + max_import_files=settings.max_import_files, + max_import_file_bytes=settings.max_import_file_bytes, + max_import_total_bytes=settings.max_import_total_bytes, + ) + sink.create() + ledger = CostLedger() + evaluation_runtime = create_evaluation_runtime( + validated=validated, + backend=backend, + custom_backend=custom_backend, + sink=sink, + ledger=ledger, + ) + stage = "audit_config" + applied = False + candidate_started = False + baseline_pair = candidate_pair = comparisons = None + baseline_attribution_pair = candidate_attribution_pair = None + proposal: Optional[CandidateProposal] = None + inner: Optional[InnerSplit] = None + gate_decision = None + report_context = create_report_context( + validated, + started_at=started_at, + started_clock=started_clock, + baseline_prompts=workspace.baseline, + baseline_hashes=workspace.baseline_hashes, + callback_spec=callback_spec, + programmatic_component=custom_components, + ) + + def build_report(status: Decision, report_stage: str, errors: tuple[RunError, ...] = ()) -> OptimizationReport: + candidate_hashes = prompt_hashes(proposal.prompts) if proposal else None + return build_optimization_report( + report_context, + status=status, + stage=report_stage, + applied=applied, + proposal=proposal, + candidate_hashes=candidate_hashes, + baseline=baseline_pair, + candidate=candidate_pair, + delta=comparisons, + baseline_attribution=baseline_attribution_pair, + candidate_attribution=candidate_attribution_pair, + inner=inner, + cost=ledger.summary(), + gate_decision=gate_decision, + artifacts=sink.records(include_report_files=False), + errors=errors, + ) + + def enter_stage(next_stage: str) -> None: + nonlocal stage + stage = next_stage + _fault(stage, fault_injector) + + def retain_progress(phase: Phase, progress: SnapshotPair) -> None: + nonlocal baseline_pair, candidate_pair + nonlocal baseline_attribution_pair, candidate_attribution_pair + if phase == Phase.BASELINE: + baseline_pair = progress + else: + candidate_pair = progress + attribution = AttributionPair( + train=(attribute_failures( + progress.train, + settings.attribution, + max_text_chars=settings.max_text_chars, + ) if progress.train is not None else None), + validation=(attribute_failures( + progress.validation, + settings.attribution, + max_text_chars=settings.max_text_chars, + ) if progress.validation is not None else None), + ) + if phase == Phase.BASELINE: + baseline_attribution_pair = attribution + else: + candidate_attribution_pair = attribution + + try: + enter_stage(stage) + sink.write_json("config.json", validated.config.model_dump(mode="json", by_alias=True)) + sink.write_json("train.evalset.json", validated.train.model_dump(mode="json", by_alias=True)) + sink.write_json("val.evalset.json", validated.validation.model_dump(mode="json", by_alias=True)) + for name, content in workspace.baseline.items(): + sink.write_text(f"baseline_prompts/{name}.md", content) + + baseline_pair = await evaluation_runtime.evaluate_phase( + train=validated.train, + validation=validated.validation, + prompts=workspace.baseline, + phase=Phase.BASELINE, + on_stage=enter_stage, + on_progress=retain_progress, + ) + if baseline_attribution_pair is None: + raise RuntimeError("baseline attribution was not retained") + baseline_train = baseline_pair.train + baseline_validation = baseline_pair.validation + baseline_train_attr = baseline_attribution_pair.train + baseline_val_attr = baseline_attribution_pair.validation + assert baseline_train and baseline_validation and baseline_train_attr and baseline_val_attr + + enter_stage("inner_split") + inner, inner_train_attr, inner_train_path, inner_selection_path = prepare_candidate_inputs( + train=validated.train, + baseline=baseline_train, + attribution=baseline_train_attr, + seed=settings.seed, + selection_ratio=settings.inner_selection_ratio, + sink=sink, + ) + + enter_stage("candidate_generation") + candidate_started = True + proposal = await generate_candidate( + generator=candidate_generator, + workspace=workspace, + sink=sink, + eval_config=validated.config.evaluate, + optimize_config=validated.config.optimize, + train_attribution=inner_train_attr, + inner_train_path=inner_train_path, + inner_selection_path=inner_selection_path, + ) + for source in proposal.cost_sources: + ledger.record( + f"{stage}.{source.name}", + cost_usd=source.cost_usd, + model_calls=source.model_calls, + metric_calls=source.metric_calls, + token_usage=source.token_usage, + ) + sink.write_json("candidate.json", proposal.model_dump(mode="json", by_alias=True)) + for name, content in proposal.prompts.items(): + sink.write_text(f"candidate_prompts/{name}.md", content) + + async with workspace.temporary(proposal.prompts): + candidate_pair = await evaluation_runtime.evaluate_phase( + train=validated.train, + validation=validated.validation, + prompts=proposal.prompts, + phase=Phase.CANDIDATE, + on_stage=enter_stage, + on_progress=retain_progress, + ) + if candidate_attribution_pair is None: + raise RuntimeError("candidate attribution was not retained") + candidate_train = candidate_pair.train + candidate_validation = candidate_pair.validation + candidate_train_attr = candidate_attribution_pair.train + candidate_val_attr = candidate_attribution_pair.validation + assert candidate_train and candidate_validation and candidate_train_attr and candidate_val_attr + + enter_stage("comparison") + train_comparison = compare_snapshots( + baseline_train, + candidate_train, + epsilon=settings.gate.epsilon, + baseline_attribution=baseline_train_attr, + candidate_attribution=candidate_train_attr, + ) + comparisons = ComparisonPair(train=train_comparison) + validation_comparison = compare_snapshots( + baseline_validation, + candidate_validation, + epsilon=settings.gate.epsilon, + critical_case_ids=settings.critical_case_ids, + hard_case_ids=settings.hard_case_ids, + baseline_attribution=baseline_val_attr, + candidate_attribution=candidate_val_attr, + ) + comparisons = ComparisonPair(train=train_comparison, validation=validation_comparison) + sink.write_json("comparison.json", comparisons.model_dump(mode="json", by_alias=True)) + + enter_stage("gate") + gate_decision = evaluate_gate( + train=train_comparison, + validation=validation_comparison, + candidate=proposal, + cost=ledger.summary(), + duration_seconds=time.monotonic() - started_clock, + baseline_prompt_hashes=workspace.baseline_hashes, + candidate_prompt_hashes=prompt_hashes(proposal.prompts), + config=settings.gate, + ) + if gate_decision.decision == Decision.ERROR: + raise ValueError("gate rejected structurally invalid regression inputs") + + enter_stage("pre_apply_report") + pre_apply_report = build_report(gate_decision.decision, stage) + persist_report(sink, pre_apply_report) + + if gate_decision.accepted and settings.apply_candidate: + enter_stage("apply") + await workspace.apply(proposal.prompts) + applied = True + + enter_stage("final_report") + final_report = build_report(gate_decision.decision, "complete") + return persist_terminal_report(sink, final_report, started_clock=started_clock) + except BaseException as error: + if candidate_started and proposal is None: + ledger.record( + "candidate_generation.unreported_failure", + cost_usd=None, + model_calls=None, + metric_calls=None, + ) + fatal_restore: Optional[PromptRestoreError] = None + try: + await workspace.restore() + applied = False + except PromptRestoreError as restore_error: + fatal_restore = restore_error + stage = "restore" + error = restore_error + message = sanitized_text(error, max_text_chars=settings.max_text_chars) + run_error = RunError(stage=stage, error_type=type(error).__name__, message=message) + error_report = build_report(Decision.ERROR, stage, (run_error, )) + try: + error_report = persist_terminal_report( + sink, + error_report, + started_clock=started_clock, + ) + except BaseException: + pass + if fatal_restore is not None: + raise fatal_restore + if isinstance(error, (asyncio.CancelledError, KeyboardInterrupt, SystemExit)): + raise + return error_report diff --git a/examples/optimization/eval_optimize_loop/pipeline/preflight.py b/examples/optimization/eval_optimize_loop/pipeline/preflight.py new file mode 100644 index 000000000..700f79814 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/preflight.py @@ -0,0 +1,176 @@ +"""Validated, content-addressed run inputs for the optimization example.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Any, Optional + +from trpc_agent_sdk.evaluation import EvalSet + +from .artifacts import load_strict_json +from .evaluation import canonical_json, dataset_fingerprint, validate_datasets +from .configuration import PipelineConfig, ValidatedRunConfig +from .schema import validate_safe_component +from .prompt_workspace import prompt_hashes + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def inside_example_root(root: Path, relative: str, label: str) -> Path: + """Resolve a configured path while preventing escape from the example root.""" + + raw = Path(relative) + if raw.is_absolute(): + raise ValueError(f"{label} must be relative to the example root") + resolved = (root / raw).resolve() + if root != resolved and root not in resolved.parents: + raise ValueError(f"{label} escapes the example root") + return resolved + + +def _component_identity(component: object | None, default: str) -> str: + if component is None: + return default + component_type = type(component) + return f"{component_type.__module__}.{component_type.__qualname__}" + + +def _callable_identity(call_agent: object | None, callback_spec: Optional[str]) -> Optional[str]: + if callback_spec: + return callback_spec + if call_agent is None: + return None + module = getattr(call_agent, "__module__", type(call_agent).__module__) + qualname = getattr(call_agent, "__qualname__", type(call_agent).__qualname__) + return f"{module}:{qualname}" + + +def _adapter_identity( + mode: str, + *, + call_agent: object | None, + callback_spec: Optional[str], + backend: object | None, + candidate_generator: object | None, +) -> str: + """Stable identity for the components that can change execution behavior.""" + + return canonical_json({ + "mode": + mode, + "backend": + _component_identity(backend, f"default:{mode}:evaluation:v1"), + "candidateGenerator": + _component_identity(candidate_generator, f"default:{mode}:candidate-generator:v1"), + "callback": + _callable_identity(call_agent, callback_spec) if mode == "live" else None, + }) + + +def preflight_run( + root_dir: str, + *, + config_path: Optional[str], + train_path: Optional[str], + validation_path: Optional[str], + mode: Optional[str], + run_id: Optional[str], + apply_candidate: Optional[bool], + call_agent: object | None, + callback_spec: Optional[str], + backend: object | None, + candidate_generator: object | None, +) -> ValidatedRunConfig: + """Load all inputs and derive the immutable run identity before side effects.""" + + root = Path(root_dir).resolve() + config_file = Path(config_path).resolve() if config_path else root / "optimizer.json" + train_file = Path(train_path).resolve() if train_path else root / "train.evalset.json" + validation_file = Path(validation_path).resolve() if validation_path else root / "val.evalset.json" + config = PipelineConfig.model_validate(load_strict_json(config_file)) + settings = config.pipeline + updates: dict[str, Any] = {} + if mode is not None: + updates["mode"] = mode + if run_id is not None: + updates["run_id"] = run_id + if apply_candidate is not None: + updates["apply_candidate"] = apply_candidate + if updates: + settings = type(settings).model_validate({**settings.model_dump(mode="python", by_alias=False), **updates}) + config = config.model_copy(update={"pipeline": settings}) + + train = EvalSet.model_validate(load_strict_json(train_file)) + validation = EvalSet.model_validate(load_strict_json(validation_file)) + metric_names = [metric.metric_name for metric in config.evaluate.get_eval_metrics()] + validate_datasets( + train, + validation, + train_path=str(train_file), + validation_path=str(validation_file), + configured_metrics=metric_names, + critical_case_ids=settings.critical_case_ids, + hard_case_ids=settings.hard_case_ids, + metric_weights=settings.metric_weights, + train_case_weights=settings.train_case_weights, + validation_case_weights=settings.validation_case_weights, + ) + + prompt_paths: dict[str, str] = {} + prompts: dict[str, str] = {} + for name, relative in settings.prompt_paths.items(): + validate_safe_component(name, name="prompt field") + path = inside_example_root(root, relative, f"prompt path {name!r}") + prompts[name] = path.read_text(encoding="utf-8") + prompt_paths[name] = str(path) + + hashes = { + "config": + _sha256_file(config_file), + "effectiveConfig": + hashlib.sha256(canonical_json(config.model_dump(mode="json", by_alias=True)).encode("utf-8")).hexdigest(), + "train": + dataset_fingerprint(train), + "validation": + dataset_fingerprint(validation), + } + trace_path: Optional[Path] = None + if settings.mode == "trace": + trace_path = inside_example_root(root, settings.trace_fixture, "trace fixture") + hashes["trace"] = _sha256_file(trace_path) + + adapter_identity = _adapter_identity( + settings.mode, + call_agent=call_agent, + callback_spec=callback_spec, + backend=backend, + candidate_generator=candidate_generator, + ) + derived = ("run-" + hashlib.sha256( + canonical_json({ + "inputHashes": hashes, + "promptHashes": prompt_hashes(prompts), + "adapterIdentity": adapter_identity, + }).encode("utf-8")).hexdigest()[:16]) + effective_run_id = settings.run_id or derived + validate_safe_component(effective_run_id, name="run ID") + artifact_root = inside_example_root(root, settings.artifact_root, "artifact root") + return ValidatedRunConfig( + root_dir=str(root), + config_path=str(config_file), + train_path=str(train_file), + validation_path=str(validation_file), + trace_fixture_path=str(trace_path) if trace_path else None, + artifact_root=str(artifact_root), + run_id=effective_run_id, + config=config, + train=train, + validation=validation, + input_hashes=hashes, + prompt_paths=prompt_paths, + prompt_hashes=prompt_hashes(prompts), + adapter_identity=adapter_identity, + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py b/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py new file mode 100644 index 000000000..295a363fb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py @@ -0,0 +1,127 @@ +"""Verified TargetPrompt mutation, restoration and controlled application.""" + +from __future__ import annotations + +import hashlib +from contextlib import asynccontextmanager +from pathlib import Path +from typing import AsyncIterator + +from trpc_agent_sdk.evaluation import TargetPrompt + +from .schema import validate_safe_component + + +class PromptRestoreError(RuntimeError): + """The baseline prompt state could not be restored and verified.""" + + +def hash_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def prompt_hashes(prompts: dict[str, str]) -> dict[str, str]: + return {name: hash_text(text) for name, text in prompts.items()} + + +class PromptWorkspace: + """Own every prompt mutation and verify all writes by reading them back.""" + + def __init__(self, target: TargetPrompt) -> None: + if not target.names(): + raise ValueError("PromptWorkspace requires at least one TargetPrompt field") + for name in target.names(): + validate_safe_component(name, name="prompt field") + self._target = target + self._baseline: dict[str, str] | None = None + self._baseline_hashes: dict[str, str] | None = None + + async def initialize(self) -> None: + baseline = await self._target.read_all() + self._validate_prompt_map(baseline) + self._baseline = dict(baseline) + self._baseline_hashes = prompt_hashes(baseline) + + @property + def baseline(self) -> dict[str, str]: + self._ensure_initialized() + return dict(self._baseline or {}) + + @property + def baseline_hashes(self) -> dict[str, str]: + self._ensure_initialized() + return dict(self._baseline_hashes or {}) + + async def current_hashes(self) -> dict[str, str]: + self._ensure_initialized() + current = await self._target.read_all() + self._validate_prompt_map(current) + return prompt_hashes(current) + + async def _write_verified(self, prompts: dict[str, str]) -> dict[str, str]: + self._validate_prompt_map(prompts) + await self._target.write_all(dict(prompts)) + actual = await self._target.read_all() + if actual != prompts: + raise IOError("TargetPrompt read-back differs from requested content") + return prompt_hashes(actual) + + async def restore(self) -> None: + self._ensure_initialized() + try: + hashes = await self._write_verified(self.baseline) + if hashes != self.baseline_hashes: + raise IOError("baseline prompt hashes differ after restoration") + except BaseException as error: + raise PromptRestoreError("baseline prompt restoration could not be verified") from error + + async def apply(self, candidate: dict[str, str]) -> dict[str, str]: + self._ensure_initialized() + try: + return await self._write_verified(candidate) + except BaseException: + await self.restore() + raise + + @asynccontextmanager + async def temporary(self, candidate: dict[str, str]) -> AsyncIterator[dict[str, str]]: + self._ensure_initialized() + primary: BaseException | None = None + try: + hashes = await self._write_verified(candidate) + yield hashes + except BaseException as error: + primary = error + raise + finally: + try: + await self.restore() + except PromptRestoreError as restore_error: + if primary is not None: + raise PromptRestoreError( + "baseline restoration failed while propagating another error") from restore_error + raise + + def create_candidate_target(self, directory: str) -> TargetPrompt: + """Create an independent file-backed TargetPrompt for AgentOptimizer.""" + + self._ensure_initialized() + root = Path(directory) + root.mkdir(parents=True, exist_ok=False) + target = TargetPrompt() + for name, content in self.baseline.items(): + path = root / f"{name}.md" + path.write_text(content, encoding="utf-8") + target.add_path(name, str(path)) + return target + + def _validate_prompt_map(self, prompts: dict[str, str]) -> None: + expected = set(self._target.names()) + if set(prompts) != expected: + raise ValueError(f"prompt keys mismatch; expected {sorted(expected)}, got {sorted(prompts)}") + if any(not isinstance(text, str) for text in prompts.values()): + raise TypeError("all prompt values must be UTF-8 text strings") + + def _ensure_initialized(self) -> None: + if self._baseline is None or self._baseline_hashes is None: + raise RuntimeError("PromptWorkspace.initialize() must be awaited first") diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporting.py b/examples/optimization/eval_optimize_loop/pipeline/reporting.py new file mode 100644 index 000000000..5943d4014 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/reporting.py @@ -0,0 +1,352 @@ +"""Environment metadata and render/persist operations for OptimizationReport.""" + +from __future__ import annotations + +import platform +import re +import shlex +import subprocess +import sys +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from trpc_agent_sdk.version import __version__ as sdk_version + +from .artifacts import AuditSink +from .configuration import ValidatedRunConfig +from .models import ( + ArtifactRecord, + AttributionPair, + CandidateProposal, + ComparisonPair, + CostSummary, + Decision, + GateDecision, + InnerSplit, + OptimizationReport, + Reproducibility, + RunError, + SnapshotPair, + SourceApplication, +) + + +@dataclass(frozen=True) +class ReportContext: + """Static run facts shared by pre-apply, terminal and error reports.""" + + validated: ValidatedRunConfig + started_at: str + started_clock: float + reproducibility: Reproducibility + baseline_prompts: dict[str, str] + baseline_hashes: dict[str, str] + + +def create_report_context( + validated: ValidatedRunConfig, + *, + started_at: str, + started_clock: float, + baseline_prompts: dict[str, str], + baseline_hashes: dict[str, str], + callback_spec: Optional[str], + programmatic_component: bool, +) -> ReportContext: + settings = validated.config.pipeline + reproducibility = build_reproducibility( + validated.root_dir, + mode=settings.mode, + config_path=validated.config_path, + train_path=validated.train_path, + validation_path=validated.validation_path, + run_id=validated.run_id, + apply_candidate=settings.apply_candidate, + callback_spec=callback_spec, + trace_fixture=validated.trace_fixture_path, + trace_hash=validated.input_hashes.get("trace"), + programmatic_component=programmatic_component, + ) + return ReportContext( + validated=validated, + started_at=started_at, + started_clock=started_clock, + reproducibility=reproducibility, + baseline_prompts=baseline_prompts, + baseline_hashes=baseline_hashes, + ) + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def build_reproducibility( + repo_root: str, + *, + mode: str, + config_path: str, + train_path: str, + validation_path: str, + run_id: str, + apply_candidate: bool, + callback_spec: Optional[str] = None, + trace_fixture: Optional[str] = None, + trace_hash: Optional[str] = None, + programmatic_component: bool = False, +) -> Reproducibility: + commit: Optional[str] = None + dirty: Optional[bool] = None + reason: Optional[str] = None + git_root: Optional[Path] = None + try: + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + dirty = bool( + subprocess.run( + ["git", "status", "--porcelain"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip()) + git_root = Path( + subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip()).resolve() + except (OSError, subprocess.SubprocessError): + reason = "git_metadata_unavailable" + if reason is None and (commit is None or not re.fullmatch(r"[0-9a-fA-F]{40}", commit)): + reason = "git_commit_invalid" + elif reason is None and dirty: + reason = "worktree_dirty" + elif reason is None and programmatic_component: + reason = "programmatic_custom_component" + elif reason is None and mode == "live" and not callback_spec: + reason = "live_callback_not_importable" + elif reason is None and mode == "trace" and (not trace_fixture or not trace_hash): + reason = "trace_fixture_not_pinned" + reproducible = reason is None + command = None + if reproducible: + assert git_root is not None + + def replay_path(path: str | Path) -> str: + resolved = Path(path).resolve() + try: + return resolved.relative_to(git_root).as_posix() + except ValueError: + return str(resolved) + + args = [ + "python", + replay_path(Path(repo_root) / "run_pipeline.py"), + "--mode", + mode, + "--config", + replay_path(config_path), + "--train", + replay_path(train_path), + "--validation", + replay_path(validation_path), + "--run-id", + f"{run_id}-replay", + "--apply-candidate" if apply_candidate else "--no-apply-candidate", + ] + if callback_spec: + args.extend(("--call-agent", callback_spec)) + command = shlex.join(args) + return Reproducibility( + reproducible=reproducible, + command=command, + reason=reason, + git_commit=commit, + git_dirty=dirty, + ) + + +def environment_metadata() -> dict[str, str]: + return { + "python": sys.version.split()[0], + "sdk": sdk_version, + "platform": platform.platform(), + } + + +def _report_path(path: str, root: Path) -> str: + resolved = Path(path) + try: + return resolved.relative_to(root).as_posix() + except ValueError: + return str(resolved) + + +def build_optimization_report( + context: ReportContext, + *, + status: Decision, + stage: str, + applied: bool, + proposal: Optional[CandidateProposal], + candidate_hashes: Optional[dict[str, str]], + baseline: Optional[SnapshotPair], + candidate: Optional[SnapshotPair], + delta: Optional[ComparisonPair], + baseline_attribution: Optional[AttributionPair], + candidate_attribution: Optional[AttributionPair], + inner: Optional[InnerSplit], + cost: CostSummary, + gate_decision: Optional[GateDecision], + artifacts: tuple[ArtifactRecord, ...], + errors: tuple[RunError, ...] = (), +) -> OptimizationReport: + """Assemble report-model facts without rerunning aggregation or gate logic.""" + + validated = context.validated + settings = validated.config.pipeline + report_root = Path(validated.root_dir) + final_hashes = candidate_hashes if applied and candidate_hashes else context.baseline_hashes + return OptimizationReport( + run_id=validated.run_id, + status=status, + mode=settings.mode, + stage=stage, + started_at=context.started_at, + finished_at=utc_now(), + duration_seconds=time.monotonic() - context.started_clock, + reproducibility=context.reproducibility, + inputs={ + "hashes": validated.input_hashes, + "paths": { + "config": _report_path(validated.config_path, report_root), + "train": _report_path(validated.train_path, report_root), + "validation": _report_path(validated.validation_path, report_root), + }, + "environment": environment_metadata(), + "seed": settings.seed, + "adapterIdentity": validated.adapter_identity, + }, + prompts={ + "baseline": context.baseline_prompts, + "baselineHashes": context.baseline_hashes, + "candidate": proposal.prompts if proposal else None, + "candidateHashes": candidate_hashes, + }, + baseline=baseline, + candidate=candidate, + delta=delta, + failure_attribution=baseline_attribution, + candidate_failure_attribution=candidate_attribution, + optimization=({ + "proposal": proposal.model_dump(mode="json", by_alias=True), + "innerSplit": inner.model_dump(mode="json", by_alias=True) if inner else None, + } if proposal else None), + cost=cost, + gate_decision=gate_decision, + source_application=SourceApplication( + requested=settings.apply_candidate, + applied=applied, + baseline_hashes=context.baseline_hashes, + final_hashes=final_hashes, + ), + artifacts=artifacts, + errors=errors, + ) + + +def render_markdown(report: OptimizationReport) -> str: + """Render only report-model facts; no business aggregation occurs here.""" + + lines = [ + f"# Optimization Report: {report.status.value}", + "", + f"- Run ID: `{report.run_id}`", + f"- Mode: `{report.mode}`", + f"- Final stage: `{report.stage}`", + f"- Duration: `{report.duration_seconds:.3f}s`", + f"- Reproducible: `{'yes' if report.reproducibility.reproducible else 'no'}`", + f"- Source prompt applied: `{'yes' if report.source_application.applied else 'no'}`", + ] + if (report.baseline and report.candidate and report.baseline.train and report.baseline.validation + and report.candidate.train and report.candidate.validation): + lines.extend([ + "", + "## Regression", + "", + "| Split | Baseline score | Candidate score | Baseline pass rate | Candidate pass rate |", + "|---|---:|---:|---:|---:|", + (f"| train | {report.baseline.train.dataset_score:.4f} | " + f"{report.candidate.train.dataset_score:.4f} | " + f"{report.baseline.train.pass_rate:.4f} | {report.candidate.train.pass_rate:.4f} |"), + (f"| validation | {report.baseline.validation.dataset_score:.4f} | " + f"{report.candidate.validation.dataset_score:.4f} | " + f"{report.baseline.validation.pass_rate:.4f} | " + f"{report.candidate.validation.pass_rate:.4f} |"), + ]) + if report.delta and report.delta.validation: + lines.extend(["", "## Validation Transitions", ""]) + for case in report.delta.validation.cases: + lines.append(f"- `{case.case_id}`: `{case.transition.value}` ({case.delta:+.4f})") + if report.gate_decision: + failed = [check.code for check in report.gate_decision.checks if not check.passed] + lines.extend([ + "", + "## Gate", + "", + f"- Decision: `{report.gate_decision.decision.value}`", + f"- Failed checks: `{', '.join(failed) if failed else 'none'}`", + f"- Reasons: `{', '.join(report.gate_decision.reasons) if report.gate_decision.reasons else 'none'}`", + ("- Overfit detected: `yes`" + if "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in report.gate_decision.reasons else "- Overfit detected: `no`"), + ]) + if report.cost: + cost = "unknown" if report.cost.total_cost_usd is None else f"${report.cost.total_cost_usd:.6f}" + lines.extend(["", "## Cost", "", f"- Total: `{cost}`"]) + if report.errors: + lines.extend(["", "## Errors", ""]) + for error in report.errors: + lines.append(f"- `{error.stage}/{error.error_type}`: {error.message}") + return "\n".join(lines) + "\n" + + +def persist_report(sink: AuditSink, report: OptimizationReport) -> None: + sink.write_json("optimization_report.json", report.model_dump(mode="json", by_alias=True)) + sink.write_text("optimization_report.md", render_markdown(report)) + + +def persist_terminal_report( + sink: AuditSink, + report: OptimizationReport, + *, + started_clock: float, +) -> OptimizationReport: + """Persist one full terminal cycle before recording the completed duration.""" + + persist_report(sink, report) + sink.write_manifest() + sink.publish_latest_snapshot("optimization_report.json", "optimization_report.json") + sink.publish_latest_snapshot("optimization_report.md", "optimization_report.md") + completed = OptimizationReport.model_validate({ + **report.model_dump(mode="python", by_alias=False), + "finished_at": + utc_now(), + "duration_seconds": + time.monotonic() - started_clock, + }) + persist_report(sink, completed) + sink.write_manifest() + sink.publish_latest_snapshot("optimization_report.json", "optimization_report.json") + sink.publish_latest_snapshot("optimization_report.md", "optimization_report.md") + return completed diff --git a/examples/optimization/eval_optimize_loop/pipeline/schema.py b/examples/optimization/eval_optimize_loop/pipeline/schema.py new file mode 100644 index 000000000..e9d153086 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/schema.py @@ -0,0 +1,166 @@ +"""Shared strict-schema primitives with no pipeline dependencies.""" + +from __future__ import annotations + +import json +import math +import re +from typing import Any + +from pydantic import BaseModel, ConfigDict +from pydantic.alias_generators import to_camel + +_WINDOWS_RESERVED = { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{i}" for i in range(1, 10)), + *(f"LPT{i}" for i in range(1, 10)), +} +_SAFE_COMPONENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_SECRET_ASSIGNMENT = re.compile(r"(?i)(?P(?P[\"']?)[A-Za-z0-9_.-]*" + r"(?:api[_-]?key|access[_-]?token|refresh[_-]?token|password|authorization|" + r"cookie|secret|credential|private[_-]?key|secret[_-]?access[_-]?key)" + r"(?P=key_quote)\s*[:=]\s*)" + r"(?P\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*'|" + r"(?:bearer\s+)?[^\s,;}\]]+)") +_BEARER_TOKEN = re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]+") +_SECRET_SUFFIXES = ( + "apikey", + "token", + "password", + "authorization", + "cookie", + "secret", + "credential", + "accesstoken", + "refreshtoken", + "privatekey", + "secretaccesskey", +) + + +def parse_strict_json(text: str) -> dict[str, Any]: + """Parse a JSON object while rejecting duplicates and non-finite values.""" + + def object_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key!r}") + result[key] = value + return result + + def invalid_constant(value: str) -> Any: + raise ValueError(f"non-finite JSON number: {value}") + + def finite_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed): + raise ValueError(f"non-finite JSON number: {value}") + return parsed + + payload = json.loads( + text, + object_pairs_hook=object_pairs, + parse_constant=invalid_constant, + parse_float=finite_float, + ) + if not isinstance(payload, dict): + raise ValueError("JSON root must be an object") + return payload + + +def add_exception_note(error: BaseException, note: str) -> None: + """Attach diagnostics without dropping Python 3.10 compatibility.""" + + add_note = getattr(error, "add_note", None) + if callable(add_note): + add_note(note) + return + notes = list(getattr(error, "__notes__", ())) + notes.append(note) + try: + error.__notes__ = notes + except (AttributeError, TypeError): + pass + + +def validate_safe_component(value: str, *, name: str = "component") -> str: + """Validate one portable path component.""" + + if not value or value != value.strip() or not _SAFE_COMPONENT.fullmatch(value): + raise ValueError(f"{name} must be a non-empty portable path component") + if ".." in value or value.endswith((".", " ")): + raise ValueError(f"{name} contains an unsafe path component") + stem = value.split(".", 1)[0].upper() + if stem in _WINDOWS_RESERVED: + raise ValueError(f"{name} is reserved on Windows") + return value + + +def _is_secret_key(value: str) -> bool: + normalized = re.sub(r"[^a-z0-9]", "", value.casefold()) + return normalized in _SECRET_SUFFIXES or normalized.endswith(_SECRET_SUFFIXES) + + +def _redact_assignment(match: re.Match[str]) -> str: + value = match.group("value") + quote = value[0] if value and value[0] in "\"'" else "" + return f"{match.group('prefix')}{quote}[REDACTED]{quote}" + + +def sanitize(value: Any, *, max_text_chars: int) -> Any: + """Recursively redact credential fields and credential-shaped text.""" + + if isinstance(value, dict): + return { + str(key): ("[REDACTED]" if _is_secret_key(str(key)) else sanitize(item, max_text_chars=max_text_chars)) + for key, item in value.items() + } + if isinstance(value, (list, tuple)): + return [sanitize(item, max_text_chars=max_text_chars) for item in value] + if isinstance(value, str): + text = _SECRET_ASSIGNMENT.sub(_redact_assignment, value) + text = _BEARER_TOKEN.sub(lambda match: match.group(1) + "[REDACTED]", text) + return text if len(text) <= max_text_chars else text[:max_text_chars] + "...[truncated]" + return value + + +def sanitized_text(value: Any, *, max_text_chars: int) -> str: + """Render one bounded text value after structured recursive sanitization.""" + + clean = sanitize(value, max_text_chars=max_text_chars) + if isinstance(clean, str): + text = clean + elif isinstance(clean, (dict, list)): + text = json.dumps(clean, ensure_ascii=False, sort_keys=True, default=str) + else: + text = str(clean) + text = str(sanitize(text, max_text_chars=max_text_chars)) + return text if len(text) <= max_text_chars else text[:max_text_chars] + "...[truncated]" + + +class StrictModel(BaseModel): + model_config = ConfigDict( + extra="forbid", + frozen=True, + populate_by_name=True, + alias_generator=to_camel, + arbitrary_types_allowed=True, + ) + + +def finite_number(value: float, name: str) -> float: + value = float(value) + if not math.isfinite(value): + raise ValueError(f"{name} must be finite") + return value + + +def non_negative_number(value: float, name: str) -> float: + value = finite_number(value, name) + if value < 0: + raise ValueError(f"{name} must be non-negative") + return value diff --git a/examples/optimization/eval_optimize_loop/prompts/system.md b/examples/optimization/eval_optimize_loop/prompts/system.md new file mode 100644 index 000000000..6b4c11388 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/system.md @@ -0,0 +1,3 @@ +You are a deterministic response agent. + +Return the requested answer exactly. Do not add commentary. diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 000000000..399396e3c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,71 @@ +"""Thin CLI for the auditable evaluation and optimization loop.""" + +from __future__ import annotations + +import argparse +import asyncio +import importlib +import sys +from pathlib import Path +from typing import Any + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from examples.optimization.eval_optimize_loop.pipeline.orchestrator import run_pipeline + + +def _load_callback(spec: str) -> Any: + if ":" not in spec: + raise ValueError("--call-agent must use MODULE:FUNCTION syntax") + module_name, function_name = spec.split(":", 1) + if not module_name or not function_name: + raise ValueError("--call-agent must use MODULE:FUNCTION syntax") + return getattr(importlib.import_module(module_name), function_name) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run the auditable prompt optimization loop.") + parser.add_argument("--mode", choices=("fake", "trace", "live")) + parser.add_argument("--config") + parser.add_argument("--train") + parser.add_argument("--validation") + parser.add_argument("--run-id") + parser.add_argument("--call-agent", metavar="MODULE:FUNCTION") + parser.add_argument( + "--apply-candidate", + action=argparse.BooleanOptionalAction, + default=None, + ) + return parser + + +async def _main(args: argparse.Namespace) -> int: + callback = _load_callback(args.call_agent) if args.call_agent else None + report = await run_pipeline( + str(_HERE), + config_path=args.config, + train_path=args.train, + validation_path=args.validation, + mode=args.mode, + run_id=args.run_id, + apply_candidate=args.apply_candidate, + call_agent=callback, + callback_spec=args.call_agent, + ) + print(f"{report.status.value}: run={report.run_id} stage={report.stage}") + if report.gate_decision and report.gate_decision.reasons: + print("reasons=" + ",".join(report.gate_decision.reasons)) + return 1 if report.status.value == "ERROR" else 0 + + +if __name__ == "__main__": + try: + raise SystemExit(asyncio.run(_main(_parser().parse_args()))) + except (KeyboardInterrupt, asyncio.CancelledError): + raise + except Exception as error: + print(f"ERROR: {type(error).__name__}: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/examples/optimization/eval_optimize_loop/traces/trace_cases.json b/examples/optimization/eval_optimize_loop/traces/trace_cases.json new file mode 100644 index 000000000..8d277d772 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/traces/trace_cases.json @@ -0,0 +1,105 @@ +{ + "schemaVersion": "v1", + "datasetHashes": { + "train": "db8df56423d916afa929a2fc1a6f7245d8a1f719e87ac292f88f346d1cb81789", + "validation": "499d3bde9cdc70cc4871d425d0dfee83ed30a67d35da15a4eb87cc9549583f84" + }, + "phases": { + "baseline": { + "train": { + "train_improve_alpha": [ + { + "invocationId": "trace-baseline-train-1", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:ALPHA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "INCORRECT"}]} + } + ], + "train_improve_beta": [ + { + "invocationId": "trace-baseline-train-2", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:BETA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "INCORRECT"}]} + } + ], + "train_stable": [ + { + "invocationId": "trace-baseline-train-3", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:stable; ANSWER:ELEVEN"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "ELEVEN"}]} + } + ] + }, + "validation": { + "validation_improve": [ + { + "invocationId": "trace-baseline-validation-1", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:GAMMA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "INCORRECT"}]} + } + ], + "validation_regress": [ + { + "invocationId": "trace-baseline-validation-2", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:regress; ANSWER:DELTA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "DELTA"}]} + } + ], + "validation_stable": [ + { + "invocationId": "trace-baseline-validation-3", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:stable; ANSWER:OMEGA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "OMEGA"}]} + } + ] + } + }, + "candidate": { + "train": { + "train_improve_alpha": [ + { + "invocationId": "trace-candidate-train-1", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:ALPHA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "ALPHA"}]} + } + ], + "train_improve_beta": [ + { + "invocationId": "trace-candidate-train-2", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:BETA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "BETA"}]} + } + ], + "train_stable": [ + { + "invocationId": "trace-candidate-train-3", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:stable; ANSWER:ELEVEN"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "ELEVEN"}]} + } + ] + }, + "validation": { + "validation_improve": [ + { + "invocationId": "trace-candidate-validation-1", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:GAMMA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "GAMMA"}]} + } + ], + "validation_regress": [ + { + "invocationId": "trace-candidate-validation-2", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:regress; ANSWER:DELTA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "INCORRECT"}]} + } + ], + "validation_stable": [ + { + "invocationId": "trace-candidate-validation-3", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:stable; ANSWER:OMEGA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "OMEGA"}]} + } + ] + } + } + } +} diff --git a/examples/optimization/eval_optimize_loop/train.evalset.json b/examples/optimization/eval_optimize_loop/train.evalset.json new file mode 100644 index 000000000..092ffb140 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/train.evalset.json @@ -0,0 +1,57 @@ +{ + "evalSetId": "eval_optimize_loop_train", + "name": "Deterministic inner optimization training cases", + "evalCases": [ + { + "evalId": "train_improve_alpha", + "conversation": [ + { + "invocationId": "train-input-1", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:improve; ANSWER:ALPHA"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "ALPHA"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + }, + { + "evalId": "train_improve_beta", + "conversation": [ + { + "invocationId": "train-input-2", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:improve; ANSWER:BETA"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "BETA"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + }, + { + "evalId": "train_stable", + "conversation": [ + { + "invocationId": "train-input-3", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:stable; ANSWER:ELEVEN"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "ELEVEN"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/val.evalset.json b/examples/optimization/eval_optimize_loop/val.evalset.json new file mode 100644 index 000000000..d3830e19b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/val.evalset.json @@ -0,0 +1,57 @@ +{ + "evalSetId": "eval_optimize_loop_validation", + "name": "Held-out deterministic regression cases", + "evalCases": [ + { + "evalId": "validation_improve", + "conversation": [ + { + "invocationId": "validation-input-1", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:improve; ANSWER:GAMMA"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "GAMMA"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + }, + { + "evalId": "validation_regress", + "conversation": [ + { + "invocationId": "validation-input-2", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:regress; ANSWER:DELTA"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "DELTA"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + }, + { + "evalId": "validation_stable", + "conversation": [ + { + "invocationId": "validation-input-3", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:stable; ANSWER:OMEGA"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "OMEGA"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + } + ] +} diff --git a/tests/examples/eval_optimize_loop/test_attribution_gate.py b/tests/examples/eval_optimize_loop/test_attribution_gate.py new file mode 100644 index 000000000..42a9ebb4c --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_attribution_gate.py @@ -0,0 +1,416 @@ +"""Deterministic attribution and gate policy tests.""" + +from __future__ import annotations + +import pytest + +from examples.optimization.eval_optimize_loop.pipeline.attribution import attribute_failures +from examples.optimization.eval_optimize_loop.pipeline.configuration import ( + AttributionConfig, + GateConfig, +) +from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate +from examples.optimization.eval_optimize_loop.pipeline.models import ( + CandidateProposal, + CandidateRound, + CaseComparison, + CaseRun, + CaseSnapshot, + ComparisonSnapshot, + CostSource, + CostSummary, + Decision, + EvaluationSnapshot, + FailureCategory, + MetricDelta, + MetricRun, + MetricSnapshot, + Phase, + RubricOutcome, + Split, + Transition, +) + +_MISSING = object() + + +def _invocation(tool: str | None = None, args: dict | None = None) -> dict: + payload = { + "invocationId": "ignored", + "userContent": { + "role": "user", + "parts": [{ + "text": "query" + }] + }, + "finalResponse": { + "role": "model", + "parts": [{ + "text": "answer" + }] + }, + } + if tool: + payload["intermediateData"] = {"toolUses": [{"name": tool, "args": args or {}}]} + return payload + + +def _failed_snapshot( + metric_name: str = "custom", + *, + reason: str | None = "unmapped failure", + error: str | None = None, + actual: dict | None = None, + expected: dict | None | object = _MISSING, + rubrics: tuple[RubricOutcome, ...] = (), +) -> EvaluationSnapshot: + metric = MetricRun( + metric_name=metric_name, + score=0, + threshold=1, + passed=False, + reason=reason, + rubrics=rubrics, + ) + run = CaseRun( + run_id=1, + passed=False, + metrics=(metric, ), + error=error, + trace=({ + "actual": actual or _invocation(), + "expected": _invocation() if expected is _MISSING else expected, + }, ), + ) + case = CaseSnapshot( + case_id="case", + passed=False, + score=0, + metrics=(MetricSnapshot(metric_name=metric_name, score=0, threshold=1, passed=False), ), + runs=(run, ), + error=error, + ) + return EvaluationSnapshot( + split=Split.TRAIN, + phase=Phase.BASELINE, + dataset_score=0, + pass_rate=0, + metric_scores={metric_name: 0}, + case_ids=("case", ), + cases=(case, ), + ) + + +_ATTRIBUTION_BENCHMARK = ( + (_failed_snapshot(error="backend failed"), AttributionConfig(), FailureCategory.EVALUATION_ERROR), + ( + _failed_snapshot(actual=_invocation("search"), expected=_invocation("calculate")), + AttributionConfig(), + FailureCategory.TOOL_CALL_ERROR, + ), + ( + _failed_snapshot( + actual=_invocation("search", {"q": "wrong"}), + expected=_invocation("search", {"q": "right"}), + ), + AttributionConfig(), + FailureCategory.TOOL_ARGUMENT_ERROR, + ), + ( + _failed_snapshot("llm_rubric_knowledge_recall"), + AttributionConfig(), + FailureCategory.KNOWLEDGE_RECALL_INSUFFICIENT, + ), + ( + _failed_snapshot("format_metric"), + AttributionConfig(metric_categories={"format_metric": FailureCategory.FORMAT_VIOLATION}), + FailureCategory.FORMAT_VIOLATION, + ), + ( + _failed_snapshot("llm_rubric_response"), + AttributionConfig(), + FailureCategory.LLM_RUBRIC_NOT_MET, + ), + ( + _failed_snapshot("final_response_avg_score"), + AttributionConfig(), + FailureCategory.FINAL_RESPONSE_MISMATCH, + ), + (_failed_snapshot(), AttributionConfig(), FailureCategory.UNKNOWN), +) + + +@pytest.mark.parametrize( + ("snapshot", "config", "expected"), + _ATTRIBUTION_BENCHMARK, +) +def test_attribution_taxonomy(snapshot, config, expected) -> None: + result = attribute_failures(snapshot, config, max_text_chars=200) + assert result.failures[0].primary == expected + assert result.failures[0].reasons + assert result.failures[0].evidence + + +def test_attribution_policy_matrix_matches_every_expected_category() -> None: + assert all( + attribute_failures(snapshot, config, max_text_chars=200).failures[0].primary == expected + for snapshot, config, expected in _ATTRIBUTION_BENCHMARK) + + +def test_reference_free_failure_uses_metric_attribution_without_expected_trace() -> None: + result = attribute_failures( + _failed_snapshot("llm_rubric_response", expected=None), + AttributionConfig(), + max_text_chars=200, + ) + assert result.failures[0].primary == FailureCategory.LLM_RUBRIC_NOT_MET + + +def test_tool_difference_evidence_is_recursively_sanitized() -> None: + result = attribute_failures( + _failed_snapshot( + actual=_invocation("search", {"apiKey": "actual-tool-secret"}), + expected=_invocation("search", {"apiKey": "expected-tool-secret"}), + ), + AttributionConfig(), + max_text_chars=1000, + ) + evidence = "\n".join(result.failures[0].evidence) + assert "actual-tool-secret" not in evidence + assert "expected-tool-secret" not in evidence + assert "[REDACTED]" in evidence + + +def test_structured_argument_difference_outranks_generic_tool_metric() -> None: + result = attribute_failures( + _failed_snapshot( + "tool_trajectory_avg_score", + actual=_invocation("search", {"q": "wrong"}), + expected=_invocation("search", {"q": "right"}), + ), + AttributionConfig(), + max_text_chars=1000, + ) + assert result.failures[0].primary == FailureCategory.TOOL_ARGUMENT_ERROR + + +def test_attribution_uses_only_failed_rubric_outcomes() -> None: + snapshot = _failed_snapshot( + "llm_rubric_response", + rubrics=( + RubricOutcome(id="passed_rule", score=1, passed=True), + RubricOutcome(id="failed_rule", score=0, passed=False), + ), + ) + config = AttributionConfig(rubric_categories={ + "passed_rule": FailureCategory.TOOL_CALL_ERROR, + "failed_rule": FailureCategory.FORMAT_VIOLATION, + }) + + result = attribute_failures(snapshot, config, max_text_chars=200) + + failure = result.failures[0] + assert failure.primary == FailureCategory.FORMAT_VIOLATION + assert failure.trigger_rubrics == ("failed_rule", ) + + +def _comparison( + split: Split, + *, + score_delta: float = 0.1, + pass_delta: float = 1.0, + critical: bool = False, + hard: bool = False, +) -> ComparisonSnapshot: + if pass_delta > 0: + baseline_passed, candidate_passed = False, True + transition = Transition.NEW_PASS + elif pass_delta < 0: + baseline_passed, candidate_passed = True, False + transition = Transition.NEW_FAIL + else: + baseline_passed = candidate_passed = True + transition = (Transition.IMPROVED + if score_delta > 0 else Transition.REGRESSED if score_delta < 0 else Transition.UNCHANGED) + return ComparisonSnapshot( + split=split, + score_delta=score_delta, + pass_rate_delta=pass_delta, + metric_deltas={"quality": score_delta}, + cases=(CaseComparison( + case_id="case", + baseline_passed=baseline_passed, + candidate_passed=candidate_passed, + baseline_score=0, + candidate_score=score_delta, + delta=score_delta, + metrics=(MetricDelta( + metric_name="quality", + baseline=0, + candidate=score_delta, + delta=score_delta, + ), ), + transition=transition, + critical=critical, + hard=hard, + new_hard_failure=hard and baseline_passed and not candidate_passed, + ), ), + ) + + +def _candidate() -> CandidateProposal: + return CandidateProposal( + algorithm="fake", + baseline_prompts={"system": "old"}, + prompts={"system": "new"}, + changed=True, + rounds=(CandidateRound( + round=1, + candidate_prompts={"system": "new"}, + accepted=True, + score=1, + ), ), + ) + + +def _gate(**overrides): + values = { + "train": _comparison(Split.TRAIN), + "validation": _comparison(Split.VALIDATION), + "candidate": _candidate(), + "cost": CostSummary( + sources=(CostSource(name="fake", cost_usd=0), ), + total_cost_usd=0, + ), + "duration_seconds": 1, + "baseline_prompt_hashes": { + "system": "old" + }, + "candidate_prompt_hashes": { + "system": "new" + }, + "config": GateConfig(metric_max_regression={"quality": 0}), + } + values.update(overrides) + return evaluate_gate(**values) + + +def test_gate_accepts_only_when_checks_pass_in_fixed_order() -> None: + result = _gate() + assert result.decision == Decision.ACCEPT + assert [check.code for check in result.checks] == [ + "REPORT_COMPLETE", + "CANDIDATE_CHANGED", + "VALIDATION_SCORE_DELTA", + "VALIDATION_PASS_RATE_DELTA", + "NO_NEW_HARD_FAIL", + "CRITICAL_CASE_NON_REGRESSION", + "METRIC_NON_REGRESSION", + "COST_BUDGET", + "DURATION_BUDGET", + "OVERFIT_GUARD", + ] + + +def test_gate_rejects_train_only_improvement_as_overfit() -> None: + validation = _comparison(Split.VALIDATION, score_delta=-0.1, pass_delta=0) + result = _gate(validation=validation) + assert result.decision == Decision.REJECT + assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in result.reasons + + +def test_gate_fails_closed_when_enabled_cost_is_unknown() -> None: + cost = CostSummary( + sources=(CostSource(name="live-agent", cost_usd=None), ), + total_cost_usd=None, + ) + result = _gate(cost=cost, config=GateConfig(max_cost_usd=1, metric_max_regression={"quality": 0})) + assert result.decision == Decision.REJECT + assert "COST_UNAVAILABLE" in result.reasons + + +def test_gate_returns_error_for_missing_configured_metric() -> None: + result = _gate(config=GateConfig(metric_max_regression={"missing": 0})) + assert result.decision == Decision.ERROR + assert result.reasons == ("GATE_INPUT_INVALID", ) + + +def test_gate_policy_matrix_matches_every_expected_decision() -> None: + unchanged = CandidateProposal( + algorithm="fake", + baseline_prompts={"system": "old"}, + prompts={"system": "old"}, + changed=False, + ) + unknown_cost = CostSummary( + sources=(CostSource(name="live", cost_usd=None), ), + total_cost_usd=None, + ) + scenarios = ( + (_gate(), Decision.ACCEPT), + (_gate(report_complete=False), Decision.REJECT), + ( + _gate( + candidate=unchanged, + candidate_prompt_hashes={"system": "old"}, + ), + Decision.REJECT, + ), + ( + _gate(config=GateConfig(min_validation_score_delta=0.2)), + Decision.REJECT, + ), + ( + _gate(config=GateConfig(min_validation_pass_rate_delta=1.1)), + Decision.REJECT, + ), + ( + _gate(validation=_comparison( + Split.VALIDATION, + score_delta=-0.1, + pass_delta=-1, + hard=True, + )), + Decision.REJECT, + ), + ( + _gate(validation=_comparison( + Split.VALIDATION, + score_delta=-0.1, + pass_delta=-1, + critical=True, + )), + Decision.REJECT, + ), + ( + _gate( + validation=_comparison(Split.VALIDATION, score_delta=-0.1, pass_delta=0), + config=GateConfig( + min_validation_score_delta=-1, + min_validation_pass_rate_delta=-1, + metric_max_regression={"quality": 0}, + ), + ), + Decision.REJECT, + ), + ( + _gate( + cost=unknown_cost, + config=GateConfig(max_cost_usd=1, metric_max_regression={"quality": 0}), + ), + Decision.REJECT, + ), + ( + _gate( + duration_seconds=2, + config=GateConfig(max_duration_seconds=1, metric_max_regression={"quality": 0}), + ), + Decision.REJECT, + ), + ( + _gate(config=GateConfig(metric_max_regression={"missing": 0})), + Decision.ERROR, + ), + ) + assert len(scenarios) >= 6 + assert all(actual.decision == expected for actual, expected in scenarios) diff --git a/tests/examples/eval_optimize_loop/test_backends.py b/tests/examples/eval_optimize_loop/test_backends.py new file mode 100644 index 000000000..c26430d86 --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_backends.py @@ -0,0 +1,741 @@ +"""Backend and candidate-generator substitution contract tests.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from trpc_agent_sdk.evaluation import AgentEvaluator, EvalConfig, EvalSet, TargetPrompt + +from examples.optimization.eval_optimize_loop.pipeline.backends import ( + DeterministicCandidateGenerator, + FakeEvaluationBackend, + LiveCandidateGenerator, + LiveEvaluationBackend, + TraceEvaluationBackend, +) +from examples.optimization.eval_optimize_loop.pipeline.evaluation import normalize_result +from examples.optimization.eval_optimize_loop.pipeline.models import ( + AttributionSnapshot, + Phase, + Split, +) + + +def _eval_set() -> EvalSet: + return EvalSet.model_validate({ + "evalSetId": + "backend_contract", + "evalCases": [{ + "evalId": + "case", + "conversation": [{ + "invocationId": "expected", + "userContent": { + "role": "user", + "parts": [{ + "text": "BEHAVIOR:stable; ANSWER:OK" + }], + }, + "finalResponse": { + "role": "model", + "parts": [{ + "text": "OK" + }] + }, + }], + "sessionInput": { + "appName": "backend", + "userId": "test", + "state": {} + }, + }], + }) + + +def _config() -> EvalConfig: + return EvalConfig( + metrics=[{ + "metric_name": "final_response_avg_score", + "threshold": 1, + "criterion": { + "final_response": { + "text": { + "match": "exact" + } + } + }, + }], + num_runs=1, + ) + + +@pytest.mark.asyncio +async def test_fake_backend_uses_public_services_and_does_not_mutate_inputs(tmp_path, monkeypatch) -> None: + backend = FakeEvaluationBackend() + eval_set = _eval_set() + config = _config() + before_set = eval_set.model_dump(mode="json") + before_config = config.model_dump(mode="json") + monkeypatch.setattr( + AgentEvaluator, + "get_executer", + lambda *args, **kwargs: pytest.fail("backend must compose public EvalService APIs directly"), + ) + raw = await backend.evaluate( + eval_set=eval_set, + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + assert snapshot.pass_rate == 1 + assert eval_set.model_dump(mode="json") == before_set + assert config.model_dump(mode="json") == before_config + + +@pytest.mark.asyncio +async def test_fake_llm_metric_uses_deterministic_offline_substitute(tmp_path, monkeypatch) -> None: + + def forbid_real_judge(*args, **kwargs): + raise AssertionError("fake mode constructed a real LLM judge") + + monkeypatch.setattr( + AgentEvaluator, + "get_executer", + lambda *args, **kwargs: pytest.fail("backend must not depend on AgentEvaluator file orchestration"), + ) + monkeypatch.setattr( + "trpc_agent_sdk.evaluation._llm_judge.LLMJudge.__init__", + forbid_real_judge, + ) + config = EvalConfig( + metrics=[ + { + "metric_name": "llm_final_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + } + } + }, + }, + { + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "response_quality", + "content": { + "text": "The response is correct." + }, + "type": "OFFLINE_RESPONSE_EXACT_REFERENCE", + }], + } + }, + }, + ], + num_runs=1, + ) + eval_set = _eval_set() + raw = await FakeEvaluationBackend().evaluate( + eval_set=eval_set, + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + assert snapshot.metric_scores == { + "llm_final_response": 1, + "llm_rubric_response": 1, + } + run_metrics = {metric.metric_name: metric for metric in snapshot.cases[0].runs[0].metrics} + assert [rubric.id for rubric in run_metrics["llm_rubric_response"].rubrics] == ["response_quality"] + + +@pytest.mark.asyncio +async def test_fake_knowledge_recall_fails_before_black_box_inference(tmp_path, monkeypatch) -> None: + + def fail_if_called(*args, **kwargs): + raise AssertionError("black-box inference ran before capability validation") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.backends.deterministic_fake_response", + fail_if_called, + ) + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_knowledge_recall", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "knowledge_evidence", + "type": "OFFLINE_KNOWLEDGE_NON_EMPTY", + }], + } + }, + }], + num_runs=1, + ) + with pytest.raises(ValueError, match="does not support metrics"): + await FakeEvaluationBackend().evaluate( + eval_set=_eval_set(), + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + + +@pytest.mark.asyncio +async def test_offline_rubric_rejects_unstructured_natural_language(tmp_path) -> None: + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "quality", + "content": { + "text": "Be correct." + } + }], + } + }, + }], + num_runs=1, + ) + with pytest.raises(ValueError, match="requires an explicit type"): + await FakeEvaluationBackend().evaluate( + eval_set=_eval_set(), + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + + +@pytest.mark.asyncio +async def test_offline_equals_requires_explicit_content(tmp_path) -> None: + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "missing_operand", + "type": "OFFLINE_RESPONSE_EQUALS", + }], + } + }, + }], + num_runs=1, + ) + with pytest.raises(ValueError, match="requires content.text"): + await FakeEvaluationBackend().evaluate( + eval_set=_eval_set(), + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + + +@pytest.mark.asyncio +async def test_offline_exact_reference_preserves_empty_response(tmp_path) -> None: + eval_set = _eval_set() + invocation = eval_set.eval_cases[0].conversation[0] + invocation.user_content.parts[0].text = "BEHAVIOR:stable; ANSWER:" + invocation.final_response.parts[0].text = "" + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "empty_reference", + "type": "OFFLINE_RESPONSE_EXACT_REFERENCE", + }], + } + }, + }], + num_runs=1, + ) + raw = await FakeEvaluationBackend().evaluate( + eval_set=eval_set, + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + assert snapshot.metric_scores == {"llm_rubric_response": 1} + + +@pytest.mark.asyncio +async def test_trace_reference_free_rubric_uses_placeholder_expected(tmp_path) -> None: + source = _eval_set().model_dump(mode="json", by_alias=True, exclude_none=True) + case = source["evalCases"][0] + actual = case.pop("conversation") + case["evalMode"] = "trace" + case["actualConversation"] = actual + eval_set = EvalSet.model_validate(source) + fixture = { + "schemaVersion": "v1", + "datasetHashes": { + "train": "train-hash", + "validation": "validation-hash" + }, + "phases": { + phase: { + "train": { + "case": actual + }, + "validation": { + "case": actual + }, + } + for phase in ("baseline", "candidate") + }, + } + fixture_path = tmp_path / "trace-reference-free.json" + fixture_path.write_text(json.dumps(fixture), encoding="utf-8") + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [ + { + "id": "reference_free_quality", + "content": { + "text": "OK" + }, + "type": "OFFLINE_RESPONSE_CONTAINS", + }, + { + "id": "missing_quality", + "content": { + "text": "MISSING" + }, + "type": "OFFLINE_RESPONSE_CONTAINS", + }, + ], + } + }, + }], + num_runs=1, + ) + raw = await TraceEvaluationBackend( + str(fixture_path), + { + "train": "train-hash", + "validation": "validation-hash" + }, + ).evaluate( + eval_set=eval_set, + eval_config=config, + prompts={"system": "unused"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + metric = snapshot.cases[0].runs[0].metrics[0] + assert metric.score == 0.5 + assert [(rubric.id, rubric.passed) for rubric in metric.rubrics] == [ + ("reference_free_quality", True), + ("missing_quality", False), + ] + assert snapshot.cases[0].runs[0].trace[0]["expected"] is None + + +@pytest.mark.asyncio +async def test_trace_knowledge_recall_scores_recorded_tool_evidence(tmp_path) -> None: + source = _eval_set().model_dump(mode="json", by_alias=True, exclude_none=True) + case = source["evalCases"][0] + actual = case.pop("conversation") + actual[0]["intermediateData"] = { + "toolResponses": [{ + "name": "knowledge_search", + "response": { + "text": "pinned source fact" + }, + }] + } + case["evalMode"] = "trace" + case["actualConversation"] = actual + eval_set = EvalSet.model_validate(source) + fixture = { + "schemaVersion": "v1", + "datasetHashes": { + "train": "train-hash", + "validation": "validation-hash" + }, + "phases": { + phase: { + "train": { + "case": actual + }, + "validation": { + "case": actual + } + } + for phase in ("baseline", "candidate") + }, + } + fixture_path = tmp_path / "trace-knowledge.json" + fixture_path.write_text(json.dumps(fixture), encoding="utf-8") + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_knowledge_recall", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "source_fact", + "content": { + "text": "pinned source fact" + }, + "type": "OFFLINE_KNOWLEDGE_CONTAINS", + }], + } + }, + }], + num_runs=1, + ) + raw = await TraceEvaluationBackend( + str(fixture_path), + { + "train": "train-hash", + "validation": "validation-hash" + }, + ).evaluate( + eval_set=eval_set, + eval_config=config, + prompts={"system": "unused"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + metric = snapshot.cases[0].runs[0].metrics[0] + assert metric.score == 1 + assert [(rubric.id, rubric.passed) for rubric in metric.rubrics] == [("source_fact", True)] + + +@pytest.mark.asyncio +async def test_trace_backend_replays_fixture_without_agent(tmp_path) -> None: + eval_set = _eval_set() + actual = eval_set.eval_cases[0].conversation[0].model_dump(mode="json", by_alias=True) + fixture = { + "schemaVersion": "v1", + "datasetHashes": { + "train": "train-hash", + "validation": "validation-hash" + }, + "phases": { + phase: { + "train": { + "case": [actual] + }, + "validation": { + "case": [actual] + }, + } + for phase in ("baseline", "candidate") + }, + } + fixture_path = tmp_path / "trace.json" + fixture_path.write_text(__import__("json").dumps(fixture), encoding="utf-8") + backend = TraceEvaluationBackend( + str(fixture_path), + { + "train": "train-hash", + "validation": "validation-hash" + }, + ) + fixture["phases"]["baseline"]["train"]["case"][0]["finalResponse"] = { + "role": "model", + "parts": [{ + "text": "CHANGED_AFTER_PREFLIGHT" + }], + } + fixture_path.write_text(json.dumps(fixture), encoding="utf-8") + raw = await backend.evaluate( + eval_set=eval_set, + eval_config=_config(), + prompts={"system": "unused"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + _config(), + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + assert snapshot.pass_rate == 1 + + +def test_live_backend_requires_async_callback() -> None: + with pytest.raises(TypeError, match="async"): + LiveEvaluationBackend(lambda query: query) # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_deterministic_generator_only_uses_failure_facts(tmp_path) -> None: + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + target = TargetPrompt().add_path("system", str(prompt_path)) + attribution = AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()) + proposal = await DeterministicCandidateGenerator().generate( + target_prompt=target, + baseline_prompts={"system": "baseline"}, + train_attribution=attribution, + inner_train_path="inner-train", + inner_selection_path="inner-selection", + config_path="config", + output_dir="output", + ) + assert proposal.changed is False + + +@pytest.mark.asyncio +async def test_live_generator_forces_update_source_false(monkeypatch, tmp_path) -> None: + + async def call_agent(query: str) -> str: + return query + + captured = {} + + async def fake_optimize(**kwargs): + captured.update(kwargs) + round_ = SimpleNamespace( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=True, + validation_pass_rate=1, + kind="merge", + optimized_field_names=["system"], + metric_breakdown={"quality": 0.9}, + acceptance_reason="accepted by optimizer", + skip_reason=None, + error_message=None, + round_llm_cost=0.035, + round_token_usage={"total": 0}, + duration_seconds=0, + ) + return SimpleNamespace( + status="SUCCEEDED", + error_message="", + rounds=[round_], + algorithm="gepa_reflective", + baseline_prompts={"system": "baseline"}, + best_prompts={"system": "candidate"}, + stop_reason="completed", + total_reflection_lm_calls=1, + total_llm_cost=0, + total_token_usage={"total": 0}, + duration_seconds=0, + ) + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.backends.AgentOptimizer.optimize", + fake_optimize, + ) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + target = TargetPrompt().add_path("system", str(prompt_path)) + generator = LiveCandidateGenerator(call_agent) + proposal = await generator.generate( + target_prompt=target, + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir="optimizer-output", + ) + assert captured["update_source"] is False + assert captured["validation_dataset_path"] == "inner-selection.json" + assert proposal.prompts == {"system": "candidate"} + assert proposal.rounds[0].kind == "merge" + assert proposal.rounds[0].cost_usd == 0.035 + assert proposal.rounds[0].metric_scores == {"quality": 0.9} + assert [(source.name, source.cost_usd, source.model_calls) for source in proposal.cost_sources] == [ + ("optimizer_reported", 0, 1), + ("judge_unreported", None, None), + ] + + +@pytest.mark.asyncio +async def test_live_generator_accepts_sdk_skipped_round(monkeypatch, tmp_path) -> None: + + async def call_agent(query: str) -> str: + return query + + async def fake_optimize(**kwargs): + return SimpleNamespace( + status="SUCCEEDED", + error_message="", + rounds=[ + SimpleNamespace( + round=1, + candidate_prompts={}, + accepted=False, + validation_pass_rate=0.0, + kind="reflective", + optimized_field_names=[], + metric_breakdown={}, + acceptance_reason="no candidate produced this round", + skip_reason="reflect-LM produced no usable new prompt", + error_message=None, + round_llm_cost=0.02, + round_token_usage={"total": 4}, + duration_seconds=0.3, + ) + ], + algorithm="gepa_reflective", + baseline_prompts={"system": "baseline"}, + best_prompts={"system": "baseline"}, + stop_reason="completed", + total_reflection_lm_calls=1, + total_llm_cost=0.02, + total_token_usage={"total": 4}, + duration_seconds=0.4, + ) + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.backends.AgentOptimizer.optimize", + fake_optimize, + ) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + proposal = await LiveCandidateGenerator(call_agent).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + ) + assert proposal.changed is False + assert proposal.rounds[0].candidate_prompts == {} + assert proposal.rounds[0].skip_reason == "reflect-LM produced no usable new prompt" + assert proposal.rounds[0].score is None + assert proposal.rounds[0].cost_usd == 0.02 + + +@pytest.mark.asyncio +async def test_live_generator_requests_cooperative_stop_before_propagating_cancel(monkeypatch, tmp_path) -> None: + + async def call_agent(query: str) -> str: + return query + + started = asyncio.Event() + stopped = asyncio.Event() + + async def fake_optimize(**kwargs): + started.set() + stop_path = Path(kwargs["output_dir"]) / "optimize.stop" + while not stop_path.exists(): + await asyncio.sleep(0) + stopped.set() + return SimpleNamespace(status="CANCELED", error_message="user stop") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.backends.AgentOptimizer.optimize", + fake_optimize, + ) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + task = asyncio.create_task( + LiveCandidateGenerator(call_agent).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + )) + await started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert stopped.is_set() diff --git a/tests/examples/eval_optimize_loop/test_candidate_runtime.py b/tests/examples/eval_optimize_loop/test_candidate_runtime.py new file mode 100644 index 000000000..16add9eff --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_candidate_runtime.py @@ -0,0 +1,57 @@ +"""Candidate-generation lifecycle tests for primary-error and artifact boundaries.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.pipeline.candidate_runtime import generate_candidate + + +class _DumpConfig: + + def model_dump(self, **kwargs): + return {} + + +class _Workspace: + baseline = {"system": "baseline"} + + def create_candidate_target(self, path: str): + Path(path).mkdir(parents=True, exist_ok=False) + return object() + + +class _ImportFailingSink: + + def phase_dir(self, name: str) -> None: + return None + + def import_tree(self, source: str | Path, destination: str) -> None: + raise json.JSONDecodeError("half-written optimizer output", "{", 1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("primary", [asyncio.CancelledError("cancel"), RuntimeError("optimizer failed")]) +async def test_artifact_import_failure_does_not_replace_generator_failure(primary) -> None: + + class FailingGenerator: + + async def generate(self, **kwargs): + raise primary + + with pytest.raises(type(primary), match=str(primary)) as raised: + await generate_candidate( + generator=FailingGenerator(), + workspace=_Workspace(), + sink=_ImportFailingSink(), + eval_config=_DumpConfig(), + optimize_config=_DumpConfig(), + train_attribution=object(), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + ) + assert any("artifact import also failed" in note for note in raised.value.__notes__) diff --git a/tests/examples/eval_optimize_loop/test_evaluation.py b/tests/examples/eval_optimize_loop/test_evaluation.py new file mode 100644 index 000000000..c76342efd --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_evaluation.py @@ -0,0 +1,232 @@ +"""Dataset, normalization and comparison tests.""" + +from __future__ import annotations + +from copy import deepcopy + +import pytest + +from trpc_agent_sdk.evaluation import ( + EvalCaseResult, + EvalConfig, + EvalMetricResult, + EvalMetricResultPerInvocation, + EvalSet, + EvalSetAggregateResult, + EvalStatus, + EvaluateResult, +) + +from examples.optimization.eval_optimize_loop.pipeline.evaluation import ( + compare_snapshots, + normalize_result, + split_train_dataset, + validate_datasets, +) +from examples.optimization.eval_optimize_loop.pipeline.models import Phase, Split, Transition + + +def _eval_set(set_id: str, case_ids: list[str]) -> EvalSet: + return EvalSet.model_validate({ + "evalSetId": + set_id, + "evalCases": [{ + "evalId": + case_id, + "conversation": [{ + "invocationId": f"input-{case_id}", + "userContent": { + "role": "user", + "parts": [{ + "text": f"query {case_id}" + }] + }, + "finalResponse": { + "role": "model", + "parts": [{ + "text": "expected" + }] + }, + }], + "sessionInput": { + "appName": "loop-test", + "userId": "test", + "state": {} + }, + } for case_id in case_ids], + }) + + +def _raw(eval_set: EvalSet, scores: dict[str, list[float]], threshold: float = 0.5) -> EvaluateResult: + by_case: dict[str, list[EvalCaseResult]] = {} + for case in eval_set.eval_cases: + runs = [] + expected = case.conversation[0] + for run_id, score in enumerate(scores[case.eval_id], 1): + passed = score >= threshold + status = EvalStatus.PASSED if passed else EvalStatus.FAILED + actual = expected.model_copy( + update={ + "invocation_id": f"actual-{case.eval_id}-{run_id}", + "final_response": expected.final_response, + }, + deep=True, + ) + metric = EvalMetricResult( + metric_name="quality", + threshold=threshold, + score=score, + eval_status=status, + ) + runs.append( + EvalCaseResult( + eval_set_id=eval_set.eval_set_id, + eval_id=case.eval_id, + run_id=run_id, + final_eval_status=status, + overall_eval_metric_results=[metric], + eval_metric_result_per_invocation=[ + EvalMetricResultPerInvocation( + actual_invocation=actual, + expected_invocation=expected, + eval_metric_results=[metric], + ) + ], + session_id=f"session-{run_id}", + user_id="test", + )) + by_case[case.eval_id] = runs + return EvaluateResult( + results_by_eval_set_id={ + eval_set.eval_set_id: + EvalSetAggregateResult( + eval_results_by_eval_id=by_case, + num_runs=len(next(iter(scores.values()))), + ) + }) + + +def test_dataset_validation_rejects_content_leakage() -> None: + train = _eval_set("train", ["a", "b", "c"]) + validation = _eval_set("validation", ["v"]) + leaked = train.eval_cases[0].model_copy(update={"eval_id": "v"}, deep=True) + validation = validation.model_copy(update={"eval_cases": [leaked]}, deep=True) + with pytest.raises(ValueError, match="duplicate case content"): + validate_datasets( + train, + validation, + train_path="train.json", + validation_path="validation.json", + configured_metrics=["quality"], + ) + + +def test_dataset_validation_rejects_same_input_with_changed_reference() -> None: + train = _eval_set("train", ["a", "b", "c"]) + validation = _eval_set("validation", ["v"]) + leaked = train.eval_cases[0].model_copy(update={"eval_id": "v"}, deep=True) + leaked.conversation[0].final_response.parts[0].text = "different label" + validation = validation.model_copy(update={"eval_cases": [leaked]}, deep=True) + with pytest.raises(ValueError, match="duplicate model inputs"): + validate_datasets( + train, + validation, + train_path="train.json", + validation_path="validation.json", + configured_metrics=["quality"], + ) + + +@pytest.mark.parametrize("metrics", [[], ["quality", "quality"]]) +def test_dataset_validation_rejects_invalid_metric_schema(metrics) -> None: + with pytest.raises(ValueError, match="metric names"): + validate_datasets( + _eval_set("train", ["a", "b", "c"]), + _eval_set("validation", ["v"]), + train_path="train.json", + validation_path="validation.json", + configured_metrics=metrics, + ) + + +def test_normalization_aggregates_runs_and_inner_split_is_deterministic() -> None: + train = _eval_set("train", ["a", "b", "c"]) + config = EvalConfig(metrics=[{"metric_name": "quality", "threshold": 0.5}], num_runs=2) + snapshot = normalize_result( + _raw(train, { + "a": [1, 1], + "b": [0, 1], + "c": [0, 0] + }), + train, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + case_weights={"a": 2}, + ) + assert snapshot.case_ids == ("a", "b", "c") + assert snapshot.cases[1].score == 0.5 + assert snapshot.cases[1].passed is False + assert snapshot.dataset_score == pytest.approx(0.625) + first = split_train_dataset(train, snapshot, seed=42, selection_ratio=0.34) + second = split_train_dataset(train, snapshot, seed=42, selection_ratio=0.34) + assert [case.eval_id for case in first[0].eval_cases] == [case.eval_id for case in second[0].eval_cases] + assert len(first[0].eval_cases) == 2 + assert len(first[1].eval_cases) == 1 + + +def test_normalization_fails_on_threshold_drift() -> None: + eval_set = _eval_set("validation", ["v"]) + config = EvalConfig(metrics=[{"metric_name": "quality", "threshold": 0.5}], num_runs=1) + raw = _raw(eval_set, {"v": [1]}, threshold=0.4) + with pytest.raises(ValueError, match="threshold drift"): + normalize_result(raw, eval_set, config, split=Split.VALIDATION, phase=Phase.BASELINE) + + +def test_comparison_classifies_all_transition_types() -> None: + eval_set = _eval_set("validation", ["a", "b", "c", "d", "e"]) + config = EvalConfig(metrics=[{"metric_name": "quality", "threshold": 0.5}], num_runs=1) + baseline = normalize_result( + _raw(eval_set, { + "a": [0], + "b": [1], + "c": [0.6], + "d": [0.9], + "e": [1] + }), + eval_set, + config, + split=Split.VALIDATION, + phase=Phase.BASELINE, + ) + candidate = normalize_result( + _raw(eval_set, { + "a": [1], + "b": [0], + "c": [0.8], + "d": [0.7], + "e": [1] + }), + eval_set, + config, + split=Split.VALIDATION, + phase=Phase.CANDIDATE, + ) + comparison = compare_snapshots(baseline, candidate, epsilon=1e-6, hard_case_ids=["b"]) + assert [case.transition for case in comparison.cases] == [ + Transition.NEW_PASS, + Transition.NEW_FAIL, + Transition.IMPROVED, + Transition.REGRESSED, + Transition.UNCHANGED, + ] + assert comparison.cases[1].new_hard_failure is True + + +def test_normalizer_does_not_mutate_sdk_result() -> None: + eval_set = _eval_set("validation", ["v"]) + config = EvalConfig(metrics=[{"metric_name": "quality", "threshold": 0.5}], num_runs=1) + raw = _raw(eval_set, {"v": [1]}) + before = deepcopy(raw.model_dump(mode="json")) + normalize_result(raw, eval_set, config, split=Split.VALIDATION, phase=Phase.BASELINE) + assert raw.model_dump(mode="json") == before diff --git a/tests/examples/eval_optimize_loop/test_models_and_imports.py b/tests/examples/eval_optimize_loop/test_models_and_imports.py new file mode 100644 index 000000000..5dac83585 --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_models_and_imports.py @@ -0,0 +1,321 @@ +"""Strict model and architecture boundary tests for the loop example.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from examples.optimization.eval_optimize_loop.pipeline.configuration import ( + GateConfig, + PipelineSettings, +) +from examples.optimization.eval_optimize_loop.pipeline.models import ( + CandidateProposal, + CandidateRound, + CaseComparison, + CostSource, + CostSummary, + MetricDelta, + MetricRun, + OptimizationReport, + RubricOutcome, + Transition, +) +from examples.optimization.eval_optimize_loop.pipeline.schema import ( + add_exception_note, + parse_strict_json, + sanitized_text, + validate_safe_component, +) + +PIPELINE_DIR = Path(__file__).resolve().parents[3] / "examples" / "optimization" / "eval_optimize_loop" / "pipeline" + + +def test_models_forbid_unknown_and_non_finite_values() -> None: + with pytest.raises(ValidationError): + PipelineSettings.model_validate({"unexpected": True}) + with pytest.raises(ValidationError): + GateConfig(epsilon=float("nan")) + with pytest.raises(ValidationError): + PipelineSettings(metric_weights={"metric": 0}) + with pytest.raises(ValidationError, match="v2"): + OptimizationReport.model_validate({"schemaVersion": "v1"}) + + +def test_pipeline_policy_scalars_reject_string_coercion() -> None: + with pytest.raises(ValidationError): + PipelineSettings.model_validate({"applyCandidate": "yes"}) + with pytest.raises(ValidationError): + PipelineSettings.model_validate({"seed": "42"}) + with pytest.raises(ValidationError): + GateConfig.model_validate({"maxNewHardFailures": "2"}) + with pytest.raises(ValidationError): + GateConfig.model_validate({"overfitGuard": "false"}) + + +def test_strict_json_and_path_components_reject_ambiguous_inputs() -> None: + with pytest.raises(ValueError, match="duplicate"): + parse_strict_json('{"key": 1, "key": 2}') + with pytest.raises(ValueError, match="non-finite"): + parse_strict_json('{"value": NaN}') + with pytest.raises(ValueError, match="non-finite"): + parse_strict_json('{"value": 1e999}') + for value in ("../escape", "a..b", "CON", "name.", "C:drive"): + with pytest.raises(ValueError): + validate_safe_component(value) + + +def test_candidate_requires_contiguous_accepted_history() -> None: + with pytest.raises(ValidationError): + CandidateProposal( + algorithm="fake", + baseline_prompts={"system": "old"}, + prompts={"system": "new"}, + changed=True, + rounds=(CandidateRound( + round=2, + candidate_prompts={"system": "new"}, + accepted=True, + score=1, + ), ), + ) + + +def test_candidate_accounting_defaults_to_unknown() -> None: + proposal = CandidateProposal( + algorithm="custom", + baseline_prompts={"system": "same"}, + prompts={"system": "same"}, + changed=False, + ) + assert proposal.cost_sources == (CostSource(name="unreported", cost_usd=None), ) + with pytest.raises(ValidationError): + CandidateRound( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=True, + score=1, + cost_usd=-1, + ) + + +def test_candidate_round_distinguishes_skipped_rounds_and_rejects_invalid_accounting() -> None: + skipped = CandidateProposal( + algorithm="gepa_reflective", + baseline_prompts={"system": "same"}, + prompts={"system": "same"}, + changed=False, + rounds=(CandidateRound( + round=1, + candidate_prompts={}, + accepted=False, + skip_reason="reflect-LM produced no usable new prompt", + ), ), + ) + assert skipped.rounds[0].score is None + with pytest.raises(ValidationError, match="skip or error"): + CandidateRound(round=1, candidate_prompts={}, accepted=False) + with pytest.raises(ValidationError, match="round score"): + CandidateRound( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=False, + score=-0.1, + ) + with pytest.raises(ValidationError, match="token usage"): + CandidateRound( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=False, + score=0, + token_usage={"total": -1}, + ) + negative_metric = CandidateRound( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=False, + score=0, + metric_scores={"log_likelihood": -2.5}, + ) + assert negative_metric.metric_scores == {"log_likelihood": -2.5} + + +def test_sanitized_text_redacts_structured_values_and_exception_messages() -> None: + assert "tool-secret" not in sanitized_text( + {"actual": { + "apiKey": "tool-secret" + }}, + max_text_chars=1000, + ) + assert "error-secret" not in sanitized_text( + RuntimeError('{"clientSecret":"error-secret"}'), + max_text_chars=1000, + ) + + +def test_exception_notes_work_without_python_311_add_note() -> None: + + class LegacyError(RuntimeError): + add_note = None + + error = LegacyError("primary") + add_exception_note(error, "secondary diagnostic") + assert error.__notes__ == ["secondary diagnostic"] + + +def test_rubric_and_cost_contracts_reject_inconsistent_aggregates() -> None: + scaled = MetricRun( + metric_name="quality_5_point", + score=4, + threshold=3, + passed=True, + rubrics=(RubricOutcome(id="quality", score=4, passed=True), ), + ) + assert scaled.rubrics[0].score == 4 + with pytest.raises(ValidationError, match="finite"): + RubricOutcome(id="quality", score=float("inf"), passed=True) + with pytest.raises(ValidationError, match="metric threshold"): + MetricRun( + metric_name="quality", + score=0, + threshold=0.5, + passed=False, + rubrics=(RubricOutcome(id="quality", score=1, passed=False), ), + ) + with pytest.raises(ValidationError, match="must be unknown"): + CostSummary( + sources=(CostSource(name="judge", cost_usd=None), ), + total_cost_usd=0, + ) + with pytest.raises(ValidationError, match="does not equal"): + CostSummary( + sources=(CostSource(name="judge", cost_usd=1), ), + total_cost_usd=2, + ) + + +def test_comparison_contract_rejects_contradictory_delta_and_failure_flags() -> None: + with pytest.raises(ValidationError, match="metric delta"): + MetricDelta(metric_name="quality", baseline=0, candidate=1, delta=0) + with pytest.raises(ValidationError, match="hard failure"): + CaseComparison( + case_id="critical", + baseline_passed=True, + candidate_passed=False, + baseline_score=1, + candidate_score=0, + delta=-1, + metrics=(MetricDelta(metric_name="quality", baseline=1, candidate=0, delta=-1), ), + transition=Transition.NEW_FAIL, + hard=True, + new_hard_failure=False, + ) + + +def _pipeline_imports(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + imports: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.level == 1 and node.module: + imports.add(node.module.split(".")[0]) + return imports + + +def test_pipeline_import_boundaries() -> None: + allowed = { + "artifacts": {"models", "schema"}, + "attribution": {"configuration", "models", "schema"}, + "backends": {"contracts", "models", "offline_evaluation", "schema"}, + "candidate_runtime": { + "artifacts", + "attribution", + "contracts", + "evaluation", + "models", + "prompt_workspace", + "schema", + }, + "configuration": {"models", "schema"}, + "contracts": {"models"}, + "costing": {"models"}, + "evaluation": {"models"}, + "evaluation_runtime": { + "artifacts", + "configuration", + "contracts", + "costing", + "evaluation", + "models", + }, + "gate": {"configuration", "models"}, + "models": {"schema"}, + "offline_evaluation": set(), + "orchestrator": { + "artifacts", + "attribution", + "backends", + "candidate_runtime", + "contracts", + "costing", + "evaluation", + "evaluation_runtime", + "gate", + "models", + "preflight", + "prompt_workspace", + "reporting", + "schema", + }, + "preflight": {"artifacts", "configuration", "evaluation", "prompt_workspace", "schema"}, + "prompt_workspace": {"schema"}, + "reporting": {"artifacts", "configuration", "models"}, + "schema": set(), + } + graph: dict[str, set[str]] = {} + for path in PIPELINE_DIR.glob("*.py"): + if path.name == "__init__.py": + continue + module = path.stem + graph[module] = _pipeline_imports(path) + assert set(graph) == set(allowed) + for module, dependencies in graph.items(): + unexpected = dependencies - allowed[module] + assert not unexpected, f"{module} imports disallowed pipeline modules: {sorted(unexpected)}" + + visiting: set[str] = set() + visited: set[str] = set() + + def visit(module: str) -> None: + if module in visiting: + raise AssertionError(f"pipeline import cycle involving {module}") + if module in visited: + return + visiting.add(module) + for dependency in graph.get(module, set()): + visit(dependency) + visiting.remove(module) + visited.add(module) + + for module in graph: + visit(module) + + +def test_only_normalizer_unfolds_raw_evaluate_result() -> None: + raw_result_attributes = { + "results_by_eval_set_id", + "eval_results_by_eval_id", + "overall_eval_metric_results", + "eval_metric_result_per_invocation", + } + for path in PIPELINE_DIR.glob("*.py"): + if path.stem in {"__init__", "evaluation"}: + continue + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + used = { + node.attr + for node in ast.walk(tree) if isinstance(node, ast.Attribute) and node.attr in raw_result_attributes + } + assert not used, f"{path.name} unfolds raw EvaluateResult fields: {sorted(used)}" diff --git a/tests/examples/eval_optimize_loop/test_orchestrator.py b/tests/examples/eval_optimize_loop/test_orchestrator.py new file mode 100644 index 000000000..7c49a9bf3 --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_orchestrator.py @@ -0,0 +1,550 @@ +"""End-to-end lifecycle tests for the fake orchestration path.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import shlex +import shutil +import subprocess +import time +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.pipeline.models import ( + CandidateProposal, + CandidateRound, + Decision, + OptimizationReport, + Transition, +) +from examples.optimization.eval_optimize_loop.pipeline.backends import ( + DeterministicCandidateGenerator, + FakeEvaluationBackend, +) +from examples.optimization.eval_optimize_loop.pipeline.orchestrator import run_pipeline +from examples.optimization.eval_optimize_loop.pipeline.preflight import preflight_run +from examples.optimization.eval_optimize_loop.pipeline import reporting as reporting_module + +SOURCE = Path(__file__).resolve().parents[3] / "examples" / "optimization" / "eval_optimize_loop" + + +def _example(tmp_path: Path, *, accept: bool = False, apply: bool = False) -> Path: + root = tmp_path / "example" + (root / "prompts").mkdir(parents=True) + (root / "traces").mkdir() + for name in ("optimizer.json", "train.evalset.json", "val.evalset.json"): + shutil.copyfile(SOURCE / name, root / name) + shutil.copyfile(SOURCE / "prompts" / "system.md", root / "prompts" / "system.md") + shutil.copyfile(SOURCE / "traces" / "trace_cases.json", root / "traces" / "trace_cases.json") + config = json.loads((root / "optimizer.json").read_text(encoding="utf-8")) + if accept: + config["pipeline"]["gate"]["minValidationScoreDelta"] = 0 + config["pipeline"]["gate"]["maxNewHardFailures"] = 1 + config["pipeline"]["applyCandidate"] = apply + (root / "optimizer.json").write_text(json.dumps(config), encoding="utf-8") + return root + + +@pytest.mark.asyncio +async def test_fake_reject_overfit_is_a_completed_audit_run(tmp_path) -> None: + root = _example(tmp_path) + report = await run_pipeline(str(root), run_id="reject-run") + assert report.status == Decision.REJECT + assert [case.transition for case in report.delta.validation.cases] == [ + Transition.NEW_PASS, + Transition.NEW_FAIL, + Transition.UNCHANGED, + ] + assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in report.gate_decision.reasons + assert report.source_application.applied is False + actual_prompt = (root / "prompts" / "system.md").read_text(encoding="utf-8") + expected_prompt = (SOURCE / "prompts" / "system.md").read_text(encoding="utf-8") + assert actual_prompt == expected_prompt + assert report.duration_seconds < 180 + artifact_paths = {record.path for record in report.artifacts} + assert "optimization_report.json" not in artifact_paths + assert "optimization_report.md" not in artifact_paths + manifest = json.loads((root / "artifacts" / "reject-run" / "manifest.json").read_text(encoding="utf-8")) + records = {record["path"]: record for record in manifest["files"]} + for name in ("optimization_report.json", "optimization_report.md"): + content = (root / "artifacts" / "reject-run" / name).read_bytes() + assert records[name]["sha256"] == hashlib.sha256(content).hexdigest() + + +@pytest.mark.asyncio +async def test_accept_without_apply_leaves_source_unchanged(tmp_path) -> None: + root = _example(tmp_path, accept=True) + baseline = (root / "prompts" / "system.md").read_text(encoding="utf-8") + report = await run_pipeline(str(root), run_id="accept-no-apply") + assert report.status == Decision.ACCEPT + assert report.source_application.requested is False + assert report.source_application.applied is False + assert (root / "prompts" / "system.md").read_text(encoding="utf-8") == baseline + + +@pytest.mark.asyncio +async def test_accept_apply_writes_and_verifies_candidate(tmp_path) -> None: + root = _example(tmp_path, accept=True, apply=True) + report = await run_pipeline(str(root), run_id="accept-apply") + prompt = (root / "prompts" / "system.md").read_text(encoding="utf-8") + assert report.status == Decision.ACCEPT + assert report.source_application.applied is True + assert "BEHAVIOR_PROFILE=precision-v2" in prompt + assert report.source_application.final_hashes["system"] == hashlib.sha256(prompt.encode("utf-8")).hexdigest() + + +@pytest.mark.asyncio +async def test_final_report_failure_after_apply_rolls_back(tmp_path) -> None: + root = _example(tmp_path, accept=True, apply=True) + baseline = (root / "prompts" / "system.md").read_text(encoding="utf-8") + + def fail_final_report(stage: str) -> None: + if stage == "final_report": + raise OSError("injected final report failure") + + report = await run_pipeline( + str(root), + run_id="failed-final-report", + fault_injector=fail_final_report, + ) + assert report.status == Decision.ERROR + assert report.stage == "final_report" + assert report.source_application.applied is False + assert (root / "prompts" / "system.md").read_text(encoding="utf-8") == baseline + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failed_stage", + ( + "audit_config", + "baseline_train", + "baseline_validation", + "inner_split", + "candidate_generation", + "candidate_train", + "candidate_validation", + "comparison", + "gate", + "pre_apply_report", + "apply", + "final_report", + ), +) +async def test_every_orchestration_stage_fails_to_an_error_audit_and_restores_prompt( + tmp_path, + failed_stage, +) -> None: + root = _example(tmp_path, accept=True, apply=True) + baseline = (root / "prompts" / "system.md").read_text(encoding="utf-8") + + def inject(stage: str) -> None: + if stage == failed_stage: + raise RuntimeError(f"injected {stage} failure") + + report = await run_pipeline( + str(root), + run_id=f"stage-{failed_stage}", + fault_injector=inject, + ) + + assert report.status == Decision.ERROR + assert report.stage == failed_stage + assert report.errors[0].stage == failed_stage + assert report.source_application.applied is False + assert (root / "prompts" / "system.md").read_text(encoding="utf-8") == baseline + persisted = OptimizationReport.model_validate_json( + (root / "artifacts" / f"stage-{failed_stage}" / "optimization_report.json").read_text(encoding="utf-8")) + assert persisted.status == Decision.ERROR + + +@pytest.mark.asyncio +async def test_run_directory_is_never_reused(tmp_path) -> None: + root = _example(tmp_path) + await run_pipeline(str(root), run_id="immutable") + report_path = root / "artifacts" / "immutable" / "optimization_report.json" + before = report_path.read_bytes() + with pytest.raises(FileExistsError): + await run_pipeline(str(root), run_id="immutable") + assert report_path.read_bytes() == before + + +@pytest.mark.asyncio +async def test_trace_drift_fails_before_artifact_creation(tmp_path) -> None: + root = _example(tmp_path) + fixture_path = root / "traces" / "trace_cases.json" + fixture = json.loads(fixture_path.read_text(encoding="utf-8")) + fixture["datasetHashes"]["train"] = "0" * 64 + fixture_path.write_text(json.dumps(fixture), encoding="utf-8") + with pytest.raises(ValueError, match="hashes"): + await run_pipeline(str(root), mode="trace", run_id="trace-drift") + assert not (root / "artifacts").exists() + + +@pytest.mark.asyncio +async def test_source_trace_fixture_completes_offline_pipeline(tmp_path) -> None: + root = _example(tmp_path) + + report = await run_pipeline( + str(root), + mode="trace", + run_id="source-trace-complete", + ) + + assert report.status == Decision.REJECT + assert report.stage == "complete" + assert report.duration_seconds < 180 + + +@pytest.mark.asyncio +async def test_cancellation_restores_prompt_and_propagates(tmp_path) -> None: + root = _example(tmp_path) + baseline = (root / "prompts" / "system.md").read_text(encoding="utf-8") + + def cancel_candidate_validation(stage: str) -> None: + if stage == "candidate_validation": + raise asyncio.CancelledError() + + with pytest.raises(asyncio.CancelledError): + await run_pipeline( + str(root), + run_id="cancelled", + fault_injector=cancel_candidate_validation, + ) + assert (root / "prompts" / "system.md").read_text(encoding="utf-8") == baseline + report = json.loads((root / "artifacts" / "cancelled" / "optimization_report.json").read_text(encoding="utf-8")) + assert report["status"] == "ERROR" + assert report["stage"] == "candidate_validation" + assert report["candidate"]["train"] is not None + assert report["candidate"]["validation"] is None + + +@pytest.mark.asyncio +async def test_error_report_preserves_completed_baseline_train(tmp_path) -> None: + root = _example(tmp_path) + + def fail_baseline_validation(stage: str) -> None: + if stage == "baseline_validation": + raise RuntimeError("validation unavailable") + + report = await run_pipeline( + str(root), + run_id="partial-baseline", + fault_injector=fail_baseline_validation, + ) + assert report.status == Decision.ERROR + assert report.baseline is not None + assert report.baseline.train is not None + assert report.baseline.validation is None + assert report.failure_attribution is not None + assert report.failure_attribution.train is not None + assert report.failure_attribution.validation is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failure_call", "failed_stage", "candidate_completed"), + ((2, "baseline_validation", False), (4, "candidate_validation", True)), +) +async def test_backend_failure_preserves_partial_facts_and_unknown_failed_call_cost( + tmp_path, + failure_call, + failed_stage, + candidate_completed, +) -> None: + root = _example(tmp_path) + delegate = FakeEvaluationBackend() + + class FailingBackend: + + def __init__(self) -> None: + self.calls = 0 + + async def evaluate(self, **kwargs): + self.calls += 1 + if self.calls == failure_call: + raise RuntimeError("backend unavailable") + return await delegate.evaluate(**kwargs) + + report = await run_pipeline( + str(root), + run_id=f"backend-failure-{failure_call}", + backend=FailingBackend(), + ) + + assert report.status == Decision.ERROR + assert report.stage == failed_stage + failed_source = next(source for source in report.cost.sources if source.name == failed_stage) + assert failed_source.cost_usd is None + assert failed_source.model_calls is None + assert failed_source.metric_calls is None + if candidate_completed: + assert report.candidate.train is not None + assert report.candidate.validation is None + assert report.candidate_failure_attribution.train is not None + assert report.candidate_failure_attribution.validation is None + else: + assert report.baseline.train is not None + assert report.baseline.validation is None + assert report.failure_attribution.train is not None + assert report.failure_attribution.validation is None + + +@pytest.mark.asyncio +async def test_generator_failure_is_recorded_as_unknown_cost_and_redacted(tmp_path) -> None: + root = _example(tmp_path) + + class FailingGenerator: + + async def generate(self, **kwargs): + raise RuntimeError('optimizer failed with {"apiKey":"candidate-secret"}') + + report = await run_pipeline( + str(root), + run_id="generator-failure-cost", + candidate_generator=FailingGenerator(), + ) + + assert report.status == Decision.ERROR + assert report.stage == "candidate_generation" + source = next(item for item in report.cost.sources if item.name == "candidate_generation.unreported_failure") + assert source.cost_usd is None + assert source.model_calls is None + assert source.metric_calls is None + assert "candidate-secret" not in report.errors[0].message + assert "[REDACTED]" in report.errors[0].message + + +@pytest.mark.asyncio +async def test_generator_receives_only_inner_train_attribution(tmp_path) -> None: + root = _example(tmp_path) + + class CapturingGenerator: + + def __init__(self) -> None: + self.checked = False + + async def generate(self, **kwargs): + inner = json.loads(Path(kwargs["inner_train_path"]).read_text(encoding="utf-8")) + inner_ids = {case["evalId"] for case in inner["evalCases"]} + failure_ids = {failure.case_id for failure in kwargs["train_attribution"].failures} + assert failure_ids <= inner_ids + output = Path(kwargs["output_dir"]) + output.mkdir(parents=True) + (output / "config.snapshot.json").write_text( + json.dumps({"apiKey": "optimizer-secret"}), + encoding="utf-8", + ) + self.checked = True + return await DeterministicCandidateGenerator().generate(**kwargs) + + generator = CapturingGenerator() + report = await run_pipeline( + str(root), + run_id="inner-attribution", + candidate_generator=generator, + ) + assert report.status != Decision.ERROR + assert generator.checked is True + audit_text = "\n".join( + path.read_text(encoding="utf-8") for path in (root / "artifacts" / "inner-attribution").rglob("*") + if path.is_file()) + assert "optimizer-secret" not in audit_text + assert "[REDACTED]" in audit_text + assert not list((root / "artifacts" / "inner-attribution").rglob("prompt_sandbox")) + + +@pytest.mark.asyncio +async def test_custom_backend_is_isolated_and_has_unknown_accounting(tmp_path) -> None: + root = _example(tmp_path) + delegate = FakeEvaluationBackend() + + class MutatingBackend: + + async def evaluate(self, **kwargs): + raw = await delegate.evaluate(**kwargs) + kwargs["eval_set"].eval_cases.clear() + kwargs["eval_config"].num_runs = 9 + kwargs["prompts"].clear() + return raw + + report = await run_pipeline( + str(root), + run_id="isolated-custom-backend", + backend=MutatingBackend(), + ) + assert report.status != Decision.ERROR + evaluation_sources = [ + source for source in report.cost.sources if not source.name.startswith("candidate_generation.") + ] + assert len(evaluation_sources) == 4 + assert all(source.cost_usd is None for source in evaluation_sources) + assert all(source.model_calls is None for source in evaluation_sources) + assert report.baseline.train.case_ids == ( + "train_improve_alpha", + "train_improve_beta", + "train_stable", + ) + + +@pytest.mark.asyncio +async def test_custom_generator_unknown_cost_fails_closed_when_budget_enabled(tmp_path) -> None: + root = _example(tmp_path) + config_path = root / "optimizer.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config["pipeline"]["gate"]["maxCostUsd"] = 1 + config_path.write_text(json.dumps(config), encoding="utf-8") + + class UnmeteredGenerator: + + async def generate(self, **kwargs): + known = await DeterministicCandidateGenerator().generate(**kwargs) + payload = known.model_dump(mode="python") + payload.pop("cost_sources") + return CandidateProposal.model_validate(payload) + + report = await run_pipeline( + str(root), + run_id="unknown-candidate-cost", + candidate_generator=UnmeteredGenerator(), + ) + source = next(item for item in report.cost.sources if item.name == "candidate_generation.unreported") + assert source.cost_usd is None + assert source.model_calls is None + assert report.status == Decision.REJECT + assert "COST_UNAVAILABLE" in report.gate_decision.reasons + + +def test_reproducibility_command_contains_all_effective_cli_inputs(tmp_path, monkeypatch) -> None: + repo = tmp_path / "repo with space" + root = repo / "examples" / "optimization" / "eval_optimize_loop" + root.mkdir(parents=True) + + def fake_run(args, **kwargs): + if args[-1] == "HEAD": + stdout = "a" * 40 + "\n" + elif args[-1] == "--porcelain": + stdout = "" + elif args[-1] == "--show-toplevel": + stdout = str(repo) + "\n" + else: + raise AssertionError(args) + return subprocess.CompletedProcess(args, 0, stdout=stdout, stderr="") + + monkeypatch.setattr(reporting_module.subprocess, "run", fake_run) + result = reporting_module.build_reproducibility( + str(root), + mode="live", + config_path=str(root / "custom config.json"), + train_path=str(root / "custom train.json"), + validation_path=str(root / "custom validation.json"), + run_id="exact-replay", + apply_candidate=True, + callback_spec="package.agent:call_agent", + ) + args = shlex.split(result.command) + assert result.reproducible is True + assert args[1] == "examples/optimization/eval_optimize_loop/run_pipeline.py" + assert args[args.index("--config") + 1].endswith("custom config.json") + assert args[args.index("--train") + 1].endswith("custom train.json") + assert args[args.index("--validation") + 1].endswith("custom validation.json") + assert args[args.index("--run-id") + 1] == "exact-replay-replay" + assert "--apply-candidate" in args + assert args[args.index("--call-agent") + 1] == "package.agent:call_agent" + + +@pytest.mark.asyncio +async def test_generator_baseline_drift_is_rejected_at_stage_boundary(tmp_path) -> None: + root = _example(tmp_path) + + class DriftingGenerator: + + async def generate(self, **kwargs): + candidate = {"system": "candidate"} + return CandidateProposal( + algorithm="drifting-test-generator", + baseline_prompts={"system": "stale baseline"}, + prompts=candidate, + changed=True, + rounds=(CandidateRound( + round=1, + candidate_prompts=candidate, + accepted=True, + score=1, + ), ), + ) + + report = await run_pipeline( + str(root), + run_id="candidate-baseline-drift", + candidate_generator=DriftingGenerator(), + ) + assert report.status == Decision.ERROR + assert report.stage == "candidate_generation" + assert "baseline prompts differ" in report.errors[0].message + + +def test_default_run_id_includes_effective_apply_setting(tmp_path) -> None: + root = _example(tmp_path) + common = { + "config_path": None, + "train_path": None, + "validation_path": None, + "mode": None, + "run_id": None, + "call_agent": None, + "callback_spec": None, + "backend": None, + "candidate_generator": None, + } + dry_run = preflight_run(str(root), apply_candidate=False, **common) + applying_run = preflight_run(str(root), apply_candidate=True, **common) + assert dry_run.run_id != applying_run.run_id + + async def call_agent(query: str) -> str: + return query + + first_callback = preflight_run( + str(root), + apply_candidate=False, + **{ + **common, + "mode": "live", + "call_agent": call_agent, + "callback_spec": "package.first:call_agent", + }, + ) + second_callback = preflight_run( + str(root), + apply_candidate=False, + **{ + **common, + "mode": "live", + "call_agent": call_agent, + "callback_spec": "package.second:call_agent", + }, + ) + assert first_callback.run_id != second_callback.run_id + + +@pytest.mark.asyncio +async def test_final_duration_includes_terminal_report_write(tmp_path, monkeypatch) -> None: + root = _example(tmp_path) + original = reporting_module.persist_report + delayed = False + + def persist_with_delay(sink, report): + nonlocal delayed + if report.stage == "complete" and not delayed: + delayed = True + time.sleep(0.05) + return original(sink, report) + + monkeypatch.setattr(reporting_module, "persist_report", persist_with_delay) + report = await run_pipeline(str(root), run_id="duration-includes-report") + assert delayed is True + assert report.duration_seconds >= 0.05 + assert report.inputs["environment"]["sdk"] diff --git a/tests/examples/eval_optimize_loop/test_workspace_artifacts.py b/tests/examples/eval_optimize_loop/test_workspace_artifacts.py new file mode 100644 index 000000000..5457bfa8d --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_workspace_artifacts.py @@ -0,0 +1,207 @@ +"""Prompt lifecycle, audit persistence, sanitization and cost tests.""" + +from __future__ import annotations + +import hashlib +import json +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from trpc_agent_sdk.evaluation import TargetPrompt + +from examples.optimization.eval_optimize_loop.pipeline.artifacts import AuditSink +from examples.optimization.eval_optimize_loop.pipeline.costing import CostLedger +from examples.optimization.eval_optimize_loop.pipeline.prompt_workspace import ( + PromptRestoreError, + PromptWorkspace, +) + + +@pytest.mark.asyncio +async def test_temporary_candidate_always_restores_baseline(tmp_path) -> None: + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + workspace = PromptWorkspace(TargetPrompt().add_path("system", str(prompt_path))) + await workspace.initialize() + with pytest.raises(RuntimeError, match="candidate failed"): + async with workspace.temporary({"system": "candidate"}): + assert prompt_path.read_text(encoding="utf-8") == "candidate" + raise RuntimeError("candidate failed") + assert prompt_path.read_text(encoding="utf-8") == "baseline" + assert await workspace.current_hashes() == workspace.baseline_hashes + + +@pytest.mark.asyncio +async def test_apply_is_verified_and_can_be_rolled_back(tmp_path) -> None: + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + workspace = PromptWorkspace(TargetPrompt().add_path("system", str(prompt_path))) + await workspace.initialize() + hashes = await workspace.apply({"system": "candidate"}) + assert hashes["system"] == hashlib.sha256(b"candidate").hexdigest() + await workspace.restore() + assert prompt_path.read_text(encoding="utf-8") == "baseline" + + +@pytest.mark.asyncio +async def test_restoration_failure_is_not_downgraded_to_reject() -> None: + state = {"value": "baseline", "fail_restore": False} + + async def read() -> str: + return state["value"] + + async def write(value: str) -> None: + if value == "baseline" and state["fail_restore"]: + raise OSError("restore unavailable") + state["value"] = value + + workspace = PromptWorkspace(TargetPrompt().add_callback("system", read=read, write=write)) + await workspace.initialize() + with pytest.raises(PromptRestoreError): + async with workspace.temporary({"system": "candidate"}): + state["fail_restore"] = True + + +def test_audit_sink_is_immutable_safe_sanitized_and_manifested(tmp_path) -> None: + sink = AuditSink(tmp_path / "artifacts", "run-1", max_text_chars=64) + sink.create() + sink.write_json( + "config.json", + { + "api_key": "secret-value", + "nested": { + "message": "Authorization: Bearer abc.def" + }, + "tokenUsage": { + "total": 12 + }, + }, + ) + sink.write_text("notes.txt", "token=abc123") + with pytest.raises(ValueError, match="unsafe|path"): + sink.write_text("../escape.txt", "bad") + records = sink.write_manifest() + payload = json.loads((sink.run_dir / "config.json").read_text(encoding="utf-8")) + assert payload["api_key"] == "[REDACTED]" + assert "abc.def" not in payload["nested"]["message"] + assert payload["tokenUsage"] == {"total": 12} + assert {record.path for record in records} == {"config.json", "notes.txt"} + for record in records: + content = (sink.run_dir / record.path).read_bytes() + assert record.sha256 == hashlib.sha256(content).hexdigest() + assert record.byte_size == len(content) + with pytest.raises(FileExistsError): + AuditSink(tmp_path / "artifacts", "run-1", max_text_chars=64).create() + + +def test_optimizer_tree_is_sanitized_before_audit_import(tmp_path) -> None: + raw = tmp_path / "raw-optimizer" + raw.mkdir() + (raw / "config.snapshot.json").write_text( + json.dumps({"optimize": { + "apiKey": "live-secret" + }}), + encoding="utf-8", + ) + (raw / "run.log").write_text( + 'Authorization: Bearer live-token\n{"apiKey":"quoted-log-secret"}', + encoding="utf-8", + ) + (raw / "summary.md").write_text( + "result: {'clientSecret': 'markdown-secret'}", + encoding="utf-8", + ) + sink = AuditSink(tmp_path / "artifacts", "run-2", max_text_chars=128) + sink.create() + sink.import_tree(raw, "candidate_generation/optimizer") + imported = "\n".join(path.read_text(encoding="utf-8") for path in sink.run_dir.rglob("*") if path.is_file()) + assert "live-secret" not in imported + assert "live-token" not in imported + assert "quoted-log-secret" not in imported + assert "markdown-secret" not in imported + assert "[REDACTED]" in imported + + +@pytest.mark.parametrize( + ("sink_kwargs", "files", "message"), + [ + ({ + "max_import_files": 1 + }, { + "a.log": "a", + "b.log": "b" + }, "count"), + ({ + "max_import_file_bytes": 3 + }, { + "large.log": "1234" + }, "per-file"), + ( + { + "max_import_file_bytes": 4, + "max_import_total_bytes": 5 + }, + { + "a.log": "123", + "b.log": "456" + }, + "total", + ), + ], +) +def test_optimizer_tree_enforces_resource_budgets(tmp_path, sink_kwargs, files, message) -> None: + raw = tmp_path / "raw-budget" + raw.mkdir() + for name, content in files.items(): + (raw / name).write_text(content, encoding="utf-8") + sink = AuditSink( + tmp_path / "artifacts", + "run-budget", + max_text_chars=128, + **sink_kwargs, + ) + sink.create() + with pytest.raises(ValueError, match=message): + sink.import_tree(raw, "candidate_generation/optimizer") + + +def test_concurrent_latest_publication_uses_collision_free_atomic_files(tmp_path) -> None: + publication_root = tmp_path / "publication" + sinks = [] + for index in range(2): + sink = AuditSink( + tmp_path / "artifacts", + f"run-{index}", + max_text_chars=128, + publication_root=publication_root, + ) + sink.create() + sink.write_text("report.md", f"run-{index}\n") + sinks.append(sink) + + with ThreadPoolExecutor(max_workers=8) as executor: + futures = [ + executor.submit( + sinks[index % 2].publish_latest_snapshot, + "report.md", + "optimization_report.md", + ) for index in range(40) + ] + for future in futures: + future.result() + + assert (publication_root / "optimization_report.md").read_text(encoding="utf-8") in { + "run-0\n", + "run-1\n", + } + assert not list(publication_root.glob("*.tmp")) + + +def test_cost_ledger_preserves_unknown_sources() -> None: + ledger = CostLedger() + ledger.record("fake", cost_usd=0, metric_calls=3) + ledger.record("live-judge", cost_usd=None, model_calls=1) + summary = ledger.summary() + assert summary.total_cost_usd is None + assert [source.name for source in summary.sources] == ["fake", "live-judge"] From ceb6ff81a7a499689ddef85f0d0c997d2e54a505 Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Thu, 30 Jul 2026 23:00:43 +0800 Subject: [PATCH 02/13] fix: align issue 91 pipeline with evaluator contracts --- .../optimization/eval_optimize_loop/DESIGN.md | 13 +++-- .../optimization/eval_optimize_loop/README.md | 4 +- .../eval_optimize_loop/SOLUTION.md | 7 +++ .../pipeline/attribution.py | 22 +++++++- .../eval_optimize_loop/pipeline/backends.py | 28 ++++++++-- .../eval_optimize_loop/pipeline/models.py | 27 ++++++++++ .../test_attribution_gate.py | 36 ++++++++++++- .../eval_optimize_loop/test_backends.py | 53 ++++++++++++++++--- .../eval_optimize_loop/test_deliverables.py | 32 +++++++++++ 9 files changed, 202 insertions(+), 20 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/SOLUTION.md create mode 100644 tests/examples/eval_optimize_loop/test_deliverables.py diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index 41008b7ba..7bf68d770 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -34,11 +34,14 @@ Only `EvaluationBackend` and `CandidateGenerator` are protocols because each has real fake, trace, and live substitutions. Attribution, comparison, normalization, and gate logic are ordinary pure functions. -The example does not extend or patch the SDK. `backends.py` composes the exported -`LocalEvalService`, `RemoteEvalService`, and `InMemoryEvalSetsManager` APIs and -rebuilds the exported `EvaluateResult` aggregate at the example boundary. A -run-local `EvaluatorRegistry` lets fake and trace modes replace LLM judges without -mutating global registry state. The SDK remains the owner of remote metric +The example does not extend or patch the SDK. Standard fake, trace, and live +evaluation calls `AgentEvaluator.evaluate_eval_set` with validated in-memory +`EvalSet` and `EvalConfig` objects. Only deterministic offline LLM metrics need a +run-local `EvaluatorRegistry`, which `AgentEvaluator` cannot receive per call; for +that narrow path `backends.py` composes the exported `LocalEvalService`, +`RemoteEvalService`, and `InMemoryEvalSetsManager` APIs and rebuilds the exported +`EvaluateResult` aggregate at the example boundary. The registry never mutates +global state. The SDK remains the owner of remote metric compatibility: black-box evaluation rejects trajectory and knowledge-recall metrics because callback results do not expose their required intermediate data. diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index d97ade400..f61501fcf 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -56,4 +56,6 @@ cancellation is cooperative. A production deployment that requires a hard deadline must run the live optimizer in a supervised worker process or container and terminate that worker at the platform boundary. -See [DESIGN.md](DESIGN.md) for stage contracts and failure semantics. +See [SOLUTION.md](SOLUTION.md) for the concise Chinese issue proposal and +[DESIGN.md](DESIGN.md) for detailed stage contracts and failure semantics. A +versioned example report is available under [sample_output](sample_output/). diff --git a/examples/optimization/eval_optimize_loop/SOLUTION.md b/examples/optimization/eval_optimize_loop/SOLUTION.md new file mode 100644 index 000000000..93fd31023 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/SOLUTION.md @@ -0,0 +1,7 @@ +# 方案设计说明 + +本方案把评测、归因、优化、回归和审计组织为单向流水线。基线阶段使用 AgentEvaluator 分别运行训练集与验证集，保留每个用例、每次运行和每项指标的分数、阈值、原因及关键轨迹。失败归因先比较结构化工具名与参数，再读取失败的指标和 rubric，最后才使用文本兜底；分类有固定优先级，每条失败均输出主类别、证据和置信度，并由模型校验分类统计与逐条事实一致。 + +候选生成只接收训练集内部切分的失败信息，不读取外部验证集。候选完成后重新评测完整训练集和保留验证集，逐用例标记新增通过、新增失败、提升、下降或不变。接受门禁按固定顺序检查验证总分与通过率阈值、新增 hard fail、关键用例退化、单项指标回退、成本和耗时；训练提升但验证下降会被明确识别为过拟合并拒绝，未知成本在启用预算时也按失败处理。 + +所有输入在运行前严格解析并哈希，随机种子、适配器身份、候选轮次、成本来源、耗时和门禁理由写入不可变运行目录。prompt 默认不回写，只有门禁接受且显式授权时才原子应用；异常或取消会恢复并校验基线。fake 与 trace 模式使用确定性替代规则，不读取真实密钥，最终同时生成机器可读 JSON、人可读 Markdown 和带哈希清单的审计产物，便于复现、比较和追责。 diff --git a/examples/optimization/eval_optimize_loop/pipeline/attribution.py b/examples/optimization/eval_optimize_loop/pipeline/attribution.py index 1f85f3283..d2e03b87f 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/attribution.py +++ b/examples/optimization/eval_optimize_loop/pipeline/attribution.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections import Counter from typing import Any from trpc_agent_sdk.evaluation import Invocation, get_all_tool_calls @@ -10,6 +11,7 @@ from .models import ( AttributionPair, AttributionSnapshot, + AttributionStatistics, EvaluationSnapshot, FailureAttribution, FailureCategory, @@ -29,6 +31,14 @@ } +def _statistics(failures: tuple[FailureAttribution, ...]) -> AttributionStatistics: + counts = Counter(failure.primary for failure in failures) + return AttributionStatistics( + total_failures=len(failures), + primary_category_counts=dict(counts), + ) + + def _safe_text(value: Any, max_chars: int) -> str: return sanitized_text(value, max_text_chars=max_chars) @@ -169,7 +179,13 @@ def attribute_failures( evidence=tuple(dict.fromkeys(evidence or ["No structured evidence was available."])), confidence=confidence, )) - return AttributionSnapshot(split=snapshot.split, phase=snapshot.phase, failures=tuple(failures)) + attributed = tuple(failures) + return AttributionSnapshot( + split=snapshot.split, + phase=snapshot.phase, + failures=attributed, + statistics=_statistics(attributed), + ) def attribute_pair( @@ -202,8 +218,10 @@ def select_attribution( ) -> AttributionSnapshot: """Restrict attribution facts to the cases visible to a downstream stage.""" + selected = tuple(failure for failure in snapshot.failures if failure.case_id in case_ids) return AttributionSnapshot( split=snapshot.split, phase=snapshot.phase, - failures=tuple(failure for failure in snapshot.failures if failure.case_id in case_ids), + failures=selected, + statistics=_statistics(selected), ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/backends.py b/examples/optimization/eval_optimize_loop/pipeline/backends.py index c8e880836..249a22f2d 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/backends.py +++ b/examples/optimization/eval_optimize_loop/pipeline/backends.py @@ -10,6 +10,7 @@ from typing import Awaitable, Callable, Optional from trpc_agent_sdk.evaluation import ( + AgentEvaluator, AgentOptimizer, EvalConfig, EvalSet, @@ -42,7 +43,7 @@ _DEFAULT_APP_NAME = "test_app" -async def _evaluate_with_service( +async def _evaluate_with_sdk( eval_set: EvalSet, eval_config: EvalConfig, *, @@ -50,7 +51,7 @@ async def _evaluate_with_service( runtime_dir: str, evaluator_registry: Optional[EvaluatorRegistry] = None, ) -> EvaluateResult: - """Compose public eval services and preserve the SDK aggregate contract.""" + """Use AgentEvaluator by default and services only for a run-local registry.""" dataset = deepcopy(eval_set) config = deepcopy(eval_config) @@ -61,6 +62,25 @@ async def _evaluate_with_service( raise ValueError("evaluation num_runs must be at least one") app_name = dataset.app_name or _DEFAULT_APP_NAME + if evaluator_registry is None: + _, _, _, results_by_case = await AgentEvaluator.evaluate_eval_set( + dataset, + call_agent=call_agent, + eval_config=config, + num_runs=config.num_runs, + print_detailed_results=False, + ) + return EvaluateResult( + results_by_eval_set_id={ + dataset.eval_set_id: + EvalSetAggregateResult( + eval_results_by_eval_id=results_by_case, + num_runs=config.num_runs, + ) + }) + + # AgentEvaluator has no per-call registry hook. Keep custom offline judges + # scoped to this run by composing the exported services directly. manager = InMemoryEvalSetsManager() manager.create_eval_set(app_name=app_name, eval_set_id=dataset.eval_set_id) for eval_case in dataset.eval_cases: @@ -121,7 +141,7 @@ async def _evaluate_offline( """Evaluate using a run-local deterministic replacement registry.""" offline_config, registry = prepare_offline_evaluation(eval_config) - return await _evaluate_with_service( + return await _evaluate_with_sdk( eval_set, offline_config, call_agent=call_agent, @@ -270,7 +290,7 @@ async def evaluate( audit_dir: str, ) -> EvaluateResult: del prompts, split, phase - return await _evaluate_with_service( + return await _evaluate_with_sdk( eval_set, eval_config, call_agent=self._call_agent, diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py index 58f7a1cc9..08e3ec1cd 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/models.py +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -230,10 +230,37 @@ def explanatory(self) -> "FailureAttribution": return self +class AttributionStatistics(StrictModel): + total_failures: int = Field(default=0, ge=0) + primary_category_counts: dict[FailureCategory, int] = Field(default_factory=dict) + + @model_validator(mode="after") + def consistent_totals(self) -> "AttributionStatistics": + if any(count < 1 for count in self.primary_category_counts.values()): + raise ValueError("attribution category counts must be positive") + if self.total_failures != sum(self.primary_category_counts.values()): + raise ValueError("attribution total must equal primary category counts") + return self + + class AttributionSnapshot(StrictModel): split: Split phase: Phase failures: tuple[FailureAttribution, ...] + statistics: AttributionStatistics = Field(default_factory=AttributionStatistics) + + @model_validator(mode="after") + def statistics_match_failures(self) -> "AttributionSnapshot": + expected = { + category: sum(failure.primary == category for failure in self.failures) + for category in FailureCategory + } + expected = {category: count for category, count in expected.items() if count} + if self.statistics.total_failures != len(self.failures): + raise ValueError("attribution statistics total does not match failures") + if self.statistics.primary_category_counts != expected: + raise ValueError("attribution category statistics do not match failures") + return self class InnerSplit(StrictModel): diff --git a/tests/examples/eval_optimize_loop/test_attribution_gate.py b/tests/examples/eval_optimize_loop/test_attribution_gate.py index 42a9ebb4c..cebb63d02 100644 --- a/tests/examples/eval_optimize_loop/test_attribution_gate.py +++ b/tests/examples/eval_optimize_loop/test_attribution_gate.py @@ -3,8 +3,12 @@ from __future__ import annotations import pytest +from pydantic import ValidationError -from examples.optimization.eval_optimize_loop.pipeline.attribution import attribute_failures +from examples.optimization.eval_optimize_loop.pipeline.attribution import ( + attribute_failures, + select_attribution, +) from examples.optimization.eval_optimize_loop.pipeline.configuration import ( AttributionConfig, GateConfig, @@ -21,6 +25,8 @@ CostSummary, Decision, EvaluationSnapshot, + AttributionSnapshot, + AttributionStatistics, FailureCategory, MetricDelta, MetricRun, @@ -149,6 +155,34 @@ def test_attribution_taxonomy(snapshot, config, expected) -> None: assert result.failures[0].primary == expected assert result.failures[0].reasons assert result.failures[0].evidence + assert result.statistics.total_failures == 1 + assert result.statistics.primary_category_counts == {expected: 1} + + +def test_attribution_statistics_cannot_diverge_from_failure_facts() -> None: + result = attribute_failures( + _failed_snapshot("final_response_avg_score"), + AttributionConfig(), + max_text_chars=200, + ) + with pytest.raises(ValidationError, match="statistics total"): + AttributionSnapshot( + split=result.split, + phase=result.phase, + failures=result.failures, + statistics=AttributionStatistics(), + ) + + +def test_select_attribution_recalculates_statistics() -> None: + result = attribute_failures( + _failed_snapshot("final_response_avg_score"), + AttributionConfig(), + max_text_chars=200, + ) + selected = select_attribution(result, set()) + assert selected.failures == () + assert selected.statistics == AttributionStatistics() def test_attribution_policy_matrix_matches_every_expected_category() -> None: diff --git a/tests/examples/eval_optimize_loop/test_backends.py b/tests/examples/eval_optimize_loop/test_backends.py index c26430d86..06c7d23a3 100644 --- a/tests/examples/eval_optimize_loop/test_backends.py +++ b/tests/examples/eval_optimize_loop/test_backends.py @@ -81,11 +81,15 @@ async def test_fake_backend_uses_public_services_and_does_not_mutate_inputs(tmp_ config = _config() before_set = eval_set.model_dump(mode="json") before_config = config.model_dump(mode="json") - monkeypatch.setattr( - AgentEvaluator, - "get_executer", - lambda *args, **kwargs: pytest.fail("backend must compose public EvalService APIs directly"), - ) + calls = 0 + original = AgentEvaluator.evaluate_eval_set + + async def recording_evaluate_eval_set(*args, **kwargs): + nonlocal calls + calls += 1 + return await original(*args, **kwargs) + + monkeypatch.setattr(AgentEvaluator, "evaluate_eval_set", recording_evaluate_eval_set) raw = await backend.evaluate( eval_set=eval_set, eval_config=config, @@ -102,6 +106,7 @@ async def test_fake_backend_uses_public_services_and_does_not_mutate_inputs(tmp_ phase=Phase.BASELINE, ) assert snapshot.pass_rate == 1 + assert calls == 1 assert eval_set.model_dump(mode="json") == before_set assert config.model_dump(mode="json") == before_config @@ -114,8 +119,8 @@ def forbid_real_judge(*args, **kwargs): monkeypatch.setattr( AgentEvaluator, - "get_executer", - lambda *args, **kwargs: pytest.fail("backend must not depend on AgentEvaluator file orchestration"), + "evaluate_eval_set", + lambda *args, **kwargs: pytest.fail("custom registry must remain run-local"), ) monkeypatch.setattr( "trpc_agent_sdk.evaluation._llm_judge.LLMJudge.__init__", @@ -556,6 +561,40 @@ def test_live_backend_requires_async_callback() -> None: LiveEvaluationBackend(lambda query: query) # type: ignore[arg-type] +@pytest.mark.asyncio +async def test_live_backend_uses_agent_evaluator_entrypoint(tmp_path, monkeypatch) -> None: + calls = 0 + original = AgentEvaluator.evaluate_eval_set + + async def recording_evaluate_eval_set(*args, **kwargs): + nonlocal calls + calls += 1 + return await original(*args, **kwargs) + + async def call_agent(query: str) -> str: + del query + return "OK" + + monkeypatch.setattr(AgentEvaluator, "evaluate_eval_set", recording_evaluate_eval_set) + raw = await LiveEvaluationBackend(call_agent).evaluate( + eval_set=_eval_set(), + eval_config=_config(), + prompts={"system": "baseline"}, + split=Split.VALIDATION, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + _eval_set(), + _config(), + split=Split.VALIDATION, + phase=Phase.BASELINE, + ) + assert snapshot.pass_rate == 1 + assert calls == 1 + + @pytest.mark.asyncio async def test_deterministic_generator_only_uses_failure_facts(tmp_path) -> None: prompt_path = tmp_path / "system.md" diff --git a/tests/examples/eval_optimize_loop/test_deliverables.py b/tests/examples/eval_optimize_loop/test_deliverables.py new file mode 100644 index 000000000..3d9f7e5a1 --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_deliverables.py @@ -0,0 +1,32 @@ +"""Issue-level deliverable contract tests for the public example.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +EXAMPLE_ROOT = (Path(__file__).resolve().parents[3] / "examples" / "optimization" / "eval_optimize_loop") + + +def _dataset(name: str) -> dict: + return json.loads((EXAMPLE_ROOT / name).read_text(encoding="utf-8")) + + +def test_public_datasets_cover_required_case_matrix() -> None: + train = _dataset("train.evalset.json")["evalCases"] + validation = _dataset("val.evalset.json")["evalCases"] + assert len(train) == 3 + assert len(validation) == 3 + assert {case["evalId"] for case in train}.isdisjoint(case["evalId"] for case in validation) + queries = [case["conversation"][0]["userContent"]["parts"][0]["text"] for case in (*train, *validation)] + assert all( + any(f"BEHAVIOR:{behavior}" in query for query in queries) for behavior in ("improve", "stable", "regress")) + + +def test_solution_is_300_to_500_chinese_characters_and_covers_required_topics() -> None: + solution = (EXAMPLE_ROOT / "SOLUTION.md").read_text(encoding="utf-8") + chinese_characters = re.findall(r"[\u4e00-\u9fff]", solution) + assert 300 <= len(chinese_characters) <= 500 + for topic in ("失败归因", "接受门禁", "过拟合", "审计"): + assert topic in solution From ae69b6311ed4106dd6a9dcd15240f06f0de0c4f4 Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Thu, 30 Jul 2026 23:32:50 +0800 Subject: [PATCH 03/13] docs: add reproducible issue 91 sample report --- .../sample_output/optimization_report.json | 2309 +++++++++++++++++ .../sample_output/optimization_report.md | 32 + .../eval_optimize_loop/test_deliverables.py | 48 + 3 files changed, 2389 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.md diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json new file mode 100644 index 000000000..3becb7ccf --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -0,0 +1,2309 @@ +{ + "artifacts": [ + { + "byteSize": 101, + "path": "baseline_prompts/system.md", + "sha256": "aacc5210d0fe4482263f69f5f83241cdf9252e44be22a70de08107daec05e639" + }, + { + "byteSize": 13921, + "path": "baseline_train/snapshot.json", + "sha256": "e3e4fa35e990241618df6d6151a1e8fc6e24ed1c98f5d19c349980964a489caf" + }, + { + "byteSize": 13955, + "path": "baseline_validation/snapshot.json", + "sha256": "6c22f2743db51fe0b604389cb3585ef8850f96c5ac40311a48a2198f77a4abfd" + }, + { + "byteSize": 1230, + "path": "candidate.json", + "sha256": "09d48df58df5b954757a9963665f61b36d981cda9c7a555b270e2d4e69dfdf53" + }, + { + "byteSize": 132, + "path": "candidate_prompts/system.md", + "sha256": "ed2dfc103a7763a126f690e536572c5d06d20e2119ef7ef707c0a72deb162be2" + }, + { + "byteSize": 13857, + "path": "candidate_train/snapshot.json", + "sha256": "81619a1694caffa8e5661b53a6838c2347e7fc5bd4947f9c0dc17ba1fe894812" + }, + { + "byteSize": 13956, + "path": "candidate_validation/snapshot.json", + "sha256": "ba0ec006116176cd6396212950b90d66f0c1b913775f0a9fb3451b845c5e40c7" + }, + { + "byteSize": 5994, + "path": "comparison.json", + "sha256": "5ad875415c7ffad80abb3428be37d34e54d0ec69d09352f97375ab89ca821d89" + }, + { + "byteSize": 2769, + "path": "config.json", + "sha256": "480c323b930dcece647f3db77038b12cd333457e8abb0983417f73c1aa5a8bde" + }, + { + "byteSize": 2163, + "path": "inner_selection.evalset.json", + "sha256": "af935deba41f9e95db01dcb4b8db86c17619a2f205d6dddd0addad2415896715" + }, + { + "byteSize": 647, + "path": "inner_train.attribution.json", + "sha256": "c6de4971edada2345c566bfe8a39258f3585bc716c6a0294b8d5d8288af0f46b" + }, + { + "byteSize": 4108, + "path": "inner_train.evalset.json", + "sha256": "b7146ee360286979afab3f62bbace6255b13293037a27757639d294a3fc7bdff" + }, + { + "byteSize": 6045, + "path": "train.evalset.json", + "sha256": "bb66729048d0076e1f6f08b4367641bb28c0c2026ca6278d304e93360a39a945" + }, + { + "byteSize": 6061, + "path": "val.evalset.json", + "sha256": "11df6126e19a1e3cc8126ba6575aa6c69ed8c014d2c0b13484ade19be3d21afb" + } + ], + "baseline": { + "train": { + "caseIds": [ + "train_improve_alpha", + "train_improve_beta", + "train_stable" + ], + "cases": [ + { + "caseId": "train_improve_alpha", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "reason": null, + "rubrics": [], + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.1023717, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "INCORRECT", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 0.0 + }, + { + "caseId": "train_improve_beta", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "reason": null, + "rubrics": [], + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.1013703, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "INCORRECT", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 0.0 + }, + { + "caseId": "train_stable", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.1013703, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + } + ], + "datasetScore": 0.3333333333333333, + "metricScores": { + "final_response_avg_score": 0.3333333333333333 + }, + "passRate": 0.3333333333333333, + "phase": "baseline", + "split": "train" + }, + "validation": { + "caseIds": [ + "validation_improve", + "validation_regress", + "validation_stable" + ], + "cases": [ + { + "caseId": "validation_improve", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "reason": null, + "rubrics": [], + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.1065273, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "INCORRECT", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 0.0 + }, + { + "caseId": "validation_regress", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.1065273, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:regress; ANSWER:DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:regress; ANSWER:DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + }, + { + "caseId": "validation_stable", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.1065273, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + } + ], + "datasetScore": 0.6666666666666666, + "metricScores": { + "final_response_avg_score": 0.6666666666666666 + }, + "passRate": 0.6666666666666666, + "phase": "baseline", + "split": "validation" + } + }, + "candidate": { + "train": { + "caseIds": [ + "train_improve_alpha", + "train_improve_beta", + "train_stable" + ], + "cases": [ + { + "caseId": "train_improve_alpha", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.120922, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + }, + { + "caseId": "train_improve_beta", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.119919, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + }, + { + "caseId": "train_stable", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.119919, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + } + ], + "datasetScore": 1.0, + "metricScores": { + "final_response_avg_score": 1.0 + }, + "passRate": 1.0, + "phase": "candidate", + "split": "train" + }, + "validation": { + "caseIds": [ + "validation_improve", + "validation_regress", + "validation_stable" + ], + "cases": [ + { + "caseId": "validation_improve", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.1244266, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + }, + { + "caseId": "validation_regress", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "reason": null, + "rubrics": [], + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.1244266, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "INCORRECT", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:regress; ANSWER:DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:regress; ANSWER:DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 0.0 + }, + { + "caseId": "validation_stable", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785423818.1244266, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + } + ], + "datasetScore": 0.6666666666666666, + "metricScores": { + "final_response_avg_score": 0.6666666666666666 + }, + "passRate": 0.6666666666666666, + "phase": "candidate", + "split": "validation" + } + }, + "candidateFailureAttribution": { + "train": { + "failures": [], + "phase": "candidate", + "split": "train", + "statistics": { + "primaryCategoryCounts": {}, + "totalFailures": 0 + } + }, + "validation": { + "failures": [ + { + "caseId": "validation_regress", + "confidence": "medium", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + } + ], + "phase": "candidate", + "split": "validation", + "statistics": { + "primaryCategoryCounts": { + "FINAL_RESPONSE_MISMATCH": 1 + }, + "totalFailures": 1 + } + } + }, + "cost": { + "sources": [ + { + "costUsd": 0.0, + "metricCalls": 3, + "modelCalls": 3, + "name": "baseline_train", + "tokenUsage": {} + }, + { + "costUsd": 0.0, + "metricCalls": 3, + "modelCalls": 3, + "name": "baseline_validation", + "tokenUsage": {} + }, + { + "costUsd": 0.0, + "metricCalls": 0, + "modelCalls": 0, + "name": "candidate_generation.deterministic", + "tokenUsage": {} + }, + { + "costUsd": 0.0, + "metricCalls": 3, + "modelCalls": 3, + "name": "candidate_train", + "tokenUsage": {} + }, + { + "costUsd": 0.0, + "metricCalls": 3, + "modelCalls": 3, + "name": "candidate_validation", + "tokenUsage": {} + } + ], + "totalCostUsd": 0.0 + }, + "delta": { + "train": { + "cases": [ + { + "baselineAttribution": { + "caseId": "train_improve_alpha", + "confidence": "medium", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + }, + "baselinePassed": false, + "baselineScore": 0.0, + "candidateAttribution": null, + "candidatePassed": true, + "candidateScore": 1.0, + "caseId": "train_improve_alpha", + "critical": false, + "delta": 1.0, + "hard": false, + "metrics": [ + { + "baseline": 0.0, + "candidate": 1.0, + "delta": 1.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": false, + "transition": "NEW_PASS" + }, + { + "baselineAttribution": { + "caseId": "train_improve_beta", + "confidence": "medium", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + }, + "baselinePassed": false, + "baselineScore": 0.0, + "candidateAttribution": null, + "candidatePassed": true, + "candidateScore": 1.0, + "caseId": "train_improve_beta", + "critical": false, + "delta": 1.0, + "hard": false, + "metrics": [ + { + "baseline": 0.0, + "candidate": 1.0, + "delta": 1.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": false, + "transition": "NEW_PASS" + }, + { + "baselineAttribution": null, + "baselinePassed": true, + "baselineScore": 1.0, + "candidateAttribution": null, + "candidatePassed": true, + "candidateScore": 1.0, + "caseId": "train_stable", + "critical": false, + "delta": 0.0, + "hard": false, + "metrics": [ + { + "baseline": 1.0, + "candidate": 1.0, + "delta": 0.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": false, + "transition": "UNCHANGED" + } + ], + "metricDeltas": { + "final_response_avg_score": 0.6666666666666667 + }, + "passRateDelta": 0.6666666666666667, + "scoreDelta": 0.6666666666666667, + "split": "train" + }, + "validation": { + "cases": [ + { + "baselineAttribution": { + "caseId": "validation_improve", + "confidence": "medium", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + }, + "baselinePassed": false, + "baselineScore": 0.0, + "candidateAttribution": null, + "candidatePassed": true, + "candidateScore": 1.0, + "caseId": "validation_improve", + "critical": false, + "delta": 1.0, + "hard": false, + "metrics": [ + { + "baseline": 0.0, + "candidate": 1.0, + "delta": 1.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": false, + "transition": "NEW_PASS" + }, + { + "baselineAttribution": null, + "baselinePassed": true, + "baselineScore": 1.0, + "candidateAttribution": { + "caseId": "validation_regress", + "confidence": "medium", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + }, + "candidatePassed": false, + "candidateScore": 0.0, + "caseId": "validation_regress", + "critical": false, + "delta": -1.0, + "hard": true, + "metrics": [ + { + "baseline": 1.0, + "candidate": 0.0, + "delta": -1.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": true, + "transition": "NEW_FAIL" + }, + { + "baselineAttribution": null, + "baselinePassed": true, + "baselineScore": 1.0, + "candidateAttribution": null, + "candidatePassed": true, + "candidateScore": 1.0, + "caseId": "validation_stable", + "critical": true, + "delta": 0.0, + "hard": false, + "metrics": [ + { + "baseline": 1.0, + "candidate": 1.0, + "delta": 0.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": false, + "transition": "UNCHANGED" + } + ], + "metricDeltas": { + "final_response_avg_score": 0.0 + }, + "passRateDelta": 0.0, + "scoreDelta": 0.0, + "split": "validation" + } + }, + "durationSeconds": 0.2190000000045984, + "errors": [], + "failureAttribution": { + "train": { + "failures": [ + { + "caseId": "train_improve_alpha", + "confidence": "medium", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + }, + { + "caseId": "train_improve_beta", + "confidence": "medium", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + } + ], + "phase": "baseline", + "split": "train", + "statistics": { + "primaryCategoryCounts": { + "FINAL_RESPONSE_MISMATCH": 2 + }, + "totalFailures": 2 + } + }, + "validation": { + "failures": [ + { + "caseId": "validation_improve", + "confidence": "medium", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + } + ], + "phase": "baseline", + "split": "validation", + "statistics": { + "primaryCategoryCounts": { + "FINAL_RESPONSE_MISMATCH": 1 + }, + "totalFailures": 1 + } + } + }, + "finishedAt": "2026-07-30T15:03:38.207789+00:00", + "gateDecision": { + "accepted": false, + "checks": [ + { + "code": "REPORT_COMPLETE", + "message": "All required regression snapshots and report inputs are present.", + "observed": true, + "passed": true, + "threshold": true + }, + { + "code": "CANDIDATE_CHANGED", + "message": "Candidate prompt hashes differ from baseline.", + "observed": true, + "passed": true, + "threshold": true + }, + { + "code": "VALIDATION_SCORE_DELTA", + "message": "Held-out validation score delta meets the configured minimum.", + "observed": 0.0, + "passed": false, + "threshold": 0.05 + }, + { + "code": "VALIDATION_PASS_RATE_DELTA", + "message": "Held-out validation pass-rate delta meets the configured minimum.", + "observed": 0.0, + "passed": true, + "threshold": 0.0 + }, + { + "code": "NO_NEW_HARD_FAIL", + "message": "New held-out hard failures stay within budget.", + "observed": 1, + "passed": false, + "threshold": 0 + }, + { + "code": "CRITICAL_CASE_NON_REGRESSION", + "message": "Critical held-out cases do not regress.", + "observed": [], + "passed": true, + "threshold": 0.0 + }, + { + "code": "METRIC_NON_REGRESSION", + "message": "Configured validation metrics stay within regression limits.", + "observed": {}, + "passed": true, + "threshold": { + "final_response_avg_score": 0.0 + } + }, + { + "code": "COST_BUDGET", + "message": "Known enabled costs stay within budget.", + "observed": 0.0, + "passed": true, + "threshold": null + }, + { + "code": "DURATION_BUDGET", + "message": "Gate-observed duration stays within budget.", + "observed": 0.14100000000325963, + "passed": true, + "threshold": 180.0 + }, + { + "code": "OVERFIT_GUARD", + "message": "Train-only improvement is rejected as overfitting.", + "observed": { + "trainScoreDelta": 0.6666666666666667, + "validationScoreDelta": 0.0 + }, + "passed": false, + "threshold": "train improvement requires held-out validation thresholds" + } + ], + "decision": "REJECT", + "reasons": [ + "VALIDATION_SCORE_DELTA_BELOW_MINIMUM", + "NEW_HARD_FAIL_BUDGET_EXCEEDED", + "OVERFIT_TRAIN_UP_VALIDATION_DOWN" + ] + }, + "inputs": { + "adapterIdentity": "{\"backend\":\"default:fake:evaluation:v1\",\"callback\":null,\"candidateGenerator\":\"default:fake:candidate-generator:v1\",\"mode\":\"fake\"}", + "environment": { + "platform": "Windows-11-10.0.22631-SP0", + "python": "3.12.7", + "sdk": "1.1.14" + }, + "hashes": { + "config": "4f1210691dcdd12c1d97a9c73e30920a39dc27c4704ec345c94717728c78637c", + "effectiveConfig": "c4525545639ffa945a68c8b601b54cd3328f70d435b2dc147807f380f70f03b0", + "train": "db8df56423d916afa929a2fc1a6f7245d8a1f719e87ac292f88f346d1cb81789", + "validation": "499d3bde9cdc70cc4871d425d0dfee83ed30a67d35da15a4eb87cc9549583f84" + }, + "paths": { + "config": "optimizer.json", + "train": "train.evalset.json", + "validation": "val.evalset.json" + }, + "seed": 42 + }, + "mode": "fake", + "optimization": { + "innerSplit": { + "selectionCaseIds": [ + "train_improve_beta" + ], + "selectionHash": "ec02d7b1c9840580943cbed756a0d3bc8e0185adf2b2fc7009ad70d61aaca92a", + "selectionPath": "inner_selection.evalset.json", + "trainCaseIds": [ + "train_improve_alpha", + "train_stable" + ], + "trainHash": "70f85fca6ab8ece3ad0e3a3b1c3a6635eab921bf29efa4ed224b22513e9d9cd2", + "trainPath": "inner_train.evalset.json" + }, + "proposal": { + "algorithm": "deterministic_failure_rewrite", + "baselinePrompts": { + "system": "You are a deterministic response agent.\n\nReturn the requested answer exactly. Do not add commentary.\n" + }, + "changed": true, + "costSources": [ + { + "costUsd": 0.0, + "metricCalls": 0, + "modelCalls": 0, + "name": "deterministic", + "tokenUsage": {} + } + ], + "durationSeconds": 0.0, + "prompts": { + "system": "You are a deterministic response agent.\n\nReturn the requested answer exactly. Do not add commentary.\n\nBEHAVIOR_PROFILE=precision-v2\n" + }, + "rounds": [ + { + "acceptanceReason": "deterministic failure-attribution rewrite", + "accepted": true, + "candidatePrompts": { + "system": "You are a deterministic response agent.\n\nReturn the requested answer exactly. Do not add commentary.\n\nBEHAVIOR_PROFILE=precision-v2\n" + }, + "costUsd": 0.0, + "durationSeconds": 0.0, + "errorMessage": null, + "kind": "deterministic", + "metricScores": {}, + "optimizedFields": [ + "system" + ], + "round": 1, + "score": 1.0, + "skipReason": null, + "tokenUsage": {} + } + ], + "status": "SUCCEEDED", + "stopReason": "completed" + } + }, + "prompts": { + "baseline": { + "system": "You are a deterministic response agent.\n\nReturn the requested answer exactly. Do not add commentary.\n" + }, + "baselineHashes": { + "system": "aacc5210d0fe4482263f69f5f83241cdf9252e44be22a70de08107daec05e639" + }, + "candidate": { + "system": "You are a deterministic response agent.\n\nReturn the requested answer exactly. Do not add commentary.\n\nBEHAVIOR_PROFILE=precision-v2\n" + }, + "candidateHashes": { + "system": "ed2dfc103a7763a126f690e536572c5d06d20e2119ef7ef707c0a72deb162be2" + } + }, + "reproducibility": { + "command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --config examples/optimization/eval_optimize_loop/optimizer.json --train examples/optimization/eval_optimize_loop/train.evalset.json --validation examples/optimization/eval_optimize_loop/val.evalset.json --run-id sample-output-v1-replay --no-apply-candidate", + "gitCommit": "ceb6ff81a7a499689ddef85f0d0c997d2e54a505", + "gitDirty": false, + "reason": null, + "reproducible": true + }, + "runId": "sample-output-v1", + "schemaVersion": "v2", + "sourceApplication": { + "applied": false, + "baselineHashes": { + "system": "aacc5210d0fe4482263f69f5f83241cdf9252e44be22a70de08107daec05e639" + }, + "finalHashes": { + "system": "aacc5210d0fe4482263f69f5f83241cdf9252e44be22a70de08107daec05e639" + }, + "requested": false + }, + "stage": "complete", + "startedAt": "2026-07-30T15:03:37.984886+00:00", + "status": "REJECT" +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md new file mode 100644 index 000000000..aa6a125d0 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md @@ -0,0 +1,32 @@ +# Optimization Report: REJECT + +- Run ID: `sample-output-v1` +- Mode: `fake` +- Final stage: `complete` +- Duration: `0.219s` +- Reproducible: `yes` +- Source prompt applied: `no` + +## Regression + +| Split | Baseline score | Candidate score | Baseline pass rate | Candidate pass rate | +|---|---:|---:|---:|---:| +| train | 0.3333 | 1.0000 | 0.3333 | 1.0000 | +| validation | 0.6667 | 0.6667 | 0.6667 | 0.6667 | + +## Validation Transitions + +- `validation_improve`: `NEW_PASS` (+1.0000) +- `validation_regress`: `NEW_FAIL` (-1.0000) +- `validation_stable`: `UNCHANGED` (+0.0000) + +## Gate + +- Decision: `REJECT` +- Failed checks: `VALIDATION_SCORE_DELTA, NO_NEW_HARD_FAIL, OVERFIT_GUARD` +- Reasons: `VALIDATION_SCORE_DELTA_BELOW_MINIMUM, NEW_HARD_FAIL_BUDGET_EXCEEDED, OVERFIT_TRAIN_UP_VALIDATION_DOWN` +- Overfit detected: `yes` + +## Cost + +- Total: `$0.000000` diff --git a/tests/examples/eval_optimize_loop/test_deliverables.py b/tests/examples/eval_optimize_loop/test_deliverables.py index 3d9f7e5a1..ef08e2c9f 100644 --- a/tests/examples/eval_optimize_loop/test_deliverables.py +++ b/tests/examples/eval_optimize_loop/test_deliverables.py @@ -6,6 +6,13 @@ import re from pathlib import Path +from examples.optimization.eval_optimize_loop.pipeline.models import ( + Decision, + OptimizationReport, + Transition, +) +from examples.optimization.eval_optimize_loop.pipeline.reporting import render_markdown + EXAMPLE_ROOT = (Path(__file__).resolve().parents[3] / "examples" / "optimization" / "eval_optimize_loop") @@ -30,3 +37,44 @@ def test_solution_is_300_to_500_chinese_characters_and_covers_required_topics() assert 300 <= len(chinese_characters) <= 500 for topic in ("失败归因", "接受门禁", "过拟合", "审计"): assert topic in solution + + +def test_committed_sample_report_is_complete_and_rendered_from_the_model() -> None: + sample_root = EXAMPLE_ROOT / "sample_output" + payload = json.loads((sample_root / "optimization_report.json").read_text(encoding="utf-8")) + assert { + "baseline", + "candidate", + "delta", + "gateDecision", + "failureAttribution", + } <= payload.keys() + + report = OptimizationReport.model_validate(payload) + assert report.status == Decision.REJECT + assert report.stage == "complete" + assert report.duration_seconds < 180 + assert report.reproducibility.reproducible is True + assert report.reproducibility.git_dirty is False + assert report.baseline and report.baseline.train and report.baseline.validation + assert report.candidate and report.candidate.train and report.candidate.validation + assert report.delta and report.delta.train and report.delta.validation + assert report.gate_decision and report.gate_decision.decision == Decision.REJECT + assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in report.gate_decision.reasons + transitions = {case.transition for case in report.delta.validation.cases} + assert transitions == { + Transition.NEW_PASS, + Transition.NEW_FAIL, + Transition.UNCHANGED, + } + assert report.failure_attribution + for snapshot in ( + report.failure_attribution.train, + report.failure_attribution.validation, + ): + assert snapshot + assert snapshot.statistics.total_failures == len(snapshot.failures) + assert all(failure.reasons and failure.evidence for failure in snapshot.failures) + + markdown = (sample_root / "optimization_report.md").read_text(encoding="utf-8") + assert markdown == render_markdown(report) From 858b196edcfb7d10b8e7283625177ad72fbaf8f4 Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Fri, 31 Jul 2026 01:07:16 +0800 Subject: [PATCH 04/13] refactor(example): harden evaluation optimization architecture --- .../optimization/eval_optimize_loop/DESIGN.md | 58 ++-- .../optimization/eval_optimize_loop/README.md | 20 +- .../eval_optimize_loop/SOLUTION.md | 6 +- .../eval_optimize_loop/optimizer.json | 4 + .../eval_optimize_loop/pipeline/artifacts.py | 28 +- .../pipeline/attribution.py | 52 ++-- .../eval_optimize_loop/pipeline/backends.py | 259 ++++++++++++------ .../pipeline/candidate_runtime.py | 7 +- .../pipeline/configuration.py | 26 ++ .../eval_optimize_loop/pipeline/costing.py | 2 + .../pipeline/evaluation_runtime.py | 30 +- .../eval_optimize_loop/pipeline/gate.py | 10 +- .../eval_optimize_loop/pipeline/models.py | 7 + .../pipeline/optimizer_worker.py | 79 ++++++ .../pipeline/orchestrator.py | 58 +++- .../eval_optimize_loop/pipeline/preflight.py | 40 ++- .../pipeline/prompt_workspace.py | 67 +++++ .../eval_optimize_loop/pipeline/reporting.py | 51 +++- .../eval_optimize_loop/pipeline/schema.py | 16 +- .../pipeline/trace_fixture.py | 59 ++++ .../test_attribution_gate.py | 37 +++ .../eval_optimize_loop/test_backends.py | 63 ++++- .../eval_optimize_loop/test_deliverables.py | 2 +- .../test_models_and_imports.py | 14 +- .../eval_optimize_loop/test_orchestrator.py | 140 +++++++++- .../test_workspace_artifacts.py | 46 +++- 26 files changed, 992 insertions(+), 189 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/trace_fixture.py diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index 7bf68d770..90559672a 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -13,7 +13,9 @@ configuration -> models / schema models -> schema evaluation_runtime -> backend contract / normalizer / cost ledger / artifact sink candidate_runtime -> generator contract / split policy / prompt workspace / artifact sink -backends -> offline_evaluation / contracts / models +backends -> offline_evaluation / trace_fixture / contracts / models +optimizer_worker -> schema +trace_fixture -> models / schema offline_evaluation -> SDK evaluation types pure policies -> models reporting -> models / artifacts @@ -25,6 +27,7 @@ report facts. `preflight.py` owns validated inputs and run identity. `evaluation_runtime.py` owns ordered backend calls, completed-call accounting, and snapshot persistence. `candidate_runtime.py` owns inner-split persistence, the OS-temporary optimizer workspace, and sanitized optimizer output import. +`trace_fixture.py` owns trace schema, hash and case-matrix validation. `offline_evaluation.py` owns offline rule parsing, evidence extraction, deterministic evaluators, and the run-local replacement registry. `backends.py` only adapts fake, trace, and live inputs to SDK calls. These are concrete flat @@ -57,13 +60,14 @@ status, and reason; attribution consumes only failed rubric outcomes. Unsupporte rules, missing operands, or unavailable evidence fail the evaluation instead of fabricating a score. -Candidate generators return independent cost sources instead of a lossy aggregate. -The live optimizer source reports only reflection calls and optimizer cost; a -separate unknown judge source records that judge calls and judge cost are not -available from the SDK result. Unknown accounting remains `null` through the cost -ledger, so the total is also unknown. Only the built-in deterministic generator -declares zero cost and zero model calls. Custom generators default to an explicit -`unreported` source; an enabled cost gate rejects it with `COST_UNAVAILABLE`. +Candidate generators return independent cost sources instead of a lossy +aggregate. The SDK defines `OptimizeResult.total_llm_cost` as total optimizer +cost, including evaluator calls, so the live adapter records it once without an +invented unknown judge source. Live evaluation remains unknown unless both +per-agent-call and per-metric-call maxima are configured; then each evaluation +source is explicitly marked as an upper bound. Unknown accounting remains +`null` through the ledger, and an enabled cost gate rejects it with +`COST_UNAVAILABLE`. The report schema is `v2`. It intentionally replaces `rubricIds` with structured `rubrics` and candidate aggregate accounting with source-level accounting; no @@ -72,8 +76,11 @@ compatibility shim reconstructs the discarded `v1` information. ## Lifecycle 1. Preflight strictly parses config and datasets, rejects duplicate JSON keys, - validates split isolation, hashes every input, and validates trace pins. -2. The artifact directory is created exclusively; an existing run ID fails. + validates split isolation, hashes every input, and validates the complete + trace phase/split/case matrix without side effects. +2. A cross-process lock is acquired for the complete prompt path set, then the + artifact directory is created exclusively; contention or a reused run ID + fails before prompt mutation. 3. Baseline train and held-out validation are evaluated sequentially. 4. A seeded inner train/selection split is persisted. Only inner-train failure attribution is passed to the candidate generator. @@ -103,11 +110,19 @@ The import boundary enforces file-count, per-file-byte and total-byte budgets before reading optimizer output. This boundary protects audit storage; it is not a sandbox for a programmatic generator, which is trusted in-process code. -Live cancellation writes the optimizer stop signal and waits for cooperative -shutdown so an optimization thread is not silently abandoned. Python cannot -safely force-kill an in-process thread. Deployments requiring a hard timeout must -place live optimization in a supervised process or container and enforce the -deadline there. +CLI live optimization uses `optimizer_worker.py`. Cancellation writes the SDK +stop signal, waits for the configured bound, then terminates and finally kills a +worker that still does not exit. Programmatically injected generators remain +trusted in-process components and make the report non-reproducible. Terminal +report persistence failures raise `AuditPersistenceError`; they are never +discarded while returning an apparently handled pipeline error. Audit writes +redact credentials without truncation and fail before publication when the +configured file-byte ceiling is exceeded. + +A replay claim is emitted only for a clean, pinned Git commit when every +effective config, dataset, prompt, trace and live callback source is inside that +repository and tracked. Absolute external or untracked inputs remain executable +but are explicitly marked non-reproducible. ## Industrial Acceptance @@ -122,11 +137,8 @@ manifests. The dependency test is an allowlist for all flat pipeline modules and rejects any new pipeline module, reverse import, or cycle until the architecture contract is updated deliberately. -The pipeline is roughly 3,700 non-empty lines and its tests roughly 2,150. That -size reflects immutable manifests, verified prompt rollback and apply, strict -SDK-result normalization, partial ERROR reports, reproducibility metadata, and -three execution modes. The implementation stays in one flat package with explicit -ownership and an import allowlist. Line count is tracked as a review signal, not -used as a reason to merge unrelated responsibilities or introduce facade layers. -The largest modules are validated result models, normalization, backend adapters, -report persistence, and the composition root. +The implementation stays in one flat package with explicit ownership and an +import allowlist. Line count is tracked as a review signal, not used as a reason +to merge unrelated responsibilities or introduce facade layers. The largest +modules remain validated result models, normalization, backend adapters, report +persistence, and the composition root. diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index f61501fcf..22abd446d 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -33,8 +33,8 @@ Reports retain each rubric's ID, score, pass status, and reason. Failure attribution uses only rubric outcomes that failed their metric threshold. Live mode requires an importable async `query -> str` callback. The optimizer is -always called with `update_source=False`; only this pipeline's gate may apply a -candidate: +run in a supervised subprocess and always called with `update_source=False`; +only this pipeline's gate may apply a candidate: ```bash python examples/optimization/eval_optimize_loop/run_pipeline.py \ @@ -45,16 +45,24 @@ Source prompts remain unchanged by default. Add `--apply-candidate` only when a gate ACCEPT should be written back. REJECT is a completed audit run and exits 0; ERROR exits non-zero. Run IDs are immutable and cannot be reused. A reproducible report command preserves all effective inputs but appends `-replay` to the run ID -so it can execute without overwriting the authoritative original run. +so it can execute without overwriting the authoritative original run. A report +is marked reproducible only when the worktree is clean and every effective +config, dataset, prompt, trace, and callback source is tracked by the pinned Git +commit. Cost is reported by source. Any unreported source makes the total cost unknown; when a cost budget is enabled, the gate fails closed with `COST_UNAVAILABLE`. +Live evaluation can participate in a cost gate by configuring both +`liveAgentCallMaxCostUsd` and `liveMetricCallMaxCostUsd`; these values produce a +conservative upper bound rather than a fabricated bill. Optimizer artifacts are accepted only through a sanitized allowlist and bounded by configurable file-count, per-file-byte, and total-byte limits. Live optimizer -cancellation is cooperative. A production deployment that requires a hard -deadline must run the live optimizer in a supervised worker process or container -and terminate that worker at the platform boundary. +cancellation first requests cooperative shutdown, then terminates the isolated +worker after `optimizerShutdownTimeoutSeconds`. Audit values are redacted but +never silently truncated; a report exceeding `maxAuditFileBytes` fails +explicitly. Concurrent runs targeting the same prompt files are rejected by a +cross-process lock. See [SOLUTION.md](SOLUTION.md) for the concise Chinese issue proposal and [DESIGN.md](DESIGN.md) for detailed stage contracts and failure semantics. A diff --git a/examples/optimization/eval_optimize_loop/SOLUTION.md b/examples/optimization/eval_optimize_loop/SOLUTION.md index 93fd31023..191c3271f 100644 --- a/examples/optimization/eval_optimize_loop/SOLUTION.md +++ b/examples/optimization/eval_optimize_loop/SOLUTION.md @@ -1,7 +1,7 @@ # 方案设计说明 -本方案把评测、归因、优化、回归和审计组织为单向流水线。基线阶段使用 AgentEvaluator 分别运行训练集与验证集，保留每个用例、每次运行和每项指标的分数、阈值、原因及关键轨迹。失败归因先比较结构化工具名与参数，再读取失败的指标和 rubric，最后才使用文本兜底；分类有固定优先级，每条失败均输出主类别、证据和置信度，并由模型校验分类统计与逐条事实一致。 +本方案把评测、归因、优化、回归和审计组织为单向流水线。预检严格解析并哈希配置、数据集、prompt 与 trace，验证训练/验证隔离和完整 trace 矩阵；只有全部有效输入受同一干净 Git 提交管理时才声明可复现。基线阶段用 AgentEvaluator 分别运行训练集与验证集，保留逐用例、逐运行、逐指标分数、阈值、原因和关键轨迹。失败归因按结构证据、显式 rubric/指标映射、原因语义和指标兜底的顺序决策，每个失败输出主类别、次类别、证据与置信度。 -候选生成只接收训练集内部切分的失败信息，不读取外部验证集。候选完成后重新评测完整训练集和保留验证集，逐用例标记新增通过、新增失败、提升、下降或不变。接受门禁按固定顺序检查验证总分与通过率阈值、新增 hard fail、关键用例退化、单项指标回退、成本和耗时；训练提升但验证下降会被明确识别为过拟合并拒绝，未知成本在启用预算时也按失败处理。 +候选生成只接收训练集内部切分的失败事实，不读取外部验证集；live 优化运行在可终止 worker 中，取消先协作停止，超时再终止进程。候选完成后重跑完整训练集和保留验证集，逐用例标记新增通过、新增失败、提升、下降或不变。接受门禁依次检查验证总分与通过率、新增 hard fail、关键用例、指标回退、成本和耗时；训练提升而验证下降是不可配置的过拟合拒绝条件，未知成本在启用预算时失败关闭，live 评测可使用显式单次调用上界。 -所有输入在运行前严格解析并哈希，随机种子、适配器身份、候选轮次、成本来源、耗时和门禁理由写入不可变运行目录。prompt 默认不回写，只有门禁接受且显式授权时才原子应用；异常或取消会恢复并校验基线。fake 与 trace 模式使用确定性替代规则，不读取真实密钥，最终同时生成机器可读 JSON、人可读 Markdown 和带哈希清单的审计产物，便于复现、比较和追责。 +同一组 prompt 由跨进程锁串行保护，默认不回写；仅门禁接受且显式授权时原子应用、回读和验哈希，异常或取消必须恢复基线。审计保存完整候选轮次、成本来源、耗时、种子、门禁理由和产物哈希，只脱敏而不静默截断，超出文件预算或终态落盘失败会显式报错。fake/trace 使用确定性规则且不读真实密钥，最终生成 JSON、Markdown 和清单，支持复现、比较与追责。 diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json index f907ae0f8..e3281b968 100644 --- a/examples/optimization/eval_optimize_loop/optimizer.json +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -45,6 +45,10 @@ "applyCandidate": false, "artifactRoot": "artifacts", "traceFixture": "traces/trace_cases.json", + "maxAuditFileBytes": 26214400, + "liveAgentCallMaxCostUsd": null, + "liveMetricCallMaxCostUsd": null, + "optimizerShutdownTimeoutSeconds": 10.0, "maxImportFiles": 256, "maxImportFileBytes": 5242880, "maxImportTotalBytes": 26214400, diff --git a/examples/optimization/eval_optimize_loop/pipeline/artifacts.py b/examples/optimization/eval_optimize_loop/pipeline/artifacts.py index 92174c201..78f369fa8 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/artifacts.py +++ b/examples/optimization/eval_optimize_loop/pipeline/artifacts.py @@ -12,7 +12,7 @@ from typing import Any from .models import ArtifactRecord -from .schema import parse_strict_json, sanitize, sanitized_text, validate_safe_component +from .schema import parse_strict_json, sanitize, validate_safe_component _REPORT_FILES = {"optimization_report.json", "optimization_report.md"} _TEXT_ARTIFACT_SUFFIXES = {".log", ".md", ".txt"} @@ -20,6 +20,10 @@ _ATOMIC_REPLACE_ATTEMPTS = 8 +class AuditPersistenceError(RuntimeError): + """Raised when a terminal audit report cannot be durably persisted.""" + + def load_strict_json(path: str | Path) -> dict[str, Any]: return parse_strict_json(Path(path).read_text(encoding="utf-8")) @@ -32,8 +36,8 @@ def __init__( artifact_root: str | Path, run_id: str, *, - max_text_chars: int, publication_root: str | Path | None = None, + max_file_bytes: int = 25 * 1024 * 1024, max_import_files: int = 256, max_import_file_bytes: int = 5 * 1024 * 1024, max_import_total_bytes: int = 25 * 1024 * 1024, @@ -43,7 +47,11 @@ def __init__( self.publication_root = Path(publication_root).resolve() if publication_root is not None else self.root.parent self.run_id = run_id self.run_dir = self.root / run_id - self.max_text_chars = max_text_chars + if min(max_file_bytes, max_import_files, max_import_file_bytes, max_import_total_bytes) < 1: + raise ValueError("audit and import limits must be positive") + if max_import_total_bytes < max_import_file_bytes: + raise ValueError("total import byte limit must be at least the per-file limit") + self.max_file_bytes = max_file_bytes self.max_import_files = max_import_files self.max_import_file_bytes = max_import_file_bytes self.max_import_total_bytes = max_import_total_bytes @@ -70,8 +78,10 @@ def _resolve(self, relative_path: str) -> Path: raise ValueError("artifact path escapes the run directory") return resolved - @staticmethod - def _atomic_write(path: Path, content: str) -> None: + def _atomic_write(self, path: Path, content: str) -> None: + byte_size = len(content.encode("utf-8")) + if byte_size > self.max_file_bytes: + raise ValueError(f"audit file exceeds byte limit: {path.name} ({byte_size} > {self.max_file_bytes})") path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") try: @@ -90,14 +100,16 @@ def _atomic_write(path: Path, content: str) -> None: def write_json(self, relative_path: str, payload: Any) -> Path: path = self._resolve(relative_path) - clean = sanitize(payload, max_text_chars=self.max_text_chars) + clean = sanitize(payload, max_text_chars=None) content = json.dumps(clean, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False) + "\n" self._atomic_write(path, content) return path def write_text(self, relative_path: str, content: str) -> Path: path = self._resolve(relative_path) - clean = sanitized_text(content, max_text_chars=self.max_text_chars) + clean = sanitize(content, max_text_chars=None) + if not isinstance(clean, str): + raise TypeError("text audit content must remain text after sanitization") self._atomic_write(path, clean) return path @@ -105,7 +117,7 @@ def write_jsonl(self, relative_path: str, payloads: list[dict[str, Any]]) -> Pat path = self._resolve(relative_path) lines = [ json.dumps( - sanitize(payload, max_text_chars=self.max_text_chars), + sanitize(payload, max_text_chars=None), ensure_ascii=False, sort_keys=True, allow_nan=False, diff --git a/examples/optimization/eval_optimize_loop/pipeline/attribution.py b/examples/optimization/eval_optimize_loop/pipeline/attribution.py index d2e03b87f..c4de09800 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/attribution.py +++ b/examples/optimization/eval_optimize_loop/pipeline/attribution.py @@ -19,7 +19,6 @@ ) from .schema import sanitized_text -_PRIORITY = tuple(FailureCategory) _KNOWN_METRICS = { "tool_trajectory_avg_score": FailureCategory.TOOL_CALL_ERROR, "llm_rubric_knowledge_recall": FailureCategory.KNOWLEDGE_RECALL_INSUFFICIENT, @@ -107,25 +106,27 @@ def attribute_failures( for case in snapshot.cases: if case.passed: continue - candidates: list[FailureCategory] = [] + candidates: list[tuple[FailureCategory, str]] = [] reasons: list[str] = [] evidence: list[str] = [] trigger_metrics: list[str] = [] trigger_rubrics: list[str] = [] - authoritative = False + + def add_candidate(category: FailureCategory, confidence: str) -> None: + if all(existing != category for existing, _ in candidates): + candidates.append((category, confidence)) + if case.error: - candidates.append(FailureCategory.EVALUATION_ERROR) + add_candidate(FailureCategory.EVALUATION_ERROR, "high") reasons.append("Evaluator reported an execution error.") evidence.append(_safe_text(case.error, max_text_chars)) - authoritative = True tool_differences = _tool_differences(case, max_text_chars) structured_tool_categories = {category for category, _ in tool_differences} for category, item_evidence in tool_differences: - candidates.append(category) + add_candidate(category, "high") reasons.append("Structured expected and actual tool trajectories differ.") evidence.append(item_evidence) - authoritative = True for run in case.runs: for metric in run.metrics: @@ -144,35 +145,36 @@ def attribute_failures( }, max_text_chars, )) - if metric.metric_name in config.metric_categories: - candidates.append(config.metric_categories[metric.metric_name]) - authoritative = True for rubric in metric.rubrics: if rubric.passed: continue trigger_rubrics.append(rubric.id) if rubric.id in config.rubric_categories: - candidates.append(config.rubric_categories[rubric.id]) - authoritative = True + add_candidate(config.rubric_categories[rubric.id], "high") + if metric.metric_name in config.metric_categories: + add_candidate(config.metric_categories[metric.metric_name], "high") + + semantic = _fallback_category(reason) + if semantic not in { + FailureCategory.UNKNOWN, + FailureCategory.FINAL_RESPONSE_MISMATCH, + }: + add_candidate(semantic, "medium") if metric.metric_name in _KNOWN_METRICS and not (metric.metric_name == "tool_trajectory_avg_score" and structured_tool_categories): - candidates.append(_KNOWN_METRICS[metric.metric_name]) - authoritative = True - elif not authoritative: - candidates.append(_fallback_category(reason)) - - categories = sorted(set(candidates or [FailureCategory.UNKNOWN]), key=_PRIORITY.index) - primary = categories[0] - confidence = ("high" if primary in { - FailureCategory.EVALUATION_ERROR, - FailureCategory.TOOL_CALL_ERROR, - FailureCategory.TOOL_ARGUMENT_ERROR, - } else "medium" if authoritative else "low") + add_candidate(_KNOWN_METRICS[metric.metric_name], "medium") + if semantic == FailureCategory.FINAL_RESPONSE_MISMATCH: + add_candidate(semantic, "medium") + + if not candidates: + candidates.append((FailureCategory.UNKNOWN, "low")) + primary, confidence = candidates[0] + categories = tuple(category for category, _ in candidates) failures.append( FailureAttribution( case_id=case.case_id, primary=primary, - secondary=tuple(categories[1:]), + secondary=categories[1:], reasons=tuple(dict.fromkeys(reasons or ["The failure has no mapped deterministic cause."])), trigger_metrics=tuple(dict.fromkeys(trigger_metrics)), trigger_rubrics=tuple(dict.fromkeys(trigger_rubrics)), diff --git a/examples/optimization/eval_optimize_loop/pipeline/backends.py b/examples/optimization/eval_optimize_loop/pipeline/backends.py index 249a22f2d..e5db93c8a 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/backends.py +++ b/examples/optimization/eval_optimize_loop/pipeline/backends.py @@ -3,8 +3,9 @@ from __future__ import annotations import asyncio -import hashlib import inspect +import json +import sys from copy import deepcopy from pathlib import Path from typing import Awaitable, Callable, Optional @@ -23,6 +24,7 @@ InferenceConfig, InferenceRequest, LocalEvalService, + OptimizeResult, RemoteEvalService, TargetPrompt, ) @@ -38,9 +40,11 @@ ) from .offline_evaluation import prepare_offline_evaluation from .schema import add_exception_note, parse_strict_json +from .trace_fixture import TraceFixture CallAgent = Callable[[str], Awaitable[str]] _DEFAULT_APP_NAME = "test_app" +_REPOSITORY_ROOT = Path(__file__).resolve().parents[4] async def _evaluate_with_sdk( @@ -210,46 +214,15 @@ def __init__( dataset_hashes: dict[str, str], fixture_hash: Optional[str] = None, ) -> None: - self._fixture_path = Path(fixture_path) - self._dataset_hashes = dict(dataset_hashes) - if not self._fixture_path.is_file(): - raise FileNotFoundError(f"trace fixture does not exist: {self._fixture_path}") - content = self._fixture_path.read_bytes() - actual_hash = hashlib.sha256(content).hexdigest() - if fixture_hash is not None and actual_hash != fixture_hash: - raise ValueError("trace fixture changed after preflight") - self._fixture_hash = actual_hash - self._payload = parse_strict_json(content.decode("utf-8")) + self._fixture = TraceFixture(fixture_path, dataset_hashes, fixture_hash) def validate_fixture(self, train: EvalSet, validation: EvalSet) -> None: """Fail before run creation when any pinned trace input has drifted.""" - for phase in Phase: - self._trace_eval_set(train, Split.TRAIN, phase) - self._trace_eval_set(validation, Split.VALIDATION, phase) + self._fixture.validate(train, validation) def _trace_eval_set(self, eval_set: EvalSet, split: Split, phase: Phase) -> EvalSet: - payload = self._payload - if payload.get("schemaVersion") != "v1": - raise ValueError("unsupported trace fixture schemaVersion") - recorded_hashes = payload.get("datasetHashes") - if recorded_hashes != self._dataset_hashes: - raise ValueError("trace fixture dataset hashes do not match validated inputs") - try: - cases = payload["phases"][phase.value][split.value] - except (KeyError, TypeError) as error: - raise ValueError(f"trace fixture is missing {phase.value}/{split.value}") from error - expected_ids = [case.eval_id for case in eval_set.eval_cases] - if not isinstance(cases, dict) or set(cases) != set(expected_ids): - raise ValueError("trace fixture case IDs do not match the dataset") - raw = eval_set.model_dump(mode="json", by_alias=True, exclude_none=True) - for case in raw["evalCases"]: - conversation = deepcopy(cases[case["evalId"]]) - if not isinstance(conversation, list) or not conversation: - raise ValueError("each trace fixture conversation must be non-empty") - case["evalMode"] = "trace" - case["actualConversation"] = conversation - return EvalSet.model_validate(raw) + return self._fixture.eval_set(eval_set, split, phase) async def evaluate( self, @@ -347,49 +320,53 @@ async def generate( class LiveCandidateGenerator: """Adapt AgentOptimizer output into the strict candidate-only contract.""" - def __init__(self, call_agent: CallAgent, *, verbose: int = 0) -> None: + def __init__( + self, + call_agent: CallAgent, + *, + callback_spec: Optional[str] = None, + shutdown_timeout_seconds: float = 10.0, + verbose: int = 0, + ) -> None: if not inspect.iscoroutinefunction(call_agent): raise TypeError("live call_agent must be an async function") + if shutdown_timeout_seconds <= 0: + raise ValueError("optimizer shutdown timeout must be positive") self._call_agent = call_agent + self._callback_spec = callback_spec + self._shutdown_timeout_seconds = shutdown_timeout_seconds self._verbose = verbose - async def generate( - self, - *, - target_prompt: TargetPrompt, - baseline_prompts: dict[str, str], - train_attribution: AttributionSnapshot, - inner_train_path: str, - inner_selection_path: str, - config_path: str, - output_dir: str, - ) -> CandidateProposal: - del train_attribution - optimize_task = asyncio.create_task( - AgentOptimizer.optimize( - config_path=config_path, - call_agent=self._call_agent, - target_prompt=target_prompt, - train_dataset_path=inner_train_path, - validation_dataset_path=inner_selection_path, - output_dir=output_dir, - update_source=False, - verbose=self._verbose, - )) + @staticmethod + def _request_stop(output_dir: str, cancellation: BaseException) -> None: + stop_path = Path(output_dir) / "optimize.stop" + try: + stop_path.parent.mkdir(parents=True, exist_ok=True) + stop_path.write_text("cancel requested\n", encoding="utf-8") + except OSError as stop_error: + add_exception_note(cancellation, f"could not request optimizer stop: {stop_error}") + + async def _optimize_in_process(self, **kwargs) -> OptimizeResult: + optimize_task = asyncio.create_task(AgentOptimizer.optimize( + **kwargs, + call_agent=self._call_agent, + )) try: - result = await asyncio.shield(optimize_task) + return await asyncio.shield(optimize_task) except asyncio.CancelledError as cancellation: - stop_path = Path(output_dir) / "optimize.stop" + self._request_stop(kwargs["output_dir"], cancellation) try: - stop_path.parent.mkdir(parents=True, exist_ok=True) - stop_path.write_text("cancel requested\n", encoding="utf-8") - except OSError as stop_error: + await asyncio.wait_for( + asyncio.shield(optimize_task), + timeout=self._shutdown_timeout_seconds, + ) + except asyncio.TimeoutError: add_exception_note( cancellation, - f"could not request optimizer stop: {stop_error}", + "programmatic optimizer did not stop before its shutdown timeout; " + "use an importable callback spec for process isolation", ) - try: - await optimize_task + optimize_task.add_done_callback(self._consume_task_result) except BaseException as shutdown_error: add_exception_note( cancellation, @@ -397,6 +374,128 @@ async def generate( f"{shutdown_error}", ) raise + + @staticmethod + def _consume_task_result(task: asyncio.Task) -> None: + try: + task.result() + except BaseException: + pass + + async def _stop_worker( + self, + process: asyncio.subprocess.Process, + *, + output_dir: str, + cancellation: BaseException, + ) -> None: + self._request_stop(output_dir, cancellation) + try: + await asyncio.wait_for( + asyncio.shield(process.wait()), + timeout=self._shutdown_timeout_seconds, + ) + return + except asyncio.TimeoutError: + if process.returncode is None: + try: + process.terminate() + except ProcessLookupError: + pass + try: + await asyncio.wait_for( + asyncio.shield(process.wait()), + timeout=self._shutdown_timeout_seconds, + ) + except asyncio.TimeoutError: + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + try: + await asyncio.wait_for( + asyncio.shield(process.wait()), + timeout=self._shutdown_timeout_seconds, + ) + except asyncio.TimeoutError: + add_exception_note(cancellation, "optimizer worker did not report exit after kill") + add_exception_note(cancellation, "optimizer worker required forced termination") + + async def _optimize_in_worker(self, **kwargs) -> OptimizeResult: + assert self._callback_spec is not None + output_dir = kwargs["output_dir"] + request_path = Path(output_dir).parent / "optimizer-worker-request.json" + prompt_paths = {name: kwargs["target_prompt"].describe_source(name) for name in kwargs["target_prompt"].names()} + request_path.write_text( + json.dumps( + { + "callbackSpec": self._callback_spec, + "promptPaths": prompt_paths, + "configPath": kwargs["config_path"], + "trainPath": kwargs["train_dataset_path"], + "validationPath": kwargs["validation_dataset_path"], + "outputDir": output_dir, + "verbose": kwargs["verbose"], + }, + ensure_ascii=False, + allow_nan=False, + ), + encoding="utf-8", + ) + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "examples.optimization.eval_optimize_loop.pipeline.optimizer_worker", + str(request_path), + cwd=str(_REPOSITORY_ROOT), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + try: + return_code = await process.wait() + except asyncio.CancelledError as cancellation: + await self._stop_worker( + process, + output_dir=output_dir, + cancellation=cancellation, + ) + raise + result_path = Path(output_dir) / "result.json" + if return_code != 0: + error_path = Path(output_dir) / "worker_error.json" + detail = "optimizer worker failed without an error artifact" + if error_path.is_file(): + payload = parse_strict_json(error_path.read_text(encoding="utf-8")) + detail = str(payload.get("message", detail)) + raise RuntimeError(f"AgentOptimizer worker failed with exit code {return_code}: {detail}") + if not result_path.is_file(): + raise RuntimeError("AgentOptimizer worker completed without result.json") + return OptimizeResult.from_file(str(result_path)) + + async def generate( + self, + *, + target_prompt: TargetPrompt, + baseline_prompts: dict[str, str], + train_attribution: AttributionSnapshot, + inner_train_path: str, + inner_selection_path: str, + config_path: str, + output_dir: str, + ) -> CandidateProposal: + del train_attribution + optimize_kwargs = { + "config_path": config_path, + "target_prompt": target_prompt, + "train_dataset_path": inner_train_path, + "validation_dataset_path": inner_selection_path, + "output_dir": output_dir, + "update_source": False, + "verbose": self._verbose, + } + result = (await self._optimize_in_worker( + **optimize_kwargs) if self._callback_spec else await self._optimize_in_process(**optimize_kwargs)) if result.status == "CANCELED": raise asyncio.CancelledError(result.error_message or "AgentOptimizer canceled") if result.status != "SUCCEEDED": @@ -425,15 +524,12 @@ async def generate( changed=result.best_prompts != baseline_prompts, stop_reason=result.stop_reason, rounds=rounds, - cost_sources=( - CostSource( - name="optimizer_reported", - cost_usd=result.total_llm_cost, - model_calls=result.total_reflection_lm_calls, - token_usage=dict(result.total_token_usage), - ), - CostSource(name="judge_unreported", cost_usd=None, model_calls=None), - ), + cost_sources=(CostSource( + name="optimizer_reported", + cost_usd=result.total_llm_cost, + model_calls=result.total_reflection_lm_calls, + token_usage=dict(result.total_token_usage), + ), ), duration_seconds=result.duration_seconds, ) return CandidateProposal.model_validate(proposal.model_dump(mode="python", by_alias=True)) @@ -446,6 +542,8 @@ def create_backends( trace_fixture_path: Optional[str] = None, trace_fixture_hash: Optional[str] = None, dataset_hashes: Optional[dict[str, str]] = None, + callback_spec: Optional[str] = None, + optimizer_shutdown_timeout_seconds: float = 10.0, ) -> tuple[EvaluationBackend, CandidateGenerator]: if mode == "fake": return FakeEvaluationBackend(), DeterministicCandidateGenerator() @@ -459,5 +557,12 @@ def create_backends( if mode == "live": if call_agent is None: raise ValueError("live mode requires call_agent") - return LiveEvaluationBackend(call_agent), LiveCandidateGenerator(call_agent) + return ( + LiveEvaluationBackend(call_agent), + LiveCandidateGenerator( + call_agent, + callback_spec=callback_spec, + shutdown_timeout_seconds=optimizer_shutdown_timeout_seconds, + ), + ) raise ValueError(f"unsupported mode: {mode!r}") diff --git a/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py b/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py index 5d726dc56..541acce4c 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py +++ b/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py @@ -19,7 +19,7 @@ InnerSplit, ) from .prompt_workspace import PromptWorkspace -from .schema import add_exception_note +from .schema import add_exception_note, validate_secret_free_text def prepare_candidate_inputs( @@ -118,4 +118,9 @@ async def generate_candidate( validated = CandidateProposal.model_validate(proposal.model_dump(mode="python", by_alias=True)) if validated.baseline_prompts != workspace.baseline: raise ValueError("candidate generator baseline prompts differ from the workspace baseline") + for name, text in validated.prompts.items(): + validate_secret_free_text(text, name=f"candidate prompt {name!r}") + for round_ in validated.rounds: + for name, text in round_.candidate_prompts.items(): + validate_secret_free_text(text, name=f"candidate round {round_.round} prompt {name!r}") return validated diff --git a/examples/optimization/eval_optimize_loop/pipeline/configuration.py b/examples/optimization/eval_optimize_loop/pipeline/configuration.py index 90230c8ab..38cd68d9e 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/configuration.py +++ b/examples/optimization/eval_optimize_loop/pipeline/configuration.py @@ -58,6 +58,13 @@ def valid_epsilon(cls, value: float) -> float: def valid_budget(cls, value: Optional[float]) -> Optional[float]: return None if value is None else _non_negative(value, "budget") + @field_validator("overfit_guard") + @classmethod + def overfit_guard_is_mandatory(cls, value: bool) -> bool: + if not value: + raise ValueError("overfit_guard is a mandatory safety invariant") + return value + @field_validator("metric_max_regression") @classmethod def valid_metric_regression(cls, value: dict[str, float]) -> dict[str, float]: @@ -83,6 +90,10 @@ class PipelineSettings(StrictModel): train_case_weights: dict[str, StrictFloat] = Field(default_factory=dict) validation_case_weights: dict[str, StrictFloat] = Field(default_factory=dict) max_text_chars: StrictInt = Field(default=4000, ge=64, le=100_000) + max_audit_file_bytes: StrictInt = Field(default=25 * 1024 * 1024, ge=1024, le=500 * 1024 * 1024) + live_agent_call_max_cost_usd: Optional[StrictFloat] = None + live_metric_call_max_cost_usd: Optional[StrictFloat] = None + optimizer_shutdown_timeout_seconds: StrictFloat = Field(default=10.0, ge=0.1, le=300.0) max_import_files: StrictInt = Field(default=256, ge=1, le=10_000) max_import_file_bytes: StrictInt = Field(default=5 * 1024 * 1024, ge=1, le=100 * 1024 * 1024) max_import_total_bytes: StrictInt = Field(default=25 * 1024 * 1024, ge=1, le=500 * 1024 * 1024) @@ -95,6 +106,19 @@ def valid_import_budget(self) -> "PipelineSettings": raise ValueError("max_import_total_bytes must be at least max_import_file_bytes") return self + @model_validator(mode="after") + def valid_live_cost_bounds(self) -> "PipelineSettings": + bounds = ( + self.live_agent_call_max_cost_usd, + self.live_metric_call_max_cost_usd, + ) + if (bounds[0] is None) != (bounds[1] is None): + raise ValueError("live Agent and metric call cost bounds must be configured together") + for value in bounds: + if value is not None: + _non_negative(value, "live call cost bound") + return self + @field_validator("inner_selection_ratio") @classmethod def valid_ratio(cls, value: float) -> float: @@ -142,3 +166,5 @@ class ValidatedRunConfig(StrictModel): prompt_paths: dict[str, str] prompt_hashes: dict[str, str] adapter_identity: str + reproducibility_paths: tuple[str, ...] + reproducibility_issues: tuple[str, ...] = () diff --git a/examples/optimization/eval_optimize_loop/pipeline/costing.py b/examples/optimization/eval_optimize_loop/pipeline/costing.py index 912cc13e3..32d7821aa 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/costing.py +++ b/examples/optimization/eval_optimize_loop/pipeline/costing.py @@ -16,6 +16,7 @@ def record( name: str, *, cost_usd: float | None, + upper_bound: bool = False, model_calls: int | None = None, metric_calls: int | None = None, token_usage: dict[str, int] | None = None, @@ -26,6 +27,7 @@ def record( CostSource( name=name, cost_usd=cost_usd, + upper_bound=upper_bound, model_calls=model_calls, metric_calls=metric_calls, token_usage=dict(token_usage or {}), diff --git a/examples/optimization/eval_optimize_loop/pipeline/evaluation_runtime.py b/examples/optimization/eval_optimize_loop/pipeline/evaluation_runtime.py index f7b61ad33..447a72942 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/evaluation_runtime.py +++ b/examples/optimization/eval_optimize_loop/pipeline/evaluation_runtime.py @@ -28,14 +28,19 @@ def create_evaluation_runtime( settings = validated.config.pipeline offline = not custom_backend and settings.mode in {"fake", "trace"} - calls_per_invocation = (1 if not custom_backend and settings.mode == "fake" else + live_with_bounds = (not custom_backend and settings.mode == "live" + and settings.live_agent_call_max_cost_usd is not None + and settings.live_metric_call_max_cost_usd is not None) + calls_per_invocation = (1 if not custom_backend and settings.mode in {"fake", "live"} else 0 if not custom_backend and settings.mode == "trace" else None) return EvaluationRuntime( backend=backend, sink=sink, ledger=ledger, eval_config=validated.config.evaluate, - cost_usd=0 if offline else None, + agent_call_cost_usd=(0 if offline else settings.live_agent_call_max_cost_usd if live_with_bounds else None), + metric_call_cost_usd=(0 if offline else settings.live_metric_call_max_cost_usd if live_with_bounds else None), + cost_is_upper_bound=live_with_bounds, model_calls_per_invocation=calls_per_invocation, metric_weights=settings.metric_weights, train_case_weights=settings.train_case_weights, @@ -51,7 +56,9 @@ class EvaluationRuntime: sink: AuditSink ledger: CostLedger eval_config: EvalConfig - cost_usd: float | None + agent_call_cost_usd: float | None + metric_call_cost_usd: float | None + cost_is_upper_bound: bool model_calls_per_invocation: int | None metric_weights: dict[str, float] train_case_weights: dict[str, float] @@ -70,6 +77,15 @@ async def evaluate( metric_count = len(self.eval_config.get_eval_metrics()) invocations = sum(len(case.conversation or case.actual_conversation or []) for case in eval_set.eval_cases) metric_calls = invocations * self.eval_config.num_runs * metric_count + model_calls = None + if self.model_calls_per_invocation is not None: + model_calls = invocations * self.eval_config.num_runs * self.model_calls_per_invocation + cost_usd = None + if self.agent_call_cost_usd is not None and self.metric_call_cost_usd is not None: + cost_usd = ( + invocations * self.eval_config.num_runs * self.agent_call_cost_usd + + metric_calls * self.metric_call_cost_usd + ) try: raw = await self.backend.evaluate( eval_set=deepcopy(eval_set), @@ -82,17 +98,15 @@ async def evaluate( except BaseException: self.ledger.record( stage, - cost_usd=self.cost_usd, + cost_usd=(0 if self.agent_call_cost_usd == self.metric_call_cost_usd == 0 else None), model_calls=(0 if self.model_calls_per_invocation == 0 else None), metric_calls=None, ) raise - model_calls = None - if self.model_calls_per_invocation is not None: - model_calls = invocations * self.eval_config.num_runs * self.model_calls_per_invocation self.ledger.record( stage, - cost_usd=self.cost_usd, + cost_usd=cost_usd, + upper_bound=self.cost_is_upper_bound, model_calls=model_calls, metric_calls=metric_calls, ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/gate.py b/examples/optimization/eval_optimize_loop/pipeline/gate.py index 4c240da85..139cb1414 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/gate.py +++ b/examples/optimization/eval_optimize_loop/pipeline/gate.py @@ -178,16 +178,18 @@ def is_critical_regression(case) -> bool: "Gate-observed duration stays within budget.", "DURATION_BUDGET_EXCEEDED", ) - overfit = config.overfit_guard and train.score_delta > config.epsilon and not (score_passed and pass_rate_passed) + validation_regressed = (validation.score_delta < -config.epsilon or validation.pass_rate_delta < -config.epsilon) + overfit = train.score_delta > config.epsilon and validation_regressed add( "OVERFIT_GUARD", not overfit, { "trainScoreDelta": train.score_delta, - "validationScoreDelta": validation.score_delta + "validationScoreDelta": validation.score_delta, + "validationPassRateDelta": validation.pass_rate_delta, }, - "train improvement requires held-out validation thresholds", - "Train-only improvement is rejected as overfitting.", + "train improvement cannot accompany held-out validation regression", + "Training improvement with held-out regression is rejected as overfitting.", "OVERFIT_TRAIN_UP_VALIDATION_DOWN", ) accepted = all(check.passed for check in checks) diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py index 08e3ec1cd..57117da2e 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/models.py +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -275,6 +275,7 @@ class InnerSplit(StrictModel): class CostSource(StrictModel): name: str cost_usd: Optional[float] + upper_bound: bool = False model_calls: Optional[int] = Field(default=None, ge=0) metric_calls: Optional[int] = Field(default=None, ge=0) token_usage: dict[str, int] = Field(default_factory=dict) @@ -291,6 +292,12 @@ def valid_name(cls, value: str) -> str: def valid_cost(cls, value: Optional[float]) -> Optional[float]: return None if value is None else _non_negative(value, "cost") + @model_validator(mode="after") + def valid_upper_bound(self) -> "CostSource": + if self.upper_bound and self.cost_usd is None: + raise ValueError("unknown cost cannot be marked as an upper bound") + return self + @field_validator("token_usage") @classmethod def valid_tokens(cls, value: dict[str, int]) -> dict[str, int]: diff --git a/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py b/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py new file mode 100644 index 000000000..0fb567407 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py @@ -0,0 +1,79 @@ +"""Isolated process entry point for a live AgentOptimizer invocation.""" + +from __future__ import annotations + +import argparse +import asyncio +import importlib +import json +from pathlib import Path +from typing import Any + +from trpc_agent_sdk.evaluation import AgentOptimizer, TargetPrompt + +from .schema import parse_strict_json, sanitize + + +def _load_callback(spec: str) -> Any: + if ":" not in spec: + raise ValueError("callbackSpec must use MODULE:FUNCTION syntax") + module_name, function_name = spec.split(":", 1) + if not module_name or not function_name: + raise ValueError("callbackSpec must use MODULE:FUNCTION syntax") + return getattr(importlib.import_module(module_name), function_name) + + +async def _run(request_path: Path) -> None: + request = parse_strict_json(request_path.read_text(encoding="utf-8")) + callback = _load_callback(str(request["callbackSpec"])) + prompt_paths = request["promptPaths"] + if not isinstance(prompt_paths, dict) or not prompt_paths: + raise ValueError("promptPaths must be a non-empty object") + target = TargetPrompt() + for name, path in prompt_paths.items(): + target.add_path(str(name), str(path)) + result = await AgentOptimizer.optimize( + config_path=str(request["configPath"]), + call_agent=callback, + target_prompt=target, + train_dataset_path=str(request["trainPath"]), + validation_dataset_path=str(request["validationPath"]), + output_dir=str(request["outputDir"]), + update_source=False, + verbose=int(request.get("verbose", 0)), + ) + output_dir = Path(str(request["outputDir"])) + output_dir.mkdir(parents=True, exist_ok=True) + result.dump_to(str(output_dir / "result.json")) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("request") + request_path = Path(parser.parse_args().request).resolve() + try: + asyncio.run(_run(request_path)) + except BaseException as error: + try: + request = parse_strict_json(request_path.read_text(encoding="utf-8")) + output_dir = Path(str(request["outputDir"])) + output_dir.mkdir(parents=True, exist_ok=True) + payload = sanitize( + { + "errorType": type(error).__name__, + "message": str(error), + }, + max_text_chars=4000, + ) + (output_dir / "worker_error.json").write_text( + json.dumps(payload, ensure_ascii=False, indent=2, allow_nan=False) + "\n", + encoding="utf-8", + ) + except BaseException: + pass + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py index 7d9b1bf5e..f64b5b020 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py +++ b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py @@ -8,10 +8,11 @@ from trpc_agent_sdk.evaluation import TargetPrompt -from .artifacts import AuditSink +from .artifacts import AuditPersistenceError, AuditSink from .attribution import attribute_failures from .backends import CallAgent, TraceEvaluationBackend, create_backends from .candidate_runtime import generate_candidate, prepare_candidate_inputs +from .configuration import ValidatedRunConfig from .contracts import CandidateGenerator, EvaluationBackend from .costing import CostLedger from .evaluation import compare_snapshots @@ -29,7 +30,12 @@ SnapshotPair, ) from .preflight import preflight_run -from .prompt_workspace import PromptRestoreError, PromptWorkspace, prompt_hashes +from .prompt_workspace import ( + PromptRestoreError, + PromptRunLock, + PromptWorkspace, + prompt_hashes, +) from .reporting import ( build_optimization_report, create_report_context, @@ -37,7 +43,7 @@ persist_terminal_report, utc_now, ) -from .schema import sanitized_text +from .schema import add_exception_note, sanitized_text FaultInjector = Callable[[str], None] @@ -79,6 +85,33 @@ async def run_pipeline( backend=backend, candidate_generator=candidate_generator, ) + run_lock = PromptRunLock(tuple(validated.prompt_paths.values()), ) + with run_lock: + return await _run_validated_pipeline( + validated, + started_at=started_at, + started_clock=started_clock, + call_agent=call_agent, + callback_spec=callback_spec, + backend=backend, + candidate_generator=candidate_generator, + fault_injector=fault_injector, + ) + + +async def _run_validated_pipeline( + validated: ValidatedRunConfig, + *, + started_at: str, + started_clock: float, + call_agent: Optional[CallAgent], + callback_spec: Optional[str], + backend: Optional[EvaluationBackend], + candidate_generator: Optional[CandidateGenerator], + fault_injector: Optional[FaultInjector], +) -> OptimizationReport: + """Execute the lifecycle while the caller owns the prompt-set lock.""" + settings = validated.config.pipeline custom_backend = backend is not None custom_components = custom_backend or candidate_generator is not None @@ -92,6 +125,8 @@ async def run_pipeline( "train": validated.input_hashes["train"], "validation": validated.input_hashes["validation"], }, + callback_spec=callback_spec, + optimizer_shutdown_timeout_seconds=settings.optimizer_shutdown_timeout_seconds, ) backend = backend or default_backend candidate_generator = candidate_generator or default_generator @@ -108,8 +143,8 @@ async def run_pipeline( sink = AuditSink( validated.artifact_root, validated.run_id, - max_text_chars=settings.max_text_chars, publication_root=validated.root_dir, + max_file_bytes=settings.max_audit_file_bytes, max_import_files=settings.max_import_files, max_import_file_bytes=settings.max_import_file_bytes, max_import_total_bytes=settings.max_import_total_bytes, @@ -241,6 +276,7 @@ def retain_progress(phase: Phase, progress: SnapshotPair) -> None: ledger.record( f"{stage}.{source.name}", cost_usd=source.cost_usd, + upper_bound=source.upper_bound, model_calls=source.model_calls, metric_calls=source.metric_calls, token_usage=source.token_usage, @@ -332,16 +368,26 @@ def retain_progress(phase: Phase, progress: SnapshotPair) -> None: message = sanitized_text(error, max_text_chars=settings.max_text_chars) run_error = RunError(stage=stage, error_type=type(error).__name__, message=message) error_report = build_report(Decision.ERROR, stage, (run_error, )) + persistence_error: Optional[BaseException] = None try: error_report = persist_terminal_report( sink, error_report, started_clock=started_clock, ) - except BaseException: - pass + except BaseException as persist_error: + persistence_error = persist_error if fatal_restore is not None: + if persistence_error is not None: + add_exception_note(fatal_restore, f"terminal audit persistence also failed: {persistence_error}") raise fatal_restore if isinstance(error, (asyncio.CancelledError, KeyboardInterrupt, SystemExit)): + if persistence_error is not None: + add_exception_note(error, f"terminal audit persistence also failed: {persistence_error}") raise + if persistence_error is not None: + audit_error = AuditPersistenceError( + f"terminal audit persistence failed at stage {stage!r}: {persistence_error}") + add_exception_note(audit_error, f"original pipeline error: {type(error).__name__}: {message}") + raise audit_error from persistence_error return error_report diff --git a/examples/optimization/eval_optimize_loop/pipeline/preflight.py b/examples/optimization/eval_optimize_loop/pipeline/preflight.py index 700f79814..5fe17b3b9 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/preflight.py +++ b/examples/optimization/eval_optimize_loop/pipeline/preflight.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import inspect from pathlib import Path from typing import Any, Optional @@ -11,8 +12,9 @@ from .artifacts import load_strict_json from .evaluation import canonical_json, dataset_fingerprint, validate_datasets from .configuration import PipelineConfig, ValidatedRunConfig -from .schema import validate_safe_component +from .schema import validate_safe_component, validate_secret_free_text from .prompt_workspace import prompt_hashes +from .trace_fixture import TraceFixture def _sha256_file(path: Path) -> str: @@ -48,6 +50,16 @@ def _callable_identity(call_agent: object | None, callback_spec: Optional[str]) return f"{module}:{qualname}" +def _callable_source_path(call_agent: object | None) -> Optional[Path]: + if call_agent is None: + return None + try: + source = inspect.getsourcefile(inspect.unwrap(call_agent)) or inspect.getfile(call_agent) + except (OSError, TypeError): + return None + return Path(source).resolve() if source else None + + def _adapter_identity( mode: str, *, @@ -124,7 +136,10 @@ def preflight_run( for name, relative in settings.prompt_paths.items(): validate_safe_component(name, name="prompt field") path = inside_example_root(root, relative, f"prompt path {name!r}") - prompts[name] = path.read_text(encoding="utf-8") + prompts[name] = validate_secret_free_text( + path.read_text(encoding="utf-8"), + name=f"prompt {name!r}", + ) prompt_paths[name] = str(path) hashes = { @@ -141,6 +156,14 @@ def preflight_run( if settings.mode == "trace": trace_path = inside_example_root(root, settings.trace_fixture, "trace fixture") hashes["trace"] = _sha256_file(trace_path) + TraceFixture( + trace_path, + { + "train": hashes["train"], + "validation": hashes["validation"], + }, + hashes["trace"], + ).validate(train, validation) adapter_identity = _adapter_identity( settings.mode, @@ -149,6 +172,17 @@ def preflight_run( backend=backend, candidate_generator=candidate_generator, ) + reproducibility_paths = [config_file, train_file, validation_file] + reproducibility_paths.extend(Path(path) for path in prompt_paths.values()) + if trace_path is not None: + reproducibility_paths.append(trace_path) + reproducibility_issues: list[str] = [] + if settings.mode == "live" and callback_spec: + callback_source = _callable_source_path(call_agent) + if callback_source is None: + reproducibility_issues.append("live_callback_source_unavailable") + else: + reproducibility_paths.append(callback_source) derived = ("run-" + hashlib.sha256( canonical_json({ "inputHashes": hashes, @@ -173,4 +207,6 @@ def preflight_run( prompt_paths=prompt_paths, prompt_hashes=prompt_hashes(prompts), adapter_identity=adapter_identity, + reproducibility_paths=tuple(str(path.resolve()) for path in reproducibility_paths), + reproducibility_issues=tuple(reproducibility_issues), ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py b/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py index 295a363fb..f65f44a5c 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py +++ b/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py @@ -3,6 +3,8 @@ from __future__ import annotations import hashlib +import os +import tempfile from contextlib import asynccontextmanager from pathlib import Path from typing import AsyncIterator @@ -16,6 +18,71 @@ class PromptRestoreError(RuntimeError): """The baseline prompt state could not be restored and verified.""" +class PromptRunLock: + """Cross-process exclusive lock for one set of file-backed prompts.""" + + def __init__( + self, + prompt_paths: tuple[str, ...], + *, + lock_root: str | None = None, + ) -> None: + if not prompt_paths: + raise ValueError("prompt run lock requires at least one prompt path") + identity = "\0".join(sorted(os.path.normcase(str(Path(path).resolve())) for path in prompt_paths)) + digest = hashlib.sha256(identity.encode("utf-8")).hexdigest() + if lock_root is not None: + root = Path(lock_root).resolve() + else: + root = Path(tempfile.gettempdir()).resolve() / "trpc-agent-eval-optimize-locks" + self.path = root / f"{digest}.lock" + self._handle = None + + def acquire(self) -> None: + if self._handle is not None: + raise RuntimeError("prompt run lock is already acquired") + self.path.parent.mkdir(parents=True, exist_ok=True) + handle = self.path.open("a+b") + if self.path.stat().st_size == 0: + handle.write(b"\0") + handle.flush() + handle.seek(0) + try: + if os.name == "nt": + import msvcrt + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except (OSError, BlockingIOError) as error: + handle.close() + raise RuntimeError("prompt sources are already owned by another pipeline run") from error + self._handle = handle + + def release(self) -> None: + handle = self._handle + if handle is None: + return + try: + handle.seek(0) + if os.name == "nt": + import msvcrt + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + finally: + handle.close() + self._handle = None + + def __enter__(self) -> "PromptRunLock": + self.acquire() + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.release() + + def hash_text(value: str) -> str: return hashlib.sha256(value.encode("utf-8")).hexdigest() diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporting.py b/examples/optimization/eval_optimize_loop/pipeline/reporting.py index 5943d4014..cb173658f 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/reporting.py +++ b/examples/optimization/eval_optimize_loop/pipeline/reporting.py @@ -69,6 +69,8 @@ def create_report_context( trace_fixture=validated.trace_fixture_path, trace_hash=validated.input_hashes.get("trace"), programmatic_component=programmatic_component, + input_paths=validated.reproducibility_paths, + input_issues=validated.reproducibility_issues, ) return ReportContext( validated=validated, @@ -85,18 +87,20 @@ def utc_now() -> str: def build_reproducibility( - repo_root: str, - *, - mode: str, - config_path: str, - train_path: str, - validation_path: str, - run_id: str, - apply_candidate: bool, - callback_spec: Optional[str] = None, - trace_fixture: Optional[str] = None, - trace_hash: Optional[str] = None, - programmatic_component: bool = False, + repo_root: str, + *, + mode: str, + config_path: str, + train_path: str, + validation_path: str, + run_id: str, + apply_candidate: bool, + callback_spec: Optional[str] = None, + trace_fixture: Optional[str] = None, + trace_hash: Optional[str] = None, + programmatic_component: bool = False, + input_paths: tuple[str, ...] = (), + input_issues: tuple[str, ...] = (), ) -> Reproducibility: commit: Optional[str] = None dirty: Optional[bool] = None @@ -138,6 +142,29 @@ def build_reproducibility( reason = "live_callback_not_importable" elif reason is None and mode == "trace" and (not trace_fixture or not trace_hash): reason = "trace_fixture_not_pinned" + elif reason is None and input_issues: + reason = input_issues[0] + + if reason is None: + assert git_root is not None + for input_path in input_paths: + resolved = Path(input_path).resolve() + try: + relative = resolved.relative_to(git_root) + except ValueError: + reason = "input_outside_git" + break + tracked = subprocess.run( + ["git", "ls-files", "--error-unmatch", "--", + relative.as_posix()], + cwd=git_root, + check=False, + capture_output=True, + text=True, + ) + if tracked.returncode != 0: + reason = "input_not_tracked" + break reproducible = reason is None command = None if reproducible: diff --git a/examples/optimization/eval_optimize_loop/pipeline/schema.py b/examples/optimization/eval_optimize_loop/pipeline/schema.py index e9d153086..df3830511 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/schema.py +++ b/examples/optimization/eval_optimize_loop/pipeline/schema.py @@ -111,8 +111,8 @@ def _redact_assignment(match: re.Match[str]) -> str: return f"{match.group('prefix')}{quote}[REDACTED]{quote}" -def sanitize(value: Any, *, max_text_chars: int) -> Any: - """Recursively redact credential fields and credential-shaped text.""" +def sanitize(value: Any, *, max_text_chars: int | None) -> Any: + """Recursively redact credentials and optionally bound individual strings.""" if isinstance(value, dict): return { @@ -124,7 +124,17 @@ def sanitize(value: Any, *, max_text_chars: int) -> Any: if isinstance(value, str): text = _SECRET_ASSIGNMENT.sub(_redact_assignment, value) text = _BEARER_TOKEN.sub(lambda match: match.group(1) + "[REDACTED]", text) - return text if len(text) <= max_text_chars else text[:max_text_chars] + "...[truncated]" + if max_text_chars is None or len(text) <= max_text_chars: + return text + return text[:max_text_chars] + "...[truncated]" + return value + + +def validate_secret_free_text(value: str, *, name: str) -> str: + """Reject prompt content that would be changed by audit redaction.""" + + if sanitize(value, max_text_chars=None) != value: + raise ValueError(f"{name} contains credential-shaped content") return value diff --git a/examples/optimization/eval_optimize_loop/pipeline/trace_fixture.py b/examples/optimization/eval_optimize_loop/pipeline/trace_fixture.py new file mode 100644 index 000000000..ca53dbf24 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/trace_fixture.py @@ -0,0 +1,59 @@ +"""Hash-pinned trace fixture contract shared by preflight and replay.""" + +from __future__ import annotations + +import hashlib +from copy import deepcopy +from pathlib import Path +from typing import Optional + +from trpc_agent_sdk.evaluation import EvalSet + +from .models import Phase, Split +from .schema import parse_strict_json + + +class TraceFixture: + """Parse, pin and validate every phase/split in a trace fixture.""" + + def __init__( + self, + path: str | Path, + dataset_hashes: dict[str, str], + expected_hash: Optional[str] = None, + ) -> None: + self.path = Path(path) + self.dataset_hashes = dict(dataset_hashes) + if not self.path.is_file(): + raise FileNotFoundError(f"trace fixture does not exist: {self.path}") + content = self.path.read_bytes() + self.sha256 = hashlib.sha256(content).hexdigest() + if expected_hash is not None and self.sha256 != expected_hash: + raise ValueError("trace fixture changed after preflight") + self.payload = parse_strict_json(content.decode("utf-8")) + + def validate(self, train: EvalSet, validation: EvalSet) -> None: + for phase in Phase: + self.eval_set(train, Split.TRAIN, phase) + self.eval_set(validation, Split.VALIDATION, phase) + + def eval_set(self, eval_set: EvalSet, split: Split, phase: Phase) -> EvalSet: + if self.payload.get("schemaVersion") != "v1": + raise ValueError("unsupported trace fixture schemaVersion") + if self.payload.get("datasetHashes") != self.dataset_hashes: + raise ValueError("trace fixture dataset hashes do not match validated inputs") + try: + cases = self.payload["phases"][phase.value][split.value] + except (KeyError, TypeError) as error: + raise ValueError(f"trace fixture is missing {phase.value}/{split.value}") from error + expected_ids = [case.eval_id for case in eval_set.eval_cases] + if not isinstance(cases, dict) or set(cases) != set(expected_ids): + raise ValueError("trace fixture case IDs do not match the dataset") + raw = eval_set.model_dump(mode="json", by_alias=True, exclude_none=True) + for case in raw["evalCases"]: + conversation = deepcopy(cases[case["evalId"]]) + if not isinstance(conversation, list) or not conversation: + raise ValueError("each trace fixture conversation must be non-empty") + case["evalMode"] = "trace" + case["actualConversation"] = conversation + return EvalSet.model_validate(raw) diff --git a/tests/examples/eval_optimize_loop/test_attribution_gate.py b/tests/examples/eval_optimize_loop/test_attribution_gate.py index cebb63d02..14b1e53d9 100644 --- a/tests/examples/eval_optimize_loop/test_attribution_gate.py +++ b/tests/examples/eval_optimize_loop/test_attribution_gate.py @@ -200,6 +200,23 @@ def test_reference_free_failure_uses_metric_attribution_without_expected_trace() assert result.failures[0].primary == FailureCategory.LLM_RUBRIC_NOT_MET +@pytest.mark.parametrize( + ("reason", "expected"), + ( + ("response violates required JSON format schema", FailureCategory.FORMAT_VIOLATION), + ("knowledge retrieval recall is insufficient", FailureCategory.KNOWLEDGE_RECALL_INSUFFICIENT), + ("final answer mismatch", FailureCategory.FINAL_RESPONSE_MISMATCH), + ), +) +def test_reason_evidence_disambiguates_failures_from_the_same_metric(reason, expected) -> None: + result = attribute_failures( + _failed_snapshot("final_response_avg_score", reason=reason), + AttributionConfig(), + max_text_chars=200, + ) + assert result.failures[0].primary == expected + + def test_tool_difference_evidence_is_recursively_sanitized() -> None: result = attribute_failures( _failed_snapshot( @@ -353,6 +370,26 @@ def test_gate_rejects_train_only_improvement_as_overfit() -> None: assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in result.reasons +def test_overfit_guard_cannot_be_bypassed_by_permissive_thresholds() -> None: + validation = _comparison(Split.VALIDATION, score_delta=-0.1, pass_delta=0) + result = _gate( + validation=validation, + config=GateConfig( + min_validation_score_delta=-0.2, + min_validation_pass_rate_delta=-1, + metric_max_regression={"quality": 1}, + overfit_guard=True, + ), + ) + assert result.decision == Decision.REJECT + assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in result.reasons + + +def test_overfit_guard_cannot_be_disabled() -> None: + with pytest.raises(ValidationError, match="mandatory safety invariant"): + GateConfig(overfit_guard=False) + + def test_gate_fails_closed_when_enabled_cost_is_unknown() -> None: cost = CostSummary( sources=(CostSource(name="live-agent", cost_usd=None), ), diff --git a/tests/examples/eval_optimize_loop/test_backends.py b/tests/examples/eval_optimize_loop/test_backends.py index 06c7d23a3..37c74147e 100644 --- a/tests/examples/eval_optimize_loop/test_backends.py +++ b/tests/examples/eval_optimize_loop/test_backends.py @@ -677,7 +677,6 @@ async def fake_optimize(**kwargs): assert proposal.rounds[0].metric_scores == {"quality": 0.9} assert [(source.name, source.cost_usd, source.model_calls) for source in proposal.cost_sources] == [ ("optimizer_reported", 0, 1), - ("judge_unreported", None, None), ] @@ -778,3 +777,65 @@ async def fake_optimize(**kwargs): with pytest.raises(asyncio.CancelledError): await task assert stopped.is_set() + + +@pytest.mark.asyncio +async def test_live_worker_is_forcibly_terminated_after_bounded_cooperative_stop(monkeypatch, tmp_path) -> None: + + async def call_agent(query: str) -> str: + return query + + class StuckProcess: + + def __init__(self) -> None: + self.returncode = None + self.waiting = asyncio.Event() + self.finished = asyncio.Event() + self.terminated = False + self.killed = False + + async def wait(self): + self.waiting.set() + await self.finished.wait() + return self.returncode + + def terminate(self): + self.terminated = True + self.returncode = -15 + self.finished.set() + + def kill(self): + self.killed = True + self.returncode = -9 + self.finished.set() + + process = StuckProcess() + + async def fake_subprocess(*args, **kwargs): + return process + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_subprocess) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + output_dir = tmp_path / "optimizer-output" + task = asyncio.create_task( + LiveCandidateGenerator( + call_agent, + callback_spec="tests.agent:call_agent", + shutdown_timeout_seconds=0.01, + ).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(output_dir), + )) + await process.waiting.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert (output_dir / "optimize.stop").read_text(encoding="utf-8") == "cancel requested\n" + assert process.terminated is True + assert process.returncode in {-15, -9} diff --git a/tests/examples/eval_optimize_loop/test_deliverables.py b/tests/examples/eval_optimize_loop/test_deliverables.py index ef08e2c9f..806ac35af 100644 --- a/tests/examples/eval_optimize_loop/test_deliverables.py +++ b/tests/examples/eval_optimize_loop/test_deliverables.py @@ -60,7 +60,7 @@ def test_committed_sample_report_is_complete_and_rendered_from_the_model() -> No assert report.candidate and report.candidate.train and report.candidate.validation assert report.delta and report.delta.train and report.delta.validation assert report.gate_decision and report.gate_decision.decision == Decision.REJECT - assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in report.gate_decision.reasons + assert "NEW_HARD_FAIL_BUDGET_EXCEEDED" in report.gate_decision.reasons transitions = {case.transition for case in report.delta.validation.cases} assert transitions == { Transition.NEW_PASS, diff --git a/tests/examples/eval_optimize_loop/test_models_and_imports.py b/tests/examples/eval_optimize_loop/test_models_and_imports.py index 5dac83585..b6dc2c1dc 100644 --- a/tests/examples/eval_optimize_loop/test_models_and_imports.py +++ b/tests/examples/eval_optimize_loop/test_models_and_imports.py @@ -228,7 +228,7 @@ def test_pipeline_import_boundaries() -> None: allowed = { "artifacts": {"models", "schema"}, "attribution": {"configuration", "models", "schema"}, - "backends": {"contracts", "models", "offline_evaluation", "schema"}, + "backends": {"contracts", "models", "offline_evaluation", "schema", "trace_fixture"}, "candidate_runtime": { "artifacts", "attribution", @@ -253,11 +253,13 @@ def test_pipeline_import_boundaries() -> None: "gate": {"configuration", "models"}, "models": {"schema"}, "offline_evaluation": set(), + "optimizer_worker": {"schema"}, "orchestrator": { "artifacts", "attribution", "backends", "candidate_runtime", + "configuration", "contracts", "costing", "evaluation", @@ -269,10 +271,18 @@ def test_pipeline_import_boundaries() -> None: "reporting", "schema", }, - "preflight": {"artifacts", "configuration", "evaluation", "prompt_workspace", "schema"}, + "preflight": { + "artifacts", + "configuration", + "evaluation", + "prompt_workspace", + "schema", + "trace_fixture", + }, "prompt_workspace": {"schema"}, "reporting": {"artifacts", "configuration", "models"}, "schema": set(), + "trace_fixture": {"models", "schema"}, } graph: dict[str, set[str]] = {} for path in PIPELINE_DIR.glob("*.py"): diff --git a/tests/examples/eval_optimize_loop/test_orchestrator.py b/tests/examples/eval_optimize_loop/test_orchestrator.py index 7c49a9bf3..ef92ee70e 100644 --- a/tests/examples/eval_optimize_loop/test_orchestrator.py +++ b/tests/examples/eval_optimize_loop/test_orchestrator.py @@ -24,8 +24,16 @@ DeterministicCandidateGenerator, FakeEvaluationBackend, ) +from examples.optimization.eval_optimize_loop.pipeline.artifacts import ( + AuditPersistenceError, + AuditSink, +) +from examples.optimization.eval_optimize_loop.pipeline.costing import CostLedger +from examples.optimization.eval_optimize_loop.pipeline.evaluation_runtime import create_evaluation_runtime +from examples.optimization.eval_optimize_loop.pipeline.models import Phase, Split from examples.optimization.eval_optimize_loop.pipeline.orchestrator import run_pipeline from examples.optimization.eval_optimize_loop.pipeline.preflight import preflight_run +from examples.optimization.eval_optimize_loop.pipeline import orchestrator as orchestrator_module from examples.optimization.eval_optimize_loop.pipeline import reporting as reporting_module SOURCE = Path(__file__).resolve().parents[3] / "examples" / "optimization" / "eval_optimize_loop" @@ -49,7 +57,7 @@ def _example(tmp_path: Path, *, accept: bool = False, apply: bool = False) -> Pa @pytest.mark.asyncio -async def test_fake_reject_overfit_is_a_completed_audit_run(tmp_path) -> None: +async def test_fake_reject_with_hard_regression_is_a_completed_audit_run(tmp_path) -> None: root = _example(tmp_path) report = await run_pipeline(str(root), run_id="reject-run") assert report.status == Decision.REJECT @@ -58,7 +66,8 @@ async def test_fake_reject_overfit_is_a_completed_audit_run(tmp_path) -> None: Transition.NEW_FAIL, Transition.UNCHANGED, ] - assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in report.gate_decision.reasons + assert "NEW_HARD_FAIL_BUDGET_EXCEEDED" in report.gate_decision.reasons + assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" not in report.gate_decision.reasons assert report.source_application.applied is False actual_prompt = (root / "prompts" / "system.md").read_text(encoding="utf-8") expected_prompt = (SOURCE / "prompts" / "system.md").read_text(encoding="utf-8") @@ -318,6 +327,80 @@ async def generate(self, **kwargs): assert "[REDACTED]" in report.errors[0].message +@pytest.mark.asyncio +async def test_terminal_audit_failure_is_raised_instead_of_silently_returned(tmp_path, monkeypatch) -> None: + root = _example(tmp_path) + + class FailingGenerator: + + async def generate(self, **kwargs): + raise RuntimeError("optimizer unavailable") + + def fail_persistence(*args, **kwargs): + raise OSError("audit volume unavailable") + + monkeypatch.setattr(orchestrator_module, "persist_terminal_report", fail_persistence) + with pytest.raises(AuditPersistenceError, match="audit volume unavailable") as captured: + await run_pipeline( + str(root), + run_id="terminal-persistence-failure", + candidate_generator=FailingGenerator(), + ) + assert any("original pipeline error: RuntimeError" in note for note in captured.value.__notes__) + + +@pytest.mark.asyncio +async def test_live_evaluation_cost_bounds_are_accounted_conservatively(tmp_path) -> None: + root = _example(tmp_path) + config_path = root / "optimizer.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config["pipeline"].update({ + "mode": "live", + "liveAgentCallMaxCostUsd": 0.01, + "liveMetricCallMaxCostUsd": 0.02, + }) + config_path.write_text(json.dumps(config), encoding="utf-8") + + async def call_agent(query: str) -> str: + return query + + validated = preflight_run( + str(root), + config_path=None, + train_path=None, + validation_path=None, + mode=None, + run_id="live-cost-bounds", + apply_candidate=False, + call_agent=call_agent, + callback_spec="tests.agent:call_agent", + backend=None, + candidate_generator=None, + ) + sink = AuditSink(validated.artifact_root, validated.run_id) + sink.create() + ledger = CostLedger() + runtime = create_evaluation_runtime( + validated=validated, + backend=FakeEvaluationBackend(), + custom_backend=False, + sink=sink, + ledger=ledger, + ) + await runtime.evaluate( + eval_set=validated.train, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + source = ledger.summary().sources[0] + invocations = sum(len(case.conversation or case.actual_conversation or []) for case in validated.train.eval_cases) + expected = invocations * validated.config.evaluate.num_runs * 0.03 + assert source.cost_usd == pytest.approx(expected) + assert source.upper_bound is True + assert source.model_calls == invocations * validated.config.evaluate.num_runs + + @pytest.mark.asyncio async def test_generator_receives_only_inner_train_attribution(tmp_path) -> None: root = _example(tmp_path) @@ -430,6 +513,8 @@ def fake_run(args, **kwargs): stdout = "" elif args[-1] == "--show-toplevel": stdout = str(repo) + "\n" + elif args[1] == "ls-files": + stdout = args[-1] + "\n" else: raise AssertionError(args) return subprocess.CompletedProcess(args, 0, stdout=stdout, stderr="") @@ -444,6 +529,11 @@ def fake_run(args, **kwargs): run_id="exact-replay", apply_candidate=True, callback_spec="package.agent:call_agent", + input_paths=( + str(root / "custom config.json"), + str(root / "custom train.json"), + str(root / "custom validation.json"), + ), ) args = shlex.split(result.command) assert result.reproducible is True @@ -456,6 +546,52 @@ def fake_run(args, **kwargs): assert args[args.index("--call-agent") + 1] == "package.agent:call_agent" +@pytest.mark.parametrize( + ("input_path", "tracked", "expected_reason"), + ( + ("outside/input.json", True, "input_outside_git"), + ("repo/untracked.json", False, "input_not_tracked"), + ), +) +def test_reproducibility_requires_all_inputs_in_the_pinned_commit( + tmp_path, + monkeypatch, + input_path, + tracked, + expected_reason, +) -> None: + repo = tmp_path / "repo" + root = repo / "example" + root.mkdir(parents=True) + resolved_input = tmp_path / input_path + + def fake_run(args, **kwargs): + if args[-1] == "HEAD": + return subprocess.CompletedProcess(args, 0, stdout="a" * 40 + "\n", stderr="") + if args[-1] == "--porcelain": + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + if args[-1] == "--show-toplevel": + return subprocess.CompletedProcess(args, 0, stdout=str(repo) + "\n", stderr="") + if args[1] == "ls-files": + return subprocess.CompletedProcess(args, 0 if tracked else 1, stdout="", stderr="") + raise AssertionError(args) + + monkeypatch.setattr(reporting_module.subprocess, "run", fake_run) + result = reporting_module.build_reproducibility( + str(root), + mode="fake", + config_path=str(root / "optimizer.json"), + train_path=str(root / "train.json"), + validation_path=str(root / "validation.json"), + run_id="replay", + apply_candidate=False, + input_paths=(str(resolved_input), ), + ) + assert result.reproducible is False + assert result.reason == expected_reason + assert result.command is None + + @pytest.mark.asyncio async def test_generator_baseline_drift_is_rejected_at_stage_boundary(tmp_path) -> None: root = _example(tmp_path) diff --git a/tests/examples/eval_optimize_loop/test_workspace_artifacts.py b/tests/examples/eval_optimize_loop/test_workspace_artifacts.py index 5457bfa8d..bf1d68de5 100644 --- a/tests/examples/eval_optimize_loop/test_workspace_artifacts.py +++ b/tests/examples/eval_optimize_loop/test_workspace_artifacts.py @@ -14,6 +14,7 @@ from examples.optimization.eval_optimize_loop.pipeline.costing import CostLedger from examples.optimization.eval_optimize_loop.pipeline.prompt_workspace import ( PromptRestoreError, + PromptRunLock, PromptWorkspace, ) @@ -44,6 +45,19 @@ async def test_apply_is_verified_and_can_be_rolled_back(tmp_path) -> None: assert prompt_path.read_text(encoding="utf-8") == "baseline" +def test_prompt_run_lock_rejects_concurrent_owner_and_releases(tmp_path) -> None: + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + lock_root = str(tmp_path / "locks") + first = PromptRunLock((str(prompt_path), ), lock_root=lock_root) + second = PromptRunLock((str(prompt_path), ), lock_root=lock_root) + with first: + with pytest.raises(RuntimeError, match="already owned"): + second.acquire() + with second: + assert second.path == first.path + + @pytest.mark.asyncio async def test_restoration_failure_is_not_downgraded_to_reject() -> None: state = {"value": "baseline", "fail_restore": False} @@ -64,7 +78,7 @@ async def write(value: str) -> None: def test_audit_sink_is_immutable_safe_sanitized_and_manifested(tmp_path) -> None: - sink = AuditSink(tmp_path / "artifacts", "run-1", max_text_chars=64) + sink = AuditSink(tmp_path / "artifacts", "run-1") sink.create() sink.write_json( "config.json", @@ -92,7 +106,23 @@ def test_audit_sink_is_immutable_safe_sanitized_and_manifested(tmp_path) -> None assert record.sha256 == hashlib.sha256(content).hexdigest() assert record.byte_size == len(content) with pytest.raises(FileExistsError): - AuditSink(tmp_path / "artifacts", "run-1", max_text_chars=64).create() + AuditSink(tmp_path / "artifacts", "run-1").create() + + +def test_audit_sink_preserves_complete_text_and_fails_explicitly_on_size_limit(tmp_path) -> None: + content = "complete-audit-content:" + "x" * 256 + sink = AuditSink(tmp_path / "artifacts", "complete", max_file_bytes=1024) + sink.create() + sink.write_text("prompt.md", content) + sink.write_json("prompt.json", {"prompt": content}) + assert (sink.run_dir / "prompt.md").read_text(encoding="utf-8") == content + assert json.loads((sink.run_dir / "prompt.json").read_text(encoding="utf-8"))["prompt"] == content + + limited = AuditSink(tmp_path / "artifacts", "limited", max_file_bytes=64) + limited.create() + with pytest.raises(ValueError, match="audit file exceeds byte limit"): + limited.write_text("prompt.md", content) + assert not (limited.run_dir / "prompt.md").exists() def test_optimizer_tree_is_sanitized_before_audit_import(tmp_path) -> None: @@ -112,7 +142,7 @@ def test_optimizer_tree_is_sanitized_before_audit_import(tmp_path) -> None: "result: {'clientSecret': 'markdown-secret'}", encoding="utf-8", ) - sink = AuditSink(tmp_path / "artifacts", "run-2", max_text_chars=128) + sink = AuditSink(tmp_path / "artifacts", "run-2") sink.create() sink.import_tree(raw, "candidate_generation/optimizer") imported = "\n".join(path.read_text(encoding="utf-8") for path in sink.run_dir.rglob("*") if path.is_file()) @@ -158,7 +188,6 @@ def test_optimizer_tree_enforces_resource_budgets(tmp_path, sink_kwargs, files, sink = AuditSink( tmp_path / "artifacts", "run-budget", - max_text_chars=128, **sink_kwargs, ) sink.create() @@ -173,7 +202,6 @@ def test_concurrent_latest_publication_uses_collision_free_atomic_files(tmp_path sink = AuditSink( tmp_path / "artifacts", f"run-{index}", - max_text_chars=128, publication_root=publication_root, ) sink.create() @@ -205,3 +233,11 @@ def test_cost_ledger_preserves_unknown_sources() -> None: summary = ledger.summary() assert summary.total_cost_usd is None assert [source.name for source in summary.sources] == ["fake", "live-judge"] + + +def test_cost_ledger_preserves_upper_bound_basis() -> None: + ledger = CostLedger() + ledger.record("live-evaluation", cost_usd=0.5, upper_bound=True, model_calls=2) + summary = ledger.summary() + assert summary.total_cost_usd == 0.5 + assert summary.sources[0].upper_bound is True From 310c3b626fc9a59a9b02c27518fe6b7e94451edb Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Fri, 31 Jul 2026 01:13:23 +0800 Subject: [PATCH 05/13] docs(example): regenerate issue 91 sample report --- .../sample_output/optimization_report.json | 110 +++++++++--------- .../sample_output/optimization_report.md | 8 +- 2 files changed, 62 insertions(+), 56 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json index 3becb7ccf..995ef47da 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -8,17 +8,17 @@ { "byteSize": 13921, "path": "baseline_train/snapshot.json", - "sha256": "e3e4fa35e990241618df6d6151a1e8fc6e24ed1c98f5d19c349980964a489caf" + "sha256": "e1e7a1d29d8495b2f97d90ddef4179aca40c7da3cd7f28e7fac51b5207273489" }, { "byteSize": 13955, "path": "baseline_validation/snapshot.json", - "sha256": "6c22f2743db51fe0b604389cb3585ef8850f96c5ac40311a48a2198f77a4abfd" + "sha256": "c56620490167625ac3aa84b67bf7914b9bbeb17a8ecb56547ac28d2b4990d5d1" }, { - "byteSize": 1230, + "byteSize": 1257, "path": "candidate.json", - "sha256": "09d48df58df5b954757a9963665f61b36d981cda9c7a555b270e2d4e69dfdf53" + "sha256": "b426767863e0ec1d1b3e7310ad9e05809362926993428e83234ec6214a61fc17" }, { "byteSize": 132, @@ -26,24 +26,24 @@ "sha256": "ed2dfc103a7763a126f690e536572c5d06d20e2119ef7ef707c0a72deb162be2" }, { - "byteSize": 13857, + "byteSize": 13860, "path": "candidate_train/snapshot.json", - "sha256": "81619a1694caffa8e5661b53a6838c2347e7fc5bd4947f9c0dc17ba1fe894812" + "sha256": "0da1efea5c6dfc843ab03d09fc687724bb24596456c1d47692d4fd70a7afb6c7" }, { - "byteSize": 13956, + "byteSize": 13953, "path": "candidate_validation/snapshot.json", - "sha256": "ba0ec006116176cd6396212950b90d66f0c1b913775f0a9fb3451b845c5e40c7" + "sha256": "d7ba2a08412cb7adc5a82ccb408a5e9ad5da3564550f13ec2468389c0993cf9f" }, { - "byteSize": 5994, + "byteSize": 5986, "path": "comparison.json", - "sha256": "5ad875415c7ffad80abb3428be37d34e54d0ec69d09352f97375ab89ca821d89" + "sha256": "ce6e77862b58a753e4fe61fcf044161dd74b0194f15e61e9cb21c1e9896fd836" }, { - "byteSize": 2769, + "byteSize": 2924, "path": "config.json", - "sha256": "480c323b930dcece647f3db77038b12cd333457e8abb0983417f73c1aa5a8bde" + "sha256": "911831d3f66d621893f03ca83840d507d29365c18a6b2c6dbbb7911d656b632e" }, { "byteSize": 2163, @@ -51,9 +51,9 @@ "sha256": "af935deba41f9e95db01dcb4b8db86c17619a2f205d6dddd0addad2415896715" }, { - "byteSize": 647, + "byteSize": 645, "path": "inner_train.attribution.json", - "sha256": "c6de4971edada2345c566bfe8a39258f3585bc716c6a0294b8d5d8288af0f46b" + "sha256": "77478a288c3f38a77677e169e5da7d1c88509c0e3091e0989c70a26a1c216aa4" }, { "byteSize": 4108, @@ -109,7 +109,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.1023717, + "creationTimestamp": 1785431480.1068149, "finalResponse": { "parts": [ { @@ -242,7 +242,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.1013703, + "creationTimestamp": 1785431480.1068149, "finalResponse": { "parts": [ { @@ -375,7 +375,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.1013703, + "creationTimestamp": 1785431480.1068149, "finalResponse": { "parts": [ { @@ -524,7 +524,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.1065273, + "creationTimestamp": 1785431480.1110127, "finalResponse": { "parts": [ { @@ -657,7 +657,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.1065273, + "creationTimestamp": 1785431480.1110127, "finalResponse": { "parts": [ { @@ -790,7 +790,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.1065273, + "creationTimestamp": 1785431480.1110127, "finalResponse": { "parts": [ { @@ -941,7 +941,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.120922, + "creationTimestamp": 1785431480.1273332, "finalResponse": { "parts": [ { @@ -1074,7 +1074,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.119919, + "creationTimestamp": 1785431480.1273332, "finalResponse": { "parts": [ { @@ -1207,7 +1207,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.119919, + "creationTimestamp": 1785431480.1273332, "finalResponse": { "parts": [ { @@ -1356,7 +1356,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.1244266, + "creationTimestamp": 1785431480.131333, "finalResponse": { "parts": [ { @@ -1489,7 +1489,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.1244266, + "creationTimestamp": 1785431480.131333, "finalResponse": { "parts": [ { @@ -1622,7 +1622,7 @@ "trace": [ { "actual": { - "creationTimestamp": 1785423818.1244266, + "creationTimestamp": 1785431480.131333, "finalResponse": { "parts": [ { @@ -1749,7 +1749,7 @@ "failures": [ { "caseId": "validation_regress", - "confidence": "medium", + "confidence": "high", "evidence": [ "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" ], @@ -1781,35 +1781,40 @@ "metricCalls": 3, "modelCalls": 3, "name": "baseline_train", - "tokenUsage": {} + "tokenUsage": {}, + "upperBound": false }, { "costUsd": 0.0, "metricCalls": 3, "modelCalls": 3, "name": "baseline_validation", - "tokenUsage": {} + "tokenUsage": {}, + "upperBound": false }, { "costUsd": 0.0, "metricCalls": 0, "modelCalls": 0, "name": "candidate_generation.deterministic", - "tokenUsage": {} + "tokenUsage": {}, + "upperBound": false }, { "costUsd": 0.0, "metricCalls": 3, "modelCalls": 3, "name": "candidate_train", - "tokenUsage": {} + "tokenUsage": {}, + "upperBound": false }, { "costUsd": 0.0, "metricCalls": 3, "modelCalls": 3, "name": "candidate_validation", - "tokenUsage": {} + "tokenUsage": {}, + "upperBound": false } ], "totalCostUsd": 0.0 @@ -1820,7 +1825,7 @@ { "baselineAttribution": { "caseId": "train_improve_alpha", - "confidence": "medium", + "confidence": "high", "evidence": [ "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" ], @@ -1857,7 +1862,7 @@ { "baselineAttribution": { "caseId": "train_improve_beta", - "confidence": "medium", + "confidence": "high", "evidence": [ "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" ], @@ -1926,7 +1931,7 @@ { "baselineAttribution": { "caseId": "validation_improve", - "confidence": "medium", + "confidence": "high", "evidence": [ "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" ], @@ -1966,7 +1971,7 @@ "baselineScore": 1.0, "candidateAttribution": { "caseId": "validation_regress", - "confidence": "medium", + "confidence": "high", "evidence": [ "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" ], @@ -2028,14 +2033,14 @@ "split": "validation" } }, - "durationSeconds": 0.2190000000045984, + "durationSeconds": 0.3899999999994179, "errors": [], "failureAttribution": { "train": { "failures": [ { "caseId": "train_improve_alpha", - "confidence": "medium", + "confidence": "high", "evidence": [ "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" ], @@ -2051,7 +2056,7 @@ }, { "caseId": "train_improve_beta", - "confidence": "medium", + "confidence": "high", "evidence": [ "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" ], @@ -2079,7 +2084,7 @@ "failures": [ { "caseId": "validation_improve", - "confidence": "medium", + "confidence": "high", "evidence": [ "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" ], @@ -2104,7 +2109,7 @@ } } }, - "finishedAt": "2026-07-30T15:03:38.207789+00:00", + "finishedAt": "2026-07-30T17:11:20.215369+00:00", "gateDecision": { "accepted": false, "checks": [ @@ -2169,26 +2174,26 @@ { "code": "DURATION_BUDGET", "message": "Gate-observed duration stays within budget.", - "observed": 0.14100000000325963, + "observed": 0.31199999999807915, "passed": true, "threshold": 180.0 }, { "code": "OVERFIT_GUARD", - "message": "Train-only improvement is rejected as overfitting.", + "message": "Training improvement with held-out regression is rejected as overfitting.", "observed": { "trainScoreDelta": 0.6666666666666667, + "validationPassRateDelta": 0.0, "validationScoreDelta": 0.0 }, - "passed": false, - "threshold": "train improvement requires held-out validation thresholds" + "passed": true, + "threshold": "train improvement cannot accompany held-out validation regression" } ], "decision": "REJECT", "reasons": [ "VALIDATION_SCORE_DELTA_BELOW_MINIMUM", - "NEW_HARD_FAIL_BUDGET_EXCEEDED", - "OVERFIT_TRAIN_UP_VALIDATION_DOWN" + "NEW_HARD_FAIL_BUDGET_EXCEEDED" ] }, "inputs": { @@ -2199,8 +2204,8 @@ "sdk": "1.1.14" }, "hashes": { - "config": "4f1210691dcdd12c1d97a9c73e30920a39dc27c4704ec345c94717728c78637c", - "effectiveConfig": "c4525545639ffa945a68c8b601b54cd3328f70d435b2dc147807f380f70f03b0", + "config": "fef5828244ea4f0058084dc427d5c42f60ffa004ee3e3ac38bee94ba08f0667e", + "effectiveConfig": "69cbd345d2e3868ab5baf39f73901fa51e137a59289bdebf6e10d31086d997e1", "train": "db8df56423d916afa929a2fc1a6f7245d8a1f719e87ac292f88f346d1cb81789", "validation": "499d3bde9cdc70cc4871d425d0dfee83ed30a67d35da15a4eb87cc9549583f84" }, @@ -2238,7 +2243,8 @@ "metricCalls": 0, "modelCalls": 0, "name": "deterministic", - "tokenUsage": {} + "tokenUsage": {}, + "upperBound": false } ], "durationSeconds": 0.0, @@ -2286,7 +2292,7 @@ }, "reproducibility": { "command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --config examples/optimization/eval_optimize_loop/optimizer.json --train examples/optimization/eval_optimize_loop/train.evalset.json --validation examples/optimization/eval_optimize_loop/val.evalset.json --run-id sample-output-v1-replay --no-apply-candidate", - "gitCommit": "ceb6ff81a7a499689ddef85f0d0c997d2e54a505", + "gitCommit": "858b196edcfb7d10b8e7283625177ad72fbaf8f4", "gitDirty": false, "reason": null, "reproducible": true @@ -2304,6 +2310,6 @@ "requested": false }, "stage": "complete", - "startedAt": "2026-07-30T15:03:37.984886+00:00", + "startedAt": "2026-07-30T17:11:19.833935+00:00", "status": "REJECT" } diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md index aa6a125d0..0372b238d 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md @@ -3,7 +3,7 @@ - Run ID: `sample-output-v1` - Mode: `fake` - Final stage: `complete` -- Duration: `0.219s` +- Duration: `0.390s` - Reproducible: `yes` - Source prompt applied: `no` @@ -23,9 +23,9 @@ ## Gate - Decision: `REJECT` -- Failed checks: `VALIDATION_SCORE_DELTA, NO_NEW_HARD_FAIL, OVERFIT_GUARD` -- Reasons: `VALIDATION_SCORE_DELTA_BELOW_MINIMUM, NEW_HARD_FAIL_BUDGET_EXCEEDED, OVERFIT_TRAIN_UP_VALIDATION_DOWN` -- Overfit detected: `yes` +- Failed checks: `VALIDATION_SCORE_DELTA, NO_NEW_HARD_FAIL` +- Reasons: `VALIDATION_SCORE_DELTA_BELOW_MINIMUM, NEW_HARD_FAIL_BUDGET_EXCEEDED` +- Overfit detected: `no` ## Cost From d39cd7163102c7b0e3774c7612a9a9db00df6f72 Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Fri, 31 Jul 2026 01:22:05 +0800 Subject: [PATCH 06/13] test(example): cover isolated optimizer worker success --- .../optimization/eval_optimize_loop/DESIGN.md | 13 +++ .../eval_optimize_loop/test_backends.py | 83 +++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index 90559672a..2d94ef206 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -75,6 +75,19 @@ compatibility shim reconstructs the discarded `v1` information. ## Lifecycle +```text +validated = preflight(config, train, validation, prompts, trace, callback_source) +with prompt_set_lock(validated.prompt_paths): + baseline = evaluate(train, validation) + failures = attribute(structured_evidence, explicit_maps, reason_semantics, metric_fallback) + candidate = optimizer_worker(inner_train(failures), inner_selection) + regression = compare(evaluate(candidate, train, validation), baseline) + decision = hard_overfit_guard(regression) -> configured_gate(regression, cost, duration) + persist_complete_redacted_audit_or_raise() + apply_verified(candidate) only when decision == ACCEPT and explicitly_authorized +on cancel: request_stop -> bounded_wait -> terminate -> bounded_wait -> kill +``` + 1. Preflight strictly parses config and datasets, rejects duplicate JSON keys, validates split isolation, hashes every input, and validates the complete trace phase/split/case matrix without side effects. diff --git a/tests/examples/eval_optimize_loop/test_backends.py b/tests/examples/eval_optimize_loop/test_backends.py index 37c74147e..e81b538df 100644 --- a/tests/examples/eval_optimize_loop/test_backends.py +++ b/tests/examples/eval_optimize_loop/test_backends.py @@ -11,6 +11,7 @@ from trpc_agent_sdk.evaluation import AgentEvaluator, EvalConfig, EvalSet, TargetPrompt +from examples.optimization.eval_optimize_loop.pipeline import backends as backends_module from examples.optimization.eval_optimize_loop.pipeline.backends import ( DeterministicCandidateGenerator, FakeEvaluationBackend, @@ -839,3 +840,85 @@ async def fake_subprocess(*args, **kwargs): assert (output_dir / "optimize.stop").read_text(encoding="utf-8") == "cancel requested\n" assert process.terminated is True assert process.returncode in {-15, -9} + + +@pytest.mark.asyncio +async def test_live_worker_success_is_loaded_through_the_strict_candidate_adapter(monkeypatch, tmp_path) -> None: + + async def call_agent(query: str) -> str: + return query + + round_ = SimpleNamespace( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=True, + validation_pass_rate=0.8, + kind="reflective", + optimized_field_names=["system"], + metric_breakdown={"quality": 0.8}, + acceptance_reason="improved selection score", + skip_reason=None, + error_message=None, + round_llm_cost=0.03, + round_token_usage={"total": 8}, + duration_seconds=0.2, + ) + result = SimpleNamespace( + status="SUCCEEDED", + error_message="", + rounds=[round_], + algorithm="gepa_reflective", + baseline_prompts={"system": "baseline"}, + best_prompts={"system": "candidate"}, + stop_reason="completed", + total_reflection_lm_calls=1, + total_llm_cost=0.03, + total_token_usage={"total": 8}, + duration_seconds=0.2, + ) + captured = {} + + class CompletedProcess: + returncode = 0 + + async def wait(self): + return self.returncode + + async def fake_subprocess(*args, **kwargs): + request_path = Path(args[-1]) + request = json.loads(request_path.read_text(encoding="utf-8")) + output_dir = Path(request["outputDir"]) + output_dir.mkdir(parents=True) + (output_dir / "result.json").write_text("{}", encoding="utf-8") + captured.update({"args": args, "kwargs": kwargs, "request": request}) + return CompletedProcess() + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_subprocess) + monkeypatch.setattr( + backends_module, + "OptimizeResult", + SimpleNamespace(from_file=lambda path: result), + ) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + proposal = await LiveCandidateGenerator( + call_agent, + callback_spec="tests.agent:call_agent", + ).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + ) + assert captured["args"][1:3] == ( + "-m", + "examples.optimization.eval_optimize_loop.pipeline.optimizer_worker", + ) + assert captured["request"]["callbackSpec"] == "tests.agent:call_agent" + assert captured["request"]["promptPaths"] == {"system": str(prompt_path)} + assert proposal.prompts == {"system": "candidate"} + assert proposal.rounds[0].metric_scores == {"quality": 0.8} + assert proposal.cost_sources[0].cost_usd == 0.03 From f40d22a5e4b8a1d761cb461d043639af4ce832b6 Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Fri, 31 Jul 2026 11:11:34 +0800 Subject: [PATCH 07/13] fix(example): harden evaluation optimization contracts --- .../optimization/eval_optimize_loop/DESIGN.md | 57 ++- .../optimization/eval_optimize_loop/README.md | 22 +- .../eval_optimize_loop/SOLUTION.md | 4 +- .../eval_optimize_loop/optimizer.json | 3 +- .../eval_optimize_loop/pipeline/backends.py | 94 ++--- .../pipeline/candidate_runtime.py | 2 +- .../pipeline/configuration.py | 11 +- .../eval_optimize_loop/pipeline/gate.py | 4 +- .../pipeline/live_adapter.py | 88 +++++ .../eval_optimize_loop/pipeline/models.py | 10 +- .../pipeline/optimizer_worker.py | 19 +- .../pipeline/orchestrator.py | 54 +-- .../eval_optimize_loop/pipeline/preflight.py | 58 ++- .../eval_optimize_loop/pipeline/reporting.py | 15 +- .../eval_optimize_loop/run_pipeline.py | 14 +- .../test_attribution_gate.py | 18 +- .../eval_optimize_loop/test_backends.py | 330 +++++++++++------- .../test_models_and_imports.py | 10 +- .../eval_optimize_loop/test_orchestrator.py | 144 +++++++- 19 files changed, 610 insertions(+), 347 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/pipeline/live_adapter.py diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index 2d94ef206..1687b0ff6 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -9,12 +9,14 @@ The dependency direction is flat and one-way: ```text CLI -> orchestrator -> preflight / evaluation_runtime / candidate_runtime / reporting -configuration -> models / schema +configuration -> live_adapter / models / schema models -> schema evaluation_runtime -> backend contract / normalizer / cost ledger / artifact sink candidate_runtime -> generator contract / split policy / prompt workspace / artifact sink -backends -> offline_evaluation / trace_fixture / contracts / models -optimizer_worker -> schema +backends -> offline_evaluation / trace_fixture / live_adapter / contracts / models +live_adapter -> none +optimizer_worker -> live_adapter / schema +preflight -> live_adapter trace_fixture -> models / schema offline_evaluation -> SDK evaluation types pure policies -> models @@ -82,9 +84,12 @@ with prompt_set_lock(validated.prompt_paths): failures = attribute(structured_evidence, explicit_maps, reason_semantics, metric_fallback) candidate = optimizer_worker(inner_train(failures), inner_selection) regression = compare(evaluate(candidate, train, validation), baseline) - decision = hard_overfit_guard(regression) -> configured_gate(regression, cost, duration) - persist_complete_redacted_audit_or_raise() - apply_verified(candidate) only when decision == ACCEPT and explicitly_authorized + preliminary = hard_overfit_guard(regression) -> configured_gate(regression, cost, duration) + persist_pre_apply_audit_or_raise() + apply_verified(candidate) only when preliminary == ACCEPT and explicitly_authorized + terminal = configured_gate(regression, cost, current_duration) + restore_verified_baseline() if terminal rejects an applied candidate + persist_terminal_audit_once_or_raise() on cancel: request_stop -> bounded_wait -> terminate -> bounded_wait -> kill ``` @@ -99,12 +104,19 @@ on cancel: request_stop -> bounded_wait -> terminate -> bounded_wait -> kill attribution is passed to the candidate generator. 5. The candidate is generated with source updates disabled, then evaluated on full train and held-out validation inside a verified temporary prompt context. -6. Pure comparison and gate functions produce the only business decision. +6. Pure comparison and gate functions produce the only business decision. The + non-configurable overfit guard treats either train score or train pass-rate + improvement combined with either validation score or pass-rate regression as + a rejection. 7. A report is persisted before optional apply. Apply occurs only on ACCEPT when explicitly enabled, and every write is read back and hash verified. -8. A complete terminal write cycle is measured before the final duration is - recorded. The report excludes its own files from its embedded artifact list; - the manifest hashes the final JSON and Markdown files. +8. The terminal gate and report duration are sampled at the final decision + boundary. If that decision changes to REJECT after an apply, the baseline is + restored before publication. The terminal report, manifest, JSON snapshot, + and Markdown snapshot are then published once. Audit I/O is outside the + business duration gate; the 180-second end-to-end requirement is measured by + the acceptance runner. The report excludes its own files from its embedded + artifact list; the manifest hashes the final JSON and Markdown files. The immutable `artifacts//` directory and its manifest are the authority. Root-level report files are atomic latest-run snapshots for interactive use. They @@ -123,14 +135,23 @@ The import boundary enforces file-count, per-file-byte and total-byte budgets before reading optimizer output. This boundary protects audit storage; it is not a sandbox for a programmatic generator, which is trusted in-process code. -CLI live optimization uses `optimizer_worker.py`. Cancellation writes the SDK -stop signal, waits for the configured bound, then terminates and finally kills a -worker that still does not exit. Programmatically injected generators remain -trusted in-process components and make the report non-reproducible. Terminal -report persistence failures raise `AuditPersistenceError`; they are never -discarded while returning an apparently handled pipeline error. Audit writes -redact credentials without truncation and fail before publication when the -configured file-byte ceiling is exceeded. +Default live evaluation and optimization resolve one importable callback through +`LiveAdapterSpec`; preflight binds its source path, file SHA-256, and callable code +fingerprint to the run, and the worker verifies all three after re-importing it. A +different callback object, source, cached code version, or source version is +rejected before optimization. Live optimization always uses +`optimizer_worker.py`. Cancellation +writes the SDK stop signal, waits for the configured bound, then terminates and +finally kills a worker that still does not exit. Programmatically injected +backends and generators remain trusted in-process components and make the report +non-reproducible. A successful SDK result must report the exact workspace +baseline and registered best-prompt keys before it can enter regression. Failed +or canceled SDK results retain their structured rounds, duration, error and cost +facts in the terminal report. Terminal report +persistence failures raise `AuditPersistenceError`; they are never discarded +while returning an apparently handled pipeline error. Audit writes redact +credentials without truncation and fail before publication when the configured +file-byte ceiling is exceeded. A replay claim is emitted only for a clean, pinned Git commit when every effective config, dataset, prompt, trace and live callback source is inside that diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 22abd446d..781b136a2 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -4,10 +4,10 @@ This example runs a baseline on train and held-out validation data, attributes failures, generates a candidate from an inner training split, reruns full regression, and makes a deterministic gate decision. Every completed run writes an immutable, authoritative audit directory under `artifacts//`. The -generated `optimization_report.json` and `optimization_report.md` files in this -directory are ignored atomic latest-run convenience snapshots, not a -transactional report pair. Consumers that need a coherent pair must read the JSON -run ID and then use the two immutable files and manifest under +root-level generated `optimization_report.json` and `optimization_report.md` +files are ignored atomic latest-run convenience snapshots, not a transactional +report pair. Consumers that need a coherent pair must read the JSON run ID and +then use the two immutable files and manifest under `artifacts//`. The default fake mode is deterministic, offline, and does not read API keys: @@ -32,9 +32,11 @@ fake black-box mode fail fast because they cannot be scored deterministically. Reports retain each rubric's ID, score, pass status, and reason. Failure attribution uses only rubric outcomes that failed their metric threshold. -Live mode requires an importable async `query -> str` callback. The optimizer is -run in a supervised subprocess and always called with `update_source=False`; -only this pipeline's gate may apply a candidate: +Live mode requires one importable async `query -> str` callback. Preflight binds +its resolved source path, file SHA-256, and callable code fingerprint to the run; +the optimizer worker reloads the callback and verifies all three before use. The +optimizer is run in a supervised subprocess and always called with +`update_source=False`; only this pipeline's gate may apply a candidate: ```bash python examples/optimization/eval_optimize_loop/run_pipeline.py \ @@ -62,7 +64,11 @@ cancellation first requests cooperative shutdown, then terminates the isolated worker after `optimizerShutdownTimeoutSeconds`. Audit values are redacted but never silently truncated; a report exceeding `maxAuditFileBytes` fails explicitly. Concurrent runs targeting the same prompt files are rejected by a -cross-process lock. +cross-process lock. Failed optimizer runs retain reported rounds, costs, duration, +and error facts. The terminal decision and duration are finalized before one +terminal audit publication, so an applied prompt cannot be paired with a stale +ACCEPT report. A successful optimizer result is rejected before regression when +its reported baseline or best-prompt key set differs from the pipeline workspace. See [SOLUTION.md](SOLUTION.md) for the concise Chinese issue proposal and [DESIGN.md](DESIGN.md) for detailed stage contracts and failure semantics. A diff --git a/examples/optimization/eval_optimize_loop/SOLUTION.md b/examples/optimization/eval_optimize_loop/SOLUTION.md index 191c3271f..697ccdcd1 100644 --- a/examples/optimization/eval_optimize_loop/SOLUTION.md +++ b/examples/optimization/eval_optimize_loop/SOLUTION.md @@ -2,6 +2,6 @@ 本方案把评测、归因、优化、回归和审计组织为单向流水线。预检严格解析并哈希配置、数据集、prompt 与 trace，验证训练/验证隔离和完整 trace 矩阵；只有全部有效输入受同一干净 Git 提交管理时才声明可复现。基线阶段用 AgentEvaluator 分别运行训练集与验证集，保留逐用例、逐运行、逐指标分数、阈值、原因和关键轨迹。失败归因按结构证据、显式 rubric/指标映射、原因语义和指标兜底的顺序决策，每个失败输出主类别、次类别、证据与置信度。 -候选生成只接收训练集内部切分的失败事实，不读取外部验证集；live 优化运行在可终止 worker 中，取消先协作停止，超时再终止进程。候选完成后重跑完整训练集和保留验证集，逐用例标记新增通过、新增失败、提升、下降或不变。接受门禁依次检查验证总分与通过率、新增 hard fail、关键用例、指标回退、成本和耗时；训练提升而验证下降是不可配置的过拟合拒绝条件，未知成本在启用预算时失败关闭，live 评测可使用显式单次调用上界。 +候选生成只接收训练集内部切分的失败事实，不读取外部验证集；live 评测与优化绑定同一回调身份，优化运行在可终止 worker 中，取消先协作停止，超时再终止进程。候选完成后重跑完整训练集和保留验证集，逐用例标记新增通过、新增失败、提升、下降或不变。接受门禁依次检查验证总分与通过率、新增 hard fail、关键用例、指标回退、成本和耗时；训练分数或通过率提升而验证下降是不可配置的过拟合拒绝条件，未知成本在启用预算时失败关闭，live 评测可使用显式单次调用上界。 -同一组 prompt 由跨进程锁串行保护，默认不回写；仅门禁接受且显式授权时原子应用、回读和验哈希，异常或取消必须恢复基线。审计保存完整候选轮次、成本来源、耗时、种子、门禁理由和产物哈希，只脱敏而不静默截断，超出文件预算或终态落盘失败会显式报错。fake/trace 使用确定性规则且不读真实密钥，最终生成 JSON、Markdown 和清单，支持复现、比较与追责。 +prompt 由跨进程锁保护，默认不回写；仅门禁接受且显式授权时原子应用、回读和验哈希，终态拒绝、异常或取消必须恢复基线。审计一次发布终态，保存各终态完整候选轮次、成本来源、耗时、种子、门禁理由和产物哈希，只脱敏而不静默截断，超出文件预算或终态落盘失败会显式报错。fake/trace 使用确定性规则且不读真实密钥，最终生成 JSON、Markdown 和清单，支持复现、比较与追责。 diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json index e3281b968..202118203 100644 --- a/examples/optimization/eval_optimize_loop/optimizer.json +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -79,8 +79,7 @@ }, "maxCostUsd": null, "maxDurationSeconds": 180.0, - "epsilon": 1e-09, - "overfitGuard": true + "epsilon": 1e-09 } } } diff --git a/examples/optimization/eval_optimize_loop/pipeline/backends.py b/examples/optimization/eval_optimize_loop/pipeline/backends.py index e5db93c8a..060ab81c5 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/backends.py +++ b/examples/optimization/eval_optimize_loop/pipeline/backends.py @@ -12,7 +12,6 @@ from trpc_agent_sdk.evaluation import ( AgentEvaluator, - AgentOptimizer, EvalConfig, EvalSet, EvalSetAggregateResult, @@ -30,6 +29,7 @@ ) from .contracts import CandidateGenerator, EvaluationBackend +from .live_adapter import LiveAdapterSpec from .models import ( AttributionSnapshot, CandidateProposal, @@ -314,7 +314,7 @@ async def generate( cost_sources=(CostSource(name="deterministic", cost_usd=0, model_calls=0, metric_calls=0), ), duration_seconds=0, ) - return CandidateProposal.model_validate(proposal.model_dump(mode="python", by_alias=True)) + return proposal class LiveCandidateGenerator: @@ -322,18 +322,14 @@ class LiveCandidateGenerator: def __init__( self, - call_agent: CallAgent, + live_adapter: LiveAdapterSpec, *, - callback_spec: Optional[str] = None, shutdown_timeout_seconds: float = 10.0, verbose: int = 0, ) -> None: - if not inspect.iscoroutinefunction(call_agent): - raise TypeError("live call_agent must be an async function") if shutdown_timeout_seconds <= 0: raise ValueError("optimizer shutdown timeout must be positive") - self._call_agent = call_agent - self._callback_spec = callback_spec + self._live_adapter = live_adapter self._shutdown_timeout_seconds = shutdown_timeout_seconds self._verbose = verbose @@ -346,42 +342,6 @@ def _request_stop(output_dir: str, cancellation: BaseException) -> None: except OSError as stop_error: add_exception_note(cancellation, f"could not request optimizer stop: {stop_error}") - async def _optimize_in_process(self, **kwargs) -> OptimizeResult: - optimize_task = asyncio.create_task(AgentOptimizer.optimize( - **kwargs, - call_agent=self._call_agent, - )) - try: - return await asyncio.shield(optimize_task) - except asyncio.CancelledError as cancellation: - self._request_stop(kwargs["output_dir"], cancellation) - try: - await asyncio.wait_for( - asyncio.shield(optimize_task), - timeout=self._shutdown_timeout_seconds, - ) - except asyncio.TimeoutError: - add_exception_note( - cancellation, - "programmatic optimizer did not stop before its shutdown timeout; " - "use an importable callback spec for process isolation", - ) - optimize_task.add_done_callback(self._consume_task_result) - except BaseException as shutdown_error: - add_exception_note( - cancellation, - f"optimizer shutdown completed with {type(shutdown_error).__name__}: " - f"{shutdown_error}", - ) - raise - - @staticmethod - def _consume_task_result(task: asyncio.Task) -> None: - try: - task.result() - except BaseException: - pass - async def _stop_worker( self, process: asyncio.subprocess.Process, @@ -423,14 +383,16 @@ async def _stop_worker( add_exception_note(cancellation, "optimizer worker required forced termination") async def _optimize_in_worker(self, **kwargs) -> OptimizeResult: - assert self._callback_spec is not None output_dir = kwargs["output_dir"] request_path = Path(output_dir).parent / "optimizer-worker-request.json" prompt_paths = {name: kwargs["target_prompt"].describe_source(name) for name in kwargs["target_prompt"].names()} request_path.write_text( json.dumps( { - "callbackSpec": self._callback_spec, + "callbackSpec": self._live_adapter.import_path, + "callbackSourcePath": str(self._live_adapter.source_path), + "callbackSourceSha256": self._live_adapter.source_sha256, + "callbackCallableSha256": self._live_adapter.callable_sha256, "promptPaths": prompt_paths, "configPath": kwargs["config_path"], "trainPath": kwargs["train_dataset_path"], @@ -494,12 +456,17 @@ async def generate( "update_source": False, "verbose": self._verbose, } - result = (await self._optimize_in_worker( - **optimize_kwargs) if self._callback_spec else await self._optimize_in_process(**optimize_kwargs)) - if result.status == "CANCELED": - raise asyncio.CancelledError(result.error_message or "AgentOptimizer canceled") - if result.status != "SUCCEEDED": - raise RuntimeError(f"AgentOptimizer candidate generation failed: {result.error_message}") + result = await self._optimize_in_worker(**optimize_kwargs) + reported_baseline = dict(result.baseline_prompts) + result_prompts = dict(result.best_prompts) + adapter_errors: list[str] = [] + if result.status == "SUCCEEDED": + if reported_baseline != baseline_prompts: + adapter_errors.append("optimizer baseline prompts differ from the workspace baseline") + if set(result_prompts) != set(baseline_prompts): + adapter_errors.append("optimizer best prompt keys differ from the workspace schema") + if result.status != "SUCCEEDED" or adapter_errors: + result_prompts = dict(baseline_prompts) rounds = tuple( CandidateRound( round=round_.round, @@ -518,10 +485,13 @@ async def generate( duration_seconds=round_.duration_seconds, ) for round_ in result.rounds) proposal = CandidateProposal( + status=result.status, + error_message=result.error_message or None, + adapter_error="; ".join(adapter_errors) or None, algorithm=result.algorithm, - baseline_prompts=dict(result.baseline_prompts), - prompts=dict(result.best_prompts), - changed=result.best_prompts != baseline_prompts, + baseline_prompts=dict(baseline_prompts), + prompts=result_prompts, + changed=result_prompts != baseline_prompts, stop_reason=result.stop_reason, rounds=rounds, cost_sources=(CostSource( @@ -532,17 +502,16 @@ async def generate( ), ), duration_seconds=result.duration_seconds, ) - return CandidateProposal.model_validate(proposal.model_dump(mode="python", by_alias=True)) + return proposal def create_backends( mode: str, *, - call_agent: Optional[CallAgent] = None, + live_adapter: Optional[LiveAdapterSpec] = None, trace_fixture_path: Optional[str] = None, trace_fixture_hash: Optional[str] = None, dataset_hashes: Optional[dict[str, str]] = None, - callback_spec: Optional[str] = None, optimizer_shutdown_timeout_seconds: float = 10.0, ) -> tuple[EvaluationBackend, CandidateGenerator]: if mode == "fake": @@ -555,13 +524,12 @@ def create_backends( DeterministicCandidateGenerator(), ) if mode == "live": - if call_agent is None: - raise ValueError("live mode requires call_agent") + if live_adapter is None: + raise ValueError("live mode requires a validated LiveAdapterSpec") return ( - LiveEvaluationBackend(call_agent), + LiveEvaluationBackend(live_adapter.callback), LiveCandidateGenerator( - call_agent, - callback_spec=callback_spec, + live_adapter, shutdown_timeout_seconds=optimizer_shutdown_timeout_seconds, ), ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py b/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py index 541acce4c..06da40312 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py +++ b/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py @@ -115,7 +115,7 @@ async def generate_candidate( raise else: sink.import_tree(optimizer_output, "candidate_generation/optimizer") - validated = CandidateProposal.model_validate(proposal.model_dump(mode="python", by_alias=True)) + validated = CandidateProposal.model_validate(proposal) if validated.baseline_prompts != workspace.baseline: raise ValueError("candidate generator baseline prompts differ from the workspace baseline") for name, text in validated.prompts.items(): diff --git a/examples/optimization/eval_optimize_loop/pipeline/configuration.py b/examples/optimization/eval_optimize_loop/pipeline/configuration.py index 38cd68d9e..100468098 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/configuration.py +++ b/examples/optimization/eval_optimize_loop/pipeline/configuration.py @@ -15,6 +15,7 @@ from trpc_agent_sdk.evaluation import EvalConfig, EvalSet, OptimizeConfig +from .live_adapter import LiveAdapterSpec from .models import FailureCategory, RunMode from .schema import ( StrictModel, @@ -37,7 +38,6 @@ class GateConfig(StrictModel): max_cost_usd: Optional[StrictFloat] = None max_duration_seconds: Optional[StrictFloat] = 180.0 epsilon: StrictFloat = 1e-9 - overfit_guard: StrictBool = True @field_validator( "min_validation_score_delta", @@ -58,13 +58,6 @@ def valid_epsilon(cls, value: float) -> float: def valid_budget(cls, value: Optional[float]) -> Optional[float]: return None if value is None else _non_negative(value, "budget") - @field_validator("overfit_guard") - @classmethod - def overfit_guard_is_mandatory(cls, value: bool) -> bool: - if not value: - raise ValueError("overfit_guard is a mandatory safety invariant") - return value - @field_validator("metric_max_regression") @classmethod def valid_metric_regression(cls, value: dict[str, float]) -> dict[str, float]: @@ -167,4 +160,4 @@ class ValidatedRunConfig(StrictModel): prompt_hashes: dict[str, str] adapter_identity: str reproducibility_paths: tuple[str, ...] - reproducibility_issues: tuple[str, ...] = () + live_adapter: Optional[LiveAdapterSpec] = Field(default=None, exclude=True, repr=False) diff --git a/examples/optimization/eval_optimize_loop/pipeline/gate.py b/examples/optimization/eval_optimize_loop/pipeline/gate.py index 139cb1414..4d134ac0d 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/gate.py +++ b/examples/optimization/eval_optimize_loop/pipeline/gate.py @@ -179,12 +179,14 @@ def is_critical_regression(case) -> bool: "DURATION_BUDGET_EXCEEDED", ) validation_regressed = (validation.score_delta < -config.epsilon or validation.pass_rate_delta < -config.epsilon) - overfit = train.score_delta > config.epsilon and validation_regressed + train_improved = train.score_delta > config.epsilon or train.pass_rate_delta > config.epsilon + overfit = train_improved and validation_regressed add( "OVERFIT_GUARD", not overfit, { "trainScoreDelta": train.score_delta, + "trainPassRateDelta": train.pass_rate_delta, "validationScoreDelta": validation.score_delta, "validationPassRateDelta": validation.pass_rate_delta, }, diff --git a/examples/optimization/eval_optimize_loop/pipeline/live_adapter.py b/examples/optimization/eval_optimize_loop/pipeline/live_adapter.py new file mode 100644 index 000000000..b7c92ec6d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/live_adapter.py @@ -0,0 +1,88 @@ +"""Single importable callback contract for live evaluation and optimization.""" + +from __future__ import annotations + +import importlib +import hashlib +import inspect +import marshal +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +def load_callback(spec: str) -> Any: + """Load one MODULE:FUNCTION callback and reject ambiguous specifications.""" + + if spec.count(":") != 1: + raise ValueError("live callback must use MODULE:FUNCTION syntax") + module_name, function_name = spec.split(":", 1) + if not module_name or not function_name or "." in function_name: + raise ValueError("live callback must use MODULE:FUNCTION syntax") + callback = getattr(importlib.import_module(module_name), function_name) + if not inspect.iscoroutinefunction(callback): + raise TypeError("live callback must be an async function") + return callback + + +def _source_identity(callback: Any) -> tuple[Path, str, str]: + unwrapped = inspect.unwrap(callback) + code = getattr(unwrapped, "__code__", None) + if code is None: + raise TypeError("live callback must expose Python function code") + source = inspect.getsourcefile(unwrapped) or inspect.getfile(unwrapped) + if not source: + raise ValueError("live callback source is unavailable") + source_path = Path(source).resolve() + if not source_path.is_file(): + raise ValueError("live callback source is not a file") + return ( + source_path, + hashlib.sha256(source_path.read_bytes()).hexdigest(), + hashlib.sha256(marshal.dumps(code)).hexdigest(), + ) + + +def load_verified_callback( + spec: str, + *, + expected_source_path: str, + expected_source_sha256: str, + expected_callable_sha256: str, +) -> Any: + """Load a worker callback and prove it is the preflight-resolved source.""" + + callback = load_callback(spec) + source_path, source_sha256, callable_sha256 = _source_identity(callback) + if source_path != Path(expected_source_path).resolve(): + raise ValueError("worker callback source path differs from preflight") + if source_sha256 != expected_source_sha256: + raise ValueError("worker callback source hash differs from preflight") + if callable_sha256 != expected_callable_sha256: + raise ValueError("worker callback code differs from preflight") + return callback + + +@dataclass(frozen=True) +class LiveAdapterSpec: + """Bind the parent callable, worker import path and tracked source as one fact.""" + + import_path: str + callback: Any + source_path: Path + source_sha256: str + callable_sha256: str + + @classmethod + def resolve(cls, import_path: str, callback: Any) -> "LiveAdapterSpec": + loaded = load_callback(import_path) + if inspect.unwrap(loaded) is not inspect.unwrap(callback): + raise ValueError("call_agent and callback_spec must resolve to the same function") + source_path, source_sha256, callable_sha256 = _source_identity(loaded) + return cls( + import_path=import_path, + callback=loaded, + source_path=source_path, + source_sha256=source_sha256, + callable_sha256=callable_sha256, + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py index 57117da2e..ff370fccc 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/models.py +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -384,7 +384,9 @@ def consistent_outcome(self) -> "CandidateRound": class CandidateProposal(StrictModel): - status: Literal["SUCCEEDED"] = "SUCCEEDED" + status: Literal["SUCCEEDED", "FAILED", "CANCELED"] = "SUCCEEDED" + error_message: Optional[str] = None + adapter_error: Optional[str] = None algorithm: str baseline_prompts: dict[str, str] prompts: dict[str, str] @@ -401,6 +403,12 @@ def valid_duration(cls, value: float) -> float: @model_validator(mode="after") def valid_round_history(self) -> "CandidateProposal": + if self.status == "SUCCEEDED" and self.error_message: + raise ValueError("successful candidate generation cannot contain an error") + if self.adapter_error and self.status != "SUCCEEDED": + raise ValueError("adapter errors only describe rejected successful backend results") + if self.status == "FAILED" and not self.error_message: + raise ValueError("failed candidate generation requires an error message") if set(self.baseline_prompts) != set(self.prompts): raise ValueError("candidate prompt keys must equal baseline keys") if self.changed != (self.prompts != self.baseline_prompts): diff --git a/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py b/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py index 0fb567407..c583826ce 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py +++ b/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py @@ -4,28 +4,23 @@ import argparse import asyncio -import importlib import json from pathlib import Path -from typing import Any from trpc_agent_sdk.evaluation import AgentOptimizer, TargetPrompt +from .live_adapter import load_verified_callback from .schema import parse_strict_json, sanitize -def _load_callback(spec: str) -> Any: - if ":" not in spec: - raise ValueError("callbackSpec must use MODULE:FUNCTION syntax") - module_name, function_name = spec.split(":", 1) - if not module_name or not function_name: - raise ValueError("callbackSpec must use MODULE:FUNCTION syntax") - return getattr(importlib.import_module(module_name), function_name) - - async def _run(request_path: Path) -> None: request = parse_strict_json(request_path.read_text(encoding="utf-8")) - callback = _load_callback(str(request["callbackSpec"])) + callback = load_verified_callback( + str(request["callbackSpec"]), + expected_source_path=str(request["callbackSourcePath"]), + expected_source_sha256=str(request["callbackSourceSha256"]), + expected_callable_sha256=str(request["callbackCallableSha256"]), + ) prompt_paths = request["promptPaths"] if not isinstance(prompt_paths, dict) or not prompt_paths: raise ValueError("promptPaths must be a non-empty object") diff --git a/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py index f64b5b020..2b31a34be 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py +++ b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py @@ -23,6 +23,7 @@ CandidateProposal, ComparisonPair, Decision, + GateDecision, InnerSplit, OptimizationReport, Phase, @@ -91,8 +92,6 @@ async def run_pipeline( validated, started_at=started_at, started_clock=started_clock, - call_agent=call_agent, - callback_spec=callback_spec, backend=backend, candidate_generator=candidate_generator, fault_injector=fault_injector, @@ -104,8 +103,6 @@ async def _run_validated_pipeline( *, started_at: str, started_clock: float, - call_agent: Optional[CallAgent], - callback_spec: Optional[str], backend: Optional[EvaluationBackend], candidate_generator: Optional[CandidateGenerator], fault_injector: Optional[FaultInjector], @@ -118,19 +115,18 @@ async def _run_validated_pipeline( if backend is None or candidate_generator is None: default_backend, default_generator = create_backends( settings.mode, - call_agent=call_agent, + live_adapter=validated.live_adapter, trace_fixture_path=validated.trace_fixture_path, trace_fixture_hash=validated.input_hashes.get("trace"), dataset_hashes={ "train": validated.input_hashes["train"], "validation": validated.input_hashes["validation"], }, - callback_spec=callback_spec, optimizer_shutdown_timeout_seconds=settings.optimizer_shutdown_timeout_seconds, ) backend = backend or default_backend candidate_generator = candidate_generator or default_generator - if isinstance(backend, TraceEvaluationBackend): + if custom_backend and isinstance(backend, TraceEvaluationBackend): backend.validate_fixture(validated.train, validated.validation) target = TargetPrompt() @@ -172,7 +168,7 @@ async def _run_validated_pipeline( started_clock=started_clock, baseline_prompts=workspace.baseline, baseline_hashes=workspace.baseline_hashes, - callback_spec=callback_spec, + callback_spec=(validated.live_adapter.import_path if validated.live_adapter else None), programmatic_component=custom_components, ) @@ -282,6 +278,12 @@ def retain_progress(phase: Phase, progress: SnapshotPair) -> None: token_usage=source.token_usage, ) sink.write_json("candidate.json", proposal.model_dump(mode="json", by_alias=True)) + if proposal.adapter_error: + raise RuntimeError(f"candidate adapter rejected optimizer result: {proposal.adapter_error}") + if proposal.status == "CANCELED": + raise asyncio.CancelledError(proposal.error_message or "AgentOptimizer canceled") + if proposal.status == "FAILED": + raise RuntimeError(f"AgentOptimizer candidate generation failed: {proposal.error_message}") for name, content in proposal.prompts.items(): sink.write_text(f"candidate_prompts/{name}.md", content) @@ -323,17 +325,20 @@ def retain_progress(phase: Phase, progress: SnapshotPair) -> None: comparisons = ComparisonPair(train=train_comparison, validation=validation_comparison) sink.write_json("comparison.json", comparisons.model_dump(mode="json", by_alias=True)) + def decide(duration_seconds: float) -> GateDecision: + return evaluate_gate( + train=train_comparison, + validation=validation_comparison, + candidate=proposal, + cost=ledger.summary(), + duration_seconds=duration_seconds, + baseline_prompt_hashes=workspace.baseline_hashes, + candidate_prompt_hashes=prompt_hashes(proposal.prompts), + config=settings.gate, + ) + enter_stage("gate") - gate_decision = evaluate_gate( - train=train_comparison, - validation=validation_comparison, - candidate=proposal, - cost=ledger.summary(), - duration_seconds=time.monotonic() - started_clock, - baseline_prompt_hashes=workspace.baseline_hashes, - candidate_prompt_hashes=prompt_hashes(proposal.prompts), - config=settings.gate, - ) + gate_decision = decide(time.monotonic() - started_clock) if gate_decision.decision == Decision.ERROR: raise ValueError("gate rejected structurally invalid regression inputs") @@ -347,8 +352,17 @@ def retain_progress(phase: Phase, progress: SnapshotPair) -> None: applied = True enter_stage("final_report") + terminal_duration = time.monotonic() - started_clock + gate_decision = decide(terminal_duration) + if gate_decision.decision == Decision.ERROR: + raise ValueError("terminal gate rejected structurally invalid regression inputs") + if applied and not gate_decision.accepted: + await workspace.restore() + applied = False + terminal_duration = time.monotonic() - started_clock + gate_decision = decide(terminal_duration) final_report = build_report(gate_decision.decision, "complete") - return persist_terminal_report(sink, final_report, started_clock=started_clock) + return persist_terminal_report(sink, final_report, duration_seconds=terminal_duration) except BaseException as error: if candidate_started and proposal is None: ledger.record( @@ -373,7 +387,7 @@ def retain_progress(phase: Phase, progress: SnapshotPair) -> None: error_report = persist_terminal_report( sink, error_report, - started_clock=started_clock, + duration_seconds=time.monotonic() - started_clock, ) except BaseException as persist_error: persistence_error = persist_error diff --git a/examples/optimization/eval_optimize_loop/pipeline/preflight.py b/examples/optimization/eval_optimize_loop/pipeline/preflight.py index 5fe17b3b9..5df803a86 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/preflight.py +++ b/examples/optimization/eval_optimize_loop/pipeline/preflight.py @@ -3,7 +3,6 @@ from __future__ import annotations import hashlib -import inspect from pathlib import Path from typing import Any, Optional @@ -12,6 +11,7 @@ from .artifacts import load_strict_json from .evaluation import canonical_json, dataset_fingerprint, validate_datasets from .configuration import PipelineConfig, ValidatedRunConfig +from .live_adapter import LiveAdapterSpec from .schema import validate_safe_component, validate_secret_free_text from .prompt_workspace import prompt_hashes from .trace_fixture import TraceFixture @@ -40,31 +40,10 @@ def _component_identity(component: object | None, default: str) -> str: return f"{component_type.__module__}.{component_type.__qualname__}" -def _callable_identity(call_agent: object | None, callback_spec: Optional[str]) -> Optional[str]: - if callback_spec: - return callback_spec - if call_agent is None: - return None - module = getattr(call_agent, "__module__", type(call_agent).__module__) - qualname = getattr(call_agent, "__qualname__", type(call_agent).__qualname__) - return f"{module}:{qualname}" - - -def _callable_source_path(call_agent: object | None) -> Optional[Path]: - if call_agent is None: - return None - try: - source = inspect.getsourcefile(inspect.unwrap(call_agent)) or inspect.getfile(call_agent) - except (OSError, TypeError): - return None - return Path(source).resolve() if source else None - - def _adapter_identity( mode: str, *, - call_agent: object | None, - callback_spec: Optional[str], + live_adapter: Optional[LiveAdapterSpec], backend: object | None, candidate_generator: object | None, ) -> str: @@ -78,7 +57,11 @@ def _adapter_identity( "candidateGenerator": _component_identity(candidate_generator, f"default:{mode}:candidate-generator:v1"), "callback": - _callable_identity(call_agent, callback_spec) if mode == "live" else None, + ({ + "importPath": live_adapter.import_path, + "sourceSha256": live_adapter.source_sha256, + "callableSha256": live_adapter.callable_sha256, + } if live_adapter is not None else None), }) @@ -165,10 +148,20 @@ def preflight_run( hashes["trace"], ).validate(train, validation) + live_adapter: Optional[LiveAdapterSpec] = None + if settings.mode == "live": + if backend is None and call_agent is None: + raise ValueError("default live evaluation requires call_agent") + if callback_spec is not None: + if call_agent is None: + raise ValueError("callback_spec requires call_agent for identity verification") + live_adapter = LiveAdapterSpec.resolve(callback_spec, call_agent) + if (backend is None or candidate_generator is None) and live_adapter is None: + raise ValueError("default live components require an importable callback_spec") + adapter_identity = _adapter_identity( settings.mode, - call_agent=call_agent, - callback_spec=callback_spec, + live_adapter=live_adapter, backend=backend, candidate_generator=candidate_generator, ) @@ -176,13 +169,10 @@ def preflight_run( reproducibility_paths.extend(Path(path) for path in prompt_paths.values()) if trace_path is not None: reproducibility_paths.append(trace_path) - reproducibility_issues: list[str] = [] - if settings.mode == "live" and callback_spec: - callback_source = _callable_source_path(call_agent) - if callback_source is None: - reproducibility_issues.append("live_callback_source_unavailable") - else: - reproducibility_paths.append(callback_source) + if live_adapter is not None: + reproducibility_paths.append(live_adapter.source_path) + hashes["callback"] = live_adapter.source_sha256 + hashes["callbackCallable"] = live_adapter.callable_sha256 derived = ("run-" + hashlib.sha256( canonical_json({ "inputHashes": hashes, @@ -208,5 +198,5 @@ def preflight_run( prompt_hashes=prompt_hashes(prompts), adapter_identity=adapter_identity, reproducibility_paths=tuple(str(path.resolve()) for path in reproducibility_paths), - reproducibility_issues=tuple(reproducibility_issues), + live_adapter=live_adapter, ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporting.py b/examples/optimization/eval_optimize_loop/pipeline/reporting.py index cb173658f..561a30f1b 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/reporting.py +++ b/examples/optimization/eval_optimize_loop/pipeline/reporting.py @@ -70,7 +70,6 @@ def create_report_context( trace_hash=validated.input_hashes.get("trace"), programmatic_component=programmatic_component, input_paths=validated.reproducibility_paths, - input_issues=validated.reproducibility_issues, ) return ReportContext( validated=validated, @@ -100,7 +99,6 @@ def build_reproducibility( trace_hash: Optional[str] = None, programmatic_component: bool = False, input_paths: tuple[str, ...] = (), - input_issues: tuple[str, ...] = (), ) -> Reproducibility: commit: Optional[str] = None dirty: Optional[bool] = None @@ -142,9 +140,6 @@ def build_reproducibility( reason = "live_callback_not_importable" elif reason is None and mode == "trace" and (not trace_fixture or not trace_hash): reason = "trace_fixture_not_pinned" - elif reason is None and input_issues: - reason = input_issues[0] - if reason is None: assert git_root is not None for input_path in input_paths: @@ -357,20 +352,16 @@ def persist_terminal_report( sink: AuditSink, report: OptimizationReport, *, - started_clock: float, + duration_seconds: float, ) -> OptimizationReport: - """Persist one full terminal cycle before recording the completed duration.""" + """Commit one authoritative terminal report and publish its latest snapshots.""" - persist_report(sink, report) - sink.write_manifest() - sink.publish_latest_snapshot("optimization_report.json", "optimization_report.json") - sink.publish_latest_snapshot("optimization_report.md", "optimization_report.md") completed = OptimizationReport.model_validate({ **report.model_dump(mode="python", by_alias=False), "finished_at": utc_now(), "duration_seconds": - time.monotonic() - started_clock, + duration_seconds, }) persist_report(sink, completed) sink.write_manifest() diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 399396e3c..77b417101 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -4,10 +4,8 @@ import argparse import asyncio -import importlib import sys from pathlib import Path -from typing import Any _HERE = Path(__file__).resolve().parent _REPO_ROOT = _HERE.parents[2] @@ -15,15 +13,7 @@ sys.path.insert(0, str(_REPO_ROOT)) from examples.optimization.eval_optimize_loop.pipeline.orchestrator import run_pipeline - - -def _load_callback(spec: str) -> Any: - if ":" not in spec: - raise ValueError("--call-agent must use MODULE:FUNCTION syntax") - module_name, function_name = spec.split(":", 1) - if not module_name or not function_name: - raise ValueError("--call-agent must use MODULE:FUNCTION syntax") - return getattr(importlib.import_module(module_name), function_name) +from examples.optimization.eval_optimize_loop.pipeline.live_adapter import load_callback def _parser() -> argparse.ArgumentParser: @@ -43,7 +33,7 @@ def _parser() -> argparse.ArgumentParser: async def _main(args: argparse.Namespace) -> int: - callback = _load_callback(args.call_agent) if args.call_agent else None + callback = load_callback(args.call_agent) if args.call_agent else None report = await run_pipeline( str(_HERE), config_path=args.config, diff --git a/tests/examples/eval_optimize_loop/test_attribution_gate.py b/tests/examples/eval_optimize_loop/test_attribution_gate.py index 14b1e53d9..58fc986b5 100644 --- a/tests/examples/eval_optimize_loop/test_attribution_gate.py +++ b/tests/examples/eval_optimize_loop/test_attribution_gate.py @@ -378,16 +378,26 @@ def test_overfit_guard_cannot_be_bypassed_by_permissive_thresholds() -> None: min_validation_score_delta=-0.2, min_validation_pass_rate_delta=-1, metric_max_regression={"quality": 1}, - overfit_guard=True, ), ) assert result.decision == Decision.REJECT assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in result.reasons -def test_overfit_guard_cannot_be_disabled() -> None: - with pytest.raises(ValidationError, match="mandatory safety invariant"): - GateConfig(overfit_guard=False) +def test_overfit_guard_includes_train_pass_rate_improvement() -> None: + train = _comparison(Split.TRAIN, score_delta=0, pass_delta=1) + validation = _comparison(Split.VALIDATION, score_delta=-0.1, pass_delta=0) + result = _gate( + train=train, + validation=validation, + config=GateConfig( + min_validation_score_delta=-0.2, + min_validation_pass_rate_delta=-1, + metric_max_regression={"quality": 1}, + ), + ) + assert result.decision == Decision.REJECT + assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in result.reasons def test_gate_fails_closed_when_enabled_cost_is_unknown() -> None: diff --git a/tests/examples/eval_optimize_loop/test_backends.py b/tests/examples/eval_optimize_loop/test_backends.py index e81b538df..d0bc96797 100644 --- a/tests/examples/eval_optimize_loop/test_backends.py +++ b/tests/examples/eval_optimize_loop/test_backends.py @@ -12,6 +12,7 @@ from trpc_agent_sdk.evaluation import AgentEvaluator, EvalConfig, EvalSet, TargetPrompt from examples.optimization.eval_optimize_loop.pipeline import backends as backends_module +from examples.optimization.eval_optimize_loop.pipeline import optimizer_worker as optimizer_worker_module from examples.optimization.eval_optimize_loop.pipeline.backends import ( DeterministicCandidateGenerator, FakeEvaluationBackend, @@ -20,6 +21,10 @@ TraceEvaluationBackend, ) from examples.optimization.eval_optimize_loop.pipeline.evaluation import normalize_result +from examples.optimization.eval_optimize_loop.pipeline.live_adapter import ( + LiveAdapterSpec, + load_verified_callback, +) from examples.optimization.eval_optimize_loop.pipeline.models import ( AttributionSnapshot, Phase, @@ -75,6 +80,70 @@ def _config() -> EvalConfig: ) +async def importable_call_agent(query: str) -> str: + return query + + +def _live_adapter() -> LiveAdapterSpec: + return LiveAdapterSpec.resolve(f"{__name__}:importable_call_agent", importable_call_agent) + + +def test_worker_callback_verification_rejects_source_drift(tmp_path) -> None: + adapter = _live_adapter() + assert load_verified_callback( + adapter.import_path, + expected_source_path=str(adapter.source_path), + expected_source_sha256=adapter.source_sha256, + expected_callable_sha256=adapter.callable_sha256, + ) is importable_call_agent + with pytest.raises(ValueError, match="source path differs"): + load_verified_callback( + adapter.import_path, + expected_source_path=str(tmp_path / "shadowed.py"), + expected_source_sha256=adapter.source_sha256, + expected_callable_sha256=adapter.callable_sha256, + ) + with pytest.raises(ValueError, match="source hash differs"): + load_verified_callback( + adapter.import_path, + expected_source_path=str(adapter.source_path), + expected_source_sha256="0" * 64, + expected_callable_sha256=adapter.callable_sha256, + ) + with pytest.raises(ValueError, match="callback code differs"): + load_verified_callback( + adapter.import_path, + expected_source_path=str(adapter.source_path), + expected_source_sha256=adapter.source_sha256, + expected_callable_sha256="0" * 64, + ) + + +def _install_worker_result(monkeypatch, result, captured=None) -> None: + class CompletedProcess: + returncode = 0 + + async def wait(self): + return self.returncode + + async def fake_subprocess(*args, **kwargs): + request_path = Path(args[-1]) + request = json.loads(request_path.read_text(encoding="utf-8")) + output_dir = Path(request["outputDir"]) + output_dir.mkdir(parents=True) + (output_dir / "result.json").write_text("{}", encoding="utf-8") + if captured is not None: + captured.update({"args": args, "kwargs": kwargs, "request": request}) + return CompletedProcess() + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_subprocess) + monkeypatch.setattr( + backends_module, + "OptimizeResult", + SimpleNamespace(from_file=lambda path: result), + ) + + @pytest.mark.asyncio async def test_fake_backend_uses_public_services_and_does_not_mutate_inputs(tmp_path, monkeypatch) -> None: backend = FakeEvaluationBackend() @@ -616,7 +685,6 @@ async def test_deterministic_generator_only_uses_failure_facts(tmp_path) -> None @pytest.mark.asyncio async def test_live_generator_forces_update_source_false(monkeypatch, tmp_path) -> None: - async def call_agent(query: str) -> str: return query @@ -624,69 +692,41 @@ async def call_agent(query: str) -> str: async def fake_optimize(**kwargs): captured.update(kwargs) - round_ = SimpleNamespace( - round=1, - candidate_prompts={"system": "candidate"}, - accepted=True, - validation_pass_rate=1, - kind="merge", - optimized_field_names=["system"], - metric_breakdown={"quality": 0.9}, - acceptance_reason="accepted by optimizer", - skip_reason=None, - error_message=None, - round_llm_cost=0.035, - round_token_usage={"total": 0}, - duration_seconds=0, - ) - return SimpleNamespace( - status="SUCCEEDED", - error_message="", - rounds=[round_], - algorithm="gepa_reflective", - baseline_prompts={"system": "baseline"}, - best_prompts={"system": "candidate"}, - stop_reason="completed", - total_reflection_lm_calls=1, - total_llm_cost=0, - total_token_usage={"total": 0}, - duration_seconds=0, - ) + return SimpleNamespace(dump_to=lambda path: Path(path).write_text("{}", encoding="utf-8")) monkeypatch.setattr( - "examples.optimization.eval_optimize_loop.pipeline.backends.AgentOptimizer.optimize", - fake_optimize, + optimizer_worker_module, + "load_verified_callback", + lambda spec, **kwargs: call_agent, ) + monkeypatch.setattr(optimizer_worker_module.AgentOptimizer, "optimize", fake_optimize) prompt_path = tmp_path / "system.md" prompt_path.write_text("baseline", encoding="utf-8") - target = TargetPrompt().add_path("system", str(prompt_path)) - generator = LiveCandidateGenerator(call_agent) - proposal = await generator.generate( - target_prompt=target, - baseline_prompts={"system": "baseline"}, - train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), - inner_train_path="inner-train.json", - inner_selection_path="inner-selection.json", - config_path="optimizer.json", - output_dir="optimizer-output", - ) + request_path = tmp_path / "request.json" + request_path.write_text( + json.dumps({ + "callbackSpec": _live_adapter().import_path, + "callbackSourcePath": str(_live_adapter().source_path), + "callbackSourceSha256": _live_adapter().source_sha256, + "callbackCallableSha256": _live_adapter().callable_sha256, + "promptPaths": { + "system": str(prompt_path) + }, + "configPath": "optimizer.json", + "trainPath": "inner-train.json", + "validationPath": "inner-selection.json", + "outputDir": str(tmp_path / "optimizer-output"), + "verbose": 0, + }), + encoding="utf-8", + ) + await optimizer_worker_module._run(request_path) assert captured["update_source"] is False assert captured["validation_dataset_path"] == "inner-selection.json" - assert proposal.prompts == {"system": "candidate"} - assert proposal.rounds[0].kind == "merge" - assert proposal.rounds[0].cost_usd == 0.035 - assert proposal.rounds[0].metric_scores == {"quality": 0.9} - assert [(source.name, source.cost_usd, source.model_calls) for source in proposal.cost_sources] == [ - ("optimizer_reported", 0, 1), - ] @pytest.mark.asyncio async def test_live_generator_accepts_sdk_skipped_round(monkeypatch, tmp_path) -> None: - - async def call_agent(query: str) -> str: - return query - async def fake_optimize(**kwargs): return SimpleNamespace( status="SUCCEEDED", @@ -718,13 +758,10 @@ async def fake_optimize(**kwargs): duration_seconds=0.4, ) - monkeypatch.setattr( - "examples.optimization.eval_optimize_loop.pipeline.backends.AgentOptimizer.optimize", - fake_optimize, - ) + _install_worker_result(monkeypatch, await fake_optimize()) prompt_path = tmp_path / "system.md" prompt_path.write_text("baseline", encoding="utf-8") - proposal = await LiveCandidateGenerator(call_agent).generate( + proposal = await LiveCandidateGenerator(_live_adapter()).generate( target_prompt=TargetPrompt().add_path("system", str(prompt_path)), baseline_prompts={"system": "baseline"}, train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), @@ -741,51 +778,120 @@ async def fake_optimize(**kwargs): @pytest.mark.asyncio -async def test_live_generator_requests_cooperative_stop_before_propagating_cancel(monkeypatch, tmp_path) -> None: +async def test_live_generator_preserves_failed_sdk_result_facts(monkeypatch, tmp_path) -> None: + result = SimpleNamespace( + status="FAILED", + error_message="optimizer request failed", + rounds=[SimpleNamespace( + round=1, + candidate_prompts={}, + accepted=False, + validation_pass_rate=0.0, + kind="reflective", + optimized_field_names=[], + metric_breakdown={}, + acceptance_reason="", + skip_reason=None, + error_message="reflection request failed", + round_llm_cost=0.04, + round_token_usage={"total": 6}, + duration_seconds=0.2, + )], + algorithm="gepa_reflective", + baseline_prompts={"system": "baseline"}, + best_prompts={"system": "baseline"}, + stop_reason=None, + total_reflection_lm_calls=1, + total_llm_cost=0.04, + total_token_usage={"total": 6}, + duration_seconds=0.3, + ) + _install_worker_result(monkeypatch, result) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") - async def call_agent(query: str) -> str: - return query + proposal = await LiveCandidateGenerator(_live_adapter()).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + ) - started = asyncio.Event() - stopped = asyncio.Event() + assert proposal.status == "FAILED" + assert proposal.error_message == "optimizer request failed" + assert proposal.rounds[0].error_message == "reflection request failed" + assert proposal.cost_sources[0].cost_usd == 0.04 + assert proposal.duration_seconds == 0.3 - async def fake_optimize(**kwargs): - started.set() - stop_path = Path(kwargs["output_dir"]) / "optimize.stop" - while not stop_path.exists(): - await asyncio.sleep(0) - stopped.set() - return SimpleNamespace(status="CANCELED", error_message="user stop") - monkeypatch.setattr( - "examples.optimization.eval_optimize_loop.pipeline.backends.AgentOptimizer.optimize", - fake_optimize, +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("reported_baseline", "best_prompts", "expected_error"), + ( + ({"system": "stale"}, {"system": "candidate"}, "baseline prompts differ"), + ({"system": "baseline"}, {}, "best prompt keys differ"), + ), +) +async def test_live_generator_rejects_invalid_success_contract( + monkeypatch, + tmp_path, + reported_baseline, + best_prompts, + expected_error, +) -> None: + result = SimpleNamespace( + status="SUCCEEDED", + error_message="", + rounds=[SimpleNamespace( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=True, + validation_pass_rate=0.9, + kind="reflective", + optimized_field_names=["system"], + metric_breakdown={"quality": 0.9}, + acceptance_reason="improved", + skip_reason=None, + error_message=None, + round_llm_cost=0.02, + round_token_usage={"total": 4}, + duration_seconds=0.1, + )], + algorithm="gepa_reflective", + baseline_prompts=reported_baseline, + best_prompts=best_prompts, + stop_reason="completed", + total_reflection_lm_calls=1, + total_llm_cost=0.02, + total_token_usage={"total": 4}, + duration_seconds=0.2, ) + _install_worker_result(monkeypatch, result) prompt_path = tmp_path / "system.md" prompt_path.write_text("baseline", encoding="utf-8") - task = asyncio.create_task( - LiveCandidateGenerator(call_agent).generate( - target_prompt=TargetPrompt().add_path("system", str(prompt_path)), - baseline_prompts={"system": "baseline"}, - train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), - inner_train_path="inner-train.json", - inner_selection_path="inner-selection.json", - config_path="optimizer.json", - output_dir=str(tmp_path / "optimizer-output"), - )) - await started.wait() - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - assert stopped.is_set() + proposal = await LiveCandidateGenerator(_live_adapter()).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + ) -@pytest.mark.asyncio -async def test_live_worker_is_forcibly_terminated_after_bounded_cooperative_stop(monkeypatch, tmp_path) -> None: + assert proposal.status == "SUCCEEDED" + assert expected_error in proposal.adapter_error + assert proposal.prompts == {"system": "baseline"} + assert proposal.changed is False + assert proposal.rounds[0].cost_usd == 0.02 - async def call_agent(query: str) -> str: - return query +@pytest.mark.asyncio +async def test_live_worker_is_forcibly_terminated_after_bounded_cooperative_stop(monkeypatch, tmp_path) -> None: class StuckProcess: def __init__(self) -> None: @@ -821,8 +927,7 @@ async def fake_subprocess(*args, **kwargs): output_dir = tmp_path / "optimizer-output" task = asyncio.create_task( LiveCandidateGenerator( - call_agent, - callback_spec="tests.agent:call_agent", + _live_adapter(), shutdown_timeout_seconds=0.01, ).generate( target_prompt=TargetPrompt().add_path("system", str(prompt_path)), @@ -844,10 +949,6 @@ async def fake_subprocess(*args, **kwargs): @pytest.mark.asyncio async def test_live_worker_success_is_loaded_through_the_strict_candidate_adapter(monkeypatch, tmp_path) -> None: - - async def call_agent(query: str) -> str: - return query - round_ = SimpleNamespace( round=1, candidate_prompts={"system": "candidate"}, @@ -877,34 +978,10 @@ async def call_agent(query: str) -> str: duration_seconds=0.2, ) captured = {} - - class CompletedProcess: - returncode = 0 - - async def wait(self): - return self.returncode - - async def fake_subprocess(*args, **kwargs): - request_path = Path(args[-1]) - request = json.loads(request_path.read_text(encoding="utf-8")) - output_dir = Path(request["outputDir"]) - output_dir.mkdir(parents=True) - (output_dir / "result.json").write_text("{}", encoding="utf-8") - captured.update({"args": args, "kwargs": kwargs, "request": request}) - return CompletedProcess() - - monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_subprocess) - monkeypatch.setattr( - backends_module, - "OptimizeResult", - SimpleNamespace(from_file=lambda path: result), - ) + _install_worker_result(monkeypatch, result, captured) prompt_path = tmp_path / "system.md" prompt_path.write_text("baseline", encoding="utf-8") - proposal = await LiveCandidateGenerator( - call_agent, - callback_spec="tests.agent:call_agent", - ).generate( + proposal = await LiveCandidateGenerator(_live_adapter()).generate( target_prompt=TargetPrompt().add_path("system", str(prompt_path)), baseline_prompts={"system": "baseline"}, train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), @@ -917,7 +994,10 @@ async def fake_subprocess(*args, **kwargs): "-m", "examples.optimization.eval_optimize_loop.pipeline.optimizer_worker", ) - assert captured["request"]["callbackSpec"] == "tests.agent:call_agent" + assert captured["request"]["callbackSpec"] == _live_adapter().import_path + assert captured["request"]["callbackSourcePath"] == str(_live_adapter().source_path) + assert captured["request"]["callbackSourceSha256"] == _live_adapter().source_sha256 + assert captured["request"]["callbackCallableSha256"] == _live_adapter().callable_sha256 assert captured["request"]["promptPaths"] == {"system": str(prompt_path)} assert proposal.prompts == {"system": "candidate"} assert proposal.rounds[0].metric_scores == {"quality": 0.8} diff --git a/tests/examples/eval_optimize_loop/test_models_and_imports.py b/tests/examples/eval_optimize_loop/test_models_and_imports.py index b6dc2c1dc..c826db406 100644 --- a/tests/examples/eval_optimize_loop/test_models_and_imports.py +++ b/tests/examples/eval_optimize_loop/test_models_and_imports.py @@ -53,7 +53,7 @@ def test_pipeline_policy_scalars_reject_string_coercion() -> None: with pytest.raises(ValidationError): GateConfig.model_validate({"maxNewHardFailures": "2"}) with pytest.raises(ValidationError): - GateConfig.model_validate({"overfitGuard": "false"}) + GateConfig.model_validate({"overfitGuard": False}) def test_strict_json_and_path_components_reject_ambiguous_inputs() -> None: @@ -228,7 +228,7 @@ def test_pipeline_import_boundaries() -> None: allowed = { "artifacts": {"models", "schema"}, "attribution": {"configuration", "models", "schema"}, - "backends": {"contracts", "models", "offline_evaluation", "schema", "trace_fixture"}, + "backends": {"contracts", "live_adapter", "models", "offline_evaluation", "schema", "trace_fixture"}, "candidate_runtime": { "artifacts", "attribution", @@ -238,7 +238,7 @@ def test_pipeline_import_boundaries() -> None: "prompt_workspace", "schema", }, - "configuration": {"models", "schema"}, + "configuration": {"live_adapter", "models", "schema"}, "contracts": {"models"}, "costing": {"models"}, "evaluation": {"models"}, @@ -251,9 +251,10 @@ def test_pipeline_import_boundaries() -> None: "models", }, "gate": {"configuration", "models"}, + "live_adapter": set(), "models": {"schema"}, "offline_evaluation": set(), - "optimizer_worker": {"schema"}, + "optimizer_worker": {"live_adapter", "schema"}, "orchestrator": { "artifacts", "attribution", @@ -275,6 +276,7 @@ def test_pipeline_import_boundaries() -> None: "artifacts", "configuration", "evaluation", + "live_adapter", "prompt_workspace", "schema", "trace_fixture", diff --git a/tests/examples/eval_optimize_loop/test_orchestrator.py b/tests/examples/eval_optimize_loop/test_orchestrator.py index ef92ee70e..5943a6ee3 100644 --- a/tests/examples/eval_optimize_loop/test_orchestrator.py +++ b/tests/examples/eval_optimize_loop/test_orchestrator.py @@ -16,6 +16,7 @@ from examples.optimization.eval_optimize_loop.pipeline.models import ( CandidateProposal, CandidateRound, + CostSource, Decision, OptimizationReport, Transition, @@ -39,6 +40,14 @@ SOURCE = Path(__file__).resolve().parents[3] / "examples" / "optimization" / "eval_optimize_loop" +async def importable_call_agent(query: str) -> str: + return query + + +async def alternate_importable_call_agent(query: str) -> str: + return query + + def _example(tmp_path: Path, *, accept: bool = False, apply: bool = False) -> Path: root = tmp_path / "example" (root / "prompts").mkdir(parents=True) @@ -327,6 +336,90 @@ async def generate(self, **kwargs): assert "[REDACTED]" in report.errors[0].message +@pytest.mark.asyncio +async def test_failed_optimizer_result_preserves_rounds_cost_and_duration(tmp_path) -> None: + root = _example(tmp_path) + + class FailedGenerator: + async def generate(self, **kwargs): + baseline = dict(kwargs["baseline_prompts"]) + return CandidateProposal( + status="FAILED", + error_message="optimizer backend failed", + algorithm="gepa_reflective", + baseline_prompts=baseline, + prompts=baseline, + changed=False, + stop_reason="error", + rounds=(CandidateRound( + round=1, + candidate_prompts={}, + accepted=False, + error_message="reflection request failed", + cost_usd=0.12, + duration_seconds=0.4, + ), ), + cost_sources=(CostSource( + name="optimizer_reported", + cost_usd=0.12, + model_calls=1, + ), ), + duration_seconds=0.4, + ) + + report = await run_pipeline( + str(root), + run_id="failed-optimizer-facts", + candidate_generator=FailedGenerator(), + ) + + assert report.status == Decision.ERROR + assert report.optimization["proposal"]["status"] == "FAILED" + assert report.optimization["proposal"]["rounds"][0]["costUsd"] == 0.12 + assert report.optimization["proposal"]["durationSeconds"] == 0.4 + source = next(item for item in report.cost.sources if item.name.endswith("optimizer_reported")) + assert source.cost_usd == 0.12 + assert all(item.name != "candidate_generation.unreported_failure" for item in report.cost.sources) + + +@pytest.mark.asyncio +async def test_adapter_rejected_success_is_audited_without_evaluation_or_apply(tmp_path) -> None: + root = _example(tmp_path, accept=True, apply=True) + prompt_path = root / "prompts" / "system.md" + baseline_prompt = prompt_path.read_text(encoding="utf-8") + + class RejectedSuccessGenerator: + async def generate(self, **kwargs): + baseline = dict(kwargs["baseline_prompts"]) + return CandidateProposal( + status="SUCCEEDED", + adapter_error="optimizer baseline prompts differ from the workspace baseline", + algorithm="gepa_reflective", + baseline_prompts=baseline, + prompts=baseline, + changed=False, + cost_sources=(CostSource( + name="optimizer_reported", + cost_usd=0.03, + model_calls=1, + ), ), + duration_seconds=0.2, + ) + + report = await run_pipeline( + str(root), + run_id="adapter-rejected-success", + candidate_generator=RejectedSuccessGenerator(), + ) + + assert report.status == Decision.ERROR + assert report.stage == "candidate_generation" + assert report.optimization["proposal"]["status"] == "SUCCEEDED" + assert report.optimization["proposal"]["adapterError"] + assert report.candidate is None + assert prompt_path.read_text(encoding="utf-8") == baseline_prompt + + @pytest.mark.asyncio async def test_terminal_audit_failure_is_raised_instead_of_silently_returned(tmp_path, monkeypatch) -> None: root = _example(tmp_path) @@ -361,9 +454,6 @@ async def test_live_evaluation_cost_bounds_are_accounted_conservatively(tmp_path }) config_path.write_text(json.dumps(config), encoding="utf-8") - async def call_agent(query: str) -> str: - return query - validated = preflight_run( str(root), config_path=None, @@ -372,8 +462,8 @@ async def call_agent(query: str) -> str: mode=None, run_id="live-cost-bounds", apply_candidate=False, - call_agent=call_agent, - callback_spec="tests.agent:call_agent", + call_agent=importable_call_agent, + callback_spec=f"{__name__}:importable_call_agent", backend=None, candidate_generator=None, ) @@ -640,17 +730,14 @@ def test_default_run_id_includes_effective_apply_setting(tmp_path) -> None: applying_run = preflight_run(str(root), apply_candidate=True, **common) assert dry_run.run_id != applying_run.run_id - async def call_agent(query: str) -> str: - return query - first_callback = preflight_run( str(root), apply_candidate=False, **{ **common, "mode": "live", - "call_agent": call_agent, - "callback_spec": "package.first:call_agent", + "call_agent": importable_call_agent, + "callback_spec": f"{__name__}:importable_call_agent", }, ) second_callback = preflight_run( @@ -659,28 +746,47 @@ async def call_agent(query: str) -> str: **{ **common, "mode": "live", - "call_agent": call_agent, - "callback_spec": "package.second:call_agent", + "call_agent": alternate_importable_call_agent, + "callback_spec": f"{__name__}:alternate_importable_call_agent", }, ) assert first_callback.run_id != second_callback.run_id +def test_preflight_rejects_mismatched_live_callback_identity(tmp_path) -> None: + root = _example(tmp_path) + with pytest.raises(ValueError, match="must resolve to the same function"): + preflight_run( + str(root), + config_path=None, + train_path=None, + validation_path=None, + mode="live", + run_id="mismatched-live-callback", + apply_candidate=False, + call_agent=importable_call_agent, + callback_spec=f"{__name__}:alternate_importable_call_agent", + backend=None, + candidate_generator=None, + ) + + @pytest.mark.asyncio -async def test_final_duration_includes_terminal_report_write(tmp_path, monkeypatch) -> None: +async def test_terminal_report_is_committed_once_with_gate_consistent_duration(tmp_path, monkeypatch) -> None: root = _example(tmp_path) original = reporting_module.persist_report - delayed = False + complete_writes = 0 def persist_with_delay(sink, report): - nonlocal delayed - if report.stage == "complete" and not delayed: - delayed = True + nonlocal complete_writes + if report.stage == "complete": + complete_writes += 1 time.sleep(0.05) return original(sink, report) monkeypatch.setattr(reporting_module, "persist_report", persist_with_delay) report = await run_pipeline(str(root), run_id="duration-includes-report") - assert delayed is True - assert report.duration_seconds >= 0.05 + duration_check = next(check for check in report.gate_decision.checks if check.code == "DURATION_BUDGET") + assert complete_writes == 1 + assert report.duration_seconds == pytest.approx(duration_check.observed, abs=1e-12) assert report.inputs["environment"]["sdk"] From 8b741cceacbe6a63d5c49023601225f954c77d08 Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Fri, 31 Jul 2026 12:30:24 +0800 Subject: [PATCH 08/13] fix(example): stabilize trace dataset fingerprints --- .../eval_optimize_loop/pipeline/evaluation.py | 4 +++- .../sample_output/optimization_report.json | 4 ++-- .../eval_optimize_loop/traces/trace_cases.json | 4 ++-- .../eval_optimize_loop/test_models_and_imports.py | 12 ++++++++++++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/pipeline/evaluation.py b/examples/optimization/eval_optimize_loop/pipeline/evaluation.py index 64c59ef14..b59815277 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/evaluation.py +++ b/examples/optimization/eval_optimize_loop/pipeline/evaluation.py @@ -83,7 +83,9 @@ def case_input_fingerprint(case: Any) -> str: def dataset_fingerprint(eval_set: EvalSet) -> str: - payload = eval_set.model_dump(mode="json", by_alias=True) + # Hash the submitted dataset contract, not defaults injected by SDK or + # Pydantic versions after validation. + payload = eval_set.model_dump(mode="json", by_alias=True, exclude_unset=True) return sha256_json(_without_execution_ids(payload)) diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json index 995ef47da..5bfd8c4d5 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -2206,8 +2206,8 @@ "hashes": { "config": "fef5828244ea4f0058084dc427d5c42f60ffa004ee3e3ac38bee94ba08f0667e", "effectiveConfig": "69cbd345d2e3868ab5baf39f73901fa51e137a59289bdebf6e10d31086d997e1", - "train": "db8df56423d916afa929a2fc1a6f7245d8a1f719e87ac292f88f346d1cb81789", - "validation": "499d3bde9cdc70cc4871d425d0dfee83ed30a67d35da15a4eb87cc9549583f84" + "train": "af410bcc6f46c041a373efb1d1dda9f0f0f20258d1ef2146eac8a583cde99b19", + "validation": "5011756743a6211c78c75f2b7cbc435e98cf4caf0e0c484bbac9c5f03951c7d9" }, "paths": { "config": "optimizer.json", diff --git a/examples/optimization/eval_optimize_loop/traces/trace_cases.json b/examples/optimization/eval_optimize_loop/traces/trace_cases.json index 8d277d772..a20db2e40 100644 --- a/examples/optimization/eval_optimize_loop/traces/trace_cases.json +++ b/examples/optimization/eval_optimize_loop/traces/trace_cases.json @@ -1,8 +1,8 @@ { "schemaVersion": "v1", "datasetHashes": { - "train": "db8df56423d916afa929a2fc1a6f7245d8a1f719e87ac292f88f346d1cb81789", - "validation": "499d3bde9cdc70cc4871d425d0dfee83ed30a67d35da15a4eb87cc9549583f84" + "train": "af410bcc6f46c041a373efb1d1dda9f0f0f20258d1ef2146eac8a583cde99b19", + "validation": "5011756743a6211c78c75f2b7cbc435e98cf4caf0e0c484bbac9c5f03951c7d9" }, "phases": { "baseline": { diff --git a/tests/examples/eval_optimize_loop/test_models_and_imports.py b/tests/examples/eval_optimize_loop/test_models_and_imports.py index c826db406..4dcdc5c63 100644 --- a/tests/examples/eval_optimize_loop/test_models_and_imports.py +++ b/tests/examples/eval_optimize_loop/test_models_and_imports.py @@ -7,11 +7,14 @@ import pytest from pydantic import ValidationError +from trpc_agent_sdk.evaluation import EvalSet +from examples.optimization.eval_optimize_loop.pipeline.artifacts import load_strict_json from examples.optimization.eval_optimize_loop.pipeline.configuration import ( GateConfig, PipelineSettings, ) +from examples.optimization.eval_optimize_loop.pipeline.evaluation import dataset_fingerprint from examples.optimization.eval_optimize_loop.pipeline.models import ( CandidateProposal, CandidateRound, @@ -32,6 +35,15 @@ ) PIPELINE_DIR = Path(__file__).resolve().parents[3] / "examples" / "optimization" / "eval_optimize_loop" / "pipeline" +EXAMPLE_DIR = PIPELINE_DIR.parent + + +def test_dataset_fingerprint_tracks_only_explicit_input_fields() -> None: + train = EvalSet.model_validate(load_strict_json(EXAMPLE_DIR / "train.evalset.json")) + validation = EvalSet.model_validate(load_strict_json(EXAMPLE_DIR / "val.evalset.json")) + + assert dataset_fingerprint(train) == "af410bcc6f46c041a373efb1d1dda9f0f0f20258d1ef2146eac8a583cde99b19" + assert dataset_fingerprint(validation) == "5011756743a6211c78c75f2b7cbc435e98cf4caf0e0c484bbac9c5f03951c7d9" def test_models_forbid_unknown_and_non_finite_values() -> None: From 485e5b9c63ec325476d2bedeccc0689f22d592bf Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Fri, 31 Jul 2026 19:49:18 +0800 Subject: [PATCH 09/13] fix(examples): harden optimization audit boundaries --- .../optimization/eval_optimize_loop/DESIGN.md | 19 +- .../eval_optimize_loop/pipeline/artifacts.py | 211 +++++++++++++++--- .../pipeline/candidate_runtime.py | 17 +- .../pipeline/configuration.py | 1 + .../eval_optimize_loop/pipeline/evaluation.py | 32 ++- .../eval_optimize_loop/pipeline/models.py | 1 + .../pipeline/orchestrator.py | 31 ++- .../eval_optimize_loop/pipeline/preflight.py | 12 +- .../pipeline/prompt_workspace.py | 26 ++- .../eval_optimize_loop/pipeline/reporting.py | 1 + .../eval_optimize_loop/pipeline/schema.py | 39 ++++ .../sample_output/optimization_report.json | 8 + .../test_candidate_runtime.py | 28 ++- .../eval_optimize_loop/test_deliverables.py | 8 + .../test_models_and_imports.py | 7 +- .../eval_optimize_loop/test_orchestrator.py | 108 +++++++++ .../test_workspace_artifacts.py | 151 ++++++++++++- 17 files changed, 636 insertions(+), 64 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index 1687b0ff6..39a38cee6 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -129,10 +129,17 @@ Any evaluator, optimizer, normalization, comparison, gate, render, apply, write, cancel, or system-exit failure restores and verifies the baseline prompt. A restoration failure raises `PromptRestoreError`; it is never downgraded to REJECT. Reports store no chain-of-thought and recursively redact credentials. +Source `inputs.hashes` bind the submitted contracts used for replay. Separate +`inputs.auditHashes` and inner-split `auditHashes` bind the redacted dataset +copies stored on disk, so credential removal cannot invalidate or misrepresent +either fact. Live optimizer config, prompt sandbox and raw optimizer output stay in an OS temporary directory; only sanitized known artifact types enter the audit tree. -The import boundary enforces file-count, per-file-byte and total-byte budgets -before reading optimizer output. This boundary protects audit storage; it is not +The import boundary snapshots only regular, single-link files. It rejects +symlinks, hard links and Windows reparse points, verifies file identity before +and after bounded reads, and enforces file-count, per-file-byte and total-byte +budgets before publishing any optimizer artifact. This boundary protects audit +storage; it is not a sandbox for a programmatic generator, which is trusted in-process code. Default live evaluation and optimization resolve one importable callback through @@ -149,9 +156,11 @@ baseline and registered best-prompt keys before it can enter regression. Failed or canceled SDK results retain their structured rounds, duration, error and cost facts in the terminal report. Terminal report persistence failures raise `AuditPersistenceError`; they are never discarded -while returning an apparently handled pipeline error. Audit writes redact -credentials without truncation and fail before publication when the configured -file-byte ceiling is exceeded. +while returning an apparently handled pipeline error. Error chains are +credential-redacted with bounded item count and total text size, retaining the +primary and restoration causes. Audit writes redact credentials without +truncation and fail before publication when the configured file-byte ceiling is +exceeded. A replay claim is emitted only for a clean, pinned Git commit when every effective config, dataset, prompt, trace and live callback source is inside that diff --git a/examples/optimization/eval_optimize_loop/pipeline/artifacts.py b/examples/optimization/eval_optimize_loop/pipeline/artifacts.py index 78f369fa8..ce868fd80 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/artifacts.py +++ b/examples/optimization/eval_optimize_loop/pipeline/artifacts.py @@ -5,9 +5,11 @@ import hashlib import json import os +import stat import threading import time import uuid +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -28,6 +30,53 @@ def load_strict_json(path: str | Path) -> dict[str, Any]: return parse_strict_json(Path(path).read_text(encoding="utf-8")) +@dataclass(frozen=True) +class _ArtifactSnapshot: + relative_path: str + content: str + + +_REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) +_NO_FOLLOW = getattr(os, "O_NOFOLLOW", 0) + + +def _stat_is_link_or_reparse(info: os.stat_result) -> bool: + return stat.S_ISLNK(info.st_mode) or bool(getattr(info, "st_file_attributes", 0) & _REPARSE_POINT) + + +def _path_is_junction(path: Path) -> bool: + is_junction = getattr(path, "is_junction", None) + return bool(is_junction()) if callable(is_junction) else False + + +def _require_safe_root(path: Path) -> os.stat_result: + try: + info = os.lstat(path) + except FileNotFoundError: + raise FileNotFoundError(path) from None + if _stat_is_link_or_reparse(info) or _path_is_junction(path): + raise ValueError("optimizer artifact symlinks, hard links and reparse points are not allowed") + if not stat.S_ISDIR(info.st_mode): + raise ValueError("optimizer artifact source must be a directory") + return info + + +def _same_file_identity(before: os.stat_result, after: os.stat_result) -> bool: + return ( + before.st_dev, + before.st_ino, + before.st_mode, + before.st_size, + before.st_mtime_ns, + ) == ( + after.st_dev, + after.st_ino, + after.st_mode, + after.st_size, + after.st_mtime_ns, + ) + + class AuditSink: """Own safe paths and atomic persistence beneath one immutable run directory.""" @@ -126,41 +175,135 @@ def write_jsonl(self, relative_path: str, payloads: list[dict[str, Any]]) -> Pat self._atomic_write(path, "\n".join(lines) + ("\n" if lines else "")) return path - def import_tree(self, source: str | Path, destination: str) -> None: - """Import known optimizer artifacts through the shared sanitization boundary.""" - - source_root = Path(source).resolve() - if not source_root.exists(): - return - paths = sorted(item for item in source_root.rglob("*") if item.is_file()) - if len(paths) > self.max_import_files: - raise ValueError(f"optimizer artifact count exceeds limit: {len(paths)} > {self.max_import_files}") - total_bytes = 0 - for path in paths: - if path.is_symlink(): - raise ValueError("optimizer artifact symlinks are not allowed") - byte_size = path.stat().st_size - if byte_size > self.max_import_file_bytes: + def _read_verified_file(self, path: Path, before: os.stat_result) -> bytes: + if before.st_nlink != 1: + raise ValueError(f"optimizer artifact hard links are not allowed: {path.name}") + flags = os.O_RDONLY | _NO_FOLLOW + try: + descriptor = os.open(path, flags) + except OSError as error: + raise ValueError(f"optimizer artifact could not be opened safely: {path.name}") from error + try: + opened = os.fstat(descriptor) + if _stat_is_link_or_reparse(opened) or not stat.S_ISREG(opened.st_mode): + raise ValueError(f"optimizer artifact is not a regular file: {path.name}") + if opened.st_nlink != 1 or not _same_file_identity(before, opened): + raise ValueError(f"optimizer artifact changed during validation: {path.name}") + chunks: list[bytes] = [] + remaining = self.max_import_file_bytes + 1 + while remaining: + chunk = os.read(descriptor, min(1024 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + after = os.fstat(descriptor) + if not _same_file_identity(opened, after): + raise ValueError(f"optimizer artifact changed while reading: {path.name}") + content = b"".join(chunks) + if len(content) > self.max_import_file_bytes: raise ValueError(f"optimizer artifact exceeds per-file byte limit: " - f"{path.name} ({byte_size} > {self.max_import_file_bytes})") - total_bytes += byte_size - if total_bytes > self.max_import_total_bytes: - raise ValueError(f"optimizer artifacts exceed total byte limit: " - f"{total_bytes} > {self.max_import_total_bytes}") - relative = path.relative_to(source_root) - target = (Path(destination) / relative).as_posix() - suffix = path.suffix.casefold() - if suffix == ".json": - self.write_json(target, parse_strict_json(path.read_text(encoding="utf-8"))) - elif suffix == ".jsonl": - payloads = [ - parse_strict_json(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip() - ] - self.write_jsonl(target, payloads) - elif suffix in _TEXT_ARTIFACT_SUFFIXES: - self.write_text(target, path.read_text(encoding="utf-8")) - else: - raise ValueError(f"unsupported optimizer artifact type: {relative.as_posix()}") + f"{path.name} ({len(content)} > {self.max_import_file_bytes})") + return content + finally: + os.close(descriptor) + + @staticmethod + def _render_json(payload: Any) -> str: + return json.dumps( + sanitize(payload, max_text_chars=None), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + "\n" + + @staticmethod + def _render_jsonl(payload: Any) -> str: + return json.dumps( + sanitize(payload, max_text_chars=None), + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + + def _snapshot_file(self, path: Path, relative_path: str, before: os.stat_result) -> _ArtifactSnapshot: + suffix = path.suffix.casefold() + if suffix not in {".json", ".jsonl", *_TEXT_ARTIFACT_SUFFIXES}: + raise ValueError(f"unsupported optimizer artifact type: {relative_path}") + if before.st_size > self.max_import_file_bytes: + raise ValueError(f"optimizer artifact exceeds per-file byte limit: " + f"{relative_path} ({before.st_size} > {self.max_import_file_bytes})") + raw = self._read_verified_file(path, before) + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as error: + raise ValueError(f"optimizer text artifact is not UTF-8: {relative_path}") from error + if suffix == ".json": + content = self._render_json(parse_strict_json(text)) + elif suffix == ".jsonl": + payloads = [parse_strict_json(line) for line in text.splitlines() if line.strip()] + content = "\n".join(self._render_jsonl(payload) for payload in payloads) + content += "\n" if payloads else "" + else: + clean = sanitize(text, max_text_chars=None) + if not isinstance(clean, str): + raise TypeError("text audit content must remain text after sanitization") + content = clean + if len(content.encode("utf-8")) > self.max_file_bytes: + raise ValueError(f"audit file exceeds byte limit: {relative_path}") + return _ArtifactSnapshot(relative_path=relative_path, content=content) + + def _snapshot_tree(self, source: str | Path) -> tuple[_ArtifactSnapshot, ...]: + source_root = Path(source) + if _path_is_junction(source_root): + raise ValueError("optimizer artifact symlinks, hard links and reparse points are not allowed") + try: + _require_safe_root(source_root) + except FileNotFoundError: + return () + pending: list[tuple[Path, str]] = [(source_root, "")] + snapshots: list[_ArtifactSnapshot] = [] + total_bytes = 0 + while pending: + directory, prefix = pending.pop() + try: + entries = sorted(os.scandir(directory), key=lambda entry: entry.name) + except OSError as error: + raise ValueError(f"optimizer artifact directory cannot be read: {directory}") from error + for entry in entries: + relative = f"{prefix}/{entry.name}" if prefix else entry.name + for component in Path(relative).parts: + validate_safe_component(component, name="optimizer artifact component") + try: + info = os.lstat(entry.path) + except OSError as error: + raise ValueError(f"optimizer artifact cannot be inspected: {relative}") from error + if _stat_is_link_or_reparse(info) or _path_is_junction(Path(entry.path)): + raise ValueError(f"optimizer artifact symlinks, hard links and reparse points are not allowed: " + f"{relative}") + if stat.S_ISDIR(info.st_mode): + pending.append((Path(entry.path), relative)) + continue + if not stat.S_ISREG(info.st_mode): + raise ValueError(f"optimizer artifact must be a regular file: {relative}") + if len(snapshots) >= self.max_import_files: + raise ValueError(f"optimizer artifact count exceeds limit: " + f"{len(snapshots) + 1} > {self.max_import_files}") + total_bytes += info.st_size + if total_bytes > self.max_import_total_bytes: + raise ValueError(f"optimizer artifacts exceed total byte limit: " + f"{total_bytes} > {self.max_import_total_bytes}") + snapshots.append(self._snapshot_file(Path(entry.path), relative, info)) + return tuple(snapshots) + + def import_tree(self, source: str | Path, destination: str) -> None: + """Validate and sanitize the complete optimizer tree before publishing it.""" + + snapshots = self._snapshot_tree(source) + for snapshot in snapshots: + target = (Path(destination) / snapshot.relative_path).as_posix() + self._atomic_write(self._resolve(target), snapshot.content) def records( self, diff --git a/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py b/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py index 06da40312..818619058 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py +++ b/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py @@ -11,7 +11,12 @@ from .artifacts import AuditSink from .attribution import select_attribution from .contracts import CandidateGenerator -from .evaluation import dataset_fingerprint, split_train_dataset +from .evaluation import ( + dataset_audit_fingerprint, + dataset_contract_payload, + dataset_fingerprint, + split_train_dataset, +) from .models import ( AttributionSnapshot, CandidateProposal, @@ -41,17 +46,21 @@ def prepare_candidate_inputs( ) train_path = sink.write_json( "inner_train.evalset.json", - inner_train.model_dump(mode="json", by_alias=True), + dataset_contract_payload(inner_train), ) selection_path = sink.write_json( "inner_selection.evalset.json", - inner_selection.model_dump(mode="json", by_alias=True), + dataset_contract_payload(inner_selection), ) inner = InnerSplit( train_case_ids=tuple(case.eval_id for case in inner_train.eval_cases), selection_case_ids=tuple(case.eval_id for case in inner_selection.eval_cases), train_hash=dataset_fingerprint(inner_train), selection_hash=dataset_fingerprint(inner_selection), + audit_hashes={ + "train": dataset_audit_fingerprint(inner_train), + "selection": dataset_audit_fingerprint(inner_selection), + }, train_path=train_path.relative_to(sink.run_dir).as_posix(), selection_path=selection_path.relative_to(sink.run_dir).as_posix(), ) @@ -106,7 +115,7 @@ async def generate_candidate( except BaseException as primary_error: try: sink.import_tree(optimizer_output, "candidate_generation/optimizer") - except Exception as import_error: + except BaseException as import_error: add_exception_note( primary_error, f"optimizer artifact import also failed with " diff --git a/examples/optimization/eval_optimize_loop/pipeline/configuration.py b/examples/optimization/eval_optimize_loop/pipeline/configuration.py index 100468098..e3196a669 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/configuration.py +++ b/examples/optimization/eval_optimize_loop/pipeline/configuration.py @@ -156,6 +156,7 @@ class ValidatedRunConfig(StrictModel): train: EvalSet validation: EvalSet input_hashes: dict[str, str] + audit_hashes: dict[str, str] prompt_paths: dict[str, str] prompt_hashes: dict[str, str] adapter_identity: str diff --git a/examples/optimization/eval_optimize_loop/pipeline/evaluation.py b/examples/optimization/eval_optimize_loop/pipeline/evaluation.py index b59815277..ef40c618c 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/evaluation.py +++ b/examples/optimization/eval_optimize_loop/pipeline/evaluation.py @@ -26,6 +26,7 @@ Split, Transition, ) +from .schema import sanitize _VOLATILE_KEYS = { "invocationId", @@ -82,13 +83,36 @@ def case_input_fingerprint(case: Any) -> str: return sha256_json(_without_execution_ids(payload)) -def dataset_fingerprint(eval_set: EvalSet) -> str: - # Hash the submitted dataset contract, not defaults injected by SDK or - # Pydantic versions after validation. - payload = eval_set.model_dump(mode="json", by_alias=True, exclude_unset=True) +def dataset_contract_payload(eval_set: EvalSet) -> dict[str, Any]: + """Return the submitted dataset contract without injected defaults.""" + + return eval_set.model_dump(mode="json", by_alias=True, exclude_unset=True) + + +def dataset_fingerprint_payload(payload: Mapping[str, Any]) -> str: + """Hash a serialized dataset contract after removing execution-only IDs.""" + return sha256_json(_without_execution_ids(payload)) +def dataset_fingerprint(eval_set: EvalSet) -> str: + """Hash the source dataset contract used for replay and trace validation.""" + + return dataset_fingerprint_payload(dataset_contract_payload(eval_set)) + + +def dataset_audit_payload(eval_set: EvalSet) -> dict[str, Any]: + """Return the exact credential-redacted payload persisted in the audit tree.""" + + return sanitize(dataset_contract_payload(eval_set), max_text_chars=None) + + +def dataset_audit_fingerprint(eval_set: EvalSet) -> str: + """Hash the redacted dataset artifact independently from its source hash.""" + + return dataset_fingerprint_payload(dataset_audit_payload(eval_set)) + + def _validate_case_ids(eval_set: EvalSet, label: str, minimum: int) -> set[str]: if len(eval_set.eval_cases) < minimum: raise ValueError(f"{label} must contain at least {minimum} cases") diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py index ff370fccc..7eddc1931 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/models.py +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -268,6 +268,7 @@ class InnerSplit(StrictModel): selection_case_ids: tuple[str, ...] train_hash: str selection_hash: str + audit_hashes: dict[str, str] = Field(default_factory=dict) train_path: str selection_path: str diff --git a/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py index 2b31a34be..e6fa59b0c 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py +++ b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py @@ -15,7 +15,7 @@ from .configuration import ValidatedRunConfig from .contracts import CandidateGenerator, EvaluationBackend from .costing import CostLedger -from .evaluation import compare_snapshots +from .evaluation import compare_snapshots, dataset_contract_payload from .evaluation_runtime import create_evaluation_runtime from .gate import evaluate_gate from .models import ( @@ -44,7 +44,7 @@ persist_terminal_report, utc_now, ) -from .schema import add_exception_note, sanitized_text +from .schema import add_exception_note, sanitized_exception_message, sanitized_text FaultInjector = Callable[[str], None] @@ -225,8 +225,8 @@ def retain_progress(phase: Phase, progress: SnapshotPair) -> None: try: enter_stage(stage) sink.write_json("config.json", validated.config.model_dump(mode="json", by_alias=True)) - sink.write_json("train.evalset.json", validated.train.model_dump(mode="json", by_alias=True)) - sink.write_json("val.evalset.json", validated.validation.model_dump(mode="json", by_alias=True)) + sink.write_json("train.evalset.json", dataset_contract_payload(validated.train)) + sink.write_json("val.evalset.json", dataset_contract_payload(validated.validation)) for name, content in workspace.baseline.items(): sink.write_text(f"baseline_prompts/{name}.md", content) @@ -376,11 +376,30 @@ def decide(duration_seconds: float) -> GateDecision: await workspace.restore() applied = False except PromptRestoreError as restore_error: + original_stage = stage + original_message = sanitized_text(str(error), max_text_chars=settings.max_text_chars) + add_exception_note( + restore_error, + f"pipeline failure before final restoration at stage {original_stage!r}: " + f"{type(error).__name__}: {original_message}", + ) + for note in getattr(error, "__notes__", ()): + add_exception_note( + restore_error, + f"prior pipeline diagnostic: {sanitized_text(str(note), max_text_chars=settings.max_text_chars)}", + ) fatal_restore = restore_error stage = "restore" error = restore_error - message = sanitized_text(error, max_text_chars=settings.max_text_chars) - run_error = RunError(stage=stage, error_type=type(error).__name__, message=message) + message = sanitized_exception_message( + error, + max_text_chars=settings.max_text_chars, + ) + run_error = RunError( + stage=stage, + error_type=type(error).__name__, + message=message, + ) error_report = build_report(Decision.ERROR, stage, (run_error, )) persistence_error: Optional[BaseException] = None try: diff --git a/examples/optimization/eval_optimize_loop/pipeline/preflight.py b/examples/optimization/eval_optimize_loop/pipeline/preflight.py index 5df803a86..619a98bef 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/preflight.py +++ b/examples/optimization/eval_optimize_loop/pipeline/preflight.py @@ -9,7 +9,12 @@ from trpc_agent_sdk.evaluation import EvalSet from .artifacts import load_strict_json -from .evaluation import canonical_json, dataset_fingerprint, validate_datasets +from .evaluation import ( + canonical_json, + dataset_audit_fingerprint, + dataset_fingerprint, + validate_datasets, +) from .configuration import PipelineConfig, ValidatedRunConfig from .live_adapter import LiveAdapterSpec from .schema import validate_safe_component, validate_secret_free_text @@ -135,6 +140,10 @@ def preflight_run( "validation": dataset_fingerprint(validation), } + audit_hashes = { + "train": dataset_audit_fingerprint(train), + "validation": dataset_audit_fingerprint(validation), + } trace_path: Optional[Path] = None if settings.mode == "trace": trace_path = inside_example_root(root, settings.trace_fixture, "trace fixture") @@ -194,6 +203,7 @@ def preflight_run( train=train, validation=validation, input_hashes=hashes, + audit_hashes=audit_hashes, prompt_paths=prompt_paths, prompt_hashes=prompt_hashes(prompts), adapter_identity=adapter_identity, diff --git a/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py b/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py index f65f44a5c..368dda1fb 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py +++ b/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py @@ -11,7 +11,7 @@ from trpc_agent_sdk.evaluation import TargetPrompt -from .schema import validate_safe_component +from .schema import add_exception_note, validate_safe_component class PromptRestoreError(RuntimeError): @@ -140,14 +140,26 @@ async def restore(self) -> None: if hashes != self.baseline_hashes: raise IOError("baseline prompt hashes differ after restoration") except BaseException as error: - raise PromptRestoreError("baseline prompt restoration could not be verified") from error + restore_error = PromptRestoreError("baseline prompt restoration could not be verified") + add_exception_note( + restore_error, + f"restoration operation failed with {type(error).__name__}: {error}", + ) + raise restore_error from error async def apply(self, candidate: dict[str, str]) -> dict[str, str]: self._ensure_initialized() try: return await self._write_verified(candidate) - except BaseException: - await self.restore() + except BaseException as primary_error: + try: + await self.restore() + except PromptRestoreError as restore_error: + add_exception_note( + restore_error, + f"prompt apply also failed with {type(primary_error).__name__}: {primary_error}", + ) + raise raise @asynccontextmanager @@ -165,8 +177,10 @@ async def temporary(self, candidate: dict[str, str]) -> AsyncIterator[dict[str, await self.restore() except PromptRestoreError as restore_error: if primary is not None: - raise PromptRestoreError( - "baseline restoration failed while propagating another error") from restore_error + add_exception_note( + restore_error, + f"prompt operation also failed with {type(primary).__name__}: {primary}", + ) raise def create_candidate_target(self, directory: str) -> TargetPrompt: diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporting.py b/examples/optimization/eval_optimize_loop/pipeline/reporting.py index 561a30f1b..2194f975f 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/reporting.py +++ b/examples/optimization/eval_optimize_loop/pipeline/reporting.py @@ -251,6 +251,7 @@ def build_optimization_report( reproducibility=context.reproducibility, inputs={ "hashes": validated.input_hashes, + "auditHashes": validated.audit_hashes, "paths": { "config": _report_path(validated.config_path, report_root), "train": _report_path(validated.train_path, report_root), diff --git a/examples/optimization/eval_optimize_loop/pipeline/schema.py b/examples/optimization/eval_optimize_loop/pipeline/schema.py index df3830511..6285131bd 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/schema.py +++ b/examples/optimization/eval_optimize_loop/pipeline/schema.py @@ -141,6 +141,11 @@ def validate_secret_free_text(value: str, *, name: str) -> str: def sanitized_text(value: Any, *, max_text_chars: int) -> str: """Render one bounded text value after structured recursive sanitization.""" + if isinstance(value, BaseException): + return sanitized_exception_message(value, max_text_chars=max_text_chars) + if isinstance(value, str) and len(value) > max_text_chars: + marker = "...[truncated]" + value = value[:max(0, max_text_chars - len(marker))] + marker clean = sanitize(value, max_text_chars=max_text_chars) if isinstance(clean, str): text = clean @@ -152,6 +157,40 @@ def sanitized_text(value: Any, *, max_text_chars: int) -> str: return text if len(text) <= max_text_chars else text[:max_text_chars] + "...[truncated]" +def sanitized_exception_message( + error: BaseException, + *, + max_text_chars: int, + max_parts: int = 8, +) -> str: + """Render an exception chain with bounded count, text and total size.""" + + notes = tuple(getattr(error, "__notes__", ())) + all_parts = [("primary", str(error))] + all_parts.extend((f"diagnostic-{index}", str(note)) for index, note in enumerate(notes, start=1)) + omitted = 0 + if len(all_parts) > max_parts: + keep_notes = max_parts - 1 + first = max(1, keep_notes // 2) + last = keep_notes - first + selected = [all_parts[0], *all_parts[1:1 + first]] + if last: + selected.extend(all_parts[-last:]) + omitted = len(all_parts) - len(selected) + all_parts = selected + omitted_text = f"; {omitted} diagnostics omitted" if omitted else "" + overhead = sum(len(label) + 2 for label, _ in all_parts) + len(omitted_text) + part_budget = max(1, (max_text_chars - overhead) // max(1, len(all_parts))) + rendered: list[str] = [] + for label, raw in all_parts: + marker = "...[truncated]" + bounded = raw if len(raw) <= part_budget else raw[:max(0, part_budget - len(marker))] + marker + clean = sanitize(bounded, max_text_chars=part_budget) + rendered.append(f"{label}: {clean if isinstance(clean, str) else str(clean)}") + result = "\n".join(rendered) + omitted_text + return result[:max_text_chars] + + class StrictModel(BaseModel): model_config = ConfigDict( extra="forbid", diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json index 5bfd8c4d5..b64df9ed0 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -2209,6 +2209,10 @@ "train": "af410bcc6f46c041a373efb1d1dda9f0f0f20258d1ef2146eac8a583cde99b19", "validation": "5011756743a6211c78c75f2b7cbc435e98cf4caf0e0c484bbac9c5f03951c7d9" }, + "auditHashes": { + "train": "af410bcc6f46c041a373efb1d1dda9f0f0f20258d1ef2146eac8a583cde99b19", + "validation": "5011756743a6211c78c75f2b7cbc435e98cf4caf0e0c484bbac9c5f03951c7d9" + }, "paths": { "config": "optimizer.json", "train": "train.evalset.json", @@ -2224,6 +2228,10 @@ ], "selectionHash": "ec02d7b1c9840580943cbed756a0d3bc8e0185adf2b2fc7009ad70d61aaca92a", "selectionPath": "inner_selection.evalset.json", + "auditHashes": { + "selection": "ec02d7b1c9840580943cbed756a0d3bc8e0185adf2b2fc7009ad70d61aaca92a", + "train": "70f85fca6ab8ece3ad0e3a3b1c3a6635eab921bf29efa4ed224b22513e9d9cd2" + }, "trainCaseIds": [ "train_improve_alpha", "train_stable" diff --git a/tests/examples/eval_optimize_loop/test_candidate_runtime.py b/tests/examples/eval_optimize_loop/test_candidate_runtime.py index 16add9eff..1bb4261f0 100644 --- a/tests/examples/eval_optimize_loop/test_candidate_runtime.py +++ b/tests/examples/eval_optimize_loop/test_candidate_runtime.py @@ -27,11 +27,14 @@ def create_candidate_target(self, path: str): class _ImportFailingSink: + def __init__(self, error: BaseException | None = None) -> None: + self.error = error or json.JSONDecodeError("half-written optimizer output", "{", 1) + def phase_dir(self, name: str) -> None: return None def import_tree(self, source: str | Path, destination: str) -> None: - raise json.JSONDecodeError("half-written optimizer output", "{", 1) + raise self.error @pytest.mark.asyncio @@ -55,3 +58,26 @@ async def generate(self, **kwargs): inner_selection_path="inner-selection.json", ) assert any("artifact import also failed" in note for note in raised.value.__notes__) + + +@pytest.mark.asyncio +async def test_base_exception_during_import_does_not_replace_generator_failure() -> None: + + class FailingGenerator: + + async def generate(self, **kwargs): + raise RuntimeError("optimizer failed first") + + sink = _ImportFailingSink(KeyboardInterrupt("import interrupted")) + with pytest.raises(RuntimeError, match="optimizer failed first") as raised: + await generate_candidate( + generator=FailingGenerator(), + workspace=_Workspace(), + sink=sink, + eval_config=_DumpConfig(), + optimize_config=_DumpConfig(), + train_attribution=object(), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + ) + assert any("KeyboardInterrupt: import interrupted" in note for note in raised.value.__notes__) diff --git a/tests/examples/eval_optimize_loop/test_deliverables.py b/tests/examples/eval_optimize_loop/test_deliverables.py index 806ac35af..36e36f13b 100644 --- a/tests/examples/eval_optimize_loop/test_deliverables.py +++ b/tests/examples/eval_optimize_loop/test_deliverables.py @@ -53,6 +53,10 @@ def test_committed_sample_report_is_complete_and_rendered_from_the_model() -> No report = OptimizationReport.model_validate(payload) assert report.status == Decision.REJECT assert report.stage == "complete" + assert report.inputs["auditHashes"] == { + "train": report.inputs["hashes"]["train"], + "validation": report.inputs["hashes"]["validation"], + } assert report.duration_seconds < 180 assert report.reproducibility.reproducible is True assert report.reproducibility.git_dirty is False @@ -68,6 +72,10 @@ def test_committed_sample_report_is_complete_and_rendered_from_the_model() -> No Transition.UNCHANGED, } assert report.failure_attribution + assert report.optimization and report.optimization["innerSplit"]["auditHashes"] == { + "train": report.optimization["innerSplit"]["trainHash"], + "selection": report.optimization["innerSplit"]["selectionHash"], + } for snapshot in ( report.failure_attribution.train, report.failure_attribution.validation, diff --git a/tests/examples/eval_optimize_loop/test_models_and_imports.py b/tests/examples/eval_optimize_loop/test_models_and_imports.py index 4dcdc5c63..0c2707691 100644 --- a/tests/examples/eval_optimize_loop/test_models_and_imports.py +++ b/tests/examples/eval_optimize_loop/test_models_and_imports.py @@ -166,6 +166,11 @@ def test_sanitized_text_redacts_structured_values_and_exception_messages() -> No RuntimeError('{"clientSecret":"error-secret"}'), max_text_chars=1000, ) + error = RuntimeError("primary failure") + add_exception_note(error, "Authorization: Bearer diagnostic-secret") + diagnostic = sanitized_text(error, max_text_chars=1000) + assert "diagnostic-secret" not in diagnostic + assert "[REDACTED]" in diagnostic def test_exception_notes_work_without_python_311_add_note() -> None: @@ -253,7 +258,7 @@ def test_pipeline_import_boundaries() -> None: "configuration": {"live_adapter", "models", "schema"}, "contracts": {"models"}, "costing": {"models"}, - "evaluation": {"models"}, + "evaluation": {"models", "schema"}, "evaluation_runtime": { "artifacts", "configuration", diff --git a/tests/examples/eval_optimize_loop/test_orchestrator.py b/tests/examples/eval_optimize_loop/test_orchestrator.py index 5943a6ee3..dd797292e 100644 --- a/tests/examples/eval_optimize_loop/test_orchestrator.py +++ b/tests/examples/eval_optimize_loop/test_orchestrator.py @@ -12,6 +12,7 @@ from pathlib import Path import pytest +from trpc_agent_sdk.evaluation import EvalSet from examples.optimization.eval_optimize_loop.pipeline.models import ( CandidateProposal, @@ -30,10 +31,17 @@ AuditSink, ) from examples.optimization.eval_optimize_loop.pipeline.costing import CostLedger +from examples.optimization.eval_optimize_loop.pipeline.evaluation import ( + dataset_fingerprint, + dataset_fingerprint_payload, +) from examples.optimization.eval_optimize_loop.pipeline.evaluation_runtime import create_evaluation_runtime from examples.optimization.eval_optimize_loop.pipeline.models import Phase, Split from examples.optimization.eval_optimize_loop.pipeline.orchestrator import run_pipeline from examples.optimization.eval_optimize_loop.pipeline.preflight import preflight_run +from examples.optimization.eval_optimize_loop.pipeline.prompt_workspace import ( + PromptRestoreError, +) from examples.optimization.eval_optimize_loop.pipeline import orchestrator as orchestrator_module from examples.optimization.eval_optimize_loop.pipeline import reporting as reporting_module @@ -92,6 +100,54 @@ async def test_fake_reject_with_hard_regression_is_a_completed_audit_run(tmp_pat assert records[name]["sha256"] == hashlib.sha256(content).hexdigest() +@pytest.mark.asyncio +async def test_audit_dataset_copies_round_trip_to_reported_fingerprints(tmp_path) -> None: + root = _example(tmp_path) + report = await run_pipeline(str(root), run_id="dataset-contract") + run_dir = root / "artifacts" / "dataset-contract" + + for source_name, audit_name in ( + ("train.evalset.json", "train.evalset.json"), + ("val.evalset.json", "val.evalset.json"), + ): + source = EvalSet.model_validate(json.loads((root / source_name).read_text(encoding="utf-8"))) + audited = EvalSet.model_validate(json.loads((run_dir / audit_name).read_text(encoding="utf-8"))) + assert dataset_fingerprint(audited) == dataset_fingerprint(source) + + assert report.inputs["auditHashes"] == { + split: report.inputs["hashes"][split] for split in ("train", "validation") + } + assert report.optimization is not None + inner = report.optimization["innerSplit"] + for split_name, hash_name, path_name in ( + ("train", "trainHash", "trainPath"), + ("selection", "selectionHash", "selectionPath"), + ): + audited_payload = json.loads((run_dir / inner[path_name]).read_text(encoding="utf-8")) + audited = EvalSet.model_validate(audited_payload) + assert inner[hash_name] == dataset_fingerprint(audited) + assert inner["auditHashes"][split_name] == dataset_fingerprint_payload(audited_payload) + + +@pytest.mark.asyncio +async def test_secret_bearing_dataset_has_distinct_source_and_audit_hashes(tmp_path) -> None: + root = _example(tmp_path) + train_path = root / "train.evalset.json" + train_payload = json.loads(train_path.read_text(encoding="utf-8")) + train_payload["evalCases"][0]["sessionInput"]["state"]["apiKey"] = "fixture-secret" + train_path.write_text(json.dumps(train_payload), encoding="utf-8") + + source = EvalSet.model_validate(train_payload) + report = await run_pipeline(str(root), run_id="redacted-dataset-contract") + audited_payload = json.loads( + (root / "artifacts" / "redacted-dataset-contract" / "train.evalset.json").read_text(encoding="utf-8")) + + assert audited_payload["evalCases"][0]["sessionInput"]["state"]["apiKey"] == "[REDACTED]" + assert report.inputs["hashes"]["train"] == dataset_fingerprint(source) + assert report.inputs["auditHashes"]["train"] == dataset_fingerprint_payload(audited_payload) + assert report.inputs["hashes"]["train"] != report.inputs["auditHashes"]["train"] + + @pytest.mark.asyncio async def test_accept_without_apply_leaves_source_unchanged(tmp_path) -> None: root = _example(tmp_path, accept=True) @@ -134,6 +190,58 @@ def fail_final_report(stage: str) -> None: assert (root / "prompts" / "system.md").read_text(encoding="utf-8") == baseline +@pytest.mark.asyncio +async def test_apply_and_restore_double_failure_is_preserved_in_terminal_audit(tmp_path, monkeypatch) -> None: + root = _example(tmp_path, accept=True, apply=True) + original_write = orchestrator_module.PromptWorkspace._write_verified + candidate_writes = 0 + + async def fail_apply_and_restore(workspace, prompts): + nonlocal candidate_writes + if prompts != workspace.baseline: + candidate_writes += 1 + if candidate_writes > 1: + raise OSError("candidate write unavailable") + elif candidate_writes > 1: + raise OSError("restore unavailable: " + "r" * 5000) + return await original_write(workspace, prompts) + + monkeypatch.setattr( + orchestrator_module.PromptWorkspace, + "_write_verified", + fail_apply_and_restore, + ) + with pytest.raises(PromptRestoreError): + await run_pipeline(str(root), run_id="apply-restore-double-failure") + + report_path = root / "artifacts" / "apply-restore-double-failure" / "optimization_report.json" + payload = json.loads(report_path.read_text(encoding="utf-8")) + message = payload["errors"][0]["message"] + assert "candidate write unavailable" in message + assert "restore unavailable" in message + assert len(message) <= 4000 + + +@pytest.mark.asyncio +async def test_error_notes_have_bounded_count_and_total_size(tmp_path) -> None: + root = _example(tmp_path) + + class FailingGenerator: + + async def generate(self, **kwargs): + error = RuntimeError("primary failure") + for index in range(100): + error.add_note(f"note-{index}: " + "x" * 10_000) + raise error + + report = await run_pipeline(str(root), run_id="bounded-error-notes", candidate_generator=FailingGenerator()) + assert report.status == Decision.ERROR + message = report.errors[0].message + assert "primary failure" in message + assert "diagnostics omitted" in message + assert len(message) <= 4000 + + @pytest.mark.asyncio @pytest.mark.parametrize( "failed_stage", diff --git a/tests/examples/eval_optimize_loop/test_workspace_artifacts.py b/tests/examples/eval_optimize_loop/test_workspace_artifacts.py index bf1d68de5..d3488a118 100644 --- a/tests/examples/eval_optimize_loop/test_workspace_artifacts.py +++ b/tests/examples/eval_optimize_loop/test_workspace_artifacts.py @@ -4,19 +4,37 @@ import hashlib import json +import os from concurrent.futures import ThreadPoolExecutor +from pathlib import Path import pytest from trpc_agent_sdk.evaluation import TargetPrompt from examples.optimization.eval_optimize_loop.pipeline.artifacts import AuditSink +from examples.optimization.eval_optimize_loop.pipeline import artifacts as artifacts_module from examples.optimization.eval_optimize_loop.pipeline.costing import CostLedger from examples.optimization.eval_optimize_loop.pipeline.prompt_workspace import ( PromptRestoreError, PromptRunLock, PromptWorkspace, ) +from examples.optimization.eval_optimize_loop.pipeline.schema import sanitized_text + + +def _directory_link_or_emulation(link: Path, target: Path, monkeypatch) -> None: + try: + link.symlink_to(target, target_is_directory=True) + except OSError: + link.mkdir() + original = getattr(Path, "is_junction", None) + monkeypatch.setattr( + Path, + "is_junction", + lambda path: path == link or (bool(original(path)) if original else False), + raising=False, + ) @pytest.mark.asyncio @@ -59,7 +77,7 @@ def test_prompt_run_lock_rejects_concurrent_owner_and_releases(tmp_path) -> None @pytest.mark.asyncio -async def test_restoration_failure_is_not_downgraded_to_reject() -> None: +async def test_restoration_failure_preserves_the_primary_operation_diagnostic() -> None: state = {"value": "baseline", "fail_restore": False} async def read() -> str: @@ -72,9 +90,32 @@ async def write(value: str) -> None: workspace = PromptWorkspace(TargetPrompt().add_callback("system", read=read, write=write)) await workspace.initialize() - with pytest.raises(PromptRestoreError): + with pytest.raises(PromptRestoreError) as raised: async with workspace.temporary({"system": "candidate"}): state["fail_restore"] = True + raise RuntimeError("candidate evaluation failed") + message = sanitized_text(raised.value, max_text_chars=4000) + assert "candidate evaluation failed" in message + assert "restore unavailable" in message + + +@pytest.mark.asyncio +async def test_apply_double_failure_preserves_write_and_restore_diagnostics() -> None: + async def read() -> str: + return "baseline" + + async def write(value: str) -> None: + if value == "candidate": + raise OSError("candidate write unavailable") + raise OSError("restore unavailable") + + workspace = PromptWorkspace(TargetPrompt().add_callback("system", read=read, write=write)) + await workspace.initialize() + with pytest.raises(PromptRestoreError) as raised: + await workspace.apply({"system": "candidate"}) + message = sanitized_text(raised.value, max_text_chars=4000) + assert "candidate write unavailable" in message + assert "restore unavailable" in message def test_audit_sink_is_immutable_safe_sanitized_and_manifested(tmp_path) -> None: @@ -153,6 +194,112 @@ def test_optimizer_tree_is_sanitized_before_audit_import(tmp_path) -> None: assert "[REDACTED]" in imported +def test_optimizer_tree_rejects_source_directory_symlink(tmp_path, monkeypatch) -> None: + raw = tmp_path / "raw-optimizer" + raw.mkdir() + (raw / "secret.log").write_text("secret", encoding="utf-8") + alias = tmp_path / "optimizer-alias" + _directory_link_or_emulation(alias, raw, monkeypatch) + + sink = AuditSink(tmp_path / "artifacts", "run-root-link") + sink.create() + with pytest.raises(ValueError, match="symlinks"): + sink.import_tree(alias, "candidate_generation/optimizer") + + +def test_optimizer_tree_rejects_nested_directory_symlink(tmp_path, monkeypatch) -> None: + raw = tmp_path / "raw-optimizer" + external = tmp_path / "external" + raw.mkdir() + external.mkdir() + (external / "secret.log").write_text("secret", encoding="utf-8") + _directory_link_or_emulation(raw / "linked", external, monkeypatch) + + sink = AuditSink(tmp_path / "artifacts", "run-nested-link") + sink.create() + with pytest.raises(ValueError, match="symlinks"): + sink.import_tree(raw, "candidate_generation/optimizer") + + +def test_optimizer_tree_rejects_nested_directory_junction(tmp_path, monkeypatch) -> None: + raw = tmp_path / "raw-optimizer" + linked = raw / "linked" + linked.mkdir(parents=True) + (linked / "outside.log").write_text("outside", encoding="utf-8") + original = getattr(Path, "is_junction", None) + + def is_junction(path: Path) -> bool: + return path == linked or (bool(original(path)) if original else False) + + monkeypatch.setattr(Path, "is_junction", is_junction, raising=False) + sink = AuditSink(tmp_path / "artifacts", "run-nested-junction") + sink.create() + + with pytest.raises(ValueError, match="reparse points"): + sink.import_tree(raw, "candidate_generation/optimizer") + + +def test_optimizer_tree_rejects_broken_source_junction(tmp_path, monkeypatch) -> None: + junction = tmp_path / "broken-junction" + original = getattr(Path, "is_junction", None) + monkeypatch.setattr( + Path, + "is_junction", + lambda path: path == junction or (bool(original(path)) if original else False), + raising=False, + ) + sink = AuditSink(tmp_path / "artifacts", "run-broken-junction") + sink.create() + + with pytest.raises(ValueError, match="reparse points"): + sink.import_tree(junction, "candidate_generation/optimizer") + + +def test_optimizer_tree_rejects_hard_link_to_external_file(tmp_path) -> None: + raw = tmp_path / "raw-optimizer" + raw.mkdir() + external = tmp_path / "external.log" + external.write_text("outside-content", encoding="utf-8") + os.link(external, raw / "outside.log") + + sink = AuditSink(tmp_path / "artifacts", "run-hard-link") + sink.create() + with pytest.raises(ValueError, match="hard links"): + sink.import_tree(raw, "candidate_generation/optimizer") + + +def test_optimizer_tree_validates_all_files_before_publishing(tmp_path) -> None: + raw = tmp_path / "raw-optimizer" + raw.mkdir() + (raw / "valid.log").write_text("valid", encoding="utf-8") + (raw / "invalid.bin").write_bytes(b"invalid") + + sink = AuditSink(tmp_path / "artifacts", "run-batch-validation") + sink.create() + with pytest.raises(ValueError, match="unsupported"): + sink.import_tree(raw, "candidate_generation/optimizer") + assert not (sink.run_dir / "candidate_generation").exists() + + +def test_optimizer_tree_rejects_file_identity_change_before_read(tmp_path, monkeypatch) -> None: + raw = tmp_path / "raw-optimizer" + raw.mkdir() + changed = raw / "changed.log" + changed.write_text("before", encoding="utf-8") + original_open = artifacts_module.os.open + + def replace_before_open(path, flags, *args): + if Path(path) == changed: + changed.write_text("after", encoding="utf-8") + return original_open(path, flags, *args) + + monkeypatch.setattr(artifacts_module.os, "open", replace_before_open) + sink = AuditSink(tmp_path / "artifacts", "run-identity-change") + sink.create() + with pytest.raises(ValueError, match="changed during validation"): + sink.import_tree(raw, "candidate_generation/optimizer") + + @pytest.mark.parametrize( ("sink_kwargs", "files", "message"), [ From decd32d551be1804e8bccc8bad76bb16d23ea8fc Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Fri, 31 Jul 2026 20:57:09 +0800 Subject: [PATCH 10/13] fix(example): fail empty offline evaluations explicitly --- .../pipeline/offline_evaluation.py | 6 ++- .../eval_optimize_loop/test_backends.py | 39 ++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/pipeline/offline_evaluation.py b/examples/optimization/eval_optimize_loop/pipeline/offline_evaluation.py index 74a057a59..d887a577e 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/offline_evaluation.py +++ b/examples/optimization/eval_optimize_loop/pipeline/offline_evaluation.py @@ -137,7 +137,11 @@ def evaluate_invocations( ], )) if not per_invocation: - return EvaluationResult() + return EvaluationResult( + overall_score=0.0, + overall_eval_status=EvalStatus.FAILED, + per_invocation_results=[], + ) overall_score = statistics.fmean(invocation_scores) return EvaluationResult( overall_score=overall_score, diff --git a/tests/examples/eval_optimize_loop/test_backends.py b/tests/examples/eval_optimize_loop/test_backends.py index d0bc96797..889266ca4 100644 --- a/tests/examples/eval_optimize_loop/test_backends.py +++ b/tests/examples/eval_optimize_loop/test_backends.py @@ -9,7 +9,13 @@ import pytest -from trpc_agent_sdk.evaluation import AgentEvaluator, EvalConfig, EvalSet, TargetPrompt +from trpc_agent_sdk.evaluation import ( + AgentEvaluator, + EvalConfig, + EvalSet, + EvalStatus, + TargetPrompt, +) from examples.optimization.eval_optimize_loop.pipeline import backends as backends_module from examples.optimization.eval_optimize_loop.pipeline import optimizer_worker as optimizer_worker_module @@ -25,6 +31,7 @@ LiveAdapterSpec, load_verified_callback, ) +from examples.optimization.eval_optimize_loop.pipeline.offline_evaluation import prepare_offline_evaluation from examples.optimization.eval_optimize_loop.pipeline.models import ( AttributionSnapshot, Phase, @@ -254,6 +261,36 @@ def forbid_real_judge(*args, **kwargs): assert [rubric.id for rubric in run_metrics["llm_rubric_response"].rubrics] == ["response_quality"] +def test_offline_rubric_empty_invocations_are_a_hard_failure() -> None: + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "response_quality", + "type": "OFFLINE_RESPONSE_NON_EMPTY", + }], + } + }, + }], + num_runs=1, + ) + offline_config, registry = prepare_offline_evaluation(config) + assert registry is not None + evaluator = registry.get_evaluator(offline_config.get_eval_metrics()[0]) + + result = evaluator.evaluate_invocations([], None) + + assert result.overall_score == 0.0 + assert result.overall_eval_status == EvalStatus.FAILED + assert result.per_invocation_results == [] + + @pytest.mark.asyncio async def test_fake_knowledge_recall_fails_before_black_box_inference(tmp_path, monkeypatch) -> None: From 9172044e00acf279dad87882fef7c01dd062a739 Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Fri, 31 Jul 2026 21:30:19 +0800 Subject: [PATCH 11/13] fix(example): harden CLI and prompt locking --- .../pipeline/prompt_workspace.py | 48 ++++++++++++------- .../eval_optimize_loop/run_pipeline.py | 15 +++++- tests/examples/eval_optimize_loop/test_cli.py | 18 +++++++ .../test_workspace_artifacts.py | 46 ++++++++++++++++++ 4 files changed, 110 insertions(+), 17 deletions(-) create mode 100644 tests/examples/eval_optimize_loop/test_cli.py diff --git a/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py b/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py index 368dda1fb..7f3f10604 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py +++ b/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py @@ -38,25 +38,47 @@ def __init__( self.path = root / f"{digest}.lock" self._handle = None + @staticmethod + def _lock(handle) -> None: + if os.name == "nt": + import msvcrt + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + + @staticmethod + def _unlock(handle) -> None: + handle.seek(0) + if os.name == "nt": + import msvcrt + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + def acquire(self) -> None: if self._handle is not None: raise RuntimeError("prompt run lock is already acquired") self.path.parent.mkdir(parents=True, exist_ok=True) handle = self.path.open("a+b") - if self.path.stat().st_size == 0: - handle.write(b"\0") - handle.flush() handle.seek(0) try: - if os.name == "nt": - import msvcrt - msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) - else: - import fcntl - fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + self._lock(handle) except (OSError, BlockingIOError) as error: handle.close() raise RuntimeError("prompt sources are already owned by another pipeline run") from error + try: + if handle.seek(0, os.SEEK_END) == 0: + handle.write(b"\0") + handle.flush() + except BaseException: + try: + self._unlock(handle) + except OSError: + pass + handle.close() + raise self._handle = handle def release(self) -> None: @@ -64,13 +86,7 @@ def release(self) -> None: if handle is None: return try: - handle.seek(0) - if os.name == "nt": - import msvcrt - msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) - else: - import fcntl - fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + self._unlock(handle) finally: handle.close() self._handle = None diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 77b417101..8adfdcd1e 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -14,6 +14,19 @@ from examples.optimization.eval_optimize_loop.pipeline.orchestrator import run_pipeline from examples.optimization.eval_optimize_loop.pipeline.live_adapter import load_callback +from examples.optimization.eval_optimize_loop.pipeline.schema import sanitized_exception_message + +_CLI_ERROR_MAX_CHARS = 4000 + + +def _format_cli_error(error: BaseException) -> str: + error_type = type(error).__name__[:128] + prefix = f"ERROR: {error_type}: " + summary = sanitized_exception_message( + error, + max_text_chars=_CLI_ERROR_MAX_CHARS - len(prefix), + ) + return prefix + summary def _parser() -> argparse.ArgumentParser: @@ -57,5 +70,5 @@ async def _main(args: argparse.Namespace) -> int: except (KeyboardInterrupt, asyncio.CancelledError): raise except Exception as error: - print(f"ERROR: {type(error).__name__}: {error}", file=sys.stderr) + print(_format_cli_error(error), file=sys.stderr) raise SystemExit(1) diff --git a/tests/examples/eval_optimize_loop/test_cli.py b/tests/examples/eval_optimize_loop/test_cli.py new file mode 100644 index 000000000..db9154e60 --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_cli.py @@ -0,0 +1,18 @@ +"""Command-line boundary tests.""" + +from examples.optimization.eval_optimize_loop.run_pipeline import _format_cli_error +from examples.optimization.eval_optimize_loop.pipeline.schema import add_exception_note + + +def test_cli_error_output_is_bounded_and_redacted() -> None: + error = RuntimeError( + "Authorization: Bearer live-token; api_key=live-key; " + "context" * 1000) + add_exception_note(error, "password=note-secret") + + rendered = _format_cli_error(error) + + assert len(rendered) <= 4000 + assert "live-token" not in rendered + assert "live-key" not in rendered + assert "note-secret" not in rendered + assert rendered.count("[REDACTED]") == 3 diff --git a/tests/examples/eval_optimize_loop/test_workspace_artifacts.py b/tests/examples/eval_optimize_loop/test_workspace_artifacts.py index d3488a118..78c120aba 100644 --- a/tests/examples/eval_optimize_loop/test_workspace_artifacts.py +++ b/tests/examples/eval_optimize_loop/test_workspace_artifacts.py @@ -4,6 +4,7 @@ import hashlib import json +import multiprocessing import os from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -37,6 +38,21 @@ def _directory_link_or_emulation(link: Path, target: Path, monkeypatch) -> None: ) +def _contend_for_prompt_lock(prompt_path: str, lock_root: str, start, release, outcomes) -> None: + lock = PromptRunLock((prompt_path, ), lock_root=lock_root) + start.wait(timeout=10) + try: + lock.acquire() + except RuntimeError: + outcomes.put("blocked") + return + try: + outcomes.put("acquired") + release.wait(timeout=10) + finally: + lock.release() + + @pytest.mark.asyncio async def test_temporary_candidate_always_restores_baseline(tmp_path) -> None: prompt_path = tmp_path / "system.md" @@ -76,6 +92,36 @@ def test_prompt_run_lock_rejects_concurrent_owner_and_releases(tmp_path) -> None assert second.path == first.path +def test_prompt_run_lock_first_acquire_has_one_sentinel_byte(tmp_path) -> None: + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + lock_root = str(tmp_path / "locks") + context = multiprocessing.get_context("spawn") + start = context.Event() + release = context.Event() + outcomes = context.Queue() + processes = [ + context.Process( + target=_contend_for_prompt_lock, + args=(str(prompt_path), lock_root, start, release, outcomes), + ) for _ in range(2) + ] + for process in processes: + process.start() + start.set() + try: + results = [outcomes.get(timeout=15) for _ in processes] + finally: + release.set() + for process in processes: + process.join(timeout=15) + + assert sorted(results) == ["acquired", "blocked"] + assert all(process.exitcode == 0 for process in processes) + lock_path = PromptRunLock((str(prompt_path), ), lock_root=lock_root).path + assert lock_path.read_bytes() == b"\0" + + @pytest.mark.asyncio async def test_restoration_failure_preserves_the_primary_operation_diagnostic() -> None: state = {"value": "baseline", "fail_restore": False} From dfdb011dae582c7a349651460de59369acde829a Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Fri, 31 Jul 2026 22:23:09 +0800 Subject: [PATCH 12/13] fix(example): harden optimizer worker shutdown --- .../optimization/eval_optimize_loop/DESIGN.md | 9 +- .../eval_optimize_loop/pipeline/backends.py | 138 ++++++++++++++---- .../pipeline/optimizer_worker.py | 13 +- .../pipeline/orchestrator.py | 8 +- .../eval_optimize_loop/pipeline/reporting.py | 8 +- .../eval_optimize_loop/test_backends.py | 128 +++++++++++++++- .../test_models_and_imports.py | 10 ++ 7 files changed, 271 insertions(+), 43 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index 39a38cee6..2a255547c 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -90,7 +90,7 @@ with prompt_set_lock(validated.prompt_paths): terminal = configured_gate(regression, cost, current_duration) restore_verified_baseline() if terminal rejects an applied candidate persist_terminal_audit_once_or_raise() -on cancel: request_stop -> bounded_wait -> terminate -> bounded_wait -> kill +on cancel: shielded_reaper(request_stop -> bounded_wait -> terminate -> bounded_wait -> kill) -> re-raise cancel ``` 1. Preflight strictly parses config and datasets, rejects duplicate JSON keys, @@ -147,9 +147,10 @@ Default live evaluation and optimization resolve one importable callback through fingerprint to the run, and the worker verifies all three after re-importing it. A different callback object, source, cached code version, or source version is rejected before optimization. Live optimization always uses -`optimizer_worker.py`. Cancellation -writes the SDK stop signal, waits for the configured bound, then terminates and -finally kills a worker that still does not exit. Programmatically injected +`optimizer_worker.py`. Cancellation starts an independent reaper that reuses one +observed wait task, writes the SDK stop signal, waits for the configured bound, +then terminates and finally kills a worker that still does not exit. Repeated +parent cancellation cannot interrupt this bounded cleanup. Programmatically injected backends and generators remain trusted in-process components and make the report non-reproducible. A successful SDK result must report the exact workspace baseline and registered best-prompt keys before it can enter regression. Failed diff --git a/examples/optimization/eval_optimize_loop/pipeline/backends.py b/examples/optimization/eval_optimize_loop/pipeline/backends.py index 060ab81c5..06e69ab00 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/backends.py +++ b/examples/optimization/eval_optimize_loop/pipeline/backends.py @@ -39,7 +39,7 @@ Split, ) from .offline_evaluation import prepare_offline_evaluation -from .schema import add_exception_note, parse_strict_json +from .schema import add_exception_note, parse_strict_json, sanitized_text from .trace_fixture import TraceFixture CallAgent = Callable[[str], Awaitable[str]] @@ -342,45 +342,127 @@ def _request_stop(output_dir: str, cancellation: BaseException) -> None: except OSError as stop_error: add_exception_note(cancellation, f"could not request optimizer stop: {stop_error}") - async def _stop_worker( + async def _wait_for_worker( self, process: asyncio.subprocess.Process, - *, - output_dir: str, cancellation: BaseException, - ) -> None: - self._request_stop(output_dir, cancellation) + wait_task: Optional[asyncio.Task[int]], + ) -> tuple[bool, Optional[asyncio.Task[int]]]: + if wait_task is None: + wait_task = asyncio.create_task(process.wait()) try: await asyncio.wait_for( - asyncio.shield(process.wait()), + asyncio.shield(wait_task), timeout=self._shutdown_timeout_seconds, ) - return + return True, None except asyncio.TimeoutError: - if process.returncode is None: - try: - process.terminate() - except ProcessLookupError: - pass + return False, wait_task + except asyncio.CancelledError: + raise + except Exception as wait_error: + detail = sanitized_text(wait_error, max_text_chars=1000) + add_exception_note( + cancellation, + f"optimizer worker wait failed with {type(wait_error).__name__}: {detail}", + ) + return process.returncode is not None, None + + @staticmethod + def _signal_worker( + process: asyncio.subprocess.Process, + signal_name: str, + cancellation: BaseException, + ) -> bool: + if process.returncode is not None: + return False try: - await asyncio.wait_for( - asyncio.shield(process.wait()), - timeout=self._shutdown_timeout_seconds, + getattr(process, signal_name)() + return True + except ProcessLookupError: + return False + except OSError as signal_error: + detail = sanitized_text(signal_error, max_text_chars=1000) + add_exception_note( + cancellation, + f"optimizer worker {signal_name} failed with {type(signal_error).__name__}: {detail}", ) - except asyncio.TimeoutError: - if process.returncode is None: + return False + + async def _stop_worker( + self, + process: asyncio.subprocess.Process, + *, + output_dir: str, + cancellation: BaseException, + ) -> None: + self._request_stop(output_dir, cancellation) + wait_task: Optional[asyncio.Task[int]] = None + forced = False + try: + exited, wait_task = await self._wait_for_worker(process, cancellation, wait_task) + if exited: + return + forced = self._signal_worker(process, "terminate", cancellation) + exited, wait_task = await self._wait_for_worker(process, cancellation, wait_task) + if exited: + if forced: + add_exception_note(cancellation, "optimizer worker required forced termination") + return + forced = self._signal_worker(process, "kill", cancellation) or forced + exited, wait_task = await self._wait_for_worker(process, cancellation, wait_task) + if not exited: + add_exception_note(cancellation, "optimizer worker did not report exit after kill") + if forced: + add_exception_note(cancellation, "optimizer worker required forced termination") + finally: + if wait_task is not None: + wait_task.cancel() try: - process.kill() - except ProcessLookupError: + await wait_task + except BaseException: pass + + async def _complete_worker_stop( + self, + process: asyncio.subprocess.Process, + *, + output_dir: str, + cancellation: BaseException, + ) -> None: + """Finish bounded child cleanup even if the parent task is canceled again.""" + + cleanup = asyncio.create_task( + self._stop_worker( + process, + output_dir=output_dir, + cancellation=cancellation, + )) + repeated_cancellations = 0 + while not cleanup.done(): try: - await asyncio.wait_for( - asyncio.shield(process.wait()), - timeout=self._shutdown_timeout_seconds, - ) - except asyncio.TimeoutError: - add_exception_note(cancellation, "optimizer worker did not report exit after kill") - add_exception_note(cancellation, "optimizer worker required forced termination") + await asyncio.shield(cleanup) + except asyncio.CancelledError: + repeated_cancellations += 1 + continue + except BaseException: + break + if repeated_cancellations: + add_exception_note( + cancellation, + f"worker cleanup resisted {repeated_cancellations} additional cancellation request(s)", + ) + if cleanup.cancelled(): + add_exception_note(cancellation, "optimizer worker cleanup task was canceled before completion") + return + try: + cleanup.result() + except BaseException as cleanup_error: + detail = sanitized_text(cleanup_error, max_text_chars=1000) + add_exception_note( + cancellation, + f"optimizer worker cleanup failed with {type(cleanup_error).__name__}: {detail}", + ) async def _optimize_in_worker(self, **kwargs) -> OptimizeResult: output_dir = kwargs["output_dir"] @@ -417,7 +499,7 @@ async def _optimize_in_worker(self, **kwargs) -> OptimizeResult: try: return_code = await process.wait() except asyncio.CancelledError as cancellation: - await self._stop_worker( + await self._complete_worker_stop( process, output_dir=output_dir, cancellation=cancellation, diff --git a/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py b/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py index c583826ce..b898bf242 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py +++ b/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py @@ -10,7 +10,7 @@ from trpc_agent_sdk.evaluation import AgentOptimizer, TargetPrompt from .live_adapter import load_verified_callback -from .schema import parse_strict_json, sanitize +from .schema import parse_strict_json, sanitize, sanitized_exception_message async def _run(request_path: Path) -> None: @@ -48,6 +48,8 @@ def main() -> int: request_path = Path(parser.parse_args().request).resolve() try: asyncio.run(_run(request_path)) + except (KeyboardInterrupt, SystemExit): + raise except BaseException as error: try: request = parse_strict_json(request_path.read_text(encoding="utf-8")) @@ -56,15 +58,18 @@ def main() -> int: payload = sanitize( { "errorType": type(error).__name__, - "message": str(error), + "message": sanitized_exception_message( + error, + max_text_chars=4000, + ), }, - max_text_chars=4000, + max_text_chars=None, ) (output_dir / "worker_error.json").write_text( json.dumps(payload, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8", ) - except BaseException: + except Exception: pass return 1 return 0 diff --git a/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py index e6fa59b0c..7d199ddcd 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py +++ b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py @@ -244,7 +244,9 @@ def retain_progress(phase: Phase, progress: SnapshotPair) -> None: baseline_validation = baseline_pair.validation baseline_train_attr = baseline_attribution_pair.train baseline_val_attr = baseline_attribution_pair.validation - assert baseline_train and baseline_validation and baseline_train_attr and baseline_val_attr + if (baseline_train is None or baseline_validation is None or baseline_train_attr is None + or baseline_val_attr is None): + raise RuntimeError("baseline evaluation did not produce complete snapshots and attribution") enter_stage("inner_split") inner, inner_train_attr, inner_train_path, inner_selection_path = prepare_candidate_inputs( @@ -302,7 +304,9 @@ def retain_progress(phase: Phase, progress: SnapshotPair) -> None: candidate_validation = candidate_pair.validation candidate_train_attr = candidate_attribution_pair.train candidate_val_attr = candidate_attribution_pair.validation - assert candidate_train and candidate_validation and candidate_train_attr and candidate_val_attr + if (candidate_train is None or candidate_validation is None or candidate_train_attr is None + or candidate_val_attr is None): + raise RuntimeError("candidate evaluation did not produce complete snapshots and attribution") enter_stage("comparison") train_comparison = compare_snapshots( diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporting.py b/examples/optimization/eval_optimize_loop/pipeline/reporting.py index 2194f975f..079ef9642 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/reporting.py +++ b/examples/optimization/eval_optimize_loop/pipeline/reporting.py @@ -140,8 +140,9 @@ def build_reproducibility( reason = "live_callback_not_importable" elif reason is None and mode == "trace" and (not trace_fixture or not trace_hash): reason = "trace_fixture_not_pinned" + elif reason is None and git_root is None: + reason = "git_root_unavailable" if reason is None: - assert git_root is not None for input_path in input_paths: resolved = Path(input_path).resolve() try: @@ -163,7 +164,8 @@ def build_reproducibility( reproducible = reason is None command = None if reproducible: - assert git_root is not None + if git_root is None: + raise RuntimeError("reproducible replay requires a resolved Git root") def replay_path(path: str | Path) -> str: resolved = Path(path).resolve() @@ -173,6 +175,8 @@ def replay_path(path: str | Path) -> str: return str(resolved) args = [ + # Keep the report portable across checkouts; environment metadata + # records the interpreter version used for the original run. "python", replay_path(Path(repo_root) / "run_pipeline.py"), "--mode", diff --git a/tests/examples/eval_optimize_loop/test_backends.py b/tests/examples/eval_optimize_loop/test_backends.py index 889266ca4..d58cec1f4 100644 --- a/tests/examples/eval_optimize_loop/test_backends.py +++ b/tests/examples/eval_optimize_loop/test_backends.py @@ -4,6 +4,7 @@ import asyncio import json +import sys from pathlib import Path from types import SimpleNamespace @@ -762,6 +763,57 @@ async def fake_optimize(**kwargs): assert captured["validation_dataset_path"] == "inner-selection.json" +@pytest.mark.parametrize("error_type", (KeyboardInterrupt, SystemExit)) +def test_optimizer_worker_rethrows_process_control(monkeypatch, tmp_path, error_type) -> None: + output_dir = tmp_path / "optimizer-output" + request_path = tmp_path / "request.json" + request_path.write_text( + json.dumps({ + "outputDir": str(output_dir) + }), + encoding="utf-8", + ) + control = error_type("stop worker") + + async def fail(_request_path): + raise control + + monkeypatch.setattr(optimizer_worker_module, "_run", fail) + monkeypatch.setattr(sys, "argv", ["optimizer_worker", str(request_path)]) + + with pytest.raises(error_type) as raised: + optimizer_worker_module.main() + + assert raised.value is control + assert not (output_dir / "worker_error.json").exists() + + +def test_optimizer_worker_error_artifact_is_bounded_and_redacted(monkeypatch, tmp_path) -> None: + output_dir = tmp_path / "optimizer-output" + request_path = tmp_path / "request.json" + request_path.write_text( + json.dumps({ + "outputDir": str(output_dir) + }), + encoding="utf-8", + ) + + async def fail(_request_path): + raise RuntimeError( + "Authorization: Bearer worker-token; api_key=worker-key; " + "context" * 1000) + + monkeypatch.setattr(optimizer_worker_module, "_run", fail) + monkeypatch.setattr(sys, "argv", ["optimizer_worker", str(request_path)]) + + assert optimizer_worker_module.main() == 1 + payload = json.loads((output_dir / "worker_error.json").read_text(encoding="utf-8")) + assert payload["errorType"] == "RuntimeError" + assert len(payload["message"]) <= 4000 + assert "worker-token" not in payload["message"] + assert "worker-key" not in payload["message"] + assert payload["message"].count("[REDACTED]") == 2 + + @pytest.mark.asyncio async def test_live_generator_accepts_sdk_skipped_round(monkeypatch, tmp_path) -> None: async def fake_optimize(**kwargs): @@ -928,18 +980,23 @@ async def test_live_generator_rejects_invalid_success_contract( @pytest.mark.asyncio -async def test_live_worker_is_forcibly_terminated_after_bounded_cooperative_stop(monkeypatch, tmp_path) -> None: +async def test_live_worker_is_reaped_after_repeated_parent_cancellation(monkeypatch, tmp_path) -> None: class StuckProcess: def __init__(self) -> None: self.returncode = None self.waiting = asyncio.Event() + self.cleanup_waiting = asyncio.Event() self.finished = asyncio.Event() + self.wait_calls = 0 self.terminated = False self.killed = False async def wait(self): + self.wait_calls += 1 self.waiting.set() + if self.wait_calls >= 2: + self.cleanup_waiting.set() await self.finished.wait() return self.returncode @@ -965,7 +1022,7 @@ async def fake_subprocess(*args, **kwargs): task = asyncio.create_task( LiveCandidateGenerator( _live_adapter(), - shutdown_timeout_seconds=0.01, + shutdown_timeout_seconds=0.1, ).generate( target_prompt=TargetPrompt().add_path("system", str(prompt_path)), baseline_prompts={"system": "baseline"}, @@ -977,11 +1034,76 @@ async def fake_subprocess(*args, **kwargs): )) await process.waiting.wait() task.cancel() - with pytest.raises(asyncio.CancelledError): + await process.cleanup_waiting.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError) as raised: await task assert (output_dir / "optimize.stop").read_text(encoding="utf-8") == "cancel requested\n" assert process.terminated is True assert process.returncode in {-15, -9} + assert any("additional cancellation" in note for note in raised.value.__notes__) + + +@pytest.mark.asyncio +async def test_live_worker_cleanup_escalates_after_wait_and_terminate_errors(monkeypatch, tmp_path) -> None: + class KillOnlyProcess: + + def __init__(self) -> None: + self.returncode = None + self.waiting = asyncio.Event() + self.finished = asyncio.Event() + self.wait_calls = 0 + self.killed = False + + async def wait(self): + self.wait_calls += 1 + self.waiting.set() + if self.wait_calls == 2: + raise OSError("Authorization: Bearer wait-secret") + await self.finished.wait() + return self.returncode + + def terminate(self): + raise PermissionError("api_key=terminate-secret") + + def kill(self): + self.killed = True + self.returncode = -9 + self.finished.set() + + process = KillOnlyProcess() + + async def fake_subprocess(*args, **kwargs): + return process + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_subprocess) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + task = asyncio.create_task( + LiveCandidateGenerator( + _live_adapter(), + shutdown_timeout_seconds=0.01, + ).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + )) + await process.waiting.wait() + task.cancel() + + with pytest.raises(asyncio.CancelledError) as raised: + await task + + diagnostics = "\n".join(raised.value.__notes__) + assert process.killed is True + assert process.returncode == -9 + assert "wait-secret" not in diagnostics + assert "terminate-secret" not in diagnostics + assert diagnostics.count("[REDACTED]") == 2 @pytest.mark.asyncio diff --git a/tests/examples/eval_optimize_loop/test_models_and_imports.py b/tests/examples/eval_optimize_loop/test_models_and_imports.py index 0c2707691..60a4496d6 100644 --- a/tests/examples/eval_optimize_loop/test_models_and_imports.py +++ b/tests/examples/eval_optimize_loop/test_models_and_imports.py @@ -241,6 +241,16 @@ def _pipeline_imports(path: Path) -> set[str]: return imports +def test_production_pipeline_has_no_optimizable_assertions() -> None: + paths = [*PIPELINE_DIR.glob("*.py"), EXAMPLE_DIR / "run_pipeline.py"] + violations = [] + for path in paths: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + violations.extend( + f"{path.name}:{node.lineno}" for node in ast.walk(tree) if isinstance(node, ast.Assert)) + assert violations == [] + + def test_pipeline_import_boundaries() -> None: allowed = { "artifacts": {"models", "schema"}, From 812a9f67b72300732f31da5b010a3fe69b671b8e Mon Sep 17 00:00:00 2001 From: Audience-jmf <2500840938@qq.com> Date: Fri, 31 Jul 2026 23:11:31 +0800 Subject: [PATCH 13/13] fix(example): preserve worker failure diagnostics --- .../eval_optimize_loop/pipeline/backends.py | 80 ++++++++++++++++--- .../eval_optimize_loop/pipeline/reporting.py | 6 +- .../eval_optimize_loop/test_backends.py | 51 ++++++++++++ .../eval_optimize_loop/test_orchestrator.py | 10 ++- 4 files changed, 132 insertions(+), 15 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/pipeline/backends.py b/examples/optimization/eval_optimize_loop/pipeline/backends.py index 06e69ab00..9c0c92a7c 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/backends.py +++ b/examples/optimization/eval_optimize_loop/pipeline/backends.py @@ -45,6 +45,7 @@ CallAgent = Callable[[str], Awaitable[str]] _DEFAULT_APP_NAME = "test_app" _REPOSITORY_ROOT = Path(__file__).resolve().parents[4] +_WORKER_STDERR_TAIL_BYTES = 4000 async def _evaluate_with_sdk( @@ -335,6 +336,8 @@ def __init__( @staticmethod def _request_stop(output_dir: str, cancellation: BaseException) -> None: + # The SDK's GepaReflectiveOptimizer registers a FileStopper for this + # exact path. Forced process termination remains the bounded fallback. stop_path = Path(output_dir) / "optimize.stop" try: stop_path.parent.mkdir(parents=True, exist_ok=True) @@ -342,6 +345,43 @@ def _request_stop(output_dir: str, cancellation: BaseException) -> None: except OSError as stop_error: add_exception_note(cancellation, f"could not request optimizer stop: {stop_error}") + @staticmethod + async def _capture_worker_stderr(stderr: Optional[asyncio.StreamReader]) -> str: + """Drain worker stderr continuously and retain only a sanitized tail.""" + + if stderr is None: + return "" + tail = bytearray() + truncated = False + try: + while True: + chunk = await stderr.read(8192) + if not chunk: + break + if len(chunk) >= _WORKER_STDERR_TAIL_BYTES: + tail[:] = chunk[-_WORKER_STDERR_TAIL_BYTES:] + truncated = True + continue + tail.extend(chunk) + overflow = len(tail) - _WORKER_STDERR_TAIL_BYTES + if overflow > 0: + del tail[:overflow] + truncated = True + except asyncio.CancelledError: + raise + except Exception as capture_error: + detail = sanitized_text(capture_error, max_text_chars=1000) + return f"stderr capture failed with {type(capture_error).__name__}: {detail}" + if not tail: + return "" + text = tail.decode("utf-8", errors="replace") + if truncated: + line_end = text.find("\n") + if line_end < 0: + return "[stderr tail omitted: final line exceeded capture limit]" + text = text[line_end + 1:] + return sanitized_text(text, max_text_chars=_WORKER_STDERR_TAIL_BYTES) + async def _wait_for_worker( self, process: asyncio.subprocess.Process, @@ -424,20 +464,31 @@ async def _stop_worker( pass async def _complete_worker_stop( - self, - process: asyncio.subprocess.Process, - *, - output_dir: str, - cancellation: BaseException, + self, + process: asyncio.subprocess.Process, + *, + output_dir: str, + cancellation: BaseException, + stderr_capture: asyncio.Task[str], ) -> None: """Finish bounded child cleanup even if the parent task is canceled again.""" - cleanup = asyncio.create_task( - self._stop_worker( - process, - output_dir=output_dir, - cancellation=cancellation, - )) + async def stop_and_discard_stderr() -> None: + try: + await self._stop_worker( + process, + output_dir=output_dir, + cancellation=cancellation, + ) + finally: + if not stderr_capture.done(): + stderr_capture.cancel() + try: + await stderr_capture + except asyncio.CancelledError: + pass + + cleanup = asyncio.create_task(stop_and_discard_stderr()) repeated_cancellations = 0 while not cleanup.done(): try: @@ -494,8 +545,9 @@ async def _optimize_in_worker(self, **kwargs) -> OptimizeResult: str(request_path), cwd=str(_REPOSITORY_ROOT), stdout=asyncio.subprocess.DEVNULL, - stderr=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, ) + stderr_capture = asyncio.create_task(self._capture_worker_stderr(getattr(process, "stderr", None))) try: return_code = await process.wait() except asyncio.CancelledError as cancellation: @@ -503,8 +555,10 @@ async def _optimize_in_worker(self, **kwargs) -> OptimizeResult: process, output_dir=output_dir, cancellation=cancellation, + stderr_capture=stderr_capture, ) raise + stderr_detail = await stderr_capture result_path = Path(output_dir) / "result.json" if return_code != 0: error_path = Path(output_dir) / "worker_error.json" @@ -512,6 +566,8 @@ async def _optimize_in_worker(self, **kwargs) -> OptimizeResult: if error_path.is_file(): payload = parse_strict_json(error_path.read_text(encoding="utf-8")) detail = str(payload.get("message", detail)) + elif stderr_detail: + detail = f"{detail}; sanitized stderr tail: {stderr_detail}" raise RuntimeError(f"AgentOptimizer worker failed with exit code {return_code}: {detail}") if not result_path.is_file(): raise RuntimeError("AgentOptimizer worker completed without result.json") diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporting.py b/examples/optimization/eval_optimize_loop/pipeline/reporting.py index 079ef9642..563b0fc50 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/reporting.py +++ b/examples/optimization/eval_optimize_loop/pipeline/reporting.py @@ -33,6 +33,8 @@ SourceApplication, ) +_GIT_OBJECT_ID_PATTERN = re.compile(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})\Z") + @dataclass(frozen=True) class ReportContext: @@ -130,7 +132,9 @@ def build_reproducibility( ).stdout.strip()).resolve() except (OSError, subprocess.SubprocessError): reason = "git_metadata_unavailable" - if reason is None and (commit is None or not re.fullmatch(r"[0-9a-fA-F]{40}", commit)): + # Git repositories may use SHA-1 or SHA-256 object IDs. ``rev-parse`` + # already verified that HEAD resolves; this check rejects malformed output. + if reason is None and (commit is None or _GIT_OBJECT_ID_PATTERN.fullmatch(commit) is None): reason = "git_commit_invalid" elif reason is None and dirty: reason = "worktree_dirty" diff --git a/tests/examples/eval_optimize_loop/test_backends.py b/tests/examples/eval_optimize_loop/test_backends.py index d58cec1f4..d42728741 100644 --- a/tests/examples/eval_optimize_loop/test_backends.py +++ b/tests/examples/eval_optimize_loop/test_backends.py @@ -1106,6 +1106,57 @@ async def fake_subprocess(*args, **kwargs): assert diagnostics.count("[REDACTED]") == 2 +@pytest.mark.asyncio +async def test_live_worker_uses_sanitized_stderr_when_error_artifact_is_missing(monkeypatch, tmp_path) -> None: + stderr = asyncio.StreamReader() + stderr.feed_data(b"".join(( + b"old-diagnostic\nAuthorization: Bearer ", + b"boundary-secret-" * 400, + b"\nImportError: failed before worker main; Authorization: Bearer startup-secret\n", + ))) + stderr.feed_eof() + + class FailedProcess: + returncode = 1 + + def __init__(self) -> None: + self.stderr = stderr + + async def wait(self): + return self.returncode + + captured = {} + + async def fake_subprocess(*args, **kwargs): + captured.update(kwargs) + return FailedProcess() + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_subprocess) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(RuntimeError) as raised: + await LiveCandidateGenerator(_live_adapter()).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + ) + + message = str(raised.value) + assert captured["stderr"] is asyncio.subprocess.PIPE + assert "failed without an error artifact" in message + assert "ImportError: failed before worker main" in message + assert "startup-secret" not in message + assert "boundary-secret" not in message + assert "[REDACTED]" in message + assert "old-diagnostic" not in message + assert len(message) < 4200 + + @pytest.mark.asyncio async def test_live_worker_success_is_loaded_through_the_strict_candidate_adapter(monkeypatch, tmp_path) -> None: round_ = SimpleNamespace( diff --git a/tests/examples/eval_optimize_loop/test_orchestrator.py b/tests/examples/eval_optimize_loop/test_orchestrator.py index dd797292e..6dd3ebde4 100644 --- a/tests/examples/eval_optimize_loop/test_orchestrator.py +++ b/tests/examples/eval_optimize_loop/test_orchestrator.py @@ -699,14 +699,19 @@ async def generate(self, **kwargs): assert "COST_UNAVAILABLE" in report.gate_decision.reasons -def test_reproducibility_command_contains_all_effective_cli_inputs(tmp_path, monkeypatch) -> None: +@pytest.mark.parametrize("object_id_length", (40, 64)) +def test_reproducibility_command_contains_all_effective_cli_inputs( + tmp_path, + monkeypatch, + object_id_length, +) -> None: repo = tmp_path / "repo with space" root = repo / "examples" / "optimization" / "eval_optimize_loop" root.mkdir(parents=True) def fake_run(args, **kwargs): if args[-1] == "HEAD": - stdout = "a" * 40 + "\n" + stdout = "a" * object_id_length + "\n" elif args[-1] == "--porcelain": stdout = "" elif args[-1] == "--show-toplevel": @@ -735,6 +740,7 @@ def fake_run(args, **kwargs): ) args = shlex.split(result.command) assert result.reproducible is True + assert result.git_commit == "a" * object_id_length assert args[1] == "examples/optimization/eval_optimize_loop/run_pipeline.py" assert args[args.index("--config") + 1].endswith("custom config.json") assert args[args.index("--train") + 1].endswith("custom train.json")