From cee4ee455926c10bad2a0937a35f120fdea72987 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 08:37:00 -0400 Subject: [PATCH 01/64] Build fresh household sets and run the tool pilot on them scripts/household_set.py samples households from the dataset the installed policyengine.py bundle certifies, renames them mc_000 onward, computes every output on the installed policyengine-us with the reference conventions, and writes a board in the frozen layout with batches. On the frozen board's own households it reproduces all 1,935 scored 2.41.0 references exactly. The tool pilot takes --board: the engine and fix pins come from the board's last engine_upgrade revision, and a household set's arms are scored against the same model's no-tools run (--baseline-dir). Co-Authored-By: Claude Opus 5.5 --- scripts/household_set.py | 421 ++++++++++++++++++++++++++++++++++++ scripts/tool_pilot_argon.py | 187 ++++++++++++---- tests/test_household_set.py | 282 ++++++++++++++++++++++++ 3 files changed, 848 insertions(+), 42 deletions(-) create mode 100644 scripts/household_set.py create mode 100644 tests/test_household_set.py diff --git a/scripts/household_set.py b/scripts/household_set.py new file mode 100644 index 00000000..5b6a01ce --- /dev/null +++ b/scripts/household_set.py @@ -0,0 +1,421 @@ +"""Build a fresh US household set for PolicyBench, in the frozen board's layout. + +Max, 2026-10-22: v2's head-to-head runs on fresh households from the latest +Microcosm build, with references from the latest policyengine-us, served as +one certified pair by the installed policyengine.py bundle. The run goes +slowly, in small batches with AI judges between them, because new microdata +can reach engine paths the frozen 100 households never did. + + household_set.py build --n 100 --seed 20261022 --batch-size 10 --out DIR + household_set.py references --board DIR --out FILE + household_set.py check-frozen --out FILE + +``build`` samples households from the dataset the installed policyengine.py +bundle certifies (:func:`policybench.scenarios.generate_scenarios`, the board's +own sampler and eligibility rules), renames them ``mc_000`` onward so they +never share an id with the frozen board, computes every output on the +installed policyengine-us with the reference conventions +(reference_audit/2026-09-28/fixes/latest_final.py), and writes a board +directory the tool pilot reads with ``--board``: + + scenarios.csv (+ .meta.json) the households, as the frozen board + reference_outputs.csv (+ .meta.json) value and impact_weight per output + reference_exclusions.json an empty exclusion record + batches.json the households in batches, in draw order + +The reference sidecar has the frozen board's shape: one ``engine_upgrade`` +revision naming the engine and pinning every fix module by sha256, so the +pilot's pin check reads either board the same way. + +``references`` recomputes a board's references and reports any output that +differs (the determinism check). ``check-frozen`` is the differential check +of the reference path: on the frozen board's own households it must reproduce +every scored reference, which holds only on the engine those references were +built with (policyengine-us 2.41.0). + +The set is not public. It stays outside the repository until a release +publishes it, so a web-search arm cannot find its answers. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import datetime +import hashlib +import importlib.util +import json +import sys +from pathlib import Path + +import pandas as pd + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +FROZEN_BOARD = ( + ROOT + / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +FIX_MODULE = ROOT / "reference_audit/2026-09-28/fixes/latest_final.py" +YEAR = 2026 +# The v1.1 reference run renamed this input for policyengine-us 1.755.4; the +# frozen households still carry the old name (reference_audit/2026-09-28/ +# scripts/sweep.py build_situation). Harmless for a household without it. +RENAME = {"partnership_se_income": "partnership_self_employment_net_earnings"} +EXCLUSIONS_RULE = ( + "An output is excluded from scoring for every model when review finds its " + "reference wrong in law or undetermined by the stated facts. A fresh household " + "set starts with no exclusions; its batch judges propose them." +) + + +def sha256_bytes(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def sha256_file(path: Path) -> str: + return sha256_bytes(Path(path).read_bytes()) + + +def installed_engine() -> str: + from importlib import metadata + + return f"policyengine-us {metadata.version('policyengine-us')}" + + +def frozen_programs() -> list[str]: + meta = json.loads((FROZEN_BOARD / "reference_outputs.csv.meta.json").read_text()) + return list(meta["programs"]) + + +def frozen_fix_pins() -> list[dict]: + """The fix modules the frozen 2.41.0 references were built with.""" + meta = json.loads((FROZEN_BOARD / "reference_outputs.csv.meta.json").read_text()) + revisions = [ + r + for r in meta["revisions"] + if r.get("kind") == "engine_upgrade" + and r.get("engine_version") == "policyengine-us 2.41.0" + ] + if len(revisions) != 1: + raise SystemExit("the frozen board has no single 2.41.0 engine_upgrade") + return revisions[0]["fix_modules"] + + +def fix_module_pins(fix_module: Path = FIX_MODULE) -> list[dict]: + """Each module the reform composes, at its current bytes. + + The set of modules is the frozen board's; each must exist and match the + frozen pin, so a household set's references follow exactly the frozen + conventions. Changing a convention for a new engine is an engine-upgrade + audit, which pins its own modules. + """ + pins, problems = [], [] + for pin in frozen_fix_pins(): + path = ROOT / pin["path"] + if not path.exists(): + problems.append(f"{pin['path']} is missing") + continue + digest = sha256_file(path) + if digest != pin["sha256"]: + problems.append(f"{pin['path']} is not the frozen {pin['sha256'][:12]}") + pins.append({"module": pin["module"], "path": pin["path"], "sha256": digest}) + if problems: + raise SystemExit("; ".join(problems)) + return pins + + +def load_reform(fix_module: Path = FIX_MODULE): + spec = importlib.util.spec_from_file_location("household_set_fixes", fix_module) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module.reform + + +def reference_system(fix_module: Path = FIX_MODULE): + from policyengine_us import CountryTaxBenefitSystem + + return CountryTaxBenefitSystem(reform=load_reform(fix_module)) + + +def situation_for(scenario) -> dict: + situation = scenario.to_pe_household() + for person in situation["people"].values(): + for old, new in RENAME.items(): + if old in person: + person[new] = person.pop(old) + return situation + + +def compute_references(scenarios, programs: list[str], system) -> pd.DataFrame: + """Every output of every household on ``system``: the ground-truth path + (policybench.ground_truth._calculate_ground_truth_us_scalar) with the + reference conventions.""" + from policyengine_us import Simulation + + from policybench.ground_truth import ( + _extract_person_impact_weight, + _extract_person_value, + _impact_weight_variable_for_output, + _pe_variable_for_output, + ) + from policybench.spec import expand_programs_for_scenario + + rows = [] + for scenario in scenarios: + simulation = Simulation( + tax_benefit_system=system, situation=situation_for(scenario) + ) + for variable in expand_programs_for_scenario(programs, scenario): + result = simulation.calculate(_pe_variable_for_output(variable, "us"), YEAR) + weight_variable = _impact_weight_variable_for_output(variable, "us") + weight = None + if weight_variable is not None: + weight = _extract_person_impact_weight( + result, + simulation.calculate(weight_variable, YEAR), + scenario, + variable, + ) + rows.append( + { + "scenario_id": scenario.id, + "variable": variable, + "value": float(_extract_person_value(result, scenario, variable)), + "impact_weight": weight, + } + ) + return pd.DataFrame( + rows, columns=["scenario_id", "variable", "value", "impact_weight"] + ) + + +def rename_scenarios(scenarios, prefix: str) -> list: + """The sampled households under fresh ids, in draw order.""" + return [ + dataclasses.replace(scenario, id=f"{prefix}_{index:03d}") + for index, scenario in enumerate(scenarios) + ] + + +def split_batches(ids: list[str], size: int) -> list[list[str]]: + if size < 1: + raise ValueError("batch size must be at least 1") + return [ids[start : start + size] for start in range(0, len(ids), size)] + + +def load_scenarios(board: Path) -> list: + from policybench.scenarios import scenario_from_dict + + frame = pd.read_csv(Path(board) / "scenarios.csv") + return [scenario_from_dict(json.loads(text)) for text in frame["scenario_json"]] + + +def _utc_now() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def command_build(args) -> None: + from policybench.policyengine_runtime import ( + materialize_certified_us_dataset, + policyengine_bundles_for_countries, + ) + from policybench.scenarios import generate_scenarios, scenario_manifest + + out = Path(args.out) + if out.exists() and any(out.iterdir()): + raise SystemExit(f"{out} is not empty; a household set is written once") + engine = installed_engine() + pins = fix_module_pins(Path(args.fix_module)) + programs = frozen_programs() + bundles = policyengine_bundles_for_countries(["us"]) + dataset_path = Path(materialize_certified_us_dataset()) + dataset_sha256 = sha256_file(dataset_path) + certified = bundles["us"].get("certified_data_artifact_sha256") + if certified and certified != dataset_sha256: + raise SystemExit( + f"{dataset_path} hashes to {dataset_sha256}, not the bundle's {certified}" + ) + + sampled = generate_scenarios(n=args.n, seed=args.seed, country="us") + scenarios = rename_scenarios(sampled, args.id_prefix) + references = compute_references( + scenarios, programs, reference_system(Path(args.fix_module)) + ) + + out.mkdir(parents=True, exist_ok=True) + manifest = scenario_manifest(scenarios) + manifest.to_csv(out / "scenarios.csv", index=False) + references.to_csv(out / "reference_outputs.csv", index=False) + batches = split_batches([s.id for s in scenarios], args.batch_size) + (out / "batches.json").write_text( + json.dumps({"batch_size": args.batch_size, "batches": batches}, indent=1) + "\n" + ) + (out / "reference_exclusions.json").write_text( + json.dumps( + { + "schema_version": 1, + "rule": EXCLUSIONS_RULE, + "derivation": f"Built {_utc_now()[:10]} with no exclusions.", + "exclusions": [], + }, + indent=1, + ) + + "\n" + ) + generated = _utc_now() + common = { + "country": "us", + "metadata_version": 1, + "board_kind": "household_set", + "generated_at_utc": generated, + "num_scenarios": len(scenarios), + "requested_num_scenarios": args.n, + "seed": args.seed, + "id_prefix": args.id_prefix, + "program_set": "headline", + "programs": programs, + "policyengine_bundles": bundles, + "dataset": { + "path_name": dataset_path.name, + "sha256": dataset_sha256, + "bundle_default_dataset_uri": bundles["us"].get("default_dataset_uri"), + }, + "source_households": [ + {"scenario_id": s.id, **{k: s.metadata[k] for k in sorted(s.metadata)}} + for s in scenarios + ], + } + (out / "scenarios.csv.meta.json").write_text( + json.dumps( + { + **common, + "task": "scenario_manifest", + "scenarios_csv_sha256": sha256_file(out / "scenarios.csv"), + }, + indent=1, + ) + + "\n" + ) + (out / "reference_outputs.csv.meta.json").write_text( + json.dumps( + { + **common, + "task": "reference_outputs", + "reference_csv_sha256": sha256_file(out / "reference_outputs.csv"), + "scenarios_csv_sha256": sha256_file(out / "scenarios.csv"), + "revisions": [ + { + "kind": "engine_upgrade", + "engine_version": engine, + "date": generated[:10], + "builder": "scripts/household_set.py build", + "fix_module": { + "path": str( + Path(args.fix_module).resolve().relative_to(ROOT) + ), + "sha256": sha256_file(Path(args.fix_module)), + }, + "fix_modules": pins, + "outputs": len(references), + } + ], + }, + indent=1, + ) + + "\n" + ) + print( + f"{len(scenarios)} households, {len(references)} outputs on {engine}; " + f"{len(batches)} batches of up to {args.batch_size} in {out}" + ) + + +def differences( + expected: pd.DataFrame, computed: pd.DataFrame, tolerance: float +) -> pd.DataFrame: + """Outputs whose computed value is missing or differs beyond ``tolerance``.""" + merged = expected.merge( + computed[["scenario_id", "variable", "value"]], + on=["scenario_id", "variable"], + how="left", + suffixes=("", "_computed"), + ) + merged["delta"] = (merged["value_computed"] - merged["value"]).abs() + return merged[merged["value_computed"].isna() | (merged["delta"] > tolerance)] + + +def command_references(args) -> None: + board = Path(args.board) + expected = pd.read_csv(board / "reference_outputs.csv") + meta = json.loads((board / "reference_outputs.csv.meta.json").read_text()) + computed = compute_references( + load_scenarios(board), meta["programs"], reference_system(Path(args.fix_module)) + ) + diff = differences(expected, computed, args.tolerance) + computed.to_csv(args.out, index=False) + print( + f"{len(computed)} outputs recomputed on {installed_engine()}; " + f"{len(diff)} differ" + ) + if len(diff): + print(diff.to_string(index=False)) + raise SystemExit(1) + + +def command_check_frozen(args) -> None: + """The reference path on the frozen board must reproduce every scored output.""" + from policybench.reference_exclusions import ( + load_reference_exclusions, + split_reference, + ) + + engine = installed_engine() + if engine != "policyengine-us 2.41.0": + raise SystemExit(f"{engine} is installed; the frozen references are 2.41.0's") + fix_module_pins(FIX_MODULE) + reference = pd.read_csv(FROZEN_BOARD / "reference_outputs.csv") + scored, _ = split_reference(reference, load_reference_exclusions(FROZEN_BOARD)) + computed = compute_references( + load_scenarios(FROZEN_BOARD), frozen_programs(), reference_system(FIX_MODULE) + ) + computed.to_csv(args.out, index=False) + diff = differences(scored, computed, args.tolerance) + print( + f"{len(scored)} scored outputs; {len(scored) - len(diff)} reproduced within " + f"{args.tolerance} on {engine}" + ) + if len(diff): + print(diff.to_string(index=False)) + raise SystemExit(1) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + build = sub.add_parser("build", help="sample a household set and its references") + build.add_argument("--n", type=int, required=True) + build.add_argument("--seed", type=int, required=True) + build.add_argument("--batch-size", type=int, default=10) + build.add_argument("--id-prefix", default="mc") + build.add_argument("--fix-module", default=str(FIX_MODULE)) + build.add_argument("--out", required=True) + build.set_defaults(func=command_build) + refs = sub.add_parser("references", help="recompute a board's references") + refs.add_argument("--board", required=True) + refs.add_argument("--fix-module", default=str(FIX_MODULE)) + refs.add_argument("--tolerance", type=float, default=1e-6) + refs.add_argument("--out", required=True) + refs.set_defaults(func=command_references) + frozen = sub.add_parser("check-frozen", help="reproduce the frozen references") + frozen.add_argument("--tolerance", type=float, default=1e-6) + frozen.add_argument("--out", required=True) + frozen.set_defaults(func=command_check_frozen) + args = parser.parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 97d880e7..ea8f85f4 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -826,44 +826,67 @@ def execute_request(system, registry, request: dict, simulation_cls=None) -> dic return {"results": results} +def _board_upgrade(snapshot: Path) -> dict: + """A board's last engine_upgrade revision: the engine its references are + on and the fix modules it pins. The frozen board's is 2.41.0's; a household + set (scripts/household_set.py) records one.""" + meta = json.loads((snapshot / "reference_outputs.csv.meta.json").read_text()) + revisions = [r for r in meta["revisions"] if r.get("kind") == "engine_upgrade"] + if not revisions: + raise SystemExit(f"{snapshot} records no engine_upgrade revision") + return revisions[-1] + + +def board_engine(snapshot: Path = SNAPSHOT) -> str: + """The policyengine-us version a board's references were computed on.""" + return _board_upgrade(snapshot)["engine_version"].removeprefix("policyengine-us ") + + +def is_household_set(snapshot: Path) -> bool: + meta = json.loads((snapshot / "reference_outputs.csv.meta.json").read_text()) + return meta.get("board_kind") == "household_set" + + def pinned_fix_modules(snapshot: Path = SNAPSHOT) -> list[dict]: - """The fix modules the 2.41.0 references were built with, as the sidecar's + """The fix modules a board's references were built with, as its sidecar's engine_upgrade revision pins them (path and sha256).""" - meta = json.loads((snapshot / "reference_outputs.csv.meta.json").read_text()) - revisions = [ - r - for r in meta["revisions"] - if r.get("kind") == "engine_upgrade" - and r.get("engine_version") == f"policyengine-us {REFERENCE_ENGINE}" - ] - if len(revisions) != 1: - raise SystemExit(f"expected one {REFERENCE_ENGINE} engine_upgrade revision") - return revisions[0]["fix_modules"] + return _board_upgrade(snapshot)["fix_modules"] + + +def board_fix_module_sha256(snapshot: Path = SNAPSHOT) -> str: + """The pinned latest_final.py: the constant for the frozen board, the + sidecar's record for a household set.""" + if not is_household_set(snapshot): + return FIX_MODULE_SHA256 + return _board_upgrade(snapshot)["fix_module"]["sha256"] -def verify_fix_pins(fixes_dir: Path = FIX_MODULE.parent) -> None: +def verify_fix_pins( + fixes_dir: Path = FIX_MODULE.parent, snapshot: Path = SNAPSHOT +) -> None: """Every module latest_final.py composes must be the pinned bytes. Each pin is checked at its recorded path; a module the reform reads from its own directory (the sales tax tables) is also checked there. """ problems = [] - for pin in pinned_fix_modules(): + for pin in pinned_fix_modules(snapshot): candidates = {ROOT / pin["path"], fixes_dir / pin["module"]} for path in sorted(candidates): if path.exists() and _sha256(path) != pin["sha256"]: problems.append(f"{path} does not match the pinned {pin['sha256']}") if not any(path.exists() for path in candidates): problems.append(f"{pin['module']} is missing") - if _sha256(FIX_MODULE) != FIX_MODULE_SHA256: - problems.append(f"{FIX_MODULE} does not match {FIX_MODULE_SHA256}") + pinned = board_fix_module_sha256(snapshot) + if _sha256(FIX_MODULE) != pinned: + problems.append(f"{FIX_MODULE} does not match {pinned}") if problems: raise SystemExit("; ".join(problems)) -def load_fix_reform(path: Path = FIX_MODULE): +def load_fix_reform(path: Path = FIX_MODULE, snapshot: Path = SNAPSHOT): if path == FIX_MODULE: - verify_fix_pins() + verify_fix_pins(snapshot=snapshot) spec = importlib.util.spec_from_file_location("tool_pilot_latest_final", path) module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module @@ -871,18 +894,21 @@ def load_fix_reform(path: Path = FIX_MODULE): return module.reform -def build_tool_system(reform: bool = True, path: Path = FIX_MODULE): +def build_tool_system( + reform: bool = True, path: Path = FIX_MODULE, snapshot: Path = SNAPSHOT +): from policyengine_us import CountryTaxBenefitSystem if not reform: return CountryTaxBenefitSystem() - return CountryTaxBenefitSystem(reform=load_fix_reform(path)) + return CountryTaxBenefitSystem(reform=load_fix_reform(path, snapshot)) -def _worker_main(conn, fix_path: str, reform: bool) -> None: +def _worker_main(conn, fix_path: str, reform: bool, board: str | None = None) -> None: for name in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS"): os.environ[name] = "1" - system = build_tool_system(reform=reform, path=Path(fix_path)) + snapshot = Path(board) if board else SNAPSHOT + system = build_tool_system(reform=reform, path=Path(fix_path), snapshot=snapshot) registry = build_registry(system) conn.send(("ready", None)) while True: @@ -910,9 +936,13 @@ def __init__( reform: bool = True, worker_target: Callable | None = None, acquire_timeout: float | None = None, + board: Path | None = None, ): self._ctx = mp.get_context("spawn") self._fix_path = str(fix_path) + # A worker gets the board only when it is not the frozen one, so a + # three-argument worker target still starts on the default. + self._board = None if board is None or board == SNAPSHOT else str(board) self._reform = reform self._target = worker_target or _worker_main self.timeout = timeout @@ -932,11 +962,10 @@ def __init__( def _start(self): parent, child = self._ctx.Pipe() - process = self._ctx.Process( - target=self._target, - args=(child, self._fix_path, self._reform), - daemon=True, - ) + args = (child, self._fix_path, self._reform) + if self._board is not None: + args += (self._board,) + process = self._ctx.Process(target=self._target, args=args, daemon=True) process.start() return process, parent @@ -2366,28 +2395,38 @@ def _versions() -> dict: return out -def _require_engine() -> None: +def board_path(args) -> Path: + """The board a command reads: --board, or the frozen snapshot.""" + return Path(getattr(args, "board", None) or SNAPSHOT) + + +def _require_engine(board: Path = SNAPSHOT) -> None: version = _versions()["policyengine-us"] - if version != REFERENCE_ENGINE: + pinned = board_engine(board) + if version != pinned: + where = ( + f"the reference engine of household set {board}" + if is_household_set(board) + else "the reference engine of release dashboard-data-20261014" + ) raise SystemExit( - f"policyengine-us {version} is installed; the pilot pins " - f"{REFERENCE_ENGINE}, the reference engine of release " - "dashboard-data-20261014." + f"policyengine-us {version} is installed; the pilot pins {pinned}, {where}." ) def command_oracle(args) -> None: """Reference situations through the tool engine: every scored reference must come back within the board's tolerance.""" - _require_engine() + snapshot = board_path(args) + _require_engine(snapshot) from policybench.ground_truth import _extract_person_value, _pe_variable_for_output - board = load_board() + board = load_board(snapshot) scored_keys = set(zip(board["scored"].scenario_id, board["scored"].variable)) values = board["reference"].set_index(["scenario_id", "variable"])["value"] rows = [] for label, reform in (("reference_engine", True), ("plain_engine", False)): - system = build_tool_system(reform=reform) + system = build_tool_system(reform=reform, snapshot=snapshot) registry = build_registry(system) for sid, scenario in board["scenarios"].items(): situation = scenario.to_pe_household() @@ -2565,7 +2604,8 @@ def records_arm(records: Mapping[str, dict]) -> str: def command_run(args) -> None: - _require_engine() + snapshot = board_path(args) + _require_engine(snapshot) profile = model_profile( args.model, args.arm, args.prompt, args.forbid_benchmark_search ) @@ -2579,7 +2619,7 @@ def command_run(args) -> None: from litellm import completion, responses completion_fn = responses if profile.api == "responses" else completion - board = load_board() + board = load_board(snapshot) out = Path(args.out_dir) households_dir = out / "households" households_dir.mkdir(parents=True, exist_ok=True) @@ -2620,6 +2660,9 @@ def command_run(args) -> None: "versions": _versions(), "fix_module": str(FIX_MODULE.relative_to(ROOT)), "fix_module_sha256": _sha256(FIX_MODULE), + "board": str(snapshot), + "board_engine": f"policyengine-us {board_engine(snapshot)}", + "board_reference_sha256": board["meta"].get("reference_csv_sha256"), "model": profile.name, "model_id": profile.model_id, "arm": profile.arm, @@ -2662,7 +2705,9 @@ def command_run(args) -> None: if not todo: return # Arms without the calculator start no engine workers. - pool = ToolPool(size=args.workers) if profile.uses_calculator else None + pool = ( + ToolPool(size=args.workers, board=snapshot) if profile.uses_calculator else None + ) def no_calculator(arguments) -> tuple[dict, float]: return {"error": "not_run", "message": "no calculator in this arm"}, 0.0 @@ -2797,11 +2842,36 @@ def arm1_predictions( return rows +def baseline_predictions( + records: Mapping[str, dict], label: str = ARM1_LABEL +) -> pd.DataFrame: + """A no-tools run's records as the board's prediction frame, with each + household's usage on its first row so per-household sums are its totals.""" + frame = arm2_predictions(records, label) + usage = { + "prompt_tokens": "prompt_tokens", + "completion_tokens": "completion_tokens", + "reasoning_tokens": "reasoning_tokens", + "total_cost_usd": "cost_usd", + "elapsed_seconds": "model_seconds", + "cached_prompt_tokens": "cached_prompt_tokens", + } + first = ~frame.duplicated("scenario_id") + for column, total in usage.items(): + frame[column] = [ + float(records[sid]["totals"].get(total) or 0.0) if is_first else 0.0 + for sid, is_first in zip(frame.scenario_id, first) + ] + frame["call_id"] = frame.scenario_id + return frame + + def command_score(args) -> None: from policybench.analysis import analyze_no_tools from policybench.spec import output_group_id - board = load_board() + snapshot = board_path(args) + board = load_board(snapshot) out = Path(args.out_dir) records = load_records(out) profile = model_profile( @@ -2825,7 +2895,28 @@ def command_score(args) -> None: f"failure and score as misses until a resume reruns them: {leftover}" ) scored = board["scored"][board["scored"].scenario_id.isin(households)] - arm1 = arm1_predictions(label=arm1_label) + if is_household_set(snapshot): + # No board run exists for a fresh household set: the baseline is the + # same model's no-tools arm on it, run with the same prompt setup. + baseline_dir = getattr(args, "baseline_dir", None) + if not baseline_dir: + raise SystemExit("a household set is scored against --baseline-dir") + baseline = load_records(Path(baseline_dir)) + if records_model(baseline) != profile.name or records_arm(baseline) != "none": + raise SystemExit( + f"--baseline-dir holds {records_model(baseline)} " + f"{records_arm(baseline)} records, not {profile.name} none" + ) + if records_prompt(baseline)[0] != profile.prompt_version: + raise SystemExit("--baseline-dir was run on another prompt version") + missing = sorted(set(households) - set(baseline)) + if missing: + raise SystemExit(f"--baseline-dir lacks households {missing}") + arm1 = baseline_predictions( + {sid: baseline[sid] for sid in households}, arm1_label + ) + else: + arm1 = arm1_predictions(label=arm1_label) arm1 = arm1[arm1.scenario_id.isin(households)] arm2 = arm2_predictions({sid: records[sid] for sid in households}, arm2_label) scenarios = board["scenarios_frame"][ @@ -4003,7 +4094,8 @@ def situation_fidelity(records, board, system, registry, defaults, out: Path) -> def command_diagnose(args) -> None: from policybench.spec import metric_type_for_output - board = load_board() + snapshot = board_path(args) + board = load_board(snapshot) out = Path(args.out_dir) records = load_records(out) rows_path = out / "scored_rows.csv" @@ -4014,8 +4106,8 @@ def command_diagnose(args) -> None: registry = None defaults = {} if not args.no_engine_defaults: - _require_engine() - system = build_tool_system(reform=True) + _require_engine(snapshot) + system = build_tool_system(reform=True, snapshot=snapshot) registry = build_registry(system) defaults = { name: system.variables[name].default_value for name in system.variables @@ -4146,6 +4238,12 @@ def main(argv: list[str] | None = None) -> None: for name in ("oracle", "run", "score", "diagnose"): p = sub.add_parser(name) p.add_argument("--out-dir", required=True) + p.add_argument( + "--board", + default=str(SNAPSHOT), + help="the frozen board (default) or a household set from " + "scripts/household_set.py", + ) if name == "run": p.add_argument( "--model", @@ -4186,6 +4284,11 @@ def main(argv: list[str] | None = None) -> None: action="store_true", help="score only households whose conversation completed", ) + p.add_argument( + "--baseline-dir", + help="on a household set, the same model's no-tools run: the " + "baseline each arm is compared with", + ) if name == "diagnose": p.add_argument("--no-engine-defaults", action="store_true") args = parser.parse_args(argv) diff --git a/tests/test_household_set.py b/tests/test_household_set.py new file mode 100644 index 00000000..ca51ea71 --- /dev/null +++ b/tests/test_household_set.py @@ -0,0 +1,282 @@ +"""The household-set builder and the tool pilot's --board path. + +The build test runs the real engine on three of the frozen board's own +households, with sampling and the dataset download faked, so the references it +writes must equal the frozen ones on the frozen engine (policyengine-us +2.41.0). It is marked slow. +""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import json +import sys +from pathlib import Path + +import pandas as pd +import pytest +from hypothesis import given +from hypothesis import strategies as st + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import household_set as hs # noqa: E402 +import tool_pilot_argon as pilot # noqa: E402 + +ON_FROZEN_ENGINE = importlib.metadata.version("policyengine-us") == "2.41.0" +SOL = "gpt-6-sol" + + +def test_rename_gives_fresh_ids_in_draw_order(): + scenarios = hs.load_scenarios(hs.FROZEN_BOARD)[:3] + renamed = hs.rename_scenarios(list(reversed(scenarios)), "mc") + assert [s.id for s in renamed] == ["mc_000", "mc_001", "mc_002"] + assert renamed[0].state == scenarios[2].state + assert renamed[0].metadata == scenarios[2].metadata + assert [s.id for s in scenarios] == ["scenario_000", "scenario_001", "scenario_002"] + + +@given(st.lists(st.text(min_size=1), unique=True, max_size=60), st.integers(1, 15)) +def test_batches_partition_the_draw_in_order(ids, size): + batches = hs.split_batches(ids, size) + assert [sid for batch in batches for sid in batch] == ids + assert all(len(batch) == size for batch in batches[:-1]) + assert all(1 <= len(batch) <= size for batch in batches) + + +def test_batches_refuse_a_size_below_one(): + with pytest.raises(ValueError): + hs.split_batches(["a"], 0) + + +def test_differences_report_missing_and_moved_outputs(): + expected = pd.DataFrame( + { + "scenario_id": ["a", "a", "b"], + "variable": ["x", "y", "x"], + "value": [1.0, 2.0, 3.0], + } + ) + computed = pd.DataFrame( + {"scenario_id": ["a", "a"], "variable": ["x", "y"], "value": [1.0, 2.5]} + ) + diff = hs.differences(expected, computed, tolerance=1e-6) + assert sorted(zip(diff.scenario_id, diff.variable)) == [("a", "y"), ("b", "x")] + assert hs.differences(expected, computed, tolerance=1.0).scenario_id.tolist() == [ + "b" + ] + + +def test_fix_module_pins_are_the_frozen_revision(): + pins = hs.fix_module_pins() + assert pins == [ + {"module": p["module"], "path": p["path"], "sha256": p["sha256"]} + for p in hs.frozen_fix_pins() + ] + + +def test_the_frozen_board_reads_as_before(): + assert pilot.board_engine() == pilot.REFERENCE_ENGINE + assert not pilot.is_household_set(pilot.SNAPSHOT) + assert pilot.board_fix_module_sha256() == pilot.FIX_MODULE_SHA256 + assert pilot.board_path(argparse.Namespace()) == pilot.SNAPSHOT + + +@pytest.fixture(scope="module") +def household_board(tmp_path_factory): + """A three-household set built from the frozen board's first households.""" + if not ON_FROZEN_ENGINE: + pytest.skip("the build test reproduces the 2.41.0 references") + import policybench.policyengine_runtime as runtime + import policybench.scenarios as scenarios_module + + root = tmp_path_factory.mktemp("set") + dataset = root / "populace_us_2024.h5" + dataset.write_bytes(b"not a real dataset") + digest = hs.sha256_file(dataset) + sampled = hs.load_scenarios(hs.FROZEN_BOARD)[:3] + patch = pytest.MonkeyPatch() + patch.setattr( + scenarios_module, "generate_scenarios", lambda n, seed, country: sampled[:n] + ) + patch.setattr(runtime, "materialize_certified_us_dataset", lambda: str(dataset)) + patch.setattr( + runtime, + "policyengine_bundles_for_countries", + lambda countries: { + "us": { + "bundle_id": "us-test", + "certified_data_artifact_sha256": digest, + "default_dataset_uri": "hf://policyengine/populace-us/x.h5@test", + } + }, + ) + out = root / "board" + try: + hs.main( + ["build", "--n", "3", "--seed", "7", "--batch-size", "2", "--out", str(out)] + ) + finally: + patch.undo() + return out, sampled + + +@pytest.mark.slow +def test_build_writes_the_frozen_references_for_frozen_households(household_board): + out, sampled = household_board + frozen = pd.read_csv(hs.FROZEN_BOARD / "reference_outputs.csv") + built = pd.read_csv(out / "reference_outputs.csv") + names = {f"mc_{i:03d}": s.id for i, s in enumerate(sampled)} + built["frozen_id"] = built.scenario_id.map(names) + merged = built.merge( + frozen, + left_on=["frozen_id", "variable"], + right_on=["scenario_id", "variable"], + suffixes=("", "_frozen"), + ) + assert ( + len(merged) + == len(built) + == len(frozen[frozen.scenario_id.isin(names.values())]) + ) + from policybench.reference_exclusions import load_reference_exclusions + + excluded = { + (e["scenario_id"], e["variable"]) + for e in load_reference_exclusions(hs.FROZEN_BOARD) + } + scored = merged[ + [(s, v) not in excluded for s, v in zip(merged.frozen_id, merged.variable)] + ] + assert (scored.value - scored.value_frozen).abs().max() < 1e-6 + + +@pytest.mark.slow +def test_build_writes_a_board_the_pilot_reads(household_board): + out, _ = household_board + meta = json.loads((out / "reference_outputs.csv.meta.json").read_text()) + assert meta["board_kind"] == "household_set" + assert meta["seed"] == 7 and meta["num_scenarios"] == 3 + assert meta["reference_csv_sha256"] == hs.sha256_file(out / "reference_outputs.csv") + assert json.loads((out / "batches.json").read_text())["batches"] == [ + ["mc_000", "mc_001"], + ["mc_002"], + ] + board = pilot.load_board(out) + assert sorted(board["scenarios"]) == ["mc_000", "mc_001", "mc_002"] + assert board["excluded"].empty + assert pilot.is_household_set(out) + assert pilot.board_engine(out) == "2.41.0" + pilot.verify_fix_pins(snapshot=out) + pilot._require_engine(out) + + +@pytest.mark.slow +def test_build_refuses_a_written_directory_and_a_dataset_off_the_bundle( + household_board, tmp_path, monkeypatch +): + out, _ = household_board + with pytest.raises(SystemExit, match="not empty"): + hs.main(["build", "--n", "3", "--seed", "7", "--out", str(out)]) + import policybench.policyengine_runtime as runtime + + dataset = tmp_path / "x.h5" + dataset.write_bytes(b"other bytes") + monkeypatch.setattr( + runtime, "materialize_certified_us_dataset", lambda: str(dataset) + ) + monkeypatch.setattr( + runtime, + "policyengine_bundles_for_countries", + lambda countries: {"us": {"certified_data_artifact_sha256": "0" * 64}}, + ) + with pytest.raises(SystemExit, match="not the bundle's"): + hs.main(["build", "--n", "1", "--seed", "1", "--out", str(tmp_path / "new")]) + + +def _record(sid, variables, predictions, model, arm, prompt="v2-pilot"): + record = { + "scenario_id": sid, + "model": model, + "arm": arm, + "prompt_version": prompt, + "benchmark_search_rule": arm in ("web", "web+pe"), + "variables": variables, + "predictions": predictions, + "explanations": {v: "x" for v in variables}, + "status": "complete", + "turns": [{"total_cost_usd": 0.02, "calls": []}], + "cap_hit": False, + "wall_seconds": 1.0, + } + record["totals"] = pilot.household_totals(record) + return record + + +def _write_run(directory, board, model, arm, miss=None, prompt="v2-pilot"): + reference = board["reference"].set_index(["scenario_id", "variable"])["value"] + (directory / "households").mkdir(parents=True) + for sid in sorted(board["scenarios"]): + variables = board["variables"][sid] + predictions = {v: float(reference[(sid, v)]) for v in variables} + if miss is not None and sid == miss[0]: + predictions[miss[1]] = float(reference[(sid, miss[1])]) + 100 + record = _record(sid, variables, predictions, model, arm, prompt) + (directory / "households" / f"{sid}.json").write_text(json.dumps(record)) + + +@pytest.mark.slow +def test_a_household_set_scores_each_arm_against_its_no_tools_run( + household_board, tmp_path +): + out, _ = household_board + board = pilot.load_board(out) + amount = next( + v + for v in board["variables"]["mc_000"] + if not v.endswith("_eligible") and v in set(board["scored"].variable) + ) + none_dir, web_dir = tmp_path / "none", tmp_path / "web" + _write_run(none_dir, board, SOL, "none", miss=("mc_000", amount)) + _write_run(web_dir, board, SOL, "web") + pilot.command_score( + argparse.Namespace( + out_dir=str(web_dir), + complete_only=False, + board=str(out), + baseline_dir=str(none_dir), + ) + ) + score = json.loads((web_dir / "score.json").read_text()) + assert score["unweighted_hits"]["arm2"] == score["scored_outputs"] + assert score["unweighted_hits"]["arm1"] == score["scored_outputs"] - 1 + usage = pd.read_csv(web_dir / "usage_by_household.csv") + assert usage.arm1_cost_usd.round(6).tolist() == [0.02, 0.02, 0.02] + + with pytest.raises(SystemExit, match="--baseline-dir"): + pilot.command_score( + argparse.Namespace( + out_dir=str(web_dir), complete_only=False, board=str(out) + ) + ) + with pytest.raises(SystemExit, match="not gpt-6-sol none"): + pilot.command_score( + argparse.Namespace( + out_dir=str(web_dir), + complete_only=False, + board=str(out), + baseline_dir=str(web_dir), + ) + ) + v1_dir = tmp_path / "none-v1" + _write_run(v1_dir, board, SOL, "none", prompt="v1") + with pytest.raises(SystemExit, match="another prompt version"): + pilot.command_score( + argparse.Namespace( + out_dir=str(web_dir), + complete_only=False, + board=str(out), + baseline_dir=str(v1_dir), + ) + ) From c62b48d893bb622c1a481e56ee7edad4a95ead40 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 08:44:34 -0400 Subject: [PATCH 02/64] Judge each household-set batch with the reference adversary scripts/batch_judge.py writes one arm's answers on a batch as a dashboard payload (each model a board model), so consensus-flags and the two-stage adversary run unchanged: two of three models agreeing off the reference flag a cell. The payload carries what the models saw: the v2-pilot household prompt (checked against each record's sent prompt hash), the v2-pilot definitions, and the condition they answered under. Derivations are pruned engine traces on the household set's own reference system. The adversary takes a payload's outputDefinitions and answerCondition; the default renders both stage headers byte for byte as before. Dry run on today's frozen-board no-tools answers (v2 wording): 23 of 1,935 scored cells flagged, 12 of them among the 2026-10-05 adversary's. Co-Authored-By: Claude Opus 5.5 --- policybench/reference_adversary.py | 44 ++++- scripts/batch_judge.py | 306 +++++++++++++++++++++++++++++ tests/conftest.py | 50 +++++ tests/test_batch_judge.py | 251 +++++++++++++++++++++++ tests/test_household_set.py | 39 ---- 5 files changed, 648 insertions(+), 42 deletions(-) create mode 100644 scripts/batch_judge.py create mode 100644 tests/test_batch_judge.py diff --git a/policybench/reference_adversary.py b/policybench/reference_adversary.py index 5dfd5740..eaba7ae7 100644 --- a/policybench/reference_adversary.py +++ b/policybench/reference_adversary.py @@ -73,6 +73,11 @@ REFERENCE_FREEZES = {"us": REFERENCE_FREEZE, "uk": "2026-10-10"} UK_FISCAL_YEAR = f"{TAX_YEAR}-{str(TAX_YEAR + 1)[-2:]}" +# How the stage headers say the consensus models answered. A payload's +# ``answerCondition`` replaces it, for clusters of tool-using arms (the tool +# pilot's web arms); the default renders the headers byte for byte as before. +DEFAULT_ANSWER_CONDITION = "answering from memory, without tools" + VERDICTS = ( "reference_holds", "reference_wrong", @@ -457,6 +462,7 @@ class AdversaryCase: metric_type: str = "amount" state: str = "" trigger: tuple[str, ...] = () + answer_condition: str = DEFAULT_ANSWER_CONDITION def to_manifest_row(self) -> dict: """The case as a ``cases.jsonl`` row, without the derivation.""" @@ -465,6 +471,8 @@ def to_manifest_row(self) -> dict: row["consensus"] = [dict(cluster) for cluster in self.consensus] row["trigger"] = list(self.trigger) row["derivation_sha256"] = sha256_text(derivation) if derivation else None + if self.answer_condition == DEFAULT_ANSWER_CONDITION: + row.pop("answer_condition") return row @classmethod @@ -486,6 +494,9 @@ def from_manifest_row(cls, row: dict, derivation: str) -> AdversaryCase: metric_type=str(row.get("metric_type", "amount")), state=str(row.get("state", "") or ""), trigger=tuple(row.get("trigger", ())), + answer_condition=str( + row.get("answer_condition") or DEFAULT_ANSWER_CONDITION + ), ) @@ -600,6 +611,10 @@ def build_adversary_cases( country = str(payload.get("country") or "us") scenarios = payload.get("scenarios") or {} predictions = payload.get("scenarioPredictions") or {} + # A payload may state the definitions its models saw (the tool pilot's + # v2-pilot wording) and the condition they answered under. + definitions = payload.get("outputDefinitions") or {} + condition = answer_condition_of(payload) cases: list[AdversaryCase] = [] seen: set[tuple[str, str]] = set() for flag in flags: @@ -673,7 +688,8 @@ def build_adversary_cases( country=country, scenario_id=scenario_id, variable=variable, - definition=get_variable_description(variable, country=country), + definition=_clean(definitions.get(variable)) + or get_variable_description(variable, country=country), household_prompt=household_prompt, reference_value=reference, consensus=tuple(clusters), @@ -684,11 +700,33 @@ def build_adversary_cases( metric_type=metric_type_for_output(variable), state=_clean(flag.get("state") or scenario.get("state")), trigger=tuple(str(t) for t in flag.get("trigger", ())), + answer_condition=condition, ) ) return cases +def answer_condition_of(payload: dict) -> str: + """The payload's ``answerCondition``, checked, or the default.""" + condition = payload.get("answerCondition") + if condition is None: + return DEFAULT_ANSWER_CONDITION + if not isinstance(condition, str) or not condition.startswith("answering "): + raise ValueError( + f"answerCondition must be a phrase starting 'answering ': {condition!r}" + ) + return condition + + +def _with_condition(header: str, case: AdversaryCase) -> str: + """A stage header stating the case's answer condition.""" + if case.answer_condition == DEFAULT_ANSWER_CONDITION: + return header + if header.count(DEFAULT_ANSWER_CONDITION) != 1: + raise ValueError(f"{case.country} header does not state the answer condition") + return header.replace(DEFAULT_ANSWER_CONDITION, case.answer_condition) + + # --- Prompts ------------------------------------------------------------------- _DOMAINS = ", ".join(BLOCKED_DOMAINS) @@ -1022,7 +1060,7 @@ def _case_sections(case: AdversaryCase) -> list[str]: def render_stage1_prompt(case: AdversaryCase) -> str: """The law-first prompt: everything about the case except the derivation.""" - header = _country_template(_STAGE1_HEADERS, case.country) + header = _with_condition(_country_template(_STAGE1_HEADERS, case.country), case) return "\n".join([header, "", *_case_sections(case)]) @@ -1036,7 +1074,7 @@ def render_stage2_prompt(case: AdversaryCase, stage1: dict, stage1_sha256: str) derivation = case.derivation or "(no derivation was recorded for this case)" return "\n".join( [ - _country_template(_STAGE2_HEADERS, case.country), + _with_condition(_country_template(_STAGE2_HEADERS, case.country), case), "", *_case_sections(case), "", diff --git a/scripts/batch_judge.py b/scripts/batch_judge.py new file mode 100644 index 00000000..77dc6ec2 --- /dev/null +++ b/scripts/batch_judge.py @@ -0,0 +1,306 @@ +"""AI judges on each batch of a household set, before the next batch runs. + +Max, 2026-10-22: "we should go slowly since the new microdata could reveal +different pe bugs - we should do ai judges in small batches or real time". + +Each batch of a household set (scripts/household_set.py) runs every model in +every arm of the tool pilot (scripts/tool_pilot_argon.py --board). This script +then hands the batch to PolicyBench's reference adversary +(policybench/reference_adversary.py), arm by arm: + + batch_judge.py payload --board DIR --runs ROOT --stage S --arm none \\ + --batch K --out payload.json + policybench consensus-flags --payload payload.json --output flags.json \\ + --min-models 2 --top-k 3 --min-top 2 --zero-cluster-min-models 2 + batch_judge.py derivations --board DIR --payload payload.json \\ + --flags flags.json --out-dir annotations/ + policybench adversary-prepare --payload payload.json --flags flags.json \\ + --adversary-dir judge/ --annotations-dir annotations/ + scripts/run_reference_adversary_claude.sh judge/ + policybench adversary-collect --adversary-dir claude=judge/ --output-dir out/ + +``payload`` writes one arm's answers as a dashboard payload, each model a +"model" of the board, so the consensus trigger and the adversary run on it +unchanged. Only the no-tools and web arms are judged: a calculator arm answers +with the reference engine itself, and its explanations name it, which the +adversary's blinding refuses. The payload carries what the models saw: the +v2-pilot household prompt (the tool contract's, checked against each record's +sent prompt hash), the v2-pilot output definitions, and the condition they +answered under (from memory, or with web search). + +The trigger's thresholds are the board's, scaled to three models: a cell is +flagged when two of the three models agree on one answer other than the +reference (``min_models`` 2, a zero cluster too), or when two of the three +(all are "top") do. + +``derivations`` writes the engine derivation of each flagged reference: the +pruned computation trace on the household set's own reference system (its +engine and fix modules), in the frozen annotations' CSV format. A trace is the +engine's own record; no model narrates it. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path + +import pandas as pd + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "scripts")) + +import tool_pilot_argon as pilot # noqa: E402 + +MODELS = ("gpt-6-sol", "claude-opus-5.5", "gemini-4-argon") +JUDGED_ARMS = { + "none": "answering from memory, without tools", + "web": "answering with web search", +} +# The tool contract's household prompt is the dashboard payload's +# ``prompt.tool``; this model is on the tool contract. +PROMPT_MODEL = "gpt-6-sol" +TRACE_MAX_LINES = 400 +YEAR = 2026 + + +def arm_folder(arm: str) -> str: + return arm.replace("+", "-") + + +def load_batch(board: Path, batch: int) -> list[str]: + batches = json.loads((board / "batches.json").read_text())["batches"] + if not 0 <= batch < len(batches): + raise SystemExit(f"{board} has batches 0 to {len(batches) - 1}") + return list(batches[batch]) + + +def household_prompt(scenario, variables: list[str], prompt_version: str) -> str: + """The household prompt the tool-contract models saw, without the arm's + notice: the board's prompt in the run's prompt version.""" + profile = pilot.model_profile(PROMPT_MODEL, "none", prompt_version, False) + prompt, kwargs = pilot.build_initial_request(scenario, variables, profile) + return prompt[: len(prompt) - len(profile.notice)] if profile.notice else prompt + + +def output_definitions(variables, prompt_version: str) -> dict[str, str]: + from policybench.prompts import get_variable_description + + definitions = {} + for variable in sorted(set(variables)): + text = get_variable_description(variable, country="us") + if prompt_version == "v2-pilot": + text = pilot.v2_pilot_text(text) + definitions[variable] = text + return definitions + + +def build_payload( + board_dir: Path, + runs: Path, + stage: str, + arm: str, + batch: int | None = None, + scenarios: list[str] | None = None, +) -> dict: + """One arm's answers on one batch (or on named households), as a + dashboard payload.""" + if arm not in JUDGED_ARMS: + raise SystemExit(f"only {sorted(JUDGED_ARMS)} are judged, not {arm}") + if (batch is None) == (not scenarios): + raise SystemExit("name a batch or households, not both") + board = pilot.load_board(board_dir) + ids = list(scenarios) if scenarios else load_batch(board_dir, batch) + unknown = sorted(set(ids) - set(board["scenarios"])) + if unknown: + raise SystemExit(f"{board_dir} has no households {unknown}") + scored = set(zip(board["scored"].scenario_id, board["scored"].variable)) + reference = board["reference"].set_index(["scenario_id", "variable"])["value"] + records = {} + for model in MODELS: + folder = runs / model / arm_folder(arm) / stage / "households" + found = {} + for sid in ids: + path = folder / f"{sid}.json" + if not path.exists(): + raise SystemExit(f"{path} is missing: the batch has not finished") + found[sid] = json.loads(path.read_text()) + if pilot.records_model(found) != model or pilot.records_arm(found) != arm: + raise SystemExit(f"{folder} does not hold {model} {arm} records") + records[model] = found + versions = {pilot.records_prompt(r)[0] for r in records.values()} + if len(versions) != 1: + raise SystemExit(f"the models ran different prompt versions: {versions}") + prompt_version = versions.pop() + + labels = {model: f"{model}+{arm}" for model in MODELS} + scenarios, predictions, variables_seen = {}, {}, set() + for sid in ids: + scenario = board["scenarios"][sid] + variables = board["variables"][sid] + prompt = household_prompt(scenario, variables, prompt_version) + sent = records[PROMPT_MODEL][sid].get("sent_board_prompt_sha256") + if sent != hashlib.sha256(prompt.encode()).hexdigest(): + raise SystemExit( + f"{sid}: the rebuilt prompt is not the one {PROMPT_MODEL} saw" + ) + scenarios[sid] = {"prompt": {"tool": prompt}, "state": scenario.state} + cells = {} + for variable in variables: + variables_seen.add(variable) + cell = {} + for model, label in labels.items(): + record = records[model][sid] + cell[label] = { + "prediction": record["predictions"].get(variable), + "explanation": record["explanations"].get(variable), + "groundTruth": float(reference[(sid, variable)]), + "scored": (sid, variable) in scored, + } + cells[variable] = cell + predictions[sid] = cells + return { + "country": "us", + "source": { + "board": str(board_dir), + "board_reference_sha256": board["meta"].get("reference_csv_sha256"), + "runs": str(runs), + "stage": stage, + "arm": arm, + "batch": batch, + "households": ids, + "prompt_version": prompt_version, + }, + "answerCondition": JUDGED_ARMS[arm], + "outputDefinitions": output_definitions(variables_seen, prompt_version), + "modelStats": [{"model": labels[model]} for model in MODELS], + "scenarios": scenarios, + "scenarioPredictions": predictions, + } + + +def render_trace(system, scenario, variable: str) -> str: + """The pruned computation trace of one output on ``system``.""" + import household_set + from policyengine_us import Simulation + + from policybench.case_reference_explanations import ( + _find_target_tree, + _render_trace, + ) + from policybench.ground_truth import ( + _extract_person_value, + _pe_variable_for_output, + _person_index_for_output, + ) + + pe_variable = _pe_variable_for_output(variable, "us") + simulation = Simulation( + tax_benefit_system=system, situation=household_set.situation_for(scenario) + ) + simulation.trace = True + result = simulation.calculate(pe_variable, YEAR) + tree = _find_target_tree(simulation.tracer.trees, pe_variable) + if tree is None: + raise RuntimeError(f"no trace for {pe_variable}") + value = float(_extract_person_value(result, scenario, variable)) + lines = _render_trace(tree) + cut = len(lines) > TRACE_MAX_LINES + person = _person_index_for_output(scenario, variable) + header = [ + f"Output {variable} = {value:g}, computed as the engine variable " + f"{pe_variable} for {YEAR}" + + (f", read for person index {person}." if person is not None else "."), + "Each line is a variable and its value; a variable's inputs are indented " + "beneath it. Zero-valued branches below the first level are omitted.", + "", + ] + footer = [f"... ({len(lines) - TRACE_MAX_LINES} more lines cut)"] if cut else [] + return "\n".join(header + lines[:TRACE_MAX_LINES] + footer) + + +def command_payload(args) -> None: + payload = build_payload( + Path(args.board), + Path(args.runs), + args.stage, + args.arm, + batch=args.batch, + scenarios=args.scenarios, + ) + Path(args.out).write_text(json.dumps(payload, indent=1) + "\n") + cells = sum(len(v) for v in payload["scenarioPredictions"].values()) + print(f"{len(payload['scenarios'])} households, {cells} cells, arm {args.arm}") + + +def command_derivations(args) -> None: + board_dir = Path(args.board) + pilot._require_engine(board_dir) + payload = json.loads(Path(args.payload).read_text()) + if payload["source"]["board"] != str(board_dir): + raise SystemExit("the payload is from another board") + flags = json.loads(Path(args.flags).read_text())["flags"] + board = pilot.load_board(board_dir) + system = pilot.build_tool_system(reform=True, snapshot=board_dir) + rows = [] + for flag in flags: + sid, variable = flag["scenario_id"], flag["variable"] + rows.append( + { + "country": "us", + "scenario_id": sid, + "variable": variable, + "reference_value": flag["reference"], + "trace_lines": None, + "explanation": render_trace(system, board["scenarios"][sid], variable), + "error": None, + } + ) + for row in rows: + row["trace_lines"] = row["explanation"].count("\n") + 1 + out = Path(args.out_dir) + out.mkdir(parents=True, exist_ok=True) + pd.DataFrame( + rows, + columns=[ + "country", + "scenario_id", + "variable", + "reference_value", + "trace_lines", + "explanation", + "error", + ], + ).to_csv(out / "us_case_reference_explanations.csv", index=False) + print(f"{len(rows)} derivations in {out}") + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + payload = sub.add_parser("payload", help="one arm's batch as a dashboard payload") + payload.add_argument("--board", required=True) + payload.add_argument("--runs", required=True) + payload.add_argument("--stage", required=True) + payload.add_argument("--arm", required=True, choices=sorted(JUDGED_ARMS)) + payload.add_argument("--batch", type=int) + payload.add_argument( + "--scenarios", nargs="*", help="households, for a board without batches" + ) + payload.add_argument("--out", required=True) + payload.set_defaults(func=command_payload) + derivations = sub.add_parser("derivations", help="traces of flagged references") + derivations.add_argument("--board", required=True) + derivations.add_argument("--payload", required=True) + derivations.add_argument("--flags", required=True) + derivations.add_argument("--out-dir", required=True) + derivations.set_defaults(func=command_derivations) + args = parser.parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py index 0802969c..79f1792b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,3 +57,53 @@ def low_income_scenario(): def sample_scenarios(simple_single_scenario, family_scenario, low_income_scenario): """A small set of test scenarios.""" return [simple_single_scenario, family_scenario, low_income_scenario] + + +# A three-household set built from the frozen board's first households, with +# sampling and the dataset download faked (tests/test_household_set.py, +# tests/test_batch_judge.py). Its references must be the frozen ones on the +# frozen engine, policyengine-us 2.41.0; on any other engine it is skipped. +@pytest.fixture(scope="session") +def household_board(tmp_path_factory): + """A three-household set built from the frozen board's first households.""" + import importlib.metadata + import sys + from pathlib import Path + + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + import household_set as hs + + if importlib.metadata.version("policyengine-us") != "2.41.0": + pytest.skip("the build test reproduces the 2.41.0 references") + import policybench.policyengine_runtime as runtime + import policybench.scenarios as scenarios_module + + root = tmp_path_factory.mktemp("set") + dataset = root / "populace_us_2024.h5" + dataset.write_bytes(b"not a real dataset") + digest = hs.sha256_file(dataset) + sampled = hs.load_scenarios(hs.FROZEN_BOARD)[:3] + patch = pytest.MonkeyPatch() + patch.setattr( + scenarios_module, "generate_scenarios", lambda n, seed, country: sampled[:n] + ) + patch.setattr(runtime, "materialize_certified_us_dataset", lambda: str(dataset)) + patch.setattr( + runtime, + "policyengine_bundles_for_countries", + lambda countries: { + "us": { + "bundle_id": "us-test", + "certified_data_artifact_sha256": digest, + "default_dataset_uri": "hf://policyengine/populace-us/x.h5@test", + } + }, + ) + out = root / "board" + try: + hs.main( + ["build", "--n", "3", "--seed", "7", "--batch-size", "2", "--out", str(out)] + ) + finally: + patch.undo() + return out, sampled diff --git a/tests/test_batch_judge.py b/tests/test_batch_judge.py new file mode 100644 index 00000000..b34ae6d0 --- /dev/null +++ b/tests/test_batch_judge.py @@ -0,0 +1,251 @@ +"""The batch judge's payload and derivations, and the adversary hooks it uses. + +The payload tests run on the three-household set from tests/conftest.py, so +they need the frozen engine; the adversary hook tests need no engine. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import batch_judge as bj # noqa: E402 +import tool_pilot_argon as pilot # noqa: E402 + +from policybench import reference_adversary as ra # noqa: E402 +from policybench.consensus import ConsensusParams, consensus_flags # noqa: E402 + +THREE = ConsensusParams(min_models=2, top_k=3, min_top=2, zero_cluster_min_models=2) + + +def _case(**overrides): + case = dict( + case_id="us__s__snap", + country="us", + scenario_id="s", + variable="snap", + definition="annual SNAP benefit", + household_prompt="A household.", + reference_value=100.0, + consensus=( + { + "answer": 0.0, + "n_models": 2, + "n_top": 2, + "top_models": ["a", "b"], + "members": [ + {"model": "a", "prediction": 0.0, "explanation": "x"}, + {"model": "b", "prediction": 0.0, "explanation": "y"}, + ], + }, + ), + models_answered=3, + models_exact=1, + derivation="snap = 100", + ) + case.update(overrides) + return ra.AdversaryCase(**case) + + +def test_the_default_condition_leaves_both_headers_unchanged(): + case = _case() + assert ra._STAGE1_HEADER in ra.render_stage1_prompt(case) + stage2 = ra.render_stage2_prompt(case, {"answer": 1}, "0" * 64) + assert stage2.startswith(ra._STAGE2_HEADER) + assert "answer_condition" not in case.to_manifest_row() + + +def test_a_web_condition_restates_both_headers_and_survives_the_manifest(): + case = _case(answer_condition="answering with web search") + stage1 = ra.render_stage1_prompt(case) + stage2 = ra.render_stage2_prompt(case, {"answer": 1}, "0" * 64) + for text in (stage1, stage2): + assert "AI models answering with web search, agreed" in text + assert ra.DEFAULT_ANSWER_CONDITION not in text + row = case.to_manifest_row() + assert row["answer_condition"] == "answering with web search" + again = ra.AdversaryCase.from_manifest_row(row, case.derivation) + assert again == case + default = _case() + assert ( + ra.AdversaryCase.from_manifest_row( + default.to_manifest_row(), default.derivation + ) + == default + ) + + +@pytest.mark.parametrize("condition", ["with web search", 3, ""]) +def test_a_malformed_condition_is_refused(condition): + with pytest.raises(ValueError, match="answerCondition"): + ra.answer_condition_of({"answerCondition": condition}) + + +def _payload(predictions, reference=100.0, definitions=None, condition=None): + payload = { + "country": "us", + "modelStats": [{"model": m} for m in ("a", "b", "c")], + "scenarios": {"s": {"prompt": {"tool": "A household."}, "state": "CA"}}, + "scenarioPredictions": { + "s": { + "snap": { + model: { + "prediction": value, + "explanation": f"{model} says {value}", + "groundTruth": reference, + "scored": True, + } + for model, value in predictions.items() + } + } + }, + } + if definitions is not None: + payload["outputDefinitions"] = definitions + if condition is not None: + payload["answerCondition"] = condition + return payload + + +def test_two_of_three_agreeing_off_the_reference_flag_a_cell_and_one_does_not(): + flagged = consensus_flags(_payload({"a": 0.0, "b": 0.0, "c": 100.0}), THREE) + assert [(f["scenario_id"], f["variable"]) for f in flagged] == [("s", "snap")] + assert flagged[0]["clusters"][0]["models"] == ["a", "b"] + alone = consensus_flags(_payload({"a": 0.0, "b": 50.0, "c": 100.0}), THREE) + assert alone == [] + exact = consensus_flags(_payload({"a": 100.4, "b": 99.6, "c": 100.0}), THREE) + assert exact == [] + + +def test_cases_take_the_payload_definition_and_condition(): + payload = _payload( + {"a": 0.0, "b": 0.0, "c": 100.0}, + definitions={"snap": "v2 SNAP definition"}, + condition="answering with web search", + ) + flags = consensus_flags(payload, THREE) + (case,) = ra.build_adversary_cases(payload, flags) + assert case.definition == "v2 SNAP definition" + assert case.answer_condition == "answering with web search" + payload.pop("outputDefinitions") + payload.pop("answerCondition") + (case,) = ra.build_adversary_cases(payload, flags) + assert case.answer_condition == ra.DEFAULT_ANSWER_CONDITION + assert case.definition != "v2 SNAP definition" + + +# --- On a household set ---------------------------------------------------- + + +def _write_arm(runs, board, model, arm, stage, wrong=None): + """Records for one model and arm whose sent prompt is the v2-pilot one.""" + reference = board["reference"].set_index(["scenario_id", "variable"])["value"] + folder = runs / model / bj.arm_folder(arm) / stage / "households" + folder.mkdir(parents=True) + for sid in sorted(board["scenarios"]): + scenario, variables = board["scenarios"][sid], board["variables"][sid] + prompt = bj.household_prompt(scenario, variables, "v2-pilot") + predictions = {v: float(reference[(sid, v)]) for v in variables} + if wrong and sid == wrong[0]: + predictions[wrong[1]] = wrong[2] + record = { + "scenario_id": sid, + "model": model, + "arm": arm, + "prompt_version": "v2-pilot", + "benchmark_search_rule": arm == "web", + "variables": variables, + "predictions": predictions, + "explanations": {v: f"{model} on {v}" for v in variables}, + "status": "complete", + "sent_board_prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + } + (folder / f"{sid}.json").write_text(json.dumps(record)) + + +@pytest.mark.slow +def test_a_batch_payload_flags_what_two_models_get_wrong_and_judges_it_blind( + household_board, tmp_path +): + out, _ = household_board + board = pilot.load_board(out) + variable = next( + v + for v in board["variables"]["mc_000"] + if v == "federal_income_tax_before_refundable_credits" + ) + runs = tmp_path / "runs" + for model in bj.MODELS: + wrong = ("mc_000", variable, 123456.0) if model != "gemini-4-argon" else None + _write_arm(runs, board, model, "web", "b0", wrong=wrong) + payload = bj.build_payload(out, runs, "b0", "web", batch=0) + assert sorted(payload["scenarios"]) == ["mc_000", "mc_001"] + assert payload["answerCondition"] == "answering with web search" + assert [m["model"] for m in payload["modelStats"]] == [ + f"{m}+web" for m in bj.MODELS + ] + medicaid = payload["outputDefinitions"].get("head_medicaid_eligible") + if medicaid is not None: + assert "PolicyEngine" not in medicaid + flags = consensus_flags(payload, THREE) + assert [(f["scenario_id"], f["variable"]) for f in flags] == [("mc_000", variable)] + (case,) = ra.build_adversary_cases(payload, flags) + assert case.household_prompt == payload["scenarios"]["mc_000"]["prompt"]["tool"] + assert "with web search" in ra.render_stage1_prompt(case) + + flags_path = tmp_path / "flags.json" + flags_path.write_text(json.dumps({"flags": flags})) + payload_path = tmp_path / "payload.json" + payload_path.write_text(json.dumps(payload)) + bj.main( + [ + "derivations", + "--board", + str(out), + "--payload", + str(payload_path), + "--flags", + str(flags_path), + "--out-dir", + str(tmp_path / "annotations"), + ] + ) + derivations = ra.load_derivations(tmp_path / "annotations") + text = derivations[("mc_000", variable)] + reference = board["reference"].set_index(["scenario_id", "variable"])["value"] + assert text.startswith( + f"Output {variable} = {float(reference[('mc_000', variable)]):g}" + ) + assert "income_tax_before_refundable_credits" in text + + +@pytest.mark.slow +def test_a_payload_refuses_an_unfinished_batch_a_calculator_arm_and_another_prompt( + household_board, tmp_path +): + out, _ = household_board + board = pilot.load_board(out) + runs = tmp_path / "runs" + with pytest.raises(SystemExit, match="only"): + bj.build_payload(out, runs, "b0", "pe", batch=0) + for model in bj.MODELS[:2]: + _write_arm(runs, board, model, "none", "b0") + with pytest.raises(SystemExit, match="has not finished"): + bj.build_payload(out, runs, "b0", "none", batch=0) + _write_arm(runs, board, bj.MODELS[2], "none", "b0") + record_path = runs / bj.PROMPT_MODEL / "none" / "b0" / "households" / "mc_000.json" + record = json.loads(record_path.read_text()) + record["sent_board_prompt_sha256"] = "0" * 64 + record_path.write_text(json.dumps(record)) + with pytest.raises(SystemExit, match="rebuilt prompt"): + bj.build_payload(out, runs, "b0", "none", batch=0) + with pytest.raises(SystemExit, match="batches 0 to 1"): + bj.build_payload(out, runs, "b0", "none", batch=2) + with pytest.raises(SystemExit, match="not both"): + bj.build_payload(out, runs, "b0", "none", batch=0, scenarios=["mc_000"]) diff --git a/tests/test_household_set.py b/tests/test_household_set.py index ca51ea71..aab84412 100644 --- a/tests/test_household_set.py +++ b/tests/test_household_set.py @@ -83,45 +83,6 @@ def test_the_frozen_board_reads_as_before(): assert pilot.board_path(argparse.Namespace()) == pilot.SNAPSHOT -@pytest.fixture(scope="module") -def household_board(tmp_path_factory): - """A three-household set built from the frozen board's first households.""" - if not ON_FROZEN_ENGINE: - pytest.skip("the build test reproduces the 2.41.0 references") - import policybench.policyengine_runtime as runtime - import policybench.scenarios as scenarios_module - - root = tmp_path_factory.mktemp("set") - dataset = root / "populace_us_2024.h5" - dataset.write_bytes(b"not a real dataset") - digest = hs.sha256_file(dataset) - sampled = hs.load_scenarios(hs.FROZEN_BOARD)[:3] - patch = pytest.MonkeyPatch() - patch.setattr( - scenarios_module, "generate_scenarios", lambda n, seed, country: sampled[:n] - ) - patch.setattr(runtime, "materialize_certified_us_dataset", lambda: str(dataset)) - patch.setattr( - runtime, - "policyengine_bundles_for_countries", - lambda countries: { - "us": { - "bundle_id": "us-test", - "certified_data_artifact_sha256": digest, - "default_dataset_uri": "hf://policyengine/populace-us/x.h5@test", - } - }, - ) - out = root / "board" - try: - hs.main( - ["build", "--n", "3", "--seed", "7", "--batch-size", "2", "--out", str(out)] - ) - finally: - patch.undo() - return out, sampled - - @pytest.mark.slow def test_build_writes_the_frozen_references_for_frozen_households(household_board): out, sampled = household_board From 078cafc48ac7fc60ccd3d1efd12606d426a838df Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 09:16:52 -0400 Subject: [PATCH 03/64] Let the Claude adversary runner bill a declared API key Max, 2026-10-22: "in case youre running any claude api stuff please use the key on my clipboard". With JUDGE_API_KEY_SECRET naming an agent-secret entry, the judges bill that key and nothing else: it refuses a lane config, a lane token, AUDIT_ACCOUNT or the desktop opt-in alongside it; every call runs in a fresh empty config directory removed at exit; the key joins only the claude calls' allowlisted environment; and check-login requires a first-party api_key login from ANTHROPIC_API_KEY in that directory. Sidecars record the mode and the secret's name, never the key. The lane and desktop paths are unchanged. Why now: the desktop login's CLI session had expired, and the Subfleet lanes were closed, so the batch judges had no login to run on. Co-Authored-By: Claude Opus 5.5 --- policybench/reference_adversary.py | 34 ++++++++++ scripts/run_reference_adversary_claude.sh | 46 +++++++++++++- tests/test_reference_adversary.py | 51 +++++++++++++++ tests/test_reference_adversary_runner.py | 75 +++++++++++++++++++++++ 4 files changed, 203 insertions(+), 3 deletions(-) diff --git a/policybench/reference_adversary.py b/policybench/reference_adversary.py index eaba7ae7..ebe7573a 100644 --- a/policybench/reference_adversary.py +++ b/policybench/reference_adversary.py @@ -2102,6 +2102,7 @@ def check_login( declared: str, config_dir: str, desktop_opt_in: bool, + api_key_secret: str = "", ) -> dict: """The Claude login a runner may judge with, or ``ValueError`` saying why not. @@ -2111,6 +2112,12 @@ def check_login( in which case it must be exactly the desktop's own claude.ai login. Unlike that script, it also refuses a token lane whose declared account (AUDIT_ACCOUNT) is the desktop login's. + + With ``api_key_secret`` (the runner's JUDGE_API_KEY_SECRET, Max + 2026-10-22) the judges bill that API key instead: the CLI must report a + first-party ``api_key`` login from ``ANTHROPIC_API_KEY`` in exactly + ``config_dir`` (the runner's fresh, empty one), with no lane token and no + desktop opt-in. """ def parse(text: str) -> dict: @@ -2130,6 +2137,31 @@ def parse(text: str) -> dict: f"the login is through {status.get('apiProvider')!r}, not a " "first-party subscription" ) + if api_key_secret: + if token or desktop_opt_in: + raise ValueError( + "an API key judge takes no lane token and no desktop opt-in" + ) + if method != "api_key" or status.get("apiKeySource") != "ANTHROPIC_API_KEY": + raise ValueError( + f"JUDGE_API_KEY_SECRET is set but the CLI logs in by {method!r} " + f"from {status.get('apiKeySource')!r}, not ANTHROPIC_API_KEY" + ) + if not ( + isinstance(reported, str) + and os.path.isdir(reported) + and os.path.samefile(reported, config_dir) + ): + raise ValueError( + f"the CLI reports config directory {reported!r}, not the run's " + f"own {config_dir}" + ) + return { + "method": "api_key", + "account": None, + "org": None, + "secret": api_key_secret, + } if method not in ("oauth_token", "claude.ai"): raise ValueError(f"the login method {method!r} is not a subscription login") same_account = bool( @@ -2226,6 +2258,7 @@ def main(argv: list[str] | None = None) -> int: login.add_argument("--declared", default="") login.add_argument("--config-dir", required=True) login.add_argument("--desktop-opt-in", choices=["0", "1"], default="0") + login.add_argument("--api-key-secret", default="") args = parser.parse_args(argv) if args.command == "render-stage2": @@ -2264,6 +2297,7 @@ def main(argv: list[str] | None = None) -> int: declared=args.declared, config_dir=args.config_dir, desktop_opt_in=args.desktop_opt_in == "1", + api_key_secret=args.api_key_secret, ) except ValueError as exc: print(str(exc), file=sys.stderr) diff --git a/scripts/run_reference_adversary_claude.sh b/scripts/run_reference_adversary_claude.sh index 69818c67..5c680040 100755 --- a/scripts/run_reference_adversary_claude.sh +++ b/scripts/run_reference_adversary_claude.sh @@ -58,6 +58,17 @@ # judge runs. A call the API refuses with 401, 403 or 429 stops the run: no # further judge starts, and the run resumes where it stopped. # +# API key (Max, 2026-10-22: "in case youre running any claude api stuff please +# use the key on my clipboard"): JUDGE_API_KEY_SECRET names an agent-secret +# entry (account JUDGE_API_KEY_ACCOUNT, default $USER; AUDIT_SECRET_BIN +# replaces agent-secret) holding an Anthropic API key. The judges then bill +# that key and nothing else: it cannot be combined with a lane, a lane token +# or the desktop login; every claude call runs in a fresh empty config +# directory, removed when the run ends; the key joins only the claude calls' +# allowlisted environment; and `claude auth status` must report a first-party +# api_key login in that directory. The sidecars record the mode and the +# secret's name, never the key. +# # Provenance: beside stage1.json and verdict.json the runner writes # stage1.meta.json and verdict.meta.json with the runner, the stage, the model # requested and the models the CLI reports, the CLI version, the effort, the @@ -165,7 +176,34 @@ for base in {home, pwd.getpwuid(os.getuid()).pw_dir}: print("other") PY } -if [ "$DESKTOP_OPT_IN" = 1 ]; then +API_SECRET="${JUDGE_API_KEY_SECRET:-}" +API_KEY="" +if [ -n "$API_SECRET" ]; then + [ "$DESKTOP_OPT_IN" = 0 ] || { + echo "JUDGE_API_KEY_SECRET and JUDGE_ALLOW_DESKTOP_LOGIN=1 are exclusive; refusing to start" >&2 + exit 1 + } + [ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] || { + echo "JUDGE_API_KEY_SECRET takes no lane token: unset CLAUDE_CODE_OAUTH_TOKEN; refusing to start" >&2 + exit 1 + } + [ -z "${AUDIT_ACCOUNT:-}" ] || { + echo "JUDGE_API_KEY_SECRET bills a key, not an account: unset AUDIT_ACCOUNT; refusing to start" >&2 + exit 1 + } + [ -z "${CLAUDE_CONFIG_DIR:-}" ] || { + echo "JUDGE_API_KEY_SECRET runs in its own empty config directory: unset CLAUDE_CONFIG_DIR; refusing to start" >&2 + exit 1 + } + API_KEY=$("${AUDIT_SECRET_BIN:-agent-secret}" get "$API_SECRET" "${JUDGE_API_KEY_ACCOUNT:-$USER}" 2>/dev/null) + [ -n "$API_KEY" ] || { + echo "could not read the API key $API_SECRET; refusing to start" >&2 + exit 1 + } + CONFIG_DIR=$(mktemp -d "${TMPDIR:-/tmp}/pb-adversary-config.XXXXXX") || exit 1 + CONFIG_DIR="$(cd "$CONFIG_DIR" && pwd -P)" + trap 'rm -rf "$CONFIG_DIR"' EXIT +elif [ "$DESKTOP_OPT_IN" = 1 ]; then [ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] || { echo "JUDGE_ALLOW_DESKTOP_LOGIN=1 takes no lane token: unset CLAUDE_CODE_OAUTH_TOKEN; refusing to start" >&2 exit 1 @@ -218,6 +256,7 @@ while IFS= read -r line; do [ -n "$line" ] && CHILD_ENV+=("$line"); done <&2 + --desktop-opt-in "$DESKTOP_OPT_IN" --api-key-secret "$API_SECRET") || { + echo "refusing to start: the judges must bill a lane's subscription login, the opted-in desktop login or the declared API key" >&2 exit 1 } DECLARED="${AUDIT_ACCOUNT:-}" [ "$DESKTOP_OPT_IN" = 1 ] && DECLARED="$DESKTOP_DECLARED" +[ -n "$API_SECRET" ] && DECLARED="API key $API_SECRET (JUDGE_API_KEY_SECRET, Max 2026-10-22)" CLI_VERSION=$(cd / && claude_child --version /dev/null | head -n 1) valid() { [ -s "$2" ] && adversary validate --schema "$1" --file "$2" >/dev/null 2>&1; } diff --git a/tests/test_reference_adversary.py b/tests/test_reference_adversary.py index 45c4352e..3496e381 100644 --- a/tests/test_reference_adversary.py +++ b/tests/test_reference_adversary.py @@ -1551,6 +1551,57 @@ def test_check_login_mirrors_the_audit_runner_rule(tmp_path: Path): assert check_login(lane, desktop, token=True, declared="claude:lane@x", **common) +def test_check_login_takes_an_api_key_only_as_declared_in_the_runs_own_directory( + tmp_path: Path, +): + own = tmp_path / "own" + own.mkdir() + other = tmp_path / "other" + other.mkdir() + + def status(**fields): + return json.dumps( + { + "loggedIn": True, + "authMethod": "api_key", + "apiProvider": "firstParty", + "apiKeySource": "ANTHROPIC_API_KEY", + "configDirectory": str(own), + **fields, + } + ) + + common = {"token": False, "declared": "", "config_dir": str(own)} + assert check_login( + status(), "", desktop_opt_in=False, api_key_secret="KEY", **common + ) == {"method": "api_key", "account": None, "org": None, "secret": "KEY"} + # Without the declaration an API key is still refused. + with pytest.raises(ValueError, match="not a subscription"): + check_login(status(), "", desktop_opt_in=False, **common) + refusals = [ + (status(authMethod="claude.ai"), "logs in by 'claude.ai'"), + (status(apiKeySource="/login managed key"), "not ANTHROPIC_API_KEY"), + (status(configDirectory=str(other)), "not the run's own"), + (status(apiProvider="bedrock"), "not a first-party"), + (status(loggedIn=False), "no login"), + ] + for text, message in refusals: + with pytest.raises(ValueError, match=message): + check_login(text, "", desktop_opt_in=False, api_key_secret="KEY", **common) + with pytest.raises(ValueError, match="no lane token and no desktop opt-in"): + check_login(status(), "", desktop_opt_in=True, api_key_secret="KEY", **common) + with pytest.raises(ValueError, match="no lane token and no desktop opt-in"): + check_login( + status(), + "", + token=True, + declared="", + config_dir=str(own), + desktop_opt_in=False, + api_key_secret="KEY", + ) + + # --- Command line ---------------------------------------------------------------- diff --git a/tests/test_reference_adversary_runner.py b/tests/test_reference_adversary_runner.py index 0a1d1e93..1dadf611 100644 --- a/tests/test_reference_adversary_runner.py +++ b/tests/test_reference_adversary_runner.py @@ -109,6 +109,13 @@ def _case(scenario_id: str) -> AdversaryCase: print("9.9.9 (Claude Code fake)") sys.exit(0) if args[:1] == ["auth"]: + if os.environ.get("ANTHROPIC_API_KEY"): + print(json.dumps({ + "loggedIn": True, "authMethod": "api_key", "apiProvider": "firstParty", + "apiKeySource": "ANTHROPIC_API_KEY", + "configDirectory": os.environ.get("CLAUDE_CONFIG_DIR"), + })) + sys.exit(0) print(json.dumps( {"loggedIn": True, "authMethod": "oauth_token", "apiProvider": "firstParty"} )) @@ -538,6 +545,74 @@ def test_claude_runner_refuses_without_a_lane_login(tmp_path: Path): assert _calls(bin_dir) == [] +def _api_key_env(tmp_path: Path, env: dict, key: str = "sk-ant-judge") -> dict: + """The runner's environment for an API key judge, with a fake secret store.""" + store = tmp_path / "bin" / "secret" + store.write_text( + f"#!{sys.executable}\n" + "import sys\n" + f"print({key!r}) if sys.argv[1:3] == ['get', 'JUDGE_KEY'] else sys.exit(1)\n" + ) + store.chmod(0o755) + env = {**env, "JUDGE_API_KEY_SECRET": "JUDGE_KEY", "AUDIT_SECRET_BIN": str(store)} + for name in ("CLAUDE_CONFIG_DIR", "AUDIT_ACCOUNT", "CLAUDE_CODE_OAUTH_TOKEN"): + env.pop(name, None) + return env + + +def test_claude_runner_judges_on_a_declared_api_key_in_its_own_config(tmp_path: Path): + adversary, bin_dir, env = _setup(tmp_path, "claude") + env = _api_key_env(tmp_path, env) + env["TMPDIR"] = str(tmp_path / "tmp") + (tmp_path / "tmp").mkdir() + result = _run("claude", adversary, env, tmp_path) + assert result.returncode == 0, result.stderr + result.stdout + calls = _calls(bin_dir) + assert calls + configs = {call["env"]["CLAUDE_CONFIG_DIR"] for call in calls} + assert len(configs) == 1 + config = configs.pop() + assert Path(config).parent == (tmp_path / "tmp").resolve() + assert not Path(config).exists() # removed when the run ended + for call in calls: + # The declared key, never the parent's ANTHROPIC_API_KEY. + assert call["env"]["ANTHROPIC_API_KEY"] == "sk-ant-judge" + assert "CLAUDE_CODE_OAUTH_TOKEN" not in call["env"] + for case in CASES: + meta = json.loads( + (adversary / "cases" / case.case_id / "verdict.meta.json").read_text() + ) + assert meta["judge_auth"] == { + "method": "api_key", + "account": None, + "org": None, + "secret": "JUDGE_KEY", + } + assert "JUDGE_KEY" in meta["judge_account_declared"] + assert "sk-ant-judge" not in json.dumps(meta) + + +def test_claude_runner_refuses_an_api_key_mixed_with_a_login(tmp_path: Path): + adversary, bin_dir, env = _setup(tmp_path, "claude") + base = _api_key_env(tmp_path, env) + mixes = { + "CLAUDE_CONFIG_DIR": (str(tmp_path / "lane-config"), "own empty config"), + "AUDIT_ACCOUNT": ("claude:lane@example.org", "unset AUDIT_ACCOUNT"), + "CLAUDE_CODE_OAUTH_TOKEN": ("token", "takes no lane token"), + "JUDGE_ALLOW_DESKTOP_LOGIN": ("1", "are exclusive"), + } + for name, (value, message) in mixes.items(): + result = _run("claude", adversary, {**base, name: value}, tmp_path) + assert result.returncode == 1 + assert message in result.stderr, (name, result.stderr) + result = _run( + "claude", adversary, {**base, "JUDGE_API_KEY_SECRET": "MISSING"}, tmp_path + ) + assert result.returncode == 1 + assert "could not read the API key MISSING" in result.stderr + assert _calls(bin_dir) == [] + + # --- Codex ------------------------------------------------------------------------- From 879fd2bc3f1420cc404413de4ea6b9eee669f084 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 13:43:26 -0400 Subject: [PATCH 04/64] Let a judge exclude the blocked domains by operator The adversary prompt tells a judge to exclude policybench.org, policyengine.org and GitHub on every search. Codex's search tool takes no blocked_domains, so GPT-6.1 Sol did it with -site: operators, and the audit rejected those stage-1 outputs as searches naming PolicyEngine (rehearsal, 2026-10-22: 2 of 3). searched_for() drops a -site: exclusion of a blocked domain before the contamination check, in both runners' audits. A query that still names the engine or the benchmark (beside exclusions, under site:, -inurl:, or a look-alike domain) is still rejected. Co-Authored-By: Claude Opus 5.5 --- policybench/reference_adversary.py | 22 ++++++++++- tests/test_reference_adversary.py | 62 ++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/policybench/reference_adversary.py b/policybench/reference_adversary.py index ebe7573a..301a2d7e 100644 --- a/policybench/reference_adversary.py +++ b/policybench/reference_adversary.py @@ -1684,6 +1684,24 @@ def _markers_in(text: str) -> list[str]: return [marker for marker in DERIVATION_MARKERS if marker in lowered] +# An exclusion of a blocked domain in a search query ("-site:policyengine.org"): +# how a judge whose search tool takes no blocked_domains (Codex) follows the +# prompt's rule to exclude those domains on every search. It keeps a domain +# out; it does not search for it. +_EXCLUDED_DOMAIN = re.compile( + r"(? str: + """A search query without its exclusions of blocked domains: what the + judge searched for, for the contamination check.""" + return _EXCLUDED_DOMAIN.sub(" ", str(query or "")) + + def codex_events_audit(events_text: str) -> tuple[list[str], dict]: """Audit a ``codex exec --json`` event log for contamination. @@ -1752,7 +1770,7 @@ def codex_events_audit(events_text: str) -> tuple[list[str], dict]: ] url = str(action.get("url") or "") for text in [str(item.get("query") or ""), *searched]: - for marker in _markers_in(text): + for marker in _markers_in(searched_for(text)): problems.append(f"line {number}: a web search names {marker!r}") for marker in _markers_in(str(action.get("pattern") or "")): problems.append(f"line {number}: an in-page search names {marker!r}") @@ -1857,7 +1875,7 @@ def claude_transcript_audit( query = str(payload.get("query") or "") activity["searches"].append(query) searches.append((str(part.get("id")), query)) - if any(n in query.lower() for n in BLOCKED_NAMES): + if any(n in searched_for(query).lower() for n in BLOCKED_NAMES): problems.append(f"the judge searched for the engine: {query}") else: problems.append(f"the judge called a tool it was not given: {name}") diff --git a/tests/test_reference_adversary.py b/tests/test_reference_adversary.py index 3496e381..89798920 100644 --- a/tests/test_reference_adversary.py +++ b/tests/test_reference_adversary.py @@ -47,6 +47,7 @@ prepare_adversary, render_stage1_prompt, render_stage2_prompt, + searched_for, sha256_text, write_stage2_prompt, ) @@ -1339,6 +1340,57 @@ def test_codex_audit_catches_contamination(item, problem): assert problems and any(problem in p for p in problems), problems +# The query GPT-6.1 Sol sent on 2026-10-22 (rehearsal case us__rh_000__snap): +# the prompt asks a judge to exclude the blocked domains on every search, and +# Codex's search tool takes no blocked_domains, so it excludes them by operator. +_EXCLUDING = ( + "site:fns.usda.gov SNAP fiscal year 2026 cost living adjustments " + + " ".join(f"-site:{domain}" for domain in BLOCKED_DOMAINS) +) + + +def test_codex_audit_accepts_a_search_that_excludes_the_blocked_domains(): + item = { + "id": "1", + "type": "web_search", + "query": _EXCLUDING, + "action": {"type": "search", "queries": [_EXCLUDING, '"7 CFR 273.10" minimum']}, + } + problems, activity = codex_events_audit(_events(item)) + assert problems == [] + assert _EXCLUDING in activity["searches"] + + +@pytest.mark.parametrize( + "query", + [ + "policyengine snap 2026 -site:policyengine.org", + "site:policyengine.org snap", + "snap -site:policyengine.org.example.com", + "snap -site:notpolicyengine.org", + "snap -inurl:policybench", + "snap -site:policybench.org policybench answers", + ], +) +def test_codex_audit_still_catches_a_search_for_the_engine_beside_exclusions(query): + problems, _ = codex_events_audit( + _events({"id": "1", "type": "web_search", "query": query}) + ) + assert any("a web search names" in p for p in problems), problems + + +@given( + st.lists(st.sampled_from(BLOCKED_DOMAINS), max_size=6), + st.text(alphabet="abcdefghijklmnopqrstuvwxyz0123456789 .:", max_size=40), +) +def test_searched_for_drops_only_blocked_domain_exclusions(domains, text): + query = " ".join([text, *(f"-site:{d}" for d in domains)]) + remaining = searched_for(query).lower() + assert all(f"-site:{d}" not in remaining.split() for d in domains) + # Everything else the judge typed survives. + assert all(word in remaining.split() for word in text.lower().split()) + + def test_codex_audit_greps_lines_that_are_not_events(): problems, _ = codex_events_audit("warning: read /x/data.json\n") assert problems == ["line 1: non-event output names 'data.json'"] @@ -1398,6 +1450,16 @@ def test_claude_audit_passes_web_research_and_returns_the_accepted_answer(): assert accepted == [_stage1()] +def test_claude_audit_takes_an_excluding_search_as_law_research(): + excluding = _use("1", "WebSearch", query=_EXCLUDING) + problems, activity, _ = claude_transcript_audit(_transcript("p", excluding), "p") + assert problems == [] + assert activity["searches"] == [_EXCLUDING] + seeking = _use("1", "WebSearch", query="policyengine snap -site:policyengine.org") + problems, _, _ = claude_transcript_audit(_transcript("p", seeking), "p") + assert any("searched for the engine" in p for p in problems), problems + + def test_claude_audit_records_a_fetch_the_deny_rule_refused(): """A blocked fetch whose result is an error returned nothing: recorded, not contamination. One that returned content is contamination.""" From b6d835a8a7324e6ac34ee2195c5d02a6fb67df48 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 07:13:06 -0400 Subject: [PATCH 05/64] Build test household sets on a named dataset; v2 needs a certified pair Max, 2026-10-23: "we can still test before that but i'd like v2 to be clean". household_set.py build takes --purpose v2 (the default) or test. A v2 set needs the installed policyengine.py bundle to certify its own dataset for the installed engine; anything else is refused unless --purpose test. A test set may name a published dataset (--dataset-repo/--dataset-file/ --dataset-revision); it records the release manifest's build engine and compatibility claim, purpose test and certified_pair false. Engines after policyengine-us 2.2.1 ignore Populace's filing_status_input and derive filing status from the household; the build records each sampled household whose stated status differs from the engine's (its scenario and reference carry the engine's). The sampler takes the dataset as an option; the default call is unchanged. Co-Authored-By: Claude Opus 5.5 --- policybench/scenarios.py | 32 ++++++-- scripts/household_set.py | 155 ++++++++++++++++++++++++++++++++++-- tests/conftest.py | 7 +- tests/test_household_set.py | 92 +++++++++++++++++++++ 4 files changed, 272 insertions(+), 14 deletions(-) diff --git a/policybench/scenarios.py b/policybench/scenarios.py index 35e88d00..4e8ae379 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -918,19 +918,29 @@ def scenario_from_dict(data: dict[str, Any]) -> Scenario: ) -def load_certified_us_person_frame() -> tuple[pd.DataFrame, int, str]: +def load_certified_us_person_frame( + dataset: str | None = None, dataset_label: str | None = None +) -> tuple[pd.DataFrame, int, str]: """Load a person-level frame from the certified US microsimulation dataset. Returns ``(person_df, dataset_year, dataset_label)``. The label is read from the live simulation's PolicyEngine bundle (``default_dataset``), so it reflects the dataset the run actually materialized -- the certified populace build (``populace_us_2024``) -- rather than a hardcoded name. + + ``dataset`` (a local H5 path) replaces the certified dataset, for test + household sets only; ``dataset_label`` then names it. """ from policybench.policyengine_runtime import make_us_microsimulation - sim = make_us_microsimulation() + if dataset is None: + sim = make_us_microsimulation() + dataset_label = str(sim.policyengine_bundle["default_dataset"]) + else: + if not dataset_label: + raise ValueError("an explicit dataset needs a dataset_label") + sim = make_us_microsimulation(dataset=dataset) dataset_year = sim.default_input_period - dataset_label = str(sim.policyengine_bundle["default_dataset"]) input_specs = get_promptable_input_specs() values = {} @@ -1822,10 +1832,22 @@ def generate_scenarios( seed: int = SEED, excluded_household_ids: set[int] | None = None, country: str = DEFAULT_COUNTRY, + dataset: str | None = None, + dataset_label: str | None = None, ) -> list[Scenario]: - """Generate benchmark scenarios for a country.""" + """Generate benchmark scenarios for a country. + + ``dataset`` and ``dataset_label`` replace the certified US dataset (see + :func:`load_certified_us_person_frame`); test household sets only. + """ + if dataset is not None and country != "us": + raise ValueError("an explicit dataset is supported for the US only") if country == "us": - person_df, dataset_year, dataset_label = load_certified_us_person_frame() + person_df, dataset_year, dataset_label = ( + load_certified_us_person_frame() + if dataset is None + else load_certified_us_person_frame(dataset, dataset_label) + ) return scenarios_from_cps_frame( person_df, n=n, diff --git a/scripts/household_set.py b/scripts/household_set.py index 5b6a01ce..ff385040 100644 --- a/scripts/household_set.py +++ b/scripts/household_set.py @@ -217,6 +217,95 @@ def _utc_now() -> str: return datetime.datetime.now(datetime.timezone.utc).isoformat() +def explicit_dataset(repo: str, filename: str, revision: str) -> tuple[Path, dict]: + """A published dataset named by repo, file and tag (not the bundle's), and + what its release manifest says about the engine it was built for.""" + from huggingface_hub import hf_hub_download + from huggingface_hub.errors import EntryNotFoundError + + path = Path(hf_hub_download(repo, filename, revision=revision, repo_type="dataset")) + record = { + "uri": f"hf://datasets/{repo}/{filename}@{revision}", + "release_manifest": None, + } + try: + manifest_path = hf_hub_download( + repo, + f"releases/{revision}/release_manifest.json", + revision=revision, + repo_type="dataset", + ) + except EntryNotFoundError: + return path, record + manifest = json.loads(Path(manifest_path).read_text()) + build = manifest.get("build") or {} + record["release_manifest"] = { + "sha256": sha256_file(Path(manifest_path)), + "built_with_model_package": build.get("built_with_model_package"), + "built_with_core_package": build.get("built_with_core_package"), + "compatible_model_packages": manifest.get("compatible_model_packages"), + } + return path, record + + +# How a dataset's stated filing status reads as a scenario's. +STATED_FILING_STATUS = { + "SINGLE": "single", + "JOINT": "joint", + "HEAD_OF_HOUSEHOLD": "head_of_household", + "SEPARATE": "separate", + "SURVIVING_SPOUSE": "surviving_spouse", +} + + +def stated_filing_status(path: Path) -> dict[int, str] | None: + """Each tax unit's filing_status_input from a Populace H5 (pandas table + format), or None when the file states none. Engines after 2.2.1 no longer + read the column; they derive filing status from the household.""" + import h5py + + if not h5py.is_hdf5(path): + return None + with h5py.File(path, "r") as handle: + if "tax_unit" not in handle or "table" not in handle["tax_unit"]: + return None + table = handle["tax_unit"]["table"][:] + columns = {} + for field in table.dtype.names: + sub = table.dtype[field] + for name in sub.names or (field,): + columns[name] = table[field][name] if sub.names else table[field] + if "filing_status_input" not in columns or "tax_unit_id" not in columns: + return None + return { + int(unit): (value.decode() if isinstance(value, bytes) else str(value)) + for unit, value in zip(columns["tax_unit_id"], columns["filing_status_input"]) + } + + +def filing_status_mismatches(scenarios, stated: dict[int, str] | None) -> list[dict]: + """Households whose scenario filing status (the engine's) is not the one + the dataset states.""" + if stated is None: + return [] + found = [] + for scenario in scenarios: + unit = scenario.metadata.get("tax_unit_id") + said = stated.get(int(unit)) if unit is not None else None + if said is None: + continue + if STATED_FILING_STATUS.get(said, said.lower()) != scenario.filing_status: + found.append( + { + "scenario_id": scenario.id, + "tax_unit_id": int(unit), + "stated": said, + "engine": scenario.filing_status, + } + ) + return found + + def command_build(args) -> None: from policybench.policyengine_runtime import ( materialize_certified_us_dataset, @@ -227,20 +316,53 @@ def command_build(args) -> None: out = Path(args.out) if out.exists() and any(out.iterdir()): raise SystemExit(f"{out} is not empty; a household set is written once") + named = [args.dataset_repo, args.dataset_file, args.dataset_revision] + if any(named) and not all(named): + raise SystemExit( + "name a dataset by --dataset-repo, --dataset-file and --dataset-revision" + ) engine = installed_engine() pins = fix_module_pins(Path(args.fix_module)) programs = frozen_programs() bundles = policyengine_bundles_for_countries(["us"]) - dataset_path = Path(materialize_certified_us_dataset()) - dataset_sha256 = sha256_file(dataset_path) - certified = bundles["us"].get("certified_data_artifact_sha256") - if certified and certified != dataset_sha256: + bundle = bundles["us"] + if all(named): + # Max, 2026-10-22: "we can still test before that but i'd like v2 to be + # clean". A dataset other than the bundle's makes a test set only. + dataset_path, dataset_record = explicit_dataset(*named) + dataset_label = f"{Path(args.dataset_file).stem}@{args.dataset_revision}" + certified_pair = False + else: + dataset_path = Path(materialize_certified_us_dataset()) + dataset_record = {"uri": bundle.get("default_dataset_uri")} + dataset_label = None + certified = bundle.get("certified_data_artifact_sha256") + if certified and certified != sha256_file(dataset_path): + raise SystemExit( + f"{dataset_path} hashes to {sha256_file(dataset_path)}, not the " + f"bundle's {certified}" + ) + # The bundle certifies its data for its own model pin; the pair is + # certified only when that pin is the installed engine. + certified_pair = bundle.get("bundled_model_version") == bundle.get( + "model_version" + ) + if not certified_pair and args.purpose != "test": raise SystemExit( - f"{dataset_path} hashes to {dataset_sha256}, not the bundle's {certified}" + "this dataset and engine are not a certified pair: build it with " + "--purpose test, or install a bundle that certifies the engine" ) + dataset_sha256 = sha256_file(dataset_path) - sampled = generate_scenarios(n=args.n, seed=args.seed, country="us") + sampled = generate_scenarios( + n=args.n, + seed=args.seed, + country="us", + dataset=str(dataset_path) if all(named) else None, + dataset_label=dataset_label, + ) scenarios = rename_scenarios(sampled, args.id_prefix) + mismatches = filing_status_mismatches(scenarios, stated_filing_status(dataset_path)) references = compute_references( scenarios, programs, reference_system(Path(args.fix_module)) ) @@ -270,6 +392,8 @@ def command_build(args) -> None: "country": "us", "metadata_version": 1, "board_kind": "household_set", + "purpose": args.purpose, + "certified_pair": certified_pair, "generated_at_utc": generated, "num_scenarios": len(scenarios), "requested_num_scenarios": args.n, @@ -279,10 +403,14 @@ def command_build(args) -> None: "programs": programs, "policyengine_bundles": bundles, "dataset": { + **dataset_record, "path_name": dataset_path.name, "sha256": dataset_sha256, - "bundle_default_dataset_uri": bundles["us"].get("default_dataset_uri"), + "bundle_default_dataset_uri": bundle.get("default_dataset_uri"), }, + # The dataset states a filing status the engine no longer reads; these + # households' scenarios carry the engine's, as their references do. + "filing_status_mismatches": mismatches, "source_households": [ {"scenario_id": s.id, **{k: s.metadata[k] for k in sorted(s.metadata)}} for s in scenarios @@ -329,7 +457,9 @@ def command_build(args) -> None: ) print( f"{len(scenarios)} households, {len(references)} outputs on {engine}; " - f"{len(batches)} batches of up to {args.batch_size} in {out}" + f"{len(batches)} batches of up to {args.batch_size} in {out}; purpose " + f"{args.purpose}, certified pair {certified_pair}; " + f"{len(mismatches)} filing status mismatches" ) @@ -402,6 +532,15 @@ def main(argv: list[str] | None = None) -> None: build.add_argument("--id-prefix", default="mc") build.add_argument("--fix-module", default=str(FIX_MODULE)) build.add_argument("--out", required=True) + build.add_argument( + "--purpose", + choices=["v2", "test"], + default="v2", + help="v2 needs a certified dataset-engine pair; test takes any", + ) + build.add_argument("--dataset-repo", help="a published dataset, for a test set") + build.add_argument("--dataset-file") + build.add_argument("--dataset-revision") build.set_defaults(func=command_build) refs = sub.add_parser("references", help="recompute a board's references") refs.add_argument("--board", required=True) diff --git a/tests/conftest.py b/tests/conftest.py index 79f1792b..27797a13 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -85,7 +85,9 @@ def household_board(tmp_path_factory): sampled = hs.load_scenarios(hs.FROZEN_BOARD)[:3] patch = pytest.MonkeyPatch() patch.setattr( - scenarios_module, "generate_scenarios", lambda n, seed, country: sampled[:n] + scenarios_module, + "generate_scenarios", + lambda n, seed, country, **kwargs: sampled[:n], ) patch.setattr(runtime, "materialize_certified_us_dataset", lambda: str(dataset)) patch.setattr( @@ -96,6 +98,9 @@ def household_board(tmp_path_factory): "bundle_id": "us-test", "certified_data_artifact_sha256": digest, "default_dataset_uri": "hf://policyengine/populace-us/x.h5@test", + # The bundle certifies its data for the installed engine. + "bundled_model_version": "2.41.0", + "model_version": "2.41.0", } }, ) diff --git a/tests/test_household_set.py b/tests/test_household_set.py index aab84412..99d42f9b 100644 --- a/tests/test_household_set.py +++ b/tests/test_household_set.py @@ -241,3 +241,95 @@ def test_a_household_set_scores_each_arm_against_its_no_tools_run( baseline_dir=str(v1_dir), ) ) + + +def _scenario(sid, filing_status, tax_unit): + from policybench.scenarios import Person, Scenario + + return Scenario( + id=sid, + state="CA", + filing_status=filing_status, + adults=[Person(name="head", age=40, employment_income=0.0)], + metadata={"household_id": 1, "tax_unit_id": tax_unit}, + ) + + +def test_stated_filing_status_is_read_from_the_tax_unit_table(tmp_path): + import h5py + import numpy as np + + path = tmp_path / "set.h5" + rows = np.array( + [(0, 11, b"JOINT"), (1, 12, b"SEPARATE"), (2, 13, b"SURVIVING_SPOUSE")], + dtype=[ + ("index", " Date: Fri, 23 Oct 2026 12:41:39 -0400 Subject: [PATCH 06/64] Add calculator tool version 2 (the pe2 arm) Max, 2026-10-23, on what test batch 0 showed about the calculator ("yes go"): - Names: a name written with spaces (as the prompt labels its facts) or a benchmark output key (free_school_meals_eligible, head_medicaid_eligible) is read as its engine variable, and the result's notes say how. Every model had a call refused for a guessed name in all 10 households. - Placement: an input set on another entity moves to the one unit that holds the people it was set for (a group left out becomes one unit of everyone, as the engine reads it); otherwise it stays an error. - Overrides: a contribution set as the realized amount is read as its _desired input (Sol and Opus put "traditional 401k contributions desired" into traditional_401k_contributions in 6 households, moving tax), and an input the engine computes and no stated fact sets is flagged as replacing its formula (Sol set rent, not pre_subsidy_rent). - Explanations: explain (up to 3 variables) returns the computation steps and the parameters read, with values and cited sources. Opus overrode a correct free-meals answer in Louisiana, which pays the reduced-price copay. Tool version 1 (pe, web+pe) is byte for byte as before. The notices and tool description now state the board's engine (a household set on 2.60.4 had been told 2.41.0); the frozen board's text is unchanged. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 562 ++++++++++++++++++++++++++++++--- tests/test_tool_pilot_argon.py | 1 + tests/test_tool_v2.py | 321 +++++++++++++++++++ 3 files changed, 835 insertions(+), 49 deletions(-) create mode 100644 tests/test_tool_v2.py diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index ea8f85f4..c8a1559b 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -152,13 +152,54 @@ # with no tools), so a prompt version other than the board's has its own # no-tools baseline. Repairs, retries and truncation then follow this pilot's # loop, not the board's; they matter only when a first answer does not parse. -ARMS = ("none", "pe", "web", "web+pe") +ARMS = ("none", "pe", "web", "web+pe", "pe2") ARM_SUFFIX = { "none": "+no-tools", "pe": "+pe-tool", "web": "+web", "web+pe": "+web+pe-tool", + "pe2": "+pe-tool-v2", } +# "pe2" is the calculator, tool version 2 (Max, 2026-10-23: "yes go"), alone: +# it reads the names the prompt uses, places inputs set on the wrong entity, +# flags inputs that replace a formula, and explains a value on request. +TOOL_VERSION_BY_ARM = {"pe": 1, "web+pe": 1, "pe2": 2} + +TOOL_NOTICE_V2 = ( + "\n\nYou also have a `calculate_household` function. It runs PolicyEngine US " + "(policyengine-us 2.41.0), an open-source rules engine for US federal and " + "state taxes and benefits, on a household situation you build from the facts " + "above. The facts above are labeled with its variable names, written with " + "spaces; it accepts them as written, and it accepts the output keys you are " + "asked for. Set every listed fact. Its result says which names it read, which " + "inputs it moved to the entity they belong to, and which inputs replace a " + "formula; ask it to `explain` up to three variables to see how it computed " + "them, with the parameters and laws it used. You may call it in up to 8 " + "turns. Then call `submit_outputs` exactly once with your final answers, as " + "instructed above." +) +JSON_TOOL_NOTICE_V2 = TOOL_NOTICE_V2.replace( + "Then call `submit_outputs` exactly once with your final answers, as " + "instructed above.", + "Then give your final answers as the single JSON object described above.", +) +# The engine version the notices and the tool description state: the board's +# (set by command_run), so a household set on a newer engine says so. The +# frozen board's 2.41.0 leaves every text byte for byte as before. +_STATED_ENGINE = {"version": REFERENCE_ENGINE} + + +def set_stated_engine(version: str) -> None: + _STATED_ENGINE["version"] = version + + +def engine_text(text: str) -> str: + return text.replace( + f"policyengine-us {REFERENCE_ENGINE}", + f"policyengine-us {_STATED_ENGINE['version']}", + ) + + # A web arm's request timeout. A search-heavy turn can outlast the board's # (300 s for Gemini 4 Argon, the harness default for its card; its web-only arm # never finished scenario_005 within it in six attempts on 2026-10-21), so the @@ -452,12 +493,43 @@ def refuse_policyengine(label: str, value) -> None: ) -def calculate_tool_schema() -> dict: +CALCULATE_DESCRIPTION_V2 = CALCULATE_DESCRIPTION.replace( + "An unknown name, a variable set on the wrong entity, or a malformed " + "situation returns an error with suggestions instead of results.", + "A name may also be written with spaces (as the facts are labeled) or be one " + "of the output keys you are asked for; the result's `notes` say how each was " + "read. An input set on the wrong entity is moved to the one unit it belongs " + "to, and the notes say so; where there is no single such unit, or a name is " + "unknown, or the situation is malformed, it returns an error with " + "suggestions instead of results. Setting a variable the engine computes " + "replaces its formula, and the notes say so; a contribution set as the " + "amount contributed is read as the amount asked for before limits " + "(`..._desired`). `explain` lists up to 3 variables to trace: each comes " + "back with the steps that produced it and the parameters, with their " + "values and sources, that it used.", +) +MAX_EXPLAIN = 3 + + +def calculate_tool_schema(version: int = 1) -> dict: + if version == 2: + schema = calculate_tool_schema(1) + function = schema["function"] + function["description"] = engine_text(CALCULATE_DESCRIPTION_V2) + function["parameters"]["properties"]["explain"] = { + "type": "array", + "items": {"type": "string"}, + "description": ( + f"Up to {MAX_EXPLAIN} variables to explain: how the engine " + "computed them." + ), + } + return schema return { "type": "function", "function": { "name": TOOL_NAME, - "description": CALCULATE_DESCRIPTION, + "description": engine_text(CALCULATE_DESCRIPTION), "parameters": { "type": "object", "properties": { @@ -499,9 +571,53 @@ class VariableInfo: label: str | None = None definition_period: str | None = None possible_values: tuple[str, ...] | None = None + # Whether the engine computes the variable (a formula or a sum of other + # variables) and no household fact the benchmark states sets it: setting + # it replaces what the engine would compute. + computed: bool = False -def build_registry(system) -> dict[str, VariableInfo]: +# Inputs the benchmark's reference situations set besides its promptable +# facts (policybench.scenarios.Scenario.to_pe_household), and the convention +# switches (take-up, filing) any household may set. +BASE_STATED_INPUTS = frozenset( + { + "age", + "employment_income", + "self_employment_income", + "long_term_capital_gains", + "partnership_s_corp_income", + "is_tax_unit_head", + "is_tax_unit_spouse", + "state_code", + "weekly_hours_worked_before_lsr", + } +) +CONVENTION_PREFIXES = ("takes_up_", "would_") + + +def stated_inputs() -> frozenset[str]: + """Every variable a household fact the benchmark states can set.""" + from policybench.scenarios import ( + PE_INPUT_ALIASES, + PE_INPUT_RENAMES, + get_promptable_input_specs, + ) + + names = {spec.source_name for spec in get_promptable_input_specs()} + names |= BASE_STATED_INPUTS + names |= set(PE_INPUT_ALIASES.values()) | set(PE_INPUT_RENAMES.values()) + return frozenset(names) + + +def build_registry( + system, stated: frozenset[str] | None = None +) -> dict[str, VariableInfo]: + """The engine's variables. ``stated`` (default :func:`stated_inputs`) marks + which ones a stated fact sets; the rest that the engine computes are + ``computed``.""" + if stated is None: + stated = stated_inputs() registry = {} for name, variable in system.variables.items(): value_type = getattr(variable.value_type, "__name__", str(variable.value_type)) @@ -514,6 +630,13 @@ def build_registry(system) -> dict[str, VariableInfo]: label=variable.label, definition_period=str(variable.definition_period), possible_values=possible, + computed=bool( + getattr(variable, "formulas", None) + or getattr(variable, "adds", None) + or getattr(variable, "subtracts", None) + ) + and name not in stated + and not name.startswith(CONVENTION_PREFIXES), ) return registry @@ -555,6 +678,44 @@ def _type_problem(value, info: VariableInfo) -> str | None: return None +def _normalize_value( + where: str, + name: str, + value, + info: VariableInfo, + period: str, + problems: list[str], +) -> dict | None: + """One input's {period: value}, or None after recording why not.""" + if isinstance(value, dict): + if not value: + problems.append( + f"{where}: {name} has no values; set it as {{period: value}}." + ) + return None + periods = value + else: + periods = {period: value} + normalized = {} + for key, item in periods.items(): + if not isinstance(key, str) or not PERIOD_RE.match(key): + problems.append( + f"{where}: {name} has period {key!r}; use a year such as " + '"2026" or a month such as "2026-01".' + ) + continue + scalar = _scalar_problem(item) + if scalar is not None: + problems.append(f"{where}: {name} for {key} {scalar}.") + continue + typed = _type_problem(item, info) + if typed is not None: + problems.append(f"{where}: {name} {typed}.") + continue + normalized[key] = item + return normalized + + def _normalize_inputs( where: str, entity: str, @@ -563,11 +724,20 @@ def _normalize_inputs( period: str, problems: list[str], allow_members: bool, + v2: dict | None = None, ) -> dict: + """Each input under its engine name. ``v2`` (tool version 2) collects the + names it read, the inputs it redirected or flagged, and the inputs set on + another entity, which normalize_request then places.""" out: dict[str, Any] = {} for name, value in data.items(): if allow_members and name == "members": continue + if v2 is not None and isinstance(name, str) and name not in registry: + resolved, note = resolve_name(name, registry, output_names=False) + if resolved is not None: + v2["notes"].append(f"{where}: {note}") + name = resolved if not isinstance(name, str) or name not in registry: problems.append( f"{where}: unknown variable {name!r}." @@ -575,52 +745,57 @@ def _normalize_inputs( ) continue info = registry[name] + if v2 is not None: + desired = f"{name}{DESIRED_SUFFIX}" + if ( + info.computed + and desired in registry + and registry[desired].entity == info.entity + ): + v2["notes"].append( + f"{where}: set {desired} (the amount asked for, before " + f"limits) for {name}, which the engine computes from it" + ) + name, info = desired, registry[desired] + elif info.computed: + v2["notes"].append( + f"{where}: {name} is computed by the engine; the value set " + "replaces its formula" + ) if info.entity != entity: + if v2 is not None: + v2["misplaced"].append((where, entity, name, value)) + continue problems.append( f"{where}: {name} is a {info.entity} variable; set it under " f"{ENTITY_GROUP.get(info.entity, info.entity)}." ) continue - if isinstance(value, dict): - if not value: - problems.append( - f"{where}: {name} has no values; set it as {{period: value}}." - ) - continue - periods = value - else: - periods = {period: value} - normalized = {} - for key, item in periods.items(): - if not isinstance(key, str) or not PERIOD_RE.match(key): - problems.append( - f"{where}: {name} has period {key!r}; use a year such as " - '"2026" or a month such as "2026-01".' - ) - continue - scalar = _scalar_problem(item) - if scalar is not None: - problems.append(f"{where}: {name} for {key} {scalar}.") - continue - typed = _type_problem(item, info) - if typed is not None: - problems.append(f"{where}: {name} {typed}.") - continue - normalized[key] = item + normalized = _normalize_value(where, name, value, info, period, problems) + if normalized is None: + continue + if v2 is not None and name in out and out[name] != normalized: + problems.append(f"{where}: {name} is set twice with different values") + continue out[name] = normalized return out def normalize_request( - args, registry: Mapping[str, VariableInfo] + args, registry: Mapping[str, VariableInfo], version: int = 1 ) -> tuple[dict | None, dict | None]: """Check a calculate_household call; return (request, None) or (None, error). Accepts only plain JSON data. Every input and output name must be a variable of the engine, on its own entity; nothing else in the situation - (axes, reforms, parameters) is accepted. + (axes, reforms, parameters) is accepted. Tool version 2 also reads a name + written with spaces or a benchmark output key, redirects a contribution to + its ``_desired`` input, flags an input that replaces a formula, places an + input set on another entity where exactly one unit holds it, and takes + ``explain``; the request's ``notes`` say what it did. """ problems: list[str] = [] + v2 = {"notes": [], "misplaced": []} if version == 2 else None def fail() -> tuple[None, dict]: return None, { @@ -643,7 +818,10 @@ def fail() -> tuple[None, dict]: if not isinstance(args, dict): problems.append("arguments must be an object") return fail() - unknown = sorted(set(args) - {"situation", "variables", "period"}) + allowed = {"situation", "variables", "period"} | ( + {"explain"} if version == 2 else set() + ) + unknown = sorted(set(args) - allowed) if unknown: problems.append(f"unknown arguments: {', '.join(map(str, unknown))}") @@ -702,7 +880,7 @@ def fail() -> tuple[None, dict]: problems.append(f"people.{name} must be an object of inputs") continue normalized_people[name] = _normalize_inputs( - f"people.{name}", "person", data, registry, period, problems, False + f"people.{name}", "person", data, registry, period, problems, False, v2 ) normalized["people"] = normalized_people @@ -746,7 +924,9 @@ def fail() -> tuple[None, dict]: seen[member] = unit_name normalized_unit = {"members": list(members)} normalized_unit.update( - _normalize_inputs(where, entity, unit, registry, period, problems, True) + _normalize_inputs( + where, entity, unit, registry, period, problems, True, v2 + ) ) normalized_units[unit_name] = normalized_unit normalized[group] = normalized_units @@ -763,18 +943,162 @@ def fail() -> tuple[None, dict]: variables = [] elif len(variables) > MAX_VARIABLES: problems.append(f"at most {MAX_VARIABLES} variables per call") + if v2 is not None: + _place_misplaced(v2, normalized, situation, registry, period, problems) requested = [] for name in variables: + if v2 is not None and isinstance(name, str) and name not in registry: + resolved, note = resolve_name(name, registry, output_names=True) + if resolved is not None: + v2["notes"].append(f"variables: {note}") + name = resolved if name not in registry: problems.append( f"variables: unknown variable {name!r}." + _suggest(name, registry) ) elif name not in requested: requested.append(name) + explain = [] + if v2 is not None and args.get("explain") is not None: + names = args["explain"] + if isinstance(names, str): + names = [names] + if not isinstance(names, list) or not all(isinstance(n, str) for n in names): + problems.append("explain must be a list of variable names") + names = [] + elif len(names) > MAX_EXPLAIN: + problems.append(f"explain at most {MAX_EXPLAIN} variables per call") + for name in names: + if name not in registry: + resolved, note = resolve_name(name, registry, output_names=True) + if resolved is not None: + v2["notes"].append(f"explain: {note}") + name = resolved + if name not in registry: + problems.append( + f"explain: unknown variable {name!r}." + _suggest(name, registry) + ) + elif name not in explain: + explain.append(name) if problems: return fail() - return {"situation": normalized, "variables": requested, "period": period}, None + request = {"situation": normalized, "variables": requested, "period": period} + if v2 is not None: + request["explain"] = explain + request["notes"] = v2["notes"] + return request, None + + +DESIRED_SUFFIX = "_desired" +_LABEL_SEPARATORS = re.compile(r"[\s\-]+") + + +def output_reading(name: str) -> tuple[str, str] | None: + """A benchmark output key's engine variable and how the output reads it, + or None when the name is not an output key.""" + from policybench.ground_truth import ( + _aggregation_for_output, + _pe_variable_for_output, + ) + from policybench.spec import find_output_spec, parse_person_output + + person = parse_person_output(name) + if person is None and find_output_spec(name, country="us") is None: + return None + engine = _pe_variable_for_output(name, "us") + if engine == name: + return None + aggregation = _aggregation_for_output(name, "us") + if person is not None: + how = f"{engine} for the person the key names ({person[0]})" + elif aggregation == "any_positive": + how = f"1 when {engine} is above 0, else 0" + else: + how = f"{engine}, summed over the household's units" + return engine, how + + +def resolve_name( + name: str, registry: Mapping[str, VariableInfo], output_names: bool +) -> tuple[str | None, str | None]: + """Tool version 2's reading of a name the engine does not know: the name + with spaces or hyphens as underscores, or (for outputs) a benchmark output + key. Returns (engine name, note) or (None, None).""" + if name in registry: + return name, None + written = _LABEL_SEPARATORS.sub("_", name.strip().lower()) + if written in registry: + return written, f"read {name!r} as {written}" + if output_names: + reading = output_reading(name) + if reading is not None and reading[0] in registry: + return reading[0], f"{name} is {reading[1]}; computed {reading[0]}" + return None, None + + +def _place_misplaced( + v2: dict, + normalized: dict, + situation: dict, + registry: Mapping[str, VariableInfo], + period: str, + problems: list[str], +) -> None: + """Move each input set on another entity to the one unit that holds the + people it was set for: a group variable set on a person goes to that + person's unit (a group left out of the situation becomes one unit of + everyone, as the engine reads it), a person variable set on a one-person + unit goes to that person, and a group variable set on another group goes + to the unit holding all of its members. Anything else stays an error.""" + for where, source_entity, name, value in v2["misplaced"]: + info = registry[name] + if source_entity == "person": + persons = [where.split(".", 1)[1]] + else: + group, unit = where.split(".", 1) + persons = list(normalized.get(group, {}).get(unit, {}).get("members", [])) + target_group = ENTITY_GROUP.get(info.entity, info.entity) + if info.entity == "person": + if len(persons) != 1: + problems.append( + f"{where}: {name} is a person variable; set it on one person " + "under people." + ) + continue + destination = normalized["people"].get(persons[0]) + to = f"people.{persons[0]}" + else: + if target_group not in situation and target_group not in normalized: + normalized[target_group] = { + "all": {"members": list(normalized["people"])} + } + units = normalized.get(target_group, {}) + holding = [ + unit + for unit, data in units.items() + if set(persons) <= set(data.get("members", [])) + ] + if len(holding) != 1: + problems.append( + f"{where}: {name} is a {info.entity} variable; set it under " + f"{target_group} (no single unit there holds " + f"{', '.join(persons) or 'its members'})." + ) + continue + destination = units[holding[0]] + to = f"{target_group}.{holding[0]}" + if destination is None: + problems.append(f"{where}: {name} has no place to go") + continue + normalized_value = _normalize_value(to, name, value, info, period, problems) + if normalized_value is None: + continue + if name in destination and destination[name] != normalized_value: + problems.append(f"{to}: {name} is set twice with different values") + continue + destination[name] = normalized_value + v2["notes"].append(f"moved {name} from {where} to {to}") def _jsonable(value): @@ -823,7 +1147,121 @@ def execute_request(system, registry, request: dict, simulation_cls=None) -> dic "period": request["period"], "values": {i: _jsonable(v) for i, v in zip(ids, list(array))}, } - return {"results": results} + out = {"results": results} + if "notes" in request: + out["notes"] = list(request["notes"]) + if request.get("explain"): + out["explanations"] = { + name: explain_variable(system, request, name, simulation_cls) + for name in request["explain"] + } + return out + + +EXPLAIN_MAX_LINES = 60 +EXPLAIN_MAX_DEPTH = 6 +EXPLAIN_MAX_PARAMETERS = 15 +# Switches every variable has; never part of a reason. +_QUIET_PARAMETERS = ("gov.abolitions.",) + + +def _trace_value(value) -> Any: + if hasattr(value, "decode_to_str"): + value = value.decode_to_str() + try: + items = list(value) + except TypeError: + return _jsonable(value) + items = [_jsonable(v) for v in items] + return items[0] if len(items) == 1 else items + + +def _all_zero(value) -> bool: + try: + return all((not v) if not isinstance(v, str) else False for v in list(value)) + except TypeError: + return not value + + +def parameter_sources(system, name: str, limit: int = 2) -> list[dict]: + """A parameter's cited sources: its own ``reference`` metadata, or the + nearest ancestor's.""" + node = system.parameters + chain = [node] + for part in name.split("."): + children = getattr(node, "children", None) + if not isinstance(children, dict) or part not in children: + break + node = children[part] + chain.append(node) + for node in reversed(chain): + refs = (getattr(node, "metadata", None) or {}).get("reference") + if refs: + refs = refs if isinstance(refs, list) else [refs] + out = [] + for ref in refs[:limit]: + if isinstance(ref, dict): + out.append({k: ref[k] for k in ("title", "href") if ref.get(k)}) + else: + out.append({"href": str(ref)}) + return out + return [] + + +def explain_variable(system, request: dict, name: str, simulation_cls=None) -> dict: + """How the engine computed ``name`` for the request's household: the + computation steps (each variable, its period and value, with its inputs + indented beneath it; zero-valued branches below the first level left out), + then the parameters it read, each with its value and cited sources.""" + if simulation_cls is None: + from policyengine_us import Simulation as simulation_cls + try: + simulation = simulation_cls( + tax_benefit_system=system, situation=copy.deepcopy(request["situation"]) + ) + simulation.trace = True + simulation.calculate(name, request["period"]) + except Exception as error: + return {"error": f"{type(error).__name__}: {error}"[:MAX_MESSAGE_CHARS]} + + def find(nodes): + for node in nodes: + if node.name == name: + return node + hit = find(node.children) + if hit is not None: + return hit + return None + + root = find(simulation.tracer.trees) + if root is None: + return {"error": f"no trace for {name}"} + steps: list[str] = [] + parameters: dict[str, Any] = {} + + def walk(node, depth): + if len(steps) >= EXPLAIN_MAX_LINES or depth > EXPLAIN_MAX_DEPTH: + return + if depth > 1 and _all_zero(node.value): + return + steps.append( + " " * depth + f"{node.name} ({node.period}) = {_trace_value(node.value)}" + ) + for parameter in node.parameters: + if parameter.name.startswith(_QUIET_PARAMETERS): + continue + parameters.setdefault(parameter.name, _trace_value(parameter.value)) + for child in node.children: + walk(child, depth + 1) + + walk(root, 0) + return { + "steps": steps, + "parameters": [ + {"name": key, "value": value, "sources": parameter_sources(system, key)} + for key, value in list(parameters.items())[:EXPLAIN_MAX_PARAMETERS] + ], + } def _board_upgrade(snapshot: Path) -> dict: @@ -918,7 +1356,10 @@ def _worker_main(conn, fix_path: str, reform: bool, board: str | None = None) -> return if args is None: return - request, error = normalize_request(args, registry) + version = 1 + if isinstance(args, dict) and args.get("__tool_version__") == 2: + version, args = 2, args.get("arguments") + request, error = normalize_request(args, registry, version) if error is not None: conn.send(("ok", error)) continue @@ -937,8 +1378,10 @@ def __init__( worker_target: Callable | None = None, acquire_timeout: float | None = None, board: Path | None = None, + tool_version: int = 1, ): self._ctx = mp.get_context("spawn") + self._tool_version = tool_version self._fix_path = str(fix_path) # A worker gets the board only when it is not the frozen one, so a # three-argument worker target still starts on the default. @@ -1005,6 +1448,8 @@ def call(self, args) -> tuple[dict, float]: time.perf_counter() - started, ) process, conn = worker + if self._tool_version == 2: + args = {"__tool_version__": 2, "arguments": args} try: conn.send(args) if conn.poll(self.timeout): @@ -1160,13 +1605,21 @@ def notice(self) -> str: notice = WEB_NOTICE if tool else JSON_WEB_NOTICE elif self.arm == "web+pe": notice = WEB_TOOL_NOTICE if tool else JSON_WEB_TOOL_NOTICE + elif self.arm == "pe2": + return engine_text(TOOL_NOTICE_V2 if tool else JSON_TOOL_NOTICE_V2) else: - return TOOL_NOTICE if tool else JSON_TOOL_NOTICE - return notice + (BENCHMARK_SEARCH_RULE if self.benchmark_search_rule else "") + return engine_text(TOOL_NOTICE if tool else JSON_TOOL_NOTICE) + return engine_text(notice) + ( + BENCHMARK_SEARCH_RULE if self.benchmark_search_rule else "" + ) @property def uses_calculator(self) -> bool: - return self.arm in ("pe", "web+pe") + return self.arm in TOOL_VERSION_BY_ARM + + @property + def tool_version(self) -> int: + return TOOL_VERSION_BY_ARM.get(self.arm, 1) @property def uses_web(self) -> bool: @@ -1845,7 +2298,7 @@ def run_household( sent_board_prompt_sha256 = base.pop("_sent_board_prompt_sha256") ceiling = base.pop("_budget_ceiling") answer_tool = base.pop("_answer_tool") - calculate_tool = calculate_tool_schema() + calculate_tool = calculate_tool_schema(profile.tool_version) offered = ([web_search_tool(profile)] if profile.uses_web else []) + ( [calculate_tool] if profile.uses_calculator else [] ) @@ -2606,6 +3059,8 @@ def records_arm(records: Mapping[str, dict]) -> str: def command_run(args) -> None: snapshot = board_path(args) _require_engine(snapshot) + # The notices and the tool description state the board's engine. + set_stated_engine(board_engine(snapshot)) profile = model_profile( args.model, args.arm, args.prompt, args.forbid_benchmark_search ) @@ -2678,7 +3133,10 @@ def command_run(args) -> None: "api": profile.api, "answer_contract": profile.contract, "tool_notice": profile.notice, - "calculate_tool": calculate_tool_schema() if profile.uses_calculator else None, + "calculate_tool": calculate_tool_schema(profile.tool_version) + if profile.uses_calculator + else None, + "tool_version": profile.tool_version if profile.uses_calculator else None, "web_search_tool": web_search_tool(profile) if profile.uses_web else None, # The timeout actually sent: the board's for the model, or the web arms'. "request_timeout_seconds": build_initial_request( @@ -2706,7 +3164,9 @@ def command_run(args) -> None: return # Arms without the calculator start no engine workers. pool = ( - ToolPool(size=args.workers, board=snapshot) if profile.uses_calculator else None + ToolPool(size=args.workers, board=snapshot, tool_version=profile.tool_version) + if profile.uses_calculator + else None ) def no_calculator(arguments) -> tuple[dict, float]: @@ -4025,7 +4485,9 @@ def main_call(record: dict, scenario, outputs: list[str]) -> dict | None: return main_call_and_count(record, scenario, outputs)[0] -def situation_fidelity(records, board, system, registry, defaults, out: Path) -> None: +def situation_fidelity( + records, board, system, registry, defaults, out: Path, version: int = 1 +) -> None: """How each household's main situation differs from the reference's. The main situation is ``main_call``'s: the successful call that computed @@ -4046,7 +4508,7 @@ def situation_fidelity(records, board, system, registry, defaults, out: Path) -> if call is None: rows.append({"scenario_id": sid, "differences": None}) continue - request, _ = normalize_request(call["arguments"], registry) + request, _ = normalize_request(call["arguments"], registry, version) if request is None: rows.append({"scenario_id": sid, "differences": None}) continue @@ -4098,6 +4560,8 @@ def command_diagnose(args) -> None: board = load_board(snapshot) out = Path(args.out_dir) records = load_records(out) + # A run's calls are read as its own tool version read them. + version = TOOL_VERSION_BY_ARM.get(records_arm(records), 1) rows_path = out / "scored_rows.csv" if not rows_path.exists(): raise SystemExit("run score first") @@ -4113,7 +4577,7 @@ def command_diagnose(args) -> None: name: system.variables[name].default_value for name in system.variables } if registry is not None: - situation_fidelity(records, board, system, registry, defaults, out) + situation_fidelity(records, board, system, registry, defaults, out, version) out_rows = [] if misses.empty: print("no arm-2 misses") @@ -4131,7 +4595,7 @@ def command_diagnose(args) -> None: if registry is not None: def board_value_for(arguments, scenario=scenario, output=row.variable): - request, _ = normalize_request(arguments, registry) + request, _ = normalize_request(arguments, registry, version) if request is None: return None return output_value(system, request["situation"], scenario, output) @@ -4155,7 +4619,7 @@ def board_value_for(arguments, scenario=scenario, output=row.variable): attribution = {} arguments = verdict.get("call_arguments") if arguments is not None and registry is not None: - request, _ = normalize_request(arguments, registry) + request, _ = normalize_request(arguments, registry, version) if request is not None: reference_situation = scenario.to_pe_household() diff = situation_diff( diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index a5d531b0..986a89d9 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -2293,6 +2293,7 @@ def test_web_notices_name_search_and_keep_the_answer_instruction(): "pe": "gpt-6-sol+pe-tool", "web": "gpt-6-sol+web", "web+pe": "gpt-6-sol+web+pe-tool", + "pe2": "gpt-6-sol+pe-tool-v2", } with pytest.raises(SystemExit): pilot.model_profile(SOL, "search") diff --git a/tests/test_tool_v2.py b/tests/test_tool_v2.py new file mode 100644 index 00000000..d6e381b5 --- /dev/null +++ b/tests/test_tool_v2.py @@ -0,0 +1,321 @@ +"""The calculator's tool version 2 (the pe2 arm). + +Version 2 reads the names the prompt uses (labels written with spaces, and the +benchmark's output keys), redirects a contribution set as the realized amount +to its ``_desired`` input, flags an input that replaces a formula, places an +input set on another entity where exactly one unit holds it, and explains a +variable on request. Version 1 must behave exactly as before. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import tool_pilot_argon as pilot # noqa: E402 + +V = pilot.VariableInfo +REGISTRY = { + "age": V("person", "int", "age", "year"), + "employment_income": V("person", "float", "employment income", "year"), + "rent": V("person", "float", "rent", "year"), + "traditional_401k_contributions": V( + "person", "float", "401(k)", "year", computed=True + ), + "traditional_401k_contributions_desired": V( + "person", "float", "401(k) asked", "year" + ), + "is_medicaid_eligible": V("person", "bool", "Medicaid", "year", computed=True), + "snap": V("spm_unit", "float", "SNAP", "month", computed=True), + "free_school_meals": V("spm_unit", "float", "free meals", "year", computed=True), + "qualified_passenger_vehicle_loan_interest": V( + "household", "float", "car loan interest", "year" + ), + "state_code": V("household", "Enum", "state", "year", ("CA", "TX")), + "filing_status": V( + "tax_unit", "Enum", "filing status", "year", ("SINGLE", "JOINT"), True + ), +} + + +def _situation(**overrides): + base = { + "people": { + "head": {"age": {"2026": 40}, "employment_income": 30000}, + "kid": {"age": {"2026": 8}}, + }, + "spm_units": {"spm": {"members": ["head", "kid"]}}, + "households": {"h": {"members": ["head", "kid"], "state_code": "CA"}}, + } + base.update(overrides) + return base + + +def _call(situation, variables=("snap",), **extra): + return {"situation": json.dumps(situation), "variables": list(variables), **extra} + + +def _v2(args): + return pilot.normalize_request(args, REGISTRY, 2) + + +def test_version_1_still_refuses_what_version_2_reads(): + situation = _situation() + situation["people"]["head"]["traditional 401k contributions desired"] = 1000 + situation["people"]["head"]["qualified_passenger_vehicle_loan_interest"] = 500 + request, error = pilot.normalize_request( + _call(situation, ("free_school_meals_eligible",), explain=["snap"]), REGISTRY + ) + assert request is None + problems = " ".join(error["problems"]) + assert "unknown arguments: explain" in problems + assert "unknown variable 'traditional 401k contributions desired'" in problems + assert "is a household variable; set it under households" in problems + assert "unknown variable 'free_school_meals_eligible'" in problems + + +def test_version_2_reads_a_label_written_with_spaces(): + situation = _situation() + situation["people"]["head"]["Traditional 401k contributions desired"] = 1000 + request, error = _v2(_call(situation)) + assert error is None + head = request["situation"]["people"]["head"] + assert head["traditional_401k_contributions_desired"] == {"2026": 1000} + assert any( + "as traditional_401k_contributions_desired" in n for n in request["notes"] + ) + + +def test_version_2_sets_the_asked_amount_for_a_computed_contribution(): + situation = _situation() + situation["people"]["head"]["traditional_401k_contributions"] = 18523 + request, error = _v2(_call(situation)) + assert error is None + head = request["situation"]["people"]["head"] + assert "traditional_401k_contributions" not in head + assert head["traditional_401k_contributions_desired"] == {"2026": 18523} + assert any("before limits" in n for n in request["notes"]) + + +def test_version_2_flags_an_input_that_replaces_a_formula(): + situation = _situation() + situation["spm_units"]["spm"]["snap"] = 100 + request, error = _v2(_call(situation)) + assert error is None + assert request["situation"]["spm_units"]["spm"]["snap"] == {"2026": 100} + assert any("snap is computed by the engine" in n for n in request["notes"]) + + +def test_version_2_moves_a_household_input_to_the_persons_household(): + situation = _situation() + situation["people"]["head"]["qualified_passenger_vehicle_loan_interest"] = 500 + request, error = _v2(_call(situation)) + assert error is None + assert request["situation"]["households"]["h"][ + "qualified_passenger_vehicle_loan_interest" + ] == {"2026": 500} + assert ( + "qualified_passenger_vehicle_loan_interest" + not in (request["situation"]["people"]["head"]) + ) + assert any( + "moved qualified_passenger_vehicle_loan_interest" in n for n in request["notes"] + ) + + +def test_version_2_makes_the_one_unit_the_engine_would_when_a_group_is_left_out(): + situation = _situation() + del situation["households"] + situation["people"]["kid"]["state_code"] = "TX" + request, error = _v2(_call(situation)) + assert error is None + households = request["situation"]["households"] + assert households == { + "all": {"members": ["head", "kid"], "state_code": {"2026": "TX"}} + } + + +def test_version_2_moves_a_person_input_off_a_one_person_unit_only(): + one = _situation( + spm_units={"a": {"members": ["head"], "rent": 9000}, "b": {"members": ["kid"]}} + ) + request, error = _v2(_call(one)) + assert error is None + assert request["situation"]["people"]["head"]["rent"] == {"2026": 9000} + both = _situation() + both["spm_units"]["spm"]["rent"] = 9000 + request, error = _v2(_call(both)) + assert request is None + assert any("set it on one person" in p for p in error["problems"]) + + +def test_version_2_refuses_two_different_values_for_one_input(): + situation = _situation() + situation["households"]["h"]["qualified_passenger_vehicle_loan_interest"] = 400 + situation["people"]["head"]["qualified_passenger_vehicle_loan_interest"] = 500 + request, error = _v2(_call(situation)) + assert request is None + assert any("set twice with different values" in p for p in error["problems"]) + + +def test_version_2_refuses_a_group_input_no_single_unit_holds(): + situation = _situation( + households={"h1": {"members": ["head"]}, "h2": {"members": ["kid"]}}, + tax_units={"t": {"members": ["head", "kid"], "state_code": "CA"}}, + ) + request, error = _v2(_call(situation)) + assert request is None + assert any("no single unit there holds" in p for p in error["problems"]) + + +def test_version_2_reads_output_keys_as_their_engine_variables(): + request, error = _v2( + _call(_situation(), ("free_school_meals_eligible", "head_medicaid_eligible")) + ) + assert error is None + assert request["variables"] == ["free_school_meals", "is_medicaid_eligible"] + notes = " ".join(request["notes"]) + assert "1 when free_school_meals is above 0" in notes + assert "is_medicaid_eligible for the person the key names (head)" in notes + + +def test_version_2_explains_at_most_three_known_variables(): + request, error = _v2(_call(_situation(), explain=["snap", "free school meals"])) + assert error is None + assert request["explain"] == ["snap", "free_school_meals"] + request, error = _v2(_call(_situation(), explain=["snap", "a", "b", "c"])) + assert request is None + assert any("explain at most 3" in p for p in error["problems"]) + request, error = _v2(_call(_situation(), explain=["no_such_thing"])) + assert request is None + assert any("explain: unknown variable" in p for p in error["problems"]) + + +@given(st.sampled_from(sorted(REGISTRY)), st.sampled_from([" ", " ", "-", " - "])) +def test_a_label_written_with_any_separator_reads_back_to_its_name(name, separator): + written = name.replace("_", separator).upper() + resolved, note = pilot.resolve_name(written, REGISTRY, output_names=False) + assert resolved == name + assert (note is None) == (written == name) + + +def test_the_stated_engine_follows_the_board_and_defaults_to_the_frozen_one(): + profile = pilot.ModelProfile("m", "gpt-x", "chat", "tool", arm="pe") + frozen_notice = profile.notice + frozen_schema = pilot.calculate_tool_schema(1) + assert "policyengine-us 2.41.0" in frozen_notice + try: + pilot.set_stated_engine("2.60.4") + assert "policyengine-us 2.60.4" in profile.notice + assert "2.41.0" not in profile.notice + assert ( + "policyengine-us 2.60.4" + in pilot.calculate_tool_schema(2)["function"]["description"] + ) + finally: + pilot.set_stated_engine(pilot.REFERENCE_ENGINE) + assert profile.notice == frozen_notice + assert pilot.calculate_tool_schema(1) == frozen_schema + + +def test_the_pe2_arm_offers_tool_version_2(): + profile = pilot.ModelProfile("m", "gpt-x", "chat", "tool", arm="pe2") + assert profile.uses_calculator and not profile.uses_web + assert profile.tool_version == 2 + assert profile.notice == pilot.TOOL_NOTICE_V2 + assert "explain" in profile.notice + assert profile.arm2_label == "m+pe-tool-v2" + json_profile = pilot.ModelProfile("m", "claude-x", "chat", "json", arm="pe2") + assert json_profile.notice.endswith("single JSON object described above.") + for arm in ("pe", "web+pe"): + assert ( + pilot.ModelProfile("m", "gpt-x", "chat", "tool", arm=arm).tool_version == 1 + ) + v2 = pilot.calculate_tool_schema(2)["function"] + assert "explain" in v2["parameters"]["properties"] + assert ( + "explain" + not in pilot.calculate_tool_schema(1)["function"]["parameters"]["properties"] + ) + + +def _echo_worker(conn, fix_path, reform): + conn.send(("ready", None)) + while True: + args = conn.recv() + if args is None: + return + conn.send(("ok", {"received": args})) + + +def test_the_pool_tells_its_workers_the_tool_version(): + pool = pilot.ToolPool( + size=1, timeout=5.0, worker_target=_echo_worker, tool_version=2 + ) + try: + result, _ = pool.call({"situation": "{}"}) + assert result == { + "received": {"__tool_version__": 2, "arguments": {"situation": "{}"}} + } + finally: + pool.close() + pool = pilot.ToolPool(size=1, timeout=5.0, worker_target=_echo_worker) + try: + result, _ = pool.call({"situation": "{}"}) + assert result == {"received": {"situation": "{}"}} + finally: + pool.close() + + +@pytest.mark.slow +def test_an_explanation_traces_the_engine_and_cites_its_parameters(): + system = pilot.build_tool_system(reform=True) + registry = pilot.build_registry(system) + # Computed: the engine derives it and no stated fact sets it. + assert registry["snap"].computed and registry["is_medicaid_eligible"].computed + assert not registry["employment_income"].computed # summed, but stated + assert not registry["age"].computed + assert not registry["takes_up_snap_if_eligible"].computed + situation = { + "people": { + "head": { + "age": {"2026": 40}, + "employment_income": {"2026": 20000}, + "traditional_401k_contributions": {"2026": 1000}, + } + }, + "households": {"h": {"members": ["head"], "state_code": "TX"}}, + } + request, error = pilot.normalize_request( + { + "situation": json.dumps(situation), + "variables": ["federal_income_tax_before_refundable_credits"], + "explain": ["income_tax_before_refundable_credits"], + }, + registry, + 2, + ) + assert error is None, error + assert request["variables"] == ["income_tax_before_refundable_credits"] + result = pilot.execute_request(system, registry, request) + if "traditional_401k_contributions_desired" in registry: + assert any("before limits" in n for n in result["notes"]) + explanation = result["explanations"]["income_tax_before_refundable_credits"] + assert explanation["steps"][0].startswith( + "income_tax_before_refundable_credits (2026)" + ) + assert len(explanation["steps"]) <= pilot.EXPLAIN_MAX_LINES + assert explanation["parameters"], explanation + assert not any( + p["name"].startswith("gov.abolitions.") for p in explanation["parameters"] + ) + assert any(p["sources"] for p in explanation["parameters"]) + value = result["results"]["income_tax_before_refundable_credits"]["values"] + assert set(value) == {"0"} or len(value) == 1 From 5bbd571a51d15ec8f5deb928fb0afacf3e72b3d8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 14:18:50 -0400 Subject: [PATCH 07/64] Judge a batch from any model set Max, 2026-10-23: "thinking we might want to do all these tests with a cheap model until we're super ready for v2 with the web and tool experiments". batch_judge.py payload takes --models (the frontier three by default). The payload's household prompt is the first tool-contract model's, checked against that model's records, and the payload records which models it holds. Co-Authored-By: Claude Opus 5.5 --- scripts/batch_judge.py | 44 +++++++++++++++++++++++++++++---------- tests/test_batch_judge.py | 35 ++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/scripts/batch_judge.py b/scripts/batch_judge.py index 77dc6ec2..17e33045 100644 --- a/scripts/batch_judge.py +++ b/scripts/batch_judge.py @@ -78,10 +78,21 @@ def load_batch(board: Path, batch: int) -> list[str]: return list(batches[batch]) -def household_prompt(scenario, variables: list[str], prompt_version: str) -> str: - """The household prompt the tool-contract models saw, without the arm's - notice: the board's prompt in the run's prompt version.""" - profile = pilot.model_profile(PROMPT_MODEL, "none", prompt_version, False) +def prompt_model_for(models) -> str: + """The model whose household prompt a payload shows: the first on the tool + contract (the dashboard's ``prompt.tool``), else the first model.""" + for model in models: + if pilot.model_profile(model, "none").contract == "tool": + return model + return models[0] + + +def household_prompt( + scenario, variables: list[str], prompt_version: str, model: str = PROMPT_MODEL +) -> str: + """The household prompt ``model`` saw, without the arm's notice: the + board's prompt in the run's prompt version.""" + profile = pilot.model_profile(model, "none", prompt_version, False) prompt, kwargs = pilot.build_initial_request(scenario, variables, profile) return prompt[: len(prompt) - len(profile.notice)] if profile.notice else prompt @@ -105,9 +116,15 @@ def build_payload( arm: str, batch: int | None = None, scenarios: list[str] | None = None, + models: tuple[str, ...] = MODELS, ) -> dict: """One arm's answers on one batch (or on named households), as a - dashboard payload.""" + dashboard payload, from ``models`` (the frontier three by default; cheap + models for development, Max 2026-10-23).""" + models = tuple(models) + if len(models) < 2: + raise SystemExit("a payload needs at least two models") + prompt_model = prompt_model_for(models) if arm not in JUDGED_ARMS: raise SystemExit(f"only {sorted(JUDGED_ARMS)} are judged, not {arm}") if (batch is None) == (not scenarios): @@ -120,7 +137,7 @@ def build_payload( scored = set(zip(board["scored"].scenario_id, board["scored"].variable)) reference = board["reference"].set_index(["scenario_id", "variable"])["value"] records = {} - for model in MODELS: + for model in models: folder = runs / model / arm_folder(arm) / stage / "households" found = {} for sid in ids: @@ -136,16 +153,16 @@ def build_payload( raise SystemExit(f"the models ran different prompt versions: {versions}") prompt_version = versions.pop() - labels = {model: f"{model}+{arm}" for model in MODELS} + labels = {model: f"{model}+{arm}" for model in models} scenarios, predictions, variables_seen = {}, {}, set() for sid in ids: scenario = board["scenarios"][sid] variables = board["variables"][sid] - prompt = household_prompt(scenario, variables, prompt_version) - sent = records[PROMPT_MODEL][sid].get("sent_board_prompt_sha256") + prompt = household_prompt(scenario, variables, prompt_version, prompt_model) + sent = records[prompt_model][sid].get("sent_board_prompt_sha256") if sent != hashlib.sha256(prompt.encode()).hexdigest(): raise SystemExit( - f"{sid}: the rebuilt prompt is not the one {PROMPT_MODEL} saw" + f"{sid}: the rebuilt prompt is not the one {prompt_model} saw" ) scenarios[sid] = {"prompt": {"tool": prompt}, "state": scenario.state} cells = {} @@ -175,8 +192,9 @@ def build_payload( "prompt_version": prompt_version, }, "answerCondition": JUDGED_ARMS[arm], + "models": list(models), "outputDefinitions": output_definitions(variables_seen, prompt_version), - "modelStats": [{"model": labels[model]} for model in MODELS], + "modelStats": [{"model": labels[model]} for model in models], "scenarios": scenarios, "scenarioPredictions": predictions, } @@ -230,6 +248,7 @@ def command_payload(args) -> None: args.arm, batch=args.batch, scenarios=args.scenarios, + models=tuple(args.models), ) Path(args.out).write_text(json.dumps(payload, indent=1) + "\n") cells = sum(len(v) for v in payload["scenarioPredictions"].values()) @@ -291,6 +310,9 @@ def main(argv: list[str] | None = None) -> None: "--scenarios", nargs="*", help="households, for a board without batches" ) payload.add_argument("--out", required=True) + payload.add_argument( + "--models", nargs="+", default=list(MODELS), help="the models to judge" + ) payload.set_defaults(func=command_payload) derivations = sub.add_parser("derivations", help="traces of flagged references") derivations.add_argument("--board", required=True) diff --git a/tests/test_batch_judge.py b/tests/test_batch_judge.py index b34ae6d0..f9e4f7d2 100644 --- a/tests/test_batch_judge.py +++ b/tests/test_batch_judge.py @@ -150,7 +150,7 @@ def _write_arm(runs, board, model, arm, stage, wrong=None): folder.mkdir(parents=True) for sid in sorted(board["scenarios"]): scenario, variables = board["scenarios"][sid], board["variables"][sid] - prompt = bj.household_prompt(scenario, variables, "v2-pilot") + prompt = bj.household_prompt(scenario, variables, "v2-pilot", model) predictions = {v: float(reference[(sid, v)]) for v in variables} if wrong and sid == wrong[0]: predictions[wrong[1]] = wrong[2] @@ -249,3 +249,36 @@ def test_a_payload_refuses_an_unfinished_batch_a_calculator_arm_and_another_prom bj.build_payload(out, runs, "b0", "none", batch=2) with pytest.raises(SystemExit, match="not both"): bj.build_payload(out, runs, "b0", "none", batch=0, scenarios=["mc_000"]) + + +CHEAP = ("claude-haiku-5.5", "gpt-6-luna", "gemini-3.1-flash-lite-preview") + + +def test_the_prompt_model_is_the_first_on_the_tool_contract(): + assert bj.prompt_model_for(bj.MODELS) == "gpt-6-sol" + assert bj.prompt_model_for(CHEAP) == "claude-haiku-5.5" + assert bj.prompt_model_for(("gemini-3.1-flash-lite-preview", "gpt-6-luna")) == ( + "gpt-6-luna" + ) + + +@pytest.mark.slow +def test_a_payload_judges_any_model_set(household_board, tmp_path): + out, _ = household_board + board = pilot.load_board(out) + runs = tmp_path / "runs" + variable = "federal_income_tax_before_refundable_credits" + for model in CHEAP: + wrong = ("mc_000", variable, 999.0) if model != "gpt-6-luna" else None + _write_arm(runs, board, model, "none", "b0", wrong=wrong) + payload = bj.build_payload(out, runs, "b0", "none", batch=0, models=CHEAP) + assert payload["models"] == list(CHEAP) + assert [m["model"] for m in payload["modelStats"]] == [f"{m}+none" for m in CHEAP] + flags = consensus_flags(payload, THREE) + assert [(f["scenario_id"], f["variable"]) for f in flags] == [("mc_000", variable)] + assert flags[0]["clusters"][0]["models"] == [ + "claude-haiku-5.5+none", + "gemini-3.1-flash-lite-preview+none", + ] + with pytest.raises(SystemExit, match="at least two models"): + bj.build_payload(out, runs, "b0", "none", batch=0, models=CHEAP[:1]) From e881a6a418b78d6164cf062f35332ae21b830516 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 17:11:22 -0400 Subject: [PATCH 08/64] Calculator features one at a time: tool v3 and ablation arms The calculator's six features (names, placement, overrides, explain, concise, flat) are each on or off, so a study can change one at a time. "pe2" keeps tool version 2 (the first four, its notice byte for byte), "pe3" offers all six, and any other set runs as "pe:" plus its features, e.g. "pe:names+flat". - concise: an explanation gives the value, its direct inputs and up to five parameters unless `detail` is "full". - flat: the household is described as a state, people with a role (head, spouse or dependent) and facts, and household facts; the tool builds the units the benchmark's own situations have and places each household fact on its entity. Tests: the v2 notice is reproduced from its features; each feature adds only its own behavior; flat builds the benchmark's units and refuses what it cannot place; and on the installed engine eight frozen-board households described flat compute every board output exactly as Scenario.to_pe_household does. Five mutants of the flat and arm logic are all caught. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 522 +++++++++++++++++++++++++++++---- tests/test_tool_pilot_argon.py | 1 + tests/test_tool_v3.py | 377 ++++++++++++++++++++++++ 3 files changed, 850 insertions(+), 50 deletions(-) create mode 100644 tests/test_tool_v3.py diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index c8a1559b..88d8c4d6 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -152,18 +152,75 @@ # with no tools), so a prompt version other than the board's has its own # no-tools baseline. Repairs, retries and truncation then follow this pilot's # loop, not the board's; they matter only when a first answer does not parse. -ARMS = ("none", "pe", "web", "web+pe", "pe2") +ARMS = ("none", "pe", "web", "web+pe", "pe2", "pe3") ARM_SUFFIX = { "none": "+no-tools", "pe": "+pe-tool", "web": "+web", "web+pe": "+web+pe-tool", "pe2": "+pe-tool-v2", + "pe3": "+pe-tool-v3", } -# "pe2" is the calculator, tool version 2 (Max, 2026-10-23: "yes go"), alone: -# it reads the names the prompt uses, places inputs set on the wrong entity, -# flags inputs that replace a formula, and explains a value on request. -TOOL_VERSION_BY_ARM = {"pe": 1, "web+pe": 1, "pe2": 2} +# The calculator's features, each on or off, so a study can change one at a +# time (Max, 2026-10-23: "measuring this as an experiment could be +# publishable"; "do all these tests with a cheap model"): +# - names: a name written with spaces, or a benchmark output key, is read as +# its engine variable; +# - placement: an input set on another entity moves to the one unit holding +# its people; +# - overrides: a realized contribution is read as its _desired input, and an +# input that replaces a formula is flagged; +# - explain: `explain` traces up to three variables; +# - concise: an explanation is brief unless `detail` is "full"; +# - flat: the household is described as people with roles and facts +# (`household`), and the tool builds its units as the benchmark does. +# "pe2" is tool version 2 (the first four); "pe3" is all six. Any other set +# runs as an arm named "pe:" plus its features joined by "+", e.g. +# "pe:names+flat". +NAMES, PLACEMENT, OVERRIDES, EXPLAIN, CONCISE, FLAT = ( + "names", + "placement", + "overrides", + "explain", + "concise", + "flat", +) +TOOL_FEATURES = (NAMES, PLACEMENT, OVERRIDES, EXPLAIN, CONCISE, FLAT) +V2_FEATURES = frozenset({NAMES, PLACEMENT, OVERRIDES, EXPLAIN}) +FEATURES_BY_ARM = { + "pe": frozenset(), + "web+pe": frozenset(), + "pe2": V2_FEATURES, + "pe3": frozenset(TOOL_FEATURES), +} +_FEATURE_ARM = re.compile(r"^pe:([a-z]+(?:\+[a-z]+)*)$") + + +def arm_features(arm: str) -> frozenset | None: + """The calculator features an arm offers, or None for an arm without the + calculator. Refuses an unknown feature, or concise without explain.""" + if arm in FEATURES_BY_ARM: + return FEATURES_BY_ARM[arm] + match = _FEATURE_ARM.match(arm or "") + if match is None: + return None + features = frozenset(match.group(1).split("+")) + unknown = sorted(features - set(TOOL_FEATURES)) + if unknown: + raise SystemExit( + f"unknown tool features {unknown}; the features are {TOOL_FEATURES}" + ) + if CONCISE in features and EXPLAIN not in features: + raise SystemExit("the concise feature needs explain") + return features + + +def arm_suffix(arm: str) -> str: + if arm in ARM_SUFFIX: + return ARM_SUFFIX[arm] + features = arm_features(arm) + return f"+pe-tool[{'+'.join(f for f in TOOL_FEATURES if f in features)}]" + TOOL_NOTICE_V2 = ( "\n\nYou also have a `calculate_household` function. It runs PolicyEngine US " @@ -183,6 +240,76 @@ "instructed above.", "Then give your final answers as the single JSON object described above.", ) + + +def _listing(items: list[str]) -> str: + if len(items) <= 2: + return " and ".join(items) + return ", ".join(items[:-1]) + ", and " + items[-1] + + +def compose_tool_notice(features: frozenset, tool_contract: bool) -> str: + """The calculator notice for a set of features: the opening, a sentence + per feature, then the turn limit and the answer instruction. No feature + gives TOOL_NOTICE and the version 2 features give TOOL_NOTICE_V2, byte for + byte.""" + if not features: + return TOOL_NOTICE if tool_contract else JSON_TOOL_NOTICE + what = ( + "household you describe" + if FLAT in features + else "household situation you build" + ) + text = ( + "\n\nYou also have a `calculate_household` function. It runs PolicyEngine US " + "(policyengine-us 2.41.0), an open-source rules engine for US federal and " + f"state taxes and benefits, on a {what} from the facts above." + ) + if FLAT in features: + text += ( + " Describe it as `household`: each person with their role (head, spouse " + "or dependent) and facts, and the facts about the household as a " + "whole; it builds the household's units as the facts imply." + ) + if NAMES in features: + text += ( + " The facts above are labeled with its variable names, written with " + "spaces; it accepts them as written, and it accepts the output keys you " + "are asked for. Set every listed fact." + ) + notes = [] + if NAMES in features: + notes.append("which names it read") + if PLACEMENT in features: + notes.append("which inputs it moved to the entity they belong to") + if OVERRIDES in features: + notes.append("which inputs replace a formula") + explain = "" + if EXPLAIN in features: + explain = ( + "ask it to `explain` up to three variables to see in brief how it " + "computed them, with the parameters and laws it used, or every step " + 'with `detail` set to "full"' + if CONCISE in features + else "ask it to `explain` up to three variables to see how it computed " + "them, with the parameters and laws it used" + ) + if notes and explain: + text += f" Its result says {_listing(notes)}; {explain}." + elif notes: + text += f" Its result says {_listing(notes)}." + elif explain: + text += f" You may {explain}." + text += " You may call it in up to 8 turns. " + text += ( + "Then call `submit_outputs` exactly once with your final answers, as " + "instructed above." + if tool_contract + else "Then give your final answers as the single JSON object described above." + ) + return text + + # The engine version the notices and the tool description state: the board's # (set by command_run), so a household set on a newer engine says so. The # frozen board's 2.41.0 leaves every text byte for byte as before. @@ -511,19 +638,86 @@ def refuse_policyengine(label: str, value) -> None: MAX_EXPLAIN = 3 -def calculate_tool_schema(version: int = 1) -> dict: - if version == 2: +FLAT_DESCRIPTION = ( + "Run PolicyEngine US (policyengine-us 2.41.0) on one household and return " + "the requested variables. `household` is a JSON object: `state` (the " + "two-letter state code), `people` (a list, each with `name`, `role` -- " + 'head, spouse or dependent -- and `facts`, for example {"age": 40, ' + '"employment income": 30000}), and `household_facts` (facts about the ' + "household as a whole: its tax unit, family, SPM unit or home). The head " + "and spouse are one couple and everyone else is on their own; everyone " + "shares one tax unit, family, SPM unit and household. A fact is a value " + 'for `period`, or {period: value} with a period such as "2026" or ' + '"2026-01". Facts you do not give take the engine\'s defaults. `variables` ' + 'lists the variables to compute for `period` (default "2026"). Each ' + "result gives the variable's entity, label and value for each member of " + "that entity; a monthly variable computed for a year is summed over its " + "months." +) + + +def calculate_tool_schema(version: int = 1, features: frozenset | None = None) -> dict: + if features is None: + features = V2_FEATURES if version == 2 else frozenset() + if features: schema = calculate_tool_schema(1) function = schema["function"] - function["description"] = engine_text(CALCULATE_DESCRIPTION_V2) - function["parameters"]["properties"]["explain"] = { - "type": "array", - "items": {"type": "string"}, - "description": ( - f"Up to {MAX_EXPLAIN} variables to explain: how the engine " - "computed them." - ), - } + properties = function["parameters"]["properties"] + if features == V2_FEATURES: + function["description"] = engine_text(CALCULATE_DESCRIPTION_V2) + else: + description = ( + FLAT_DESCRIPTION if FLAT in features else CALCULATE_DESCRIPTION + ) + notes = [] + if NAMES in features: + notes.append( + "A name may also be written with spaces (as the facts are " + "labeled) or be one of the output keys you are asked for." + ) + if PLACEMENT in features and FLAT not in features: + notes.append( + "An input set on the wrong entity is moved to the one unit it " + "belongs to." + ) + if OVERRIDES in features: + notes.append( + "Setting a variable the engine computes replaces its formula; a " + "contribution set as the amount contributed is read as the " + "amount asked for before limits (`..._desired`)." + ) + if notes: + notes.append("The result's `notes` say what it did.") + function["description"] = engine_text(" ".join([description, *notes])) + if FLAT in features: + del properties["situation"] + properties["household"] = { + "type": "string", + "description": ( + "The household as a JSON object (state, people with role and " + "facts, household_facts), written as a JSON string." + ), + } + function["parameters"]["required"] = ["household", "variables"] + if EXPLAIN in features: + properties["explain"] = { + "type": "array", + "items": {"type": "string"}, + "description": ( + f"Up to {MAX_EXPLAIN} variables to explain: how the engine " + "computed them." + ), + } + if CONCISE in features: + properties["detail"] = { + "type": "string", + "enum": ["brief", "full"], + "description": ( + 'How much an explanation shows: "brief" (the default: the ' + 'value, what it came from, and the main parameters) or "full" ' + "(every step)." + ), + } return schema return { "type": "function", @@ -733,7 +927,12 @@ def _normalize_inputs( for name, value in data.items(): if allow_members and name == "members": continue - if v2 is not None and isinstance(name, str) and name not in registry: + if ( + v2 is not None + and NAMES in v2["features"] + and isinstance(name, str) + and name not in registry + ): resolved, note = resolve_name(name, registry, output_names=False) if resolved is not None: v2["notes"].append(f"{where}: {note}") @@ -745,7 +944,7 @@ def _normalize_inputs( ) continue info = registry[name] - if v2 is not None: + if v2 is not None and OVERRIDES in v2["features"]: desired = f"{name}{DESIRED_SUFFIX}" if ( info.computed @@ -763,7 +962,7 @@ def _normalize_inputs( "replaces its formula" ) if info.entity != entity: - if v2 is not None: + if v2 is not None and PLACEMENT in v2["features"]: v2["misplaced"].append((where, entity, name, value)) continue problems.append( @@ -782,7 +981,10 @@ def _normalize_inputs( def normalize_request( - args, registry: Mapping[str, VariableInfo], version: int = 1 + args, + registry: Mapping[str, VariableInfo], + version: int = 1, + features: frozenset | None = None, ) -> tuple[dict | None, dict | None]: """Check a calculate_household call; return (request, None) or (None, error). @@ -795,7 +997,9 @@ def normalize_request( ``explain``; the request's ``notes`` say what it did. """ problems: list[str] = [] - v2 = {"notes": [], "misplaced": []} if version == 2 else None + if features is None: + features = V2_FEATURES if version == 2 else frozenset() + v2 = {"notes": [], "misplaced": [], "features": features} if features else None def fail() -> tuple[None, dict]: return None, { @@ -818,9 +1022,10 @@ def fail() -> tuple[None, dict]: if not isinstance(args, dict): problems.append("arguments must be an object") return fail() - allowed = {"situation", "variables", "period"} | ( - {"explain"} if version == 2 else set() - ) + allowed = {"variables", "period"} + allowed |= {"household"} if FLAT in features else {"situation"} + allowed |= {"explain"} if EXPLAIN in features else set() + allowed |= {"detail"} if CONCISE in features else set() unknown = sorted(set(args) - allowed) if unknown: problems.append(f"unknown arguments: {', '.join(map(str, unknown))}") @@ -836,6 +1041,13 @@ def fail() -> tuple[None, dict]: ) period = str(YEAR) + if FLAT in features: + args = dict(args) + args["situation"] = flat_situation( + args.get("household"), registry, problems, v2 + ) + if args["situation"] is None: + return fail() situation = args.get("situation") if isinstance(situation, str): if len(situation) > MAX_SITUATION_CHARS: @@ -943,11 +1155,16 @@ def fail() -> tuple[None, dict]: variables = [] elif len(variables) > MAX_VARIABLES: problems.append(f"at most {MAX_VARIABLES} variables per call") - if v2 is not None: + if v2 is not None and PLACEMENT in features: _place_misplaced(v2, normalized, situation, registry, period, problems) requested = [] for name in variables: - if v2 is not None and isinstance(name, str) and name not in registry: + if ( + v2 is not None + and NAMES in features + and isinstance(name, str) + and name not in registry + ): resolved, note = resolve_name(name, registry, output_names=True) if resolved is not None: v2["notes"].append(f"variables: {note}") @@ -959,7 +1176,7 @@ def fail() -> tuple[None, dict]: elif name not in requested: requested.append(name) explain = [] - if v2 is not None and args.get("explain") is not None: + if EXPLAIN in features and args.get("explain") is not None: names = args["explain"] if isinstance(names, str): names = [names] @@ -969,7 +1186,7 @@ def fail() -> tuple[None, dict]: elif len(names) > MAX_EXPLAIN: problems.append(f"explain at most {MAX_EXPLAIN} variables per call") for name in names: - if name not in registry: + if NAMES in features and name not in registry: resolved, note = resolve_name(name, registry, output_names=True) if resolved is not None: v2["notes"].append(f"explain: {note}") @@ -981,15 +1198,154 @@ def fail() -> tuple[None, dict]: elif name not in explain: explain.append(name) + detail = "full" + if CONCISE in features: + detail = args.get("detail") or "brief" + if detail not in ("brief", "full"): + problems.append('detail must be "brief" or "full"') + if problems: return fail() request = {"situation": normalized, "variables": requested, "period": period} if v2 is not None: request["explain"] = explain request["notes"] = v2["notes"] + request["detail"] = detail return request, None +FLAT_ROLES = ("head", "spouse", "dependent") + + +def flat_situation(household, registry, problems: list[str], v2: dict | None): + """The flat feature's household -- state, people with role and facts, and + household facts -- as a situation, with the units the benchmark's own + reference situations have (policybench.scenarios.Scenario.to_pe_household): + the head and spouse one marital unit and everyone else alone, and one tax + unit, SPM unit, family and household holding everyone. A household fact + goes to the unit of its variable's entity. Returns None after recording + problems.""" + if isinstance(household, str): + if len(household) > MAX_SITUATION_CHARS: + problems.append("household is too large") + return None + try: + household = json.loads(household) + except (ValueError, RecursionError) as error: + problems.append(f"household is not valid JSON: {error}"[:MAX_MESSAGE_CHARS]) + return None + if not isinstance(household, dict): + problems.append( + "household must be a JSON object with state, people and household_facts" + ) + return None + extra = sorted(set(household) - {"state", "people", "household_facts"}) + if extra: + problems.append( + "household accepts state, people and household_facts; remove " + + ", ".join(map(str, extra)) + ) + people = household.get("people") + if not isinstance(people, list) or not people: + problems.append("household.people must be a non-empty list of people") + return None + if len(people) > MAX_PEOPLE: + problems.append(f"at most {MAX_PEOPLE} people") + return None + situation: dict[str, Any] = {"people": {}} + roles: dict[str, str] = {} + for index, person in enumerate(people): + if not isinstance(person, dict): + problems.append( + f"people[{index}] must be an object with name, role and facts" + ) + continue + name, role, facts = ( + person.get("name"), + person.get("role"), + person.get("facts", {}), + ) + if not isinstance(name, str) or not name or len(name) > 64: + problems.append(f"people[{index}].name must be a short string") + continue + if name in roles: + problems.append(f"people[{index}]: {name!r} is listed twice") + continue + if role not in FLAT_ROLES: + problems.append( + f"people[{index}].role must be one of {', '.join(FLAT_ROLES)}" + ) + continue + if not isinstance(facts, dict): + problems.append(f"people[{index}].facts must be an object") + continue + roles[name] = role + situation["people"][name] = dict(facts) + heads = [n for n, r in roles.items() if r == "head"] + spouses = [n for n, r in roles.items() if r == "spouse"] + if len(heads) != 1: + problems.append("the household needs exactly one head") + if len(spouses) > 1: + problems.append("the household has at most one spouse") + if problems: + return None + for name, role in roles.items(): + facts = situation["people"][name] + facts.setdefault("is_tax_unit_head", role == "head") + facts.setdefault("is_tax_unit_spouse", role == "spouse") + everyone = list(situation["people"]) + couple = heads + spouses + groups = [couple] + [[n] for n in everyone if n not in couple] + situation["marital_units"] = { + f"marital_unit_{i}": {"members": members} + for i, members in enumerate(groups, start=1) + } + situation["tax_units"] = {"tax_unit": {"members": list(everyone)}} + situation["spm_units"] = {"spm_unit": {"members": list(everyone)}} + situation["families"] = {"family": {"members": list(everyone)}} + situation["households"] = {"household": {"members": list(everyone)}} + state = household.get("state") + if state is not None: + situation["households"]["household"]["state_code"] = state + unit_of = { + "tax_unit": situation["tax_units"]["tax_unit"], + "spm_unit": situation["spm_units"]["spm_unit"], + "family": situation["families"]["family"], + "household": situation["households"]["household"], + "marital_unit": situation["marital_units"]["marital_unit_1"], + } + facts = household.get("household_facts") or {} + if not isinstance(facts, dict): + problems.append("household_facts must be an object") + return None + for name, value in facts.items(): + resolved = name + if ( + isinstance(name, str) + and name not in registry + and v2 is not None + and NAMES in v2["features"] + ): + resolved, note = resolve_name(name, registry, output_names=False) + if resolved is not None: + v2["notes"].append(f"household_facts: {note}") + if not isinstance(resolved, str) or resolved not in registry: + problems.append( + f"household_facts: unknown variable {name!r}." + + (_suggest(name, registry) if isinstance(name, str) else "") + ) + continue + entity = registry[resolved].entity + if entity == "person": + problems.append( + f"household_facts: {resolved} is a person's fact; give it under " + "that person" + ) + continue + unit_of[entity][resolved] = value + return None if problems else situation + + DESIRED_SUFFIX = "_desired" _LABEL_SEPARATORS = re.compile(r"[\s\-]+") @@ -1152,12 +1508,40 @@ def execute_request(system, registry, request: dict, simulation_cls=None) -> dic out["notes"] = list(request["notes"]) if request.get("explain"): out["explanations"] = { - name: explain_variable(system, request, name, simulation_cls) + name: ( + brief_explanation( + explain_variable(system, request, name, simulation_cls) + ) + if request.get("detail") == "brief" + else explain_variable(system, request, name, simulation_cls) + ) for name in request["explain"] } return out +BRIEF_INPUTS = 8 +BRIEF_PARAMETERS = 5 + + +def brief_explanation(full: dict) -> dict: + """The concise feature's explanation: the value, the variables it came + from directly (one level down, zero-valued ones left out), and the first + parameters it read with their sources.""" + if "error" in full: + return full + steps = full["steps"] + direct = [ + s.strip() for s in steps[1:] if s.startswith(" ") and not s.startswith(" ") + ] + return { + "value": steps[0].strip() if steps else "", + "from": direct[:BRIEF_INPUTS], + "parameters": full["parameters"][:BRIEF_PARAMETERS], + "more": 'set detail to "full" for every step', + } + + EXPLAIN_MAX_LINES = 60 EXPLAIN_MAX_DEPTH = 6 EXPLAIN_MAX_PARAMETERS = 15 @@ -1356,10 +1740,13 @@ def _worker_main(conn, fix_path: str, reform: bool, board: str | None = None) -> return if args is None: return - version = 1 - if isinstance(args, dict) and args.get("__tool_version__") == 2: - version, args = 2, args.get("arguments") - request, error = normalize_request(args, registry, version) + features = frozenset() + if isinstance(args, dict) and "__tool_features__" in args: + features = frozenset(args.get("__tool_features__") or ()) + args = args.get("arguments") + elif isinstance(args, dict) and args.get("__tool_version__") == 2: + features, args = V2_FEATURES, args.get("arguments") + request, error = normalize_request(args, registry, features=features) if error is not None: conn.send(("ok", error)) continue @@ -1379,9 +1766,15 @@ def __init__( acquire_timeout: float | None = None, board: Path | None = None, tool_version: int = 1, + tool_features: frozenset | None = None, ): self._ctx = mp.get_context("spawn") self._tool_version = tool_version + self._tool_features = ( + tool_features + if tool_features is not None + else (V2_FEATURES if tool_version == 2 else frozenset()) + ) self._fix_path = str(fix_path) # A worker gets the board only when it is not the frozen one, so a # three-argument worker target still starts on the default. @@ -1448,8 +1841,13 @@ def call(self, args) -> tuple[dict, float]: time.perf_counter() - started, ) process, conn = worker - if self._tool_version == 2: + if self._tool_features == V2_FEATURES: args = {"__tool_version__": 2, "arguments": args} + elif self._tool_features: + args = { + "__tool_features__": sorted(self._tool_features), + "arguments": args, + } try: conn.send(args) if conn.poll(self.timeout): @@ -1607,6 +2005,8 @@ def notice(self) -> str: notice = WEB_TOOL_NOTICE if tool else JSON_WEB_TOOL_NOTICE elif self.arm == "pe2": return engine_text(TOOL_NOTICE_V2 if tool else JSON_TOOL_NOTICE_V2) + elif self.arm != "pe" and arm_features(self.arm) is not None: + return engine_text(compose_tool_notice(arm_features(self.arm), tool)) else: return engine_text(TOOL_NOTICE if tool else JSON_TOOL_NOTICE) return engine_text(notice) + ( @@ -1615,11 +2015,15 @@ def notice(self) -> str: @property def uses_calculator(self) -> bool: - return self.arm in TOOL_VERSION_BY_ARM + return arm_features(self.arm) is not None + + @property + def tool_features(self) -> frozenset: + return arm_features(self.arm) or frozenset() @property def tool_version(self) -> int: - return TOOL_VERSION_BY_ARM.get(self.arm, 1) + return 2 if self.tool_features == V2_FEATURES else 1 @property def uses_web(self) -> bool: @@ -1637,7 +2041,7 @@ def arm1_label(self) -> str: def arm2_label(self) -> str: version = "" if self.prompt_version == "v1" else f"@{self.prompt_version}" rule = "+no-benchmark-search" if self.benchmark_search_rule else "" - return f"{self.name}{ARM_SUFFIX[self.arm]}{version}{rule}" + return f"{self.name}{arm_suffix(self.arm)}{version}{rule}" def provider_family(model_id: str) -> str: @@ -1663,8 +2067,11 @@ def model_profile( _uses_responses_api, ) - if arm not in ARMS: - raise SystemExit(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}") + if arm not in ARMS and arm_features(arm) is None: + raise SystemExit( + f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}, or pe: plus " + f"calculator features joined by + ({', '.join(TOOL_FEATURES)})" + ) if prompt_version not in PROMPT_VERSIONS: raise SystemExit( f"unknown prompt {prompt_version!r}; the prompts are " @@ -2298,7 +2705,7 @@ def run_household( sent_board_prompt_sha256 = base.pop("_sent_board_prompt_sha256") ceiling = base.pop("_budget_ceiling") answer_tool = base.pop("_answer_tool") - calculate_tool = calculate_tool_schema(profile.tool_version) + calculate_tool = calculate_tool_schema(features=profile.tool_features) offered = ([web_search_tool(profile)] if profile.uses_web else []) + ( [calculate_tool] if profile.uses_calculator else [] ) @@ -3133,7 +3540,10 @@ def command_run(args) -> None: "api": profile.api, "answer_contract": profile.contract, "tool_notice": profile.notice, - "calculate_tool": calculate_tool_schema(profile.tool_version) + "calculate_tool": calculate_tool_schema(features=profile.tool_features) + if profile.uses_calculator + else None, + "tool_features": sorted(profile.tool_features) if profile.uses_calculator else None, "tool_version": profile.tool_version if profile.uses_calculator else None, @@ -3164,7 +3574,12 @@ def command_run(args) -> None: return # Arms without the calculator start no engine workers. pool = ( - ToolPool(size=args.workers, board=snapshot, tool_version=profile.tool_version) + ToolPool( + size=args.workers, + board=snapshot, + tool_version=profile.tool_version, + tool_features=profile.tool_features, + ) if profile.uses_calculator else None ) @@ -3879,6 +4294,12 @@ def _successful_calls(record: dict) -> list[dict]: def _person_names(arguments) -> list[str]: try: args = json.loads(arguments) if isinstance(arguments, str) else arguments + if "household" in args: # the flat feature's people list + household = args["household"] + household = ( + json.loads(household) if isinstance(household, str) else household + ) + return [person["name"] for person in household["people"]] situation = args["situation"] situation = json.loads(situation) if isinstance(situation, str) else situation return list(situation["people"]) @@ -4486,7 +4907,7 @@ def main_call(record: dict, scenario, outputs: list[str]) -> dict | None: def situation_fidelity( - records, board, system, registry, defaults, out: Path, version: int = 1 + records, board, system, registry, defaults, out: Path, features=frozenset() ) -> None: """How each household's main situation differs from the reference's. @@ -4508,7 +4929,7 @@ def situation_fidelity( if call is None: rows.append({"scenario_id": sid, "differences": None}) continue - request, _ = normalize_request(call["arguments"], registry, version) + request, _ = normalize_request(call["arguments"], registry, features=features) if request is None: rows.append({"scenario_id": sid, "differences": None}) continue @@ -4561,7 +4982,7 @@ def command_diagnose(args) -> None: out = Path(args.out_dir) records = load_records(out) # A run's calls are read as its own tool version read them. - version = TOOL_VERSION_BY_ARM.get(records_arm(records), 1) + features = arm_features(records_arm(records)) or frozenset() rows_path = out / "scored_rows.csv" if not rows_path.exists(): raise SystemExit("run score first") @@ -4577,7 +4998,7 @@ def command_diagnose(args) -> None: name: system.variables[name].default_value for name in system.variables } if registry is not None: - situation_fidelity(records, board, system, registry, defaults, out, version) + situation_fidelity(records, board, system, registry, defaults, out, features) out_rows = [] if misses.empty: print("no arm-2 misses") @@ -4595,7 +5016,7 @@ def command_diagnose(args) -> None: if registry is not None: def board_value_for(arguments, scenario=scenario, output=row.variable): - request, _ = normalize_request(arguments, registry, version) + request, _ = normalize_request(arguments, registry, features=features) if request is None: return None return output_value(system, request["situation"], scenario, output) @@ -4619,7 +5040,7 @@ def board_value_for(arguments, scenario=scenario, output=row.variable): attribution = {} arguments = verdict.get("call_arguments") if arguments is not None and registry is not None: - request, _ = normalize_request(arguments, registry, version) + request, _ = normalize_request(arguments, registry, features=features) if request is not None: reference_situation = scenario.to_pe_household() diff = situation_diff( @@ -4716,9 +5137,10 @@ def main(argv: list[str] | None = None) -> None: ) p.add_argument( "--arm", - choices=ARMS, default="pe", - help="the tools: none, PolicyEngine (pe), web search (web), or both", + help="the tools: none, PolicyEngine (pe, pe2, pe3, or pe: plus " + "calculator features joined by +), web search (web), or both " + "(web+pe)", ) p.add_argument( "--prompt", diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index 986a89d9..4e586374 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -2294,6 +2294,7 @@ def test_web_notices_name_search_and_keep_the_answer_instruction(): "web": "gpt-6-sol+web", "web+pe": "gpt-6-sol+web+pe-tool", "pe2": "gpt-6-sol+pe-tool-v2", + "pe3": "gpt-6-sol+pe-tool-v3", } with pytest.raises(SystemExit): pilot.model_profile(SOL, "search") diff --git a/tests/test_tool_v3.py b/tests/test_tool_v3.py new file mode 100644 index 00000000..5c018fd1 --- /dev/null +++ b/tests/test_tool_v3.py @@ -0,0 +1,377 @@ +"""The calculator's features one at a time (tool v3 and the ablation arms). + +Each feature (names, placement, overrides, explain, concise, flat) must work +on its own and change nothing else, so a study can turn one on at a time. The +version 2 feature set must reproduce tool v2's notice byte for byte, and the +flat household format must build the units the benchmark's own reference +situations have. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import tool_pilot_argon as pilot # noqa: E402 + +V = pilot.VariableInfo +REGISTRY = { + "age": V("person", "int", "age", "year"), + "employment_income": V("person", "float", "employment income", "year"), + "is_tax_unit_head": V("person", "bool", "head", "year"), + "is_tax_unit_spouse": V("person", "bool", "spouse", "year"), + "traditional_401k_contributions": V( + "person", "float", "401(k)", "year", computed=True + ), + "traditional_401k_contributions_desired": V("person", "float", "asked", "year"), + "snap": V("spm_unit", "float", "SNAP", "month", computed=True), + "pre_subsidy_rent": V("spm_unit", "float", "rent", "year"), + "qualified_passenger_vehicle_loan_interest": V( + "household", "float", "car loan", "year" + ), + "state_code": V("household", "Enum", "state", "year", ("CA", "LA")), + "tax_unit_childcare_expenses": V("tax_unit", "float", "childcare", "year"), + "marital_unit_flag": V("marital_unit", "bool", "flag", "year"), +} +F = pilot + + +def _situation(): + return { + "people": {"head": {"age": 40}, "kid": {"age": 8}}, + "spm_units": {"spm": {"members": ["head", "kid"]}}, + "households": {"h": {"members": ["head", "kid"], "state_code": "CA"}}, + } + + +def _norm(args, *features): + return pilot.normalize_request(args, REGISTRY, features=frozenset(features)) + + +def test_the_version_2_set_reproduces_tool_v2s_notice(): + assert F.compose_tool_notice(F.V2_FEATURES, True) == F.TOOL_NOTICE_V2 + assert F.compose_tool_notice(F.V2_FEATURES, False) == F.JSON_TOOL_NOTICE_V2 + assert F.compose_tool_notice(frozenset(), True) == F.TOOL_NOTICE + assert F.compose_tool_notice(frozenset(), False) == F.JSON_TOOL_NOTICE + + +def test_each_feature_adds_its_own_sentence_only(): + base = F.compose_tool_notice(frozenset({F.EXPLAIN}), True) + assert "`explain`" in base and "names it read" not in base + assert "written with spaces" not in base + names = F.compose_tool_notice(frozenset({F.NAMES}), True) + assert "written with spaces" in names and "`explain`" not in names + flat = F.compose_tool_notice(frozenset({F.FLAT}), True) + assert "household you describe" in flat and "`household`" in flat + concise = F.compose_tool_notice(frozenset({F.EXPLAIN, F.CONCISE}), True) + assert '`detail` set to "full"' in concise + + +def test_arms_name_their_features(): + assert F.arm_features("pe") == frozenset() + assert F.arm_features("pe2") == F.V2_FEATURES + assert F.arm_features("pe3") == frozenset(F.TOOL_FEATURES) + assert F.arm_features("pe:flat+names") == frozenset({"flat", "names"}) + assert F.arm_features("none") is None and F.arm_features("web") is None + assert F.arm_suffix("pe:flat+names") == "+pe-tool[names+flat]" + with pytest.raises(SystemExit, match="unknown tool features"): + F.arm_features("pe:names+telepathy") + with pytest.raises(SystemExit, match="concise feature needs explain"): + F.arm_features("pe:concise") + profile = F.ModelProfile("m", "gpt-x", "chat", "tool", arm="pe:names") + assert profile.uses_calculator and profile.tool_features == frozenset({"names"}) + assert profile.tool_version == 1 + assert "written with spaces" in profile.notice + assert profile.arm2_label == "m+pe-tool[names]" + + +def test_names_alone_reads_labels_but_places_and_redirects_nothing(): + situation = _situation() + situation["people"]["head"]["Employment income"] = 30000 + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.NAMES) + assert error is None + assert request["situation"]["people"]["head"]["employment_income"] == { + "2026": 30000 + } + misplaced = _situation() + misplaced["people"]["head"]["qualified_passenger_vehicle_loan_interest"] = 5 + request, error = _norm({"situation": misplaced, "variables": ["snap"]}, F.NAMES) + assert request is None and any( + "set it under households" in p for p in error["problems"] + ) + realized = _situation() + realized["people"]["head"]["traditional_401k_contributions"] = 100 + request, error = _norm({"situation": realized, "variables": ["snap"]}, F.NAMES) + assert "traditional_401k_contributions" in request["situation"]["people"]["head"] + request, error = _norm( + {"situation": _situation(), "variables": ["snap"], "explain": ["snap"]}, F.NAMES + ) + assert request is None and any( + "unknown arguments: explain" in p for p in error["problems"] + ) + + +def test_placement_alone_moves_inputs_but_reads_no_labels(): + situation = _situation() + situation["people"]["head"]["qualified_passenger_vehicle_loan_interest"] = 5 + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.PLACEMENT) + assert error is None + assert request["situation"]["households"]["h"][ + "qualified_passenger_vehicle_loan_interest" + ] == {"2026": 5} + labeled = _situation() + labeled["people"]["head"]["employment income"] = 1 + request, error = _norm({"situation": labeled, "variables": ["snap"]}, F.PLACEMENT) + assert request is None + + +def test_overrides_alone_redirects_and_flags(): + situation = _situation() + situation["people"]["head"]["traditional_401k_contributions"] = 100 + situation["spm_units"]["spm"]["snap"] = 5 + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.OVERRIDES) + assert error is None + head = request["situation"]["people"]["head"] + assert head == { + "age": {"2026": 40}, + "traditional_401k_contributions_desired": {"2026": 100}, + } + assert any("snap is computed" in n for n in request["notes"]) + + +def test_concise_takes_a_detail_and_defaults_to_brief(): + request, error = _norm( + {"situation": _situation(), "variables": ["snap"], "explain": ["snap"]}, + F.EXPLAIN, + F.CONCISE, + ) + assert error is None and request["detail"] == "brief" + request, _ = _norm( + { + "situation": _situation(), + "variables": ["snap"], + "explain": ["snap"], + "detail": "full", + }, + F.EXPLAIN, + F.CONCISE, + ) + assert request["detail"] == "full" + request, error = _norm( + {"situation": _situation(), "variables": ["snap"], "detail": "short"}, + F.EXPLAIN, + F.CONCISE, + ) + assert request is None + request, _ = _norm( + {"situation": _situation(), "variables": ["snap"], "explain": ["snap"]}, + F.EXPLAIN, + ) + assert request["detail"] == "full" + + +def test_a_brief_explanation_keeps_the_value_its_direct_inputs_and_main_parameters(): + full = { + "steps": [ + "snap (2026) = 288", + " snap_normal_allotment (2026) = 288", + " snap_net_income (2026) = 1700", + " is_snap_eligible (2026) = True", + ], + "parameters": [{"name": f"p{i}", "value": i, "sources": []} for i in range(9)], + } + brief = F.brief_explanation(full) + assert brief["value"] == "snap (2026) = 288" + assert brief["from"] == [ + "snap_normal_allotment (2026) = 288", + "is_snap_eligible (2026) = True", + ] + assert [p["name"] for p in brief["parameters"]] == ["p0", "p1", "p2", "p3", "p4"] + assert F.brief_explanation({"error": "x"}) == {"error": "x"} + + +def _flat(**overrides): + household = { + "state": "LA", + "people": [ + { + "name": "Head", + "role": "head", + "facts": {"age": 40, "employment_income": 30000}, + }, + {"name": "Spouse", "role": "spouse", "facts": {"age": 38}}, + {"name": "Kid", "role": "dependent", "facts": {"age": 6}}, + ], + "household_facts": { + "pre_subsidy_rent": 9000, + "tax_unit_childcare_expenses": 1200, + "qualified_passenger_vehicle_loan_interest": 300, + }, + } + household.update(overrides) + return {"household": json.dumps(household), "variables": ["snap"]} + + +def test_flat_builds_the_benchmarks_units(): + request, error = _norm(_flat(), F.FLAT) + assert error is None, error + s = request["situation"] + assert s["marital_units"] == { + "marital_unit_1": {"members": ["Head", "Spouse"]}, + "marital_unit_2": {"members": ["Kid"]}, + } + for group, unit in ( + ("tax_units", "tax_unit"), + ("spm_units", "spm_unit"), + ("families", "family"), + ): + assert s[group][unit]["members"] == ["Head", "Spouse", "Kid"] + assert s["households"]["household"]["state_code"] == {"2026": "LA"} + assert s["spm_units"]["spm_unit"]["pre_subsidy_rent"] == {"2026": 9000} + assert s["tax_units"]["tax_unit"]["tax_unit_childcare_expenses"] == {"2026": 1200} + assert s["households"]["household"][ + "qualified_passenger_vehicle_loan_interest" + ] == {"2026": 300} + assert s["people"]["Head"]["is_tax_unit_head"] == {"2026": True} + assert s["people"]["Spouse"]["is_tax_unit_spouse"] == {"2026": True} + assert s["people"]["Kid"]["is_tax_unit_head"] == {"2026": False} + + +def test_flat_refuses_what_it_cannot_place(): + people = [ + {"name": "A", "role": "head", "facts": {}}, + {"name": "B", "role": "head", "facts": {}}, + ] + request, error = _norm(_flat(people=people), F.FLAT) + assert any("exactly one head" in p for p in error["problems"]) + request, error = _norm(_flat(household_facts={"age": 3}), F.FLAT) + assert any("person's fact" in p for p in error["problems"]) + people = [{"name": "A", "role": "boss", "facts": {}}] + request, error = _norm(_flat(people=people), F.FLAT) + assert any("role must be one of" in p for p in error["problems"]) + request, error = _norm({"situation": _situation(), "variables": ["snap"]}, F.FLAT) + assert any("unknown arguments: situation" in p for p in error["problems"]) + request, error = _norm(_flat(), F.NAMES) + assert any("unknown arguments: household" in p for p in error["problems"]) + + +def test_flat_with_names_reads_labeled_household_facts(): + request, error = _norm( + _flat(household_facts={"pre subsidy rent": 9000}), F.FLAT, F.NAMES + ) + assert error is None + assert request["situation"]["spm_units"]["spm_unit"]["pre_subsidy_rent"] == { + "2026": 9000 + } + assert any("as pre_subsidy_rent" in n for n in request["notes"]) + + +def test_the_schema_offers_each_features_arguments(): + flat = F.calculate_tool_schema(features=frozenset({F.FLAT}))["function"] + props = flat["parameters"]["properties"] + assert "household" in props and "situation" not in props + assert flat["parameters"]["required"] == ["household", "variables"] + concise = F.calculate_tool_schema(features=frozenset({F.EXPLAIN, F.CONCISE})) + props = concise["function"]["parameters"]["properties"] + assert "explain" in props and props["detail"]["enum"] == ["brief", "full"] + assert F.calculate_tool_schema(features=F.V2_FEATURES) == F.calculate_tool_schema(2) + assert F.calculate_tool_schema(features=frozenset()) == F.calculate_tool_schema(1) + + +def _echo_worker(conn, fix_path, reform): + conn.send(("ready", None)) + while True: + args = conn.recv() + if args is None: + return + conn.send(("ok", {"received": args})) + + +def test_the_pool_tells_its_workers_the_features(): + pool = F.ToolPool( + size=1, + timeout=5.0, + worker_target=_echo_worker, + tool_features=frozenset({F.FLAT, F.NAMES}), + ) + try: + result, _ = pool.call({"x": 1}) + assert result == { + "received": {"__tool_features__": ["flat", "names"], "arguments": {"x": 1}} + } + finally: + pool.close() + + +@pytest.mark.slow +def test_a_household_described_flat_computes_what_the_benchmarks_own_does(): + """Differential: on the installed engine, each frozen-board household + described in the flat format computes every board output exactly as the + benchmark's own situation (Scenario.to_pe_household) does.""" + import household_set as hs + import pandas as pd + from policyengine_us import Simulation + + from policybench.ground_truth import _extract_person_value, _pe_variable_for_output + + system = F.build_tool_system(reform=True) + registry = F.build_registry(system) + reference = pd.read_csv(hs.FROZEN_BOARD / "reference_outputs.csv") + variables = reference.groupby("scenario_id").variable.agg(list) + relations = {"is_tax_unit_head", "is_tax_unit_spouse"} + checked = 0 + for scenario in hs.load_scenarios(hs.FROZEN_BOARD)[:8]: + situation = hs.situation_for(scenario) + couple = scenario.marital_couple() or () + head = couple[0] if couple else scenario.adults[0].name + people = [ + { + "name": name, + "role": "head" + if name == head + else ("spouse" if name in couple else "dependent"), + "facts": {k: v for k, v in facts.items() if k not in relations}, + } + for name, facts in situation["people"].items() + ] + household_facts = {} + for group in ("tax_units", "spm_units", "families", "households"): + for unit in situation[group].values(): + household_facts.update( + { + k: v + for k, v in unit.items() + if k not in ("members", "state_code") + } + ) + household = { + "state": scenario.state, + "people": people, + "household_facts": household_facts, + } + request, error = F.normalize_request( + {"household": json.dumps(household), "variables": ["snap"]}, + registry, + features=frozenset({F.FLAT}), + ) + assert error is None, (scenario.id, error) + own = Simulation(tax_benefit_system=system, situation=situation) + flat = Simulation(tax_benefit_system=system, situation=request["situation"]) + for variable in variables[scenario.id]: + name = _pe_variable_for_output(variable, "us") + values = [ + float( + _extract_person_value( + simulation.calculate(name, 2026), scenario, variable + ) + ) + for simulation in (own, flat) + ] + assert values[0] == values[1], (scenario.id, variable, values) + checked += 1 + assert checked > 80 From b3ec406e62b19d59f285150ddc8a6207a2cf0bf0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 17:25:47 -0400 Subject: [PATCH 09/64] Read facts as the prompt labels them (labels feature) The prompts label some facts in words rather than as the variable name with spaces: employment_income is "gross wages and salaries", hours_worked_last_week "usual weekly hours worked", has_esi "has employer-sponsored insurance" (policybench.prompts.INPUT_LABEL_OVERRIDES). In the 20-household test set nearly every household has at least one; hours appear in 17, employer coverage and premiums in 11 each. Tool version 2's names feature cannot read these, and its notice says the facts carry the variable names. A Flash-Lite smoke test of the flat format met three such errors and fell back to setting employer_sponsored_insurance_premiums to 0 and weekly_hours_worked, which the SNAP work rules do not read. The new labels feature reads a fact under its prompt label (case and punctuation aside, renames applied) and, as the reference situations do, also sets stated usual weekly hours as weekly_hours_worked_before_lsr (PE_INPUT_ALIASES). pe3 now offers all seven features. pe2 is unchanged and keeps its notice as it ran; every other feature set's notice says only what the tool reads. Tests: label reading, the alias (and leaving a set value alone), pe2 unchanged, the variable-name claim only in pe2's notice across every feature set, every prompt label reading back to its engine name, and a differential: twelve frozen-board households described flat with each fact as its prompt writes it compute every board output as the benchmark does. Three mutants (no alias, no renames, strict label matching) are caught. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 130 +++++++++++++++++++++------ tests/test_tool_v3.py | 175 ++++++++++++++++++++++++++++++++++++ 2 files changed, 279 insertions(+), 26 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 88d8c4d6..58ec78bb 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -51,6 +51,7 @@ import copy import decimal import difflib +import functools import hashlib import importlib.metadata import importlib.util @@ -166,6 +167,12 @@ # publishable"; "do all these tests with a cheap model"): # - names: a name written with spaces, or a benchmark output key, is read as # its engine variable; +# - labels: a fact written under the label the prompt gives it (a prompt +# labels some facts in words, e.g. employment_income as "gross wages and +# salaries"; policybench.prompts.INPUT_LABEL_OVERRIDES) is read as its +# variable, and a fact the reference also sets under another name +# (policybench.scenarios.PE_INPUT_ALIASES: usual weekly hours) is set there +# too; # - placement: an input set on another entity moves to the one unit holding # its people; # - overrides: a realized contribution is read as its _desired input, and an @@ -174,18 +181,20 @@ # - concise: an explanation is brief unless `detail` is "full"; # - flat: the household is described as people with roles and facts # (`household`), and the tool builds its units as the benchmark does. -# "pe2" is tool version 2 (the first four); "pe3" is all six. Any other set +# "pe2" is tool version 2 (names, placement, overrides and explain); "pe3" is +# all seven. Any other set # runs as an arm named "pe:" plus its features joined by "+", e.g. # "pe:names+flat". -NAMES, PLACEMENT, OVERRIDES, EXPLAIN, CONCISE, FLAT = ( +NAMES, LABELS, PLACEMENT, OVERRIDES, EXPLAIN, CONCISE, FLAT = ( "names", + "labels", "placement", "overrides", "explain", "concise", "flat", ) -TOOL_FEATURES = (NAMES, PLACEMENT, OVERRIDES, EXPLAIN, CONCISE, FLAT) +TOOL_FEATURES = (NAMES, LABELS, PLACEMENT, OVERRIDES, EXPLAIN, CONCISE, FLAT) V2_FEATURES = frozenset({NAMES, PLACEMENT, OVERRIDES, EXPLAIN}) FEATURES_BY_ARM = { "pe": frozenset(), @@ -251,10 +260,14 @@ def _listing(items: list[str]) -> str: def compose_tool_notice(features: frozenset, tool_contract: bool) -> str: """The calculator notice for a set of features: the opening, a sentence per feature, then the turn limit and the answer instruction. No feature - gives TOOL_NOTICE and the version 2 features give TOOL_NOTICE_V2, byte for - byte.""" + gives TOOL_NOTICE and the version 2 features TOOL_NOTICE_V2, as those arms + ran. TOOL_NOTICE_V2 says the facts are labeled with the engine's variable + names, which holds for most facts but not those the prompt labels in words + ("gross wages and salaries"); every other set says only what it reads.""" if not features: return TOOL_NOTICE if tool_contract else JSON_TOOL_NOTICE + if features == V2_FEATURES: + return TOOL_NOTICE_V2 if tool_contract else JSON_TOOL_NOTICE_V2 what = ( "household you describe" if FLAT in features @@ -271,14 +284,24 @@ def compose_tool_notice(features: frozenset, tool_contract: bool) -> str: "or dependent) and facts, and the facts about the household as a " "whole; it builds the household's units as the facts imply." ) - if NAMES in features: + if LABELS in features and NAMES in features: + text += ( + " It accepts the facts above under the labels they have there, its " + "variable names written with spaces, and the output keys you are asked " + "for. Set every listed fact." + ) + elif LABELS in features: + text += ( + " It accepts the facts above under the labels they have there. Set " + "every listed fact." + ) + elif NAMES in features: text += ( - " The facts above are labeled with its variable names, written with " - "spaces; it accepts them as written, and it accepts the output keys you " - "are asked for. Set every listed fact." + " It accepts its variable names written with spaces, and the output " + "keys you are asked for. Set every listed fact." ) notes = [] - if NAMES in features: + if NAMES in features or LABELS in features: notes.append("which names it read") if PLACEMENT in features: notes.append("which inputs it moved to the entity they belong to") @@ -670,10 +693,15 @@ def calculate_tool_schema(version: int = 1, features: frozenset | None = None) - FLAT_DESCRIPTION if FLAT in features else CALCULATE_DESCRIPTION ) notes = [] + if LABELS in features: + notes.append( + "A fact may be written under the label it has in the facts " + "you were given." + ) if NAMES in features: notes.append( - "A name may also be written with spaces (as the facts are " - "labeled) or be one of the output keys you are asked for." + "A name may also be written with spaces or be one of the " + "output keys you are asked for." ) if PLACEMENT in features and FLAT not in features: notes.append( @@ -927,13 +955,8 @@ def _normalize_inputs( for name, value in data.items(): if allow_members and name == "members": continue - if ( - v2 is not None - and NAMES in v2["features"] - and isinstance(name, str) - and name not in registry - ): - resolved, note = resolve_name(name, registry, output_names=False) + if v2 is not None and isinstance(name, str) and name not in registry: + resolved, note = read_fact_name(name, registry, v2["features"]) if resolved is not None: v2["notes"].append(f"{where}: {note}") name = resolved @@ -1157,6 +1180,8 @@ def fail() -> tuple[None, dict]: problems.append(f"at most {MAX_VARIABLES} variables per call") if v2 is not None and PLACEMENT in features: _place_misplaced(v2, normalized, situation, registry, period, problems) + if v2 is not None and LABELS in features: + _set_aliases(v2, normalized["people"], registry) requested = [] for name in variables: if ( @@ -1320,13 +1345,8 @@ def flat_situation(household, registry, problems: list[str], v2: dict | None): return None for name, value in facts.items(): resolved = name - if ( - isinstance(name, str) - and name not in registry - and v2 is not None - and NAMES in v2["features"] - ): - resolved, note = resolve_name(name, registry, output_names=False) + if isinstance(name, str) and name not in registry and v2 is not None: + resolved, note = read_fact_name(name, registry, v2["features"]) if resolved is not None: v2["notes"].append(f"household_facts: {note}") if not isinstance(resolved, str) or resolved not in registry: @@ -1375,6 +1395,64 @@ def output_reading(name: str) -> tuple[str, str] | None: return engine, how +def _label_key(text: str) -> str: + """A label as compared: lower case, letters and digits only, so "Roth + 401(k) contributions" and "roth 401k contributions" read alike.""" + return re.sub(r"[^0-9a-z]", "", text.casefold()) + + +@functools.lru_cache(maxsize=1) +def prompt_labels() -> dict[str, str]: + """The labels the prompts give facts in words, keyed by _label_key, as the + engine names the facts (policybench.scenarios.PE_INPUT_RENAMES applied). A + label two facts share is left out.""" + from policybench.prompts import INPUT_LABEL_OVERRIDES + from policybench.scenarios import PE_INPUT_RENAMES + + labels: dict[str, str] = {} + shared = set() + for source, label in INPUT_LABEL_OVERRIDES.items(): + key = _label_key(label) + name = PE_INPUT_RENAMES.get(source, source) + if key in labels and labels[key] != name: + shared.add(key) + labels[key] = name + for source, name in PE_INPUT_RENAMES.items(): + labels.setdefault(_label_key(source), name) + return {key: name for key, name in labels.items() if key not in shared} + + +def read_fact_name( + name: str, registry: Mapping[str, VariableInfo], features: frozenset +) -> tuple[str | None, str | None]: + """A fact's engine variable under the labels feature (the label the prompt + gives it) and then the names feature (spaces for underscores). Returns + (engine name, note) or (None, None).""" + if LABELS in features: + labeled = prompt_labels().get(_label_key(name)) + if labeled is not None and labeled in registry: + return labeled, f"read {name!r} as {labeled}" + if NAMES in features: + return resolve_name(name, registry, output_names=False) + return None, None + + +def _set_aliases(v2: dict, people: dict, registry: Mapping[str, VariableInfo]): + """Set each fact the reference also sets under another name there too + (policybench.scenarios.PE_INPUT_ALIASES: the SNAP work rules read stated + usual weekly hours as weekly_hours_worked_before_lsr), unless it is set.""" + from policybench.scenarios import PE_INPUT_ALIASES + + for person, inputs in people.items(): + for source, target in PE_INPUT_ALIASES.items(): + if source in inputs and target in registry and target not in inputs: + inputs[target] = dict(inputs[source]) + v2["notes"].append( + f"people.{person}: also set {target} from {source}, as the " + "benchmark's reference does" + ) + + def resolve_name( name: str, registry: Mapping[str, VariableInfo], output_names: bool ) -> tuple[str | None, str | None]: diff --git a/tests/test_tool_v3.py b/tests/test_tool_v3.py index 5c018fd1..c3a5eb36 100644 --- a/tests/test_tool_v3.py +++ b/tests/test_tool_v3.py @@ -9,6 +9,7 @@ from __future__ import annotations +import itertools import json import sys from pathlib import Path @@ -37,6 +38,10 @@ "state_code": V("household", "Enum", "state", "year", ("CA", "LA")), "tax_unit_childcare_expenses": V("tax_unit", "float", "childcare", "year"), "marital_unit_flag": V("marital_unit", "bool", "flag", "year"), + "has_esi": V("person", "bool", "ESI", "year"), + "hours_worked_last_week": V("person", "float", "hours", "year"), + "weekly_hours_worked_before_lsr": V("person", "float", "hours", "year"), + "partnership_self_employment_net_earnings": V("person", "float", "p", "year"), } F = pilot @@ -76,6 +81,7 @@ def test_arms_name_their_features(): assert F.arm_features("pe") == frozenset() assert F.arm_features("pe2") == F.V2_FEATURES assert F.arm_features("pe3") == frozenset(F.TOOL_FEATURES) + assert len(F.TOOL_FEATURES) == 7 and F.LABELS in F.arm_features("pe3") assert F.arm_features("pe:flat+names") == frozenset({"flat", "names"}) assert F.arm_features("none") is None and F.arm_features("web") is None assert F.arm_suffix("pe:flat+names") == "+pe-tool[names+flat]" @@ -195,6 +201,82 @@ def test_a_brief_explanation_keeps_the_value_its_direct_inputs_and_main_paramete assert F.brief_explanation({"error": "x"}) == {"error": "x"} +def test_labels_reads_the_facts_as_the_prompt_labels_them(): + situation = _situation() + situation["people"]["head"].update( + { + "gross wages and salaries": 30000, + "Has Employer-Sponsored Insurance": True, + "usual weekly hours worked": 40, + "self-employment partnership income": 5, + } + ) + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.LABELS) + assert error is None, error + head = request["situation"]["people"]["head"] + assert head["employment_income"] == {"2026": 30000} + assert head["has_esi"] == {"2026": True} + assert head["hours_worked_last_week"] == {"2026": 40} + assert head["partnership_self_employment_net_earnings"] == {"2026": 5} + # The reference also sets stated hours where the SNAP work rules read them. + assert head["weekly_hours_worked_before_lsr"] == {"2026": 40} + assert any("also set weekly_hours_worked_before_lsr" in n for n in request["notes"]) + # A name written with spaces is the names feature's, not this one's. + spaced = _situation() + spaced["people"]["head"]["employment income"] = 1 + request, error = _norm({"situation": spaced, "variables": ["snap"]}, F.LABELS) + assert request is None and "unknown variable 'employment income'" in str(error) + + +def test_labels_leaves_hours_already_set_where_the_rules_read_them(): + situation = _situation() + situation["people"]["head"].update( + {"hours_worked_last_week": 40, "weekly_hours_worked_before_lsr": 20} + ) + request, _ = _norm({"situation": situation, "variables": ["snap"]}, F.LABELS) + head = request["situation"]["people"]["head"] + assert head["weekly_hours_worked_before_lsr"] == {"2026": 20} + + +def test_pe2_reads_no_labels_and_sets_no_alias_as_it_ran(): + situation = _situation() + situation["people"]["head"]["gross wages and salaries"] = 30000 + request, error = _norm( + {"situation": situation, "variables": ["snap"]}, *F.V2_FEATURES + ) + assert request is None and "gross wages and salaries" in str(error) + situation = _situation() + situation["people"]["head"]["hours_worked_last_week"] = 40 + request, _ = _norm({"situation": situation, "variables": ["snap"]}, *F.V2_FEATURES) + assert ( + "weekly_hours_worked_before_lsr" not in request["situation"]["people"]["head"] + ) + + +def test_only_pe2_says_the_facts_carry_variable_names(): + claim = "labeled with its variable names" + assert claim in F.TOOL_NOTICE_V2 + for size in range(1, len(F.TOOL_FEATURES) + 1): + for features in itertools.combinations(F.TOOL_FEATURES, size): + features = frozenset(features) + if F.CONCISE in features and F.EXPLAIN not in features: + continue + notice = F.compose_tool_notice(features, True) + assert (claim in notice) == (features == F.V2_FEATURES), features + assert notice.endswith("as instructed above.") + assert ("names it read" in notice) == bool(features & {F.NAMES, F.LABELS}) + + +def test_every_prompt_label_reads_back_to_its_engine_name(): + from policybench.prompts import INPUT_LABEL_OVERRIDES + from policybench.scenarios import PE_INPUT_RENAMES + + labels = F.prompt_labels() + for field, label in INPUT_LABEL_OVERRIDES.items(): + assert labels[F._label_key(label)] == PE_INPUT_RENAMES.get(field, field) + assert labels[F._label_key("roth 401k contributions")] == "roth_401k_contributions" + + def _flat(**overrides): household = { "state": "LA", @@ -375,3 +457,96 @@ def test_a_household_described_flat_computes_what_the_benchmarks_own_does(): assert values[0] == values[1], (scenario.id, variable, values) checked += 1 assert checked > 80 + + +@pytest.mark.slow +def test_a_household_described_under_the_prompts_labels_computes_the_same(): + """Differential: each frozen-board household described flat with every + fact as its prompt writes it (a label in words, or the name with spaces), + and stated hours only where the prompt states them, computes every board + output as the benchmark does. Take-up conventions, which the prompt does + not list, keep their engine names.""" + import household_set as hs + import pandas as pd + from policyengine_us import Simulation + + from policybench.ground_truth import _extract_person_value, _pe_variable_for_output + from policybench.prompts import _humanize_input_label + from policybench.scenarios import PE_INPUT_ALIASES, PE_INPUT_RENAMES + + system = F.build_tool_system(reform=True) + registry = F.build_registry(system) + reference = pd.read_csv(hs.FROZEN_BOARD / "reference_outputs.csv") + variables = reference.groupby("scenario_id").variable.agg(list) + field_of = {name: field for field, name in PE_INPUT_RENAMES.items()} + dropped = {"is_tax_unit_head", "is_tax_unit_spouse", *PE_INPUT_ALIASES.values()} + + worded = [] + + def labeled(facts): + out = {} + for k, v in facts.items(): + if k in dropped: + continue + if k.startswith(F.CONVENTION_PREFIXES): + out[k] = v + continue + label = _humanize_input_label(field_of.get(k, k)) + if label != k.replace("_", " "): + worded.append(label) + out[label] = v + return out + + checked = 0 + for scenario in hs.load_scenarios(hs.FROZEN_BOARD)[:12]: + situation = hs.situation_for(scenario) + couple = scenario.marital_couple() or () + head = couple[0] if couple else scenario.adults[0].name + people = [ + { + "name": name, + "role": "head" + if name == head + else ("spouse" if name in couple else "dependent"), + "facts": labeled(facts), + } + for name, facts in situation["people"].items() + ] + household_facts = {} + for group in ("tax_units", "spm_units", "families", "households"): + for unit in situation[group].values(): + household_facts.update( + labeled( + { + k: v + for k, v in unit.items() + if k not in ("members", "state_code") + } + ) + ) + household = { + "state": scenario.state, + "people": people, + "household_facts": household_facts, + } + request, error = F.normalize_request( + {"household": json.dumps(household), "variables": ["snap"]}, + registry, + features=frozenset({F.FLAT, F.LABELS, F.NAMES}), + ) + assert error is None, (scenario.id, error) + own = Simulation(tax_benefit_system=system, situation=situation) + flat = Simulation(tax_benefit_system=system, situation=request["situation"]) + for variable in variables[scenario.id]: + name = _pe_variable_for_output(variable, "us") + values = [ + float( + _extract_person_value( + simulation.calculate(name, 2026), scenario, variable + ) + ) + for simulation in (own, flat) + ] + assert values[0] == values[1], (scenario.id, variable, values) + checked += 1 + assert checked > 120 and len(worded) > 10, (checked, worded) From fd3f5d096562754da767819bf6334a84449acf73 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 17:31:44 -0400 Subject: [PATCH 10/64] Labels: read the v2-pilot relabelings; flat: names optional The v2-pilot prompt relabels two facts itself ("home energy assistance received" for spm_unit_energy_subsidy, "employer-paid health insurance premiums ..." for employer_sponsored_insurance_premiums); the labels feature now reads them, derived from V2_PILOT_CONVENTIONS so the two stay in step. In the flat format a person given no name is named by role (head, spouse, dependent_1, ...): Flash-Lite's first flat call in two smoke households left the names out. The label differential now writes each fact as the v2-pilot prompt does. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 46 +++++++++++++++++++++++++++---------- tests/test_tool_v3.py | 44 +++++++++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 58ec78bb..775725c9 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -664,9 +664,10 @@ def refuse_policyengine(label: str, value) -> None: FLAT_DESCRIPTION = ( "Run PolicyEngine US (policyengine-us 2.41.0) on one household and return " "the requested variables. `household` is a JSON object: `state` (the " - "two-letter state code), `people` (a list, each with `name`, `role` -- " - 'head, spouse or dependent -- and `facts`, for example {"age": 40, ' - '"employment income": 30000}), and `household_facts` (facts about the ' + "two-letter state code), `people` (a list, each with `role` -- head, " + 'spouse or dependent --, `facts`, for example {"age": 40, "employment ' + 'income": 30000}, and optionally `name`, by default the role, numbered for ' + "dependents), and `household_facts` (facts about the " "household as a whole: its tax unit, family, SPM unit or home). The head " "and spouse are one couple and everyone else is on their own; everyone " "shares one tax unit, family, SPM unit and household. A fact is a value " @@ -1248,7 +1249,8 @@ def flat_situation(household, registry, problems: list[str], v2: dict | None): reference situations have (policybench.scenarios.Scenario.to_pe_household): the head and spouse one marital unit and everyone else alone, and one tax unit, SPM unit, family and household holding everyone. A household fact - goes to the unit of its variable's entity. Returns None after recording + goes to the unit of its variable's entity. A person given no name is named + by role (head, spouse, dependent_1, ...). Returns None after recording problems.""" if isinstance(household, str): if len(household) > MAX_SITUATION_CHARS: @@ -1279,6 +1281,7 @@ def flat_situation(household, registry, problems: list[str], v2: dict | None): return None situation: dict[str, Any] = {"people": {}} roles: dict[str, str] = {} + dependents = 0 for index, person in enumerate(people): if not isinstance(person, dict): problems.append( @@ -1290,17 +1293,23 @@ def flat_situation(household, registry, problems: list[str], v2: dict | None): person.get("role"), person.get("facts", {}), ) + if role not in FLAT_ROLES: + problems.append( + f"people[{index}].role must be one of {', '.join(FLAT_ROLES)}" + ) + continue + if role == "dependent": + dependents += 1 + if name is None: + name = role if role != "dependent" else f"dependent_{dependents}" + if v2 is not None: + v2["notes"].append(f"people[{index}]: named {name}") if not isinstance(name, str) or not name or len(name) > 64: problems.append(f"people[{index}].name must be a short string") continue if name in roles: problems.append(f"people[{index}]: {name!r} is listed twice") continue - if role not in FLAT_ROLES: - problems.append( - f"people[{index}].role must be one of {', '.join(FLAT_ROLES)}" - ) - continue if not isinstance(facts, dict): problems.append(f"people[{index}].facts must be an object") continue @@ -1404,14 +1413,15 @@ def _label_key(text: str) -> str: @functools.lru_cache(maxsize=1) def prompt_labels() -> dict[str, str]: """The labels the prompts give facts in words, keyed by _label_key, as the - engine names the facts (policybench.scenarios.PE_INPUT_RENAMES applied). A - label two facts share is left out.""" + engine names the facts (policybench.scenarios.PE_INPUT_RENAMES applied): + the board's labels and the v2-pilot prompt's relabelings. A label two facts + share is left out.""" from policybench.prompts import INPUT_LABEL_OVERRIDES from policybench.scenarios import PE_INPUT_RENAMES labels: dict[str, str] = {} shared = set() - for source, label in INPUT_LABEL_OVERRIDES.items(): + for source, label in {**INPUT_LABEL_OVERRIDES, **v2_pilot_labels()}.items(): key = _label_key(label) name = PE_INPUT_RENAMES.get(source, source) if key in labels and labels[key] != name: @@ -1422,6 +1432,18 @@ def prompt_labels() -> dict[str, str]: return {key: name for key, name in labels.items() if key not in shared} +def v2_pilot_labels() -> dict[str, str]: + """The facts the v2-pilot prompt relabels (a V2_PILOT_CONVENTIONS rewrite + of ": " to ": "), by input name.""" + relabeled = {} + for pattern, replacement in V2_PILOT_CONVENTIONS: + old = pattern.pattern + if pattern.groups or not old.endswith(": ") or not replacement.endswith(": "): + continue + relabeled[old[:-2].replace(" ", "_")] = replacement[:-2] + return relabeled + + def read_fact_name( name: str, registry: Mapping[str, VariableInfo], features: frozenset ) -> tuple[str | None, str | None]: diff --git a/tests/test_tool_v3.py b/tests/test_tool_v3.py index c3a5eb36..d8903c93 100644 --- a/tests/test_tool_v3.py +++ b/tests/test_tool_v3.py @@ -274,6 +274,16 @@ def test_every_prompt_label_reads_back_to_its_engine_name(): labels = F.prompt_labels() for field, label in INPUT_LABEL_OVERRIDES.items(): assert labels[F._label_key(label)] == PE_INPUT_RENAMES.get(field, field) + # The v2-pilot prompt's own relabelings, as that prompt writes them. + relabeled = F.v2_pilot_labels() + assert set(relabeled) == { + "employer_sponsored_insurance_premiums", + "spm_unit_energy_subsidy", + } + for field, label in relabeled.items(): + written = F.v2_pilot_text(f"- {field.replace('_', ' ')}: $5") + assert written == f"- {label}: $5" + assert labels[F._label_key(label)] == field assert labels[F._label_key("roth 401k contributions")] == "roth_401k_contributions" @@ -324,6 +334,33 @@ def test_flat_builds_the_benchmarks_units(): assert s["people"]["Kid"]["is_tax_unit_head"] == {"2026": False} +def test_flat_names_a_person_by_role_when_no_name_is_given(): + people = [ + {"role": "head", "facts": {"age": 40}}, + {"role": "dependent", "facts": {"age": 6}}, + {"name": "Kid", "role": "dependent", "facts": {"age": 4}}, + {"role": "dependent", "facts": {"age": 2}}, + ] + request, error = _norm(_flat(people=people), F.FLAT) + assert error is None, error + assert list(request["situation"]["people"]) == [ + "head", + "dependent_1", + "Kid", + "dependent_3", + ] + assert "people[0]: named head" in request["notes"] + people = [ + {"role": "head", "facts": {}}, + {"name": "head", "role": "dependent", "facts": {}}, + ] + request, error = _norm(_flat(people=people), F.FLAT) + assert any("listed twice" in p for p in error["problems"]) + people = [{"name": "", "role": "head", "facts": {}}] + request, error = _norm(_flat(people=people), F.FLAT) + assert any("name must be a short string" in p for p in error["problems"]) + + def test_flat_refuses_what_it_cannot_place(): people = [ {"name": "A", "role": "head", "facts": {}}, @@ -462,7 +499,8 @@ def test_a_household_described_flat_computes_what_the_benchmarks_own_does(): @pytest.mark.slow def test_a_household_described_under_the_prompts_labels_computes_the_same(): """Differential: each frozen-board household described flat with every - fact as its prompt writes it (a label in words, or the name with spaces), + fact as its v2-pilot prompt writes it (a label in words, or the name with + spaces), and stated hours only where the prompt states them, computes every board output as the benchmark does. Take-up conventions, which the prompt does not list, keep their engine names.""" @@ -491,7 +529,9 @@ def labeled(facts): if k.startswith(F.CONVENTION_PREFIXES): out[k] = v continue - label = _humanize_input_label(field_of.get(k, k)) + label = F.v2_pilot_text(f"- {_humanize_input_label(field_of.get(k, k))}: ")[ + 2:-2 + ] if label != k.replace("_", " "): worded.append(label) out[label] = v From 5c0f3ec5843b4a839b8255f9836a359e4d6f3c80 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 17:39:43 -0400 Subject: [PATCH 11/64] Flat: a household of one's person fact goes to that person Two of three Flash-Lite smoke households gave the head's rent as a household fact. pre_subsidy_rent is a person variable in policyengine-us, so the flat format refused it. Under the placement feature a person fact given for a household of one now goes to that person, as placement moves a person variable off a one-person unit; with two or more people it is still refused, now saying to give it under the person whose facts list it. The test registry's entities now match the engine's (pre_subsidy_rent is a person variable). Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 18 ++++++++++++++++- tests/test_tool_v3.py | 39 +++++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 775725c9..e3e3f1b4 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -1366,9 +1366,25 @@ def flat_situation(household, registry, problems: list[str], v2: dict | None): continue entity = registry[resolved].entity if entity == "person": + # As placement moves a person variable off a one-person unit: only + # a household of one says whose fact it is. + if v2 is not None and PLACEMENT in v2["features"] and len(everyone) == 1: + facts_of = situation["people"][everyone[0]] + if resolved in facts_of and facts_of[resolved] != value: + problems.append( + f"household_facts: {resolved} is also set for " + f"{everyone[0]} with a different value" + ) + continue + facts_of[resolved] = value + v2["notes"].append( + f"household_facts: moved {resolved}, a person's fact, to " + f"{everyone[0]}" + ) + continue problems.append( f"household_facts: {resolved} is a person's fact; give it under " - "that person" + "the person whose facts list it" ) continue unit_of[entity][resolved] = value diff --git a/tests/test_tool_v3.py b/tests/test_tool_v3.py index d8903c93..be60efdf 100644 --- a/tests/test_tool_v3.py +++ b/tests/test_tool_v3.py @@ -31,7 +31,8 @@ ), "traditional_401k_contributions_desired": V("person", "float", "asked", "year"), "snap": V("spm_unit", "float", "SNAP", "month", computed=True), - "pre_subsidy_rent": V("spm_unit", "float", "rent", "year"), + "pre_subsidy_rent": V("person", "float", "rent", "year"), + "spm_unit_energy_subsidy": V("spm_unit", "float", "energy", "year"), "qualified_passenger_vehicle_loan_interest": V( "household", "float", "car loan", "year" ), @@ -300,7 +301,7 @@ def _flat(**overrides): {"name": "Kid", "role": "dependent", "facts": {"age": 6}}, ], "household_facts": { - "pre_subsidy_rent": 9000, + "spm_unit_energy_subsidy": 900, "tax_unit_childcare_expenses": 1200, "qualified_passenger_vehicle_loan_interest": 300, }, @@ -324,7 +325,7 @@ def test_flat_builds_the_benchmarks_units(): ): assert s[group][unit]["members"] == ["Head", "Spouse", "Kid"] assert s["households"]["household"]["state_code"] == {"2026": "LA"} - assert s["spm_units"]["spm_unit"]["pre_subsidy_rent"] == {"2026": 9000} + assert s["spm_units"]["spm_unit"]["spm_unit_energy_subsidy"] == {"2026": 900} assert s["tax_units"]["tax_unit"]["tax_unit_childcare_expenses"] == {"2026": 1200} assert s["households"]["household"][ "qualified_passenger_vehicle_loan_interest" @@ -361,6 +362,29 @@ def test_flat_names_a_person_by_role_when_no_name_is_given(): assert any("name must be a short string" in p for p in error["problems"]) +def test_flat_placement_gives_a_household_of_ones_person_fact_to_that_person(): + one = [{"name": "Solo", "role": "head", "facts": {"age": 70}}] + args = _flat(people=one, household_facts={"pre_subsidy_rent": 9000}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert error is None, error + solo = request["situation"]["people"]["Solo"] + assert solo["pre_subsidy_rent"] == {"2026": 9000} + assert "pre_subsidy_rent" not in request["situation"]["spm_units"]["spm_unit"] + assert any("moved pre_subsidy_rent" in n for n in request["notes"]) + one[0]["facts"]["pre_subsidy_rent"] = 100 + conflict = _flat(people=one, household_facts={"pre_subsidy_rent": 9000}) + request, error = _norm(conflict, F.FLAT, F.PLACEMENT) + assert any("different value" in p for p in error["problems"]) + # Two people: whose rent it is is not said, so it is refused. + request, error = _norm( + _flat(household_facts={"pre_subsidy_rent": 9000}), F.FLAT, F.PLACEMENT + ) + assert any("whose facts list it" in p for p in error["problems"]) + # Without placement, even a household of one is refused. + request, error = _norm(args, F.FLAT) + assert any("whose facts list it" in p for p in error["problems"]) + + def test_flat_refuses_what_it_cannot_place(): people = [ {"name": "A", "role": "head", "facts": {}}, @@ -381,13 +405,12 @@ def test_flat_refuses_what_it_cannot_place(): def test_flat_with_names_reads_labeled_household_facts(): request, error = _norm( - _flat(household_facts={"pre subsidy rent": 9000}), F.FLAT, F.NAMES + _flat(household_facts={"spm unit energy subsidy": 900}), F.FLAT, F.NAMES ) assert error is None - assert request["situation"]["spm_units"]["spm_unit"]["pre_subsidy_rent"] == { - "2026": 9000 - } - assert any("as pre_subsidy_rent" in n for n in request["notes"]) + spm_unit = request["situation"]["spm_units"]["spm_unit"] + assert spm_unit["spm_unit_energy_subsidy"] == {"2026": 900} + assert any("as spm_unit_energy_subsidy" in n for n in request["notes"]) def test_the_schema_offers_each_features_arguments(): From 9c43f4f7ea0e576e8686af00b962a1896fe458a3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 17:43:24 -0400 Subject: [PATCH 12/64] Keep tool version 2's request as it was: detail only under concise A differential against 5bbd571a (the commit the test batches ran) found version 2 requests carrying detail "full"; only execute_request reads it, through .get, so results were unchanged, but the request now matches exactly. 4,000 generated calls through versions 1 and 2, the notices, descriptions, schemas and 15 model profiles are identical to 5bbd571a. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 3 ++- tests/test_tool_v3.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index e3e3f1b4..2712b046 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -1224,7 +1224,7 @@ def fail() -> tuple[None, dict]: elif name not in explain: explain.append(name) - detail = "full" + detail = None if CONCISE in features: detail = args.get("detail") or "brief" if detail not in ("brief", "full"): @@ -1236,6 +1236,7 @@ def fail() -> tuple[None, dict]: if v2 is not None: request["explain"] = explain request["notes"] = v2["notes"] + if detail is not None: request["detail"] = detail return request, None diff --git a/tests/test_tool_v3.py b/tests/test_tool_v3.py index be60efdf..e0064022 100644 --- a/tests/test_tool_v3.py +++ b/tests/test_tool_v3.py @@ -179,7 +179,7 @@ def test_concise_takes_a_detail_and_defaults_to_brief(): {"situation": _situation(), "variables": ["snap"], "explain": ["snap"]}, F.EXPLAIN, ) - assert request["detail"] == "full" + assert "detail" not in request # every step, as tool version 2 explains def test_a_brief_explanation_keeps_the_value_its_direct_inputs_and_main_parameters(): From 6478e8f40f01b2253245f23a8e872d4a11784642 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 18:35:53 -0400 Subject: [PATCH 13/64] Tool v3 review r1: frame, resume, labels, flat and text fixes GPT-6.1 Sol's review (REQUEST_CHANGES, 7 findings) at 9c43f4f7: 1. A model's arguments could turn on features its arm does not offer: the worker read "__tool_features__" (and, as at 5bbd571a, "__tool_version__") from the arguments dict. A pool now sends features in a tuple frame no JSON can be, the worker validates them (names, concise beside explain), and keys like "__tool_version__" in a model's arguments are refused as unknown arguments. No run record ever carried them. 2. A resume could mix treatments. Each household record now keeps its tool features and calculator schema hash; a resume compares features, notice and schema, and refuses a non-legacy record without them. 3. Labels missed what the prompt renders. The label index is now built from the prompt's own renderer (_format_input_line, then v2_pilot_text) for every promptable, labeled and reference-stated input: fallback labels, the Marketplace plan line, "state", partnership or S-corp income. Facts the reference drops (PE_DROPPED_INPUTS) are ignored with a note. A label that names two of the engine's inputs is not read. 4. Flat refuses person keys other than name, role and facts. 5. Flat stores household facts under the name written, so normalization compares them once normalized; roles set the relationship inputs and a stated one that contradicts its role is refused. 6. The flat example uses employment_income; placement replaces the wrong-entity error sentence; "full" detail is the trace, capped. 7. Diagnostics read flat people by their default names. Evidence: 210 passed, 4 frozen-engine skips. A labels-only differential builds 30 frozen households from their prompts' facts and matches the reference situation on every output, and so do all 100 frozen and all 20 test-set households in a one-off run, with or without take-up inputs. pe and pe2 are identical to 5bbd571a over 4,000 calls and 2,000 pool messages. 11 mutants of the fixes are caught. An end-to-end Flash-Lite pe3 smoke run had 0 tool errors on 3 households. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 454 +++++++++++++++++++++++++++--------- tests/test_tool_v2.py | 7 +- tests/test_tool_v3.py | 354 ++++++++++++++++++++++------ 3 files changed, 629 insertions(+), 186 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 2712b046..5f586522 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -311,8 +311,8 @@ def compose_tool_notice(features: frozenset, tool_contract: bool) -> str: if EXPLAIN in features: explain = ( "ask it to `explain` up to three variables to see in brief how it " - "computed them, with the parameters and laws it used, or every step " - 'with `detail` set to "full"' + "computed them, with the parameters and laws it used, or the fuller " + 'trace with `detail` set to "full"' if CONCISE in features else "ask it to `explain` up to three variables to see how it computed " "them, with the parameters and laws it used" @@ -665,9 +665,9 @@ def refuse_policyengine(label: str, value) -> None: "Run PolicyEngine US (policyengine-us 2.41.0) on one household and return " "the requested variables. `household` is a JSON object: `state` (the " "two-letter state code), `people` (a list, each with `role` -- head, " - 'spouse or dependent --, `facts`, for example {"age": 40, "employment ' - 'income": 30000}, and optionally `name`, by default the role, numbered for ' - "dependents), and `household_facts` (facts about the " + 'spouse or dependent --, `facts`, for example {"age": 40, ' + '"employment_income": 30000}, and optionally `name`, by default the role, ' + "numbered for dependents), and `household_facts` (facts about the " "household as a whole: its tax unit, family, SPM unit or home). The head " "and spouse are one couple and everyone else is on their own; everyone " "shares one tax unit, family, SPM unit and household. A fact is a value " @@ -676,7 +676,13 @@ def refuse_policyengine(label: str, value) -> None: 'lists the variables to compute for `period` (default "2026"). Each ' "result gives the variable's entity, label and value for each member of " "that entity; a monthly variable computed for a year is summed over its " - "months." + "months. An unknown name or a malformed household returns an error with " + "suggestions instead of results." +) +# The version 1 description's last sentence, which placement amends. +_ENTITY_ERROR_SENTENCE = ( + "An unknown name, a variable set on the wrong entity, or a malformed " + "situation returns an error with suggestions instead of results." ) @@ -693,6 +699,12 @@ def calculate_tool_schema(version: int = 1, features: frozenset | None = None) - description = ( FLAT_DESCRIPTION if FLAT in features else CALCULATE_DESCRIPTION ) + if PLACEMENT in features and FLAT not in features: + description = description.replace( + _ENTITY_ERROR_SENTENCE, + "An unknown name or a malformed situation returns an error " + "with suggestions instead of results.", + ) notes = [] if LABELS in features: notes.append( @@ -706,8 +718,9 @@ def calculate_tool_schema(version: int = 1, features: frozenset | None = None) - ) if PLACEMENT in features and FLAT not in features: notes.append( - "An input set on the wrong entity is moved to the one unit it " - "belongs to." + "An input set on the wrong entity is moved to the one unit " + "holding its people; where no single unit holds them, it " + "returns an error." ) if OVERRIDES in features: notes.append( @@ -744,7 +757,8 @@ def calculate_tool_schema(version: int = 1, features: frozenset | None = None) - "description": ( 'How much an explanation shows: "brief" (the default: the ' 'value, what it came from, and the main parameters) or "full" ' - "(every step)." + f"(the trace, up to {EXPLAIN_MAX_LINES} steps and " + f"{EXPLAIN_MAX_PARAMETERS} parameters)." ), } return schema @@ -821,13 +835,9 @@ class VariableInfo: def stated_inputs() -> frozenset[str]: """Every variable a household fact the benchmark states can set.""" - from policybench.scenarios import ( - PE_INPUT_ALIASES, - PE_INPUT_RENAMES, - get_promptable_input_specs, - ) + from policybench.scenarios import PE_INPUT_ALIASES, PE_INPUT_RENAMES - names = {spec.source_name for spec in get_promptable_input_specs()} + names = {spec.source_name for spec in _promptable_specs()} names |= BASE_STATED_INPUTS names |= set(PE_INPUT_ALIASES.values()) | set(PE_INPUT_RENAMES.values()) return frozenset(names) @@ -961,6 +971,9 @@ def _normalize_inputs( if resolved is not None: v2["notes"].append(f"{where}: {note}") name = resolved + elif (field := dropped_fact(name, registry, v2["features"])) is not None: + v2["notes"].append(_dropped_note(where, name, field)) + continue if not isinstance(name, str) or name not in registry: problems.append( f"{where}: unknown variable {name!r}." @@ -1068,7 +1081,7 @@ def fail() -> tuple[None, dict]: if FLAT in features: args = dict(args) args["situation"] = flat_situation( - args.get("household"), registry, problems, v2 + args.get("household"), registry, problems, v2, period ) if args["situation"] is None: return fail() @@ -1242,17 +1255,67 @@ def fail() -> tuple[None, dict]: FLAT_ROLES = ("head", "spouse", "dependent") +FLAT_PERSON_KEYS = ("name", "role", "facts") +# The relationship inputs a role sets (as Scenario.to_pe_household's people +# carry them). +FLAT_RELATIONS = {"is_tax_unit_head": "head", "is_tax_unit_spouse": "spouse"} + + +def flat_names(people) -> list: + """The name each listed person goes by: the one given, or, for a person + given none, the role (head, spouse, dependent_1, ...: dependents are + numbered by their place among the dependents listed).""" + names, dependents = [], 0 + for person in people if isinstance(people, list) else []: + if not isinstance(person, dict): + names.append(None) + continue + role = person.get("role") + if role == "dependent": + dependents += 1 + name = person.get("name") + if name is None and role in FLAT_ROLES: + name = role if role != "dependent" else f"dependent_{dependents}" + names.append(name) + return names -def flat_situation(household, registry, problems: list[str], v2: dict | None): +def _same_value(registry, name: str, a, b, period: str) -> bool | None: + """Whether two values of an input are the same once normalized (None when + either is malformed; normalization reports it).""" + scratch: list[str] = [] + info = registry[name] + first = _normalize_value("", name, a, info, period, scratch) + second = _normalize_value("", name, b, info, period, scratch) + return None if scratch else first == second + + +def flat_situation( + household, + registry, + problems: list[str], + v2: dict | None, + period: str = str(YEAR), +): """The flat feature's household -- state, people with role and facts, and household facts -- as a situation, with the units the benchmark's own reference situations have (policybench.scenarios.Scenario.to_pe_household): the head and spouse one marital unit and everyone else alone, and one tax - unit, SPM unit, family and household holding everyone. A household fact - goes to the unit of its variable's entity. A person given no name is named - by role (head, spouse, dependent_1, ...). Returns None after recording - problems.""" + unit, SPM unit, family and household holding everyone. Roles set the + relationship inputs; a stated one that contradicts its role is refused. + A household fact goes to the unit of its variable's entity, under the + name it was written with, so normalization reads and compares it as any + other input. A person given no name is named by role (flat_names). + Returns None after recording problems.""" + features = v2["features"] if v2 is not None else frozenset() + + def engine_name(name): + if isinstance(name, str) and name in registry: + return name + if isinstance(name, str): + return read_fact_name(name, registry, features)[0] + return None + if isinstance(household, str): if len(household) > MAX_SITUATION_CHARS: problems.append("household is too large") @@ -1282,29 +1345,27 @@ def flat_situation(household, registry, problems: list[str], v2: dict | None): return None situation: dict[str, Any] = {"people": {}} roles: dict[str, str] = {} - dependents = 0 - for index, person in enumerate(people): + for index, (person, name) in enumerate(zip(people, flat_names(people))): if not isinstance(person, dict): problems.append( f"people[{index}] must be an object with name, role and facts" ) continue - name, role, facts = ( - person.get("name"), - person.get("role"), - person.get("facts", {}), - ) + stray = sorted(map(str, set(person) - set(FLAT_PERSON_KEYS))) + if stray: + problems.append( + f"people[{index}] takes only name, role and facts; give " + f"{', '.join(stray)} under facts" + ) + continue + role, facts = person.get("role"), person.get("facts", {}) if role not in FLAT_ROLES: problems.append( f"people[{index}].role must be one of {', '.join(FLAT_ROLES)}" ) continue - if role == "dependent": - dependents += 1 - if name is None: - name = role if role != "dependent" else f"dependent_{dependents}" - if v2 is not None: - v2["notes"].append(f"people[{index}]: named {name}") + if person.get("name") is None and v2 is not None: + v2["notes"].append(f"people[{index}]: named {name}") if not isinstance(name, str) or not name or len(name) > 64: problems.append(f"people[{index}].name must be a short string") continue @@ -1326,8 +1387,21 @@ def flat_situation(household, registry, problems: list[str], v2: dict | None): return None for name, role in roles.items(): facts = situation["people"][name] - facts.setdefault("is_tax_unit_head", role == "head") - facts.setdefault("is_tax_unit_spouse", role == "spouse") + for relation, relation_role in FLAT_RELATIONS.items(): + if relation not in registry: + continue + implied = role == relation_role + stated = [k for k in facts if engine_name(k) == relation] + if not stated: + facts[relation] = implied + elif any( + _same_value(registry, relation, facts[k], implied, period) is False + for k in stated + ): + problems.append( + f"people.{name}: {relation} contradicts the role {role}; the " + "role sets it" + ) everyone = list(situation["people"]) couple = heads + spouses groups = [couple] + [[n] for n in everyone if n not in couple] @@ -1354,12 +1428,16 @@ def flat_situation(household, registry, problems: list[str], v2: dict | None): problems.append("household_facts must be an object") return None for name, value in facts.items(): - resolved = name - if isinstance(name, str) and name not in registry and v2 is not None: - resolved, note = read_fact_name(name, registry, v2["features"]) - if resolved is not None: - v2["notes"].append(f"household_facts: {note}") - if not isinstance(resolved, str) or resolved not in registry: + resolved = engine_name(name) + if resolved is None: + field = ( + dropped_fact(name, registry, features) + if isinstance(name, str) + else None + ) + if field is not None: + v2["notes"].append(_dropped_note("household_facts", name, field)) + continue problems.append( f"household_facts: unknown variable {name!r}." + (_suggest(name, registry) if isinstance(name, str) else "") @@ -1369,26 +1447,25 @@ def flat_situation(household, registry, problems: list[str], v2: dict | None): if entity == "person": # As placement moves a person variable off a one-person unit: only # a household of one says whose fact it is. - if v2 is not None and PLACEMENT in v2["features"] and len(everyone) == 1: - facts_of = situation["people"][everyone[0]] - if resolved in facts_of and facts_of[resolved] != value: - problems.append( - f"household_facts: {resolved} is also set for " - f"{everyone[0]} with a different value" - ) - continue - facts_of[resolved] = value - v2["notes"].append( - f"household_facts: moved {resolved}, a person's fact, to " - f"{everyone[0]}" + if v2 is None or PLACEMENT not in features or len(everyone) != 1: + problems.append( + f"household_facts: {resolved} is a person's fact; give it " + "under the person whose facts list it" ) continue - problems.append( - f"household_facts: {resolved} is a person's fact; give it under " - "the person whose facts list it" + target = situation["people"][everyone[0]] + v2["notes"].append( + f"household_facts: moved {resolved}, a person's fact, to {everyone[0]}" ) + else: + target = unit_of[entity] + if name in target: + if _same_value(registry, resolved, target[name], value, period) is False: + problems.append( + f"household_facts: {resolved} is set twice with different values" + ) continue - unit_of[entity][resolved] = value + target[name] = value return None if problems else situation @@ -1427,55 +1504,123 @@ def _label_key(text: str) -> str: return re.sub(r"[^0-9a-z]", "", text.casefold()) +# The prompt's household block gives the state as "state: CT". +PROMPT_EXTRA_LABELS = {"state": "state_code"} + + +@functools.lru_cache(maxsize=1) +def _promptable_specs(): + from policybench.scenarios import get_promptable_input_specs + + return get_promptable_input_specs() + + +def _sample_value(value_type: str): + if value_type == "bool": + return True + if value_type in ("str", "Enum"): + return "x" + return 1.0 + + +def rendered_labels(field: str, value_type: str = "float") -> tuple[str, ...]: + """The labels a fact is shown under: in the board's prompt + (policybench.prompts._format_input_line, special renderers included) and + in the v2-pilot prompt (v2_pilot_text applied to that line).""" + from policybench.prompts import _format_input_line, _humanize_input_label + + line = _format_input_line(field, _sample_value(value_type)) + label = _humanize_input_label(field) + if not (line == f"- {label}" or line.startswith(f"- {label}: ")): + label = line[2:].split(": ", 1)[0] + labels = [label] + suffix = line[2 + len(label) :] + rewritten = v2_pilot_text(line) + if rewritten != line and rewritten.startswith("- ") and rewritten.endswith(suffix): + labels.append(rewritten[2 : len(rewritten) - len(suffix)]) + return tuple(labels) + + @functools.lru_cache(maxsize=1) -def prompt_labels() -> dict[str, str]: - """The labels the prompts give facts in words, keyed by _label_key, as the - engine names the facts (policybench.scenarios.PE_INPUT_RENAMES applied): - the board's labels and the v2-pilot prompt's relabelings. A label two facts - share is left out.""" +def prompt_labels() -> dict[str, frozenset[tuple[str, bool]]]: + """Every label the prompts show a fact under, keyed by _label_key, each + with the facts it may name as (engine name, dropped): the promptable + inputs, every labeled input (policybench.prompts.INPUT_LABEL_OVERRIDES), + the inputs the reference sets besides them (BASE_STATED_INPUTS), inputs + the engine has renamed and inputs the reference leaves out + (policybench.scenarios.PE_INPUT_RENAMES, PE_DROPPED_INPUTS), each in the + board's and the v2-pilot prompt's wording. Dropped facts are shown but are + not engine inputs. A label may name facts of another country; a lookup + reads only the one its engine has.""" from policybench.prompts import INPUT_LABEL_OVERRIDES - from policybench.scenarios import PE_INPUT_RENAMES - - labels: dict[str, str] = {} - shared = set() - for source, label in {**INPUT_LABEL_OVERRIDES, **v2_pilot_labels()}.items(): - key = _label_key(label) - name = PE_INPUT_RENAMES.get(source, source) - if key in labels and labels[key] != name: - shared.add(key) - labels[key] = name - for source, name in PE_INPUT_RENAMES.items(): - labels.setdefault(_label_key(source), name) - return {key: name for key, name in labels.items() if key not in shared} - - -def v2_pilot_labels() -> dict[str, str]: - """The facts the v2-pilot prompt relabels (a V2_PILOT_CONVENTIONS rewrite - of ": " to ": "), by input name.""" - relabeled = {} - for pattern, replacement in V2_PILOT_CONVENTIONS: - old = pattern.pattern - if pattern.groups or not old.endswith(": ") or not replacement.endswith(": "): - continue - relabeled[old[:-2].replace(" ", "_")] = replacement[:-2] - return relabeled + from policybench.scenarios import PE_DROPPED_INPUTS, PE_INPUT_RENAMES + + fields = {spec.output_name: spec.value_type for spec in _promptable_specs()} + dropped = {f for names in PE_DROPPED_INPUTS.values() for f in names} + for source in ( + "age", + *BASE_STATED_INPUTS, + *INPUT_LABEL_OVERRIDES, + *PE_INPUT_RENAMES, + *dropped, + ): + fields.setdefault(source, "int" if source == "age" else "float") + labels: dict[str, set[tuple[str, bool]]] = {} + for source, value_type in fields.items(): + target = (PE_INPUT_RENAMES.get(source, source), source in dropped) + keys = {_label_key(label) for label in rendered_labels(source, value_type)} + if source in PE_INPUT_RENAMES or source in dropped: + keys.add(_label_key(source)) + for key in keys: + labels.setdefault(key, set()).add(target) + for label, name in PROMPT_EXTRA_LABELS.items(): + labels.setdefault(_label_key(label), set()).add((name, False)) + return {key: frozenset(targets) for key, targets in labels.items()} def read_fact_name( name: str, registry: Mapping[str, VariableInfo], features: frozenset ) -> tuple[str | None, str | None]: - """A fact's engine variable under the labels feature (the label the prompt - gives it) and then the names feature (spaces for underscores). Returns - (engine name, note) or (None, None).""" + """A fact's engine variable under the labels feature (any label the + prompt shows it under, when exactly one fact it may name is the engine's) + and then the names feature (spaces for underscores). Returns (engine + name, note) or (None, None).""" if LABELS in features: - labeled = prompt_labels().get(_label_key(name)) - if labeled is not None and labeled in registry: - return labeled, f"read {name!r} as {labeled}" + found = [ + engine + for engine, dropped in prompt_labels().get(_label_key(name), ()) + if not dropped and engine in registry + ] + if len(found) == 1: + return found[0], f"read {name!r} as {found[0]}" if NAMES in features: return resolve_name(name, registry, output_names=False) return None, None +def dropped_fact( + name: str, registry: Mapping[str, VariableInfo], features: frozenset +) -> str | None: + """Under the labels feature, the input a fact names when the prompt shows + it but the reference leaves it out (policybench.scenarios.PE_DROPPED_INPUTS: + no longer an engine input); otherwise None.""" + if LABELS not in features: + return None + found = [ + engine + for engine, dropped in prompt_labels().get(_label_key(name), ()) + if dropped + ] + return found[0] if len(found) == 1 else None + + +def _dropped_note(where: str, name: str, field: str) -> str: + return ( + f"{where}: {name!r} ({field}) is not an input of this engine; the " + "benchmark's reference leaves it out too, so it was ignored" + ) + + def _set_aliases(v2: dict, people: dict, registry: Mapping[str, VariableInfo]): """Set each fact the reference also sets under another name there too (policybench.scenarios.PE_INPUT_ALIASES: the SNAP work rules read stated @@ -1643,7 +1788,7 @@ def execute_request(system, registry, request: dict, simulation_cls=None) -> dic def brief_explanation(full: dict) -> dict: """The concise feature's explanation: the value, the variables it came - from directly (one level down, zero-valued ones left out), and the first + from directly (one level down, the first BRIEF_INPUTS), and the first parameters it read with their sources.""" if "error" in full: return full @@ -1655,7 +1800,7 @@ def brief_explanation(full: dict) -> dict: "value": steps[0].strip() if steps else "", "from": direct[:BRIEF_INPUTS], "parameters": full["parameters"][:BRIEF_PARAMETERS], - "more": 'set detail to "full" for every step', + "more": 'set detail to "full" for the trace', } @@ -1843,6 +1988,46 @@ def build_tool_system( return CountryTaxBenefitSystem(reform=load_fix_reform(path, snapshot)) +# How a pool tells its worker the calculator's features: a tuple, which a +# model's JSON arguments can never be, so a model cannot turn on a feature its +# arm does not offer. A pool without features sends the arguments as they are. +CALL_FRAME = "calculate" + + +def tool_call_frame(features: frozenset, arguments) -> tuple: + return (CALL_FRAME, tuple(sorted(features)), arguments) + + +def frame_features(names) -> frozenset: + """A frame's features, refused (ValueError) unless they are a tuple of + known feature names with concise only beside explain.""" + if not isinstance(names, tuple) or not all(isinstance(n, str) for n in names): + raise ValueError("the features must be a tuple of names") + features = frozenset(names) + if features - set(TOOL_FEATURES): + raise ValueError( + f"unknown tool features {sorted(features - set(TOOL_FEATURES))}" + ) + if CONCISE in features and EXPLAIN not in features: + raise ValueError("the concise feature needs explain") + return features + + +def worker_request(message, registry) -> tuple[dict | None, dict | None]: + """A worker's reading of one pool message: a call frame's arguments under + its features, or anything else as a model's arguments with no features + (keys such as "__tool_features__" or "__tool_version__" in them are the + model's, and refused as unknown arguments).""" + features = frozenset() + if isinstance(message, tuple) and len(message) == 3 and message[0] == CALL_FRAME: + try: + features = frame_features(message[1]) + except ValueError as error: + return None, {"error": "configuration", "message": str(error)} + message = message[2] + return normalize_request(message, registry, features=features) + + def _worker_main(conn, fix_path: str, reform: bool, board: str | None = None) -> None: for name in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS"): os.environ[name] = "1" @@ -1857,13 +2042,7 @@ def _worker_main(conn, fix_path: str, reform: bool, board: str | None = None) -> return if args is None: return - features = frozenset() - if isinstance(args, dict) and "__tool_features__" in args: - features = frozenset(args.get("__tool_features__") or ()) - args = args.get("arguments") - elif isinstance(args, dict) and args.get("__tool_version__") == 2: - features, args = V2_FEATURES, args.get("arguments") - request, error = normalize_request(args, registry, features=features) + request, error = worker_request(args, registry) if error is not None: conn.send(("ok", error)) continue @@ -1958,13 +2137,8 @@ def call(self, args) -> tuple[dict, float]: time.perf_counter() - started, ) process, conn = worker - if self._tool_features == V2_FEATURES: - args = {"__tool_version__": 2, "arguments": args} - elif self._tool_features: - args = { - "__tool_features__": sorted(self._tool_features), - "arguments": args, - } + if self._tool_features: + args = tool_call_frame(self._tool_features, args) try: conn.send(args) if conn.poll(self.timeout): @@ -2863,6 +3037,7 @@ def run_household( "benchmark_search_rule": profile.benchmark_search_rule, "sent_board_prompt_sha256": sent_board_prompt_sha256, "tool_notice_sha256": hashlib.sha256(profile.notice.encode()).hexdigest(), + **profile_treatment(profile), "turns": [], "status": "running", } @@ -3571,6 +3746,52 @@ def records_wording(records: Mapping[str, dict]) -> str | None: return hashes.pop() if hashes else None +# Arms whose calculator features never changed, so a record written before +# households recorded their features has these. +LEGACY_TOOL_ARMS = {"pe": frozenset(), "web+pe": frozenset(), "pe2": V2_FEATURES} + + +def profile_treatment(profile: "ModelProfile") -> dict: + """What a household record keeps of the treatment it ran under, beyond + the notice hash: the calculator's features and its schema's hash.""" + if not profile.uses_calculator: + return {"tool_features": None, "calculate_tool_sha256": None} + schema = calculate_tool_schema(features=profile.tool_features) + return { + "tool_features": sorted(profile.tool_features), + "calculate_tool_sha256": hashlib.sha256( + json.dumps(schema, sort_keys=True).encode() + ).hexdigest(), + } + + +def record_treatment_problem(record: dict, profile: "ModelProfile") -> str | None: + """Why a household record ran under another treatment than the profile's + (features, notice or calculator schema), or None. A record written before + a field existed counts as matching only for an arm whose treatment never + changed (none, web, and LEGACY_TOOL_ARMS); otherwise it is unrecorded.""" + sid = record.get("scenario_id") + want = profile_treatment(profile) + legacy = profile.arm in ("none", "web") or profile.arm in LEGACY_TOOL_ARMS + if "tool_features" in record: + if record["tool_features"] != want["tool_features"]: + return f"{sid} ran with tool features {record['tool_features']}" + elif not legacy: + return f"{sid} has no recorded tool features" + notice = record.get("tool_notice_sha256") + if notice is None and not legacy: + return f"{sid} has no recorded tool notice" + if ( + notice is not None + and notice != hashlib.sha256(profile.notice.encode()).hexdigest() + ): + return f"{sid} ran under another tool notice" + schema = record.get("calculate_tool_sha256") + if schema is not None and schema != want["calculate_tool_sha256"]: + return f"{sid} ran with another calculator schema" + return None + + def records_arm(records: Mapping[str, dict]) -> str: """The one arm a run directory's records belong to; records written before the web arms are the PE tool's.""" @@ -3629,6 +3850,17 @@ def command_run(args) -> None: f"{households_dir} holds records sent under other v2-pilot wording " f"({records_wording(previous)}); use a fresh --out-dir" ) + mixed = [ + problem + for record in previous.values() + if (problem := record_treatment_problem(record, profile)) is not None + ] + if mixed: + raise SystemExit( + f"{households_dir} holds records from another treatment (" + + "; ".join(mixed[:3]) + + "); use a fresh --out-dir" + ) spend = Spend(args.budget_usd, spent_before) todo = [sid for sid in ids if sid not in done] manifest = { @@ -4416,7 +4648,7 @@ def _person_names(arguments) -> list[str]: household = ( json.loads(household) if isinstance(household, str) else household ) - return [person["name"] for person in household["people"]] + return [name or "" for name in flat_names(household["people"])] situation = args["situation"] situation = json.loads(situation) if isinstance(situation, str) else situation return list(situation["people"]) diff --git a/tests/test_tool_v2.py b/tests/test_tool_v2.py index d6e381b5..68248090 100644 --- a/tests/test_tool_v2.py +++ b/tests/test_tool_v2.py @@ -261,8 +261,13 @@ def test_the_pool_tells_its_workers_the_tool_version(): ) try: result, _ = pool.call({"situation": "{}"}) + # A frame no model's JSON arguments can be (tool v3 review r1). assert result == { - "received": {"__tool_version__": 2, "arguments": {"situation": "{}"}} + "received": ( + "calculate", + ("explain", "names", "overrides", "placement"), + {"situation": "{}"}, + ) } finally: pool.close() diff --git a/tests/test_tool_v3.py b/tests/test_tool_v3.py index e0064022..4b1e9c99 100644 --- a/tests/test_tool_v3.py +++ b/tests/test_tool_v3.py @@ -179,7 +179,7 @@ def test_concise_takes_a_detail_and_defaults_to_brief(): {"situation": _situation(), "variables": ["snap"], "explain": ["snap"]}, F.EXPLAIN, ) - assert "detail" not in request # every step, as tool version 2 explains + assert "detail" not in request # the trace, as tool version 2 explains def test_a_brief_explanation_keeps_the_value_its_direct_inputs_and_main_parameters(): @@ -268,24 +268,75 @@ def test_only_pe2_says_the_facts_carry_variable_names(): assert ("names it read" in notice) == bool(features & {F.NAMES, F.LABELS}) -def test_every_prompt_label_reads_back_to_its_engine_name(): - from policybench.prompts import INPUT_LABEL_OVERRIDES - from policybench.scenarios import PE_INPUT_RENAMES +def test_every_label_a_prompt_shows_reads_back_to_its_engine_name(): + from policybench.prompts import _format_input_line + from policybench.scenarios import PE_DROPPED_INPUTS, PE_INPUT_RENAMES labels = F.prompt_labels() - for field, label in INPUT_LABEL_OVERRIDES.items(): - assert labels[F._label_key(label)] == PE_INPUT_RENAMES.get(field, field) - # The v2-pilot prompt's own relabelings, as that prompt writes them. - relabeled = F.v2_pilot_labels() - assert set(relabeled) == { - "employer_sponsored_insurance_premiums", - "spm_unit_energy_subsidy", + fields = {spec.output_name: spec.value_type for spec in F._promptable_specs()} + fields |= {"age": "int", "employment_income": "float"} + dropped = {f for names in PE_DROPPED_INPUTS.values() for f in names} + for field, value_type in {**fields, **dict.fromkeys(dropped, "float")}.items(): + for label in F.rendered_labels(field, value_type): + want = (PE_INPUT_RENAMES.get(field, field), field in dropped) + assert want in labels[F._label_key(label)], (field, label) + # Fallback labels (the name with spaces) are the prompt's too. + assert ("first_home_mortgage_balance", False) in labels[ + F._label_key("first home mortgage balance") + ] + # Special renderers: the Marketplace plan line is labeled on its own. + line = _format_input_line("selected_marketplace_plan_benchmark_ratio", 0.8) + assert line.startswith("- selected Marketplace plan: ") + assert ("selected_marketplace_plan_benchmark_ratio", False) in labels[ + F._label_key("selected Marketplace plan") + ] + # The v2-pilot prompt's relabelings, as that prompt writes them. + for field, label in ( + ("spm_unit_energy_subsidy", "home energy assistance received"), + ( + "employer_sponsored_insurance_premiums", + "employer-paid health insurance premiums (paid by the employer, " + "not by the household)", + ), + ): + assert label in F.rendered_labels(field) + assert (field, False) in labels[F._label_key(label)] + assert ("state_code", False) in labels[F._label_key("state")] + assert ("first_home_mortgage_interest", True) in labels[ + F._label_key("first home mortgage interest") + ] + assert ("roth_401k_contributions_desired", False) in labels[ + F._label_key("Roth 401k contributions desired") + ] + assert ("partnership_s_corp_income", False) in labels[ + F._label_key("partnership or S-corp income") + ] + + +def test_labels_reads_no_label_that_names_two_of_the_engines_inputs(monkeypatch): + two = frozenset({("has_esi", False), ("age", False)}) + monkeypatch.setattr(F, "prompt_labels", lambda: {F._label_key("covered"): two}) + situation = _situation() + situation["people"]["head"]["covered"] = True + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.LABELS) + assert request is None and "unknown variable 'covered'" in str(error) + one = frozenset({("has_esi", False), ("uk_only_input", False)}) + monkeypatch.setattr(F, "prompt_labels", lambda: {F._label_key("covered"): one}) + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.LABELS) + assert error is None and request["situation"]["people"]["head"]["has_esi"] + + +def test_labels_ignores_a_fact_the_reference_leaves_out(): + situation = _situation() + situation["tax_units"] = { + "t": {"members": ["head", "kid"], "first home mortgage interest": 900} } - for field, label in relabeled.items(): - written = F.v2_pilot_text(f"- {field.replace('_', ' ')}: $5") - assert written == f"- {label}: $5" - assert labels[F._label_key(label)] == field - assert labels[F._label_key("roth 401k contributions")] == "roth_401k_contributions" + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.LABELS) + assert error is None, error + assert request["situation"]["tax_units"]["t"] == {"members": ["head", "kid"]} + assert any("leaves it out too" in n for n in request["notes"]) + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.NAMES) + assert request is None and "first home mortgage interest" in str(error) def _flat(**overrides): @@ -385,6 +436,74 @@ def test_flat_placement_gives_a_household_of_ones_person_fact_to_that_person(): assert any("whose facts list it" in p for p in error["problems"]) +def test_flat_refuses_facts_given_beside_the_person(): + people = [{"role": "head", "age": 40, "employment_income": 30000}] + request, error = _norm(_flat(people=people), F.FLAT) + assert request is None + assert any( + "takes only name, role and facts; give age, employment_income under facts" in p + for p in error["problems"] + ) + + +def test_flat_reads_a_fact_written_two_ways_once_whatever_the_order(): + for facts in ( + {"spm_unit_energy_subsidy": 900, "spm unit energy subsidy": 800}, + {"spm unit energy subsidy": 800, "spm_unit_energy_subsidy": 900}, + ): + request, error = _norm(_flat(household_facts=facts), F.FLAT, F.NAMES) + assert request is None + assert any("set twice with different values" in p for p in error["problems"]) + same = {"spm_unit_energy_subsidy": 900, "spm unit energy subsidy": {"2026": 900}} + request, error = _norm(_flat(household_facts=same), F.FLAT, F.NAMES) + assert error is None, error + + +def test_flat_roles_set_the_relationship_inputs_and_refuse_a_contradiction(): + for written in ("is_tax_unit_head", "is tax unit head"): + people = [{"name": "A", "role": "head", "facts": {written: False}}] + request, error = _norm(_flat(people=people), F.FLAT, F.NAMES) + assert request is None, written + assert any("contradicts the role head" in p for p in error["problems"]) + people = [{"name": "A", "role": "head", "facts": {written: True}}] + request, error = _norm(_flat(people=people), F.FLAT, F.NAMES) + assert error is None, (written, error) + head = request["situation"]["people"]["A"] + assert head["is_tax_unit_head"] == {"2026": True} + assert head["is_tax_unit_spouse"] == {"2026": False} + + +def test_flat_placement_compares_values_once_normalized(): + one = [ + {"name": "Solo", "role": "head", "facts": {"pre_subsidy_rent": {"2026": 9000}}} + ] + args = _flat(people=one, household_facts={"pre_subsidy_rent": 9000}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert error is None, error + assert request["situation"]["people"]["Solo"]["pre_subsidy_rent"] == {"2026": 9000} + + +def test_the_descriptions_promise_only_what_the_features_do(): + flat = F.calculate_tool_schema(features=frozenset({F.FLAT}))["function"] + assert '"employment_income": 30000' in flat["description"] + example = [ + {"role": "head", "facts": {"age": 40, "employment_income": 30000}}, + ] + request, error = _norm(_flat(people=example), F.FLAT) + assert error is None, error + entity_error = "a variable set on the wrong entity" + placed = F.calculate_tool_schema(features=frozenset({F.PLACEMENT}))["function"] + assert entity_error not in placed["description"] + assert "moved to the one unit holding its people" in placed["description"] + named = F.calculate_tool_schema(features=frozenset({F.NAMES}))["function"] + assert entity_error in named["description"] + concise = F.calculate_tool_schema(features=frozenset({F.EXPLAIN, F.CONCISE})) + detail = concise["function"]["parameters"]["properties"]["detail"]["description"] + assert f"up to {F.EXPLAIN_MAX_LINES} steps" in detail + for features in (frozenset({F.EXPLAIN, F.CONCISE}), frozenset(F.TOOL_FEATURES)): + assert "every step" not in F.compose_tool_notice(features, True) + + def test_flat_refuses_what_it_cannot_place(): people = [ {"name": "A", "role": "head", "facts": {}}, @@ -434,7 +553,7 @@ def _echo_worker(conn, fix_path, reform): conn.send(("ok", {"received": args})) -def test_the_pool_tells_its_workers_the_features(): +def test_the_pool_tells_its_workers_the_features_in_a_frame(): pool = F.ToolPool( size=1, timeout=5.0, @@ -443,13 +562,91 @@ def test_the_pool_tells_its_workers_the_features(): ) try: result, _ = pool.call({"x": 1}) - assert result == { - "received": {"__tool_features__": ["flat", "names"], "arguments": {"x": 1}} - } + assert result == {"received": ("calculate", ("flat", "names"), {"x": 1})} finally: pool.close() +def test_a_model_cannot_turn_on_a_feature_its_arm_does_not_offer(): + forged = { + "__tool_features__": ["names"], + "arguments": { + "situation": {"people": {"head": {"Employment income": 12345}}}, + "variables": ["age"], + }, + } + for message in (forged, {"__tool_version__": 2, "arguments": forged["arguments"]}): + request, error = F.worker_request(message, REGISTRY) + assert request is None + assert "unknown arguments: " in error["problems"][0] + request, error = F.worker_request( + F.tool_call_frame(frozenset({F.NAMES}), forged["arguments"]), REGISTRY + ) + assert error is None + assert request["situation"]["people"]["head"]["employment_income"] == { + "2026": 12345 + } + for names in (("telepathy",), ("concise",), ({},), ["names"]): + request, error = F.worker_request(("calculate", names, {}), REGISTRY) + assert request is None and error["error"] == "configuration", names + + +def _profile(arm): + return F.ModelProfile("m", "gpt-x", "chat", "tool", arm=arm) + + +def _record(profile, **changes): + record = { + "scenario_id": "mt_000", + "tool_notice_sha256": __import__("hashlib") + .sha256(profile.notice.encode()) + .hexdigest(), + **F.profile_treatment(profile), + } + record.update(changes) + return record + + +def test_a_resume_refuses_households_run_under_another_treatment(): + pe3 = _profile("pe3") + assert F.record_treatment_problem(_record(pe3), pe3) is None + six = sorted(set(F.TOOL_FEATURES) - {F.LABELS}) + assert "tool features" in F.record_treatment_problem( + _record(pe3, tool_features=six), pe3 + ) + assert "another tool notice" in F.record_treatment_problem( + _record(pe3, tool_notice_sha256="0" * 64), pe3 + ) + assert "calculator schema" in F.record_treatment_problem( + _record(pe3, calculate_tool_sha256="0" * 64), pe3 + ) + old = _record(pe3) + del old["tool_features"] + assert "no recorded tool features" in F.record_treatment_problem(old, pe3) + # Arms whose treatment never changed accept records from before the fields. + for arm in ("pe", "pe2", "web+pe", "none", "web"): + profile = _profile(arm) + legacy = _record(profile) + for key in ("tool_features", "calculate_tool_sha256", "tool_notice_sha256"): + legacy.pop(key) + assert F.record_treatment_problem(legacy, profile) is None, arm + pe2 = _profile("pe2") + assert "tool features" in F.record_treatment_problem(_record(_profile("pe")), pe2) + + +def test_diagnostics_read_flat_people_by_their_default_names(): + household = { + "people": [ + {"role": "head", "facts": {}}, + {"role": "spouse", "facts": {}}, + {"name": "Kid", "role": "dependent", "facts": {}}, + {"role": "dependent", "facts": {}}, + ] + } + names = F._person_names({"household": json.dumps(household)}) + assert names == ["head", "spouse", "Kid", "dependent_2"] + + @pytest.mark.slow def test_a_household_described_flat_computes_what_the_benchmarks_own_does(): """Differential: on the installed engine, each frozen-board household @@ -520,73 +717,72 @@ def test_a_household_described_flat_computes_what_the_benchmarks_own_does(): @pytest.mark.slow -def test_a_household_described_under_the_prompts_labels_computes_the_same(): - """Differential: each frozen-board household described flat with every - fact as its v2-pilot prompt writes it (a label in words, or the name with - spaces), - and stated hours only where the prompt states them, computes every board - output as the benchmark does. Take-up conventions, which the prompt does - not list, keep their engine names.""" +def test_a_household_described_from_its_prompt_computes_the_same(): + """Differential, with the labels feature alone: each of 30 frozen-board + households described flat from the facts its v2-pilot prompt shows, each + under the label the prompt shows it under (dropped facts included), with + the state and roles the prompt gives, computes every board output as the + benchmark's own situation does. The take-up conventions, which the prompt + states in words, are set under their engine names.""" import household_set as hs import pandas as pd from policyengine_us import Simulation from policybench.ground_truth import _extract_person_value, _pe_variable_for_output - from policybench.prompts import _humanize_input_label - from policybench.scenarios import PE_INPUT_ALIASES, PE_INPUT_RENAMES + from policybench.scenarios import ( + DEFAULT_TAKEUP_INPUTS, + is_excluded_prompt_input_name, + ) system = F.build_tool_system(reform=True) registry = F.build_registry(system) reference = pd.read_csv(hs.FROZEN_BOARD / "reference_outputs.csv") variables = reference.groupby("scenario_id").variable.agg(list) - field_of = {name: field for field, name in PE_INPUT_RENAMES.items()} - dropped = {"is_tax_unit_head", "is_tax_unit_spouse", *PE_INPUT_ALIASES.values()} + shown = set() - worded = [] - - def labeled(facts): - out = {} - for k, v in facts.items(): - if k in dropped: - continue - if k.startswith(F.CONVENTION_PREFIXES): - out[k] = v - continue - label = F.v2_pilot_text(f"- {_humanize_input_label(field_of.get(k, k))}: ")[ - 2:-2 - ] - if label != k.replace("_", " "): - worded.append(label) - out[label] = v - return out + def label(field, value): + value_type = ( + "bool" + if isinstance(value, bool) + else ("str" if isinstance(value, str) else "float") + ) + written = F.rendered_labels(field, value_type)[-1] + shown.add((field, written)) + return written + + def prompt_facts(inputs): + return { + label(k, v): v + for k, v in inputs.items() + if not is_excluded_prompt_input_name(k) + } checked = 0 - for scenario in hs.load_scenarios(hs.FROZEN_BOARD)[:12]: - situation = hs.situation_for(scenario) + for scenario in hs.load_scenarios(hs.FROZEN_BOARD)[:30]: couple = scenario.marital_couple() or () head = couple[0] if couple else scenario.adults[0].name - people = [ - { - "name": name, - "role": "head" - if name == head - else ("spouse" if name in couple else "dependent"), - "facts": labeled(facts), - } - for name, facts in situation["people"].items() - ] + people = [] + for person in scenario.all_people: + facts = {"age": person.age} + if person.name.startswith("adult") or abs(person.employment_income) > 1e-6: + facts[label("employment_income", 1.0)] = person.employment_income + facts |= prompt_facts(person.inputs) + facts |= DEFAULT_TAKEUP_INPUTS["person"] + role = ( + "head" + if person.name == head + else ("spouse" if person.name in couple else "dependent") + ) + people.append({"name": person.name, "role": role, "facts": facts}) household_facts = {} - for group in ("tax_units", "spm_units", "families", "households"): - for unit in situation[group].values(): - household_facts.update( - labeled( - { - k: v - for k, v in unit.items() - if k not in ("members", "state_code") - } - ) - ) + for inputs in ( + scenario.tax_unit_inputs, + scenario.spm_unit_inputs, + scenario.household_inputs, + ): + household_facts |= prompt_facts(inputs) + for entity in ("tax_unit", "spm_unit", "household"): + household_facts |= DEFAULT_TAKEUP_INPUTS[entity] household = { "state": scenario.state, "people": people, @@ -595,10 +791,12 @@ def labeled(facts): request, error = F.normalize_request( {"household": json.dumps(household), "variables": ["snap"]}, registry, - features=frozenset({F.FLAT, F.LABELS, F.NAMES}), + features=frozenset({F.FLAT, F.LABELS}), ) assert error is None, (scenario.id, error) - own = Simulation(tax_benefit_system=system, situation=situation) + own = Simulation( + tax_benefit_system=system, situation=hs.situation_for(scenario) + ) flat = Simulation(tax_benefit_system=system, situation=request["situation"]) for variable in variables[scenario.id]: name = _pe_variable_for_output(variable, "us") @@ -612,4 +810,12 @@ def labeled(facts): ] assert values[0] == values[1], (scenario.id, variable, values) checked += 1 - assert checked > 120 and len(worded) > 10, (checked, worded) + worded = {w for f, w in shown if w != f.replace("_", " ")} + assert checked > 300 and len(worded) > 10, (checked, sorted(worded)) + # No label a US prompt can show names two of this engine's inputs. + ambiguous = { + key: targets + for key, targets in F.prompt_labels().items() + if len({e for e, dropped in targets if not dropped and e in registry}) > 1 + } + assert not ambiguous, ambiguous From d570b8e7af46a99f0dd4110be2ec8ea9147de835 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 20:09:47 -0400 Subject: [PATCH 14/64] Tool v3 review r2: report malformed repeats, resumable harness errors - flat: a household fact repeated with a malformed value was dropped without a word. _same_value now reports the repeat's validation problems, and only a repeat validated as equal is dropped quietly. A malformed first value is reported once, by normal validation, and never as a conflict. - The fallback record for a household the harness failed on carried no treatment, so one failure blocked resuming any custom arm. The record is built by harness_error_record, which now includes the notice hash and profile_treatment. resume_treatment_problems checks only the records a resume keeps, since a rerunnable record is replaced under the current profile. pe, web+pe and pe2 still behave exactly as at 5bbd571a: 4,000 calls and 2,000 pool messages match. 5 mutants of these fixes were all caught. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 102 +++++++++++++++++++++++++----------- tests/test_tool_v3.py | 54 +++++++++++++++++++ 2 files changed, 126 insertions(+), 30 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 5f586522..b7b66b72 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -1280,14 +1280,29 @@ def flat_names(people) -> list: return names -def _same_value(registry, name: str, a, b, period: str) -> bool | None: - """Whether two values of an input are the same once normalized (None when - either is malformed; normalization reports it).""" - scratch: list[str] = [] +def _same_value( + registry, + name: str, + a, + b, + period: str, + problems: list[str] | None = None, + where: str = "", +) -> bool | None: + """Whether two values of an input are the same once normalized, or None + when either is malformed. ``a`` stays in the situation, so normalization + reports it later; a caller that drops ``b`` passes ``problems`` so a + malformed ``b`` is reported, not lost.""" info = registry[name] - first = _normalize_value("", name, a, info, period, scratch) - second = _normalize_value("", name, b, info, period, scratch) - return None if scratch else first == second + first_problems: list[str] = [] + second_problems: list[str] = [] + first = _normalize_value(where, name, a, info, period, first_problems) + second = _normalize_value(where, name, b, info, period, second_problems) + if problems is not None: + problems.extend(second_problems) + if first_problems or second_problems: + return None + return first == second def flat_situation( @@ -1460,7 +1475,12 @@ def engine_name(name): else: target = unit_of[entity] if name in target: - if _same_value(registry, resolved, target[name], value, period) is False: + # The repeat is dropped, so only one validated as equal is kept quiet. + same = _same_value( + registry, resolved, target[name], value, period, problems, + "household_facts", + ) + if same is False: problems.append( f"household_facts: {resolved} is set twice with different values" ) @@ -3792,6 +3812,46 @@ def record_treatment_problem(record: dict, profile: "ModelProfile") -> str | Non return None +def resume_treatment_problems( + previous: Mapping[str, dict], done: set[str], profile: "ModelProfile" +) -> list[str]: + """record_treatment_problem for each record a resume keeps. A rerunnable + record (RERUNNABLE_STATUSES) is replaced under this profile, so it cannot + mix treatments and does not block the resume.""" + return [ + problem + for sid, record in previous.items() + if sid in done + and (problem := record_treatment_problem(record, profile)) is not None + ] + + +def harness_error_record( + sid: str, variables: list[str], profile: "ModelProfile", error: Exception +) -> dict: + """The record of a household the harness itself failed on: rerunnable, + with every setup and treatment key a resume and score check, so a folder + holding it still resumes and scores.""" + record = { + "scenario_id": sid, + "model": profile.name, + "arm": profile.arm, + "prompt_version": profile.prompt_version, + "prompt_wording_sha256": prompt_wording_sha256(profile), + "benchmark_search_rule": profile.benchmark_search_rule, + "tool_notice_sha256": hashlib.sha256(profile.notice.encode()).hexdigest(), + **profile_treatment(profile), + "variables": variables, + "status": "harness_error", + "error": f"{type(error).__name__}: {error}"[:MAX_MESSAGE_CHARS], + "turns": [], + "predictions": {v: None for v in variables}, + "explanations": {v: None for v in variables}, + } + record["totals"] = household_totals(record) + return record + + def records_arm(records: Mapping[str, dict]) -> str: """The one arm a run directory's records belong to; records written before the web arms are the PE tool's.""" @@ -3850,11 +3910,7 @@ def command_run(args) -> None: f"{households_dir} holds records sent under other v2-pilot wording " f"({records_wording(previous)}); use a fresh --out-dir" ) - mixed = [ - problem - for record in previous.values() - if (problem := record_treatment_problem(record, profile)) is not None - ] + mixed = resume_treatment_problems(previous, done, profile) if mixed: raise SystemExit( f"{households_dir} holds records from another treatment (" @@ -3959,23 +4015,9 @@ def one(sid: str) -> dict: profile=profile, ) except Exception as error: # one household must not end the run - record = { - "scenario_id": sid, - "model": profile.name, - # The setup keys a resume and score check, so a folder with - # this record still resumes and scores. - "arm": profile.arm, - "prompt_version": profile.prompt_version, - "prompt_wording_sha256": prompt_wording_sha256(profile), - "benchmark_search_rule": profile.benchmark_search_rule, - "variables": board["variables"][sid], - "status": "harness_error", - "error": f"{type(error).__name__}: {error}"[:MAX_MESSAGE_CHARS], - "turns": [], - "predictions": {v: None for v in board["variables"][sid]}, - "explanations": {v: None for v in board["variables"][sid]}, - } - record["totals"] = household_totals(record) + record = harness_error_record( + sid, board["variables"][sid], profile, error + ) if sid in previous: prior = previous[sid] record["prior_attempts"] = prior.get("prior_attempts", []) + [ diff --git a/tests/test_tool_v3.py b/tests/test_tool_v3.py index 4b1e9c99..dcf6317a 100644 --- a/tests/test_tool_v3.py +++ b/tests/test_tool_v3.py @@ -819,3 +819,57 @@ def prompt_facts(inputs): if len({e for e, dropped in targets if not dropped and e in registry}) > 1 } assert not ambiguous, ambiguous + + +def test_flat_reports_a_malformed_repeat_of_a_fact_it_drops(): + # Review r2: a repeat that fails validation was dropped without a word. + one = [{"name": "Solo", "role": "head", "facts": {"employment_income": 30000}}] + for bad in ("40000", [40000], {"26": 40000}, {}): + args = _flat(people=one, household_facts={"employment_income": bad}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert request is None, bad + assert any("employment_income" in m for m in error["problems"]), (bad, error) + # An equal repeat, scalar or by period, still merges quietly; a different + # one is still refused. + for same in (30000, {"2026": 30000}): + args = _flat(people=one, household_facts={"employment_income": same}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert error is None, (same, error) + args = _flat(people=one, household_facts={"employment_income": 40000}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert request is None + assert any("set twice with different values" in m for m in error["problems"]) + + +def test_a_harness_error_record_resumes_under_its_own_arm(): + # Review r2: the fallback record carried no treatment, so one harness + # failure blocked every custom-arm resume. + for arm in ("pe3", "pe:labels+flat", "pe", "pe2", "web+pe", "none", "web"): + profile = _profile(arm) + record = F.harness_error_record("mt_000", ["snap"], profile, RuntimeError("x")) + assert record["status"] in F.RERUNNABLE_STATUSES + assert F.record_treatment_problem(record, profile) is None, arm + assert record["predictions"] == {"snap": None} + assert record["totals"]["cost_usd"] == 0 + + +def test_a_resume_checks_the_treatment_only_of_records_it_keeps(): + pe3 = _profile("pe3") + stale = _record(pe3, tool_features=["names"]) + previous = { + "mt_000": dict(stale, status="complete"), + "mt_001": dict(stale, scenario_id="mt_001", status="harness_error"), + } + # Kept, so refused. + assert len(F.resume_treatment_problems(previous, {"mt_000"}, pe3)) == 1 + # Rerun under this profile, so no mixing and no block. + assert F.resume_treatment_problems(previous, set(), pe3) == [] + + +def test_flat_reports_a_malformed_first_value_once_and_not_as_a_conflict(): + one = [{"name": "Solo", "role": "head", "facts": {"employment_income": "30000"}}] + args = _flat(people=one, household_facts={"employment_income": 40000}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert request is None + assert not any("set twice" in m for m in error["problems"]), error + assert sum("employment_income" in m for m in error["problems"]) == 1, error From fb35d8273b0d1f6fd5df1aea76b6e22d15af0e8f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 20:45:58 -0400 Subject: [PATCH 15/64] Format the r2 fix Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index b7b66b72..abab108f 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -1477,7 +1477,12 @@ def engine_name(name): if name in target: # The repeat is dropped, so only one validated as equal is kept quiet. same = _same_value( - registry, resolved, target[name], value, period, problems, + registry, + resolved, + target[name], + value, + period, + problems, "household_facts", ) if same is False: From 6b00a6f0c4997e73b9968b412b57646957cad631 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 01:29:28 -0400 Subject: [PATCH 16/64] Tool v3 review r3: check every record's treatment, at resume and at score A rerun is not sure to replace a rerunnable record: a budget stop, lost workers or a narrower --scenarios leave it in place, and scoring read whatever was there. treatment_problems now checks every record in the folder, rerunnable ones included, and command_score refuses a folder that mixes treatments, rebuilding the notices on the board's engine. Runs written before 6e1578fc (test batch 0's frontier calculator arms, b6d835a8) were told policyengine-us 2.41.0 while computing on 2.60.4. A folder written wholly so is one treatment and scores with a note; a mix of the two texts is refused, and a resume into it stays refused, since its new records would state the board's engine. All 39 run folders score (628 records); the six frontier calculator folders take the note. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 63 ++++++++++++--- tests/test_tool_v3.py | 154 ++++++++++++++++++++++++++++++++++-- 2 files changed, 201 insertions(+), 16 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index abab108f..f09e0b6f 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -48,6 +48,7 @@ import argparse import concurrent.futures +import contextlib import copy import decimal import difflib @@ -343,6 +344,18 @@ def set_stated_engine(version: str) -> None: _STATED_ENGINE["version"] = version +@contextlib.contextmanager +def stating_engine(version: str): + """The texts as a run on `version` sent them, for a check that rebuilds a + notice; the stated engine is restored afterwards.""" + before = _STATED_ENGINE["version"] + set_stated_engine(version) + try: + yield + finally: + set_stated_engine(before) + + def engine_text(text: str) -> str: return text.replace( f"policyengine-us {REFERENCE_ENGINE}", @@ -3817,20 +3830,43 @@ def record_treatment_problem(record: dict, profile: "ModelProfile") -> str | Non return None -def resume_treatment_problems( - previous: Mapping[str, dict], done: set[str], profile: "ModelProfile" +def treatment_problems( + records: Mapping[str, dict], profile: "ModelProfile" ) -> list[str]: - """record_treatment_problem for each record a resume keeps. A rerunnable - record (RERUNNABLE_STATUSES) is replaced under this profile, so it cannot - mix treatments and does not block the resume.""" + """record_treatment_problem for every record in a run directory, a + rerunnable one (RERUNNABLE_STATUSES) too. A rerun is not sure to replace + it: a budget stop, lost workers or a narrower --scenarios leave it in + place, and scoring reads whatever is there (review r3). So a resume and + a score both refuse a directory holding any record from another treatment.""" return [ problem - for sid, record in previous.items() - if sid in done - and (problem := record_treatment_problem(record, profile)) is not None + for record in records.values() + if (problem := record_treatment_problem(record, profile)) is not None ] +def scored_treatment_problems( + records: Mapping[str, dict], profile: "ModelProfile", engine: str +) -> tuple[list[str], str | None]: + """treatment_problems as score applies it, with the note score prints. + The records' notices state the board's engine, as command_run set it. + Runs before the texts stated the board's engine (6e1578fc) named the + reference engine whatever the board's: a directory written wholly so is + one treatment and scores as it ran. A mix of the two texts does not. A + resume stays strict, since its new records would state the board's.""" + with stating_engine(engine): + problems = treatment_problems(records, profile) + if not problems or engine == REFERENCE_ENGINE: + return problems, None + with stating_engine(REFERENCE_ENGINE): + if treatment_problems(records, profile): + return problems, None + return [], ( + f"NOTE: these records' notices name policyengine-us {REFERENCE_ENGINE}, " + f"as runs before 6e1578fc stated it; the board's engine is {engine}" + ) + + def harness_error_record( sid: str, variables: list[str], profile: "ModelProfile", error: Exception ) -> dict: @@ -3915,7 +3951,7 @@ def command_run(args) -> None: f"{households_dir} holds records sent under other v2-pilot wording " f"({records_wording(previous)}); use a fresh --out-dir" ) - mixed = resume_treatment_problems(previous, done, profile) + mixed = treatment_problems(previous, profile) if mixed: raise SystemExit( f"{households_dir} holds records from another treatment (" @@ -4148,6 +4184,15 @@ def command_score(args) -> None: profile = model_profile( records_model(records), records_arm(records), *records_prompt(records) ) + mixed, note = scored_treatment_problems(records, profile, board_engine(snapshot)) + if note: + print(note) + if mixed: + raise SystemExit( + f"{out / 'households'} holds records from another treatment (" + + "; ".join(mixed[:3]) + + "); scoring would mix them" + ) arm1_label, arm2_label = profile.arm1_label, profile.arm2_label complete = sorted( sid for sid, r in records.items() if r.get("status") == "complete" diff --git a/tests/test_tool_v3.py b/tests/test_tool_v3.py index dcf6317a..71bc73c3 100644 --- a/tests/test_tool_v3.py +++ b/tests/test_tool_v3.py @@ -853,17 +853,157 @@ def test_a_harness_error_record_resumes_under_its_own_arm(): assert record["totals"]["cost_usd"] == 0 -def test_a_resume_checks_the_treatment_only_of_records_it_keeps(): +def test_a_rerunnable_record_from_another_treatment_is_refused_too(): + # Review r3: a rerun is not sure to replace a rerunnable record (a budget + # stop, lost workers, a narrower --scenarios), so every record is checked. pe3 = _profile("pe3") stale = _record(pe3, tool_features=["names"]) - previous = { + records = { "mt_000": dict(stale, status="complete"), - "mt_001": dict(stale, scenario_id="mt_001", status="harness_error"), + "mt_001": dict(stale, scenario_id="mt_001", status="provider_error"), + "mt_002": F.harness_error_record("mt_002", ["snap"], pe3, RuntimeError("x")), } - # Kept, so refused. - assert len(F.resume_treatment_problems(previous, {"mt_000"}, pe3)) == 1 - # Rerun under this profile, so no mixing and no block. - assert F.resume_treatment_problems(previous, set(), pe3) == [] + problems = F.treatment_problems(records, pe3) + assert [p.split()[0] for p in problems] == ["mt_000", "mt_001"] + + +SIX = ("concise", "explain", "flat", "names", "overrides", "placement") + + +def _scored_record(sid, profile, treatment_of=None, status="complete", **changes): + """A household record as command_run writes it, with the treatment of + `treatment_of` (another arm's profile) when given.""" + import hashlib + + board = pilot.load_board() + reference = board["reference"].set_index(["scenario_id", "variable"])["value"] + variables = board["variables"][sid] + source = treatment_of or profile + record = { + "scenario_id": sid, + "model": profile.name, + "arm": profile.arm, + "prompt_version": profile.prompt_version, + "benchmark_search_rule": profile.benchmark_search_rule, + "tool_notice_sha256": hashlib.sha256(source.notice.encode()).hexdigest(), + **F.profile_treatment(source), + "variables": variables, + "predictions": {v: float(reference[(sid, v)]) for v in variables}, + "explanations": {v: "x" for v in variables}, + "status": status, + "turns": [{"total_cost_usd": 1.25, "calls": []}], + "cap_hit": False, + "wall_seconds": 1.0, + } + record.update(changes) + record["totals"] = F.household_totals(record) + return record + + +def _write(directory, *records): + (directory / "households").mkdir(parents=True, exist_ok=True) + for record in records: + path = directory / "households" / f"{record['scenario_id']}.json" + path.write_text(json.dumps(record)) + + +def test_a_resume_refuses_an_old_treatments_provider_error_before_running( + tmp_path, monkeypatch +): + # The review's case: a pe3 folder holding a provider_error written under + # six features, with the budget already spent. Before the fix the resume + # passed, `one` stopped on the budget without replacing it, and scoring + # read its predictions as pe3's. + import argparse + + pe3 = F.model_profile(F.MODEL_NAME, "pe3") + six = F.model_profile(F.MODEL_NAME, "pe:" + "+".join(SIX)) + _write(tmp_path, _scored_record("scenario_000", pe3, six, status="provider_error")) + monkeypatch.setattr(F, "_require_engine", lambda board: None) + monkeypatch.setenv(F.provider_key(pe3.model_id), "test-key") + + def no_pool(**kwargs): + raise AssertionError("the run got as far as starting the calculator") + + monkeypatch.setattr(F, "ToolPool", no_pool) + args = argparse.Namespace( + out_dir=str(tmp_path), + board=None, + model=F.MODEL_NAME, + arm="pe3", + prompt="v1", + forbid_benchmark_search=False, + scenarios=None, + limit=None, + workers=1, + budget_usd=1.0, + reserve_usd=0.0, + ) + with pytest.raises(SystemExit, match="another treatment"): + F.command_run(args) + assert not list(tmp_path.glob("manifest-*.json")) + + +def test_score_refuses_a_folder_mixing_treatments(tmp_path): + import argparse + + pe3 = F.model_profile(F.MODEL_NAME, "pe3") + six = F.model_profile(F.MODEL_NAME, "pe:" + "+".join(SIX)) + _write( + tmp_path, + _scored_record("scenario_000", pe3), + _scored_record("scenario_001", pe3, six, status="provider_error"), + ) + args = argparse.Namespace(out_dir=str(tmp_path), complete_only=False) + with pytest.raises(SystemExit, match="another treatment"): + F.command_score(args) + assert not (tmp_path / "score.json").exists() + # The same folder without the stray record scores. + (tmp_path / "households" / "scenario_001.json").unlink() + F.command_score(args) + assert json.loads((tmp_path / "score.json").read_text())["households"] == 1 + + +def test_score_rebuilds_notices_on_the_boards_engine_and_restores_it( + tmp_path, monkeypatch +): + # A household set's records state its engine (2.60.4) in their notices and + # schema, so a check on the reference engine's text would refuse them all. + import argparse + + monkeypatch.setattr(F, "board_engine", lambda snapshot=None: "2.60.4") + pe3 = F.model_profile(F.MODEL_NAME, "pe3") + with F.stating_engine("2.60.4"): + current = _scored_record("scenario_000", pe3) + _write(tmp_path, current) + args = argparse.Namespace(out_dir=str(tmp_path), complete_only=False) + F.command_score(args) + assert F._STATED_ENGINE["version"] == F.REFERENCE_ENGINE + # Beside it, a record sent under the reference engine's text mixes two. + _write(tmp_path, _scored_record("scenario_001", pe3)) + with pytest.raises(SystemExit, match="another tool notice"): + F.command_score(args) + assert F._STATED_ENGINE["version"] == F.REFERENCE_ENGINE + + +def test_score_takes_a_folder_written_wholly_before_the_texts_stated_the_engine( + tmp_path, monkeypatch, capsys +): + # Test batch 0's frontier calculator arms (b6d835a8) were told 2.41.0 while + # the board and calculator ran 2.60.4: one treatment, scored as it ran. A + # resume into it is still refused, since new records would say 2.60.4. + import argparse + + monkeypatch.setattr(F, "board_engine", lambda snapshot=None: "2.60.4") + pe = F.model_profile(F.MODEL_NAME, "pe") + _write( + tmp_path, _scored_record("scenario_000", pe), _scored_record("scenario_001", pe) + ) + F.command_score(argparse.Namespace(out_dir=str(tmp_path), complete_only=False)) + assert "name policyengine-us 2.41.0" in capsys.readouterr().out + assert json.loads((tmp_path / "score.json").read_text())["households"] == 2 + with F.stating_engine("2.60.4"): + assert len(F.treatment_problems(F.load_records(tmp_path), pe)) == 2 def test_flat_reports_a_malformed_first_value_once_and_not_as_a_conflict(): From 6e15d713b0a9a095f5a00d63a331abbaadc5966c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 02:48:28 -0400 Subject: [PATCH 17/64] Confirmatory runner: lanes, one engine pool per model, admission, the ceiling scripts/confirmatory_run.py runs the frozen pre-registration's run procedure on the treatment harness (6b00a6f0's tool_pilot_argon.py, unchanged) and writes the run records the confirmatory report checks: - two lanes per model, households alternating in draw order, each household's arms in its drawn order, then reruns in passes; - one ToolPool per model with no features of its own, each call framed with its arm's features; - admission against each model's cap and reserve, waiting while a pass's reruns are undecided; - the hard ceiling over every model's spend, through the harness's Spend; - the failure classes and actions, with operator pauses, harness and inconclusive stops, and replaced attempts saved before their reruns; - the 30-second host log and the guard's and power log's lines. Tests follow. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 1388 +++++++++++++++++++++++++++++++++++ 1 file changed, 1388 insertions(+) create mode 100644 scripts/confirmatory_run.py diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py new file mode 100644 index 00000000..cc61b52b --- /dev/null +++ b/scripts/confirmatory_run.py @@ -0,0 +1,1388 @@ +"""The confirmatory calculator-interface run. + +Usage: + python scripts/confirmatory_run.py run DESIGN [--workers N] [--dry-run] + python scripts/confirmatory_run.py resume DESIGN MODEL [--note TEXT] + +Runs the frozen pre-registration's "Run procedure" and writes its "Run +records", which the confirmatory report checks before it scores anything. +DESIGN is the JSON file written at the freeze, the file the report reads. + +- **Lanes.** Each model has two lanes. Households alternate between them in + draw order, the first drawn to lane 1. A lane runs one household's arms one + after another, in the order drawn for that household and model (the stream + ``order/``), then its reruns in passes (second attempts before + third, each pass in the household's order), and only then decides its next + household. The six models run at once. +- **One engine pool per model**, started once, with no features of its own. + Each call carries its arm's features in a call frame (``tool_call_frame``), + the message a pool started with those features would send. +- **Admission.** A lane admits its next household when the spend of the + model's finished households (every attempt, replaced ones included) plus + the reserve for each household under way, this one included, is at most + the model's cap plus $1e-9. A household is finished when its last attempt + has ended. After one refusal every later decision is a refusal. A decision + waits while an operator pause holds, and while any of the model's + households has ended a pass whose reruns are undecided, so that whether it + has finished is known. +- **Spend.** No conversation is stopped for spend, except at the hard + ceiling: the overall cap plus $25, over the spend before the calculator + stage and every model's spend since. At the crossing, no attempt starts, + the conversations under way end ``budget_stop`` at their next turn, and + every model's run log gets a ``ceiling`` event at the crossing's time. +- **Failures.** Each attempt's class follows the pre-registration's table, in + the precedence the report's ``classify`` uses, from its record and the host + evidence for its start and end; its action follows from its class and its + attempt number. An operator failure pauses the model's lanes until + ``resume``. A harness failure stops them, and so does one inconclusive + error that ends runs in more than five of the model's households. A + replaced attempt is saved, before its rerun, as + ``households/attempts/-.json``. +- **The host log.** Every 30 seconds: the time, the load average and the + kernel's memory-pressure level. At the end, the guard's and the power log's + lines from a day before the booking to its end. An attempt's class waits + for its host evidence to be complete: a sample more than 60 seconds after + the pass's last end, or no sample for more than 90 seconds. + +The records' format is the confirmatory report's (its module docstring). The +report recomputes every class, action, order and admission decision from the +records, with its own copy of these rules; the study's differential test runs +this runner on fakes and checks that its records pass the report. + +The runner runs the treatment harness's code. It refuses to start unless its +``tool_pilot_argon.py``, ``policybench`` package and fix module are the +treatment commit's (6b00a6f0), its tree is clean at the design's run commit, +the machine is on AC power with the lid open, and the runs folder holds no +records: a run is never resumed. An interrupt (SIGINT or SIGTERM) stops every +model's lanes with a stop event and lets the conversations under way end; a +second interrupt exits at once. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import shutil +import signal +import subprocess +import sys +import threading +import time +import traceback +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Callable +from zoneinfo import ZoneInfo + +import numpy as np + +# The harness and policybench come from this checkout, whatever else is on the +# path: the preflight refuses any other. +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT), str(ROOT / "scripts")] + +import tool_pilot_argon as pilot # noqa: E402 + +TREATMENT_COMMIT = "6b00a6f0c4997e73b9968b412b57646957cad631" +# The treatment's code, which must be the treatment commit's. +TREATMENT_PATHS = ( + "scripts/tool_pilot_argon.py", + "policybench", + "reference_audit/2026-09-28/fixes", +) +PROMPT_VERSION = "v2-pilot" +CHEAP = ("claude-haiku-5.5", "gpt-6-luna", "gemini-3.5-flash-lite") +FRONTIER = ("gpt-6-sol", "claude-opus-5.5", "gemini-4-argon") +ARMS = ( + "pe", + "pe3", + "pe3-b", + "loo-labels", + "loo-names", + "loo-flat", + "loo-labels-flat", +) +FRONTIER_ARMS = ("pe", "pe3") +# The features each label drops from pe3; None for the tool v1 arm. +DROPPED = { + "pe": None, + "pe3": (), + "pe3-b": (), + "loo-labels": ("labels",), + "loo-names": ("names",), + "loo-flat": ("flat",), + "loo-labels-flat": ("labels", "flat"), +} +HOUSEHOLDS = 120 +MAX_RERUNS = 2 +INCONCLUSIVE_STOP = 5 # households of a model one inconclusive error may end +PRESSURE_WINDOW = 60.0 # seconds +HOST_LOG_GAP = 90.0 # seconds +SAMPLE_SECONDS = 30.0 +CEILING_MARGIN = 25.0 # dollars above the overall cap +ADMISSION_SLACK = 1e-9 # dollars +EXCERPT_LEAD = 86_400.0 # host logs are kept from a day before the booking +WAIT_SECONDS = 1.0 # how often a waiting lane looks again + +# The pre-registration's error classes (its "Failures" table). +TRANSPORT = frozenset( + { + "APIConnectionError", + "APIError", + "APIResponseValidationError", + "BadGatewayError", + "InternalServerError", + "RateLimitError", + "ServiceUnavailableError", + "Timeout", + "RequestWallTimeoutError", + } +) +REQUEST = frozenset( + { + "BadRequestError", + "ContextWindowExceededError", + "InvalidRequestError", + "UnprocessableEntityError", + "UnsupportedParamsError", + } +) +ACCOUNT = frozenset({"AuthenticationError", "PermissionDeniedError", "NotFoundError"}) +HOST_ENGINE = frozenset({"MemoryError", "OSError"}) +RERUN = frozenset({"provider", "host", "none", "operator"}) +FINISHED = frozenset({"complete", "incomplete", "turn_limit"}) +# Classes decided without the host evidence: they come before host trouble. +EARLY = frozenset({"operator", "harness", "ceiling", "none"}) + +REQUIRED = ( + "preregistration", + "frozen_sha256", + "runs", + "stage", + "board", + "engine", + "eligible_pool", + "draw_order", + "arms", + "frontier_arms", + "treatment_harness", + "run_commit", + "caps", + "reserves", + "overall_cap", + "spent_before", +) + + +# ---------------------------------------------------------------- streams + + +def master_seed(frozen_sha256: str) -> int: + if not re.fullmatch(r"[0-9a-f]{64}", frozen_sha256 or ""): + raise ValueError(f"not a sha256: {frozen_sha256!r}") + return int(frozen_sha256[:16], 16) + + +def stream(master: int, name: str) -> np.random.Generator: + key = int.from_bytes(hashlib.sha256(name.encode()).digest()[:8], "big") + return np.random.default_rng(np.random.SeedSequence(master, spawn_key=(key,))) + + +def draw(master: int, pool_ids: list[str], n: int | None = None) -> list[str]: + ids = np.array(sorted(pool_ids)) + size = HOUSEHOLDS if n is None else n + return [ + str(s) for s in stream(master, "draw").choice(ids, size=size, replace=False) + ] + + +def orders( + master: int, model: str, draw_order: list[str], arms: tuple[str, ...] +) -> dict[str, list[str]]: + """Each drawn household's arm order for a model, drawn for every household + in draw order, so no order depends on which households run.""" + generator = stream(master, f"order/{model}") + return { + sid: [str(a) for a in generator.permutation(list(arms))] for sid in draw_order + } + + +# ---------------------------------------------------------------- arms + + +def arm_string(label: str) -> str: + """The harness's arm string for a design label (`TOOL_FEATURES` order).""" + dropped = DROPPED[label] + if dropped is None: + return "pe" + if not dropped: + return "pe3" + return "pe:" + "+".join(f for f in pilot.TOOL_FEATURES if f not in dropped) + + +def arm_profile(model: str, label: str) -> pilot.ModelProfile: + """The treatment harness's profile for a model and design label; refused + unless its calculator features are the ones the label registers.""" + profile = pilot.model_profile( + model, arm=arm_string(label), prompt_version=PROMPT_VERSION + ) + dropped = DROPPED[label] + want = set() if dropped is None else set(pilot.TOOL_FEATURES) - set(dropped) + if not profile.uses_calculator or set(profile.tool_features) != want: + raise ValueError( + f"{model}/{label}: the harness gives features " + f"{sorted(profile.tool_features)}, not {sorted(want)}" + ) + return profile + + +# ---------------------------------------------------------------- classes + + +def error_class(record: dict) -> str: + return str(record.get("error") or "").split(":", 1)[0].strip() + + +def completed_turns(record: dict) -> list: + return [t for t in record.get("turns") or [] if not t.get("truncated_and_retried")] + + +def call_failures(record: dict) -> tuple[list[str], list[str]]: + """(the inconclusive tool failures, the model's engine exceptions) in its + calls.""" + host, model = [], [] + for turn in record.get("turns") or []: + for call in turn.get("calls") or []: + result = call.get("result") + if not isinstance(result, dict): + continue + error, message = result.get("error"), str(result.get("message") or "") + name = message.split(":", 1)[0] + if error == "timeout": + host.append("tool timeout") + elif error == "engine" and message.startswith("tool worker failed:"): + host.append("tool worker failure") + elif error == "engine": + (host if name in HOST_ENGINE else model).append(name) + for value in (result.get("results") or {}).values(): + if isinstance(value, dict) and value.get("error"): + name = str(value["error"]).split(":", 1)[0] + (host if name in HOST_ENGINE else model).append(name) + return host, model + + +def classify(record: dict, start: float, end: float, host) -> tuple[str, str]: + """(class, detail) of one attempt. In order of precedence: operator (an + account class or insufficient_quota); harness (a request class with no + model request yet returned); ceiling (budget_stop); none (no model request + returned); host (host trouble during the attempt); provider (a transport + class); model (a request class); inconclusive (any other provider error, + any harness error, a tool timeout, a worker failure, MemoryError or + OSError); otherwise ok, or model when the engine raised on a household the + model built. `host` is None to decide only the classes before host.""" + status, error = record.get("status"), str(record.get("error") or "") + name = error_class(record) + reached = bool(completed_turns(record)) + if status == "provider_error" and ( + name in ACCOUNT or "insufficient_quota" in error + ): + return "operator", name + if status == "provider_error" and name in REQUEST and not reached: + return "harness", name + if status == "budget_stop": + return "ceiling", status + if not reached: + return "none", f"{status}: {name}" if name else str(status) + trouble = host.trouble(start, end) if host is not None else None + if trouble: + return "host", trouble + if status == "provider_error": + if name in TRANSPORT: + return "provider", name + if name in REQUEST: + return "model", name + return "inconclusive", name or "provider_error" + if status == "harness_error": + return "inconclusive", error or "harness_error" + tool, engine = call_failures(record) + if tool: + return "inconclusive", tool[0] + if status not in FINISHED: + return "inconclusive", f"status {status}" + return ("model", engine[0]) if engine else ("ok", str(status)) + + +def action(cls: str, attempt: int) -> str: + if cls == "harness": + return "stop" + if cls == "ceiling": + return "stopped" + if cls in RERUN and attempt <= MAX_RERUNS: + return "pause-rerun" if cls == "operator" else "rerun" + return "stands" + + +def attempt_cost(record: dict) -> float: + return float((record.get("totals") or {}).get("cost_usd") or 0.0) + + +# ---------------------------------------------------------------- host evidence + + +GUARD_LINE = re.compile(r"^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d) \[[^\]]*\] (.*)$") +POWER_LINE = re.compile(r"^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d [+-]\d{4}) ([^\t]*?)\s*\t") +GUARD_LOG = Path.home() / "Library/Logs/clamshell-guard.log" + + +def guard_time(line: str, zone: ZoneInfo) -> float | None: + match = GUARD_LINE.match(line) + if not match: + return None + stamp = datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S") + return stamp.replace(tzinfo=zone).timestamp() + + +def power_time(line: str) -> float | None: + match = POWER_LINE.match(line) + if not match: + return None + return datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S %z").timestamp() + + +def guard_spans(text: str, timezone: str, end: float) -> list: + """Pauses: from a 'paused pids:' line to the next 'resumed pids:' line (or + `end`).""" + zone, spans, opened = ZoneInfo(timezone), [], None + for line in text.splitlines(): + t = guard_time(line, zone) + if t is None: + continue + what = GUARD_LINE.match(line).group(2) + if "paused pids:" in what and opened is None: + opened = t + elif "resumed pids:" in what and opened is not None: + spans.append((opened, t, "guard pause")) + opened = None + if opened is not None: + spans.append((opened, end, "guard pause")) + return spans + + +def power_spans(text: str, end: float) -> list: + """Sleeps: from a Sleep line to the next Wake or DarkWake line (or `end`).""" + spans, opened = [], None + for line in text.splitlines(): + t = power_time(line) + if t is None: + continue + kind = POWER_LINE.match(line).group(2).strip() + if kind == "Sleep" and opened is None: + opened = t + elif kind in ("Wake", "DarkWake") and opened is not None: + spans.append((opened, t, "sleep")) + opened = None + if opened is not None: + spans.append((opened, end, "sleep")) + return spans + + +def excerpt(text: str, when: Callable[[str], float | None], since: float, until: float): + """The lines of a log stamped within [since, until].""" + keep = [] + for line in text.splitlines(): + t = when(line) + if t is not None and since <= t <= until: + keep.append(line + "\n") + return "".join(keep) + + +def memory_pressure() -> int | None: + """`sysctl kern.memorystatus_vm_pressure_level`: 1 when normal on this host.""" + try: + out = subprocess.run( + ["sysctl", "-n", "kern.memorystatus_vm_pressure_level"], + capture_output=True, + text=True, + timeout=10, + check=True, + ) + return int(out.stdout.strip()) + except (OSError, subprocess.SubprocessError, ValueError): + return None + + +def read_guard_log() -> str: + try: + return GUARD_LOG.read_text(errors="replace") + except OSError: + return "" + + +def read_power_log() -> str: + out = subprocess.run( + ["pmset", "-g", "log"], capture_output=True, text=True, timeout=120, check=True + ) + return out.stdout + + +def local_timezone() -> str: + """The IANA name of the host's zone, which the guard log's times are in.""" + target = os.path.realpath("/etc/localtime") + if "zoneinfo/" not in target: + raise SystemExit(f"cannot tell the host's time zone from {target}") + return target.split("zoneinfo/", 1)[1] + + +class HostLog: + """The host log, written every `interval` seconds, and the host evidence an + attempt's class reads: the guard's pauses, the power log's sleeps, the + host log's gaps and its memory-pressure samples.""" + + def __init__( + self, + folder: Path, + timezone: str | None = None, + interval: float = SAMPLE_SECONDS, + read_load: Callable[[], tuple] = os.getloadavg, + read_pressure: Callable[[], int | None] = memory_pressure, + read_guard: Callable[[], str] = read_guard_log, + read_power: Callable[[], str] = read_power_log, + ): + self.folder = Path(folder) + self.folder.mkdir(parents=True, exist_ok=True) + self.timezone = timezone or local_timezone() + ZoneInfo(self.timezone) # refused here, not at the first pause + self.interval = interval + self._read_load, self._read_pressure = read_load, read_pressure + self._read_guard, self._read_power = read_guard, read_power + self._cond = threading.Condition() + self._samples: list[tuple[float, int | None]] = [] + self._spans: list = [] # guard pauses and sleeps at the last refresh + self._read_at = -math.inf + self._refresh_lock = threading.Lock() + self._stop = threading.Event() + self.start = time.time() + self.end: float | None = None + self._write_meta() + self.sample() + self._thread = threading.Thread(target=self._loop, name="host-log", daemon=True) + self._thread.start() + + def _write_meta(self) -> None: + meta = {"timezone": self.timezone, "start": self.start} + if self.end is not None: + meta["end"] = self.end + write_json(self.folder / "host.json", meta) + + def sample(self) -> None: + load1, load5, load15 = self._read_load() + pressure = self._read_pressure() + with self._cond: + t = time.time() + line = { + "time": t, + "load1": load1, + "load5": load5, + "load15": load15, + "pressure": pressure, + } + with (self.folder / "host-log.jsonl").open("a") as handle: + handle.write(json.dumps(line) + "\n") + self._samples.append((t, pressure)) + self._cond.notify_all() + + def _loop(self) -> None: + k = 1 + while not self._stop.wait( + max(0.0, self.start + k * self.interval - time.time()) + ): + self.sample() + # After a stall, the next sample is the next tick, with no burst. + k = max(k + 1, math.ceil((time.time() - self.start) / self.interval)) + + def settled(self, end: float) -> bool: + """Whether every sample that could bear on an attempt ending at `end` is + in: one later than the pressure window, or a silence longer than a + gap (which is itself host trouble when it covers the attempt).""" + with self._cond: + latest = self._samples[-1][0] + return latest > end + PRESSURE_WINDOW or time.time() - latest > HOST_LOG_GAP + + def wait_settled(self, end: float) -> None: + with self._cond: + while not self.settled(end): + self._cond.wait(timeout=min(self.interval, 5.0)) + + def refresh(self, after: float) -> None: + """Read the guard's and the power log again unless read since `after`.""" + with self._refresh_lock: + if self._read_at > after: + return + now = time.time() + since = self.start - EXCERPT_LEAD + zone = ZoneInfo(self.timezone) + guard = excerpt( + self._read_guard(), lambda x: guard_time(x, zone), since, now + ) + power = excerpt(self._read_power(), power_time, since, now) + spans = guard_spans(guard, self.timezone, now) + power_spans(power, now) + with self._cond: + self._spans, self._read_at = spans, now + + def _evidence(self) -> tuple[list, list[float]]: + """(spans, the times of samples above 1), as the report builds them + from the archived logs, with a silence so far as an open gap.""" + with self._cond: + spans = list(self._spans) + times = [t for t, _ in self._samples] + pressure = [t for t, p in self._samples if int(p or 0) > 1] + for a, b in zip([self.start] + times, times): + if b - a > HOST_LOG_GAP: + spans.append((a, b, "host log gap")) + now = time.time() if self.end is None else self.end + if now - times[-1] > HOST_LOG_GAP: + spans.append((times[-1], now, "host log gap")) + return spans, pressure + + def trouble(self, start: float, end: float) -> str | None: + spans, pressure = self._evidence() + for a, b, what in spans: + if a <= end and b >= start: + return what + for t in pressure: + if start - PRESSURE_WINDOW <= t <= end + PRESSURE_WINDOW: + return "memory pressure" + return None + + def describe(self, start: float, end: float) -> str: + """The host evidence behind an attempt's host class, for its rerun's + reason.""" + spans, _ = self._evidence() + parts = [ + f"{what} {a:.1f}-{b:.1f}" for a, b, what in spans if a <= end and b >= start + ] + with self._cond: + parts += [ + f"memory pressure {p} at {t:.1f}" + for t, p in self._samples + if int(p or 0) > 1 + and start - PRESSURE_WINDOW <= t <= end + PRESSURE_WINDOW + ] + return "; ".join(parts) + + def close(self) -> None: + """A last sample, the booking's end, and the guard's and power log's + lines for the booking.""" + self._stop.set() + self._thread.join(timeout=self.interval + 10) + self.sample() + with self._cond: + self.end = time.time() + self._write_meta() + since, zone = self.start - EXCERPT_LEAD, ZoneInfo(self.timezone) + guard = excerpt( + self._read_guard(), lambda x: guard_time(x, zone), since, self.end + ) + (self.folder / "clamshell-guard.log").write_text(guard) + power = excerpt(self._read_power(), power_time, since, self.end) + (self.folder / "pmset.log").write_text(power) + + +# ---------------------------------------------------------------- spend + + +class Ceiling(pilot.Spend): + """The hard ceiling over every model's spend: once the total reaches it, + `exhausted` is true for good, which ends each conversation under way at + its next turn and lets no attempt start.""" + + def __init__(self, ceiling_usd: float, spent_before: float): + super().__init__(ceiling_usd, spent_before) + self.crossed_at: float | None = None + + @property + def lock(self) -> threading.Lock: + return self._lock + + def add(self, usd: float | None) -> None: + with self._lock: + self.spent_usd += float(usd or 0.0) + if self.crossed_at is None and self.spent_usd >= self.budget_usd: + self.crossed_at = time.time() + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + with self._lock: + return self.crossed_at is not None + + +# ---------------------------------------------------------------- files + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + return sha256_bytes(Path(path).read_bytes()) + + +def write_json(path: Path, payload, indent: int | None = 1) -> str: + """Write a file whole (to a temporary name, then renamed); its sha256.""" + data = json.dumps(payload, indent=indent, default=str).encode() + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.tmp") + tmp.write_bytes(data) + os.replace(tmp, path) + return sha256_bytes(data) + + +# ---------------------------------------------------------------- one model + + +@dataclass +class Attempt: + label: str + n: int + start: float + end: float + record: dict + cls: str = "" + detail: str = "" + reason: str = "" + + +def converse_with_harness(lanes: "Lanes", sid: str, label: str, tool_call) -> dict: + """One attempt: the treatment harness's conversation for the arm, under the + hard ceiling.""" + board = lanes.run.board + return pilot.run_household( + board["scenarios"][sid], + board["variables"][sid], + tool_call, + lanes.completion, + lanes.run.ceiling, + profile=lanes.profiles[label], + ) + + +class Lanes: + """One model's two lanes: its admission, its attempts and its records.""" + + def __init__(self, run: "Run", model: str, labels: tuple[str, ...], folders: dict): + self.run, self.model, self.labels = run, model, labels + design = run.design + self.folder = Path(design["runs"]) / model + self.households_dir = { + label: self.folder / folders[label] / design["stage"] / "households" + for label in labels + } + self.profiles = {label: arm_profile(model, label) for label in labels} + self.cap = float(design["caps"][model]) + self.reserve = float(design["reserves"][model]) + self.orders = orders(run.master, model, run.draw_order, labels) + self.cond = threading.Condition() + self._log_lock = threading.Lock() + self.state: dict[str, str] = {} # household -> running, settling, finished + self.spent: dict[str, float] = {} # household -> its attempts' cost so far + self.refused = False + self.paused = False + self.stopped = False + self.inconclusive: dict[str, list[str]] = {} + self.pool = None + self._pool_lock = threading.Lock() + self.completion = None + + # The records. + + def households(self, lane: int) -> list[str]: + return [ + sid + for i, sid in enumerate(self.run.draw_order) + if i % 2 == lane - 1 and sid not in self.run.no_output + ] + + def write_line(self, line: dict, name: str = "run-log.jsonl") -> None: + with self._log_lock: + self.folder.mkdir(parents=True, exist_ok=True) + with (self.folder / name).open("a") as handle: + handle.write(json.dumps(line) + "\n") + handle.flush() + os.fsync(handle.fileno()) + + def event(self, kind: str, reason: str, at: float | None = None) -> None: + self.write_line( + {"event": kind, "time": time.time() if at is None else at, "reason": reason} + ) + self.run.say(f"{self.model}: {kind} ({reason})") + + def attempt_line(self, sid: str, lane: int, a: Attempt, sha: str) -> None: + self.write_line( + { + "household": sid, + "arm": a.label, + "lane": lane, + "attempt": a.n, + "start": a.start, + "end": a.end, + "status": a.record.get("status"), + "class": a.cls, + "action": action(a.cls, a.n), + "sha256": sha, + } + ) + + # Stops and pauses. + + def halted(self) -> bool: + return self.stopped or self.run.ceiling.exhausted() + + def stop(self, reason: str, kind: str = "stop", at: float | None = None) -> None: + """Stop the lanes. Every rule that stops them writes its own event.""" + with self.cond: + self.stopped = True + self.event(kind, reason, at) + self.cond.notify_all() + + def resume(self, note: str) -> None: + with self.cond: + if self.paused: + self.paused = False + self.event("resume", note) + self.cond.notify_all() + + # One household. + + def admit(self, sid: str, lane: int) -> bool: + with self.cond: + while not self.halted() and ( + self.paused or "settling" in self.state.values() + ): + self.cond.wait(timeout=WAIT_SECONDS) + if self.halted(): + return False + now = time.time() + finished = [h for h, s in self.state.items() if s == "finished"] + spend = sum(self.spent[h] for h in finished) + under_way = len(self.state) - len(finished) + 1 + ok = ( + not self.refused + and spend + under_way * self.reserve <= self.cap + ADMISSION_SLACK + ) + self.write_line( + { + "household": sid, + "lane": lane, + "time": now, + "decision": "admit" if ok else "refuse", + "finished_spend": spend, + "under_way": under_way, + "reserve": self.reserve, + "cap": self.cap, + }, + "admission-log.jsonl", + ) + if ok: + self.state[sid], self.spent[sid] = "running", 0.0 + else: + self.refused = True + self.cond.notify_all() + return ok + + def start_attempt(self) -> float | None: + """The attempt's start, or None when the lanes have stopped. Waits out a + pause. The start is taken under the ceiling's lock, so no attempt + starts after the crossing.""" + with self.cond: + while self.paused and not self.halted(): + self.cond.wait(timeout=WAIT_SECONDS) + with self.run.ceiling.lock: + if self.stopped or self.run.ceiling.crossed_at is not None: + return None + return time.time() + + def end_attempt( + self, sid: str, label: str, n: int, start: float, record: dict, last: bool + ) -> Attempt: + """The attempt's end. A class decided without the host evidence acts at + once: an operator failure pauses the lanes, a harness failure stops + them, before any other attempt can start.""" + with self.cond: + end = time.time() + self.spent[sid] += attempt_cost(record) + attempt = Attempt(label, n, start, end, record) + cls, detail = classify(record, start, end, None) + if cls in EARLY: + attempt.cls, attempt.detail = cls, detail + if cls == "operator": + self.paused = True + self.event("pause", f"{label}/{sid} attempt {n}: {detail}") + if cls == "harness": + self.stopped = True + self.event( + "stop", + f"{label}/{sid}: a request class on the first request ({detail})", + ) + if last: + self.state[sid] = "settling" + self.cond.notify_all() + return attempt + + def settle(self, sid: str, done: list[Attempt]) -> None: + """Each attempt's class once the host evidence for the pass is in.""" + last = max(a.end for a in done) + host = self.run.host + host.wait_settled(last) + host.refresh(after=last) + stop = None + for a in done: + cls, detail = classify(a.record, a.start, a.end, host) + if a.cls and a.cls != cls: + raise RuntimeError(f"{a.label}/{sid}: class {a.cls} became {cls}") + a.cls, a.detail = cls, detail + a.reason = ( + f"{detail}: {host.describe(a.start, a.end)}" + if cls == "host" + else detail + ) + status = a.record.get("status") + if cls == "inconclusive" and status in ("provider_error", "harness_error"): + key = f"{status}: {error_class(a.record) or a.record.get('error')}" + with self.cond: + seen = self.inconclusive.setdefault(key, []) + if sid not in seen: + seen.append(sid) + if len(seen) > INCONCLUSIVE_STOP and stop is None: + stop = f"{key} ended runs in {len(seen)} households" + if stop: + self.stop(stop) + + def converse(self, sid: str, label: str) -> dict: + profile = self.profiles[label] + board = self.run.board + features = frozenset(profile.tool_features) + + def tool_call(arguments): + message = ( + pilot.tool_call_frame(features, arguments) if features else arguments + ) + return self.pool.call(message) + + try: + return self.run.converse(self, sid, label, tool_call) + except Exception as error: # an attempt that left no record of its own + return pilot.harness_error_record( + sid, board["variables"][sid], profile, error + ) + + def ensure_pool(self) -> bool: + """The model's pool, started again if it has lost every worker; False + if it cannot be.""" + with self._pool_lock: + if ( + self.pool is not None + and self.pool.counters()["lost"] < self.run.workers + ): + return True + old = self.pool + try: + self.pool = self.run.pool_factory(self) + except RuntimeError as error: + self.run.say(f"{self.model}: the engine pool did not start: {error}") + return False + if old is not None: + self.run.say( + f"{self.model}: the engine pool lost every worker; started again" + ) + old.close() + return True + + def replace(self, sid: str, lane: int, previous: Attempt, entries: list) -> None: + """Save the attempt a rerun replaces, before the rerun.""" + folder = self.households_dir[previous.label] / "attempts" + name = f"{sid}-{previous.n}.json" + sha = write_json(folder / name, previous.record) + entries.append( + { + "n": previous.n, + "file": f"attempts/{name}", + "sha256": sha, + "status": previous.record.get("status"), + "class": previous.cls, + "reason": previous.reason, + } + ) + self.attempt_line(sid, lane, previous, sha) + + def run_household(self, sid: str, lane: int) -> None: + order = self.orders[sid] + attempts: dict[str, list[Attempt]] = {label: [] for label in order} + entries: dict[str, list[dict]] = {label: [] for label in order} + in_flight = self.folder / "in-flight" / sid + todo = [(1, label) for label in order] + while todo: + done: list[Attempt] = [] + for i, (n, label) in enumerate(todo): + if not self.ensure_pool(): + self.stop("the engine pool could not be started again") + break + start = self.start_attempt() + if start is None: + break + if n > 1: + self.replace(sid, lane, attempts[label][-1], entries[label]) + record = self.converse(sid, label) + a = self.end_attempt( + sid, label, n, start, record, last=i == len(todo) - 1 + ) + write_json(in_flight / f"{label}-{n}.json", record) + attempts[label].append(a) + done.append(a) + self.run.say( + f"{self.model} lane {lane} {sid} {label} #{n}: " + f"{record.get('status')} ${attempt_cost(record):.4f} " + f"(total ${self.run.ceiling.spent_usd:.2f})" + ) + if not done: + break + self.settle(sid, done) + cut = len(done) < len(todo) + todo = ( + [] + if cut or self.halted() + else [ + (a.n + 1, a.label) + for a in done + if a.n <= MAX_RERUNS and a.cls in RERUN + ] + ) + with self.cond: + self.state[sid] = "running" if todo else "finished" + self.cond.notify_all() + for label, mine in attempts.items(): + if mine: + final = mine[-1] + sha = write_json( + self.households_dir[label] / f"{sid}.json", + {**final.record, "attempts": entries[label]}, + ) + self.attempt_line(sid, lane, final, sha) + with self.cond: + self.state[sid] = "finished" + self.cond.notify_all() + shutil.rmtree(in_flight, ignore_errors=True) + + def lane(self, lane: int) -> None: + try: + for sid in self.households(lane): + if not self.admit(sid, lane): + return + self.run_household(sid, lane) + except Exception as error: # a runner defect: this model stops, the rest run on + self.run.say(traceback.format_exc()) + self.stop(f"runner error on lane {lane}: {type(error).__name__}: {error}") + + def manifest(self) -> dict: + board, design = self.run.board, self.run.design + pe3 = arm_profile(self.model, "pe3") + first = sorted(board["scenarios"])[0] + + def timeout(profile): + built = pilot.build_initial_request( + board["scenarios"][first], board["variables"][first], profile + ) + return built[1].get("timeout") + + board_path = Path(design["board"]) + return { + "started_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "git_head": pilot._git("rev-parse", "HEAD"), + "git_dirty": bool(pilot._git("status", "--porcelain")), + "script_sha256": sha256_file(Path(__file__)), + "harness_sha256": sha256_file(Path(pilot.__file__)), + "treatment_commit": TREATMENT_COMMIT, + "versions": pilot._versions(), + "fix_module": str(pilot.FIX_MODULE.relative_to(pilot.ROOT)), + "fix_module_sha256": sha256_file(pilot.FIX_MODULE), + "board": str(board_path), + "board_engine": f"policyengine-us {pilot.board_engine(board_path)}", + "board_reference_sha256": sha256_file(board_path / "reference_outputs.csv"), + "model": self.model, + "model_id": pe3.model_id, + "api": pe3.api, + "answer_contract": pe3.contract, + "prompt_version": PROMPT_VERSION, + "prompt_wording_sha256": pilot.prompt_wording_sha256(pe3), + "request_timeout_seconds": timeout(pe3), + "turn_cap": pilot.TOOL_TURN_CAP, + "max_calculations_per_turn": pilot.MAX_CALCULATIONS_PER_TURN, + "tool_timeout_seconds": pilot.TOOL_TIMEOUT_SECONDS, + "arms": { + label: { + "arm": arm_string(label), + "features": sorted(self.profiles[label].tool_features), + } + for label in self.labels + }, + "arm_settings": { + label: { + "tool_notice_sha256": hashlib.sha256( + profile.notice.encode() + ).hexdigest(), + **pilot.profile_treatment(profile), + "tool_version": profile.tool_version, + "request_timeout_seconds": timeout(profile), + } + for label, profile in self.profiles.items() + }, + "folders": { + label: str(path) for label, path in self.households_dir.items() + }, + "cap_usd": self.cap, + "reserve_usd": self.reserve, + "overall_cap_usd": float(design["overall_cap"]), + "ceiling_usd": self.run.ceiling.budget_usd, + "spent_before_usd": float(design["spent_before"]), + "lanes": {lane: self.households(lane) for lane in (1, 2)}, + "no_scored_output": sorted(self.run.no_output), + "workers": self.run.workers, + } + + def summary(self) -> dict: + records = [] + for label, folder in self.households_dir.items(): + for path in sorted(folder.glob("*.json")) if folder.is_dir() else []: + records.append(json.loads(path.read_text())) + lines = [] + path = self.folder / "run-log.jsonl" + if path.exists(): + lines = [json.loads(x) for x in path.read_text().splitlines() if x.strip()] + attempts = [x for x in lines if "event" not in x] + count: dict[str, int] = {} + for x in attempts: + count[x["class"]] = count.get(x["class"], 0) + 1 + return { + "finished_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "model": self.model, + "households_finished": sorted( + h for h, s in self.state.items() if s == "finished" + ), + "refused": self.refused, + "stopped": self.stopped, + "spent_usd": sum(self.spent.values()), + "final_records": len(records), + "attempts": len(attempts), + "classes": dict(sorted(count.items())), + "events": [x for x in lines if "event" in x], + "tool_pool": self.pool.counters() if self.pool is not None else None, + } + + +# ---------------------------------------------------------------- the run + + +def default_pool(lanes: Lanes): + return pilot.ToolPool( + size=lanes.run.workers, + board=Path(lanes.run.design["board"]), + tool_features=frozenset(), + ) + + +def default_completion(profile: pilot.ModelProfile): + from litellm import completion, responses + + return responses if profile.api == "responses" else completion + + +class Run: + """The six models' lanes, the host log and the hard ceiling.""" + + def __init__( + self, + design: dict, + workers: int = 2, + models: tuple[str, ...] | None = None, + converse: Callable = converse_with_harness, + pool_factory: Callable = default_pool, + completion_for: Callable = default_completion, + host_factory: Callable | None = None, + out=sys.stdout, + ): + self.design, self.workers, self.out = design, workers, out + self.converse, self.pool_factory = converse, pool_factory + self.completion_for = completion_for + self.host_factory = host_factory or (lambda folder: HostLog(folder)) + self._say_lock = threading.Lock() + self.master = master_seed(design["frozen_sha256"]) + self.draw_order = list(design["draw_order"]) + board_path = Path(design["board"]) + pilot.set_stated_engine(pilot.board_engine(board_path)) + self.board = pilot.load_board(board_path) + scored = {str(s) for s in self.board["scored"]["scenario_id"]} + self.no_output = {sid for sid in self.draw_order if sid not in scored} + self.ceiling = Ceiling( + float(design["overall_cap"]) + CEILING_MARGIN, float(design["spent_before"]) + ) + self.models = tuple(models or CHEAP + FRONTIER) + self.lanes = { + model: Lanes( + self, + model, + ARMS if model in CHEAP else FRONTIER_ARMS, + design["arms"] if model in CHEAP else design["frontier_arms"], + ) + for model in self.models + } + self.host: HostLog | None = None + self._ceiling_written = False + self.interrupts = 0 + + def say(self, text: str) -> None: + with self._say_lock: + print(text, file=self.out, flush=True) + + def stop_all(self, reason: str) -> None: + for lanes in self.lanes.values(): + lanes.stop(reason) + + def watch(self) -> None: + """The ceiling's events, and resumes after operator pauses.""" + crossed = self.ceiling.crossed_at + if crossed is not None and not self._ceiling_written: + self._ceiling_written = True + reason = ( + f"total spend ${self.ceiling.spent_usd:.2f} reached the hard ceiling " + f"${self.ceiling.budget_usd:.2f}" + ) + for lanes in self.lanes.values(): + lanes.stop(reason, kind="ceiling", at=crossed) + for lanes in self.lanes.values(): + path = lanes.folder / "RESUME.json" + if lanes.paused and path.exists(): + note = json.loads(path.read_text()).get("note") or "resumed" + path.rename(lanes.folder / f"resume-{time.time():.0f}.json") + lanes.resume(note) + + def execute(self) -> None: + runs = Path(self.design["runs"]) + self.host = self.host_factory(runs / "host") + try: + starting = {} + for model, lanes in self.lanes.items(): + lanes.completion = self.completion_for(lanes.profiles["pe3"]) + stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) + write_json( + lanes.folder / f"manifest-{stamp}.json", lanes.manifest(), indent=2 + ) + starting[model] = threading.Thread( + target=lanes.ensure_pool, daemon=True + ) + starting[model].start() + for thread in starting.values(): + thread.join() + threads = [ + threading.Thread( + target=lanes.lane, args=(lane,), name=f"{model}-{lane}", daemon=True + ) + for model, lanes in self.lanes.items() + for lane in (1, 2) + ] + for thread in threads: + thread.start() + while any(t.is_alive() for t in threads): + self.watch() + for t in threads: + t.join(timeout=0.5 / len(threads)) + self.watch() + finally: + for model, lanes in self.lanes.items(): + stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) + write_json( + lanes.folder / f"run-summary-{stamp}.json", + lanes.summary(), + indent=2, + ) + if lanes.pool is not None: + lanes.pool.close() + self.host.close() + + def interrupt(self, signum, frame) -> None: + self.interrupts += 1 + if self.interrupts > 1: + raise KeyboardInterrupt + self.say( + f"signal {signum}: stopping every model's lanes; " + "the conversations under way end" + ) + self.stop_all(f"operator stop (signal {signum})") + + +# ---------------------------------------------------------------- before a run + + +def host_power() -> tuple[bool, bool]: + """(on AC power, lid open).""" + batt = subprocess.run( + ["pmset", "-g", "batt"], capture_output=True, text=True + ).stdout + lid = subprocess.run( + ["ioreg", "-r", "-k", "AppleClamshellState", "-d", "1"], + capture_output=True, + text=True, + ).stdout + return "'AC Power'" in batt, '"AppleClamshellState" = No' in lid + + +def preflight(design: dict, models: tuple[str, ...]) -> list[str]: + """What stops the run from starting.""" + missing = [k for k in REQUIRED if k not in design] + if missing: + return [f"the design has no {', '.join(missing)}"] + problems = [] + if design.get("stated_engine_overrides"): + problems.append("the confirmatory run states the board's engine: no overrides") + if sha256_file(Path(design["preregistration"])) != design["frozen_sha256"]: + problems.append("the frozen copy does not hash to frozen_sha256") + pool = design["eligible_pool"] + if sha256_file(Path(pool["path"])) != pool["sha256"]: + problems.append("the eligible pool does not hash as frozen") + else: + ids = [e["id"] for e in json.loads(Path(pool["path"]).read_text())] + if draw(master_seed(design["frozen_sha256"]), ids) != list( + design["draw_order"] + ): + problems.append( + "the draw, recomputed from the stream 'draw', is not the design's" + ) + head, dirty = pilot._git("rev-parse", "HEAD"), pilot._git("status", "--porcelain") + if head != design["run_commit"] or dirty: + problems.append( + f"the runner is at {head[:12]} (dirty {bool(dirty)}), " + "not the run commit, clean" + ) + changed = subprocess.run( + [ + "git", + "-C", + str(pilot.ROOT), + "diff", + "--quiet", + TREATMENT_COMMIT, + "HEAD", + "--", + *TREATMENT_PATHS, + ] + ).returncode + if changed: + problems.append(f"the treatment's code differs from {TREATMENT_COMMIT[:12]}'s") + import policybench + + places = [Path(p).resolve() for p in policybench.__path__] + if pilot.ROOT != ROOT or not all(p.is_relative_to(ROOT) for p in places): + problems.append( + f"the harness ({pilot.__file__}) or policybench ({places}) is not " + "this checkout's" + ) + harness = design["treatment_harness"] + harness_head = subprocess.run( + ["git", "-C", harness, "rev-parse", "HEAD"], capture_output=True, text=True + ).stdout.strip() + if harness_head != TREATMENT_COMMIT: + problems.append( + f"the treatment harness is at {harness_head[:12]}, " + f"not {TREATMENT_COMMIT[:12]}" + ) + board = Path(design["board"]) + engine = pilot.board_engine(board) + if f"policyengine-us {engine}" != design["engine"]: + problems.append( + f"the board's engine {engine} is not the design's {design['engine']}" + ) + installed = pilot._versions()["policyengine-us"] + if installed != engine: + problems.append(f"policyengine-us {installed} is installed, not {engine}") + if set(design["arms"]) != set(ARMS) or set(design["frontier_arms"]) != set( + FRONTIER_ARMS + ): + problems.append("the design's arms are not the registered ones") + for name in ("arms", "frontier_arms"): + if len(set(design[name].values())) != len(design[name]): + problems.append(f"two {name} share a folder") + for model in models: + if model not in design["caps"] or model not in design["reserves"]: + problems.append(f"{model}: no cap or reserve") + for label in ARMS if model in CHEAP else FRONTIER_ARMS: + try: + arm_profile(model, label) + except (ValueError, SystemExit) as error: + problems.append(str(error)) + key = pilot.provider_key(pilot.model_profile(model).model_id) + if not os.environ.get(key): + problems.append(f"{model}: {key} is not set") + folder = Path(design["runs"]) / model + if (folder / "run-log.jsonl").exists() or any( + folder.glob("*/*/households/*.json") + ): + problems.append(f"{folder} already holds records; a run is never resumed") + if (Path(design["runs"]) / "host" / "host-log.jsonl").exists(): + problems.append("the runs folder already holds a host log") + ac, lid = host_power() + if not (ac and lid): + problems.append( + f"the host is not on AC power with the lid open (AC {ac}, lid open {lid})" + ) + return problems + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run", help="run the confirmatory stage") + run.add_argument("design", type=Path) + run.add_argument("--workers", type=int, default=2, help="engine workers per model") + run.add_argument("--dry-run", action="store_true", help="check and print the plan") + resume = sub.add_parser("resume", help="end an operator pause") + resume.add_argument("design", type=Path) + resume.add_argument("model") + resume.add_argument("--note", default="the account was fixed") + args = parser.parse_args(argv) + design = json.loads(args.design.read_text()) + if args.command == "resume": + folder = Path(design["runs"]) / args.model + write_json(folder / "RESUME.json", {"time": time.time(), "note": args.note}) + print(f"{args.model}: resume requested; the runner resumes within a second") + return 0 + models = CHEAP + FRONTIER + problems = preflight(design, models) + pilot._require_engine(Path(design["board"])) + if problems: + print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) + return 1 + plan = Run(design, workers=args.workers) + if args.dry_run: + for model, lanes in plan.lanes.items(): + sizes = [len(lanes.households(lane)) for lane in (1, 2)] + print( + f"{model}: arms {', '.join(lanes.labels)}; lanes {sizes[0]} and " + f"{sizes[1]} households; cap ${lanes.cap:.2f}, " + f"reserve ${lanes.reserve:.2f}" + ) + print( + f"no scored output: {sorted(plan.no_output)}; hard ceiling " + f"${plan.ceiling.budget_usd:.2f} " + f"(${plan.ceiling.spent_usd:.2f} spent before)" + ) + return 0 + signal.signal(signal.SIGINT, plan.interrupt) + signal.signal(signal.SIGTERM, plan.interrupt) + plan.execute() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From c04ffe5b4178b8bde3f2d6d3e7af8ce39d313287 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 02:56:22 -0400 Subject: [PATCH 18/64] Confirmatory runner: tests Unit tests for the streams, arm strings and features, the classes and actions, the host evidence and the ceiling. Fake runs (a fake provider, pool and host log, on a fake board, with the windows scaled down) plant reruns, host trouble, an operator pause, a harness stop, the inconclusive stop, a refusal, the hard ceiling, a raising attempt, pool restarts and an interrupt. run_problems recomputes every rule of the run procedure from the files the runner wrote, and a property test checks it over random failures and spend. Two tests run the harness's own conversation under the runner. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 1435 ++++++++++++++++++++++++++++++++ 1 file changed, 1435 insertions(+) create mode 100644 tests/test_confirmatory_run.py diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py new file mode 100644 index 00000000..f42a1c8e --- /dev/null +++ b/tests/test_confirmatory_run.py @@ -0,0 +1,1435 @@ +"""The confirmatory runner (scripts/confirmatory_run.py). + +The runs here are fakes: a fake provider whose conversations a test plants, +a fake engine pool and a host log sampled every 20 ms, on a fake board. Each +run is checked by `run_problems`, which recomputes, from the files the runner +wrote and nothing else, every rule the run procedure states: the records and +their hashes, each attempt's class and action, the lanes and the schedule, +admission, pauses, stops and the ceiling. The study's differential test runs +the runner and checks its records with the confirmatory report itself. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import math +import sys +import threading +import time +from datetime import datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +import numpy as np +import pandas as pd +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import confirmatory_run as cr # noqa: E402 +import tool_pilot_argon as pilot # noqa: E402 + +from policybench.scenarios import Person, Scenario # noqa: E402 + +VARIABLES = ["snap", "federal_income_tax_before_refundable_credits"] +POOL = [f"h{i:02d}" for i in range(12)] +N_DRAW = 6 +COST = 0.01 +TZ = "America/New_York" +PREREG = "A frozen pre-registration for a test.\n" +ENGINE = "2.60.4" + + +# ---------------------------------------------------------------- streams + + +def test_the_master_seed_is_the_first_sixteen_hex_digits(): + sha = hashlib.sha256(b"x").hexdigest() + assert cr.master_seed(sha) == int(sha[:16], 16) + with pytest.raises(ValueError): + cr.master_seed("not a sha") + + +def test_a_stream_is_keyed_by_its_name_alone(): + a = cr.stream(7, "order/m").integers(0, 1 << 30, 5) + b = cr.stream(7, "order/m").integers(0, 1 << 30, 5) + c = cr.stream(7, "order/n").integers(0, 1 << 30, 5) + assert list(a) == list(b) and list(a) != list(c) + key = int.from_bytes(hashlib.sha256(b"draw").digest()[:8], "big") + want = np.random.default_rng(np.random.SeedSequence(7, spawn_key=(key,))) + assert list(cr.stream(7, "draw").integers(0, 100, 8)) == list( + want.integers(0, 100, 8) + ) + + +@given(st.integers(0, (1 << 64) - 1), st.integers(1, 12)) +def test_the_draw_is_seeded_and_without_replacement(master, n): + got = cr.draw(master, list(reversed(POOL)), n=n) + assert got == cr.draw(master, POOL, n=n) # the pool is sorted first + assert len(set(got)) == n and set(got) <= set(POOL) + + +def test_orders_are_permutations_drawn_for_every_household_in_draw_order(): + got = cr.orders(9, "m", POOL, cr.ARMS) + assert list(got) == POOL + assert all(sorted(o) == sorted(cr.ARMS) for o in got.values()) + generator = cr.stream(9, "order/m") + assert got[POOL[0]] == [str(a) for a in generator.permutation(list(cr.ARMS))] + # The first households' orders do not depend on the later ones. + assert cr.orders(9, "m", POOL[:3], cr.ARMS) == {s: got[s] for s in POOL[:3]} + + +# ---------------------------------------------------------------- arms + + +def test_arm_strings_follow_the_harness_feature_order(): + assert cr.arm_string("pe") == "pe" + assert cr.arm_string("pe3") == cr.arm_string("pe3-b") == "pe3" + for label, dropped in cr.DROPPED.items(): + if dropped: + kept = [f for f in pilot.TOOL_FEATURES if f not in dropped] + assert cr.arm_string(label) == "pe:" + "+".join(kept) + + +@pytest.mark.parametrize("model", cr.CHEAP + cr.FRONTIER) +def test_each_arm_has_its_registered_features(model): + for label in cr.ARMS if model in cr.CHEAP else cr.FRONTIER_ARMS: + profile = cr.arm_profile(model, label) + dropped = cr.DROPPED[label] + want = set() if dropped is None else set(pilot.TOOL_FEATURES) - set(dropped) + assert profile.uses_calculator and set(profile.tool_features) == want + assert profile.prompt_version == cr.PROMPT_VERSION + + +def test_an_arm_whose_features_are_not_registered_is_refused(monkeypatch): + # The harness builds an arm's features from its arm string: a string that + # does not give the label's registered features is refused. + monkeypatch.setattr(cr, "arm_string", lambda label: "pe:names+labels") + with pytest.raises(ValueError, match="features"): + cr.arm_profile("gpt-6-luna", "loo-flat") + + +def test_the_error_classes_are_policybench_s(): + from policybench import eval_no_tools as e + + assert cr.TRANSPORT == {c.__name__ for c in e.RETRYABLE_PROVIDER_ERRORS} + assert cr.REQUEST | cr.ACCOUNT == {c.__name__ for c in e.MODEL_FATAL_ERRORS} + assert not cr.REQUEST & cr.ACCOUNT + assert not cr.TRANSPORT & (cr.REQUEST | cr.ACCOUNT) + + +# ---------------------------------------------------------------- classes + + +def record(status="complete", error=None, turns=1, calls=(), truncated=False): + turn = {"calls": [{"result": r} for r in calls]} + out = [dict(turn) for _ in range(turns)] + if truncated: + out = [{"truncated_and_retried": True}] + return {"status": status, "error": error, "turns": out} + + +class AlwaysTrouble: + def trouble(self, start, end): + return "memory pressure" + + +CASES = [ + (record(), "ok"), + (record("incomplete"), "ok"), + (record("turn_limit"), "ok"), + (record("provider_error", "RateLimitError: slow down"), "provider"), + (record("provider_error", "BadRequestError: too long"), "model"), + (record("provider_error", "BadRequestError: bad", turns=0), "harness"), + (record("provider_error", "BadRequestError: x", truncated=True), "harness"), + (record("provider_error", "AuthenticationError: key", turns=0), "operator"), + (record("provider_error", "RateLimitError: insufficient_quota"), "operator"), + (record("provider_error", "ProviderEmpty: no choices"), "inconclusive"), + (record("provider_error", "RateLimitError: x", turns=0), "none"), + ( + record("harness_error", "RuntimeError: every tool worker was lost", turns=0), + "none", + ), + (record("harness_error", "the tool pool had no worker"), "inconclusive"), + (record("budget_stop"), "ceiling"), + (record("budget_stop", turns=0), "ceiling"), + (record(calls=[{"error": "timeout", "message": "slow"}]), "inconclusive"), + ( + record(calls=[{"error": "engine", "message": "tool worker failed: EOF"}]), + "inconclusive", + ), + (record(calls=[{"error": "engine", "message": "MemoryError: x"}]), "inconclusive"), + (record(calls=[{"error": "engine", "message": "ValueError: no person"}]), "model"), + (record(calls=[{"results": {"snap": {"error": "OSError: disk"}}}]), "inconclusive"), + (record(calls=[{"results": {"snap": {"error": "KeyError: x"}}}]), "model"), + (record("running"), "inconclusive"), +] + + +@pytest.mark.parametrize("rec, want", CASES) +def test_each_failure_takes_its_class(rec, want): + assert cr.classify(rec, 0.0, 1.0, None)[0] == want + + +@pytest.mark.parametrize("rec, want", CASES) +def test_host_trouble_outranks_every_class_but_the_early_four(rec, want): + got = cr.classify(rec, 0.0, 1.0, AlwaysTrouble())[0] + assert got == (want if want in cr.EARLY else "host") + + +@pytest.mark.parametrize("rec, want", CASES) +def test_an_early_class_needs_no_host_evidence(rec, want): + early = cr.classify(rec, 0.0, 1.0, None)[0] + assert (early in cr.EARLY) == (want in cr.EARLY) + + +def test_actions_follow_the_class_and_the_attempt(): + for cls in ("ok", "model", "provider", "host", "none", "operator", "harness"): + for n in (1, 2, 3): + got = cr.action(cls, n) + if cls == "harness": + assert got == "stop" + elif cls in ("provider", "host", "none") and n <= 2: + assert got == "rerun" + elif cls == "operator" and n <= 2: + assert got == "pause-rerun" + else: + assert got == "stands" + assert cr.action("ceiling", 1) == "stopped" + assert cr.action("inconclusive", 1) == "stands" + + +# ---------------------------------------------------------------- host evidence + + +def guard_line(t: float, what: str) -> str: + stamp = datetime.fromtimestamp(t, ZoneInfo(TZ)).strftime("%Y-%m-%d %H:%M:%S") + return f"{stamp} [poll] lid closed on battery: {what} pids: 1 2" + + +def power_line(t: float, kind: str) -> str: + stamp = datetime.fromtimestamp(t, ZoneInfo(TZ)).strftime("%Y-%m-%d %H:%M:%S %z") + return f"{stamp} {kind:<20}\t{kind} due to a test" + + +def test_guard_pauses_and_power_sleeps_are_spans(): + t = 1_800_000_000.0 + guard = "\n".join( + [ + guard_line(t, "paused"), + guard_line(t + 10, "resumed"), + "junk", + guard_line(t + 20, "paused"), + ] + ) + assert cr.guard_spans(guard, TZ, end=t + 99) == [ + (t, t + 10, "guard pause"), + (t + 20, t + 99, "guard pause"), + ] + power = "\n".join( + [ + power_line(t, "Sleep"), + power_line(t + 5, "DarkWake"), + power_line(t + 7, "Sleep"), + ] + ) + assert cr.power_spans(power, end=t + 50) == [ + (t, t + 5, "sleep"), + (t + 7, t + 50, "sleep"), + ] + + +def test_an_excerpt_keeps_the_lines_stamped_in_its_window(): + t = 1_800_000_000.0 + text = "\n".join(guard_line(t + k, "paused") for k in (-5, 0, 5, 10)) + "\nx" + zone = ZoneInfo(TZ) + kept = cr.excerpt(text, lambda x: cr.guard_time(x, zone), t, t + 5) + assert kept.splitlines() == [guard_line(t, "paused"), guard_line(t + 5, "paused")] + + +class Readers: + """The host log's inputs, which a test can change while it runs.""" + + def __init__(self): + self.pressure = 1 + self.pressing = 0 # attempts under way that raise the pressure + self.stall = 0.0 + self.guard: list[str] = [] + self.power: list[str] = [] + self.lock = threading.Lock() + + def read_pressure(self): + with self.lock: + stall, self.stall = self.stall, 0.0 + if stall: + time.sleep(stall) + return 2 if self.pressing else self.pressure + + def host(self, folder): + return cr.HostLog( + folder, + timezone=TZ, + interval=0.02, + read_load=lambda: (1.0, 2.0, 3.0), + read_pressure=self.read_pressure, + read_guard=lambda: "\n".join(self.guard), + read_power=lambda: "\n".join(self.power), + ) + + +@pytest.fixture +def fast(monkeypatch): + """Windows scaled down a thousandfold, so a fake run takes a second.""" + monkeypatch.setattr(cr, "PRESSURE_WINDOW", 0.06) + monkeypatch.setattr(cr, "HOST_LOG_GAP", 0.09) + monkeypatch.setattr(cr, "WAIT_SECONDS", 0.005) + monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) + + +def test_the_host_log_samples_and_closes(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + time.sleep(0.1) + host.close() + meta = json.loads((tmp_path / "host.json").read_text()) + samples = [ + json.loads(x) for x in (tmp_path / "host-log.jsonl").read_text().splitlines() + ] + assert meta["timezone"] == TZ and meta["start"] <= samples[0]["time"] + assert samples[-1]["time"] <= meta["end"] + assert len(samples) >= 4 + assert samples[0]["load1"] == 1.0 and samples[0]["pressure"] == 1 + times = [s["time"] for s in samples] + assert times == sorted(times) + + +def test_pressure_near_an_attempt_is_host_trouble_once_settled(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + start = time.time() + readers.pressure = 2 + time.sleep(0.05) + readers.pressure = 1 + end = time.time() + host.wait_settled(end) + assert host.settled(end) + host.refresh(after=end) + assert host.trouble(start, end) == "memory pressure" + assert host.trouble(end + 1, end + 2) is None + host.close() + + +def test_a_stalled_host_log_is_a_gap(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + time.sleep(0.03) + start = time.time() + readers.stall = 0.2 + time.sleep(0.25) + end = time.time() + host.wait_settled(end) + host.refresh(after=end) + assert host.trouble(start, end) == "host log gap" + host.close() + + +def test_guard_pauses_and_sleeps_reach_an_attempt_after_a_refresh(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + start = time.time() + readers.guard += [guard_line(start, "paused"), guard_line(start + 1, "resumed")] + end = time.time() + host.wait_settled(end) + assert host.trouble(start, end) is None # not read yet + host.refresh(after=end) + assert host.trouble(start, end) == "guard pause" + readers.guard.clear() + readers.power += [power_line(start, "Sleep"), power_line(start + 1, "Wake")] + host.refresh(after=time.time()) + assert host.trouble(start, end) == "sleep" + host.close() + assert "Sleep" in (tmp_path / "pmset.log").read_text() + + +# ---------------------------------------------------------------- the ceiling + + +def test_the_ceiling_crosses_at_its_amount_and_stays_crossed(): + ceiling = cr.Ceiling(1.0, 0.25) + ceiling.add(0.5) + assert not ceiling.exhausted() and ceiling.crossed_at is None + ceiling.add(0.25) + assert ceiling.exhausted() and ceiling.crossed_at is not None + first = ceiling.crossed_at + ceiling.add(None) + assert ceiling.crossed_at == first and ceiling.spent_usd == pytest.approx(1.0) + + +def test_the_ceiling_ignores_a_reserve(): + # No conversation stops for a model's spend: only the ceiling stops one. + assert not cr.Ceiling(1.0, 0.0).exhausted(reserve_usd=5.0) + + +# ---------------------------------------------------------------- fake runs + + +def scenario(sid: str) -> Scenario: + i = POOL.index(sid) + return Scenario( + id=sid, + state="CO", + filing_status="single", + adults=[Person(name="head", age=30 + i, employment_income=10_000.0 * (i + 1))], + children=[], + ) + + +class FakePool: + """An engine pool that answers every call and keeps what it was sent.""" + + def __init__(self, lanes, fail: bool = False): + if fail: + raise RuntimeError("tool worker failed to start: a test") + self.messages: list = [] + self.lost = 0 + self.closed = False + + def call(self, message): + self.messages.append(message) + return {"results": {"snap": {"values": {"spm": 1.0}}}}, 0.0 + + def counters(self): + return {"timeouts": 0, "crashes": 0, "lost": self.lost} + + def close(self): + self.closed = True + + +class FakeProvider: + """Conversations a test plants. specs[(model, label, household, n)] holds a + record's `status` and `error`, `turns` (default 1), `cost` per turn, + `sleep` per turn, `calls` (tool arguments, one call each, per turn), + `pressure` (high pressure during the attempt), `stall` (a host-log stall, + seconds), `guard` (a guard pause over the attempt), `resume_after` + (seconds after which the operator resumes) and `raise` (the conversation + raises).""" + + def __init__(self, world, specs=None): + self.world, self.specs = world, specs or {} + self.count: dict = {} + self.lock = threading.Lock() + + def __call__(self, lanes, sid, label, tool_call): + model = lanes.model + with self.lock: + n = self.count[(model, label, sid)] = ( + self.count.get((model, label, sid), 0) + 1 + ) + spec = self.specs.get((model, label, sid, n), {}) + readers = self.world.readers + if spec.get("raise"): + raise RuntimeError("a test raised") + if spec.get("pressure"): + with readers.lock: + readers.pressing += 1 + if spec.get("stall"): + readers.stall = spec["stall"] + start = time.time() + rec = { + "scenario_id": sid, + "model": model, + "arm": cr.arm_string(label), + "variables": list(VARIABLES), + "status": "running", + "turns": [], + "predictions": {v: 0.0 for v in VARIABLES}, + } + spend = lanes.run.ceiling + for i in range(spec.get("turns", 1)): + if spend.exhausted(): + rec["status"] = "budget_stop" + break + started = time.time() + time.sleep(spec.get("sleep", 0.0)) + completed = time.time() + cost = spec.get("cost", COST) + spend.add(cost) + turn = { + "index": i, + "request_started_at": started, + "request_completed_at": completed, + "total_cost_usd": cost, + "calls": [], + } + for arguments in spec.get("calls", []): + result, latency = tool_call(arguments) + turn["calls"].append({"arguments": arguments, "result": result}) + rec["turns"].append(turn) + if rec["status"] == "running": + rec["status"] = spec.get("status", "complete") + if spec.get("error"): + rec["error"] = spec["error"] + if spec.get("guard"): + readers.guard += [ + guard_line(math.floor(start) - 1, "paused"), + guard_line(math.ceil(time.time()) + 1, "resumed"), + ] + if spec.get("stall"): + time.sleep(spec["stall"] + 0.02) + if spec.get("pressure"): + with readers.lock: + readers.pressing -= 1 + rec["totals"] = {"cost_usd": sum(t["total_cost_usd"] for t in rec["turns"])} + if spec.get("resume_after") is not None: + folder = lanes.folder + + def resume(): + cr.write_json(folder / "RESUME.json", {"note": "fixed in a test"}) + + threading.Timer(spec["resume_after"], resume).start() + return rec + + +class World: + """A fake board, design and run.""" + + def __init__( + self, + tmp: Path, + monkeypatch, + specs=None, + caps=None, + reserve=7 * 3 * COST, + overall_cap=100.0, + spent_before=1.0, + no_output=(), + models=None, + pool=FakePool, + ): + self.tmp = tmp + prereg = tmp / "prereg.md" + prereg.write_text(PREREG) + frozen = cr.sha256_file(prereg) + master = cr.master_seed(frozen) + self.draw_order = cr.draw(master, POOL, n=N_DRAW) + board_path = tmp / "board" + board_path.mkdir() + (board_path / "reference_outputs.csv").write_text( + "scenario_id,variable,value\n" + ) + self.no_output = set(no_output) + scored = [ + (sid, v) for sid in POOL if sid not in self.no_output for v in VARIABLES + ] + self.board = { + "scenarios": {sid: scenario(sid) for sid in POOL}, + "variables": {sid: list(VARIABLES) for sid in POOL}, + "scored": pd.DataFrame(scored, columns=["scenario_id", "variable"]), + } + monkeypatch.setattr(pilot, "load_board", lambda path: self.board) + monkeypatch.setattr(pilot, "board_engine", lambda path=None: ENGINE) + monkeypatch.setitem( + pilot._STATED_ENGINE, "version", pilot._STATED_ENGINE["version"] + ) + self.models = tuple(models or cr.CHEAP + cr.FRONTIER) + self.design = { + "preregistration": str(prereg), + "frozen_sha256": frozen, + "runs": str(tmp / "runs"), + "stage": "confirm", + "board": str(board_path), + "engine": f"policyengine-us {ENGINE}", + "draw_order": self.draw_order, + "arms": {a: a for a in cr.ARMS}, + "frontier_arms": {a: f"f-{a}" for a in cr.FRONTIER_ARMS}, + "caps": {m: (caps or {}).get(m, 10.0) for m in cr.CHEAP + cr.FRONTIER}, + "reserves": {m: reserve for m in cr.CHEAP + cr.FRONTIER}, + "overall_cap": overall_cap, + "spent_before": spent_before, + } + self.readers = Readers() + self.provider = FakeProvider(self, specs) + self.run = cr.Run( + self.design, + models=self.models, + converse=self.provider, + pool_factory=pool, + completion_for=lambda profile: None, + host_factory=self.readers.host, + out=io.StringIO(), + ) + + def execute(self): + self.run.execute() + return self + + # What the runner wrote. + + def runs(self) -> Path: + return Path(self.design["runs"]) + + def lines(self, model, name="run-log.jsonl") -> list[dict]: + path = self.runs() / model / name + if not path.exists(): + return [] + return [json.loads(x) for x in path.read_text().splitlines() if x.strip()] + + def attempts(self, model) -> list[dict]: + return [x for x in self.lines(model) if "event" not in x] + + def events(self, model, kind=None) -> list[dict]: + return [ + x + for x in self.lines(model) + if x.get("event") and kind in (None, x["event"]) + ] + + def folder(self, model, label) -> Path: + arms = ( + self.design["arms"] if model in cr.CHEAP else self.design["frontier_arms"] + ) + return self.runs() / model / arms[label] / "confirm" / "households" + + +def labels_of(model): + return cr.ARMS if model in cr.CHEAP else cr.FRONTIER_ARMS + + +def host_from_files(folder: Path): + """The host evidence as the report builds it, from the archived files.""" + meta = json.loads((folder / "host.json").read_text()) + start, end = float(meta["start"]), float(meta["end"]) + samples = [ + json.loads(x) for x in (folder / "host-log.jsonl").read_text().splitlines() if x + ] + spans = cr.guard_spans( + (folder / "clamshell-guard.log").read_text(), meta["timezone"], end + ) + spans += cr.power_spans((folder / "pmset.log").read_text(), end) + times = sorted(float(s["time"]) for s in samples) + for a, b in zip([start] + times, times + [end]): + if b - a > cr.HOST_LOG_GAP: + spans.append((a, b, "host log gap")) + pressure = [float(s["time"]) for s in samples if int(s.get("pressure") or 0) > 1] + + class Host: + def trouble(self, s, e): + for a, b, what in spans: + if a <= e and b >= s: + return what + for t in pressure: + if s - cr.PRESSURE_WINDOW <= t <= e + cr.PRESSURE_WINDOW: + return "memory pressure" + return None + + return Host() + + +def run_problems(world: World) -> list[str]: + """Every rule of the run procedure, recomputed from the runner's files.""" + problems: list[str] = [] + host = host_from_files(world.runs() / "host") + position = {sid: i for i, sid in enumerate(world.draw_order)} + master = cr.master_seed(world.design["frozen_sha256"]) + all_attempts = [] + ceiling_events = {} + for model in world.models: + labels = labels_of(model) + drawn = cr.orders(master, model, world.draw_order, labels) + lines = world.attempts(model) + events = world.events(model) + stops = [e["time"] for e in events if e["event"] in ("stop", "ceiling")] + stopped_at = min(stops) if stops else math.inf + ceiling_events[model] = [e["time"] for e in events if e["event"] == "ceiling"] + cost: dict[str, float] = {} + span: dict[str, list[float]] = {} + by_run: dict = {} + for x in lines: + by_run.setdefault((x["household"], x["arm"]), []).append(x) + # Records and hashes. + for label in labels: + folder = world.folder(model, label) + finals = sorted(folder.glob("*.json")) if folder.is_dir() else [] + for path in finals: + if (path.stem, label) not in by_run: + problems.append( + f"{model}/{label}/{path.stem}: a record with no line" + ) + for (sid, label), xs in sorted(by_run.items()): + at = f"{model}/{label}/{sid}" + xs.sort(key=lambda x: x["attempt"]) + if [x["attempt"] for x in xs] != list(range(1, len(xs) + 1)): + problems.append(f"{at}: attempts {[x['attempt'] for x in xs]}") + continue + final_path = world.folder(model, label) / f"{sid}.json" + final = json.loads(final_path.read_text()) + entries = final.get("attempts") or [] + if len(entries) != len(xs) - 1: + problems.append(f"{at}: {len(entries)} entries for {len(xs)} attempts") + continue + for k, x in enumerate(xs, start=1): + if k < len(xs): + entry = entries[k - 1] + path = world.folder(model, label) / entry["file"] + rec = json.loads(path.read_text()) + if entry["n"] != k or entry["sha256"] != cr.sha256_file(path): + problems.append(f"{at}#{k}: its entry does not hash its file") + if entry["class"] != x["class"] or entry["status"] != rec["status"]: + problems.append(f"{at}#{k}: its entry is not its line's") + if x["sha256"] != entry["sha256"]: + problems.append(f"{at}#{k}: its line's hash is not its file's") + else: + rec = final + if x["sha256"] != cr.sha256_file(final_path): + problems.append( + f"{at}#{k}: its line's hash is not its record's" + ) + cls, _ = cr.classify(rec, x["start"], x["end"], host) + rule = cr.action(cls, k) + if x["class"] != cls or x["action"] != rule: + problems.append( + f"{at}#{k}: logged {x['class']}/{x['action']}, " + f"recomputed {cls}/{rule}" + ) + if x["status"] != rec["status"] or x["lane"] != 1 + position[sid] % 2: + problems.append(f"{at}#{k}: status or lane") + if k < len(xs) and rule not in ("rerun", "pause-rerun"): + problems.append(f"{at}#{k}: rerun after {cls}") + if k == len(xs) and rule in ("rerun", "pause-rerun") and not stops: + problems.append( + f"{at}#{k}: a rerun due and never run, with no stop" + ) + times = [ + t[key] + for t in rec.get("turns") or [] + for key in ("request_started_at", "request_completed_at") + ] + if times and (min(times) < x["start"] or max(times) > x["end"]): + problems.append(f"{at}#{k}: requests outside its start and end") + if x["start"] > stopped_at: + problems.append(f"{at}#{k}: started after a stop") + cost[sid] = cost.get(sid, 0.0) + cr.attempt_cost(rec) + s, e = span.get(sid, [math.inf, -math.inf]) + span[sid] = [min(s, x["start"]), max(e, x["end"])] + all_attempts.append((model, sid, label, k, x, rec)) + # The schedule. + for lane in (1, 2): + mine = sorted( + (x for x in lines if x["lane"] == lane), key=lambda x: x["start"] + ) + for a, b in zip(mine, mine[1:]): + if b["start"] < a["end"]: + problems.append(f"{model} lane {lane}: attempts overlap") + blocks: list = [] + for x in mine: + if blocks and blocks[-1][0] == x["household"]: + blocks[-1][1].append(x) + else: + blocks.append((x["household"], [x])) + sids = [s for s, _ in blocks] + if len(set(sids)) != len(sids) or sids != sorted(sids, key=position.get): + problems.append( + f"{model} lane {lane}: households split or out of order" + ) + for i, (sid, block) in enumerate(blocks): + got = [(x["attempt"], x["arm"]) for x in block] + firsts = [a for n, a in got if n == 1] + want = [(1, a) for a in firsts] + for n in range(2, cr.MAX_RERUNS + 2): + again = {x["arm"] for x in block if x["attempt"] == n} + want += [(n, a) for a in drawn[sid] if a in again] + cut = firsts != drawn[sid] + if got != want or firsts != drawn[sid][: len(firsts)]: + problems.append(f"{model}/{sid}: ran {got}, drawn {drawn[sid]}") + if cut and not (stops and i == len(blocks) - 1): + problems.append(f"{model}/{sid}: cut short without a stop") + # Admission. + decisions = sorted( + world.lines(model, "admission-log.jsonl"), key=lambda x: x["time"] + ) + cap = world.design["caps"][model] + reserve = world.design["reserves"][model] + admitted: dict[str, float] = {} + refused = False + for x in decisions: + sid, t = x["household"], x["time"] + if sid in world.no_output: + problems.append( + f"{model}/{sid}: a household with no scored output decided" + ) + finished = [h for h in admitted if h in span and span[h][1] <= t] + spend = sum(cost[h] for h in finished) + under = sum(1 for h in admitted if h not in finished) + 1 + ok = not refused and spend + under * reserve <= cap + cr.ADMISSION_SLACK + if x["decision"] != ("admit" if ok else "refuse"): + problems.append(f"{model}/{sid}: logged {x['decision']}, rule {ok}") + if abs(x["finished_spend"] - spend) > 1e-9 or x["under_way"] != under: + problems.append( + f"{model}/{sid}: spend {x['finished_spend']} vs {spend}" + ) + if x["lane"] != 1 + position[sid] % 2: + problems.append(f"{model}/{sid}: decided on the other lane") + if x["decision"] == "admit": + admitted[sid] = t + else: + refused = True + for sid, (s, _) in span.items(): + if sid not in admitted or s < admitted[sid]: + problems.append(f"{model}/{sid}: ran without, or before, its admission") + for lane in (1, 2): + expect = [ + s + for s in world.draw_order + if 1 + position[s] % 2 == lane and s not in world.no_output + ] + decided = [x["household"] for x in decisions if x["lane"] == lane] + if decided != expect[: len(decided)]: + problems.append(f"{model} lane {lane}: decisions skip a household") + if len(decided) < len(expect) and not (refused or stops): + problems.append(f"{model} lane {lane}: stopped deciding early") + if not stops and set( + a for a in admitted if 1 + position[a] % 2 == lane + ) - set(span): + problems.append(f"{model} lane {lane}: admitted and never run") + # Pauses. + resumes = sorted(e["time"] for e in events if e["event"] == "resume") + for x in lines: + if x["class"] != "operator": + continue + pauses = [ + e for e in events if e["event"] == "pause" and e["time"] >= x["end"] - 1 + ] + if not pauses: + problems.append( + f"{model}/{x['household']}: an operator failure, no pause" + ) + continue + until = next((t for t in resumes if t >= pauses[0]["time"]), math.inf) + if any(x["end"] < y["start"] < until for y in lines): + problems.append(f"{model}: an attempt started during a pause") + # The ceiling, over every model. + total = world.design["spent_before"] + sum( + cr.attempt_cost(r) for *_, r in all_attempts + ) + ceiling = world.design["overall_cap"] + cr.CEILING_MARGIN + if total >= ceiling and not all(ceiling_events.values()): + problems.append(f"spend {total} reached the ceiling and a model has no event") + for times in ceiling_events.values(): + for t in times: + if any(x["start"] > t for *_, x, _ in all_attempts): + problems.append("an attempt started after the ceiling") + # Conservation: the ceiling counted exactly the records' spend. + if abs(world.run.ceiling.spent_usd - total) > 1e-9: + problems.append( + f"the ceiling counted {world.run.ceiling.spent_usd}, records {total}" + ) + return problems + + +@pytest.fixture +def world(tmp_path, monkeypatch, fast): + """A fake run to plant failures in, then execute.""" + + def make(**kwargs): + return World(tmp_path, monkeypatch, **kwargs) + + return make + + +def test_an_untouched_run_runs_every_household_and_arm(world): + w = world().execute() + assert run_problems(w) == [] + for model in w.models: + labels = labels_of(model) + attempts = w.attempts(model) + assert len(attempts) == N_DRAW * len(labels) + assert {x["class"] for x in attempts} == {"ok"} + assert all( + x["decision"] == "admit" for x in w.lines(model, "admission-log.jsonl") + ) + manifests = list((w.runs() / model).glob("manifest-*.json")) + assert len(manifests) == 1 + m = json.loads(manifests[0].read_text()) + assert m["arms"] == { + label: { + "arm": cr.arm_string(label), + "features": sorted(cr.arm_profile(model, label).tool_features), + } + for label in labels + } + assert m["treatment_commit"] == cr.TREATMENT_COMMIT + assert m["cap_usd"] == 10.0 and m["reserve_usd"] == pytest.approx(0.21) + assert m["board_engine"] == f"policyengine-us {ENGINE}" + assert not (w.runs() / model / "in-flight").exists() or not any( + (w.runs() / model / "in-flight").iterdir() + ) + host = json.loads((w.runs() / "host" / "host.json").read_text()) + assert host["timezone"] == TZ and host["end"] > host["start"] + + +def test_households_alternate_lanes_in_draw_order(world): + w = world(models=("gpt-6-luna",)).execute() + lanes = {x["household"]: x["lane"] for x in w.attempts("gpt-6-luna")} + assert [lanes[s] for s in w.draw_order] == [1, 2] * (N_DRAW // 2) + + +def test_each_household_runs_its_arms_in_its_drawn_order(world): + w = world(models=("claude-haiku-5.5",)).execute() + master = cr.master_seed(w.design["frozen_sha256"]) + drawn = cr.orders(master, "claude-haiku-5.5", w.draw_order, cr.ARMS) + for sid in w.draw_order: + mine = sorted( + (x for x in w.attempts("claude-haiku-5.5") if x["household"] == sid), + key=lambda x: x["start"], + ) + assert [x["arm"] for x in mine] == drawn[sid] + + +def test_each_call_carries_its_arm_s_features(world): + calls = {"situation": "{}", "variables": ["snap"]} + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + w.provider.specs = {(model, label, sid, 1): {"calls": [calls]} for label in cr.ARMS} + w.execute() + messages = w.run.lanes[model].pool.messages + assert len(messages) == len(cr.ARMS) + raw = [m for m in messages if not isinstance(m, tuple)] + framed = [m for m in messages if isinstance(m, tuple)] + assert raw == [calls] # the tool v1 arm: as a featureless pool sends it + want = { + tuple(sorted(cr.arm_profile(model, label).tool_features)) + for label in cr.ARMS + if label != "pe" + } + assert {m[1] for m in framed} == want + assert all(m[0] == pilot.CALL_FRAME and m[2] == calls for m in framed) + # The worker reads each frame's features as the arm's. + for m in framed: + assert pilot.frame_features(m[1]) == frozenset(m[1]) + assert run_problems(w) == [] + + +def test_households_with_no_scored_output_are_never_decided(world): + model = "gemini-4-argon" + master = cr.master_seed(hashlib.sha256(PREREG.encode()).hexdigest()) + skipped = cr.draw(master, POOL, n=N_DRAW)[1] + w = world(models=(model,), no_output=(skipped,)).execute() + assert skipped not in { + x["household"] for x in w.lines(model, "admission-log.jsonl") + } + assert skipped not in {x["household"] for x in w.attempts(model)} + assert run_problems(w) == [] + + +def test_a_provider_error_is_rerun_after_the_household_s_first_pass(world): + model = "claude-haiku-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + master = cr.master_seed(w.design["frozen_sha256"]) + first = cr.orders(master, model, w.draw_order, cr.ARMS)[sid][0] + error = {"status": "provider_error", "error": "RateLimitError: slow down"} + w.provider.specs = {(model, first, sid, 1): error, (model, first, sid, 2): error} + w.execute() + mine = sorted( + (x for x in w.attempts(model) if x["household"] == sid), + key=lambda x: x["start"], + ) + assert [(x["attempt"], x["arm"]) for x in mine][-2:] == [(2, first), (3, first)] + assert [x["class"] for x in mine if x["arm"] == first] == ["provider"] * 2 + ["ok"] + final = json.loads((w.folder(model, first) / f"{sid}.json").read_text()) + assert [e["n"] for e in final["attempts"]] == [1, 2] + assert final["attempts"][0]["reason"] == "RateLimitError" + assert (w.folder(model, first) / "attempts" / f"{sid}-1.json").exists() + assert run_problems(w) == [] + + +def test_a_third_attempt_stands_whatever_its_class(world): + model = "gpt-6-sol" + w = world(models=(model,)) + sid = w.draw_order[1] + error = {"status": "provider_error", "error": "APIConnectionError: reset"} + w.provider.specs = {(model, "pe3", sid, n): error for n in (1, 2, 3)} + w.execute() + mine = [x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe3"] + assert sorted((x["attempt"], x["action"]) for x in mine) == [ + (1, "rerun"), + (2, "rerun"), + (3, "stands"), + ] + assert run_problems(w) == [] + + +def test_host_trouble_during_an_attempt_reruns_it(world): + model = "gemini-3.5-flash-lite" + w = world(models=(model,)) + pressured, stalled = w.draw_order[0], w.draw_order[2] # both lane 1 + w.provider.specs = { + (model, "loo-flat", pressured, 1): {"pressure": True, "sleep": 0.05}, + (model, "pe", stalled, 1): {"stall": 0.15}, + } + w.execute() + + def first(sid, label): + line = next( + x + for x in w.attempts(model) + if (x["household"], x["arm"], x["attempt"]) == (sid, label, 1) + ) + final = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + return line, final["attempts"][0] + + line, entry = first(pressured, "loo-flat") + assert line["class"] == entry["class"] == "host" and line["action"] == "rerun" + assert "memory pressure 2 at" in entry["reason"] + line, entry = first(stalled, "pe") + assert line["class"] == "host" and "host log gap" in entry["reason"] + assert run_problems(w) == [] + + +def test_a_guard_pause_over_an_attempt_reruns_it(world): + model = "gpt-6-sol" + w = world(models=(model,)) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe3", sid, 1): {"guard": True}} + w.execute() + line = next( + x + for x in w.attempts(model) + if (x["household"], x["arm"], x["attempt"]) == (sid, "pe3", 1) + ) + assert line["class"] == "host" + final = json.loads((w.folder(model, "pe3") / f"{sid}.json").read_text()) + assert final["attempts"][0]["reason"].startswith("guard pause: guard pause ") + archived = (w.runs() / "host" / "clamshell-guard.log").read_text() + assert "paused pids" in archived + assert run_problems(w) == [] + + +def test_an_operator_failure_pauses_the_model_until_resumed(world): + model = "gpt-6-luna" + w = world(models=(model, "claude-haiku-5.5")) + sid = w.draw_order[0] + w.provider.specs = { + (model, "pe3", sid, 1): { + "status": "provider_error", + "error": "AuthenticationError: bad key", + "turns": 0, + "resume_after": 0.4, + } + } + w.execute() + pauses, resumes = w.events(model, "pause"), w.events(model, "resume") + assert len(pauses) == 1 and len(resumes) == 1 + assert resumes[0]["reason"] == "fixed in a test" + started = [x["start"] for x in w.attempts(model)] + assert not [t for t in started if pauses[0]["time"] < t < resumes[0]["time"]] + assert resumes[0]["time"] - pauses[0]["time"] >= 0.35 + rerun = [ + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe3" + ] + assert [x["action"] for x in sorted(rerun, key=lambda x: x["attempt"])] == [ + "pause-rerun", + "stands", + ] + assert not w.events("claude-haiku-5.5", "pause") # another model runs on + assert list((w.runs() / model).glob("resume-*.json")) + assert run_problems(w) == [] + + +def test_a_request_class_on_the_first_request_stops_the_model(world): + model = "claude-opus-5.5" + w = world(models=(model, "gpt-6-sol")) + sid = w.draw_order[2] + w.provider.specs = { + (model, "pe", sid, 1): { + "status": "provider_error", + "error": "BadRequestError: bad tools", + "turns": 0, + } + } + w.execute() + stops = w.events(model, "stop") + assert len(stops) == 1 and "first request" in stops[0]["reason"] + assert not [x for x in w.attempts(model) if x["start"] > stops[0]["time"]] + assert len(w.attempts("gpt-6-sol")) == N_DRAW * 2 # the other model finishes + assert run_problems(w) == [] + + +def test_one_inconclusive_error_in_six_households_stops_the_model(world): + model = "gpt-6-luna" + w = world(models=(model,)) + error = {"status": "provider_error", "error": "ProviderEmpty: no choices"} + w.provider.specs = {(model, "pe", sid, 1): error for sid in w.draw_order} + w.execute() + stops = w.events(model, "stop") + assert len(stops) == 1 + assert ( + stops[0]["reason"] == "provider_error: ProviderEmpty ended runs in 6 households" + ) + assert run_problems(w) == [] + + +def test_five_households_with_one_inconclusive_error_do_not_stop(world): + model = "gpt-6-luna" + w = world(models=(model,)) + error = {"status": "provider_error", "error": "ProviderEmpty: no choices"} + w.provider.specs = {(model, "pe", sid, 1): error for sid in w.draw_order[:5]} + w.execute() + assert not w.events(model, "stop") + assert len(w.attempts(model)) == N_DRAW * len(cr.ARMS) + assert run_problems(w) == [] + + +def test_a_refusal_ends_both_lanes_decisions(world): + model = "claude-haiku-5.5" + # Each household costs 7 arms x $0.01; the reserve is $0.21 per household. + w = world(models=(model,), caps={model: 0.5}).execute() + decisions = w.lines(model, "admission-log.jsonl") + refused = [x for x in decisions if x["decision"] == "refuse"] + after = decisions[decisions.index(refused[0]) :] + assert refused and all(x["decision"] == "refuse" for x in after) + admitted = {x["household"] for x in decisions if x["decision"] == "admit"} + assert {x["household"] for x in w.attempts(model)} == admitted + assert run_problems(w) == [] + + +def test_spend_never_stops_a_conversation_below_the_ceiling(world): + model = "gemini-4-argon" + w = world(models=(model,), caps={model: 0.05}, reserve=0.01) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe", sid, 1): {"turns": 4, "cost": 0.5}} + w.execute() + # $2 on one arm, far past the cap: it ran to the end; later households + # were refused. + ran = [x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe"] + assert ran[0]["status"] == "complete" + assert any(x["decision"] == "refuse" for x in w.lines(model, "admission-log.jsonl")) + assert run_problems(w) == [] + + +def test_the_hard_ceiling_stops_every_model(world): + w = world( + models=("claude-haiku-5.5", "gpt-6-sol", "gemini-4-argon"), + overall_cap=0.0, + spent_before=24.0, + ) + sid = w.draw_order[1] + # One long conversation crosses $25 while the others run. + w.provider.specs = { + ("gpt-6-sol", "pe3", sid, 1): {"turns": 3, "cost": 0.5, "sleep": 0.02} + } + w.execute() + events = {m: w.events(m, "ceiling") for m in w.models} + assert all(len(e) == 1 for e in events.values()) + crossed = events["gpt-6-sol"][0]["time"] + assert {e[0]["time"] for e in events.values()} == {crossed} + assert w.run.ceiling.spent_usd >= 25.0 + for m in w.models: + assert not [x for x in w.attempts(m) if x["start"] > crossed] + assert run_problems(w) == [] + + +def test_a_conversation_under_way_at_the_ceiling_ends_budget_stop(world): + model = "gpt-6-luna" + w = world(models=(model,), overall_cap=0.0, spent_before=24.9) + sid = w.draw_order[0] + master = cr.master_seed(w.design["frozen_sha256"]) + first = cr.orders(master, model, w.draw_order, cr.ARMS)[sid][0] + w.provider.specs = {(model, first, sid, 1): {"turns": 5, "cost": 0.06}} + w.execute() + line = next( + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == first + ) + assert line["status"] == "budget_stop" and line["class"] == "ceiling" + assert line["action"] == "stopped" + assert run_problems(w) == [] + + +def test_an_attempt_that_raises_is_saved_as_a_harness_error_record(world): + model = "claude-opus-5.5" + w = world(models=(model,)) + sid = w.draw_order[3] + w.provider.specs = {(model, "pe3", sid, 1): {"raise": True}} + w.execute() + replaced = json.loads( + (w.folder(model, "pe3") / "attempts" / f"{sid}-1.json").read_text() + ) + assert replaced["status"] == "harness_error" and replaced["turns"] == [] + assert replaced["error"] == "RuntimeError: a test raised" + assert replaced["tool_features"] == sorted( + cr.arm_profile(model, "pe3").tool_features + ) + line = next( + x + for x in w.attempts(model) + if x["household"] == sid and x["attempt"] == 1 and x["arm"] == "pe3" + ) + assert line["class"] == "none" and line["action"] == "rerun" + assert run_problems(w) == [] + + +def test_a_pool_that_lost_every_worker_is_started_again(world): + model = "gpt-6-sol" + starts = [] + + class Losing(FakePool): + def __init__(self, lanes): + super().__init__(lanes) + starts.append(self) + + def call(self, message): + self.lost = 2 # every worker gone after this call + return super().call(message) + + w = world(models=(model,), pool=Losing) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe", sid, 1): {"calls": [{"situation": "{}"}]}} + w.execute() + assert len(starts) == 2 and starts[0].closed + assert run_problems(w) == [] + + +def test_a_pool_that_cannot_start_again_stops_the_model(world, monkeypatch): + model = "gpt-6-sol" + made = [] + + def pool(lanes): + made.append(1) + return FakePool(lanes, fail=len(made) > 1) + + w = world(models=(model,), pool=pool) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe", sid, 1): {"calls": [{"situation": "{}"}]}} + original = FakePool.call + + def losing(self, message): + self.lost = 2 + return original(self, message) + + monkeypatch.setattr(FakePool, "call", losing) + w.execute() + stops = w.events(model, "stop") + assert stops and "engine pool" in stops[0]["reason"] + assert run_problems(w) == [] + + +def test_an_interrupt_stops_every_model_and_lets_attempts_end(world): + w = world(models=("gpt-6-luna", "gpt-6-sol")) + sid = w.draw_order[0] + w.provider.specs = {("gpt-6-luna", lbl, sid, 1): {"sleep": 0.1} for lbl in cr.ARMS} + threading.Timer(0.15, lambda: w.run.interrupt(2, None)).start() + w.execute() + for m in w.models: + stops = w.events(m, "stop") + assert len(stops) == 1 and stops[0]["reason"] == "operator stop (signal 2)" + assert not [x for x in w.attempts(m) if x["start"] > stops[0]["time"]] + luna = [x for x in w.attempts("gpt-6-luna") if x["household"] == sid] + assert luna and len(luna) < len(cr.ARMS) # cut by the stop + assert run_problems(w) == [] + + +KINDS = { + "ok": {}, + "provider": {"status": "provider_error", "error": "RateLimitError: slow"}, + "none": {"status": "harness_error", "error": "RuntimeError: lost", "turns": 0}, + "model": {"status": "provider_error", "error": "BadRequestError: long"}, + "inconclusive": {"status": "provider_error", "error": "ProviderEmpty: none"}, + "pressure": {"pressure": True, "sleep": 0.04}, + "incomplete": {"status": "incomplete"}, + "harness": {"status": "provider_error", "error": "BadRequestError: x", "turns": 0}, + "costly": {"turns": 3, "cost": 0.2}, +} + + +@settings( + max_examples=12, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture, HealthCheck.too_slow], +) +@given( + planted=st.lists( + st.tuples( + st.sampled_from(("claude-haiku-5.5", "gpt-6-sol")), + st.sampled_from(cr.FRONTIER_ARMS), + st.integers(0, N_DRAW - 1), + st.integers(1, 3), + st.sampled_from(sorted(KINDS)), + ), + max_size=10, + ), + cap=st.sampled_from([0.3, 0.6, 10.0]), + spent_before=st.sampled_from([0.0, 24.0]), +) +def test_every_run_keeps_the_run_procedure( + tmp_path_factory, fast, planted, cap, spent_before +): + """Whatever fails, and wherever spend lands, the records follow every rule.""" + tmp = tmp_path_factory.mktemp("run") + mp = pytest.MonkeyPatch() + try: + models = ("claude-haiku-5.5", "gpt-6-sol") + w = World( + tmp, + mp, + models=models, + caps={m: cap for m in models}, + overall_cap=0.0, + spent_before=spent_before, + ) + w.provider.specs = { + (model, label, w.draw_order[i], n): KINDS[kind] + for model, label, i, n, kind in planted + } + w.execute() + assert run_problems(w) == [] + finally: + mp.undo() + + +def test_the_checker_catches_a_planted_fault(world): + """run_problems is not vacuous: an edited log line fails it.""" + w = world(models=("gpt-6-sol",)).execute() + path = w.runs() / "gpt-6-sol" / "run-log.jsonl" + lines = path.read_text().splitlines() + first = json.loads(lines[0]) + first["class"] = "provider" + path.write_text("\n".join([json.dumps(first)] + lines[1:]) + "\n") + assert any("recomputed" in p for p in run_problems(w)) + + +# ---------------------------------------------------------------- before a run + + +def test_the_preflight_names_a_missing_field(): + assert cr.preflight({}, cr.CHEAP) == [f"the design has no {', '.join(cr.REQUIRED)}"] + + +def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( + world, tmp_path, monkeypatch +): + w = world(models=("gpt-6-luna",)) + pool = tmp_path / "pool.json" + pool.write_text(json.dumps([{"id": s} for s in POOL])) + design = dict( + w.design, + eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, + treatment_harness=str(cr.ROOT), + run_commit="0" * 40, + draw_order=list(reversed(w.draw_order)), + ) + monkeypatch.setattr(cr, "host_power", lambda: (True, False)) + problems = cr.preflight(design, ("gpt-6-luna",)) + text = "\n".join(problems) + assert "the draw, recomputed" in text + assert "not the run commit, clean" in text + assert "lid open False" in text + design["draw_order"] = w.draw_order + assert "the draw" not in "\n".join(cr.preflight(design, ("gpt-6-luna",))) + + +def test_the_preflight_refuses_a_runs_folder_with_records(world, tmp_path): + w = world(models=("gpt-6-luna",)).execute() + pool = tmp_path / "pool.json" + pool.write_text(json.dumps([{"id": s} for s in POOL])) + design = dict( + w.design, + eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, + treatment_harness=str(cr.ROOT), + run_commit="0" * 40, + ) + text = "\n".join(cr.preflight(design, ("gpt-6-luna",))) + assert "already holds records" in text and "already holds a host log" in text + + +def test_resume_writes_the_operator_s_note(tmp_path): + design = tmp_path / "design.json" + design.write_text(json.dumps({"runs": str(tmp_path / "runs")})) + assert cr.main(["resume", str(design), "gpt-6-luna", "--note", "key rotated"]) == 0 + note = json.loads((tmp_path / "runs" / "gpt-6-luna" / "RESUME.json").read_text()) + assert note["note"] == "key rotated" + + +# ---------------------------------------------------------------- the harness + + +def chat_response(calls): + from types import SimpleNamespace + + message = SimpleNamespace( + role="assistant", content=None, tool_calls=calls, function_call=None + ) + usage = SimpleNamespace( + prompt_tokens=1000, + completion_tokens=200, + total_tokens=1200, + reasoning_tokens=0, + prompt_tokens_details=SimpleNamespace(cached_tokens=0), + completion_tokens_details=None, + ) + return SimpleNamespace( + choices=[SimpleNamespace(message=message, finish_reason="tool_calls")], + usage=usage, + model="claude-haiku-5-5", + ) + + +def chat_call(name, arguments, call_id): + from types import SimpleNamespace + + return SimpleNamespace( + id=call_id, + type="function", + function=SimpleNamespace(name=name, arguments=json.dumps(arguments)), + ) + + +def stateless_model(**kwargs): + """A tool-contract model: one calculation, then its answers. It reads the + conversation so far, so the two lanes can share it.""" + if not any(m.get("role") == "tool" for m in kwargs["messages"]): + arguments = {"situation": json.dumps({"people": {}}), "variables": ["snap"]} + return chat_response([chat_call(pilot.TOOL_NAME, arguments, "c1")]) + outputs = {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + return chat_response([chat_call("submit_outputs", {"outputs": outputs}, "s1")]) + + +def test_the_harness_conversation_runs_under_the_runner(world): + model = "claude-haiku-5.5" + w = world(models=(model,)) + w.run.converse = cr.converse_with_harness + w.run.completion_for = lambda profile: stateless_model + w.execute() + attempts = w.attempts(model) + assert len(attempts) == N_DRAW * len(cr.ARMS) + assert {x["status"] for x in attempts} == {"complete"} + messages = w.run.lanes[model].pool.messages + assert len(messages) == N_DRAW * len(cr.ARMS) # one calculation per attempt + frames = [m for m in messages if isinstance(m, tuple)] + assert len(frames) == N_DRAW * (len(cr.ARMS) - 1) # every arm but pe + sid = w.draw_order[0] + rec = json.loads((w.folder(model, "loo-names") / f"{sid}.json").read_text()) + profile = cr.arm_profile(model, "loo-names") + assert rec["arm"] == cr.arm_string("loo-names") and rec["attempts"] == [] + assert rec["tool_features"] == sorted(profile.tool_features) + assert rec["predictions"] == {v: 0.0 for v in VARIABLES} + assert run_problems(w) == [] + + +def test_the_ceiling_ends_a_harness_conversation_at_its_next_turn(world): + model = "claude-haiku-5.5" + w = world(models=(model,), overall_cap=0.0, spent_before=25.0 - 1e-9) + w.run.converse = cr.converse_with_harness + w.run.completion_for = lambda profile: stateless_model + w.execute() + # The first turns cross the ceiling; each conversation under way ends + # budget_stop before its next request, and no attempt starts after. + attempts = w.attempts(model) + assert attempts and {x["status"] for x in attempts} == {"budget_stop"} + assert len(attempts) <= 2 # one per lane at most + assert len(w.events(model, "ceiling")) == 1 + assert run_problems(w) == [] From ec31e835ece3adc83dd6cbed9a5972542a9d7a6e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 03:22:37 -0400 Subject: [PATCH 19/64] Confirmatory runner: create both logs at the start A model whose lanes never decide, because the ceiling was crossed before its first admission, wrote no admission log, and the confirmatory report reads a missing log as a lost one. The runner now creates each model's run and admission logs when it writes the model's manifest. Found by the differential test in the study's scripts repo, which runs this runner and checks its records with the report. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 3 +++ tests/test_confirmatory_run.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index cc61b52b..1b7b0566 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -1179,6 +1179,9 @@ def execute(self) -> None: write_json( lanes.folder / f"manifest-{stamp}.json", lanes.manifest(), indent=2 ) + # A model whose lanes never decide still has both logs. + for name in ("run-log.jsonl", "admission-log.jsonl"): + (lanes.folder / name).touch() starting[model] = threading.Thread( target=lanes.ensure_pool, daemon=True ) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index f42a1c8e..c7651586 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -1134,6 +1134,31 @@ def test_the_hard_ceiling_stops_every_model(world): assert run_problems(w) == [] +def test_a_model_that_decides_nothing_still_has_both_logs(world): + """haiku's first attempt crosses the ceiling before gpt-6-sol's lanes, + held back, decide: gpt-6-sol's run log holds only the ceiling event and + its admission log is empty.""" + w = world( + models=("claude-haiku-5.5", "gpt-6-sol"), overall_cap=0.0, spent_before=24.5 + ) + sid = w.draw_order[0] + first = w.run.lanes["claude-haiku-5.5"].orders[sid][0] + w.provider.specs = {("claude-haiku-5.5", first, sid, 1): {"cost": 1.0}} + held = w.run.lanes["gpt-6-sol"] + admit = held.admit + + def late(sid, lane): + time.sleep(0.3) + return admit(sid, lane) + + held.admit = late + w.execute() + admission = w.runs() / "gpt-6-sol" / "admission-log.jsonl" + assert admission.exists() and admission.read_text() == "" + assert [x["event"] for x in w.lines("gpt-6-sol")] == ["ceiling"] + assert run_problems(w) == [] + + def test_a_conversation_under_way_at_the_ceiling_ends_budget_stop(world): model = "gpt-6-luna" w = world(models=(model,), overall_cap=0.0, spent_before=24.9) From 808ccc8abc3962e534d52633f2ff5909d02a64da Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 04:11:22 -0400 Subject: [PATCH 20/64] Confirmatory runner: a clock that never goes back, and three test-oracle fixes The wall clock steps back by up to about 0.1 ms between threads on the run's host (40 backward steps in 190 million readings over 12 threads, the largest 91 us). The run records order attempts, stops, admission decisions and the ceiling's crossing by their times, and the confirmatory report compares them exactly: an attempt's end and the next start in its lane can be microseconds apart, so a step back could make a valid run fail the report. `clock()` gives time.time(), never earlier than a time it gave before, and each of those times is taken under the lock that orders it. A test reads the wall clock up to 0.2 ms early everywhere; without `clock()` it fails every time (attempts overlap, a household runs before its admission). The test oracle had three faults, two of them exposed by the full suite: - A household a stop cut short was taken to finish at its last attempt. The runner counts it under way until the stop, as the report now does. A new test makes the case deterministic (it failed the old oracle with the flake's message, spend 0.02 vs 0.03). - The scaled host-log gap was 90 ms, and under a load of 12 the sampler stalled past it, so an untouched run got host reruns. The gap is 0.5 s in the tests, and the stall tests stall 0.8 s. - The oracle allows the harness's raw request times a second outside their attempt, as the report does. 121 tests pass, 8 runs of the full suite in a row at a load of 12 to 146. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 25 ++++++-- tests/test_confirmatory_run.py | 113 +++++++++++++++++++++++++++++---- 2 files changed, 121 insertions(+), 17 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 1b7b0566..0b7f0f9b 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -331,6 +331,21 @@ def attempt_cost(record: dict) -> float: return float((record.get("totals") or {}).get("cost_usd") or 0.0) +_clock_lock = threading.Lock() +_clock_last = 0.0 + + +def clock() -> float: + """`time.time()`, never earlier than a time it gave before. The wall clock + can step back by about 0.1 ms, even within a thread, and the run records order + attempts, stops, admission decisions and the ceiling's crossing by the + times this gives; each is taken under the lock that orders it.""" + global _clock_last + with _clock_lock: + _clock_last = max(time.time(), _clock_last) + return _clock_last + + # ---------------------------------------------------------------- host evidence @@ -613,7 +628,7 @@ def add(self, usd: float | None) -> None: with self._lock: self.spent_usd += float(usd or 0.0) if self.crossed_at is None and self.spent_usd >= self.budget_usd: - self.crossed_at = time.time() + self.crossed_at = clock() def exhausted(self, reserve_usd: float = 0.0) -> bool: with self._lock: @@ -716,7 +731,7 @@ def write_line(self, line: dict, name: str = "run-log.jsonl") -> None: def event(self, kind: str, reason: str, at: float | None = None) -> None: self.write_line( - {"event": kind, "time": time.time() if at is None else at, "reason": reason} + {"event": kind, "time": clock() if at is None else at, "reason": reason} ) self.run.say(f"{self.model}: {kind} ({reason})") @@ -765,7 +780,7 @@ def admit(self, sid: str, lane: int) -> bool: self.cond.wait(timeout=WAIT_SECONDS) if self.halted(): return False - now = time.time() + now = clock() finished = [h for h, s in self.state.items() if s == "finished"] spend = sum(self.spent[h] for h in finished) under_way = len(self.state) - len(finished) + 1 @@ -803,7 +818,7 @@ def start_attempt(self) -> float | None: with self.run.ceiling.lock: if self.stopped or self.run.ceiling.crossed_at is not None: return None - return time.time() + return clock() def end_attempt( self, sid: str, label: str, n: int, start: float, record: dict, last: bool @@ -812,7 +827,7 @@ def end_attempt( once: an operator failure pauses the lanes, a harness failure stops them, before any other attempt can start.""" with self.cond: - end = time.time() + end = clock() self.spent[sid] += attempt_cost(record) attempt = Attempt(label, n, start, end, record) cls, detail = classify(record, start, end, None) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index c7651586..63acc702 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -42,6 +42,10 @@ TZ = "America/New_York" PREREG = "A frozen pre-registration for a test.\n" ENGINE = "2.60.4" +# The harness times requests with the raw wall clock, which steps back by +# about 0.1 ms on this host, even within a thread; the report allows a second, as +# here, which also covers a small correction of the clock. +CLOCK_SLACK = 1.0 # ---------------------------------------------------------------- streams @@ -283,9 +287,11 @@ def host(self, folder): @pytest.fixture def fast(monkeypatch): - """Windows scaled down a thousandfold, so a fake run takes a second.""" + """Windows scaled down a thousandfold, so a fake run takes a second; the + host-log gap less, so a sampler a busy host stalls past 90 ms (seen at a + load of 12) is not a gap.""" monkeypatch.setattr(cr, "PRESSURE_WINDOW", 0.06) - monkeypatch.setattr(cr, "HOST_LOG_GAP", 0.09) + monkeypatch.setattr(cr, "HOST_LOG_GAP", 0.5) monkeypatch.setattr(cr, "WAIT_SECONDS", 0.005) monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) @@ -328,8 +334,8 @@ def test_a_stalled_host_log_is_a_gap(tmp_path, fast): host = readers.host(tmp_path) time.sleep(0.03) start = time.time() - readers.stall = 0.2 - time.sleep(0.25) + readers.stall = 0.8 + time.sleep(0.85) end = time.time() host.wait_settled(end) host.refresh(after=end) @@ -358,6 +364,34 @@ def test_guard_pauses_and_sleeps_reach_an_attempt_after_a_refresh(tmp_path, fast # ---------------------------------------------------------------- the ceiling +def test_the_clock_never_goes_back(monkeypatch): + monkeypatch.setattr(cr, "_clock_last", cr._clock_last) # restored after + base = time.time() + 1000.0 + readings = iter([base, base - 1e-4, base + 2e-4, base - 5.0, base + 3e-4]) + monkeypatch.setattr(time, "time", lambda: next(readings)) + assert [cr.clock() for _ in range(5)] == [ + base, + base, + base + 2e-4, + base + 2e-4, + base + 3e-4, + ] + + +def test_a_wall_clock_that_steps_back_keeps_the_run_procedure(world, monkeypatch): + """Every reading of the wall clock up to 0.2 ms early (steps back of up to + 0.1 ms seen on this host): the run's times still order its attempts, + stops and decisions.""" + import random + + jitter, real = random.Random(7), time.time + monkeypatch.setattr(time, "time", lambda: real() - jitter.uniform(0.0, 2e-4)) + w = world().execute() + assert run_problems(w) == [] + for model in w.models: + assert len(w.attempts(model)) == N_DRAW * len(labels_of(model)) + + def test_the_ceiling_crosses_at_its_amount_and_stays_crossed(): ceiling = cr.Ceiling(1.0, 0.25) ceiling.add(0.5) @@ -648,6 +682,7 @@ def run_problems(world: World) -> list[str]: cost: dict[str, float] = {} span: dict[str, list[float]] = {} by_run: dict = {} + cut_short: set[str] = set() # an arm or a rerun the stop kept from running for x in lines: by_run.setdefault((x["household"], x["arm"]), []).append(x) # Records and hashes. @@ -699,17 +734,25 @@ def run_problems(world: World) -> list[str]: problems.append(f"{at}#{k}: status or lane") if k < len(xs) and rule not in ("rerun", "pause-rerun"): problems.append(f"{at}#{k}: rerun after {cls}") - if k == len(xs) and rule in ("rerun", "pause-rerun") and not stops: - problems.append( - f"{at}#{k}: a rerun due and never run, with no stop" - ) + if k == len(xs) and rule in ("rerun", "pause-rerun"): + cut_short.add(sid) + if not stops: + problems.append( + f"{at}#{k}: a rerun due and never run, with no stop" + ) times = [ t[key] for t in rec.get("turns") or [] for key in ("request_started_at", "request_completed_at") ] - if times and (min(times) < x["start"] or max(times) > x["end"]): - problems.append(f"{at}#{k}: requests outside its start and end") + if times and ( + min(times) < x["start"] - CLOCK_SLACK + or max(times) > x["end"] + CLOCK_SLACK + ): + problems.append( + f"{at}#{k}: requests {min(times)!r}-{max(times)!r} outside " + f"its start and end {x['start']!r}-{x['end']!r}" + ) if x["start"] > stopped_at: problems.append(f"{at}#{k}: started after a stop") cost[sid] = cost.get(sid, 0.0) + cr.attempt_cost(rec) @@ -743,6 +786,8 @@ def run_problems(world: World) -> list[str]: again = {x["arm"] for x in block if x["attempt"] == n} want += [(n, a) for a in drawn[sid] if a in again] cut = firsts != drawn[sid] + if cut: + cut_short.add(sid) if got != want or firsts != drawn[sid][: len(firsts)]: problems.append(f"{model}/{sid}: ran {got}, drawn {drawn[sid]}") if cut and not (stops and i == len(blocks) - 1): @@ -755,13 +800,17 @@ def run_problems(world: World) -> list[str]: reserve = world.design["reserves"][model] admitted: dict[str, float] = {} refused = False + # A household a stop cut short is under way until the stop. + ended = { + h: max(e, stopped_at) if h in cut_short else e for h, (_, e) in span.items() + } for x in decisions: sid, t = x["household"], x["time"] if sid in world.no_output: problems.append( f"{model}/{sid}: a household with no scored output decided" ) - finished = [h for h in admitted if h in span and span[h][1] <= t] + finished = [h for h in admitted if h in ended and ended[h] <= t] spend = sum(cost[h] for h in finished) under = sum(1 for h in admitted if h not in finished) + 1 ok = not refused and spend + under * reserve <= cap + cr.ADMISSION_SLACK @@ -970,7 +1019,7 @@ def test_host_trouble_during_an_attempt_reruns_it(world): pressured, stalled = w.draw_order[0], w.draw_order[2] # both lane 1 w.provider.specs = { (model, "loo-flat", pressured, 1): {"pressure": True, "sleep": 0.05}, - (model, "pe", stalled, 1): {"stall": 0.15}, + (model, "pe", stalled, 1): {"stall": 0.8}, } w.execute() @@ -1243,6 +1292,46 @@ def losing(self, message): assert run_problems(w) == [] +def test_a_household_a_stop_cuts_short_is_under_way_until_the_stop(world): + """Lane 1's first household loses the pool in its first arm, and the + restart fails 0.6 s later. In between, lane 2 finishes its household and + decides its next: the household lane 1 cut short has ended its last + attempt, but it is still under way, so its spend is not counted.""" + model = "gpt-6-sol" + made = [] + + class Losing(FakePool): + def call(self, message): + self.lost = 2 + return super().call(message) + + def pool(lanes): + made.append(1) + if len(made) > 1: + time.sleep(0.6) + raise RuntimeError("tool worker failed to start: a test") + return Losing(lanes) + + w = world(models=(model,), pool=pool) + cut, other, after = w.draw_order[0], w.draw_order[1], w.draw_order[3] + orders = w.run.lanes[model].orders + w.provider.specs = { + (model, orders[cut][0], cut, 1): {"sleep": 0.1, "calls": [{"situation": "{}"}]}, + (model, orders[other][0], other, 1): {"sleep": 0.02}, + (model, orders[other][1], other, 1): {"sleep": 0.3}, + } + w.execute() + stop = w.events(model, "stop")[0]["time"] + decided = {x["household"]: x for x in w.lines(model, "admission-log.jsonl")} + assert decided[after]["time"] < stop # decided while the cut household waited + assert decided[after]["finished_spend"] == pytest.approx(2 * COST) # `other` only + assert decided[after]["under_way"] == 2 # `cut` and `after` + assert [x["arm"] for x in w.attempts(model) if x["household"] == cut] == [ + orders[cut][0] + ] + assert run_problems(w) == [] + + def test_an_interrupt_stops_every_model_and_lets_attempts_end(world): w = world(models=("gpt-6-luna", "gpt-6-sol")) sid = w.draw_order[0] From fdbd411111d83556be2b8848a57052c3073529d0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 08:11:31 -0400 Subject: [PATCH 21/64] Confirmatory runner: the review's eight runner findings, stop causes, chains and anchors Answers runner-r1 (R1-R8) and the runner side of the pre-registration and report reviews: - A failed read of the guard's or the power log is tried five times; one that still fails is a runner error, which stops every model's lanes after settling and writing what had ended. A failed memory-pressure reading is no sample. - An attempt still running at an hour is abandoned: a harness-error record (AttemptDeadline) with its spend so far, a stop of cause deadline, and its later spend in late-cost events. An attempt that raises keeps the spend it counted. - Admission takes its time and checks the ceiling under the ceiling's lock. - A resume is accepted only if written after its pause began. - The signal handler only notes the signal; the main loop stops the lanes. - A sampler whose samples fail for five minutes, or whose thread ends, stops every model. - --workers must equal the design's. - Every stop names its cause. The error classes add the litellm subclasses the harness groups by isinstance, explain traces count as calls, and a Timeout on the escalated request after a truncated turn is class model. - Run, admission and host logs are hash chains; anchors.jsonl records every log's head every 15 minutes and at the end, and the design's anchor command (confirmatory_anchor.py) commits and pushes it. - In-flight records are fsynced before their rename. 189 tests pass (tests/test_confirmatory_run.py, tests/test_confirmatory_anchor.py). 19 mutants of the new rules each fail a named test. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 49 ++ scripts/confirmatory_run.py | 711 +++++++++++++++++++++++++----- tests/fake_anchor.py | 13 + tests/test_confirmatory_anchor.py | 91 ++++ tests/test_confirmatory_run.py | 641 ++++++++++++++++++++++++++- 5 files changed, 1378 insertions(+), 127 deletions(-) create mode 100644 scripts/confirmatory_anchor.py create mode 100644 tests/fake_anchor.py create mode 100644 tests/test_confirmatory_anchor.py diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py new file mode 100644 index 00000000..a4103d8a --- /dev/null +++ b/scripts/confirmatory_anchor.py @@ -0,0 +1,49 @@ +"""Push the confirmatory run's anchors outside the run. + +Usage: + python scripts/confirmatory_anchor.py --repo DIR ANCHORS + +The runner calls this every 15 minutes, and once more at its end, with the +path of its `anchors.jsonl`: each line holds every log's line count and chain +head at a time. This copies the file into DIR, a clone checked out on a branch +of its own, commits it and pushes the branch to its remote. The remote's record +of each push is the anchor's time from outside the run: a log edited after an +anchor was pushed no longer matches that anchor's head. +""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +import subprocess +import sys +from pathlib import Path + + +def git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), *args], check=True, capture_output=True, text=True + ).stdout + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--repo", type=Path, required=True, help="the anchors clone") + parser.add_argument("anchors", type=Path, help="the run's anchors.jsonl") + args = parser.parse_args(argv) + repo = args.repo.expanduser() + target = repo / "anchors.jsonl" + shutil.copyfile(args.anchors, target) + lines = target.read_text().splitlines() + git(repo, "add", "anchors.jsonl") + if git(repo, "status", "--porcelain", "--", "anchors.jsonl"): + head = hashlib.sha256(lines[-1].encode()).hexdigest() if lines else "none" + git(repo, "commit", "-q", "-m", f"anchor {len(lines)}: {head}") + git(repo, "push", "-q", "origin", "HEAD") + print(f"pushed {len(lines)} anchors") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 0b7f0f9b..de5f4089 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -30,19 +30,48 @@ stage and every model's spend since. At the crossing, no attempt starts, the conversations under way end ``budget_stop`` at their next turn, and every model's run log gets a ``ceiling`` event at the crossing's time. + The decision to admit a household and the crossing share one lock, so no + household is decided after the crossing. +- **The attempt deadline.** A conversation still running an hour after it + started is abandoned: its attempt gets a harness-error record + (``AttemptDeadline``) holding the spend so far, the model's lanes stop, + and the conversation ends ``budget_stop`` at its next turn. Its spend after + the deadline still counts toward the ceiling, and goes to the run log as + ``late-cost`` events. - **Failures.** Each attempt's class follows the pre-registration's table, in the precedence the report's ``classify`` uses, from its record and the host evidence for its start and end; its action follows from its class and its attempt number. An operator failure pauses the model's lanes until - ``resume``. A harness failure stops them, and so does one inconclusive - error that ends runs in more than five of the model's households. A - replaced attempt is saved, before its rerun, as - ``households/attempts/-.json``. -- **The host log.** Every 30 seconds: the time, the load average and the - kernel's memory-pressure level. At the end, the guard's and the power log's - lines from a day before the booking to its end. An attempt's class waits - for its host evidence to be complete: a sample more than 60 seconds after - the pass's last end, or no sample for more than 90 seconds. + ``resume``, which is accepted only if written after the pause began. A + harness failure stops them, and so does one inconclusive error that ends + runs in more than five of the model's households. A replaced attempt is + saved, before its rerun, as ``households/attempts/-.json``. +- **Stops.** Every ``stop`` event names its cause: ``harness``, + ``inconclusive``, ``deadline``, ``pool`` (the engine pool could not be + started again), ``runner-error`` (an exception in the runner itself, which + stops every model's lanes after writing what had ended), ``interrupt`` or + ``sampler``. The ceiling's events are ``ceiling`` events of cause + ``ceiling``. +- **The host log.** Every 30 seconds: the time, the monotonic clock, the + load average and the kernel's memory-pressure level (None when it could not + be read, which is no sample). At the end, the guard's and the power log's + lines from a day before the booking to its end. A read of those logs is + tried five times; one that still fails during the run is a runner error, + and at the end leaves ``host-error.txt`` in place of the excerpts. An + attempt's class waits for its host evidence to + be complete: a good sample more than 60 seconds after the pass's last end, + or no good sample for more than 90 seconds. A sampler whose samples have + failed for five minutes, or whose thread has ended, stops every model. +- **Chains and anchors.** Every line of the run logs, the admission logs and + the host log carries ``prev``, the sha256 of the line before it. The first + line's is a seed: for a model's logs, the sha256 of the frozen hash, the + model and the file's name; for the host log, of its name. Every 15 + minutes, and at the end, ``anchors.jsonl`` gets a line with every log's + line count and head and every manifest's sha256, itself chained, and the + design's ``anchor_command`` is run on it in the background; each run's + result goes to ``anchor-pushes.jsonl``. ``confirmatory_anchor.py`` is that + command: it commits the file to a clone and pushes it, so the remote holds + each anchor's time from outside the run. The records' format is the confirmatory report's (its module docstring). The report recomputes every class, action, order and admission decision from the @@ -53,9 +82,11 @@ ``tool_pilot_argon.py``, ``policybench`` package and fix module are the treatment commit's (6b00a6f0), its tree is clean at the design's run commit, the machine is on AC power with the lid open, and the runs folder holds no -records: a run is never resumed. An interrupt (SIGINT or SIGTERM) stops every -model's lanes with a stop event and lets the conversations under way end; a -second interrupt exits at once. +records: a run is never resumed. ``--workers`` must equal the design's +engine workers per model. An interrupt (SIGINT or SIGTERM) only notes the +signal; the main loop then stops every model's lanes with cause +``interrupt`` and lets the conversations under way end. A second interrupt +exits at once. """ from __future__ import annotations @@ -128,8 +159,29 @@ ADMISSION_SLACK = 1e-9 # dollars EXCERPT_LEAD = 86_400.0 # host logs are kept from a day before the booking WAIT_SECONDS = 1.0 # how often a waiting lane looks again +ATTEMPT_DEADLINE = 3600.0 # seconds; the longest development attempt took 714 +SAMPLER_SILENCE = 300.0 # seconds without a host-log sample that stop the run +READ_TRIES = 5 # reads of the guard's and the power log before a runner error +READ_WAIT = 10.0 # seconds between them +ANCHOR_SECONDS = 900.0 # how often the logs' chain heads are anchored +LATE_JOIN = 60.0 # seconds the run's end waits for an abandoned attempt +# What a stop event names as its cause. +STOP_CAUSES = frozenset( + { + "harness", + "inconclusive", + "pool", + "deadline", + "runner-error", + "interrupt", + "sampler", + } +) -# The pre-registration's error classes (its "Failures" table). +# The pre-registration's error classes (its "Failures" table): the names of +# litellm's exception classes under the harness's retryable classes (transport) +# and under its model-fatal ones (request and account). A record keeps only +# the raised class's name. TRANSPORT = frozenset( { "APIConnectionError", @@ -137,6 +189,8 @@ "APIResponseValidationError", "BadGatewayError", "InternalServerError", + "JSONSchemaValidationError", + "MidStreamFallbackError", "RateLimitError", "ServiceUnavailableError", "Timeout", @@ -146,13 +200,20 @@ REQUEST = frozenset( { "BadRequestError", + "ContentPolicyViolationError", "ContextWindowExceededError", + "ImageFetchError", "InvalidRequestError", + "LiteLLMUnknownProvider", + "RejectedRequestError", "UnprocessableEntityError", "UnsupportedParamsError", } ) ACCOUNT = frozenset({"AuthenticationError", "PermissionDeniedError", "NotFoundError"}) +# A timeout on the escalated request, the one asking for the escalation +# ceiling after a truncated turn, is the model's long generation timing out. +TIMEOUTS = frozenset({"Timeout", "RequestWallTimeoutError"}) HOST_ENGINE = frozenset({"MemoryError", "OSError"}) RERUN = frozenset({"provider", "host", "none", "operator"}) FINISHED = frozenset({"complete", "incomplete", "turn_limit"}) @@ -176,6 +237,8 @@ "reserves", "overall_cap", "spent_before", + "workers", + "anchor_command", ) @@ -254,7 +317,8 @@ def completed_turns(record: dict) -> list: def call_failures(record: dict) -> tuple[list[str], list[str]]: """(the inconclusive tool failures, the model's engine exceptions) in its - calls.""" + calls: a call's own error, and the errors in its results and in its + explanations (the explain feature's trace of a variable).""" host, model = [], [] for turn in record.get("turns") or []: for call in turn.get("calls") or []: @@ -269,39 +333,59 @@ def call_failures(record: dict) -> tuple[list[str], list[str]]: host.append("tool worker failure") elif error == "engine": (host if name in HOST_ENGINE else model).append(name) - for value in (result.get("results") or {}).values(): - if isinstance(value, dict) and value.get("error"): - name = str(value["error"]).split(":", 1)[0] - (host if name in HOST_ENGINE else model).append(name) + for group in ("results", "explanations"): + values = result.get(group) + for value in values.values() if isinstance(values, dict) else (): + if isinstance(value, dict) and value.get("error"): + name = str(value["error"]).split(":", 1)[0] + (host if name in HOST_ENGINE else model).append(name) return host, model +def escalation_timed_out(record: dict) -> bool: + """Whether the attempt ended on a timeout of its escalated request: the + harness records a truncated turn, then asks again at the escalation + ceiling, and an error there leaves the truncated turn last.""" + turns = record.get("turns") or [] + return ( + record.get("status") == "provider_error" + and error_class(record) in TIMEOUTS + and bool(turns) + and bool(turns[-1].get("truncated_and_retried")) + ) + + def classify(record: dict, start: float, end: float, host) -> tuple[str, str]: """(class, detail) of one attempt. In order of precedence: operator (an account class or insufficient_quota); harness (a request class with no - model request yet returned); ceiling (budget_stop); none (no model request - returned); host (host trouble during the attempt); provider (a transport - class); model (a request class); inconclusive (any other provider error, - any harness error, a tool timeout, a worker failure, MemoryError or - OSError); otherwise ok, or model when the engine raised on a household the - model built. `host` is None to decide only the classes before host.""" + completed model turn yet); ceiling (budget_stop); none (no turn recorded: + no model request returned, truncated or not); host (host trouble during + the attempt); model (a timeout of the escalated request); provider (a + transport class); model (a request class); inconclusive (any other + provider error, any harness error, a tool timeout, a worker failure, + MemoryError or OSError); otherwise ok, or model when the engine raised on + a household the model built. `host` is None to decide only the classes + before host.""" status, error = record.get("status"), str(record.get("error") or "") name = error_class(record) - reached = bool(completed_turns(record)) + returned = bool(record.get("turns")) + completed = bool(completed_turns(record)) if status == "provider_error" and ( name in ACCOUNT or "insufficient_quota" in error ): return "operator", name - if status == "provider_error" and name in REQUEST and not reached: + if status == "provider_error" and name in REQUEST and not completed: return "harness", name if status == "budget_stop": return "ceiling", status - if not reached: + if not returned: return "none", f"{status}: {name}" if name else str(status) trouble = host.trouble(start, end) if host is not None else None if trouble: return "host", trouble if status == "provider_error": + if escalation_timed_out(record): + return "model", f"{name} on the escalated request" if name in TRANSPORT: return "provider", name if name in REQUEST: @@ -432,9 +516,11 @@ def memory_pressure() -> int | None: def read_guard_log() -> str: + """The guard's log; empty when there is none (the guard never paused + anything). Any other error is raised, for the reader to try again.""" try: return GUARD_LOG.read_text(errors="replace") - except OSError: + except FileNotFoundError: return "" @@ -453,10 +539,19 @@ def local_timezone() -> str: return target.split("zoneinfo/", 1)[1] +# The `prev` of the host log's first line. +HOST_SEED = hashlib.sha256(b"host-log.jsonl").hexdigest() + + +class HostEvidenceError(RuntimeError): + """The guard's or the power log could not be read: a runner error.""" + + class HostLog: """The host log, written every `interval` seconds, and the host evidence an attempt's class reads: the guard's pauses, the power log's sleeps, the - host log's gaps and its memory-pressure samples.""" + host log's gaps and its memory-pressure samples. A sample whose pressure + could not be read is no sample: the silence around it is a gap.""" def __init__( self, @@ -481,7 +576,11 @@ def __init__( self._read_at = -math.inf self._refresh_lock = threading.Lock() self._stop = threading.Event() + self.failure: str | None = None # the sampler's last error + self.failing_since: float | None = None # failed samples since then + self._head, self._lines = HOST_SEED, 0 # the log's hash chain self.start = time.time() + self.written = self.start # the last sample written self.end: float | None = None self._write_meta() self.sample() @@ -495,37 +594,82 @@ def _write_meta(self) -> None: write_json(self.folder / "host.json", meta) def sample(self) -> None: - load1, load5, load15 = self._read_load() + """One sample. A load or pressure the host would not give is written + as null; a sample that cannot be written is the sampler's failure, + and the silence it leaves is a gap.""" + try: + load1, load5, load15 = self._read_load() + except OSError: + load1 = load5 = load15 = None pressure = self._read_pressure() with self._cond: t = time.time() line = { "time": t, + "monotonic": time.monotonic(), "load1": load1, "load5": load5, "load15": load15, "pressure": pressure, } - with (self.folder / "host-log.jsonl").open("a") as handle: - handle.write(json.dumps(line) + "\n") + text = json.dumps({**line, "prev": self._head}) + try: + with (self.folder / "host-log.jsonl").open("a") as handle: + handle.write(text + "\n") + except OSError as error: + self.failure = f"{type(error).__name__}: {error}" + self.failing_since = self.failing_since or t + return + self._head, self._lines = line_sha(text), self._lines + 1 + self.written = t + if pressure is None: + self.failure = "no memory-pressure reading" + self.failing_since = self.failing_since or t + else: + self.failing_since = None self._samples.append((t, pressure)) self._cond.notify_all() + def heads(self) -> dict: + """The host log's line count and last line's sha256, for an anchor.""" + with self._cond: + return {"lines": self._lines, "sha256": self._head} + + def dead(self) -> str | None: + """Why the sampler is taken for dead: no good sample for + SAMPLER_SILENCE seconds while its samples fail, or its thread gone. + A host that slept or was paused is not this: its sampler failed + nothing.""" + if self.end is None and not self._thread.is_alive(): + return "the sampler's thread ended" + since = self.failing_since + if since is not None and time.time() - since > SAMPLER_SILENCE: + return f"samples failing since {since:.1f} ({self.failure})" + return None + def _loop(self) -> None: k = 1 while not self._stop.wait( max(0.0, self.start + k * self.interval - time.time()) ): - self.sample() + try: + self.sample() + except Exception as error: # the sampler runs on; watch() sees the silence + self.failure = f"{type(error).__name__}: {error}" # After a stall, the next sample is the next tick, with no burst. k = max(k + 1, math.ceil((time.time() - self.start) / self.interval)) + def _good(self) -> list[float]: + """The times of the samples with a pressure reading.""" + return [t for t, p in self._samples if p is not None] + def settled(self, end: float) -> bool: """Whether every sample that could bear on an attempt ending at `end` is in: one later than the pressure window, or a silence longer than a gap (which is itself host trouble when it covers the attempt).""" with self._cond: - latest = self._samples[-1][0] + good = self._good() + latest = good[-1] if good else self.start return latest > end + PRESSURE_WINDOW or time.time() - latest > HOST_LOG_GAP def wait_settled(self, end: float) -> None: @@ -533,18 +677,30 @@ def wait_settled(self, end: float) -> None: while not self.settled(end): self._cond.wait(timeout=min(self.interval, 5.0)) + def _read_logs(self) -> tuple[str, str]: + """The guard's and the power log, read up to READ_TRIES times.""" + for attempt in range(READ_TRIES): + try: + return self._read_guard(), self._read_power() + except (OSError, subprocess.SubprocessError) as error: + failure = f"{type(error).__name__}: {error}" + if attempt < READ_TRIES - 1: + time.sleep(READ_WAIT) + raise HostEvidenceError(f"the guard's or the power log: {failure}") + def refresh(self, after: float) -> None: - """Read the guard's and the power log again unless read since `after`.""" + """Read the guard's and the power log again unless read since `after`. + A failed read keeps the last spans and tries again; HostEvidenceError + after READ_TRIES.""" with self._refresh_lock: if self._read_at > after: return now = time.time() since = self.start - EXCERPT_LEAD zone = ZoneInfo(self.timezone) - guard = excerpt( - self._read_guard(), lambda x: guard_time(x, zone), since, now - ) - power = excerpt(self._read_power(), power_time, since, now) + guard_text, power_text = self._read_logs() + guard = excerpt(guard_text, lambda x: guard_time(x, zone), since, now) + power = excerpt(power_text, power_time, since, now) spans = guard_spans(guard, self.timezone, now) + power_spans(power, now) with self._cond: self._spans, self._read_at = spans, now @@ -554,14 +710,15 @@ def _evidence(self) -> tuple[list, list[float]]: from the archived logs, with a silence so far as an open gap.""" with self._cond: spans = list(self._spans) - times = [t for t, _ in self._samples] - pressure = [t for t, p in self._samples if int(p or 0) > 1] + times = self._good() + pressure = [t for t, p in self._samples if p is not None and int(p) > 1] for a, b in zip([self.start] + times, times): if b - a > HOST_LOG_GAP: spans.append((a, b, "host log gap")) now = time.time() if self.end is None else self.end - if now - times[-1] > HOST_LOG_GAP: - spans.append((times[-1], now, "host log gap")) + last = times[-1] if times else self.start + if now - last > HOST_LOG_GAP: + spans.append((last, now, "host log gap")) return spans, pressure def trouble(self, start: float, end: float) -> str | None: @@ -591,20 +748,28 @@ def describe(self, start: float, end: float) -> str: return "; ".join(parts) def close(self) -> None: - """A last sample, the booking's end, and the guard's and power log's - lines for the booking.""" + """A last sample, the run's end, and the guard's and power log's lines + from a day before the run's start to its end. If the logs cannot be + read, `host-error.txt` says why and the excerpts are not written: the + report then has no host evidence to check against.""" self._stop.set() self._thread.join(timeout=self.interval + 10) - self.sample() + try: + self.sample() + except Exception as error: + self.failure = f"{type(error).__name__}: {error}" with self._cond: self.end = time.time() self._write_meta() since, zone = self.start - EXCERPT_LEAD, ZoneInfo(self.timezone) - guard = excerpt( - self._read_guard(), lambda x: guard_time(x, zone), since, self.end - ) + try: + guard_text, power_text = self._read_logs() + except HostEvidenceError as error: + (self.folder / "host-error.txt").write_text(f"{error}\n") + return + guard = excerpt(guard_text, lambda x: guard_time(x, zone), since, self.end) (self.folder / "clamshell-guard.log").write_text(guard) - power = excerpt(self._read_power(), power_time, since, self.end) + power = excerpt(power_text, power_time, since, self.end) (self.folder / "pmset.log").write_text(power) @@ -635,6 +800,38 @@ def exhausted(self, reserve_usd: float = 0.0) -> bool: return self.crossed_at is not None +class AttemptSpend: + """The hard ceiling as one attempt's conversation sees it. Every cost goes + to the ceiling, and the attempt keeps its own total, so an attempt that + raises still records what it spent. Once the attempt is abandoned at its + deadline, the conversation ends `budget_stop` at its next turn, and what + it spends after is kept apart as late spend.""" + + def __init__(self, ceiling: Ceiling): + self.ceiling = ceiling + self._lock = threading.Lock() + self.spent_usd = 0.0 + self.late_usd = 0.0 + self.abandoned = False + + def add(self, usd: float | None) -> None: + with self._lock: + if self.abandoned: + self.late_usd += float(usd or 0.0) + else: + self.spent_usd += float(usd or 0.0) + self.ceiling.add(usd) + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + return self.abandoned or self.ceiling.exhausted() + + def abandon(self) -> float: + """Abandon the attempt; what it had spent.""" + with self._lock: + self.abandoned = True + return self.spent_usd + + # ---------------------------------------------------------------- files @@ -647,15 +844,29 @@ def sha256_file(path: Path) -> str: def write_json(path: Path, payload, indent: int | None = 1) -> str: - """Write a file whole (to a temporary name, then renamed); its sha256.""" + """Write a file whole (to a temporary name, synced, then renamed); its + sha256.""" data = json.dumps(payload, indent=indent, default=str).encode() path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f".{path.name}.tmp") - tmp.write_bytes(data) + with tmp.open("wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) os.replace(tmp, path) return sha256_bytes(data) +def chain_seed(frozen_sha256: str, model: str, name: str) -> str: + """The `prev` of a log's first line: the log's place in the design.""" + return sha256_bytes(f"{frozen_sha256}/{model}/{name}".encode()) + + +def line_sha(text: str) -> str: + """A log line's hash, the next line's `prev`: its JSON, no newline.""" + return sha256_bytes(text.encode()) + + # ---------------------------------------------------------------- one model @@ -669,9 +880,17 @@ class Attempt: cls: str = "" detail: str = "" reason: str = "" + settled: bool = False # classed with its pass's host evidence + logged: bool = False # its run-log line written + + +class AttemptDeadline(RuntimeError): + """An attempt ran past ATTEMPT_DEADLINE and was abandoned.""" -def converse_with_harness(lanes: "Lanes", sid: str, label: str, tool_call) -> dict: +def converse_with_harness( + lanes: "Lanes", sid: str, label: str, tool_call, spend: AttemptSpend +) -> dict: """One attempt: the treatment harness's conversation for the arm, under the hard ceiling.""" board = lanes.run.board @@ -680,7 +899,7 @@ def converse_with_harness(lanes: "Lanes", sid: str, label: str, tool_call) -> di board["variables"][sid], tool_call, lanes.completion, - lanes.run.ceiling, + spend, profile=lanes.profiles[label], ) @@ -706,11 +925,21 @@ def __init__(self, run: "Run", model: str, labels: tuple[str, ...], folders: dic self.spent: dict[str, float] = {} # household -> its attempts' cost so far self.refused = False self.paused = False + self.paused_at: float | None = None self.stopped = False self.inconclusive: dict[str, list[str]] = {} self.pool = None self._pool_lock = threading.Lock() self.completion = None + self.manifest_sha256: str | None = None + # Each log is a hash chain: a line's `prev` is the line before's sha256. + self._heads = { + name: chain_seed(design["frozen_sha256"], model, name) + for name in ("run-log.jsonl", "admission-log.jsonl") + } + self._counts = {name: 0 for name in self._heads} + # Attempts abandoned at their deadline, whose spend after it is logged. + self.abandoned: list[dict] = [] # The records. @@ -723,16 +952,30 @@ def households(self, lane: int) -> list[str]: def write_line(self, line: dict, name: str = "run-log.jsonl") -> None: with self._log_lock: + text = json.dumps({**line, "prev": self._heads[name]}) self.folder.mkdir(parents=True, exist_ok=True) with (self.folder / name).open("a") as handle: - handle.write(json.dumps(line) + "\n") + handle.write(text + "\n") handle.flush() os.fsync(handle.fileno()) + self._heads[name] = line_sha(text) + self._counts[name] += 1 - def event(self, kind: str, reason: str, at: float | None = None) -> None: - self.write_line( - {"event": kind, "time": clock() if at is None else at, "reason": reason} - ) + def heads(self) -> dict: + """Each log's line count and last line's sha256, for an anchor.""" + with self._log_lock: + return { + name: {"lines": self._counts[name], "sha256": self._heads[name]} + for name in self._heads + } + + def event( + self, kind: str, reason: str, at: float | None = None, cause=None, **extra + ) -> None: + line = {"event": kind, "time": clock() if at is None else at, "reason": reason} + if cause is not None: + line["cause"] = cause + self.write_line({**line, **extra}) self.run.say(f"{self.model}: {kind} ({reason})") def attempt_line(self, sid: str, lane: int, a: Attempt, sha: str) -> None: @@ -750,17 +993,23 @@ def attempt_line(self, sid: str, lane: int, a: Attempt, sha: str) -> None: "sha256": sha, } ) + a.logged = True # Stops and pauses. def halted(self) -> bool: return self.stopped or self.run.ceiling.exhausted() - def stop(self, reason: str, kind: str = "stop", at: float | None = None) -> None: - """Stop the lanes. Every rule that stops them writes its own event.""" + def stop( + self, reason: str, cause: str, kind: str = "stop", at: float | None = None + ) -> None: + """Stop the lanes. Every rule that stops them writes its own event, + naming its cause.""" + if (kind, cause) != ("ceiling", "ceiling") and cause not in STOP_CAUSES: + raise ValueError(f"no stop has the cause {cause!r}") with self.cond: self.stopped = True - self.event(kind, reason, at) + self.event(kind, reason, at, cause=cause) self.cond.notify_all() def resume(self, note: str) -> None: @@ -778,9 +1027,11 @@ def admit(self, sid: str, lane: int) -> bool: self.paused or "settling" in self.state.values() ): self.cond.wait(timeout=WAIT_SECONDS) - if self.halted(): - return False - now = clock() + # Taken under the ceiling's lock, so no decision follows the crossing. + with self.run.ceiling.lock: + if self.stopped or self.run.ceiling.crossed_at is not None: + return False + now = clock() finished = [h for h, s in self.state.items() if s == "finished"] spend = sum(self.spent[h] for h in finished) under_way = len(self.state) - len(finished) + 1 @@ -821,11 +1072,19 @@ def start_attempt(self) -> float | None: return clock() def end_attempt( - self, sid: str, label: str, n: int, start: float, record: dict, last: bool + self, + sid: str, + label: str, + n: int, + start: float, + record: dict, + last: bool, + deadline: bool = False, ) -> Attempt: """The attempt's end. A class decided without the host evidence acts at once: an operator failure pauses the lanes, a harness failure stops - them, before any other attempt can start.""" + them, before any other attempt can start; so does an attempt + abandoned at its deadline.""" with self.cond: end = clock() self.spent[sid] += attempt_cost(record) @@ -834,13 +1093,24 @@ def end_attempt( if cls in EARLY: attempt.cls, attempt.detail = cls, detail if cls == "operator": - self.paused = True - self.event("pause", f"{label}/{sid} attempt {n}: {detail}") + self.paused, self.paused_at = True, clock() + self.event( + "pause", f"{label}/{sid} attempt {n}: {detail}", at=self.paused_at + ) if cls == "harness": self.stopped = True self.event( "stop", f"{label}/{sid}: a request class on the first request ({detail})", + cause="harness", + ) + if deadline: + self.stopped = True + self.event( + "stop", + f"{label}/{sid} attempt {n} ran past the attempt deadline " + f"of {ATTEMPT_DEADLINE:.0f} s", + cause="deadline", ) if last: self.state[sid] = "settling" @@ -858,7 +1128,7 @@ def settle(self, sid: str, done: list[Attempt]) -> None: cls, detail = classify(a.record, a.start, a.end, host) if a.cls and a.cls != cls: raise RuntimeError(f"{a.label}/{sid}: class {a.cls} became {cls}") - a.cls, a.detail = cls, detail + a.cls, a.detail, a.settled = cls, detail, True a.reason = ( f"{detail}: {host.describe(a.start, a.end)}" if cls == "host" @@ -874,12 +1144,19 @@ def settle(self, sid: str, done: list[Attempt]) -> None: if len(seen) > INCONCLUSIVE_STOP and stop is None: stop = f"{key} ended runs in {len(seen)} households" if stop: - self.stop(stop) - - def converse(self, sid: str, label: str) -> dict: + self.stop(stop, cause="inconclusive") + + def converse(self, sid: str, label: str, n: int) -> tuple[dict, bool]: + """(the attempt's record, whether it was abandoned). The conversation + runs in a thread of its own. One that raises leaves a harness-error + record with the spend it counted; one still running at + ATTEMPT_DEADLINE is abandoned with a harness-error record holding + the spend so far, ends `budget_stop` at its next turn, and writes + nothing.""" profile = self.profiles[label] - board = self.run.board + variables = self.run.board["variables"][sid] features = frozenset(profile.tool_features) + spend = AttemptSpend(self.run.ceiling) def tool_call(arguments): message = ( @@ -887,12 +1164,67 @@ def tool_call(arguments): ) return self.pool.call(message) - try: - return self.run.converse(self, sid, label, tool_call) - except Exception as error: # an attempt that left no record of its own - return pilot.harness_error_record( - sid, board["variables"][sid], profile, error + box: dict = {} + + def work(): + try: + box["record"] = self.run.converse(self, sid, label, tool_call, spend) + except Exception as error: # an attempt that left no record of its own + box["error"] = error + + thread = threading.Thread( + target=work, name=f"{self.model}-{sid}-{label}-{n}", daemon=True + ) + thread.start() + thread.join(timeout=ATTEMPT_DEADLINE) + if thread.is_alive(): + spent = spend.abandon() + with self.cond: + self.abandoned.append( + { + "household": sid, + "arm": label, + "attempt": n, + "spend": spend, + "thread": thread, + "logged": 0.0, + } + ) + error = AttemptDeadline( + f"past the attempt deadline of {ATTEMPT_DEADLINE:.0f} s" ) + record = pilot.harness_error_record(sid, variables, profile, error) + record["totals"]["cost_usd"] = spent + return record, True + if "error" in box: + record = pilot.harness_error_record(sid, variables, profile, box["error"]) + record["totals"]["cost_usd"] = spend.spent_usd + return record, False + return box["record"], False + + def log_late(self, final: bool = False) -> None: + """A `late-cost` event for each abandoned attempt that has ended (or, + at the end of the run, for each one still running) with spend after + its deadline not yet logged.""" + with self.cond: + pending = list(self.abandoned) + for item in pending: + ended = not item["thread"].is_alive() + late = item["spend"].late_usd + if (ended or final) and late > item["logged"]: + self.event( + "late-cost", + f"{item['arm']}/{item['household']} attempt {item['attempt']}: " + "spend after its deadline", + household=item["household"], + arm=item["arm"], + attempt=item["attempt"], + cost_usd=late - item["logged"], + ) + item["logged"] = late + if ended: + with self.cond: + self.abandoned.remove(item) def ensure_pool(self) -> bool: """The model's pool, started again if it has lost every worker; False @@ -933,30 +1265,30 @@ def replace(self, sid: str, lane: int, previous: Attempt, entries: list) -> None ) self.attempt_line(sid, lane, previous, sha) - def run_household(self, sid: str, lane: int) -> None: - order = self.orders[sid] - attempts: dict[str, list[Attempt]] = {label: [] for label in order} - entries: dict[str, list[dict]] = {label: [] for label in order} - in_flight = self.folder / "in-flight" / sid - todo = [(1, label) for label in order] + def passes( + self, sid: str, lane: int, attempts: dict, entries: dict, in_flight: Path + ) -> None: + """The household's arms in its drawn order, then its reruns in passes, + each pass classed once its host evidence is in.""" + todo = [(1, label) for label in self.orders[sid]] while todo: done: list[Attempt] = [] for i, (n, label) in enumerate(todo): if not self.ensure_pool(): - self.stop("the engine pool could not be started again") + self.stop("the engine pool could not be started again", "pool") break start = self.start_attempt() if start is None: break if n > 1: self.replace(sid, lane, attempts[label][-1], entries[label]) - record = self.converse(sid, label) + record, abandoned = self.converse(sid, label, n) a = self.end_attempt( - sid, label, n, start, record, last=i == len(todo) - 1 + sid, label, n, start, record, i == len(todo) - 1, abandoned ) - write_json(in_flight / f"{label}-{n}.json", record) attempts[label].append(a) done.append(a) + write_json(in_flight / f"{label}-{n}.json", record) self.run.say( f"{self.model} lane {lane} {sid} {label} #{n}: " f"{record.get('status')} ${attempt_cost(record):.4f} " @@ -978,8 +1310,39 @@ def run_household(self, sid: str, lane: int) -> None: with self.cond: self.state[sid] = "running" if todo else "finished" self.cond.notify_all() + + def run_household(self, sid: str, lane: int) -> bool: + """One household; False when the runner failed part way. Then every + model stops, and the attempts that ended are classed (as `unsettled` + when their host evidence cannot be read) and written before the lane + ends.""" + attempts: dict[str, list[Attempt]] = {label: [] for label in self.orders[sid]} + entries: dict[str, list[dict]] = {label: [] for label in self.orders[sid]} + in_flight = self.folder / "in-flight" / sid + failed = False + try: + self.passes(sid, lane, attempts, entries, in_flight) + except Exception as error: + failed = True + self.run.say(traceback.format_exc()) + self.run.stop_all( + f"runner error on {self.model} lane {lane} ({sid}): " + f"{type(error).__name__}: {error}", + cause="runner-error", + ) + pending = [a for mine in attempts.values() for a in mine if not a.settled] + if pending: + try: + self.settle(sid, pending) + except Exception as again: + self.run.say(traceback.format_exc()) + for a in pending: + a.cls, a.settled = "unsettled", True + a.detail = a.reason = ( + f"unsettled: {type(again).__name__}: {again}" + ) for label, mine in attempts.items(): - if mine: + if mine and not mine[-1].logged: final = mine[-1] sha = write_json( self.households_dir[label] / f"{sid}.json", @@ -989,17 +1352,22 @@ def run_household(self, sid: str, lane: int) -> None: with self.cond: self.state[sid] = "finished" self.cond.notify_all() - shutil.rmtree(in_flight, ignore_errors=True) + if not failed: + shutil.rmtree(in_flight, ignore_errors=True) + return not failed def lane(self, lane: int) -> None: try: for sid in self.households(lane): - if not self.admit(sid, lane): + if not self.admit(sid, lane) or not self.run_household(sid, lane): return - self.run_household(sid, lane) - except Exception as error: # a runner defect: this model stops, the rest run on + except Exception as error: # a runner defect: every model stops self.run.say(traceback.format_exc()) - self.stop(f"runner error on lane {lane}: {type(error).__name__}: {error}") + self.run.stop_all( + f"runner error on {self.model} lane {lane}: " + f"{type(error).__name__}: {error}", + cause="runner-error", + ) def manifest(self) -> dict: board, design = self.run.board, self.run.design @@ -1065,6 +1433,9 @@ def timeout(profile): "lanes": {lane: self.households(lane) for lane in (1, 2)}, "no_scored_output": sorted(self.run.no_output), "workers": self.run.workers, + "attempt_deadline_seconds": ATTEMPT_DEADLINE, + "anchor_seconds": ANCHOR_SECONDS, + "anchor_command": list(design["anchor_command"]), } def summary(self) -> dict: @@ -1094,6 +1465,7 @@ def summary(self) -> dict: "classes": dict(sorted(count.items())), "events": [x for x in lines if "event" in x], "tool_pool": self.pool.counters() if self.pool is not None else None, + "abandoned_running": len(self.abandoned), } @@ -1120,7 +1492,7 @@ class Run: def __init__( self, design: dict, - workers: int = 2, + workers: int | None = None, models: tuple[str, ...] | None = None, converse: Callable = converse_with_harness, pool_factory: Callable = default_pool, @@ -1128,7 +1500,8 @@ def __init__( host_factory: Callable | None = None, out=sys.stdout, ): - self.design, self.workers, self.out = design, workers, out + self.design, self.out = design, out + self.workers = int(design["workers"]) if workers is None else workers self.converse, self.pool_factory = converse, pool_factory self.completion_for = completion_for self.host_factory = host_factory or (lambda folder: HostLog(folder)) @@ -1156,17 +1529,83 @@ def __init__( self.host: HostLog | None = None self._ceiling_written = False self.interrupts = 0 + self.signalled: int | None = None # set by the signal handler alone + self._interrupted = False + self._sampler_stopped = False + self._anchor_head = chain_seed(design["frozen_sha256"], "", "anchors.jsonl") + self._anchored = -math.inf + self._pushing = threading.Lock() + self._push: threading.Thread | None = None def say(self, text: str) -> None: with self._say_lock: print(text, file=self.out, flush=True) - def stop_all(self, reason: str) -> None: + def stop_all(self, reason: str, cause: str) -> None: for lanes in self.lanes.values(): - lanes.stop(reason) + lanes.stop(reason, cause) + + # Anchors: the logs' chain heads, written and pushed outside the run. + + def anchor(self, wait: bool = False) -> None: + """Every log's chain head to `anchors.jsonl`, itself a chain, then the + design's anchor command on that file, in the background (one at a + time; an anchor whose push would wait is written but not pushed until + the next). Each push's result goes to `anchor-pushes.jsonl`.""" + runs = Path(self.design["runs"]) + line = { + "time": clock(), + "models": { + model: {**lanes.heads(), "manifest": lanes.manifest_sha256} + for model, lanes in self.lanes.items() + }, + "host-log.jsonl": self.host.heads() if self.host is not None else None, + "prev": self._anchor_head, + } + text = json.dumps(line) + with (runs / "anchors.jsonl").open("a") as handle: + handle.write(text + "\n") + handle.flush() + os.fsync(handle.fileno()) + self._anchor_head, self._anchored = line_sha(text), time.time() + if self._push is not None and self._push.is_alive(): + if not wait: + return + self._push.join() + self._push = threading.Thread( + target=self._push_anchor, args=(self._anchor_head,), daemon=True + ) + self._push.start() + if wait: + self._push.join() + + def _push_anchor(self, head: str) -> None: + runs = Path(self.design["runs"]) + command = [str(x) for x in self.design["anchor_command"]] + command.append(str(runs / "anchors.jsonl")) + started = time.time() + try: + out = subprocess.run(command, capture_output=True, text=True, timeout=300) + code, text = out.returncode, (out.stdout + out.stderr)[-2000:] + except (OSError, subprocess.SubprocessError) as error: + code, text = None, f"{type(error).__name__}: {error}" + line = { + "anchor": head, + "started": started, + "ended": time.time(), + "returncode": code, + "output": text, + } + with (runs / "anchor-pushes.jsonl").open("a") as handle: + handle.write(json.dumps(line) + "\n") + if code != 0: + self.say(f"the anchor push failed ({code}): {text}") + + # The main loop's work. def watch(self) -> None: - """The ceiling's events, and resumes after operator pauses.""" + """The ceiling's events, an interrupt, a dead sampler, resumes after + operator pauses, late spend and the anchors.""" crossed = self.ceiling.crossed_at if crossed is not None and not self._ceiling_written: self._ceiling_written = True @@ -1175,13 +1614,40 @@ def watch(self) -> None: f"${self.ceiling.budget_usd:.2f}" ) for lanes in self.lanes.values(): - lanes.stop(reason, kind="ceiling", at=crossed) + lanes.stop(reason, "ceiling", kind="ceiling", at=crossed) + if self.signalled is not None and not self._interrupted: + self._interrupted = True + self.say( + f"signal {self.signalled}: stopping every model's lanes; " + "the conversations under way finish" + ) + self.stop_all(f"operator stop (signal {self.signalled})", "interrupt") + dead = self.host.dead() if self.host is not None else None + if dead and not self._sampler_stopped: + self._sampler_stopped = True + self.stop_all(f"the host log stopped: {dead}", "sampler") for lanes in self.lanes.values(): path = lanes.folder / "RESUME.json" - if lanes.paused and path.exists(): - note = json.loads(path.read_text()).get("note") or "resumed" - path.rename(lanes.folder / f"resume-{time.time():.0f}.json") - lanes.resume(note) + if not path.exists(): + continue + try: + note = json.loads(path.read_text()) + except (OSError, ValueError): + continue # still being written + with lanes.cond: + paused, since = lanes.paused, lanes.paused_at + stamp = f"{time.time():.3f}" + if paused and float(note.get("time") or 0.0) >= since: + path.rename(lanes.folder / f"resume-{stamp}.json") + lanes.resume(note.get("note") or "resumed") + else: + # Written before the pause it would end, or with no pause. + path.rename(lanes.folder / f"resume-ignored-{stamp}.json") + self.say(f"{lanes.model}: a resume written before any pause, ignored") + for lanes in self.lanes.values(): + lanes.log_late() + if time.time() - self._anchored >= ANCHOR_SECONDS: + self.anchor() def execute(self) -> None: runs = Path(self.design["runs"]) @@ -1191,7 +1657,7 @@ def execute(self) -> None: for model, lanes in self.lanes.items(): lanes.completion = self.completion_for(lanes.profiles["pe3"]) stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) - write_json( + lanes.manifest_sha256 = write_json( lanes.folder / f"manifest-{stamp}.json", lanes.manifest(), indent=2 ) # A model whose lanes never decide still has both logs. @@ -1219,6 +1685,9 @@ def execute(self) -> None: self.watch() finally: for model, lanes in self.lanes.items(): + for item in list(lanes.abandoned): + item["thread"].join(timeout=LATE_JOIN) + lanes.log_late(final=True) stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) write_json( lanes.folder / f"run-summary-{stamp}.json", @@ -1228,16 +1697,15 @@ def execute(self) -> None: if lanes.pool is not None: lanes.pool.close() self.host.close() + self.anchor(wait=True) def interrupt(self, signum, frame) -> None: + """The signal handler. It only notes the signal; the main loop stops + every model's lanes. A second signal exits at once.""" self.interrupts += 1 if self.interrupts > 1: - raise KeyboardInterrupt - self.say( - f"signal {signum}: stopping every model's lanes; " - "the conversations under way end" - ) - self.stop_all(f"operator stop (signal {signum})") + os._exit(130) + self.signalled = signum # ---------------------------------------------------------------- before a run @@ -1262,6 +1730,13 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: if missing: return [f"the design has no {', '.join(missing)}"] problems = [] + command = design["anchor_command"] + if ( + not isinstance(command, list) + or not command + or shutil.which(str(command[0])) is None + ): + problems.append(f"the anchor command {command!r} is not a command here") if design.get("stated_engine_overrides"): problems.append("the confirmatory run states the board's engine: no overrides") if sha256_file(Path(design["preregistration"])) != design["frozen_sha256"]: @@ -1362,7 +1837,9 @@ def main(argv=None) -> int: sub = parser.add_subparsers(dest="command", required=True) run = sub.add_parser("run", help="run the confirmatory stage") run.add_argument("design", type=Path) - run.add_argument("--workers", type=int, default=2, help="engine workers per model") + run.add_argument( + "--workers", type=int, help="engine workers per model: the design's alone" + ) run.add_argument("--dry-run", action="store_true", help="check and print the plan") resume = sub.add_parser("resume", help="end an operator pause") resume.add_argument("design", type=Path) @@ -1375,13 +1852,19 @@ def main(argv=None) -> int: write_json(folder / "RESUME.json", {"time": time.time(), "note": args.note}) print(f"{args.model}: resume requested; the runner resumes within a second") return 0 + if args.workers is not None and args.workers != design.get("workers"): + print( + f"The run cannot start: --workers {args.workers}, and the design " + f"runs {design.get('workers')} engine workers per model." + ) + return 1 models = CHEAP + FRONTIER problems = preflight(design, models) pilot._require_engine(Path(design["board"])) if problems: print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) return 1 - plan = Run(design, workers=args.workers) + plan = Run(design) if args.dry_run: for model, lanes in plan.lanes.items(): sizes = [len(lanes.households(lane)) for lane in (1, 2)] diff --git a/tests/fake_anchor.py b/tests/fake_anchor.py new file mode 100644 index 00000000..73b5c32f --- /dev/null +++ b/tests/fake_anchor.py @@ -0,0 +1,13 @@ +"""The confirmatory runner's anchor command in its tests: a copy of the run's +anchors.jsonl to FOLDER/anchored.jsonl. + +Usage: python fake_anchor.py FOLDER ANCHORS +""" + +import shutil +import sys +from pathlib import Path + +folder, anchors = Path(sys.argv[1]), Path(sys.argv[2]) +shutil.copyfile(anchors, folder / "anchored.jsonl") +print("anchored") diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py new file mode 100644 index 00000000..626f3ba2 --- /dev/null +++ b/tests/test_confirmatory_anchor.py @@ -0,0 +1,91 @@ +"""The anchor command: each call commits the run's anchors and pushes them.""" + +import hashlib +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import confirmatory_anchor # noqa: E402 + +SCRIPT = Path(confirmatory_anchor.__file__) + + +@pytest.fixture +def clone(tmp_path, monkeypatch): + """A clone of an empty bare remote, with git's own settings only.""" + empty = tmp_path / "gitconfig" + empty.write_text("") + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(empty)) + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + for role in ("AUTHOR", "COMMITTER"): + monkeypatch.setenv(f"GIT_{role}_NAME", "anchor test") + monkeypatch.setenv(f"GIT_{role}_EMAIL", "anchor@example.invalid") + remote = tmp_path / "remote.git" + subprocess.run(["git", "init", "-q", "--bare", str(remote)], check=True) + repo = tmp_path / "anchors" + subprocess.run(["git", "clone", "-q", str(remote), str(repo)], check=True) + return remote, repo + + +def remote_log(remote: Path) -> list[str]: + out = subprocess.run( + ["git", "-C", str(remote), "log", "--format=%s", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + return out.stdout.splitlines() + + +def remote_file(remote: Path) -> str: + out = subprocess.run( + ["git", "-C", str(remote), "show", "HEAD:anchors.jsonl"], + check=True, + capture_output=True, + text=True, + ) + return out.stdout + + +def test_each_anchor_is_committed_and_pushed(clone, tmp_path): + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + lines = ['{"time": 1.0, "prev": "a"}', '{"time": 2.0, "prev": "b"}'] + shas = [hashlib.sha256(line.encode()).hexdigest() for line in lines] + anchors.write_text(lines[0] + "\n") + assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 + anchors.write_text("\n".join(lines) + "\n") + assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 + assert remote_log(remote) == [f"anchor 2: {shas[1]}", f"anchor 1: {shas[0]}"] + assert remote_file(remote) == anchors.read_text() + + +def test_an_unchanged_file_makes_no_commit(clone, tmp_path): + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + anchors.write_text('{"time": 1.0}\n') + for _ in range(2): + assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 + assert len(remote_log(remote)) == 1 + + +def test_a_failed_push_exits_nonzero(clone, tmp_path): + """The runner records the command's return code; a push that fails must + not look like one that landed.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + anchors.write_text('{"time": 1.0}\n') + subprocess.run( + ["git", "-C", str(repo), "remote", "set-url", "origin", str(tmp_path / "gone")], + check=True, + ) + out = subprocess.run( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + capture_output=True, + text=True, + ) + assert out.returncode != 0 diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 63acc702..e6034e12 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -15,6 +15,8 @@ import io import json import math +import os +import subprocess import sys import threading import time @@ -46,6 +48,12 @@ # about 0.1 ms on this host, even within a thread; the report allows a second, as # here, which also covers a small correction of the clock. CLOCK_SLACK = 1.0 +# The anchor command in a test: a copy of the anchors to the test's folder. +ANCHOR_SCRIPT = Path(__file__).resolve().parent / "fake_anchor.py" +# How far apart anchors and their triggers' stops may be in a test, on a +# loaded host. +ANCHOR_SLACK = 5.0 +TRIGGER_BOUND = 5.0 # ---------------------------------------------------------------- streams @@ -117,23 +125,50 @@ def test_an_arm_whose_features_are_not_registered_is_refused(monkeypatch): cr.arm_profile("gpt-6-luna", "loo-flat") +def litellm_names(groups) -> set[str]: + """The names of litellm's exception classes under any of `groups`.""" + import inspect + + import litellm + + found = set() + for module in (litellm, litellm.exceptions): + for obj in vars(module).values(): + if inspect.isclass(obj) and issubclass(obj, BaseException): + if issubclass(obj, groups): + found.add(obj.__name__) + return found + + def test_the_error_classes_are_policybench_s(): + """A record keeps the raised class's name, and the harness groups classes + by inheritance: so each list holds every litellm class under the + harness's group.""" from policybench import eval_no_tools as e - assert cr.TRANSPORT == {c.__name__ for c in e.RETRYABLE_PROVIDER_ERRORS} - assert cr.REQUEST | cr.ACCOUNT == {c.__name__ for c in e.MODEL_FATAL_ERRORS} + fatal = litellm_names(e.MODEL_FATAL_ERRORS) + retryable = litellm_names(e.RETRYABLE_PROVIDER_ERRORS) - fatal + assert cr.REQUEST | cr.ACCOUNT == fatal + assert cr.TRANSPORT - {"RequestWallTimeoutError"} == retryable + assert "RequestWallTimeoutError" in { + c.__name__ for c in e.RETRYABLE_PROVIDER_ERRORS + } + assert {"ContentPolicyViolationError", "RejectedRequestError"} <= cr.REQUEST assert not cr.REQUEST & cr.ACCOUNT assert not cr.TRANSPORT & (cr.REQUEST | cr.ACCOUNT) + assert cr.TIMEOUTS <= cr.TRANSPORT # ---------------------------------------------------------------- classes def record(status="complete", error=None, turns=1, calls=(), truncated=False): + """`truncated`: the record's last turn is a truncated one, its escalated + request then failing (with `turns` - 1 completed turns before it).""" turn = {"calls": [{"result": r} for r in calls]} out = [dict(turn) for _ in range(turns)] if truncated: - out = [{"truncated_and_retried": True}] + out = out[: max(turns - 1, 0)] + [{"truncated_and_retried": True}] return {"status": status, "error": error, "turns": out} @@ -150,6 +185,22 @@ def trouble(self, start, end): (record("provider_error", "BadRequestError: too long"), "model"), (record("provider_error", "BadRequestError: bad", turns=0), "harness"), (record("provider_error", "BadRequestError: x", truncated=True), "harness"), + (record("provider_error", "BadRequestError: x", 2, truncated=True), "model"), + (record("provider_error", "ContentPolicyViolationError: x", turns=0), "harness"), + (record("provider_error", "ContentPolicyViolationError: x"), "model"), + (record("provider_error", "RejectedRequestError: x", turns=0), "harness"), + # The escalated request's timeout is the model's long generation. + (record("provider_error", "Timeout: 300 s", truncated=True), "model"), + (record("provider_error", "Timeout: 300 s", 3, truncated=True), "model"), + (record("provider_error", "Timeout: 300 s"), "provider"), + # A truncated return is a return: no longer `none`. + (record("provider_error", "RateLimitError: x", truncated=True), "provider"), + ( + record("provider_error", "ProviderEmpty: no choices", truncated=True), + "inconclusive", + ), + (record("provider_error", "ProviderEmpty: no choices", turns=0), "none"), + (record("provider_error", "JSONSchemaValidationError: x"), "provider"), (record("provider_error", "AuthenticationError: key", turns=0), "operator"), (record("provider_error", "RateLimitError: insufficient_quota"), "operator"), (record("provider_error", "ProviderEmpty: no choices"), "inconclusive"), @@ -170,6 +221,17 @@ def trouble(self, start, end): (record(calls=[{"error": "engine", "message": "ValueError: no person"}]), "model"), (record(calls=[{"results": {"snap": {"error": "OSError: disk"}}}]), "inconclusive"), (record(calls=[{"results": {"snap": {"error": "KeyError: x"}}}]), "model"), + # The explain feature's trace of a variable, which catches its own errors. + ( + record(calls=[{"explanations": {"snap": {"error": "MemoryError: x"}}}]), + "inconclusive", + ), + ( + record(calls=[{"explanations": {"snap": {"error": "OSError: disk"}}}]), + "inconclusive", + ), + (record(calls=[{"explanations": {"snap": {"error": "KeyError: x"}}}]), "model"), + (record(calls=[{"explanations": {"snap": {"value": 1.0}}}]), "ok"), (record("running"), "inconclusive"), ] @@ -294,6 +356,9 @@ def fast(monkeypatch): monkeypatch.setattr(cr, "HOST_LOG_GAP", 0.5) monkeypatch.setattr(cr, "WAIT_SECONDS", 0.005) monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) + monkeypatch.setattr(cr, "SAMPLER_SILENCE", 1.0) + monkeypatch.setattr(cr, "READ_WAIT", 0.01) + monkeypatch.setattr(cr, "ANCHOR_SECONDS", 0.3) def test_the_host_log_samples_and_closes(tmp_path, fast): @@ -361,6 +426,100 @@ def test_guard_pauses_and_sleeps_reach_an_attempt_after_a_refresh(tmp_path, fast assert "Sleep" in (tmp_path / "pmset.log").read_text() +def test_a_failed_pressure_reading_is_no_sample(tmp_path, fast): + """A sample with no pressure reading is written, but it is no sample: a + failing reading under pressure would hide the evidence the rule reads, + so the silence it leaves is a gap.""" + readers = Readers() + host = readers.host(tmp_path) + time.sleep(0.03) + start = time.time() + readers.pressure = None + time.sleep(0.7) + readers.pressure = 1 + end = time.time() + host.wait_settled(end) + host.refresh(after=end) + assert host.trouble(start, end) == "host log gap" + host.close() + samples = [ + json.loads(x) for x in (tmp_path / "host-log.jsonl").read_text().splitlines() + ] + assert any(x["pressure"] is None for x in samples) + folder_host = host_from_files(tmp_path) + assert folder_host.trouble(start, end) == "host log gap" # the report's reading + + +def test_the_host_log_is_a_hash_chain_with_a_monotonic_time(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + time.sleep(0.1) + host.close() + path = tmp_path / "host-log.jsonl" + assert chain_problems(path, cr.HOST_SEED) == [] + lines = path.read_text().splitlines() + assert host.heads() == {"lines": len(lines), "sha256": cr.line_sha(lines[-1])} + ticks = [json.loads(x)["monotonic"] for x in lines] + assert ticks == sorted(ticks) + + +def test_a_failed_log_read_is_tried_again(tmp_path, fast): + calls = [] + + def power(): + calls.append(1) + if len(calls) <= 2: + raise subprocess.CalledProcessError(1, ["pmset"]) + return "" + + host = cr.HostLog( + tmp_path, + timezone=TZ, + interval=0.02, + read_load=lambda: (1.0, 2.0, 3.0), + read_pressure=lambda: 1, + read_guard=lambda: "", + read_power=power, + ) + host.refresh(after=time.time()) + assert len(calls) == 3 # two failures, then a read + host.close() + assert (tmp_path / "pmset.log").exists() + + +def test_logs_that_cannot_be_read_are_a_runner_error(tmp_path, fast): + def power(): + raise subprocess.TimeoutExpired(["pmset"], 120) + + host = cr.HostLog( + tmp_path, + timezone=TZ, + interval=0.02, + read_load=lambda: (1.0, 2.0, 3.0), + read_pressure=lambda: 1, + read_guard=lambda: "", + read_power=power, + ) + with pytest.raises(cr.HostEvidenceError): + host.refresh(after=time.time()) + host.close() + assert "TimeoutExpired" in (tmp_path / "host-error.txt").read_text() + assert not (tmp_path / "pmset.log").exists() + + +def test_a_sampler_whose_samples_fail_is_dead_after_its_silence(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + assert host.dead() is None + readers.pressure = None + time.sleep(cr.SAMPLER_SILENCE + 0.2) + assert "samples failing since" in host.dead() + readers.pressure = 1 + time.sleep(0.1) + assert host.dead() is None # a good sample: alive again + host.close() + + # ---------------------------------------------------------------- the ceiling @@ -449,15 +608,16 @@ class FakeProvider: `sleep` per turn, `calls` (tool arguments, one call each, per turn), `pressure` (high pressure during the attempt), `stall` (a host-log stall, seconds), `guard` (a guard pause over the attempt), `resume_after` - (seconds after which the operator resumes) and `raise` (the conversation - raises).""" + (seconds after which the operator resumes), `raise` (the conversation + raises before its first turn), `raise_after` (it raises after its turns) + and `check` (a function of the lanes, called first).""" def __init__(self, world, specs=None): self.world, self.specs = world, specs or {} self.count: dict = {} self.lock = threading.Lock() - def __call__(self, lanes, sid, label, tool_call): + def __call__(self, lanes, sid, label, tool_call, spend): model = lanes.model with self.lock: n = self.count[(model, label, sid)] = ( @@ -465,6 +625,8 @@ def __call__(self, lanes, sid, label, tool_call): ) spec = self.specs.get((model, label, sid, n), {}) readers = self.world.readers + if spec.get("check"): + spec["check"](lanes) if spec.get("raise"): raise RuntimeError("a test raised") if spec.get("pressure"): @@ -482,7 +644,6 @@ def __call__(self, lanes, sid, label, tool_call): "turns": [], "predictions": {v: 0.0 for v in VARIABLES}, } - spend = lanes.run.ceiling for i in range(spec.get("turns", 1)): if spend.exhausted(): rec["status"] = "budget_stop" @@ -503,6 +664,8 @@ def __call__(self, lanes, sid, label, tool_call): result, latency = tool_call(arguments) turn["calls"].append({"arguments": arguments, "result": result}) rec["turns"].append(turn) + if spec.get("raise_after"): + raise RuntimeError("a test raised after spending") if rec["status"] == "running": rec["status"] = spec.get("status", "complete") if spec.get("error"): @@ -522,7 +685,10 @@ def __call__(self, lanes, sid, label, tool_call): folder = lanes.folder def resume(): - cr.write_json(folder / "RESUME.json", {"note": "fixed in a test"}) + cr.write_json( + folder / "RESUME.json", + {"time": time.time(), "note": "fixed in a test"}, + ) threading.Timer(spec["resume_after"], resume).start() return rec @@ -584,6 +750,8 @@ def __init__( "reserves": {m: reserve for m in cr.CHEAP + cr.FRONTIER}, "overall_cap": overall_cap, "spent_before": spent_before, + "workers": 2, + "anchor_command": [sys.executable, str(ANCHOR_SCRIPT), str(tmp)], } self.readers = Readers() self.provider = FakeProvider(self, specs) @@ -644,11 +812,13 @@ def host_from_files(folder: Path): (folder / "clamshell-guard.log").read_text(), meta["timezone"], end ) spans += cr.power_spans((folder / "pmset.log").read_text(), end) - times = sorted(float(s["time"]) for s in samples) + # A sample with no pressure reading is no sample. + good = [s for s in samples if s.get("pressure") is not None] + times = sorted(float(s["time"]) for s in good) for a, b in zip([start] + times, times + [end]): if b - a > cr.HOST_LOG_GAP: spans.append((a, b, "host log gap")) - pressure = [float(s["time"]) for s in samples if int(s.get("pressure") or 0) > 1] + pressure = [float(s["time"]) for s in good if int(s["pressure"]) > 1] class Host: def trouble(self, s, e): @@ -663,10 +833,78 @@ def trouble(self, s, e): return Host() +def chain_problems(path: Path, seed: str) -> list[str]: + """Whether each line's `prev` is the line before's sha256 (the first's, + `seed`).""" + head = seed + for i, text in enumerate(path.read_text().splitlines() if path.exists() else []): + if json.loads(text).get("prev") != head: + return [f"{path.name} line {i + 1}: its prev is not the line before's hash"] + head = cr.line_sha(text) + return [] + + +def chain_heads(path: Path, seed: str) -> list[str]: + """The chain's head after each count of lines, from none.""" + heads = [seed] + for text in path.read_text().splitlines() if path.exists() else []: + heads.append(cr.line_sha(text)) + return heads + + +def anchor_problems(world: World) -> list[str]: + """The anchors: a chain of their own; each holds every log's head at its + count; the last holds every log whole; the last was pushed.""" + problems: list[str] = [] + runs, frozen = world.runs(), world.design["frozen_sha256"] + path = runs / "anchors.jsonl" + problems += chain_problems(path, cr.chain_seed(frozen, "", "anchors.jsonl")) + anchors = [json.loads(x) for x in path.read_text().splitlines()] + heads = { + (model, name): chain_heads( + runs / model / name, cr.chain_seed(frozen, model, name) + ) + for model in world.models + for name in ("run-log.jsonl", "admission-log.jsonl") + } + host = chain_heads(runs / "host" / "host-log.jsonl", cr.HOST_SEED) + for i, anchor in enumerate(anchors): + for (model, name), chain in heads.items(): + got = anchor["models"][model][name] + if got["lines"] >= len(chain) or chain[got["lines"]] != got["sha256"]: + problems.append(f"anchor {i}: {model}/{name} is not the log's head") + got = anchor["host-log.jsonl"] + if got["lines"] >= len(host) or host[got["lines"]] != got["sha256"]: + problems.append(f"anchor {i}: the host log is not the log's head") + last = anchors[-1] + for (model, name), chain in heads.items(): + if last["models"][model][name]["lines"] != len(chain) - 1: + problems.append(f"the last anchor leaves out lines of {model}/{name}") + if last["host-log.jsonl"]["lines"] != len(host) - 1: + problems.append("the last anchor leaves out host-log lines") + for a, b in zip(anchors, anchors[1:]): + if b["time"] - a["time"] > cr.ANCHOR_SECONDS + ANCHOR_SLACK: + problems.append(f"anchors {a['time']!r} and {b['time']!r} too far apart") + pushes = [ + json.loads(x) for x in (runs / "anchor-pushes.jsonl").read_text().splitlines() + ] + final = cr.line_sha(path.read_text().splitlines()[-1]) + if not any(x["anchor"] == final and x["returncode"] == 0 for x in pushes): + problems.append("the last anchor was not pushed") + copy = world.tmp / "anchored.jsonl" + if not copy.exists() or copy.read_text() != path.read_text(): + problems.append("the anchor command's copy is not the anchors") + return problems + + def run_problems(world: World) -> list[str]: """Every rule of the run procedure, recomputed from the runner's files.""" problems: list[str] = [] host = host_from_files(world.runs() / "host") + frozen = world.design["frozen_sha256"] + problems += chain_problems(world.runs() / "host" / "host-log.jsonl", cr.HOST_SEED) + problems += anchor_problems(world) + late_total = 0.0 position = {sid: i for i, sid in enumerate(world.draw_order)} master = cr.master_seed(world.design["frozen_sha256"]) all_attempts = [] @@ -676,6 +914,17 @@ def run_problems(world: World) -> list[str]: drawn = cr.orders(master, model, world.draw_order, labels) lines = world.attempts(model) events = world.events(model) + for name in ("run-log.jsonl", "admission-log.jsonl"): + problems += chain_problems( + world.runs() / model / name, cr.chain_seed(frozen, model, name) + ) + for e in events: + if e["event"] == "stop" and e.get("cause") not in cr.STOP_CAUSES: + problems.append(f"{model}: a stop with no cause ({e})") + if e["event"] == "ceiling" and e.get("cause") != "ceiling": + problems.append(f"{model}: a ceiling event with another cause") + late_total += sum(e["cost_usd"] for e in events if e["event"] == "late-cost") + causes = {e.get("cause") for e in events if e["event"] == "stop"} stops = [e["time"] for e in events if e["event"] in ("stop", "ceiling")] stopped_at = min(stops) if stops else math.inf ceiling_events[model] = [e["time"] for e in events if e["event"] == "ceiling"] @@ -724,6 +973,8 @@ def run_problems(world: World) -> list[str]: f"{at}#{k}: its line's hash is not its record's" ) cls, _ = cr.classify(rec, x["start"], x["end"], host) + if x["class"] == "unsettled" and "runner-error" in causes: + cls = "unsettled" # classed when the runner failed: it stands rule = cr.action(cls, k) if x["class"] != cls or x["action"] != rule: problems.append( @@ -871,11 +1122,61 @@ def run_problems(world: World) -> list[str]: for t in times: if any(x["start"] > t for *_, x, _ in all_attempts): problems.append("an attempt started after the ceiling") - # Conservation: the ceiling counted exactly the records' spend. - if abs(world.run.ceiling.spent_usd - total) > 1e-9: + # Conservation: the ceiling counted exactly the records' spend, and the + # spend of abandoned attempts after their deadline. + if abs(world.run.ceiling.spent_usd - total - late_total) > 1e-9: problems.append( - f"the ceiling counted {world.run.ceiling.spent_usd}, records {total}" + f"the ceiling counted {world.run.ceiling.spent_usd}, records {total} " + f"and late spend {late_total}" ) + problems += trigger_problems(world, all_attempts) + return problems + + +def trigger_problems(world: World, all_attempts) -> list[str]: + """Each stop rule's trigger has its stop, and each stop of those causes + its trigger: a harness failure and an abandoned attempt stop the model at + their end, before any other attempt starts; one inconclusive error ending + runs in a sixth household stops it once that household's pass is classed.""" + problems: list[str] = [] + for model in world.models: + events = world.events(model) + mine = [(sid, k, x, rec) for m, sid, _, k, x, rec in all_attempts if m == model] + triggers: list[tuple[str, float]] = [] + joined: dict[str, dict[str, float]] = {} # key -> household -> pass end + for sid, k, x, rec in mine: + if x["class"] == "harness": + triggers.append(("harness", x["end"])) + if str(rec.get("error") or "").startswith("AttemptDeadline"): + triggers.append(("deadline", x["end"])) + status = rec.get("status") + if x["class"] == "inconclusive" and status in ( + "provider_error", + "harness_error", + ): + key = f"{status}: {cr.error_class(rec) or rec.get('error')}" + end = max(y["end"] for s, n, y, _ in mine if s == sid and n == k) + households = joined.setdefault(key, {}) + households[sid] = min(households.get(sid, math.inf), end) + for households in joined.values(): + ends = sorted(households.values()) + if len(ends) > cr.INCONCLUSIVE_STOP: + triggers.append(("inconclusive", ends[cr.INCONCLUSIVE_STOP])) + stops = [e for e in events if e["event"] == "stop"] + for cause, t in triggers: + if not any( + e.get("cause") == cause and t <= e["time"] <= t + TRIGGER_BOUND + for e in stops + ): + problems.append(f"{model}: a {cause} trigger at {t!r} with no stop") + if cause != "inconclusive" and any(x["start"] > t for _, _, x, _ in mine): + problems.append(f"{model}: an attempt started after a {cause} stop") + for e in stops: + if e.get("cause") in ("harness", "deadline", "inconclusive") and not any( + cause == e["cause"] and t <= e["time"] <= t + TRIGGER_BOUND + for cause, t in triggers + ): + problems.append(f"{model}: a {e['cause']} stop with no trigger") return problems @@ -1104,6 +1405,7 @@ def test_a_request_class_on_the_first_request_stops_the_model(world): w.execute() stops = w.events(model, "stop") assert len(stops) == 1 and "first request" in stops[0]["reason"] + assert stops[0]["cause"] == "harness" assert not [x for x in w.attempts(model) if x["start"] > stops[0]["time"]] assert len(w.attempts("gpt-6-sol")) == N_DRAW * 2 # the other model finishes assert run_problems(w) == [] @@ -1120,6 +1422,7 @@ def test_one_inconclusive_error_in_six_households_stops_the_model(world): assert ( stops[0]["reason"] == "provider_error: ProviderEmpty ended runs in 6 households" ) + assert stops[0]["cause"] == "inconclusive" assert run_problems(w) == [] @@ -1289,6 +1592,7 @@ def losing(self, message): w.execute() stops = w.events(model, "stop") assert stops and "engine pool" in stops[0]["reason"] + assert stops[0]["cause"] == "pool" assert run_problems(w) == [] @@ -1341,12 +1645,323 @@ def test_an_interrupt_stops_every_model_and_lets_attempts_end(world): for m in w.models: stops = w.events(m, "stop") assert len(stops) == 1 and stops[0]["reason"] == "operator stop (signal 2)" + assert stops[0]["cause"] == "interrupt" assert not [x for x in w.attempts(m) if x["start"] > stops[0]["time"]] luna = [x for x in w.attempts("gpt-6-luna") if x["household"] == sid] assert luna and len(luna) < len(cr.ARMS) # cut by the stop assert run_problems(w) == [] +def test_a_failed_explain_trace_is_inconclusive(world): + """The explain feature's trace catches its own errors and returns them + under `explanations`: a MemoryError there is the host's, not a miss.""" + + class Explaining(FakePool): + def call(self, message): + self.messages.append(message) + return { + "results": {"snap": {"values": {"spm": 1.0}}}, + "explanations": {"snap": {"error": "MemoryError: a test"}}, + }, 0.0 + + model = "gpt-6-luna" + w = world(models=(model,), pool=Explaining) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe3", sid, 1): {"calls": [{"situation": "{}"}]}} + w.execute() + line = next( + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe3" + ) + assert line["class"] == "inconclusive" and line["action"] == "stands" + assert run_problems(w) == [] + + +def test_no_household_is_decided_after_the_crossing(world): + """The crossing lands between the lanes' halt check and the decision: the + decision's time is taken under the ceiling's lock, after a second look.""" + w = world(models=("gpt-6-luna",)) + lanes = w.run.lanes["gpt-6-luna"] + halted = lanes.halted + + def crossing(): + result = halted() + if not result: + w.run.ceiling.add(1e6) # another lane's spend, just after the check + return result + + lanes.halted = crossing + decided = lanes.admit(w.draw_order[0], 1) + lines = w.lines("gpt-6-luna", "admission-log.jsonl") + assert decided is False and lines == [] + + +@pytest.mark.parametrize( + "when", ["before the run", "before the run, misdated", "during the pause"] +) +def test_a_resume_written_before_the_pause_is_ignored(world, when): + """A resume left from before a pause would end that pause unseen: it is + set aside, and the pause holds until a resume written during it. One + found while no pause holds is set aside at once, whatever time it + states; one found during the pause but stamped before it is set aside + too.""" + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + folder = w.runs() / model + folder.mkdir(parents=True) + stale = {"time": time.time(), "note": "stale"} + if when.startswith("before the run"): + if when.endswith("misdated"): + stale["time"] += 3600.0 # a writer whose clock runs ahead + cr.write_json(folder / "RESUME.json", stale) + + def plant(lanes): + # Lands once the pause holds: the conversation returns at once. + threading.Timer( + 0.1, lambda: cr.write_json(folder / "RESUME.json", stale) + ).start() + + w.provider.specs = { + (model, "pe3", sid, 1): { + "status": "provider_error", + "error": "AuthenticationError: bad key", + "turns": 0, + "resume_after": 1.5, # the main loop looks every 0.5 s + **({"check": plant} if when == "during the pause" else {}), + } + } + w.execute() + pauses, resumes = w.events(model, "pause"), w.events(model, "resume") + assert len(pauses) == 1 and len(resumes) == 1 + assert resumes[0]["reason"] == "fixed in a test" + assert resumes[0]["time"] - pauses[0]["time"] >= 1.4 + assert list(folder.glob("resume-ignored-*.json")) + assert run_problems(w) == [] + + +def test_the_signal_handler_only_notes_the_signal(world, monkeypatch): + """The handler runs on the main thread between any two of its bytecodes, + perhaps holding a lock it would need: it takes none. A second signal + exits at once.""" + w = world(models=("gpt-6-luna",)) + lanes = w.run.lanes["gpt-6-luna"] + exits = [] + monkeypatch.setattr(os, "_exit", lambda code: exits.append(code)) + with w.run._say_lock, lanes.cond, lanes._log_lock: + w.run.interrupt(2, None) # would deadlock if it took any of these + assert w.run.signalled == 2 and not lanes.stopped + assert w.lines("gpt-6-luna") == [] + w.run.interrupt(15, None) + assert exits == [130] + + +def test_an_attempt_that_raises_after_spending_keeps_its_cost(world): + model = "claude-haiku-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = { + (model, first, sid, 1): {"turns": 2, "cost": 0.03, "raise_after": True} + } + w.execute() + replaced = json.loads( + (w.folder(model, first) / "attempts" / f"{sid}-1.json").read_text() + ) + assert replaced["status"] == "harness_error" and replaced["turns"] == [] + assert replaced["totals"]["cost_usd"] == pytest.approx(0.06) + assert run_problems(w) == [] # the ceiling counted it, and so do the records + + +def test_an_attempt_past_its_deadline_is_abandoned_and_stops_the_model( + world, monkeypatch +): + """The conversation runs on in its own thread; at the deadline its record + is a harness error with the spend so far, the model stops, and the + conversation ends `budget_stop` at its next turn. Its spend after the + deadline is a late-cost event.""" + monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 0.3) + model = "gpt-6-luna" + w = world(models=(model, "claude-haiku-5.5")) + sid = w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = { + (model, first, sid, 1): {"turns": 4, "cost": 0.01, "sleep": 0.2}, + **{ + ("claude-haiku-5.5", label, s, 1): {"sleep": 0.05} + for label in cr.ARMS + for s in w.draw_order[:2] + }, + } + w.execute() + line = next( + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == first + ) + final = json.loads((w.folder(model, first) / f"{sid}.json").read_text()) + assert final["error"].startswith("AttemptDeadline: past the attempt deadline") + assert final["totals"]["cost_usd"] == pytest.approx(0.01) # turn 1 only + assert line["class"] == "none" # its rerun is cut short by the stop + stops = w.events(model, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "deadline" + late = w.events(model, "late-cost") + assert len(late) == 1 and late[0]["cost_usd"] == pytest.approx(0.01) + assert not [x for x in w.attempts(model) if x["start"] > line["end"]] + assert len(w.attempts("claude-haiku-5.5")) == N_DRAW * len(cr.ARMS) + assert run_problems(w) == [] + + +def test_a_sampler_whose_samples_fail_stops_every_model(world): + models = ("gpt-6-luna", "gpt-6-sol") + w = world(models=models) + w.provider.specs = { + (m, label, sid, 1): {"sleep": 0.1} + for m in models + for label in labels_of(m) + for sid in w.draw_order + } + threading.Timer(0.3, lambda: setattr(w.readers, "pressure", None)).start() + w.execute() + for m in models: + stops = w.events(m, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "sampler" + assert "samples failing since" in stops[0]["reason"] + assert run_problems(w) == [] + + +def failing_settle(lanes, sid, times): + """Lanes.settle, failing its first `times` calls for household `sid`.""" + original, calls = lanes.settle, [] + + def settle(at, done): + if at == sid and len(calls) < times: + calls.append(at) + raise RuntimeError("a test's runner defect") + return original(at, done) + + lanes.settle = settle + return calls + + +@pytest.mark.parametrize("times", [1, 2]) +def test_a_runner_error_stops_every_model_and_writes_what_ended(world, times): + """A runner error in a household's pass: every model stops, and that + household's ended attempts are classed, or `unsettled` if classing fails + again, and written with their lines before the lane ends.""" + models = ("gpt-6-luna", "gpt-6-sol") + w = world(models=models) + sid = w.draw_order[2] + failing_settle(w.run.lanes["gpt-6-luna"], sid, times) + w.execute() + for m in models: + stops = w.events(m, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "runner-error" + mine = [x for x in w.attempts("gpt-6-luna") if x["household"] == sid] + assert len(mine) == len(cr.ARMS) # the pass ended before the error + assert {x["class"] for x in mine} == ({"ok"} if times == 1 else {"unsettled"}) + for label in cr.ARMS: + assert (w.folder("gpt-6-luna", label) / f"{sid}.json").exists() + assert run_problems(w) == [] + + +def test_each_attempt_is_on_disk_before_the_next_starts(world, monkeypatch): + """An ended attempt is written, synced, before its household's next + attempt starts; a replaced one is saved, with its line, before its rerun + starts.""" + model = "claude-haiku-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + order = w.run.lanes[model].orders[sid] + import fcntl + + if not hasattr(fcntl, "F_GETPATH"): + pytest.skip("no F_GETPATH: the paths synced cannot be told") + synced: list[str] = [] + real = os.fsync + + def fsync(fd): + path = fcntl.fcntl(fd, fcntl.F_GETPATH, bytes(1024)) + synced.append(path.rstrip(b"\0").decode()) + return real(fd) + + monkeypatch.setattr(os, "fsync", fsync) + seen = {} + + def before_second(lanes): + seen["in-flight"] = ( + lanes.folder / "in-flight" / sid / f"{order[0]}-1.json" + ).exists() + + def before_rerun(lanes): + seen["replaced"] = ( + w.folder(model, order[1]) / "attempts" / f"{sid}-1.json" + ).exists() + seen["line"] = any( + x["household"] == sid and x["arm"] == order[1] and x["attempt"] == 1 + for x in w.attempts(model) + ) + + error = {"status": "provider_error", "error": "RateLimitError: slow"} + w.provider.specs = { + (model, order[1], sid, 1): {**error, "check": before_second}, + (model, order[1], sid, 2): {"check": before_rerun}, + } + w.execute() + assert seen == {"in-flight": True, "replaced": True, "line": True} + # Each whole-file write is synced before its rename. + assert any(f"/in-flight/{sid}/.{order[0]}-1.json.tmp" in p for p in synced) + assert any(f"/attempts/.{sid}-1.json.tmp" in p for p in synced) + assert run_problems(w) == [] + + +def test_the_worker_count_is_the_design_s(tmp_path, capsys): + design = tmp_path / "design.json" + design.write_text(json.dumps({"runs": str(tmp_path / "runs"), "workers": 2})) + assert cr.main(["run", str(design), "--workers", "3"]) == 1 + assert "the design runs 2 engine workers" in capsys.readouterr().out + + +def test_the_checker_catches_a_runner_that_ignores_a_harness_failure( + world, monkeypatch +): + """trigger_problems is not vacuous: a runner whose harness failure does + not stop the model fails it.""" + model = "claude-opus-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + w.provider.specs = { + (model, "pe", sid, 1): { + "status": "provider_error", + "error": "BadRequestError: bad tools", + "turns": 0, + } + } + lanes = w.run.lanes[model] + real = lanes.event + + def no_stop(kind, reason, at=None, cause=None, **extra): + if cause == "harness": + lanes.stopped = False # the fault: the stop is not kept + return + real(kind, reason, at, cause, **extra) + + lanes.event = no_stop + w.execute() + problems = run_problems(w) + assert any("a harness trigger" in p and "with no stop" in p for p in problems) + assert any("an attempt started after a harness stop" in p for p in problems) + + +def test_the_checker_catches_an_edited_log_line(world): + w = world(models=("gpt-6-sol",)).execute() + path = w.runs() / "gpt-6-sol" / "admission-log.jsonl" + lines = path.read_text().splitlines() + edited = json.loads(lines[1]) + edited["finished_spend"] = 0.0 + path.write_text("\n".join([lines[0], json.dumps(edited)] + lines[2:]) + "\n") + problems = run_problems(w) + assert any("its prev is not the line before's hash" in p for p in problems) + assert any("is not the log's head" in p for p in problems) + + KINDS = { "ok": {}, "provider": {"status": "provider_error", "error": "RateLimitError: slow"}, From 06abd1fdf92c3047096161ec71b01b5b187b12e0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 11:11:03 -0400 Subject: [PATCH 22/64] Confirmatory runner: log excerpts from a day before the run's start The docstring and EXCERPT_LEAD's comment said "the booking"; the code keeps the guard's and power log's lines from a day before the host log's start. No behaviour changes. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index de5f4089..02048665 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -55,7 +55,7 @@ - **The host log.** Every 30 seconds: the time, the monotonic clock, the load average and the kernel's memory-pressure level (None when it could not be read, which is no sample). At the end, the guard's and the power log's - lines from a day before the booking to its end. A read of those logs is + lines from a day before the run's start to its end. A read of those logs is tried five times; one that still fails during the run is a runner error, and at the end leaves ``host-error.txt`` in place of the excerpts. An attempt's class waits for its host evidence to @@ -157,7 +157,7 @@ SAMPLE_SECONDS = 30.0 CEILING_MARGIN = 25.0 # dollars above the overall cap ADMISSION_SLACK = 1e-9 # dollars -EXCERPT_LEAD = 86_400.0 # host logs are kept from a day before the booking +EXCERPT_LEAD = 86_400.0 # host logs are kept from a day before the run's start WAIT_SECONDS = 1.0 # how often a waiting lane looks again ATTEMPT_DEADLINE = 3600.0 # seconds; the longest development attempt took 714 SAMPLER_SILENCE = 300.0 # seconds without a host-log sample that stop the run From f5099dbcc26fc6747e147eb8d413c15a40a56e54 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 11:23:22 -0400 Subject: [PATCH 23/64] Confirmatory runner tests: the interrupt test without a timer The test sent its interrupt on a 0.15 s timer, and at load 49 the run had not started the household's first attempt by then, so no attempt was cut short and the test failed. The interrupt now arrives from inside that first attempt, which finishes only once the main loop has stopped the lanes. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index e6034e12..4616ff93 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -1639,8 +1639,18 @@ def pool(lanes): def test_an_interrupt_stops_every_model_and_lets_attempts_end(world): w = world(models=("gpt-6-luna", "gpt-6-sol")) sid = w.draw_order[0] - w.provider.specs = {("gpt-6-luna", lbl, sid, 1): {"sleep": 0.1} for lbl in cr.ARMS} - threading.Timer(0.15, lambda: w.run.interrupt(2, None)).start() + first = w.run.lanes["gpt-6-luna"].orders[sid][0] + + def interrupt_inside(lanes): + # The signal arrives during the household's first attempt, which + # finishes only once the main loop has stopped the lanes, so no + # timing on a loaded host lets the household's later arms start first. + w.run.interrupt(2, None) + until = time.time() + 30 + while not lanes.stopped and time.time() < until: + time.sleep(0.01) + + w.provider.specs = {("gpt-6-luna", first, sid, 1): {"check": interrupt_inside}} w.execute() for m in w.models: stops = w.events(m, "stop") From f210b6e556fe6341052d1c9b71335210da9dda3d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 13:14:01 -0400 Subject: [PATCH 24/64] Confirmatory runner: round-2 fixes, part 1 (not yet tested) Answers the first part of review runner-r2: - The ledger can close. Ceiling.charge counts spend with the lock held and, once close() has run, counts nothing more and keeps the amount apart as unrecorded_usd. AttemptSpend now changes its totals under the ceiling's lock, so the attempt's totals and the ceiling's always agree. - The sampler's failure timer starts on every caught sampling exception (finding 3), so a reader that keeps raising stops the run after the silence limit instead of never. - Guard-log lines carry no offset, so guard_times reads each as both readings of a repeated or skipped local hour (New York repeats 01:00-02:00 on 2026-11-01). A pause opens at its earliest reading and closes at its latest; excerpts keep a line if either reading falls in the window. - ANCHOR_RETRY and ANCHOR_PUSH_TIMEOUT are defined for the anchor changes that follow; LATE_JOIN becomes one overall deadline. The suite has not run on this commit: the next commit runs it and finishes the round (stop_all, end_attempt, runner errors, startup, finish(), the resume race and the anchor push). Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 98 +++++++++++++++++++++++----------- tests/test_confirmatory_run.py | 2 +- 2 files changed, 69 insertions(+), 31 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 02048665..3dfe346f 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -164,7 +164,9 @@ READ_TRIES = 5 # reads of the guard's and the power log before a runner error READ_WAIT = 10.0 # seconds between them ANCHOR_SECONDS = 900.0 # how often the logs' chain heads are anchored -LATE_JOIN = 60.0 # seconds the run's end waits for an abandoned attempt +ANCHOR_RETRY = 60.0 # seconds to the next anchor after a failed push +ANCHOR_PUSH_TIMEOUT = 300.0 # seconds; then the command's process group is killed +LATE_JOIN = 60.0 # seconds the run's end waits, in all, for abandoned attempts # What a stop event names as its cause. STOP_CAUSES = frozenset( { @@ -438,34 +440,42 @@ def clock() -> float: GUARD_LOG = Path.home() / "Library/Logs/clamshell-guard.log" -def guard_time(line: str, zone: ZoneInfo) -> float | None: +def guard_times(line: str, zone: ZoneInfo) -> tuple[float, float] | None: + """The earliest and latest instants a guard line's local time can be. They + differ only within an hour that a change of offset repeats or skips: on + 2026-11-01, New York's 01:00-02:00 comes twice, and the guard's lines + carry no offset.""" match = GUARD_LINE.match(line) if not match: return None stamp = datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S") - return stamp.replace(tzinfo=zone).timestamp() + both = [stamp.replace(tzinfo=zone, fold=fold).timestamp() for fold in (0, 1)] + return min(both), max(both) -def power_time(line: str) -> float | None: +def power_times(line: str) -> tuple[float, float] | None: + """A power-log line's instant, twice (its time carries its offset).""" match = POWER_LINE.match(line) if not match: return None - return datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S %z").timestamp() + t = datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S %z").timestamp() + return t, t def guard_spans(text: str, timezone: str, end: float) -> list: """Pauses: from a 'paused pids:' line to the next 'resumed pids:' line (or - `end`).""" + `end`), each from the earliest reading of its pause to the latest of its + resume.""" zone, spans, opened = ZoneInfo(timezone), [], None for line in text.splitlines(): - t = guard_time(line, zone) - if t is None: + times = guard_times(line, zone) + if times is None: continue what = GUARD_LINE.match(line).group(2) if "paused pids:" in what and opened is None: - opened = t + opened = times[0] elif "resumed pids:" in what and opened is not None: - spans.append((opened, t, "guard pause")) + spans.append((opened, times[1], "guard pause")) opened = None if opened is not None: spans.append((opened, end, "guard pause")) @@ -476,9 +486,10 @@ def power_spans(text: str, end: float) -> list: """Sleeps: from a Sleep line to the next Wake or DarkWake line (or `end`).""" spans, opened = [], None for line in text.splitlines(): - t = power_time(line) - if t is None: + times = power_times(line) + if times is None: continue + t = times[0] kind = POWER_LINE.match(line).group(2).strip() if kind == "Sleep" and opened is None: opened = t @@ -490,12 +501,18 @@ def power_spans(text: str, end: float) -> list: return spans -def excerpt(text: str, when: Callable[[str], float | None], since: float, until: float): - """The lines of a log stamped within [since, until].""" +def excerpt( + text: str, + when: Callable[[str], tuple[float, float] | None], + since: float, + until: float, +): + """The lines of a log that one reading of its time puts within [since, + until].""" keep = [] for line in text.splitlines(): - t = when(line) - if t is not None and since <= t <= until: + times = when(line) + if times is not None and times[0] <= until and times[1] >= since: keep.append(line + "\n") return "".join(keep) @@ -656,6 +673,7 @@ def _loop(self) -> None: self.sample() except Exception as error: # the sampler runs on; watch() sees the silence self.failure = f"{type(error).__name__}: {error}" + self.failing_since = self.failing_since or time.time() # After a stall, the next sample is the next tick, with no burst. k = max(k + 1, math.ceil((time.time() - self.start) / self.interval)) @@ -699,8 +717,8 @@ def refresh(self, after: float) -> None: since = self.start - EXCERPT_LEAD zone = ZoneInfo(self.timezone) guard_text, power_text = self._read_logs() - guard = excerpt(guard_text, lambda x: guard_time(x, zone), since, now) - power = excerpt(power_text, power_time, since, now) + guard = excerpt(guard_text, lambda x: guard_times(x, zone), since, now) + power = excerpt(power_text, power_times, since, now) spans = guard_spans(guard, self.timezone, now) + power_spans(power, now) with self._cond: self._spans, self._read_at = spans, now @@ -767,9 +785,9 @@ def close(self) -> None: except HostEvidenceError as error: (self.folder / "host-error.txt").write_text(f"{error}\n") return - guard = excerpt(guard_text, lambda x: guard_time(x, zone), since, self.end) + guard = excerpt(guard_text, lambda x: guard_times(x, zone), since, self.end) (self.folder / "clamshell-guard.log").write_text(guard) - power = excerpt(power_text, power_time, since, self.end) + power = excerpt(power_text, power_times, since, self.end) (self.folder / "pmset.log").write_text(power) @@ -784,16 +802,33 @@ class Ceiling(pilot.Spend): def __init__(self, ceiling_usd: float, spent_before: float): super().__init__(ceiling_usd, spent_before) self.crossed_at: float | None = None + self.closed = False # the records are final: spend is no longer counted + self.unrecorded_usd = 0.0 # spend after that, in no record @property def lock(self) -> threading.Lock: return self._lock + def charge(self, usd: float) -> bool: + """Count `usd`, with the lock held; False, and the spend unrecorded, + once the ledger has closed.""" + if self.closed: + self.unrecorded_usd += usd + return False + self.spent_usd += usd + if self.crossed_at is None and self.spent_usd >= self.budget_usd: + self.crossed_at = clock() + return True + def add(self, usd: float | None) -> None: with self._lock: - self.spent_usd += float(usd or 0.0) - if self.crossed_at is None and self.spent_usd >= self.budget_usd: - self.crossed_at = clock() + self.charge(float(usd or 0.0)) + + def close(self) -> None: + """Close the ledger at the run's end: what the ceiling has counted is + then what the records hold.""" + with self._lock: + self.closed = True def exhausted(self, reserve_usd: float = 0.0) -> bool: with self._lock: @@ -805,29 +840,32 @@ class AttemptSpend: to the ceiling, and the attempt keeps its own total, so an attempt that raises still records what it spent. Once the attempt is abandoned at its deadline, the conversation ends `budget_stop` at its next turn, and what - it spends after is kept apart as late spend.""" + it spends after is kept apart as late spend. The attempt's totals and the + ceiling's change under the ceiling's lock, so they always agree; once + the ledger has closed, neither counts more.""" def __init__(self, ceiling: Ceiling): self.ceiling = ceiling - self._lock = threading.Lock() self.spent_usd = 0.0 self.late_usd = 0.0 self.abandoned = False def add(self, usd: float | None) -> None: - with self._lock: + usd = float(usd or 0.0) + with self.ceiling.lock: + if not self.ceiling.charge(usd): + return if self.abandoned: - self.late_usd += float(usd or 0.0) + self.late_usd += usd else: - self.spent_usd += float(usd or 0.0) - self.ceiling.add(usd) + self.spent_usd += usd def exhausted(self, reserve_usd: float = 0.0) -> bool: return self.abandoned or self.ceiling.exhausted() def abandon(self) -> float: """Abandon the attempt; what it had spent.""" - with self._lock: + with self.ceiling.lock: self.abandoned = True return self.spent_usd diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 4616ff93..1e962383 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -313,7 +313,7 @@ def test_an_excerpt_keeps_the_lines_stamped_in_its_window(): t = 1_800_000_000.0 text = "\n".join(guard_line(t + k, "paused") for k in (-5, 0, 5, 10)) + "\nx" zone = ZoneInfo(TZ) - kept = cr.excerpt(text, lambda x: cr.guard_time(x, zone), t, t + 5) + kept = cr.excerpt(text, lambda x: cr.guard_times(x, zone), t, t + 5) assert kept.splitlines() == [guard_line(t, "paused"), guard_line(t + 5, "paused")] From 36d553a10a4b193eb47720b0c8ad3622eac3e2dd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 07:26:56 -0700 Subject: [PATCH 25/64] Confirmatory runner: round-2 fixes, part 2 Answers review runner-r2 and finishes f210b6e5: - Set and notify every model's stop state before event I/O. Retain ended attempts before pause/stop writes; recover their classes and final records best-effort before runner-error stops, and block new starts during recovery. - Finish against one abandoned-request join deadline. Close pools and host before the ledger, then record late costs, crossings and interrupts before the final anchor. Snapshot unresolved requests under the ledger lock and record their unknown future spend explicitly as a runner error. - Confirm persistent sampler exceptions stop the run and a good sample clears the failure timer. Validate a resume against the current pause under the condition that clears it. - Poll pool startup with supervision and process pending stops before every lane launch. Record startup/restart spans for the report's idle calculation; drain supervisors on runner errors before closing their host and ledger. - Kill the whole anchor command process group at ANCHOR_PUSH_TIMEOUT. Retry failed pushes using ANCHOR_RETRY and sync each outcome, anchor timestamp and push interval. Surface push-log I/O failures and retain final local coverage. - Exercise disk-error recovery, late cleanup spend, unresolved requests, persistent sampler errors, the resume race, startup signals and slow pools, failed pushes and real child/grandchild timeout termination. The report must subtract validated setup spans from lane idle time and compute remote exposure from successful pushed snapshots. Local cadence alone cannot support the draft's claimed fifteen-minute maximum; neither report nor draft is changed here. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 475 +++++++++++++++++---- tests/test_confirmatory_anchor.py | 184 ++++++++ tests/test_confirmatory_run.py | 676 ++++++++++++++++++++++++++++-- 3 files changed, 1212 insertions(+), 123 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 3dfe346f..fd3d01da 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -16,7 +16,9 @@ household. The six models run at once. - **One engine pool per model**, started once, with no features of its own. Each call carries its arm's features in a call frame (``tool_call_frame``), - the message a pool started with those features would send. + the message a pool started with those features would send. ``setup-start`` + and ``setup-end`` events bracket startup and pool restarts, so setup time + can be distinguished from an idle lane. - **Admission.** A lane admits its next household when the spend of the model's finished households (every attempt, replaced ones included) plus the reserve for each household under way, this one included, is at most @@ -643,6 +645,7 @@ def sample(self) -> None: self.failure = "no memory-pressure reading" self.failing_since = self.failing_since or t else: + self.failure = None self.failing_since = None self._samples.append((t, pressure)) self._cond.notify_all() @@ -824,11 +827,13 @@ def add(self, usd: float | None) -> None: with self._lock: self.charge(float(usd or 0.0)) - def close(self) -> None: + def close(self, pending: list[dict] | None = None) -> list[dict]: """Close the ledger at the run's end: what the ceiling has counted is - then what the records hold.""" + then what the records hold. Snapshot unresolved requests under the + same lock: one finishing afterward cannot disappear from recovery.""" with self._lock: self.closed = True + return [item for item in pending or [] if item["thread"].is_alive()] def exhausted(self, reserve_usd: float = 0.0) -> bool: with self._lock: @@ -1036,7 +1041,7 @@ def attempt_line(self, sid: str, lane: int, a: Attempt, sha: str) -> None: # Stops and pauses. def halted(self) -> bool: - return self.stopped or self.run.ceiling.exhausted() + return self.stopped or self.run._failed.is_set() or self.run.ceiling.exhausted() def stop( self, reason: str, cause: str, kind: str = "stop", at: float | None = None @@ -1047,15 +1052,19 @@ def stop( raise ValueError(f"no stop has the cause {cause!r}") with self.cond: self.stopped = True - self.event(kind, reason, at, cause=cause) self.cond.notify_all() + self.event(kind, reason, at, cause=cause) - def resume(self, note: str) -> None: + def resume(self, note: str, noted_at: float) -> bool: + """A note may end only a pause it was written during; another lane's + later operator failure cannot be cleared by the older note.""" with self.cond: - if self.paused: - self.paused = False - self.event("resume", note) - self.cond.notify_all() + if not (self.paused and noted_at >= self.paused_at): + return False + self.paused = False + self.cond.notify_all() + self.event("resume", note) + return True # One household. @@ -1067,7 +1076,11 @@ def admit(self, sid: str, lane: int) -> bool: self.cond.wait(timeout=WAIT_SECONDS) # Taken under the ceiling's lock, so no decision follows the crossing. with self.run.ceiling.lock: - if self.stopped or self.run.ceiling.crossed_at is not None: + if ( + self.stopped + or self.run._failed.is_set() + or self.run.ceiling.crossed_at is not None + ): return False now = clock() finished = [h for h, s in self.state.items() if s == "finished"] @@ -1105,7 +1118,11 @@ def start_attempt(self) -> float | None: while self.paused and not self.halted(): self.cond.wait(timeout=WAIT_SECONDS) with self.run.ceiling.lock: - if self.stopped or self.run.ceiling.crossed_at is not None: + if ( + self.stopped + or self.run._failed.is_set() + or self.run.ceiling.crossed_at is not None + ): return None return clock() @@ -1118,6 +1135,7 @@ def end_attempt( record: dict, last: bool, deadline: bool = False, + retained: list[Attempt] | None = None, ) -> Attempt: """The attempt's end. A class decided without the host evidence acts at once: an operator failure pauses the lanes, a harness failure stops @@ -1127,32 +1145,35 @@ def end_attempt( end = clock() self.spent[sid] += attempt_cost(record) attempt = Attempt(label, n, start, end, record) + if retained is not None: + retained.append(attempt) # recovery owns it before any event I/O cls, detail = classify(record, start, end, None) if cls in EARLY: attempt.cls, attempt.detail = cls, detail if cls == "operator": self.paused, self.paused_at = True, clock() + if cls == "harness" or deadline: + self.stopped = True + if last: + self.state[sid] = "settling" + self.cond.notify_all() + if cls == "operator": self.event( "pause", f"{label}/{sid} attempt {n}: {detail}", at=self.paused_at ) if cls == "harness": - self.stopped = True self.event( "stop", f"{label}/{sid}: a request class on the first request ({detail})", cause="harness", ) if deadline: - self.stopped = True self.event( "stop", f"{label}/{sid} attempt {n} ran past the attempt deadline " f"of {ATTEMPT_DEADLINE:.0f} s", cause="deadline", ) - if last: - self.state[sid] = "settling" - self.cond.notify_all() return attempt def settle(self, sid: str, done: list[Attempt]) -> None: @@ -1264,27 +1285,52 @@ def log_late(self, final: bool = False) -> None: with self.cond: self.abandoned.remove(item) - def ensure_pool(self) -> bool: + def ensure_pool(self, startup: bool = False) -> bool: """The model's pool, started again if it has lost every worker; False if it cannot be.""" with self._pool_lock: + if self.halted(): + return False if ( self.pool is not None and self.pool.counters()["lost"] < self.run.workers ): return True old = self.pool - try: - self.pool = self.run.pool_factory(self) - except RuntimeError as error: - self.run.say(f"{self.model}: the engine pool did not start: {error}") - return False - if old is not None: - self.run.say( - f"{self.model}: the engine pool lost every worker; started again" + start, outcome = clock(), "failed" + if not startup: + self.event( + "setup-start", + "engine pool restart", + at=start, + phase="restart", + start=start, ) - old.close() - return True + try: + try: + self.pool = self.run.pool_factory(self) + except RuntimeError as error: + self.run.say( + f"{self.model}: the engine pool did not start: {error}" + ) + return False + if old is not None: + self.run.say( + f"{self.model}: the engine pool lost every worker; " + "started again" + ) + old.close() + outcome = "stopped" if self.halted() else "ready" + return outcome == "ready" + finally: + if not startup: + self.event( + "setup-end", + "engine pool restart", + phase="restart", + start=start, + outcome=outcome, + ) def replace(self, sid: str, lane: int, previous: Attempt, entries: list) -> None: """Save the attempt a rerun replaces, before the rerun.""" @@ -1313,7 +1359,8 @@ def passes( done: list[Attempt] = [] for i, (n, label) in enumerate(todo): if not self.ensure_pool(): - self.stop("the engine pool could not be started again", "pool") + if not self.halted(): + self.stop("the engine pool could not be started again", "pool") break start = self.start_attempt() if start is None: @@ -1322,9 +1369,15 @@ def passes( self.replace(sid, lane, attempts[label][-1], entries[label]) record, abandoned = self.converse(sid, label, n) a = self.end_attempt( - sid, label, n, start, record, i == len(todo) - 1, abandoned + sid, + label, + n, + start, + record, + i == len(todo) - 1, + abandoned, + retained=attempts[label], ) - attempts[label].append(a) done.append(a) write_json(in_flight / f"{label}-{n}.json", record) self.run.say( @@ -1357,17 +1410,16 @@ def run_household(self, sid: str, lane: int) -> bool: attempts: dict[str, list[Attempt]] = {label: [] for label in self.orders[sid]} entries: dict[str, list[dict]] = {label: [] for label in self.orders[sid]} in_flight = self.folder / "in-flight" / sid - failed = False + failure = None try: self.passes(sid, lane, attempts, entries, in_flight) except Exception as error: - failed = True - self.run.say(traceback.format_exc()) - self.run.stop_all( + self.run._failed.set() # block starts while records are recovered + failure = ( f"runner error on {self.model} lane {lane} ({sid}): " - f"{type(error).__name__}: {error}", - cause="runner-error", + f"{type(error).__name__}: {error}" ) + self.run.say(traceback.format_exc()) pending = [a for mine in attempts.values() for a in mine if not a.settled] if pending: try: @@ -1382,17 +1434,34 @@ def run_household(self, sid: str, lane: int) -> bool: for label, mine in attempts.items(): if mine and not mine[-1].logged: final = mine[-1] - sha = write_json( - self.households_dir[label] / f"{sid}.json", - {**final.record, "attempts": entries[label]}, - ) - self.attempt_line(sid, lane, final, sha) + try: + sha = write_json( + self.households_dir[label] / f"{sid}.json", + { + **final.record, + "attempts": [ + entry + for entry in entries[label] + if entry["n"] < final.n + ], + }, + ) + self.attempt_line(sid, lane, final, sha) + except Exception as error: # the other arms still get written + self.run._failed.set() + failure = failure or ( + f"runner error writing {self.model} lane {lane} ({sid}): " + f"{type(error).__name__}: {error}" + ) + self.run.say(traceback.format_exc()) + if failure is not None: + self.run.stop_all(failure, cause="runner-error") with self.cond: self.state[sid] = "finished" self.cond.notify_all() - if not failed: + if failure is None: shutil.rmtree(in_flight, ignore_errors=True) - return not failed + return failure is None def lane(self, lane: int) -> None: try: @@ -1570,22 +1639,49 @@ def __init__( self.signalled: int | None = None # set by the signal handler alone self._interrupted = False self._sampler_stopped = False + self._failed = threading.Event() # recovery blocks starts before stop I/O self._anchor_head = chain_seed(design["frozen_sha256"], "", "anchors.jsonl") self._anchored = -math.inf self._pushing = threading.Lock() + self._next_anchor = -math.inf self._push: threading.Thread | None = None + self._push_error: OSError | None = None def say(self, text: str) -> None: with self._say_lock: print(text, file=self.out, flush=True) - def stop_all(self, reason: str, cause: str) -> None: + def stop_all( + self, reason: str, cause: str, kind: str = "stop", at: float | None = None + ) -> None: + # Set every state before the first fallible write; notify even if a + # disk error prevents a model's event from being recorded. for lanes in self.lanes.values(): - lanes.stop(reason, cause) + with lanes.cond: + lanes.stopped = True + lanes.cond.notify_all() + for lanes in self.lanes.values(): + try: + lanes.event(kind, reason, at, cause=cause) + except Exception: + self.say(traceback.format_exc()) # Anchors: the logs' chain heads, written and pushed outside the run. - def anchor(self, wait: bool = False) -> None: + def _check_push_error(self) -> None: + with self._pushing: + error = self._push_error + if error is not None: + raise OSError( + f"anchor push outcome could not be recorded: {error}" + ) from error + + def anchor_due(self) -> bool: + self._check_push_error() + with self._pushing: + return time.time() >= self._next_anchor + + def anchor(self, wait: bool = False, push: bool = True) -> None: """Every log's chain head to `anchors.jsonl`, itself a chain, then the design's anchor command on that file, in the background (one at a time; an anchor whose push would wait is written but not pushed until @@ -1606,42 +1702,83 @@ def anchor(self, wait: bool = False) -> None: handle.flush() os.fsync(handle.fileno()) self._anchor_head, self._anchored = line_sha(text), time.time() + with self._pushing: + self._next_anchor = self._anchored + ANCHOR_SECONDS + if not push: + return # a push-log failure still leaves the final local heads if self._push is not None and self._push.is_alive(): if not wait: return self._push.join() + self._check_push_error() self._push = threading.Thread( - target=self._push_anchor, args=(self._anchor_head,), daemon=True + target=self._push_anchor, + args=(self._anchor_head, line["time"]), + daemon=True, ) self._push.start() if wait: self._push.join() + self._check_push_error() - def _push_anchor(self, head: str) -> None: + def _push_anchor(self, head: str, anchor_time: float) -> None: runs = Path(self.design["runs"]) command = [str(x) for x in self.design["anchor_command"]] command.append(str(runs / "anchors.jsonl")) started = time.time() try: - out = subprocess.run(command, capture_output=True, text=True, timeout=300) - code, text = out.returncode, (out.stdout + out.stderr)[-2000:] + child = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + stdout, stderr = child.communicate(timeout=ANCHOR_PUSH_TIMEOUT) + outcome = "success" if child.returncode == 0 else "failed" + except subprocess.TimeoutExpired: + # The command is a session leader: git and ssh share its group. + # Killing just the command leaves its push running after timeout. + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, stderr = child.communicate() + outcome = "timeout" + code, text = child.returncode, (stdout + stderr)[-2000:] except (OSError, subprocess.SubprocessError) as error: - code, text = None, f"{type(error).__name__}: {error}" + code, outcome, text = None, "error", f"{type(error).__name__}: {error}" + ended = time.time() + if outcome != "success": + with self._pushing: + self._next_anchor = min(self._next_anchor, ended + ANCHOR_RETRY) line = { "anchor": head, + "anchor_time": anchor_time, "started": started, - "ended": time.time(), + "ended": ended, + "outcome": outcome, "returncode": code, "output": text, } - with (runs / "anchor-pushes.jsonl").open("a") as handle: - handle.write(json.dumps(line) + "\n") - if code != 0: - self.say(f"the anchor push failed ({code}): {text}") + try: + with (runs / "anchor-pushes.jsonl").open("a") as handle: + handle.write(json.dumps(line) + "\n") + handle.flush() + os.fsync(handle.fileno()) + except OSError as error: + # A daemon's exception cannot stop the run: hand this one back + # to the supervisor, or to the final caller after it joins us. + with self._pushing: + self._push_error = self._push_error or error + return + if outcome != "success": + self.say(f"the anchor push failed ({outcome}, {code}): {text}") # The main loop's work. - def watch(self) -> None: + def watch(self, anchor: bool = True) -> None: """The ceiling's events, an interrupt, a dead sampler, resumes after operator pauses, late spend and the anchors.""" crossed = self.ceiling.crossed_at @@ -1651,8 +1788,7 @@ def watch(self) -> None: f"total spend ${self.ceiling.spent_usd:.2f} reached the hard ceiling " f"${self.ceiling.budget_usd:.2f}" ) - for lanes in self.lanes.values(): - lanes.stop(reason, "ceiling", kind="ceiling", at=crossed) + self.stop_all(reason, "ceiling", kind="ceiling", at=crossed) if self.signalled is not None and not self._interrupted: self._interrupted = True self.say( @@ -1672,27 +1808,39 @@ def watch(self) -> None: note = json.loads(path.read_text()) except (OSError, ValueError): continue # still being written - with lanes.cond: - paused, since = lanes.paused, lanes.paused_at stamp = f"{time.time():.3f}" - if paused and float(note.get("time") or 0.0) >= since: + if lanes.resume( + note.get("note") or "resumed", float(note.get("time") or 0.0) + ): path.rename(lanes.folder / f"resume-{stamp}.json") - lanes.resume(note.get("note") or "resumed") else: # Written before the pause it would end, or with no pause. path.rename(lanes.folder / f"resume-ignored-{stamp}.json") self.say(f"{lanes.model}: a resume written before any pause, ignored") for lanes in self.lanes.values(): lanes.log_late() - if time.time() - self._anchored >= ANCHOR_SECONDS: + if anchor and self.anchor_due(): self.anchor() def execute(self) -> None: runs = Path(self.design["runs"]) - self.host = self.host_factory(runs / "host") + starting, threads = {}, [] + setup_open = set() try: - starting = {} + self.host = self.host_factory(runs / "host") + ready, failures = {}, {} + startup = clock() + for lanes in self.lanes.values(): + lanes.event( + "setup-start", + "run startup", + at=startup, + phase="startup", + start=startup, + ) + setup_open.add(lanes.model) for model, lanes in self.lanes.items(): + self.watch(anchor=False) lanes.completion = self.completion_for(lanes.profiles["pe3"]) stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) lanes.manifest_sha256 = write_json( @@ -1701,41 +1849,188 @@ def execute(self) -> None: # A model whose lanes never decide still has both logs. for name in ("run-log.jsonl", "admission-log.jsonl"): (lanes.folder / name).touch() + + def start_pool(model, lanes): + try: + ready[model] = lanes.ensure_pool(startup=True) + except Exception as error: + failures[model] = error + self._failed.set() + + for model, lanes in self.lanes.items(): + self.watch() + if lanes.halted(): + continue starting[model] = threading.Thread( - target=lanes.ensure_pool, daemon=True + target=start_pool, args=(model, lanes), daemon=True ) starting[model].start() - for thread in starting.values(): - thread.join() - threads = [ - threading.Thread( - target=lanes.lane, args=(lane,), name=f"{model}-{lane}", daemon=True + while any(t.is_alive() for t in starting.values()): + self.watch() + for thread in starting.values(): + thread.join(timeout=0.5 / len(starting)) + self.watch() + for model, lanes in self.lanes.items(): + outcome = ( + "failed" + if model in failures + else "stopped" + if lanes.halted() + else "ready" + if ready.get(model) + else "failed" ) - for model, lanes in self.lanes.items() - for lane in (1, 2) - ] - for thread in threads: - thread.start() + lanes.event( + "setup-end", + "run startup", + phase="startup", + start=startup, + outcome=outcome, + ) + setup_open.discard(model) + if outcome == "failed" and model not in failures: + lanes.stop("the engine pool could not be started", "pool") + if failures: + raise next(iter(failures.values())) + threads = [] + for model, lanes in self.lanes.items(): + for lane in (1, 2): + self.watch() + with lanes.cond: + if lanes.halted() or self.signalled is not None: + continue + thread = threading.Thread( + target=lanes.lane, + args=(lane,), + name=f"{model}-{lane}", + daemon=True, + ) + thread.start() + threads.append(thread) while any(t.is_alive() for t in threads): self.watch() for t in threads: t.join(timeout=0.5 / len(threads)) self.watch() + except Exception as error: + self._failed.set() + # Supervisors finish their bounded attempts and recovery before + # the ledger, host and final anchor can close underneath them. + for lanes in self.lanes.values(): + with lanes.cond: + lanes.cond.notify_all() + for thread in [*starting.values(), *threads]: + while thread.is_alive(): + thread.join(timeout=WAIT_SECONDS) + try: + self.watch(anchor=False) + except Exception: + self.say(traceback.format_exc()) + for model in setup_open: + try: + self.lanes[model].event( + "setup-end", + "run startup", + phase="startup", + start=startup, + outcome="failed", + ) + except Exception: + self.say(traceback.format_exc()) + self.stop_all( + f"runner error: {type(error).__name__}: {error}", "runner-error" + ) + raise finally: - for model, lanes in self.lanes.items(): - for item in list(lanes.abandoned): - item["thread"].join(timeout=LATE_JOIN) - lanes.log_late(final=True) + self.finish() + + def finish(self) -> None: + """One join deadline, then close pools and host before closing the + ledger. Its final late costs, crossings and signals precede the last + anchor. An unresolved request makes future spend unknowable: record + it explicitly, with a runner-error stop, rather than claim it settled.""" + errors = [] + + def attempt(work): + try: + work() + return True + except Exception as error: + errors.append(error) + self._failed.set() + self.say(traceback.format_exc()) + self.stop_all( + f"runner error during cleanup: {type(error).__name__}: {error}", + "runner-error", + ) + return False + + until = time.monotonic() + LATE_JOIN + for lanes in self.lanes.values(): + for item in list(lanes.abandoned): + while item["thread"].is_alive() and time.monotonic() < until: + item["thread"].join( + timeout=min(WAIT_SECONDS, max(0.0, until - time.monotonic())) + ) + attempt(lambda: self.watch(anchor=False)) + for lanes in self.lanes.values(): + if lanes.pool is not None: + attempt(lanes.pool.close) + if self.host is not None: + attempt(self.host.close) + pending = [] + for lanes in self.lanes.values(): + with lanes.cond: + pending.extend({**item, "lanes": lanes} for item in lanes.abandoned) + unresolved = self.ceiling.close(pending) + for lanes in self.lanes.values(): + attempt(lambda: lanes.log_late(final=True)) + for item in unresolved: + attempt( + lambda: item["lanes"].event( + "late-unresolved", + "request unresolved when the ledger closed; " + "future spend is unrecorded", + household=item["household"], + arm=item["arm"], + attempt=item["attempt"], + outcome="unresolved-at-ledger-close", + counted_usd=item["spend"].spent_usd + item["spend"].late_usd, + ) + ) + if unresolved: + self._failed.set() + self.stop_all( + f"{len(unresolved)} abandoned requests unresolved at ledger close; " + "future spend is unrecorded", + "runner-error", + ) + # A first signal can also arrive while the final push is running. + while True: + attempt(lambda: self.watch(anchor=False)) + for lanes in self.lanes.values(): stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) - write_json( - lanes.folder / f"run-summary-{stamp}.json", - lanes.summary(), - indent=2, + attempt( + lambda: write_json( + lanes.folder / f"run-summary-{stamp}.json", + lanes.summary(), + indent=2, + ) ) - if lanes.pool is not None: - lanes.pool.close() - self.host.close() - self.anchor(wait=True) + if not attempt(lambda: self.anchor(wait=True)): + attempt(lambda: self.watch(anchor=False)) + # The error handler appended stops after that snapshot. Keep + # them covered locally even if pushing cannot be recorded. + try: + self.anchor(push=False) + except Exception as error: + errors.append(error) + self.say(traceback.format_exc()) + break + if self.signalled is None or self._interrupted: + break + if errors: + raise errors[0] def interrupt(self, signum, frame) -> None: """The signal handler. It only notes the signal; the main loop stops diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index 626f3ba2..8bc4c175 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -1,8 +1,15 @@ """The anchor command: each call commits the run's anchors and pushes them.""" import hashlib +import io +import json +import math +import os +import signal import subprocess import sys +import threading +import time from pathlib import Path import pytest @@ -10,6 +17,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) import confirmatory_anchor # noqa: E402 +import confirmatory_run as cr # noqa: E402 SCRIPT = Path(confirmatory_anchor.__file__) @@ -89,3 +97,179 @@ def test_a_failed_push_exits_nonzero(clone, tmp_path): text=True, ) assert out.returncode != 0 + + +def bare_run(folder, command): + """Just the anchor writer, without the board, engines or providers.""" + run = object.__new__(cr.Run) + run.design = {"runs": str(folder), "anchor_command": command} + run.out, run._say_lock = io.StringIO(), threading.Lock() + run.lanes, run.host = {}, None + run._anchor_head = cr.chain_seed("a" * 64, "", "anchors.jsonl") + run._anchored = run._next_anchor = -math.inf + run._pushing, run._push = threading.Lock(), None + run._push_error = None + return run + + +def test_failed_pushes_retry_and_keep_every_outcome(tmp_path, monkeypatch): + """A failed remote never looks anchored; retry follows the failure's end, + then a successful push restores the ordinary cadence.""" + now, calls = [1000.0], [] + monkeypatch.setattr(cr.time, "time", lambda: now[0]) + monkeypatch.setattr(cr, "clock", lambda: now[0]) + monkeypatch.setattr(cr, "ANCHOR_SECONDS", 900.0) + monkeypatch.setattr(cr, "ANCHOR_RETRY", 60.0) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 123.0) + + class Push: + def __init__(self, command, **kwargs): + calls.append((command, kwargs)) + self.returncode = 1 if len(calls) < 4 else 0 + + def communicate(self, timeout): + assert timeout == cr.ANCHOR_PUSH_TIMEOUT + now[0] += 7.0 + return "", "remote unavailable" if self.returncode else "pushed" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + run = bare_run(tmp_path, ["push-anchors"]) + assert run.anchor_due() + for i in range(4): + started = now[0] + run.anchor(wait=True) + assert not run.anchor_due() + expected = now[0] + cr.ANCHOR_RETRY if i < 3 else started + cr.ANCHOR_SECONDS + assert run._next_anchor == expected + now[0] = expected + assert run.anchor_due() + anchors = [ + json.loads(x) for x in (tmp_path / "anchors.jsonl").read_text().splitlines() + ] + pushes = [ + json.loads(x) + for x in (tmp_path / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert [x["outcome"] for x in pushes] == ["failed"] * 3 + ["success"] + assert [x["returncode"] for x in pushes] == [1, 1, 1, 0] + assert [x["anchor_time"] for x in pushes] == [x["time"] for x in anchors] + assert all(x["started"] == x["anchor_time"] < x["ended"] for x in pushes) + assert all(x[1]["start_new_session"] for x in calls) + assert len({x["anchor"] for x in pushes}) == 4 + + +def test_an_anchor_command_that_cannot_start_is_recorded(tmp_path): + run = bare_run(tmp_path, [str(tmp_path / "missing-command")]) + run.anchor(wait=True) + push = json.loads((tmp_path / "anchor-pushes.jsonl").read_text()) + assert push["outcome"] == "error" and push["returncode"] is None + assert "FileNotFoundError" in push["output"] + assert run._next_anchor == push["ended"] + cr.ANCHOR_RETRY + + +@pytest.mark.parametrize("wait", [False, True]) +def test_push_log_io_failure_reaches_the_supervisor(tmp_path, monkeypatch, wait): + """A successful remote push whose outcome could not be synced is still a + runner error, both in supervision and in the final blocking anchor.""" + calls, fsync = [], os.fsync + + class Push: + returncode = 0 + + def __init__(self, *args, **kwargs): + pass + + def communicate(self, timeout): + return "pushed", "" + + def fail_push_sync(fd): + calls.append(fd) + if len(calls) == 2: # The anchor itself was synced; its push outcome was not. + raise OSError("push-log fsync failed") + return fsync(fd) + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + monkeypatch.setattr(cr.os, "fsync", fail_push_sync) + run = bare_run(tmp_path, ["push-anchors"]) + if wait: + with pytest.raises(OSError, match="anchor push outcome could not be recorded"): + run.anchor(wait=True) + else: + run.anchor() + run._push.join() + with pytest.raises(OSError, match="push-log fsync failed"): + run.anchor_due() + assert not run._push.is_alive() + assert len(calls) == 2 + + +def test_a_push_timeout_kills_its_children_and_grandchildren(tmp_path, monkeypatch): + """All descendants inherit the captured pipe: reaping the command cannot + finish until they close it. The timeout starts after both have launched.""" + script = tmp_path / "push.py" + script.write_text( + "import os, subprocess, sys, threading\n" + "from pathlib import Path\n" + "folder, level = Path(sys.argv[1]), int(sys.argv[2])\n" + "(folder / f'pid-{level}').write_text(str(os.getpid()))\n" + "if level < 2:\n" + " child = subprocess.Popen(\n" + " [sys.executable, __file__, str(folder), str(level + 1)])\n" + "threading.Event().wait()\n" + ) + run = bare_run(tmp_path, [sys.executable, str(script), str(tmp_path), "0"]) + real_popen, real_killpg = subprocess.Popen, os.killpg + children, kills, errors = [], [], [] + done = threading.Event() + + def popen(*args, **kwargs): + child = real_popen(*args, **kwargs) + children.append(child) + communicate = child.communicate + + def ready_communicate(timeout=None): + if timeout is not None: + deadline = time.monotonic() + 30.0 + ready = tmp_path / "pid-2" + while not ready.exists() or not ready.read_text().strip(): + assert time.monotonic() < deadline, "the descendants did not start" + done.wait(0.01) + return communicate(timeout=timeout) + + child.communicate = ready_communicate + return child + + def killpg(pid, signum): + kills.append((pid, signum)) + return real_killpg(pid, signum) + + def push(): + try: + run.anchor(wait=True) + except BaseException as error: + errors.append(error) + finally: + done.set() + + with monkeypatch.context() as patch: + patch.setattr(cr.subprocess, "Popen", popen) + patch.setattr(cr.os, "killpg", killpg) + patch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 0.05) + thread = threading.Thread(target=push, daemon=True) + thread.start() + finished = done.wait(45.0) + if not finished: + for child in children: + try: + real_killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + thread.join(timeout=5.0) + assert finished, "the timed-out command left descendants holding its pipes" + assert not errors + assert len(children) == 1 and kills == [(children[0].pid, signal.SIGKILL)] + push = json.loads((tmp_path / "anchor-pushes.jsonl").read_text()) + assert push["outcome"] == "timeout" and push["returncode"] == -signal.SIGKILL + assert push["started"] <= push["ended"] + for level in (1, 2): + assert int((tmp_path / f"pid-{level}").read_text()) > 0 diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 1e962383..1be358f9 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -52,7 +52,7 @@ ANCHOR_SCRIPT = Path(__file__).resolve().parent / "fake_anchor.py" # How far apart anchors and their triggers' stops may be in a test, on a # loaded host. -ANCHOR_SLACK = 5.0 +ANCHOR_SLACK = 30.0 TRIGGER_BOUND = 5.0 @@ -78,6 +78,7 @@ def test_a_stream_is_keyed_by_its_name_alone(): ) +@settings(deadline=None) @given(st.integers(0, (1 << 64) - 1), st.integers(1, 12)) def test_the_draw_is_seeded_and_without_replacement(master, n): got = cr.draw(master, list(reversed(POOL)), n=n) @@ -349,11 +350,11 @@ def host(self, folder): @pytest.fixture def fast(monkeypatch): - """Windows scaled down a thousandfold, so a fake run takes a second; the - host-log gap less, so a sampler a busy host stalls past 90 ms (seen at a - load of 12) is not a gap.""" + """Short evidence windows, with room for the shared host's scheduler. + Intentional gaps scale with this bound; ordinary stalls must not turn + an untouched fake run into a host-failure test.""" monkeypatch.setattr(cr, "PRESSURE_WINDOW", 0.06) - monkeypatch.setattr(cr, "HOST_LOG_GAP", 0.5) + monkeypatch.setattr(cr, "HOST_LOG_GAP", 5.0) monkeypatch.setattr(cr, "WAIT_SECONDS", 0.005) monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) monkeypatch.setattr(cr, "SAMPLER_SILENCE", 1.0) @@ -399,8 +400,9 @@ def test_a_stalled_host_log_is_a_gap(tmp_path, fast): host = readers.host(tmp_path) time.sleep(0.03) start = time.time() - readers.stall = 0.8 - time.sleep(0.85) + stall = cr.HOST_LOG_GAP + 0.3 + readers.stall = stall + time.sleep(stall + 0.05) end = time.time() host.wait_settled(end) host.refresh(after=end) @@ -435,7 +437,7 @@ def test_a_failed_pressure_reading_is_no_sample(tmp_path, fast): time.sleep(0.03) start = time.time() readers.pressure = None - time.sleep(0.7) + time.sleep(cr.HOST_LOG_GAP + 0.2) readers.pressure = 1 end = time.time() host.wait_settled(end) @@ -512,10 +514,11 @@ def test_a_sampler_whose_samples_fail_is_dead_after_its_silence(tmp_path, fast): host = readers.host(tmp_path) assert host.dead() is None readers.pressure = None + host.sample() time.sleep(cr.SAMPLER_SILENCE + 0.2) assert "samples failing since" in host.dead() readers.pressure = 1 - time.sleep(0.1) + host.sample() assert host.dead() is None # a good sample: alive again host.close() @@ -632,6 +635,7 @@ def __call__(self, lanes, sid, label, tool_call, spend): if spec.get("pressure"): with readers.lock: readers.pressing += 1 + lanes.run.host.sample() # the planted pressure is actually observed if spec.get("stall"): readers.stall = spec["stall"] start = time.time() @@ -685,12 +689,21 @@ def __call__(self, lanes, sid, label, tool_call, spend): folder = lanes.folder def resume(): + with lanes.cond: + if ( + not lanes.cond.wait_for( + lambda: lanes.paused or lanes.halted(), timeout=30 + ) + or not lanes.paused + ): + return + time.sleep(spec["resume_after"]) cr.write_json( folder / "RESUME.json", - {"time": time.time(), "note": "fixed in a test"}, + {"time": cr.clock(), "note": "fixed in a test"}, ) - threading.Timer(spec["resume_after"], resume).start() + threading.Thread(target=resume, daemon=True).start() return rec @@ -1320,7 +1333,7 @@ def test_host_trouble_during_an_attempt_reruns_it(world): pressured, stalled = w.draw_order[0], w.draw_order[2] # both lane 1 w.provider.specs = { (model, "loo-flat", pressured, 1): {"pressure": True, "sleep": 0.05}, - (model, "pe", stalled, 1): {"stall": 0.8}, + (model, "pe", stalled, 1): {"stall": cr.HOST_LOG_GAP + 0.3}, } w.execute() @@ -1341,7 +1354,10 @@ def first(sid, label): assert run_problems(w) == [] -def test_a_guard_pause_over_an_attempt_reruns_it(world): +def test_a_guard_pause_over_an_attempt_reruns_it(world, monkeypatch): + # Guard lines have whole seconds and the fake brackets by another + # second: keep the lookahead longer than its possible two-second backdate. + monkeypatch.setattr(cr, "PRESSURE_WINDOW", 2.1) model = "gpt-6-sol" w = world(models=(model,)) sid = w.draw_order[0] @@ -1507,7 +1523,11 @@ def late(sid, lane): w.execute() admission = w.runs() / "gpt-6-sol" / "admission-log.jsonl" assert admission.exists() and admission.read_text() == "" - assert [x["event"] for x in w.lines("gpt-6-sol")] == ["ceiling"] + assert [ + x["event"] + for x in w.lines("gpt-6-sol") + if x["event"] not in ("setup-start", "setup-end") + ] == ["ceiling"] assert run_problems(w) == [] @@ -1568,6 +1588,11 @@ def call(self, message): w.provider.specs = {(model, "pe", sid, 1): {"calls": [{"situation": "{}"}]}} w.execute() assert len(starts) == 2 and starts[0].closed + setup = [e for e in w.events(model) if e.get("phase") == "restart"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[0]["start"] == setup[0]["time"] == setup[1]["start"] + assert setup[1]["time"] >= setup[0]["time"] + assert setup[1]["outcome"] == "ready" assert run_problems(w) == [] @@ -1593,36 +1618,62 @@ def losing(self, message): stops = w.events(model, "stop") assert stops and "engine pool" in stops[0]["reason"] assert stops[0]["cause"] == "pool" + setup = [e for e in w.events(model) if e.get("phase") == "restart"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[1]["outcome"] == "failed" assert run_problems(w) == [] def test_a_household_a_stop_cuts_short_is_under_way_until_the_stop(world): """Lane 1's first household loses the pool in its first arm, and the - restart fails 0.6 s later. In between, lane 2 finishes its household and + restart waits while lane 2 finishes its household and decides its next: the household lane 1 cut short has ended its last attempt, but it is still under way, so its spend is not counted.""" model = "gpt-6-sol" made = [] + other_second, restarting, decided_after = ( + threading.Event(), + threading.Event(), + threading.Event(), + ) class Losing(FakePool): def call(self, message): + assert other_second.wait(30) self.lost = 2 return super().call(message) def pool(lanes): made.append(1) if len(made) > 1: - time.sleep(0.6) + restarting.set() + assert decided_after.wait(30) raise RuntimeError("tool worker failed to start: a test") return Losing(lanes) w = world(models=(model,), pool=pool) cut, other, after = w.draw_order[0], w.draw_order[1], w.draw_order[3] - orders = w.run.lanes[model].orders + lanes = w.run.lanes[model] + orders, admit = lanes.orders, lanes.admit + + def in_second(lanes): + other_second.set() + assert restarting.wait(30) + + def admitted(sid, lane): + ok = admit(sid, lane) + if sid == after and ok: + decided_after.set() + # Keep this admitted household from racing the failed restart + # for the pool lock before lane 1 records its stop. + with lanes.cond: + assert lanes.cond.wait_for(lambda: lanes.stopped, timeout=30) + return ok + + lanes.admit = admitted w.provider.specs = { - (model, orders[cut][0], cut, 1): {"sleep": 0.1, "calls": [{"situation": "{}"}]}, - (model, orders[other][0], other, 1): {"sleep": 0.02}, - (model, orders[other][1], other, 1): {"sleep": 0.3}, + (model, orders[cut][0], cut, 1): {"calls": [{"situation": "{}"}]}, + (model, orders[other][1], other, 1): {"check": in_second}, } w.execute() stop = w.events(model, "stop")[0]["time"] @@ -1633,6 +1684,11 @@ def pool(lanes): assert [x["arm"] for x in w.attempts(model) if x["household"] == cut] == [ orders[cut][0] ] + setup = [e for e in w.events(model) if e.get("phase") == "restart"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[0]["start"] == setup[0]["time"] == setup[1]["start"] + assert setup[1]["time"] >= decided[after]["time"] + assert setup[1]["outcome"] == "failed" assert run_problems(w) == [] @@ -1708,7 +1764,7 @@ def crossing(): @pytest.mark.parametrize( "when", ["before the run", "before the run, misdated", "during the pause"] ) -def test_a_resume_written_before_the_pause_is_ignored(world, when): +def test_a_resume_written_before_the_pause_is_ignored(world, when, monkeypatch): """A resume left from before a pause would end that pause unseen: it is set aside, and the pause holds until a resume written during it. One found while no pause holds is set aside at once, whatever time it @@ -1725,27 +1781,46 @@ def test_a_resume_written_before_the_pause_is_ignored(world, when): stale["time"] += 3600.0 # a writer whose clock runs ahead cr.write_json(folder / "RESUME.json", stale) - def plant(lanes): - # Lands once the pause holds: the conversation returns at once. - threading.Timer( - 0.1, lambda: cr.write_json(folder / "RESUME.json", stale) - ).start() + lanes, watch = w.run.lanes[model], w.run.watch + planted, fresh = False, False + + def supervised(*args, **kwargs): + nonlocal planted, fresh + with lanes.cond: + if when == "during the pause" and lanes.paused and not planted: + cr.write_json(folder / "RESUME.json", stale) + planted = True + watch(*args, **kwargs) + with lanes.cond: + if ( + lanes.paused + and list(folder.glob("resume-ignored-*.json")) + and not fresh + ): + # The stale note has been read and archived before its + # replacement exists; the next watch sees only the fresh one. + cr.write_json( + folder / "RESUME.json", + {"time": cr.clock(), "note": "fixed in a test"}, + ) + fresh = True + + monkeypatch.setattr(w.run, "watch", supervised) w.provider.specs = { (model, "pe3", sid, 1): { "status": "provider_error", "error": "AuthenticationError: bad key", "turns": 0, - "resume_after": 1.5, # the main loop looks every 0.5 s - **({"check": plant} if when == "during the pause" else {}), } } w.execute() pauses, resumes = w.events(model, "pause"), w.events(model, "resume") assert len(pauses) == 1 and len(resumes) == 1 assert resumes[0]["reason"] == "fixed in a test" - assert resumes[0]["time"] - pauses[0]["time"] >= 1.4 - assert list(folder.glob("resume-ignored-*.json")) + assert resumes[0]["time"] >= pauses[0]["time"] + ignored = list(folder.glob("resume-ignored-*.json")) + assert len(ignored) == 1 and json.loads(ignored[0].read_text()) == stale assert run_problems(w) == [] @@ -1965,10 +2040,16 @@ def test_the_checker_catches_an_edited_log_line(world): path = w.runs() / "gpt-6-sol" / "admission-log.jsonl" lines = path.read_text().splitlines() edited = json.loads(lines[1]) - edited["finished_spend"] = 0.0 + edited["finished_spend"] += 1.0 path.write_text("\n".join([lines[0], json.dumps(edited)] + lines[2:]) + "\n") problems = run_problems(w) assert any("its prev is not the line before's hash" in p for p in problems) + # An interior edit need not change a later anchor's head: its successor + # still has its old bytes. A tail edit separately exercises the anchor. + edited = json.loads(lines[-1]) + edited["finished_spend"] += 1.0 + path.write_text("\n".join(lines[:-1] + [json.dumps(edited)]) + "\n") + problems = run_problems(w) assert any("is not the log's head" in p for p in problems) @@ -2035,9 +2116,11 @@ def test_the_checker_catches_a_planted_fault(world): w = world(models=("gpt-6-sol",)).execute() path = w.runs() / "gpt-6-sol" / "run-log.jsonl" lines = path.read_text().splitlines() - first = json.loads(lines[0]) + at = next(i for i, line in enumerate(lines) if "event" not in json.loads(line)) + first = json.loads(lines[at]) first["class"] = "provider" - path.write_text("\n".join([json.dumps(first)] + lines[1:]) + "\n") + lines[at] = json.dumps(first) + path.write_text("\n".join(lines) + "\n") assert any("recomputed" in p for p in run_problems(w)) @@ -2172,3 +2255,530 @@ def test_the_ceiling_ends_a_harness_conversation_at_its_next_turn(world): assert len(attempts) <= 2 # one per lane at most assert len(w.events(model, "ceiling")) == 1 assert run_problems(w) == [] + + +def test_a_reader_that_keeps_raising_starts_the_sampler_silence( + tmp_path, fast, monkeypatch +): + readers = Readers() + host = readers.host(tmp_path) + host._stop.set() + host._thread.join() + now, reads = [time.time()], [] + + def pressure(): + reads.append(now[0]) + raise RuntimeError("pressure reader failed") + + class Ticks: + def wait(self, timeout): + now[0] += cr.SAMPLER_SILENCE + 1.0 + return len(reads) == 3 + + # Run the real sampling loop across three failures with a controlled + # clock; no scheduling or sleep decides whether its silence has elapsed. + with monkeypatch.context() as patch: + patch.setattr(cr.time, "time", lambda: now[0]) + patch.setattr(host, "_stop", Ticks()) + patch.setattr(host, "_read_pressure", pressure) + patch.setattr(host._thread, "is_alive", lambda: True) + host._loop() + assert len(reads) == 3 + assert host.failing_since == reads[0] + assert "samples failing since" in host.dead() + assert "RuntimeError: pressure reader failed" in host.dead() + patch.setattr(host, "_read_pressure", lambda: 1) + host.sample() + assert host.failing_since is None and host.failure is None + assert host.dead() is None + host.close() + + +def test_a_second_lane_s_authentication_failure_invalidates_a_resume( + world, monkeypatch +): + model = "gpt-6-luna" + w = world(models=(model,)) + lanes = w.run.lanes[model] + first, second = w.draw_order[:2] + lanes.spent.update({first: 0.0, second: 0.0}) + now = [time.time()] + + def tick(): + now[0] += 1.0 + return now[0] + + monkeypatch.setattr(cr, "clock", tick) + auth = record("provider_error", "AuthenticationError: bad key", turns=0) + lanes.end_attempt(first, "pe3", 1, now[0], auth, False) + note = {"time": lanes.paused_at, "note": "first key fixed"} + cr.write_json(lanes.folder / "RESUME.json", note) + resume = lanes.resume + + def second_failure(*args, **kwargs): + # The other lane finishes after watch reads the note and immediately + # before resume validates it. The second pause requires its own note. + lanes.end_attempt(second, "pe3", 1, now[0], auth, False) + return resume(*args, **kwargs) + + monkeypatch.setattr(lanes, "resume", second_failure) + w.run.watch(anchor=False) + assert lanes.paused and lanes.paused_at > note["time"] + assert len(w.events(model, "pause")) == 2 + assert w.events(model, "resume") == [] + assert list(lanes.folder.glob("resume-ignored-*.json")) + monkeypatch.setattr(lanes, "resume", resume) + cr.write_json( + lanes.folder / "RESUME.json", + {"time": lanes.paused_at, "note": "both keys fixed"}, + ) + w.run.watch(anchor=False) + assert not lanes.paused + assert [event["reason"] for event in w.events(model, "resume")] == [ + "both keys fixed" + ] + + +def test_an_interrupt_during_setup_starts_no_pool_or_lane(world, monkeypatch): + started = [] + w = world( + models=("gpt-6-luna", "gpt-6-sol"), + pool=lambda lanes: started.append((lanes.model, "pool")), + ) + + def completion(profile): + if w.run.signalled is None: + w.run.interrupt(2, None) + + w.run.completion_for = completion + for model, lanes in w.run.lanes.items(): + monkeypatch.setattr( + lanes, "lane", lambda lane, m=model: started.append((m, lane)) + ) + w.execute() + assert started == [] and w.provider.count == {} + for model in w.models: + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + setup = [e for e in w.events(model) if e.get("phase") == "startup"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[0]["start"] == setup[0]["time"] == setup[1]["start"] + stop = w.events(model, "stop")[0]["time"] + assert setup[0]["time"] <= stop <= setup[1]["time"] + assert setup[1]["outcome"] == "stopped" + assert run_problems(w) == [] + + +def test_a_slow_pool_start_is_polled_and_anchored_before_it_finishes( + world, monkeypatch +): + model = "gpt-6-sol" + entered, release, finished = threading.Event(), threading.Event(), threading.Event() + polls, anchors = [], [] + + def pool(lanes): + entered.set() + # Readiness depends on supervision, never on which sleep wins. + assert release.wait(10), "startup was not supervised" + finished.set() + return FakePool(lanes) + + w = world(models=(model,), pool=pool) + watch, anchor = w.run.watch, w.run.anchor + + def supervised(*args, **kwargs): + watch(*args, **kwargs) + if entered.is_set() and not finished.is_set(): + polls.append(cr.clock()) + if len(polls) == 3: + release.set() + + def anchored(*args, **kwargs): + if entered.is_set() and not finished.is_set(): + anchors.append(cr.clock()) + return anchor(*args, **kwargs) + + monkeypatch.setattr(w.run, "watch", supervised) + monkeypatch.setattr(w.run, "anchor", anchored) + w.execute() + assert len(polls) >= 3 and anchors + setup = [e for e in w.events(model) if e.get("phase") == "startup"] + assert setup[0]["time"] <= min(polls + anchors) + assert max(polls + anchors) <= setup[1]["time"] + assert setup[1]["outcome"] == "ready" + assert all(a["start"] >= setup[1]["time"] for a in w.attempts(model)) + assert run_problems(w) == [] + + +def test_an_interrupt_during_pool_start_is_recorded_before_readiness( + world, monkeypatch +): + model = "gpt-6-sol" + stopped = threading.Event() + started = [] + + def pool(lanes): + w.run.interrupt(2, None) + assert stopped.wait(10), "the setup interrupt was not processed" + return FakePool(lanes) + + w = world(models=(model,), pool=pool) + watch = w.run.watch + + def supervised(*args, **kwargs): + watch(*args, **kwargs) + if w.run.lanes[model].stopped: + stopped.set() + + monkeypatch.setattr(w.run, "watch", supervised) + monkeypatch.setattr(w.run.lanes[model], "lane", started.append) + w.execute() + assert stopped.is_set() and started == [] + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + assert w.events(model, "setup-end")[0]["outcome"] == "stopped" + assert w.run.lanes[model].pool.closed + assert run_problems(w) == [] + + +def test_an_interrupt_between_lane_launches_prevents_the_next_lane(world, monkeypatch): + model = "gpt-6-sol" + w = world(models=(model,)) + signalled = threading.Event() + started = [] + start = threading.Thread.start + + def lane(number): + started.append(number) + w.run.interrupt(2, None) + signalled.set() + + def start_and_signal(thread): + start(thread) + if thread.name == f"{model}-1": + assert signalled.wait(10) + + monkeypatch.setattr(w.run.lanes[model], "lane", lane) + monkeypatch.setattr(threading.Thread, "start", start_and_signal) + w.execute() + assert started == [1] + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + assert run_problems(w) == [] + + +def test_stop_all_notifies_every_model_before_a_failed_event_write(world, monkeypatch): + w = world(models=("gpt-6-luna", "gpt-6-sol")) + notified = [] + for model, lanes in w.run.lanes.items(): + notify = lanes.cond.notify_all + + def observed(model=model, notify=notify): + notified.append(model) + notify() + + monkeypatch.setattr(lanes.cond, "notify_all", observed) + first = w.run.lanes[w.models[0]] + + def broken(line, name="run-log.jsonl"): + assert all(lanes.stopped for lanes in w.run.lanes.values()) + raise OSError("the first model's event log is unavailable") + + monkeypatch.setattr(first, "write_line", broken) + w.run.stop_all("disk failure", "runner-error") + assert notified == list(w.models) + assert all(lanes.stopped for lanes in w.run.lanes.values()) + assert w.events(w.models[1], "stop")[0]["cause"] == "runner-error" + + +@pytest.mark.parametrize( + "kind,error", [("pause", "AuthenticationError"), ("stop", "BadRequestError")] +) +def test_end_attempt_retains_the_record_before_a_failed_event_write( + world, monkeypatch, kind, error +): + model = "gpt-6-luna" + w = world(models=(model, "gpt-6-sol")) + sid = w.draw_order[0] + lanes = w.run.lanes[model] + label = lanes.orders[sid][0] + w.provider.specs = { + (model, label, sid, 1): { + "status": "provider_error", + "error": f"{error}: a test", + "turns": 0 if kind == "stop" else 1, + } + } + write = lanes.write_line + failed = [] + + def broken(line, name="run-log.jsonl"): + if line.get("event") == kind and not failed: + failed.append(line) + raise OSError("end_attempt's event write failed") + write(line, name) + + monkeypatch.setattr(lanes, "write_line", broken) + w.execute() + assert len(failed) == 1 + record = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + assert record["error"] == f"{error}: a test" + assert record["totals"]["cost_usd"] == pytest.approx(0 if kind == "stop" else COST) + lines = w.lines(model) + saved = next( + i + for i, x in enumerate(lines) + if x.get("household") == sid and x.get("arm") == label + ) + stopped = next(i for i, x in enumerate(lines) if x.get("cause") == "runner-error") + assert saved < stopped + assert all(lanes.stopped for lanes in w.run.lanes.values()) + assert ( + lanes.folder / "in-flight" / sid + ).exists() is False # event failed before disk write + + +def test_a_final_record_write_failure_still_recovers_the_other_arms(world, monkeypatch): + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + w.run.no_output.update(w.draw_order[1:]) + first = w.run.lanes[model].orders[sid][0] + target = w.folder(model, first) / f"{sid}.json" + write = cr.write_json + + def broken(path, payload, indent=1): + if Path(path) == target: + raise OSError("one final record cannot be written") + return write(path, payload, indent) + + monkeypatch.setattr(cr, "write_json", broken) + w.execute() + assert len(w.attempts(model)) == len(cr.ARMS) - 1 + assert all( + (w.folder(model, a) / f"{sid}.json").exists() for a in cr.ARMS if a != first + ) + assert (w.run.lanes[model].folder / "in-flight" / sid / f"{first}-1.json").exists() + assert w.lines(model)[-1]["cause"] == "runner-error" + + +def blocked_cleanup_request(w, monkeypatch): + """One real request, released by the test after the abandoned join.""" + monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 0.02) + monkeypatch.setattr(cr, "LATE_JOIN", 0.0) + monkeypatch.setattr(cr, "CEILING_MARGIN", 0.0) + w.design.update(overall_cap=0.10, spent_before=0.0) + w.run.no_output.update(w.draw_order[1:]) + w.run.ceiling = cr.Ceiling(0.10, 0.0) + release, returned = threading.Event(), threading.Event() + + def converse(lanes, sid, label, tool_call, spend): + assert release.wait(30) + spend.add(0.20) + returned.set() + return {"status": "budget_stop", "turns": [], "totals": {"cost_usd": 0.20}} + + w.run.converse = converse + return release, returned + + +def test_cleanup_counts_a_request_returning_during_host_close(world, monkeypatch): + model = "gpt-6-luna" + w = world(models=(model,)) + release, returned = blocked_cleanup_request(w, monkeypatch) + host_factory = w.run.host_factory + + def host(folder): + evidence = host_factory(folder) + close = evidence.close + + def finish(): + release.set() + assert returned.wait(10) + for lanes in w.run.lanes.values(): + for item in list(lanes.abandoned): + item["thread"].join(timeout=10) + assert not item["thread"].is_alive() + close() + + evidence.close = finish + return evidence + + w.run.host_factory = host + try: + w.execute() + finally: + release.set() + assert returned.is_set() + assert w.run.ceiling.spent_usd == pytest.approx(0.20) + assert w.run.ceiling.closed + assert [e["cost_usd"] for e in w.events(model, "late-cost")] == [ + pytest.approx(0.20) + ] + assert len(w.events(model, "ceiling")) == 1 + assert w.events(model, "late-unresolved") == [] + assert anchor_problems(w) == [] + assert run_problems(w) == [] + + +@pytest.mark.parametrize("returns_at_close", [False, True]) +def test_cleanup_records_an_unresolved_request_before_closing_the_ledger( + world, monkeypatch, returns_at_close +): + model = "gpt-6-luna" + w = world(models=(model,)) + release, returned = blocked_cleanup_request(w, monkeypatch) + if returns_at_close: + close = w.run.ceiling.close + + def finish(pending=None): + unresolved = close(pending) + release.set() + assert returned.wait(10) + return unresolved + + monkeypatch.setattr(w.run.ceiling, "close", finish) + try: + w.execute() + unresolved = w.events(model, "late-unresolved") + assert len(unresolved) == 1 + assert unresolved[0]["outcome"] == "unresolved-at-ledger-close" + assert unresolved[0]["counted_usd"] == 0.0 + assert w.events(model, "stop")[-1]["cause"] == "runner-error" + assert w.run.ceiling.closed + assert anchor_problems(w) == [] + before = (w.runs() / model / "run-log.jsonl").read_bytes() + finally: + release.set() + assert returned.wait(10) + assert w.run.ceiling.spent_usd == 0.0 + assert w.run.ceiling.unrecorded_usd == pytest.approx(0.20) + assert (w.runs() / model / "run-log.jsonl").read_bytes() == before + + +@pytest.mark.parametrize("during", ["host-close", "final-push"]) +def test_a_first_interrupt_during_cleanup_is_recorded_and_anchored( + world, monkeypatch, during +): + model = "gpt-6-sol" + w = world(models=(model,)) + w.run.no_output.update(w.draw_order) + if during == "host-close": + host_factory = w.run.host_factory + + def host(folder): + evidence = host_factory(folder) + close = evidence.close + + def finish(): + w.run.interrupt(2, None) + close() + + evidence.close = finish + return evidence + + w.run.host_factory = host + else: + anchor = w.run.anchor + signalled = [] + + def push(wait=False): + anchor(wait=wait) + if wait and not signalled: + signalled.append(True) + w.run.interrupt(2, None) + + monkeypatch.setattr(w.run, "anchor", push) + w.execute() + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + assert w.run._interrupted + assert anchor_problems(w) == [] + + +def test_a_replacement_log_failure_keeps_a_valid_final_history(world, monkeypatch): + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + w.run.no_output.update(w.draw_order[1:]) + lanes = w.run.lanes[model] + label = lanes.orders[sid][0] + w.provider.specs = { + (model, label, sid, 1): { + "status": "provider_error", + "error": "RateLimitError: a test", + } + } + write, failed = lanes.write_line, [] + + def broken(line, name="run-log.jsonl"): + if line.get("arm") == label and line.get("attempt") == 1 and not failed: + failed.append(line) + raise OSError("the replaced attempt's log write failed") + write(line, name) + + monkeypatch.setattr(lanes, "write_line", broken) + w.execute() + rec = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + assert rec["attempts"] == [] + assert w.provider.count[(model, label, sid)] == 1 + logged = [x for x in w.attempts(model) if x["arm"] == label] + assert len(logged) == 1 + assert all( + logged[0][key] == value for key, value in failed[0].items() if key != "sha256" + ) + assert logged[0]["sha256"] == cr.sha256_file(w.folder(model, label) / f"{sid}.json") + assert run_problems(w) == [] + + +def test_a_supervision_error_drains_active_lanes_before_final_anchor( + world, monkeypatch +): + model = "gpt-6-sol" + w = world(models=(model,)) + w.run.no_output.update(w.draw_order[1:]) + started, returned = threading.Event(), threading.Event() + converse, watch = w.run.converse, w.run.watch + failed = [] + + def request(*args): + started.set() + assert w.run._failed.wait(10) + rec = converse(*args) + returned.set() + return rec + + def broken_watch(anchor=True): + if started.is_set() and not failed: + failed.append(True) + raise OSError("supervision failed while a request was active") + watch(anchor=anchor) + + w.run.converse = request + monkeypatch.setattr(w.run, "watch", broken_watch) + with pytest.raises(OSError, match="supervision failed"): + w.execute() + assert returned.is_set() + assert len(w.attempts(model)) == 1 + assert w.lines(model)[-1]["cause"] == "runner-error" + assert w.run.ceiling.spent_usd == pytest.approx(1.0 + COST) + assert anchor_problems(w) == [] + assert run_problems(w) == [] + + +def test_a_final_push_log_failure_is_stopped_and_locally_anchored(world, monkeypatch): + model = "gpt-6-sol" + w = world(models=(model,)) + w.run.no_output.update(w.draw_order) + push = w.run._push_anchor + + def broken(head, anchor_time): + if w.run.host.end is not None: + with w.run._pushing: + w.run._push_error = OSError("the final push outcome cannot be written") + else: + push(head, anchor_time) + + monkeypatch.setattr(w.run, "_push_anchor", broken) + with pytest.raises(OSError, match="push outcome could not be recorded"): + w.execute() + assert w.events(model, "stop")[-1]["cause"] == "runner-error" + last = json.loads((w.runs() / "anchors.jsonl").read_text().splitlines()[-1]) + assert last["models"][model]["run-log.jsonl"]["lines"] == len(w.lines(model)) + assert last["host-log.jsonl"] == w.run.host.heads() From 00d267ae17e124f4c62d4f8169d6ce1402ced638 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 09:55:38 -0700 Subject: [PATCH 26/64] Confirmatory runner: round-3 fixes Answers review runner-r3 findings 2, 4, 5 and 6, and orders push times: - Push a synced, atomically installed snapshot of the anchors file. Give each push its own path, including after a thread is abandoned, and keep the remote record named anchors.jsonl. - Try the final push up to three times per set of heads, a minute apart, writing a new anchor for each try without changing the covered logs. - Bound pipe draining after process-group termination and every push join. Record a stuck thread once as an error and reject its late outcome. - Ignore signals before the last cleanup check, then log and anchor a first signal that arrived in the gap. Keep first and second signal handling active throughout retries. - Take anchor cadence and push intervals from the runner's ordered clock. Exercise overlapping pushes, retry exhaustion, escaped descendants, abandoned threads, cleanup signals and a backward wall-clock step. The report's record fields and outcome names are unchanged. Push snapshots are copies, not records; the draft should distinguish final retry exhaustion from the ongoing run's retry cadence, and retries after newly logged signals. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 9 +- scripts/confirmatory_run.py | 167 ++++++++++++++----- tests/test_confirmatory_anchor.py | 260 +++++++++++++++++++++++++++++- tests/test_confirmatory_run.py | 150 +++++++++++++++++ 4 files changed, 536 insertions(+), 50 deletions(-) diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py index a4103d8a..68b4f37f 100644 --- a/scripts/confirmatory_anchor.py +++ b/scripts/confirmatory_anchor.py @@ -3,9 +3,10 @@ Usage: python scripts/confirmatory_anchor.py --repo DIR ANCHORS -The runner calls this every 15 minutes, and once more at its end, with the -path of its `anchors.jsonl`: each line holds every log's line count and chain -head at a time. This copies the file into DIR, a clone checked out on a branch +The runner calls this every 15 minutes and at its end (with up to three final +tries), with a snapshot of its `anchors.jsonl`: each line holds every log's +line count and chain head at a time. The snapshot is a copy, not a record. +This copies it to `anchors.jsonl` in DIR, a clone checked out on a branch of its own, commits it and pushes the branch to its remote. The remote's record of each push is the anchor's time from outside the run: a log edited after an anchor was pushed no longer matches that anchor's head. @@ -30,7 +31,7 @@ def git(repo: Path, *args: str) -> str: def main(argv=None) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--repo", type=Path, required=True, help="the anchors clone") - parser.add_argument("anchors", type=Path, help="the run's anchors.jsonl") + parser.add_argument("anchors", type=Path, help="a snapshot of the run's anchors.jsonl") args = parser.parse_args(argv) repo = args.repo.expanduser() target = repo / "anchors.jsonl" diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index fd3d01da..eff73e76 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -70,7 +70,9 @@ model and the file's name; for the host log, of its name. Every 15 minutes, and at the end, ``anchors.jsonl`` gets a line with every log's line count and head and every manifest's sha256, itself chained, and the - design's ``anchor_command`` is run on it in the background; each run's + design's ``anchor_command`` is run on a synced, atomic copy in the + background. Each ``anchors..push-snapshot.jsonl`` is a copy, not a + record, and is never replaced by a later push's snapshot. Each push's result goes to ``anchor-pushes.jsonl``. ``confirmatory_anchor.py`` is that command: it commits the file to a clone and pushes it, so the remote holds each anchor's time from outside the run. @@ -168,6 +170,8 @@ ANCHOR_SECONDS = 900.0 # how often the logs' chain heads are anchored ANCHOR_RETRY = 60.0 # seconds to the next anchor after a failed push ANCHOR_PUSH_TIMEOUT = 300.0 # seconds; then the command's process group is killed +KILL_GRACE = 10.0 # seconds to drain pipes after killing the process group +FINAL_PUSH_TRIES = 3 # includes the first final push LATE_JOIN = 60.0 # seconds the run's end waits, in all, for abandoned attempts # What a stop event names as its cause. STOP_CAUSES = frozenset( @@ -1646,6 +1650,8 @@ def __init__( self._next_anchor = -math.inf self._push: threading.Thread | None = None self._push_error: OSError | None = None + self._push_pending: dict | None = None + self._last_push_outcome: str | None = None def say(self, text: str) -> None: with self._say_lock: @@ -1679,13 +1685,15 @@ def _check_push_error(self) -> None: def anchor_due(self) -> bool: self._check_push_error() with self._pushing: - return time.time() >= self._next_anchor + return clock() >= self._next_anchor def anchor(self, wait: bool = False, push: bool = True) -> None: """Every log's chain head to `anchors.jsonl`, itself a chain, then the - design's anchor command on that file, in the background (one at a + design's anchor command on a synced snapshot, in the background (one at a time; an anchor whose push would wait is written but not pushed until - the next). Each push's result goes to `anchor-pushes.jsonl`.""" + the next). A snapshot is a copy, not a record, and its path is unique + even if a push thread has to be abandoned. Each push's result goes to + `anchor-pushes.jsonl`.""" runs = Path(self.design["runs"]) line = { "time": clock(), @@ -1701,7 +1709,7 @@ def anchor(self, wait: bool = False, push: bool = True) -> None: handle.write(text + "\n") handle.flush() os.fsync(handle.fileno()) - self._anchor_head, self._anchored = line_sha(text), time.time() + self._anchor_head, self._anchored = line_sha(text), clock() with self._pushing: self._next_anchor = self._anchored + ANCHOR_SECONDS if not push: @@ -1709,8 +1717,21 @@ def anchor(self, wait: bool = False, push: bool = True) -> None: if self._push is not None and self._push.is_alive(): if not wait: return - self._push.join() - self._check_push_error() + self._join_push() + snapshot = runs / f"anchors.{self._anchor_head}.push-snapshot.jsonl" + tmp = snapshot.with_suffix(".tmp") + with tmp.open("wb") as handle: + handle.write((runs / "anchors.jsonl").read_bytes()) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, snapshot) + with self._pushing: + self._push_pending = { + "anchor": self._anchor_head, + "anchor_time": line["time"], + "started": clock(), + } + self._last_push_outcome = None self._push = threading.Thread( target=self._push_anchor, args=(self._anchor_head, line["time"]), @@ -1718,14 +1739,64 @@ def anchor(self, wait: bool = False, push: bool = True) -> None: ) self._push.start() if wait: - self._push.join() - self._check_push_error() + self._join_push() + + def _join_push(self) -> None: + """Bound the push thread's join and claim an error if it cannot end.""" + self._push.join(timeout=ANCHOR_PUSH_TIMEOUT + KILL_GRACE + 30) + if self._push.is_alive(): + with self._pushing: + pending = self._push_pending + if pending is not None: + self._record_push( + { + **pending, + "ended": clock(), + "outcome": "error", + "returncode": None, + "output": "the anchor push thread did not end after its time limit", + } + ) + self._check_push_error() + + def _record_push(self, line: dict) -> None: + """Claim one outcome under the push lock; a late worker cannot repeat it.""" + with self._pushing: + pending = self._push_pending + if pending is None or pending["anchor"] != line["anchor"]: + return + self._push_pending = None + self._last_push_outcome = line["outcome"] + if line["outcome"] != "success": + self._next_anchor = min( + self._next_anchor, line["ended"] + ANCHOR_RETRY + ) + try: + path = Path(self.design["runs"]) / "anchor-pushes.jsonl" + with path.open("a") as handle: + handle.write(json.dumps(line) + "\n") + handle.flush() + os.fsync(handle.fileno()) + except OSError as error: + # A daemon's exception must reach the supervisor or final join. + self._push_error = self._push_error or error + return + if line["outcome"] != "success": + self.say( + f"the anchor push failed ({line['outcome']}, " + f"{line['returncode']}): {line['output']}" + ) def _push_anchor(self, head: str, anchor_time: float) -> None: + """Push one immutable snapshot and record its bounded command's outcome.""" runs = Path(self.design["runs"]) command = [str(x) for x in self.design["anchor_command"]] - command.append(str(runs / "anchors.jsonl")) - started = time.time() + command.append(str(runs / f"anchors.{head}.push-snapshot.jsonl")) + with self._pushing: + pending = self._push_pending + if pending is None or pending["anchor"] != head: + return + started = pending["started"] = clock() try: child = subprocess.Popen( command, @@ -1744,15 +1815,23 @@ def _push_anchor(self, head: str, anchor_time: float) -> None: os.killpg(child.pid, signal.SIGKILL) except ProcessLookupError: pass - stdout, stderr = child.communicate() + try: + stdout, stderr = child.communicate(timeout=KILL_GRACE) + except subprocess.TimeoutExpired as error: + for pipe in (child.stdout, child.stderr): + if pipe is not None: + pipe.close() + stdout, stderr = error.output or b"", error.stderr or b"" + if isinstance(stdout, bytes): + stdout = stdout.decode(errors="replace") + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + stderr += "\npipes were held open after the kill" outcome = "timeout" code, text = child.returncode, (stdout + stderr)[-2000:] except (OSError, subprocess.SubprocessError) as error: code, outcome, text = None, "error", f"{type(error).__name__}: {error}" - ended = time.time() - if outcome != "success": - with self._pushing: - self._next_anchor = min(self._next_anchor, ended + ANCHOR_RETRY) + ended = clock() line = { "anchor": head, "anchor_time": anchor_time, @@ -1762,19 +1841,7 @@ def _push_anchor(self, head: str, anchor_time: float) -> None: "returncode": code, "output": text, } - try: - with (runs / "anchor-pushes.jsonl").open("a") as handle: - handle.write(json.dumps(line) + "\n") - handle.flush() - os.fsync(handle.fileno()) - except OSError as error: - # A daemon's exception cannot stop the run: hand this one back - # to the supervisor, or to the final caller after it joins us. - with self._pushing: - self._push_error = self._push_error or error - return - if outcome != "success": - self.say(f"the anchor push failed ({outcome}, {code}): {text}") + self._record_push(line) # The main loop's work. @@ -1948,7 +2015,9 @@ def finish(self) -> None: """One join deadline, then close pools and host before closing the ledger. Its final late costs, crossings and signals precede the last anchor. An unresolved request makes future spend unknowable: record - it explicitly, with a runner-error stop, rather than claim it settled.""" + it explicitly, with a runner-error stop, rather than claim it settled. + Retry the final push up to three times per set of heads, then ignore + signals and check once more for a first signal needing coverage.""" errors = [] def attempt(work): @@ -2017,21 +2086,35 @@ def attempt(work): indent=2, ) ) - if not attempt(lambda: self.anchor(wait=True)): - attempt(lambda: self.watch(anchor=False)) - # The error handler appended stops after that snapshot. Keep - # them covered locally even if pushing cannot be recorded. - try: - self.anchor(push=False) - except Exception as error: - errors.append(error) - self.say(traceback.format_exc()) - break - if self.signalled is None or self._interrupted: - break + for push_try in range(FINAL_PUSH_TRIES): + if push_try: + time.sleep(ANCHOR_RETRY) + if not attempt(lambda: self.anchor(wait=True)): + attempt(lambda: self.watch(anchor=False)) + # The error handler appended stops after that snapshot. Keep + # them covered locally even if pushing cannot be recorded. + try: + self.anchor(push=False) + except Exception as error: + errors.append(error) + self.say(traceback.format_exc()) + break + if self._last_push_outcome == "success": + break + if self.signalled is not None and not self._interrupted: + continue + self._ignore_signals() + if self.signalled is not None and not self._interrupted: + continue + break if errors: raise errors[0] + def _ignore_signals(self) -> None: + """Close signal delivery before the last check; the process is ending.""" + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + def interrupt(self, signum, frame) -> None: """The signal handler. It only notes the signal; the main loop stops every model's lanes. A second signal exits at once.""" diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index 8bc4c175..f6944c42 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -59,9 +59,11 @@ def remote_file(remote: Path) -> str: return out.stdout -def test_each_anchor_is_committed_and_pushed(clone, tmp_path): +@pytest.mark.parametrize("name", ["anchors.jsonl", "anchors.push-snapshot.jsonl"]) +def test_each_anchor_is_committed_and_pushed(clone, tmp_path, name): + """A snapshot's local name never changes the remote record's name.""" remote, repo = clone - anchors = tmp_path / "anchors.jsonl" + anchors = tmp_path / name lines = ['{"time": 1.0, "prev": "a"}', '{"time": 2.0, "prev": "b"}'] shas = [hashlib.sha256(line.encode()).hexdigest() for line in lines] anchors.write_text(lines[0] + "\n") @@ -70,6 +72,9 @@ def test_each_anchor_is_committed_and_pushed(clone, tmp_path): assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 assert remote_log(remote) == [f"anchor 2: {shas[1]}", f"anchor 1: {shas[0]}"] assert remote_file(remote) == anchors.read_text() + assert sorted(path.name for path in repo.iterdir() if path.name != ".git") == [ + "anchors.jsonl" + ] def test_an_unchanged_file_makes_no_commit(clone, tmp_path): @@ -112,6 +117,20 @@ def bare_run(folder, command): return run +def bare_finish_run(folder, command): + """An anchor writer with an empty, already closed run to finish.""" + + class Ledger: + def close(self, pending): + return [] + + run = bare_run(folder, command) + run.ceiling, run._failed = Ledger(), threading.Event() + run.signalled, run._interrupted = None, False + run.watch = lambda anchor=True: None + return run + + def test_failed_pushes_retry_and_keep_every_outcome(tmp_path, monkeypatch): """A failed remote never looks anchored; retry follows the failure's end, then a successful push restores the ordinary cadence.""" @@ -184,7 +203,8 @@ def communicate(self, timeout): def fail_push_sync(fd): calls.append(fd) - if len(calls) == 2: # The anchor itself was synced; its push outcome was not. + if (tmp_path / "anchor-pushes.jsonl").exists(): + # The anchor and its copy synced; only the outcome's sync fails. raise OSError("push-log fsync failed") return fsync(fd) @@ -200,7 +220,92 @@ def fail_push_sync(fd): with pytest.raises(OSError, match="push-log fsync failed"): run.anchor_due() assert not run._push.is_alive() - assert len(calls) == 2 + assert len(calls) == 3 + + +def test_a_running_push_reads_a_snapshot_while_the_final_anchor_is_appended( + tmp_path, monkeypatch +): + """A push released after the final append still sees its own whole prefix.""" + reading, release, joining = threading.Event(), threading.Event(), threading.Event() + calls, copies, errors = [], [], [] + + class Push: + returncode = 0 + + def __init__(self, command, **kwargs): + self.source = Path(command[-1]) + calls.append(self.source) + self.first = len(calls) == 1 + + def communicate(self, timeout): + if self.first: + reading.set() + assert release.wait(15.0), "the first push was never released" + copies.append(self.source.read_bytes()) + return "pushed", "" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + run = bare_run(tmp_path, ["push-anchors"]) + run.anchor() + assert reading.wait(15.0), "the first push did not start" + first, join = run._push, run._push.join + + def joining_push(timeout=None): + joining.set() + return join(timeout=timeout) + + def final_anchor(): + try: + run.anchor(wait=True) + except BaseException as error: + errors.append(error) + + monkeypatch.setattr(first, "join", joining_push) + final = threading.Thread(target=final_anchor, daemon=True) + final.start() + try: + assert joining.wait(15.0), "the final anchor did not join the blocked push" + live = (tmp_path / "anchors.jsonl").read_bytes() + assert len(live.splitlines()) == 2 + assert calls[0] != tmp_path / "anchors.jsonl" + assert calls[0].name.endswith("push-snapshot.jsonl") + assert len(calls[0].read_bytes().splitlines()) == 1 + finally: + release.set() + final.join(timeout=15.0) + assert not final.is_alive() and not errors + live = (tmp_path / "anchors.jsonl").read_bytes() + assert len(copies) == 2 and copies[-1] == live + assert all(copy.endswith(b"\n") and live.startswith(copy) for copy in copies) + assert len(set(calls)) == 2 + + +def test_push_times_use_the_ordered_clock_after_a_wall_clock_step_back( + tmp_path, monkeypatch +): + """The anchor, its cadence and its push never precede that anchor's time.""" + base = time.time() + 1000.0 + readings = iter([base, base - 0.0001]) + monkeypatch.setattr(cr, "_clock_last", 0.0) + monkeypatch.setattr(cr.time, "time", lambda: next(readings, base - 0.0001)) + + class Push: + returncode = 0 + + def __init__(self, *args, **kwargs): + pass + + def communicate(self, timeout): + return "pushed", "" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + run = bare_run(tmp_path, ["push-anchors"]) + run.anchor(wait=True) + anchor = json.loads((tmp_path / "anchors.jsonl").read_text()) + push = json.loads((tmp_path / "anchor-pushes.jsonl").read_text()) + assert anchor["time"] <= run._anchored <= push["started"] <= push["ended"] + assert push["anchor_time"] == anchor["time"] def test_a_push_timeout_kills_its_children_and_grandchildren(tmp_path, monkeypatch): @@ -273,3 +378,150 @@ def push(): assert push["started"] <= push["ended"] for level in (1, 2): assert int((tmp_path / f"pid-{level}").read_text()) > 0 + + +def test_finish_abandons_pipes_held_by_a_descendant_outside_the_push_group( + tmp_path, monkeypatch +): + """An escaped grandchild cannot hold finish open after the kill's grace.""" + script = tmp_path / "escaped-push.py" + script.write_text( + "import os, subprocess, sys, threading\n" + "from pathlib import Path\n" + "folder = Path(sys.argv[1])\n" + "if len(sys.argv) > 2 and sys.argv[2] == 'escaped':\n" + " (folder / 'escaped-pid').write_text(str(os.getpid()))\n" + "else:\n" + " subprocess.Popen(\n" + " [sys.executable, __file__, str(folder), 'escaped'],\n" + " start_new_session=True)\n" + "threading.Event().wait()\n" + ) + run = bare_finish_run(tmp_path, [sys.executable, str(script), str(tmp_path)]) + real_popen = subprocess.Popen + ready, done = threading.Event(), threading.Event() + children, errors = [], [] + + def popen(*args, **kwargs): + child = real_popen(*args, **kwargs) + children.append(child) + communicate = child.communicate + + def ready_communicate(timeout=None): + if not ready.is_set(): + deadline = time.monotonic() + 15.0 + pid = tmp_path / "escaped-pid" + while not pid.exists() or not pid.read_text().strip(): + assert time.monotonic() < deadline, "the grandchild did not start" + done.wait(0.01) + ready.set() + return communicate(timeout=timeout) + + child.communicate = ready_communicate + return child + + def finish(): + try: + run.finish() + except BaseException as error: + errors.append(error) + finally: + done.set() + + monkeypatch.setattr(cr.subprocess, "Popen", popen) + monkeypatch.setattr(cr.signal, "signal", lambda *args: None) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 0.05) + monkeypatch.setattr(cr, "KILL_GRACE", 0.05) + monkeypatch.setattr(cr, "FINAL_PUSH_TRIES", 1) + thread = threading.Thread(target=finish, daemon=True) + thread.start() + try: + assert ready.wait(15.0), "the push never launched its grandchild" + started = time.monotonic() + finished = done.wait(10.0) + elapsed = time.monotonic() - started + finally: + pid = tmp_path / "escaped-pid" + if pid.exists() and pid.read_text().strip(): + try: + os.kill(int(pid.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass + for child in children: + if child.poll() is None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + thread.join(timeout=15.0) + assert finished and elapsed < 10.0, "finish waited for the escaped child's pipe" + assert not errors and not thread.is_alive() + rows = [ + json.loads(line) + for line in (tmp_path / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert len(rows) == 1 and rows[0]["outcome"] == "timeout" + assert "pipes" in rows[0]["output"] and "held open after the kill" in rows[0]["output"] + + +def test_finish_records_a_stuck_push_once_even_if_its_thread_later_ends( + tmp_path, monkeypatch +): + """A bounded join retires the worker's right to append a second outcome.""" + started, release = threading.Event(), threading.Event() + calls, waits, copies = [], [], {} + + class Push: + returncode = 0 + + def __init__(self, command, **kwargs): + calls.append(command) + self.first = len(calls) == 1 + self.source = Path(command[-1]) + self.returncode = 1 if self.first else 0 + + def communicate(self, timeout): + if self.first: + started.set() + assert release.wait(15.0), "the abandoned worker was never released" + copies[self.source] = self.source.read_bytes() + return "pushed", "" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + monkeypatch.setattr(cr.signal, "signal", lambda *args: None) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 0.05) + monkeypatch.setattr(cr, "KILL_GRACE", 0.05) + run = bare_finish_run(tmp_path, ["push-anchors"]) + run.anchor() + assert started.wait(15.0), "the periodic push did not start" + worker, join = run._push, run._push.join + + def short_join(timeout=None): + waits.append(timeout) + return join(timeout=0.02) + + monkeypatch.setattr(worker, "join", short_join) + try: + began = time.monotonic() + run.finish() + assert time.monotonic() - began < 10.0 + path = tmp_path / "anchor-pushes.jsonl" + before = path.read_bytes() + rows = [json.loads(line) for line in before.splitlines()] + assert waits == [pytest.approx(cr.ANCHOR_PUSH_TIMEOUT + cr.KILL_GRACE + 30)] + assert len(rows) == 2 + assert rows[0]["outcome"] == "error" and rows[0]["returncode"] is None + assert "push thread did not end" in rows[0]["output"] + assert rows[1]["outcome"] == "success" + next_anchor = run._next_anchor + finally: + release.set() + join(timeout=15.0) + assert not worker.is_alive() + assert path.read_bytes() == before + assert len({row["anchor"] for row in rows}) == len(rows) + assert run._last_push_outcome == "success" and run._next_anchor == next_anchor + live = (tmp_path / "anchors.jsonl").read_bytes() + assert len(copies[Path(calls[0][-1])].splitlines()) == 1 + assert len(copies[Path(calls[1][-1])].splitlines()) == 2 + assert all(copy.endswith(b"\n") and live.startswith(copy) for copy in copies.values()) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 1be358f9..3f3b2b71 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -16,6 +16,7 @@ import json import math import os +import signal import subprocess import sys import threading @@ -360,6 +361,10 @@ def fast(monkeypatch): monkeypatch.setattr(cr, "SAMPLER_SILENCE", 1.0) monkeypatch.setattr(cr, "READ_WAIT", 0.01) monkeypatch.setattr(cr, "ANCHOR_SECONDS", 0.3) + handlers = {s: signal.getsignal(s) for s in (signal.SIGINT, signal.SIGTERM)} + yield + for signum, handler in handlers.items(): + signal.signal(signum, handler) def test_the_host_log_samples_and_closes(tmp_path, fast): @@ -2762,6 +2767,151 @@ def broken_watch(anchor=True): assert run_problems(w) == [] +def final_push_world(world, monkeypatch, fail_all=False, models=("gpt-6-luna",)): + """A run with no periodic pushes, whose first final push fails.""" + w = world(models=models) + w.run.no_output.update(w.draw_order) + monkeypatch.setattr(w.run, "anchor_due", lambda: False) + monkeypatch.setattr(cr, "ANCHOR_RETRY", 0.013) + script = w.tmp / "retrying-anchor.py" + script.write_text( + "import shutil, sys\n" + "from pathlib import Path\n" + "folder = Path(sys.argv[1])\n" + "counter = folder / 'push-count'\n" + "n = int(counter.read_text()) + 1 if counter.exists() else 1\n" + "counter.write_text(str(n))\n" + "shutil.copyfile(sys.argv[3], folder / f'pushed-{n}.jsonl')\n" + "raise SystemExit(1 if n == 1 or sys.argv[2] == '1' else 0)\n" + ) + w.design["anchor_command"] = [ + sys.executable, + str(script), + str(w.tmp), + str(int(fail_all)), + ] + return w + + +@pytest.mark.parametrize("fail_all", [False, True], ids=["then-success", "all-fail"]) +def test_the_final_push_is_retried_until_success_or_three_tries( + world, monkeypatch, fail_all +): + """Final retries stop at the first success, or after three failures.""" + w = final_push_world(world, monkeypatch, fail_all=fail_all) + w.execute() + anchors = [ + json.loads(x) for x in (w.runs() / "anchors.jsonl").read_text().splitlines() + ] + pushes = [ + json.loads(x) + for x in (w.runs() / "anchor-pushes.jsonl").read_text().splitlines() + ] + expected = 3 if fail_all else 2 + assert cr.FINAL_PUSH_TRIES == 3 + assert len(pushes) == len(anchors) == expected + assert [row["outcome"] for row in pushes] == ( + ["failed"] * 3 if fail_all else ["failed", "success"] + ) + final = (w.runs() / "anchors.jsonl").read_bytes() + assert (w.tmp / f"pushed-{expected}.jsonl").read_bytes() == final + assert pushes[-1]["anchor"] == cr.line_sha(final.decode().splitlines()[-1]) + assert pushes[-1]["anchor_time"] == anchors[-1]["time"] + assert all(a["models"] == anchors[0]["models"] for a in anchors) + assert all(a["host-log.jsonl"] == anchors[0]["host-log.jsonl"] for a in anchors) + assert all(a["time"] < b["time"] for a, b in zip(anchors, anchors[1:])) + + +def test_a_first_signal_during_final_push_retries_is_logged_and_anchored( + world, monkeypatch +): + """A first signal during the retry wait is covered by the last push.""" + w = final_push_world(world, monkeypatch, models=("gpt-6-luna", "gpt-6-sol")) + sleep, delivered = time.sleep, [] + + def during_retry(seconds): + if seconds == cr.ANCHOR_RETRY and not delivered: + delivered.append(True) + w.run.interrupt(signal.SIGTERM, None) + sleep(seconds) + + monkeypatch.setattr(time, "sleep", during_retry) + w.execute() + assert delivered and w.run._interrupted + anchors = [ + json.loads(x) for x in (w.runs() / "anchors.jsonl").read_text().splitlines() + ] + pushes = [ + json.loads(x) + for x in (w.runs() / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert [row["outcome"] for row in pushes] == ["failed", "success", "success"] + for model in w.models: + assert [row["cause"] for row in w.events(model, "stop")] == ["interrupt"] + assert anchors[-1]["models"][model]["run-log.jsonl"] == ( + w.run.lanes[model].heads()["run-log.jsonl"] + ) + assert anchors[-2]["models"][model]["run-log.jsonl"]["lines"] == ( + len(w.lines(model)) - 1 + ) + assert (w.tmp / "pushed-3.jsonl").read_bytes() == ( + w.runs() / "anchors.jsonl" + ).read_bytes() + + +def test_a_second_signal_during_final_push_retries_exits_at_once(world, monkeypatch): + """The retry wait leaves the second signal's immediate exit intact.""" + w = final_push_world(world, monkeypatch) + sleep, delivered = time.sleep, [] + + def exit_now(code): + raise SystemExit(code) + + def during_retry(seconds): + if seconds == cr.ANCHOR_RETRY and not delivered: + delivered.append(True) + w.run.interrupt(signal.SIGINT, None) + w.run.interrupt(signal.SIGTERM, None) + sleep(seconds) + + monkeypatch.setattr(os, "_exit", exit_now) + monkeypatch.setattr(time, "sleep", during_retry) + with pytest.raises(SystemExit) as stopped: + w.execute() + assert stopped.value.code == 130 and delivered + assert len((w.runs() / "anchor-pushes.jsonl").read_text().splitlines()) == 1 + assert all(not w.events(model, "stop") for model in w.models) + + +def test_a_signal_at_the_final_ignore_hook_is_logged_and_anchored(world, monkeypatch): + """A signal between the last ordinary check and SIG_IGN is not lost.""" + w = world(models=("gpt-6-luna", "gpt-6-sol")) + w.run.no_output.update(w.draw_order) + monkeypatch.setattr(w.run, "anchor_due", lambda: False) + ignore = w.run._ignore_signals + ignored, delivered = [], [] + monkeypatch.setattr( + signal, "signal", lambda signum, handler: ignored.append((signum, handler)) + ) + + def at_end(): + if not delivered: + delivered.append(True) + w.run.interrupt(signal.SIGTERM, None) + ignore() + + monkeypatch.setattr(w.run, "_ignore_signals", at_end) + w.execute() + assert delivered and w.run._interrupted + assert (signal.SIGINT, signal.SIG_IGN) in ignored + assert (signal.SIGTERM, signal.SIG_IGN) in ignored + for model in w.models: + stops = w.events(model, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "interrupt" + assert stops[0]["reason"] == f"operator stop (signal {signal.SIGTERM})" + assert anchor_problems(w) == [] + + def test_a_final_push_log_failure_is_stopped_and_locally_anchored(world, monkeypatch): model = "gpt-6-sol" w = world(models=(model,)) From 019f8424449e6385567b5a753e7726fe323be7e1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 13:31:42 -0700 Subject: [PATCH 27/64] Confirmatory runner: round-4 fixes Answers review runner-r4 finding 1 and tests the remaining clock and retry spacing requirements: - Hold an exclusive, non-blocking lock in the anchor clone's .git directory through the push. Refuse busy helpers and snapshots that do not extend the clone's anchors byte for byte; allow equality and never force-push. Leave a stale index.lock to fail normally. - Keep worker retirement and the next push, but say the retired command may still be running. A helper holding the lock makes later pushes fail as busy; final anchors left local are a deviation. - Exercise older helpers after newer pushes, shared-clone retirement, lock contention, interior edits, equality and stale git locks against a temporary clone and local bare remote. - Test anchor_due across a backward wall-clock step and measure each final retry from the preceding try's end, with a controlled clock. - Clarify that final retries wait a minute after a failed try ends. First and second signal handling stays active only until cleanup ignores signals before its last check; signals during re-anchoring after that check are ignored. State that write/fsync under _pushing can delay supervision and retirement beyond the thread join bound. The report's record fields and outcome names are unchanged. Retired-worker output now identifies the surviving-command risk; helper refusals are failed pushes. The clone's persistent lock file is auxiliary, outside run records. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 42 +++++-- scripts/confirmatory_run.py | 43 +++++--- tests/test_confirmatory_anchor.py | 177 ++++++++++++++++++++++++++++++ tests/test_confirmatory_run.py | 54 +++++++++ 4 files changed, 293 insertions(+), 23 deletions(-) diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py index 68b4f37f..838a4bba 100644 --- a/scripts/confirmatory_anchor.py +++ b/scripts/confirmatory_anchor.py @@ -10,39 +10,63 @@ of its own, commits it and pushes the branch to its remote. The remote's record of each push is the anchor's time from outside the run: a log edited after an anchor was pushed no longer matches that anchor's head. + +An exclusive non-blocking lock in the clone's `.git` directory covers every +read, copy, commit and push. A busy clone fails without changing its records. +Under the lock, the clone's current file must be a byte prefix of the snapshot: +a delayed older command cannot replace newer evidence. Git's own index.lock +is left alone; a git failure is a failed push. """ from __future__ import annotations import argparse +import fcntl import hashlib -import shutil import subprocess import sys from pathlib import Path def git(repo: Path, *args: str) -> str: + """Run git in the anchor clone, retaining failures for the caller.""" return subprocess.run( ["git", "-C", str(repo), *args], check=True, capture_output=True, text=True ).stdout def main(argv=None) -> int: + """Push a whole snapshot only while exclusively extending the clone's file.""" parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--repo", type=Path, required=True, help="the anchors clone") parser.add_argument("anchors", type=Path, help="a snapshot of the run's anchors.jsonl") args = parser.parse_args(argv) repo = args.repo.expanduser() target = repo / "anchors.jsonl" - shutil.copyfile(args.anchors, target) - lines = target.read_text().splitlines() - git(repo, "add", "anchors.jsonl") - if git(repo, "status", "--porcelain", "--", "anchors.jsonl"): - head = hashlib.sha256(lines[-1].encode()).hexdigest() if lines else "none" - git(repo, "commit", "-q", "-m", f"anchor {len(lines)}: {head}") - git(repo, "push", "-q", "origin", "HEAD") - print(f"pushed {len(lines)} anchors") + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print("anchor clone busy: another helper holds its lock", file=sys.stderr) + return 1 + snapshot = args.anchors.read_bytes() + current = target.read_bytes() if target.exists() else b"" + if not snapshot.startswith(current): + print( + "refusing anchor snapshot: the clone's anchors.jsonl is not a byte " + "prefix of the snapshot", + file=sys.stderr, + ) + return 1 + if snapshot != current or not target.exists(): + target.write_bytes(snapshot) + lines = snapshot.decode().splitlines() + git(repo, "add", "anchors.jsonl") + if git(repo, "status", "--porcelain", "--", "anchors.jsonl"): + head = hashlib.sha256(lines[-1].encode()).hexdigest() if lines else "none" + git(repo, "commit", "-q", "-m", f"anchor {len(lines)}: {head}") + git(repo, "push", "-q", "origin", "HEAD") + print(f"pushed {len(lines)} anchors") return 0 diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index eff73e76..1ee029c4 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -74,8 +74,13 @@ background. Each ``anchors..push-snapshot.jsonl`` is a copy, not a record, and is never replaced by a later push's snapshot. Each push's result goes to ``anchor-pushes.jsonl``. ``confirmatory_anchor.py`` is that - command: it commits the file to a clone and pushes it, so the remote holds - each anchor's time from outside the run. + command: under an exclusive, non-blocking clone lock, it accepts only a + snapshot extending the clone's anchors, commits it and pushes it, so the + remote holds each anchor's time from outside the run. Retiring a worker + does not ensure its command has ended. If an old helper still holds the + lock, later pushes fail as busy; a final segment left local is a deviation. + Final pushes get up to three tries per set of heads, waiting a minute + after each failed try ends before starting the next. The records' format is the confirmatory report's (its module docstring). The report recomputes every class, action, order and admission decision from the @@ -90,7 +95,9 @@ engine workers per model. An interrupt (SIGINT or SIGTERM) only notes the signal; the main loop then stops every model's lanes with cause ``interrupt`` and lets the conversations under way end. A second interrupt -exits at once. +exits at once until cleanup ignores signals before its final check. Signals +during any re-anchoring after that check are ignored, including a second +interrupt. """ from __future__ import annotations @@ -1689,11 +1696,11 @@ def anchor_due(self) -> bool: def anchor(self, wait: bool = False, push: bool = True) -> None: """Every log's chain head to `anchors.jsonl`, itself a chain, then the - design's anchor command on a synced snapshot, in the background (one at a - time; an anchor whose push would wait is written but not pushed until - the next). A snapshot is a copy, not a record, and its path is unique - even if a push thread has to be abandoned. Each push's result goes to - `anchor-pushes.jsonl`.""" + design's anchor command on a synced snapshot, in the background (one + current worker at a time; an anchor whose push would wait is written + but not pushed until the next). A snapshot is a copy, not a record, + and its path is unique even if a push thread has to be abandoned. + Each push's result goes to `anchor-pushes.jsonl`.""" runs = Path(self.design["runs"]) line = { "time": clock(), @@ -1742,7 +1749,9 @@ def anchor(self, wait: bool = False, push: bool = True) -> None: self._join_push() def _join_push(self) -> None: - """Bound the push thread's join and claim an error if it cannot end.""" + """Bound the thread join and retire its outcome if it cannot end. + Its command may survive; the helper's lock and prefix check protect + the clone. Waiting for _pushing still assumes a responsive filesystem.""" self._push.join(timeout=ANCHOR_PUSH_TIMEOUT + KILL_GRACE + 30) if self._push.is_alive(): with self._pushing: @@ -1754,13 +1763,16 @@ def _join_push(self) -> None: "ended": clock(), "outcome": "error", "returncode": None, - "output": "the anchor push thread did not end after its time limit", + "output": "the anchor push thread did not end after its time limit; " + "its command may still be running", } ) self._check_push_error() def _record_push(self, line: dict) -> None: - """Claim one outcome under the push lock; a late worker cannot repeat it.""" + """Claim one outcome under the push lock; a late worker cannot repeat it. + The lock covers writing and fsync, so a wedged filesystem can delay + supervision and retirement beyond the thread join's bound.""" with self._pushing: pending = self._push_pending if pending is None or pending["anchor"] != line["anchor"]: @@ -2016,8 +2028,10 @@ def finish(self) -> None: ledger. Its final late costs, crossings and signals precede the last anchor. An unresolved request makes future spend unknowable: record it explicitly, with a runner-error stop, rather than claim it settled. - Retry the final push up to three times per set of heads, then ignore - signals and check once more for a first signal needing coverage.""" + Retry the final push up to three times per set of heads, waiting + ANCHOR_RETRY after each failed try ends. Then ignore signals and check + once more for a first signal needing coverage. Signals during the + resulting re-anchoring are ignored, including a second interrupt.""" errors = [] def attempt(work): @@ -2117,7 +2131,8 @@ def _ignore_signals(self) -> None: def interrupt(self, signum, frame) -> None: """The signal handler. It only notes the signal; the main loop stops - every model's lanes. A second signal exits at once.""" + every model's lanes. A second signal exits at once while this handler + is installed; cleanup ignores signals before its final check.""" self.interrupts += 1 if self.interrupts > 1: os._exit(130) diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index f6944c42..5fa338ff 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -1,10 +1,12 @@ """The anchor command: each call commits the run's anchors and pushes them.""" +import fcntl import hashlib import io import json import math import os +import select import signal import subprocess import sys @@ -59,6 +61,15 @@ def remote_file(remote: Path) -> str: return out.stdout +def anchor_command(repo: Path, anchors: Path): + """Run the real helper and retain its failure message and exit status.""" + return subprocess.run( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + capture_output=True, + text=True, + ) + + @pytest.mark.parametrize("name", ["anchors.jsonl", "anchors.push-snapshot.jsonl"]) def test_each_anchor_is_committed_and_pushed(clone, tmp_path, name): """A snapshot's local name never changes the remote record's name.""" @@ -78,12 +89,84 @@ def test_each_anchor_is_committed_and_pushed(clone, tmp_path, name): def test_an_unchanged_file_makes_no_commit(clone, tmp_path): + """An equal snapshot succeeds without changing the file or remote history.""" remote, repo = clone anchors = tmp_path / "anchors.jsonl" anchors.write_text('{"time": 1.0}\n') for _ in range(2): assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 assert len(remote_log(remote)) == 1 + assert remote_file(remote) == anchors.read_text() + + +def test_an_older_snapshot_cannot_replace_a_newer_remote_file(clone, tmp_path): + """A delayed helper cannot commit or push a prefix after the newer snapshot.""" + remote, repo = clone + older, newer = tmp_path / "older.jsonl", tmp_path / "newer.jsonl" + older.write_bytes(b'{"time": 1.0}\n') + newer.write_bytes(older.read_bytes() + b'{"time": 2.0}\n') + assert anchor_command(repo, newer).returncode == 0 + history, index = remote_log(remote), (repo / ".git" / "index").read_bytes() + out = anchor_command(repo, older) + assert out.returncode != 0 and "not a byte prefix" in out.stderr + assert (repo / "anchors.jsonl").read_bytes() == newer.read_bytes() + assert (repo / ".git" / "index").read_bytes() == index + assert remote_file(remote).encode() == newer.read_bytes() + assert remote_log(remote) == history + + +def test_a_busy_clone_is_refused_without_changing_its_records(clone, tmp_path): + """A held flock prevents target, index, commit and remote writes.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + history, index = remote_log(remote), (repo / ".git" / "index").read_bytes() + anchors.write_bytes(first + b'{"time": 2.0}\n') + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + out = anchor_command(repo, anchors) + assert out.returncode != 0 and "busy" in out.stderr + assert (repo / "anchors.jsonl").read_bytes() == first + assert (repo / ".git" / "index").read_bytes() == index + assert remote_file(remote).encode() == first + assert remote_log(remote) == history + + +def test_a_changed_middle_line_is_refused_without_writing(clone, tmp_path): + """A longer snapshot still fails when it rewrites an existing line.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n{"time": 2.0}\n{"time": 3.0}\n' + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + history, index = remote_log(remote), (repo / ".git" / "index").read_bytes() + anchors.write_bytes(first.replace(b"2.0", b"9.0") + b'{"time": 4.0}\n') + out = anchor_command(repo, anchors) + assert out.returncode != 0 and "not a byte prefix" in out.stderr + assert (repo / "anchors.jsonl").read_bytes() == first + assert (repo / ".git" / "index").read_bytes() == index + assert remote_file(remote).encode() == first + assert remote_log(remote) == history + + +def test_a_stale_git_index_lock_is_left_alone(clone, tmp_path): + """A failed git command never removes another command's index lock.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + history = remote_log(remote) + index_lock = repo / ".git" / "index.lock" + index_lock.write_bytes(b"stale lock kept for its owner\n") + anchors.write_bytes(first + b'{"time": 2.0}\n') + out = anchor_command(repo, anchors) + assert out.returncode != 0 and "CalledProcessError" in out.stderr + assert index_lock.read_bytes() == b"stale lock kept for its owner\n" + assert remote_file(remote).encode() == first + assert remote_log(remote) == history def test_a_failed_push_exits_nonzero(clone, tmp_path): @@ -512,6 +595,7 @@ def short_join(timeout=None): assert len(rows) == 2 assert rows[0]["outcome"] == "error" and rows[0]["returncode"] is None assert "push thread did not end" in rows[0]["output"] + assert "command may still be running" in rows[0]["output"] assert rows[1]["outcome"] == "success" next_anchor = run._next_anchor finally: @@ -525,3 +609,96 @@ def short_join(timeout=None): assert len(copies[Path(calls[0][-1])].splitlines()) == 1 assert len(copies[Path(calls[1][-1])].splitlines()) == 2 assert all(copy.endswith(b"\n") and live.startswith(copy) for copy in copies.values()) + + +def test_a_retired_command_cannot_shrink_the_remote_after_the_final_push( + clone, tmp_path, monkeypatch +): + """A live old command released after finish cannot replace newer evidence.""" + remote, repo = clone + folder = tmp_path / "run" + folder.mkdir() + delayed = tmp_path / "delayed-helper.py" + delayed.write_text( + "import os, sys\n" + "ready, release = map(int, sys.argv[1:3])\n" + "os.write(ready, b'1')\n" + "os.close(ready)\n" + "os.read(release, 1)\n" + "os.close(release)\n" + "os.execv(sys.executable, [sys.executable, *sys.argv[3:]])\n" + ) + ready_read, ready_write = os.pipe() + release_read, release_write = os.pipe() + real_popen, children, waits = subprocess.Popen, [], [] + run = bare_finish_run( + folder, [sys.executable, str(SCRIPT), "--repo", str(repo)] + ) + + def popen(command, **kwargs): + """Delay only the first real helper, before it can acquire the clone lock.""" + if len(command) > 1 and command[1] == str(SCRIPT): + if not children: + command = [ + sys.executable, + str(delayed), + str(ready_write), + str(release_read), + *command[1:], + ] + kwargs["pass_fds"] = (ready_write, release_read) + child = real_popen(command, **kwargs) + children.append(child) + return child + return real_popen(command, **kwargs) + + def short_join(timeout=None): + """Exercise retirement without waiting for the production join bound.""" + waits.append(timeout) + return join(timeout=0.02) + + monkeypatch.setattr(cr.subprocess, "Popen", popen) + monkeypatch.setattr(cr.signal, "signal", lambda *args: None) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 30.0) + worker = join = None + try: + run.anchor() + worker, join = run._push, run._push.join + assert select.select([ready_read], [], [], 15.0)[0], ( + "the old command did not start" + ) + assert os.read(ready_read, 1) == b"1" + monkeypatch.setattr(worker, "join", short_join) + run.finish() + assert children[0].poll() is None + newer = (folder / "anchors.jsonl").read_bytes() + assert len(newer.splitlines()) == 2 + assert remote_file(remote).encode() == newer + history = remote_log(remote) + path = folder / "anchor-pushes.jsonl" + before = path.read_bytes() + rows = [json.loads(line) for line in before.splitlines()] + assert waits == [pytest.approx(cr.ANCHOR_PUSH_TIMEOUT + cr.KILL_GRACE + 30)] + assert [row["outcome"] for row in rows] == ["error", "success"] + assert rows[0]["returncode"] is None + assert "command may still be running" in rows[0]["output"] + finally: + try: + os.write(release_write, b"1") + except BrokenPipeError: + pass + if join is not None: + join(timeout=45.0) + for child in children: + if child.poll() is None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + for fd in (ready_read, ready_write, release_read, release_write): + os.close(fd) + assert not worker.is_alive() and len(children) == 2 + assert children[0].returncode == 1 + assert path.read_bytes() == before + assert remote_file(remote).encode() == newer + assert remote_log(remote) == history diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 3f3b2b71..ed103b04 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -545,6 +545,20 @@ def test_the_clock_never_goes_back(monkeypatch): ] +def test_a_backward_wall_clock_does_not_delay_a_due_anchor(monkeypatch): + """Once the ordered clock reaches the due time, a wall-clock step cannot + make the anchor not due again.""" + run = cr.Run.__new__(cr.Run) + run._pushing = threading.Lock() + run._push_error = None + base = time.time() + 1000.0 + run._next_anchor = base + monkeypatch.setattr(cr, "_clock_last", base - 1.0) + readings = iter([base - 1.0, base + 1.0, base - 5.0]) + monkeypatch.setattr(time, "time", lambda: next(readings)) + assert [run.anchor_due() for _ in range(3)] == [False, True, True] + + def test_a_wall_clock_that_steps_back_keeps_the_run_procedure(world, monkeypatch): """Every reading of the wall clock up to 0.2 ms early (steps back of up to 0.1 ms seen on this host): the run's times still order its attempts, @@ -2822,6 +2836,46 @@ def test_the_final_push_is_retried_until_success_or_three_tries( assert all(a["time"] < b["time"] for a, b in zip(anchors, anchors[1:])) +def test_each_final_push_retry_waits_from_the_previous_try_s_end(world, monkeypatch): + """Each final retry starts at least ANCHOR_RETRY after the prior try ends, + even when the failed command itself takes time.""" + from types import SimpleNamespace + + w = world(models=("gpt-6-luna",)) + now = [1000.0] + durations = iter([7.0, 11.0, 5.0]) + monkeypatch.setattr(cr, "clock", lambda: now[0]) + + def sleep(seconds): + """Advance the controlled clock by the requested wait.""" + now[0] += seconds + + def failed_command(*args, **kwargs): + """A failed anchor command with a controlled, nonzero duration.""" + + def communicate(timeout): + """Finish the command after its planted duration.""" + now[0] += next(durations) + return "", "controlled push failure" + + return SimpleNamespace(returncode=1, communicate=communicate) + + monkeypatch.setattr(time, "sleep", sleep) + monkeypatch.setattr(subprocess, "Popen", failed_command) + w.run.finish() + pushes = [ + json.loads(x) + for x in (w.runs() / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert len(pushes) == cr.FINAL_PUSH_TRIES == 3 + assert [p["outcome"] for p in pushes] == ["failed"] * 3 + assert [p["ended"] - p["started"] for p in pushes] == [7.0, 11.0, 5.0] + assert all( + b["started"] >= a["ended"] + cr.ANCHOR_RETRY + for a, b in zip(pushes, pushes[1:]) + ) + + def test_a_first_signal_during_final_push_retries_is_logged_and_anchored( world, monkeypatch ): From 9b30a778a2e85d5b6bce983ef2b07342b97df840 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 23:21:24 -0700 Subject: [PATCH 28/64] Confirmatory runner: round-5 fixes Answers review runner-r5 finding 1 and fills the retry and conversation coverage gaps: - Pass the clone's flock descriptor to every Git command. A surviving Git writer, hook or remote helper retains the lock after helper-only death; later helpers still refuse busy clones and non-extending snapshots. Disable fsmonitor startup, whose daemon closes inherited descriptors. Keep ordinary, non-force pushes and leave Git's own index.lock alone. - Kill the real helper after Git has entered pre-commit, pre-push and local pre-receive hooks. Check busy refusals, inherited hook descriptors and unchanged clone file, index, HEAD and remote, then newer pushes and every remote commit's byte-prefix extension. Test that configured fsmonitor is never started. - Pin the registered 900-second anchor interval and 60-second retry wait independently of production constants, citing draft line 218. - Compare fake OpenAI Responses and Gemini JSON conversations against the unchanged treatment harness, including complete requests and calculator inputs. Exercise both through the runner's lanes. Use the existing Gemini 3.1 JSON profile; both confirmatory Geminis retain their tool contracts. - Describe surviving Git lock holders and advisory/custom-command limits. The report's record fields and outcome names are unchanged. Busy refusal output now identifies a helper or Git process as the holder. The existing clone lock remains auxiliary, outside run records. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 45 ++++++-- scripts/confirmatory_run.py | 5 +- tests/test_confirmatory_anchor.py | 152 +++++++++++++++++++++++++ tests/test_confirmatory_run.py | 183 ++++++++++++++++++++++++++++-- 4 files changed, 360 insertions(+), 25 deletions(-) diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py index 838a4bba..2dd7f2b3 100644 --- a/scripts/confirmatory_anchor.py +++ b/scripts/confirmatory_anchor.py @@ -12,10 +12,17 @@ anchor was pushed no longer matches that anchor's head. An exclusive non-blocking lock in the clone's `.git` directory covers every -read, copy, commit and push. A busy clone fails without changing its records. -Under the lock, the clone's current file must be a byte prefix of the snapshot: -a delayed older command cannot replace newer evidence. Git's own index.lock -is left alone; a git failure is a failed push. +read, copy, commit and push. Every Git command inherits its descriptor, so a +surviving command keeps the clone busy even if this helper alone is killed. +Git's hooks and remote helpers inherit it too; automatic fsmonitor startup, +which closes inherited descriptors, is disabled. Configured commands must +preserve the descriptor in any background clone writer. This advisory lock +does not cover independent writers or replacement of the lock file. +A busy clone fails without changing its records. Under the lock, the clone's +current file must be a byte prefix of the snapshot (equality is allowed): a +delayed older command cannot replace newer evidence. This checks bytes, not +JSON or line boundaries; the runner supplies complete, synced lines. Git's +own index.lock is left alone; a git failure is a failed push. Never force-push. """ from __future__ import annotations @@ -28,10 +35,14 @@ from pathlib import Path -def git(repo: Path, *args: str) -> str: - """Run git in the anchor clone, retaining failures for the caller.""" +def git(repo: Path, lock_fd: int, *args: str) -> str: + """Run Git with the clone lock inherited, retaining failures for the caller.""" return subprocess.run( - ["git", "-C", str(repo), *args], check=True, capture_output=True, text=True + ["git", "-c", "core.fsmonitor=false", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + pass_fds=(lock_fd,), ).stdout @@ -47,7 +58,10 @@ def main(argv=None) -> int: try: fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: - print("anchor clone busy: another helper holds its lock", file=sys.stderr) + print( + "anchor clone busy: another helper or Git process holds its lock", + file=sys.stderr, + ) return 1 snapshot = args.anchors.read_bytes() current = target.read_bytes() if target.exists() else b"" @@ -61,11 +75,18 @@ def main(argv=None) -> int: if snapshot != current or not target.exists(): target.write_bytes(snapshot) lines = snapshot.decode().splitlines() - git(repo, "add", "anchors.jsonl") - if git(repo, "status", "--porcelain", "--", "anchors.jsonl"): + git(repo, lock.fileno(), "add", "anchors.jsonl") + if git(repo, lock.fileno(), "status", "--porcelain", "--", "anchors.jsonl"): head = hashlib.sha256(lines[-1].encode()).hexdigest() if lines else "none" - git(repo, "commit", "-q", "-m", f"anchor {len(lines)}: {head}") - git(repo, "push", "-q", "origin", "HEAD") + git( + repo, + lock.fileno(), + "commit", + "-q", + "-m", + f"anchor {len(lines)}: {head}", + ) + git(repo, lock.fileno(), "push", "-q", "origin", "HEAD") print(f"pushed {len(lines)} anchors") return 0 diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 1ee029c4..d4dd254b 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -77,8 +77,9 @@ command: under an exclusive, non-blocking clone lock, it accepts only a snapshot extending the clone's anchors, commits it and pushes it, so the remote holds each anchor's time from outside the run. Retiring a worker - does not ensure its command has ended. If an old helper still holds the - lock, later pushes fail as busy; a final segment left local is a deviation. + does not ensure its command has ended. If an old helper or its Git child + still holds the lock, later pushes fail as busy; a final segment left local + is a deviation. Final pushes get up to three tries per set of heads, waiting a minute after each failed try ends before starting the next. diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index 5fa338ff..153a2f1d 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -134,6 +134,135 @@ def test_a_busy_clone_is_refused_without_changing_its_records(clone, tmp_path): assert remote_log(remote) == history +@pytest.mark.parametrize("hook_name", ["pre-commit", "pre-push", "pre-receive"]) +def test_killing_only_the_helper_keeps_its_git_writer_locked( + clone, tmp_path, hook_name +): + """A Git child already using the clone retains the lock after helper death.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + older = first + b'{"time": 2.0}\n' + snapshots = [older + b'{"time": 3.0}\n', older + b'{"time": 3.0}\n{"time": 4.0}\n'] + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + ready, release = tmp_path / "ready", tmp_path / "release" + for path in (ready, release): + os.mkfifo(path) + hook_lock = tmp_path / "hook-lock.json" + hooks = remote / "hooks" if hook_name == "pre-receive" else repo / ".git" / "hooks" + hook = hooks / hook_name + hook.write_text( + f"#!{sys.executable}\n" + "import json, os\n" + "from pathlib import Path\n" + "if os.environ.get('ANCHOR_TEST_BLOCK') == '1':\n" + f" lock = Path({str(repo / '.git' / 'confirmatory-anchor.lock')!r}).stat()\n" + " inherited = []\n" + " for fd in range(3, 256):\n" + " try:\n" + " info = os.fstat(fd)\n" + " except OSError:\n" + " continue\n" + " if (info.st_dev, info.st_ino) == (lock.st_dev, lock.st_ino):\n" + " inherited.append(os.get_inheritable(fd))\n" + f" Path({str(hook_lock)!r}).write_text(json.dumps(inherited))\n" + f" with open({str(ready)!r}, 'wb', buffering=0) as pipe:\n" + " pipe.write(b'1')\n" + f" with open({str(release)!r}, 'rb', buffering=0) as pipe:\n" + " pipe.read(1)\n" + ) + hook.chmod(0o755) + + def records(): + """Read the clone's file, index, HEAD and the remote without writing.""" + objects = [ + subprocess.run( + ["git", "-C", str(repo), "show", obj], check=True, capture_output=True + ).stdout + for obj in (":anchors.jsonl", "HEAD:anchors.jsonl") + ] + return ( + (repo / "anchors.jsonl").read_bytes(), + (repo / ".git" / "index").read_bytes(), + *objects, + remote_file(remote).encode(), + remote_log(remote), + ) + + ready_fd, release_fd = (os.open(path, os.O_RDWR) for path in (ready, release)) + child = None + try: + anchors.write_bytes(older) + child = subprocess.Popen( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + env={**os.environ, "ANCHOR_TEST_BLOCK": "1"}, + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert select.select([ready_fd], [], [], 15.0)[0], "Git did not reach its hook" + assert os.read(ready_fd, 1) == b"1" + before = records() + assert before[0] == before[2] == older + child.kill() + assert child.wait(timeout=15.0) == -signal.SIGKILL + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 1 and "busy" in out.stderr, ( + "a surviving Git writer lost the helper's clone lock: " + out.stderr + ) + assert records() == before + assert json.loads(hook_lock.read_text()) == [True] + os.write(release_fd, b"1") + child.communicate(timeout=15.0) + # Pipe closure need not mean every inherited lock holder has ended. + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + deadline = time.monotonic() + 15.0 + while True: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, "a Git child did not finish" + threading.Event().wait(0.01) + after = records() + assert after[0] == after[2] == after[3] == older + assert after[4] in (first, older) + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + state = records() + assert state[0] == state[2] == state[3] == state[4] == snapshot + history = subprocess.run( + ["git", "-C", str(remote), "rev-list", "--reverse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + previous = b"" + for commit in history: + content = subprocess.run( + ["git", "-C", str(remote), "show", f"{commit}:anchors.jsonl"], + check=True, + capture_output=True, + ).stdout + assert content.startswith(previous) + previous = content + assert previous == snapshots[-1] + finally: + if child is not None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.communicate(timeout=15.0) + for fd in (ready_fd, release_fd): + os.close(fd) + + def test_a_changed_middle_line_is_refused_without_writing(clone, tmp_path): """A longer snapshot still fails when it rewrites an existing line.""" remote, repo = clone @@ -169,6 +298,29 @@ def test_a_stale_git_index_lock_is_left_alone(clone, tmp_path): assert remote_log(remote) == history +def test_git_fsmonitor_is_disabled_even_when_the_clone_enables_it(clone, tmp_path): + """Git must not launch a monitor that can discard the inherited clone lock.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + anchors.write_bytes(b'{"time": 1.0}\n') + assert anchor_command(repo, anchors).returncode == 0 + invoked, monitor = tmp_path / "monitor-invoked", tmp_path / "fsmonitor" + monitor.write_text( + f"#!{sys.executable}\n" + "from pathlib import Path\n" + f"Path({str(invoked)!r}).touch()\n" + ) + monitor.chmod(0o755) + subprocess.run( + ["git", "-C", str(repo), "config", "core.fsmonitor", str(monitor)], check=True + ) + anchors.write_bytes(anchors.read_bytes() + b'{"time": 2.0}\n') + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + assert not invoked.exists() + assert remote_file(remote).encode() == anchors.read_bytes() + + def test_a_failed_push_exits_nonzero(clone, tmp_path): """The runner records the command's return code; a push that fails must not look like one that landed.""" diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index ed103b04..d4e3622f 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -11,6 +11,7 @@ from __future__ import annotations +import copy import hashlib import io import json @@ -559,6 +560,13 @@ def test_a_backward_wall_clock_does_not_delay_a_due_anchor(monkeypatch): assert [run.anchor_due() for _ in range(3)] == [False, True, True] +def test_the_anchor_interval_and_retry_wait_are_registered_durations(): + """paper/preregistration-draft.md:218: anchors every 15 minutes, and retries + a minute after the failed push ends.""" + assert cr.ANCHOR_SECONDS == 900.0 + assert cr.ANCHOR_RETRY == 60.0 + + def test_a_wall_clock_that_steps_back_keeps_the_run_procedure(world, monkeypatch): """Every reading of the wall clock up to 0.2 ms early (steps back of up to 0.1 ms seen on this host): the run's times still order its attempts, @@ -768,6 +776,7 @@ def __init__( pilot._STATED_ENGINE, "version", pilot._STATED_ENGINE["version"] ) self.models = tuple(models or cr.CHEAP + cr.FRONTIER) + budget_models = tuple(dict.fromkeys(cr.CHEAP + cr.FRONTIER + self.models)) self.design = { "preregistration": str(prereg), "frozen_sha256": frozen, @@ -778,8 +787,8 @@ def __init__( "draw_order": self.draw_order, "arms": {a: a for a in cr.ARMS}, "frontier_arms": {a: f"f-{a}" for a in cr.FRONTIER_ARMS}, - "caps": {m: (caps or {}).get(m, 10.0) for m in cr.CHEAP + cr.FRONTIER}, - "reserves": {m: reserve for m in cr.CHEAP + cr.FRONTIER}, + "caps": {m: (caps or {}).get(m, 10.0) for m in budget_models}, + "reserves": {m: reserve for m in budget_models}, "overall_cap": overall_cap, "spent_before": spent_before, "workers": 2, @@ -2239,28 +2248,180 @@ def stateless_model(**kwargs): return chat_response([chat_call("submit_outputs", {"outputs": outputs}, "s1")]) -def test_the_harness_conversation_runs_under_the_runner(world): - model = "claude-haiku-5.5" +def stateless_responses_model(**kwargs): + """A Responses tool-contract model: calculate, then submit the answers.""" + from types import SimpleNamespace + + if "previous_response_id" not in kwargs: + name, call_id = pilot.TOOL_NAME, "c1" + arguments = {"situation": json.dumps({"people": {}}), "variables": ["snap"]} + else: + name, call_id = "submit_outputs", "s1" + arguments = { + "outputs": {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + } + return SimpleNamespace( + id=f"resp_{call_id}", + output=[ + SimpleNamespace( + type="function_call", + id=f"fc_{call_id}", + call_id=call_id, + name=name, + arguments=json.dumps(arguments), + status="completed", + ) + ], + output_text=None, + status="completed", + incomplete_details=None, + usage=SimpleNamespace( + input_tokens=1000, + output_tokens=200, + total_tokens=1200, + input_tokens_details=SimpleNamespace(cached_tokens=0), + output_tokens_details=SimpleNamespace(reasoning_tokens=0), + ), + model=kwargs["model"], + ) + + +def stateless_json_model(**kwargs): + """A Gemini JSON-contract model: calculate, then answer in JSON text.""" + if not any(m.get("role") == "tool" for m in kwargs["messages"]): + response = stateless_model(**kwargs) + response.model = kwargs["model"] + return response + response = chat_response([]) + response.model = kwargs["model"] + response.choices[0].finish_reason = "stop" + response.choices[0].message.content = json.dumps( + {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + ) + return response + + +@pytest.mark.parametrize( + "model, completion", + [ + ("claude-haiku-5.5", stateless_model), + ("gpt-6-luna", stateless_responses_model), + ("gemini-3.1-pro-preview", stateless_json_model), + ], +) +def test_the_harness_conversation_runs_under_the_runner(world, model, completion): + """Each API and answer contract completes real harness conversations in + both runner lanes, with the registered feature frame for every arm. + + The confirmatory roster's Geminis use tools. The older board profile + covers the harness's Gemini JSON path in a fake design, without changing + the production roster or any model's answer contract.""" w = world(models=(model,)) w.run.converse = cr.converse_with_harness - w.run.completion_for = lambda profile: stateless_model + w.run.completion_for = lambda profile: completion w.execute() + labels = labels_of(model) attempts = w.attempts(model) - assert len(attempts) == N_DRAW * len(cr.ARMS) + assert len(attempts) == N_DRAW * len(labels) assert {x["status"] for x in attempts} == {"complete"} messages = w.run.lanes[model].pool.messages - assert len(messages) == N_DRAW * len(cr.ARMS) # one calculation per attempt + assert len(messages) == N_DRAW * len(labels) # one calculation per attempt frames = [m for m in messages if isinstance(m, tuple)] - assert len(frames) == N_DRAW * (len(cr.ARMS) - 1) # every arm but pe + assert len(frames) == N_DRAW * (len(labels) - 1) # every arm but pe sid = w.draw_order[0] - rec = json.loads((w.folder(model, "loo-names") / f"{sid}.json").read_text()) - profile = cr.arm_profile(model, "loo-names") - assert rec["arm"] == cr.arm_string("loo-names") and rec["attempts"] == [] + label = "loo-names" if "loo-names" in labels else "pe3" + rec = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + profile = cr.arm_profile(model, label) + assert rec["arm"] == cr.arm_string(label) and rec["attempts"] == [] assert rec["tool_features"] == sorted(profile.tool_features) assert rec["predictions"] == {v: 0.0 for v in VARIABLES} assert run_problems(w) == [] +@pytest.mark.parametrize( + "model, completion, api, contract, label", + [ + (model, completion, api, contract, label) + for model, completion, api, contract in [ + ("gpt-6-luna", stateless_responses_model, "responses", "tool"), + ("gemini-3.1-pro-preview", stateless_json_model, "chat", "json"), + ] + for label in labels_of(model) + ], +) +def test_the_runner_sends_the_treatment_conversation( + world, label, model, completion, api, contract +): + """Compare every provider request and calculation against the unchanged + treatment harness at 6b00a6f0, including the tool-result continuation. + + Gemini 3.1's board profile exercises JSON; both confirmatory Geminis use + the tool contract. No contract or model card is replaced for this test.""" + w = world(models=(model,)) + lanes, sid = w.run.lanes[model], w.draw_order[0] + requests, calculations = [], [] + + def provider(**kwargs): + """Keep each request before the conversation can extend its messages.""" + requests.append(copy.deepcopy(kwargs)) + return completion(**kwargs) + + def tool(arguments): + """Keep exactly what the conversation sends to the calculator.""" + calculations.append(arguments) + return {"results": {"snap": {"values": {"spm": 1.0}}}}, 0.0 + + lanes.completion = provider + record = cr.converse_with_harness( + lanes, sid, label, tool, cr.AttemptSpend(w.run.ceiling) + ) + sent, called = requests[:], calculations[:] + requests.clear() + calculations.clear() + profile = pilot.model_profile( + model, arm=cr.arm_string(label), prompt_version="v2-pilot" + ) + treatment = pilot.run_household( + w.board["scenarios"][sid], + w.board["variables"][sid], + tool, + provider, + pilot.Spend(100.0), + profile=profile, + ) + assert record["api"] == treatment["api"] == api + assert record["contract"] == treatment["contract"] == contract + assert record["status"] == treatment["status"] == "complete" + assert record["predictions"] == treatment["predictions"] == { + v: 0.0 for v in VARIABLES + } + assert len(sent) == len(requests) == 2 + assert sent == requests and called == calculations and len(called) == 1 + assert record["messages"] == treatment["messages"] + first, second = sent + if api == "responses": + assert "messages" not in first and "previous_response_id" not in first + assert first["tool_choice"] == "required" + assert [t["name"] for t in first["tools"]] == [ + pilot.TOOL_NAME, + "submit_outputs", + ] + assert "max_output_tokens" in first + assert second["previous_response_id"] == "resp_c1" + assert second["input"] == [ + { + "type": "function_call_output", + "call_id": "c1", + "output": json.dumps({"results": {"snap": {"values": {"spm": 1.0}}}}), + } + ] + else: + assert "input" not in first and first["tool_choice"] == "auto" + assert [t["function"]["name"] for t in first["tools"]] == [pilot.TOOL_NAME] + assert second["messages"][-1]["role"] == "tool" + assert second["messages"][-1]["tool_call_id"] == "c1" + + def test_the_ceiling_ends_a_harness_conversation_at_its_next_turn(world): model = "claude-haiku-5.5" w = world(models=(model,), overall_cap=0.0, spent_before=25.0 - 1e-9) From 4002f8ee4454f06c2373d9742aa2d365a629e920 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 23:48:23 -0700 Subject: [PATCH 29/64] Confirmatory runner: round-6 wording and coverage Answers review runner-r6's two P3 items: - Narrow the fsmonitor claim to disabling normal index-triggered startup. A configured hook can explicitly start the descriptor-closing daemon; configured background clone writers must retain the locked descriptor without unlocking it. The flock is local, while an external remote's safety rests on the byte-prefix check and ordinary non-force pushes. - Describe credential-cache and other descriptor holders keeping later helpers busy for their lifetimes, including the default 900-second cache. Every try while the lock is retained, including all three final tries, fails as busy and is reported as a failed push. - Block real Git clean filters during add and status, kill only the helper, and check later busy refusals leave file, raw index, staged contents, HEAD and remote history unchanged. Release the writer and verify newer snapshots push with prefix-preserving history. - Double-fork a hook descendant into a new session. Let its hook and helper finish, check the detached holder still prevents later pushes, and verify pushes succeed after it exits. Check each new regression against a temporary helper with inheritance removed only from its target command. Production behavior, report fields and outcomes are unchanged. This adds no dependencies and leaves the report implementation and draft untouched. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 29 ++- tests/test_confirmatory_anchor.py | 295 +++++++++++++++++++++++++++++- 2 files changed, 313 insertions(+), 11 deletions(-) diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py index 2dd7f2b3..8cd02865 100644 --- a/scripts/confirmatory_anchor.py +++ b/scripts/confirmatory_anchor.py @@ -14,15 +14,26 @@ An exclusive non-blocking lock in the clone's `.git` directory covers every read, copy, commit and push. Every Git command inherits its descriptor, so a surviving command keeps the clone busy even if this helper alone is killed. -Git's hooks and remote helpers inherit it too; automatic fsmonitor startup, -which closes inherited descriptors, is disabled. Configured commands must -preserve the descriptor in any background clone writer. This advisory lock -does not cover independent writers or replacement of the lock file. -A busy clone fails without changing its records. Under the lock, the clone's -current file must be a byte prefix of the snapshot (equality is allowed): a -delayed older command cannot replace newer evidence. This checks bytes, not -JSON or line boundaries; the runner supplies complete, synced lines. Git's -own index.lock is left alone; a git failure is a failed push. Never force-push. +Ordinary local hooks and remote helpers inherit it too. The per-command +`core.fsmonitor=false` disables normal index-triggered fsmonitor startup. +A configured hook can still run `git fsmonitor--daemon start`, whose child +closes inherited descriptors; this falls under the configured-command limit: +configured commands must retain the descriptor without unlocking it in any +background clone writer. This advisory lock does not cover independent +writers or replacement of the lock file. The lock is local: processes on an +external remote server never hold it. The remote's safety rests on the byte-prefix +check and ordinary non-force pushes. + +A credential-cache daemon or any other descriptor holder can keep the clone +busy up to the holder's lifetime (the credential cache defaults to 900 +seconds), indefinitely if it never exits or closes the descriptor. Every +later try, including all three final tries, then fails as busy and is reported +as a failed push. A busy clone fails without changing its records. Under the +lock, the clone's current file must be a byte prefix of the snapshot (equality +is allowed): a delayed older command cannot replace newer evidence. This +checks bytes, not JSON or line boundaries; the runner supplies complete, +synced lines. Git's own index.lock is left alone; a git failure is a failed +push. Never force-push. """ from __future__ import annotations diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index 153a2f1d..fbc5a367 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -263,6 +263,297 @@ def records(): os.close(fd) +@pytest.mark.parametrize("command", ["add", "status"]) +def test_killing_only_the_helper_during_add_or_status_keeps_the_clone_locked( + clone, tmp_path, command +): + """A clean filter pins each real Git command across helper-only death.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + older = first + b'{"time": 2.0}\n' + snapshots = [older + b'{"time": 3.0}\n', older + b'{"time": 3.0}\n{"time": 4.0}\n'] + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + ready, release = tmp_path / "ready", tmp_path / "release" + for path in (ready, release): + os.mkfifo(path) + invocation = tmp_path / "filter-invocation" + filter_lock = tmp_path / "filter-lock.json" + clean = tmp_path / "clean-filter" + block_at = 1 if command == "add" else 2 + clean.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "content = sys.stdin.buffer.read()\n" + "if os.environ.get('ANCHOR_TEST_BLOCK') == '1':\n" + f" invocation = Path({str(invocation)!r})\n" + " count = int(invocation.read_text()) + 1 if invocation.exists() else 1\n" + " invocation.write_text(str(count))\n" + f" if count == {block_at}:\n" + f" lock = Path({str(repo / '.git' / 'confirmatory-anchor.lock')!r})" + ".stat()\n" + " inherited = []\n" + " for fd in range(3, 256):\n" + " try:\n" + " info = os.fstat(fd)\n" + " except OSError:\n" + " continue\n" + " if (info.st_dev, info.st_ino) == (lock.st_dev, lock.st_ino):\n" + " inherited.append(os.get_inheritable(fd))\n" + f" Path({str(filter_lock)!r}).write_text(json.dumps(inherited))\n" + f" with open({str(ready)!r}, 'wb', buffering=0) as pipe:\n" + " pipe.write(b'1')\n" + f" with open({str(release)!r}, 'rb', buffering=0) as pipe:\n" + " pipe.read(1)\n" + f" elif count == 1 and {command == 'status'!r}:\n" + f" target = Path({str(repo / 'anchors.jsonl')!r})\n" + " info = target.stat()\n" + " os.utime(target, ns=(info.st_atime_ns, " + "info.st_mtime_ns - 2000000000))\n" + "sys.stdout.buffer.write(content)\n" + ) + clean.chmod(0o755) + (repo / ".git" / "info" / "attributes").write_text("anchors.jsonl filter=anchor\n") + for name, value in (("clean", str(clean)), ("required", "true")): + subprocess.run( + ["git", "-C", str(repo), "config", f"filter.anchor.{name}", value], + check=True, + ) + + def records(): + """Read file, raw index, staged and committed bytes and remote history.""" + objects = [ + subprocess.run( + ["git", "-C", str(repo), "show", obj], check=True, capture_output=True + ).stdout + for obj in (":anchors.jsonl", "HEAD:anchors.jsonl") + ] + return ( + (repo / "anchors.jsonl").read_bytes(), + (repo / ".git" / "index").read_bytes(), + *objects, + remote_file(remote).encode(), + remote_log(remote), + ) + + ready_fd, release_fd = (os.open(path, os.O_RDWR) for path in (ready, release)) + child = None + try: + anchors.write_bytes(older) + child = subprocess.Popen( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + env={**os.environ, "ANCHOR_TEST_BLOCK": "1"}, + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert select.select([ready_fd], [], [], 15.0)[0], ( + f"Git {command} did not reach its clean filter" + ) + assert os.read(ready_fd, 1) == b"1" + # The first conversion runs in add; changing its stat forces status's second. + assert invocation.read_text() == str(block_at) + before = records() + assert before[0] == older + assert before[2] == (first if command == "add" else older) + assert before[3] == before[4] == first + child.kill() + assert child.wait(timeout=15.0) == -signal.SIGKILL + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 1 and "busy" in out.stderr, ( + f"a surviving Git {command} lost the helper's clone lock: " + out.stderr + ) + assert records() == before + assert json.loads(filter_lock.read_text()) == [True] + os.write(release_fd, b"1") + child.communicate(timeout=15.0) + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + deadline = time.monotonic() + 15.0 + while True: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, f"Git {command} did not finish" + threading.Event().wait(0.01) + after = records() + assert after[0] == after[2] == older + assert after[3] == after[4] == first + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + state = records() + assert state[0] == state[2] == state[3] == state[4] == snapshot + history = subprocess.run( + ["git", "-C", str(remote), "rev-list", "--reverse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + previous = b"" + for commit in history: + content = subprocess.run( + ["git", "-C", str(remote), "show", f"{commit}:anchors.jsonl"], + check=True, + capture_output=True, + ).stdout + assert content.startswith(previous) + previous = content + assert previous == snapshots[-1] + finally: + if child is not None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.communicate(timeout=15.0) + for fd in (ready_fd, release_fd): + os.close(fd) + + +def test_a_detached_hook_descendant_keeps_the_clone_locked(clone, tmp_path): + """A double-forked holder outlives its hook and helper until explicitly released.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + older = first + b'{"time": 2.0}\n' + snapshots = [older + b'{"time": 3.0}\n', older + b'{"time": 3.0}\n{"time": 4.0}\n'] + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + ready, release = tmp_path / "ready", tmp_path / "release" + for path in (ready, release): + os.mkfifo(path) + holder = tmp_path / "detached-holder.json" + hook = repo / ".git" / "hooks" / "pre-push" + hook.write_text( + f"#!{sys.executable}\n" + "import json, os\n" + "from pathlib import Path\n" + "if os.environ.get('ANCHOR_TEST_DETACH') == '1':\n" + f" ready = os.open({str(ready)!r}, os.O_WRONLY)\n" + f" release = os.open({str(release)!r}, os.O_RDONLY)\n" + " hook_pid = os.getpid()\n" + " child = os.fork()\n" + " if child:\n" + " os.waitpid(child, 0)\n" + " raise SystemExit(0)\n" + " os.setsid()\n" + " if os.fork():\n" + " os._exit(0)\n" + " with open(os.devnull, 'r+b', buffering=0) as null:\n" + " for fd in range(3):\n" + " os.dup2(null.fileno(), fd)\n" + f" lock = Path({str(repo / '.git' / 'confirmatory-anchor.lock')!r}).stat()\n" + " inherited = []\n" + " for fd in range(3, 256):\n" + " try:\n" + " info = os.fstat(fd)\n" + " except OSError:\n" + " continue\n" + " if (info.st_dev, info.st_ino) == (lock.st_dev, lock.st_ino):\n" + " inherited.append(os.get_inheritable(fd))\n" + " state = dict(pid=os.getpid(), sid=os.getsid(0), pgid=os.getpgrp(),\n" + " hook_pid=hook_pid, inherited=inherited)\n" + f" Path({str(holder)!r}).write_text(json.dumps(state))\n" + " os.write(ready, b'1')\n" + " os.read(release, 1)\n" + " os._exit(0)\n" + ) + hook.chmod(0o755) + + def records(): + """Read the clone's file, index, HEAD and remote without writing.""" + objects = [ + subprocess.run( + ["git", "-C", str(repo), "show", obj], check=True, capture_output=True + ).stdout + for obj in (":anchors.jsonl", "HEAD:anchors.jsonl") + ] + return ( + (repo / "anchors.jsonl").read_bytes(), + (repo / ".git" / "index").read_bytes(), + *objects, + remote_file(remote).encode(), + remote_log(remote), + ) + + ready_fd, release_fd = (os.open(path, os.O_RDWR) for path in (ready, release)) + child = None + descendant = None + try: + anchors.write_bytes(older) + child = subprocess.Popen( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + env={**os.environ, "ANCHOR_TEST_DETACH": "1"}, + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert select.select([ready_fd], [], [], 15.0)[0], ( + "hook did not detach its holder" + ) + assert os.read(ready_fd, 1) == b"1" + state = json.loads(holder.read_text()) + descendant = state["pid"] + stdout, stderr = child.communicate(timeout=15.0) + assert child.returncode == 0, stderr.decode() + assert stdout == b"pushed 2 anchors\n" + assert state["sid"] != child.pid and state["pgid"] != child.pid + assert descendant != state["sid"] + os.kill(descendant, 0) + with pytest.raises(ProcessLookupError): + os.kill(state["hook_pid"], 0) + before = records() + assert before[0] == before[2] == before[3] == before[4] == older + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 1 and "busy" in out.stderr, ( + "a detached hook descendant lost the helper's clone lock: " + out.stderr + ) + assert records() == before + assert state["inherited"] == [True] + os.write(release_fd, b"1") + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + deadline = time.monotonic() + 15.0 + while True: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, ( + "the detached holder did not exit" + ) + threading.Event().wait(0.01) + descendant = None + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + state = records() + assert state[0] == state[2] == state[3] == state[4] == snapshot + finally: + os.write(release_fd, b"1") + if descendant is not None: + try: + os.kill(descendant, signal.SIGKILL) + except ProcessLookupError: + pass + if child is not None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.communicate(timeout=15.0) + for fd in (ready_fd, release_fd): + os.close(fd) + + def test_a_changed_middle_line_is_refused_without_writing(clone, tmp_path): """A longer snapshot still fails when it rewrites an existing line.""" remote, repo = clone @@ -298,8 +589,8 @@ def test_a_stale_git_index_lock_is_left_alone(clone, tmp_path): assert remote_log(remote) == history -def test_git_fsmonitor_is_disabled_even_when_the_clone_enables_it(clone, tmp_path): - """Git must not launch a monitor that can discard the inherited clone lock.""" +def test_normal_index_fsmonitor_startup_is_disabled(clone, tmp_path): + """The per-command override suppresses a configured index-triggered monitor.""" remote, repo = clone anchors = tmp_path / "anchors.jsonl" anchors.write_bytes(b'{"time": 1.0}\n') From 047c8682f426381687ae93321cddc8732b9cd635 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 00:52:57 -0700 Subject: [PATCH 30/64] Run the no-tools stage with recoverable attempts and cumulative spend. Run each selected model in its own thread, one drawn household at a time, using the pinned harness's none profile and a tool callback that raises. Reuse the calculator runner's classification and action rules without host evidence. Rerun eligible provider and unanswered attempts immediately. Pause account failures until re-invocation, stop a model on a harness class, and stop every model on a budget stop. Limit reruns to two while retaining the action helper's harness and ceiling stop precedence. Move each replaced final's exact bytes into consecutively numbered attempt files before rerunning. Atomically write finals with complete ordered number, path, hash, status, class and reason links. Recover an interrupted archive from the files alone. Refuse native prior_attempts, missing attempts lists, numbering gaps, altered history links and ineligible archives. Carry every model's final and archived spend across invocations, including optional model folders. Share the five-dollar cap across all model threads, check it before each attempt, and retain an exception's spend in the harness fallback record. Show each attempt, model state and cumulative spend; dry runs show standing and owed households. Return success only when every selected model has a standing final for every drawn household. Cover synthetic providers, retry exhaustion, operator resume, harness and budget stops, interrupted archives, corrupted histories, exception spend, optional models and agreement with the separate report's registered cap. The existing calculator runner remains unchanged. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 411 +++++++++++++++++++++++++++ tests/test_confirmatory_no_tools.py | 426 ++++++++++++++++++++++++++++ 2 files changed, 837 insertions(+) create mode 100644 scripts/confirmatory_no_tools.py create mode 100644 tests/test_confirmatory_no_tools.py diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py new file mode 100644 index 00000000..c48f06ac --- /dev/null +++ b/scripts/confirmatory_no_tools.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +"""Run the confirmatory no-tools stage with complete, recoverable attempts. + +Every model runs in its own thread, in draw order, under a shared $5 ceiling. +A replaced final is moved byte for byte into ``households/attempts`` before +its rerun; the final links every archive by number, hash, status and reason. +Re-invocation resumes solely from those files and is an operator's approval +to rerun a paused account failure. Native harness resume is never used. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import re +import sys +import threading +from pathlib import Path + +# confirmatory_run pins the harness and policybench to this checkout. +from confirmatory_run import ( + CHEAP, + MAX_RERUNS, + PROMPT_VERSION, + action, + attempt_cost, + classify, + default_completion, + pilot, + sha256_file, + write_json, +) + +NO_TOOLS_CAP = 5.0 + + +class StateError(ValueError): + """The files cannot describe an intact no-tools attempt history.""" + + +def component(value: str) -> str: + """A record/model name must name one ordinary path component.""" + if ( + not isinstance(value, str) + or value in ("", ".", "..") + or Path(value).name != value + ): + raise StateError(f"invalid path component {value!r}") + return value + + +def read_record(path: Path) -> dict: + try: + record = json.loads(path.read_text()) + if not isinstance(record, dict): + raise ValueError("record is not a JSON object") + if "prior_attempts" in record: + raise ValueError("native prior_attempts cannot preserve full attempts") + if not isinstance(record.get("attempts"), list): + raise ValueError("record lacks an attempts list") + cost = attempt_cost(record) + if not math.isfinite(cost) or cost < 0: + raise ValueError("attempt cost must be finite and nonnegative") + return record + except (OSError, ValueError, TypeError, AttributeError) as error: + raise StateError(f"{path}: {error}") from error + + +def entry(path: Path, record: dict, n: int) -> dict: + cls, reason = classify(record, 0.0, 0.0, None) + return { + "n": n, + "file": f"attempts/{path.name}", + "sha256": sha256_file(path), + "status": record.get("status"), + "class": cls, + "reason": reason, + } + + +def history(folder: Path) -> tuple[dict[str, dict], float]: + """Validate a model's histories, including an interrupted archive, and sum spend.""" + archives: dict[str, list[tuple[int, Path]]] = {} + for path in sorted((folder / "attempts").glob("*.json")): + match = re.fullmatch(r"(.+)-([1-9][0-9]*)\.json", path.name) + if not match: + raise StateError(f"{path}: invalid attempt number") + archives.setdefault(match[1], []).append((int(match[2]), path)) + finals = {path.stem: path for path in sorted(folder.glob("*.json"))} + states, spent = {}, 0.0 + for sid in sorted(set(finals) | set(archives)): + paths = sorted(archives.get(sid, [])) + numbers = [n for n, _ in paths] + if numbers != list(range(1, len(paths) + 1)): + raise StateError( + f"{folder}: attempt-number gap for {sid}: " + + ", ".join(path.name for _, path in paths) + ) + if len(paths) > MAX_RERUNS: + raise StateError(f"{folder}: {sid} exceeds {MAX_RERUNS} reruns") + entries = [] + for n, path in paths: + record = read_record(path) + if record.get("scenario_id") != sid: + raise StateError(f"{path}: household differs from archive name {sid}") + if record["attempts"] != entries: + raise StateError(f"{path}: attempts disagree with earlier archives") + cls, _ = classify(record, 0.0, 0.0, None) + if action(cls, n) not in ("rerun", "pause-rerun"): + raise StateError(f"{path}: archived attempt was ineligible for rerun") + entries.append(entry(path, record, n)) + spent += attempt_cost(record) + final = None + if sid in finals: + path = finals[sid] + final = read_record(path) + if final.get("scenario_id") != sid: + raise StateError(f"{path}: household differs from final name {sid}") + if final["attempts"] != entries: + raise StateError(f"{path}: attempts disagree with archived files") + spent += attempt_cost(final) + states[sid] = {"entries": entries, "final": final} + return states, spent + + +class Spend: + """The stage's recorded spend, shared by every model and every invocation.""" + + def __init__(self, spent: float, stopped: threading.Event): + self.spent_usd, self.budget_usd = spent, NO_TOOLS_CAP + self.lock, self.stopped = threading.Lock(), stopped + if spent >= self.budget_usd: + stopped.set() + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + with self.lock: + return ( + self.stopped.is_set() or self.spent_usd + reserve_usd >= self.budget_usd + ) + + +class AttemptSpend: + """Count a request in both the stage and its own exception fallback record.""" + + def __init__(self, stage: Spend): + self.stage, self.spent_usd = stage, 0.0 + + def add(self, usd: float | None) -> None: + cost = float(usd or 0.0) + with self.stage.lock: + self.spent_usd += cost + self.stage.spent_usd += cost + if self.stage.spent_usd >= self.stage.budget_usd: + self.stage.stopped.set() + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + return self.stage.exhausted(reserve_usd) + + +class Stage: + """One stage, with injectable providers/board for a provider-free rehearsal.""" + + def __init__( + self, + design: dict, + models=None, + run_household=None, + completion_for=default_completion, + board=None, + out=None, + ): + self.design = design + self.models = tuple(dict.fromkeys(models if models is not None else CHEAP)) + if not self.models: + raise StateError("at least one model is required") + for model in self.models: + component(model) + self.draw_order = [component(sid) for sid in design["draw_order"]] + if len(self.draw_order) != len(set(self.draw_order)): + raise StateError("draw_order repeats a household") + spec = design["no_tools"] + self.root = Path(spec["runs"]) + self.folder, self.stage = component(spec["folder"]), component(spec["stage"]) + self.out = sys.stdout if out is None else out + self._say_lock = threading.Lock() + self.stopped = threading.Event() + self.completion_for = completion_for + self.run_household = ( + run_household if run_household is not None else pilot.run_household + ) + pilot.set_stated_engine(design["engine"].split()[-1]) + if board is None: + board_path = Path(design["board"]) + if f"policyengine-us {pilot.board_engine(board_path)}" != design["engine"]: + raise StateError("the board's engine differs from the design") + pilot._require_engine(board_path) + board = pilot.load_board(board_path) + self.board = board + missing = set(self.draw_order) - set(board["scenarios"]) + missing |= set(self.draw_order) - set(board["variables"]) + if missing: + raise StateError( + f"drawn households missing from the board: {sorted(missing)}" + ) + self.histories = {} + spent = 0.0 + folders = ( + {p.name for p in self.root.iterdir() if p.is_dir()} + if self.root.is_dir() + else set() + ) + for model in sorted(folders | set(self.models)): + states, cost = history(self.households(model)) + self.histories[model] = states + spent += cost + # A saved ceiling failure is a stage-wide stop on every invocation. + if any( + state["final"] is not None + and action( + classify(state["final"], 0.0, 0.0, None)[0], + len(state["entries"]) + 1, + ) + == "stopped" + for state in states.values() + ): + self.stopped.set() + self.spend = Spend(spent, self.stopped) + self.states = {model: "pending" for model in self.models} + + def households(self, model: str) -> Path: + return self.root / model / self.folder / self.stage / "households" + + def say(self, text: str) -> None: + with self._say_lock: + print(text, file=self.out, flush=True) + + def standing(self, model: str, sid: str) -> bool: + state = self.histories[model].get(sid) + return bool( + state + and state["final"] is not None + and action( + classify(state["final"], 0.0, 0.0, None)[0], len(state["entries"]) + 1 + ) + == "stands" + ) + + def dry_run(self) -> int: + for model in self.models: + done = [sid for sid in self.draw_order if self.standing(model, sid)] + owed = [] + for sid in self.draw_order: + if sid in done: + continue + state = self.histories[model].get(sid, {"entries": [], "final": None}) + n = len(state["entries"]) + 1 + if state["final"] is not None: + outcome = action(classify(state["final"], 0.0, 0.0, None)[0], n) + if outcome in ("stop", "stopped"): + owed.append(f"{sid}:blocked {outcome} at attempt {n}") + continue + n += 1 + owed.append(f"{sid}:{n}") + self.say(f"{model}: done {done}; owed (household:next attempt) {owed}") + self.say(f"stage spend ${self.spend.spent_usd:.6f} of ${NO_TOOLS_CAP:.2f}") + return ( + 0 + if all( + self.standing(model, sid) + for model in self.models + for sid in self.draw_order + ) + else 1 + ) + + def archive(self, model: str, sid: str) -> None: + """Move an eligible final unchanged, then extend its reconstructed links.""" + state = self.histories[model][sid] + record, entries = state["final"], state["entries"] + n = len(entries) + 1 + cls, _ = classify(record, 0.0, 0.0, None) + if action(cls, n) not in ("rerun", "pause-rerun"): + raise StateError(f"{model}/{sid}: attempt {n} is ineligible for rerun") + folder = self.households(model) + target = folder / "attempts" / f"{sid}-{n}.json" + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + raise StateError(f"{target}: would overwrite an archived attempt") + os.replace(folder / f"{sid}.json", target) + entries.append(entry(target, record, n)) + state["final"] = None + + def run_model(self, model: str) -> None: + try: + profile = pilot.model_profile( + model, arm="none", prompt_version=PROMPT_VERSION + ) + if profile.uses_calculator or profile.uses_web: + raise StateError(f"{model}: the none profile offers a tool") + completion = None + completion_loaded = False + for sid in self.draw_order: + state = self.histories[model].setdefault( + sid, {"entries": [], "final": None} + ) + while True: + if state["final"] is not None: + cls, _ = classify(state["final"], 0.0, 0.0, None) + outcome = action(cls, len(state["entries"]) + 1) + if outcome == "stands": + break + if outcome == "stop": + self.states[model] = "harness-stopped" + return + if outcome == "stopped": + self.stopped.set() + if self.spend.exhausted(): + self.states[model] = "budget-stopped" + return + if state["final"] is not None: + self.archive(model, sid) + if not completion_loaded: + completion = self.completion_for(profile) + completion_loaded = True + if self.spend.exhausted(): + self.states[model] = "budget-stopped" + return + spend = AttemptSpend(self.spend) + + def no_tool(arguments): + raise RuntimeError("the no-tools arm cannot call a tool") + + try: + record = self.run_household( + self.board["scenarios"][sid], + self.board["variables"][sid], + no_tool, + completion, + spend, + profile=profile, + ) + except Exception as error: + record = pilot.harness_error_record( + sid, self.board["variables"][sid], profile, error + ) + record["totals"]["cost_usd"] = spend.spent_usd + record["attempts"] = list(state["entries"]) + write_json(self.households(model) / f"{sid}.json", record) + state["final"] = record + n = len(state["entries"]) + 1 + cls, _ = classify(record, 0.0, 0.0, None) + outcome = action(cls, n) + self.say( + f"{model} {sid} {n} {cls} {outcome} ${attempt_cost(record):.6f}" + ) + if outcome == "pause-rerun": + self.states[model] = "operator-paused" + return + if outcome == "stop": + self.states[model] = "harness-stopped" + return + if outcome == "stopped": + self.stopped.set() + self.states[model] = "budget-stopped" + return + if outcome == "stands": + break + self.states[model] = "complete" + except Exception as error: + self.states[model] = "error" + self.say(f"{model}: {type(error).__name__}: {error}") + + def execute(self) -> int: + threads = [ + threading.Thread( + target=self.run_model, args=(model,), name=f"no-tools-{model}" + ) + for model in self.models + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + for model in self.models: + self.say(f"{model}: {self.states[model]}") + self.say(f"stage spend ${self.spend.spent_usd:.6f} of ${NO_TOOLS_CAP:.2f}") + return ( + 0 if all(self.states[model] == "complete" for model in self.models) else 1 + ) + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run") + run.add_argument("design", type=Path) + run.add_argument("--model", action="extend", nargs="+") + run.add_argument("--dry-run", action="store_true") + args = parser.parse_args(argv) + try: + stage = Stage(json.loads(args.design.read_text()), models=args.model) + return stage.dry_run() if args.dry_run else stage.execute() + except (StateError, OSError, ValueError, KeyError, TypeError) as error: + print(f"The no-tools stage cannot start: {error}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py new file mode 100644 index 00000000..1a2b388e --- /dev/null +++ b/tests/test_confirmatory_no_tools.py @@ -0,0 +1,426 @@ +"""Provider-free tests of the no-tools stage's files, retries and shared spend.""" + +from __future__ import annotations + +import ast +import io +import json +import os +import sys +import threading +from pathlib import Path +from types import SimpleNamespace + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import confirmatory_no_tools as nt # noqa: E402 + + +@pytest.fixture +def setup(tmp_path): + draw = ["h2", "h1"] + return { + "design": { + "board": "synthetic.csv", + "engine": "policyengine-us 2.60.4", + "draw_order": draw, + "no_tools": { + "runs": str(tmp_path / "no-tools"), + "folder": "none", + "stage": "confirmatory", + }, + }, + "board": { + "scenarios": {sid: SimpleNamespace(id=sid) for sid in draw}, + "variables": {sid: ["snap"] for sid in draw}, + }, + } + + +def record(sid, profile, status="complete", error=None, cost=0.1): + return { + "scenario_id": sid, + "model": profile.name, + "arm": profile.arm, + "status": status, + "error": error, + "turns": [] if status == "harness_error" else [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": cost}, + } + + +class Provider: + def __init__(self, planted=None): + self.planted, self.calls = planted or {}, [] + self.lock = threading.Lock() + + def __call__( + self, scenario, variables, tool_call, completion_fn, spend, *, profile + ): + key = profile.name, scenario.id + with self.lock: + n = 1 + sum(call[:2] == key for call in self.calls) + self.calls.append((*key, n)) + spec = self.planted.get((*key, n), {}) + cost = spec.get("cost", 0.1) + spend.add(cost) + if spec.get("raise"): + raise RuntimeError("synthetic harness failure") + return record( + scenario.id, + profile, + cost=cost, + status=spec.get("status", "complete"), + error=spec.get("error"), + ) + + +def stage(setup, provider=None, models=None, **kwargs): + return nt.Stage( + setup["design"], + models=models, + board=setup["board"], + run_household=provider or Provider(), + completion_for=lambda profile: None, + out=io.StringIO(), + **kwargs, + ) + + +def final(plan, model=nt.CHEAP[0], sid="h2"): + return plan.households(model) / f"{sid}.json" + + +def load(path): + return json.loads(path.read_text()) + + +def plant(setup, status, error, n=1, **extra): + return {(nt.CHEAP[0], "h2", n): {"status": status, "error": error, **extra}} + + +def test_every_household_ok_in_draw_order_and_each_model_has_its_thread(setup): + provider, threads = Provider(), {} + + def fake(*args, **kwargs): + threads.setdefault(kwargs["profile"].name, threading.current_thread().name) + return provider(*args, **kwargs) + + plan = stage(setup, fake) + assert plan.execute() == 0 + assert all(plan.states[m] == "complete" for m in nt.CHEAP) + assert len(set(threads.values())) == len(nt.CHEAP) + for model in nt.CHEAP: + assert [call[1] for call in provider.calls if call[0] == model] == ["h2", "h1"] + for sid in setup["design"]["draw_order"]: + assert load(final(plan, model, sid))["attempts"] == [] + assert plan.spend.spent_usd == pytest.approx(0.6) + + +def test_provider_rerun_links_the_unchanged_archived_bytes(setup): + provider = Provider(plant(setup, "provider_error", "APIConnectionError: synthetic")) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + written = {} + original = nt.write_json + + def capture(path, payload, *args, **kwargs): + result = original(path, payload, *args, **kwargs) + written.setdefault(path.name, path.read_bytes()) + return result + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(nt, "write_json", capture) + assert plan.execute() == 0 + path = final(plan) + archive = path.parent / "attempts/h2-1.json" + assert archive.read_bytes() == written["h2.json"] + assert load(path)["attempts"] == [ + { + "n": 1, + "file": "attempts/h2-1.json", + "sha256": nt.sha256_file(archive), + "status": "provider_error", + "class": "provider", + "reason": "APIConnectionError", + } + ] + assert stage(setup, models=[nt.CHEAP[0]]).spend.spent_usd == pytest.approx(0.3) + + +def test_third_provider_failure_stands_and_has_only_two_archives(setup): + planted = { + **plant(setup, "provider_error", "APIConnectionError", 1), + **plant(setup, "provider_error", "APIConnectionError", 2), + **plant(setup, "provider_error", "APIConnectionError", 3), + } + provider = Provider(planted) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.execute() == 0 + assert [n for m, sid, n in provider.calls if sid == "h2"] == [1, 2, 3] + assert len(load(final(plan))["attempts"]) == nt.MAX_RERUNS + assert load(final(plan))["status"] == "provider_error" + + +def test_operator_pause_stops_its_model_and_reinvocation_resumes(setup): + provider = Provider( + plant(setup, "provider_error", "AuthenticationError: synthetic") + ) + first = stage(setup, provider) + assert first.execute() == 1 + assert first.states[nt.CHEAP[0]] == "operator-paused" + assert not final(first, sid="h1").exists() + assert first.states[nt.CHEAP[1]] == "complete" + before = final(first).read_bytes() + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.execute() == 0 + assert (final(resumed).parent / "attempts/h2-1.json").read_bytes() == before + assert load(final(resumed))["attempts"][0]["class"] == "operator" + + +def test_harness_stop_prevents_later_households_and_persists(setup): + provider = Provider(plant(setup, "provider_error", "BadRequestError: synthetic")) + + # A request error is a harness class only before a completed turn. + def early_failure(*args, **kwargs): + result = provider(*args, **kwargs) + if result["status"] == "provider_error": + result["turns"] = [] + return result + + plan = stage(setup, early_failure, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert plan.states[nt.CHEAP[0]] == "harness-stopped" + assert not final(plan, sid="h1").exists() + resumed_provider = Provider() + resumed = stage(setup, resumed_provider, models=[nt.CHEAP[0]]) + assert resumed.execute() == 1 + assert resumed_provider.calls == [] + + +def paused(setup): + plan = stage( + setup, + Provider(plant(setup, "provider_error", "AuthenticationError")), + models=[nt.CHEAP[0]], + ) + assert plan.execute() == 1 + return plan + + +def test_crash_after_archive_resumes_at_the_next_attempt_number(setup, monkeypatch): + first = paused(setup) + original = nt.os.replace + + def interrupted(source, target): + original(source, target) + if Path(target).parent.name == "attempts": + raise RuntimeError("synthetic crash after archive") + + monkeypatch.setattr(nt.os, "replace", interrupted) + with pytest.raises(RuntimeError, match="synthetic crash"): + first.archive(nt.CHEAP[0], "h2") + monkeypatch.setattr(nt.os, "replace", original) + assert not final(first).exists() + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.execute() == 0 + assert load(final(resumed))["attempts"][0]["n"] == 1 + assert " h2 2 ok stands " in resumed.out.getvalue() + + +def test_spend_in_archives_and_optional_models_carries_across_invocations(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + optional = stage(setup, models=["gpt-6-sol"]) + assert optional.execute() == 0 + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.spend.spent_usd == pytest.approx(0.3) + assert resumed.execute() == 0 + assert resumed.spend.spent_usd == pytest.approx(0.5) + + +def test_reaching_the_cap_prevents_every_new_attempt_across_invocations( + setup, monkeypatch +): + monkeypatch.setattr(nt, "NO_TOOLS_CAP", 0.1) + plan = stage(setup, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert len(list(plan.households(nt.CHEAP[0]).glob("*.json"))) == 1 + provider = Provider() + resumed = stage(setup, provider, models=[nt.CHEAP[1]]) + assert resumed.execute() == 1 + assert provider.calls == [] + assert resumed.spend.spent_usd == pytest.approx(0.1) + + +def test_budget_stop_is_shared_by_all_models(setup): + provider = Provider(plant(setup, "budget_stop", None)) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert plan.stopped.is_set() + resumed = stage(setup, Provider(), models=[nt.CHEAP[1]]) + assert resumed.execute() == 1 + assert resumed.states[nt.CHEAP[1]] == "budget-stopped" + + +@pytest.mark.parametrize( + "mutation,match", + [ + (lambda r: r.update(prior_attempts=[]), "prior_attempts"), + (lambda r: r.pop("attempts"), "attempts list"), + ], +) +def test_refuse_a_final_not_written_by_this_script(setup, mutation, match): + first = paused(setup) + path = final(first) + payload = load(path) + mutation(payload) + path.write_text(json.dumps(payload)) + with pytest.raises(nt.StateError, match=match): + stage(setup) + + +def test_refuse_an_attempt_number_gap_and_name_its_files(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + archive = final(first).parent / "attempts/h2-1.json" + archive.rename(archive.with_name("h2-2.json")) + with pytest.raises(nt.StateError, match="attempt-number gap.*h2-2.json"): + stage(setup) + + +@pytest.mark.parametrize( + "field,value", [("sha256", "wrong"), ("n", 2), ("reason", "edited")] +) +def test_refuse_a_final_with_corrupted_attempt_links(setup, field, value): + paused(setup) + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.execute() == 0 + path = final(resumed) + payload = load(path) + payload["attempts"][0][field] = value + path.write_text(json.dumps(payload)) + with pytest.raises(nt.StateError, match="attempts disagree"): + stage(setup) + + +def test_an_attempt_that_raises_keeps_its_spend(setup): + provider = Provider(plant(setup, "complete", None, **{"raise": True, "cost": 0.27})) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.execute() == 0 + archive = final(plan).parent / "attempts/h2-1.json" + assert load(archive)["status"] == "harness_error" + assert load(archive)["totals"]["cost_usd"] == 0.27 + assert plan.spend.spent_usd == pytest.approx(0.47) + assert stage(setup, models=[nt.CHEAP[0]]).spend.spent_usd == pytest.approx(0.47) + + +def test_dry_run_lists_done_households_and_next_owed_attempt_without_providers(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + provider = Provider() + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.dry_run() == 1 + text = plan.out.getvalue() + assert "done []" in text and "h2:2" in text and "h1:1" in text + assert "$0.100000" in text + assert provider.calls == [] + + +def test_each_attempt_uses_none_profile_and_refuses_a_tool_call(setup): + def fake(scenario, variables, tool_call, completion, spend, *, profile): + assert profile.arm == "none" and profile.prompt_version == nt.PROMPT_VERSION + assert variables == ["snap"] + with pytest.raises(RuntimeError, match="cannot call a tool"): + tool_call({}) + return record(scenario.id, profile, cost=0) + + assert stage(setup, fake, models=[nt.CHEAP[0]]).execute() == 0 + + +def test_registered_cap_matches_the_report_when_available(): + """The local report is separate from this public repository's dependency tree.""" + configured = os.environ.get("CONFIRMATORY_REPORT") + candidates = ( + [Path(configured)] + if configured + else [Path(__file__).resolve().parents[2] / "report" / "confirmatory_report.py"] + ) + path = next((p for p in candidates if p.is_file()), None) + if path is None: + pytest.skip("set CONFIRMATORY_REPORT to compare the separate report checkout") + tree = ast.parse(path.read_text()) + cap = next( + ast.literal_eval(node.value) + for node in tree.body + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "NO_TOOLS_CAP" + for target in node.targets + ) + ) + assert nt.NO_TOOLS_CAP == cap == 5.0 + + +def test_main_supports_multiple_model_options_and_dry_run( + setup, tmp_path, monkeypatch, capsys +): + path = tmp_path / "design.json" + path.write_text(json.dumps(setup["design"])) + monkeypatch.setattr(nt.pilot, "board_engine", lambda path: "2.60.4") + monkeypatch.setattr(nt.pilot, "_require_engine", lambda path: None) + monkeypatch.setattr(nt.pilot, "load_board", lambda path: setup["board"]) + assert ( + nt.main( + [ + "run", + str(path), + "--model", + nt.CHEAP[0], + nt.CHEAP[1], + "--model", + nt.CHEAP[2], + "--dry-run", + ] + ) + == 1 + ) + output = capsys.readouterr().out + assert all(model in output for model in nt.CHEAP) + + +def test_in_flight_models_observe_the_shared_cap_before_their_next_turn( + setup, monkeypatch +): + monkeypatch.setattr(nt, "NO_TOOLS_CAP", 0.1) + ready, charged = threading.Barrier(3), threading.Barrier(3) + calls = [] + + def fake(scenario, variables, tool_call, completion, spend, *, profile): + calls.append((profile.name, scenario.id)) + ready.wait(timeout=30) + cost = 0.1 if profile.name == nt.CHEAP[0] else 0.0 + spend.add(cost) + charged.wait(timeout=30) + assert spend.exhausted() + return record(scenario.id, profile, cost=cost) + + plan = stage(setup, fake) + assert plan.execute() == 1 + assert len(calls) == 3 and all(sid == "h2" for _, sid in calls) + assert plan.spend.spent_usd == pytest.approx(0.1) + + +def test_dry_run_marks_harness_and_budget_finals_as_blocked(setup): + plan = stage(setup, models=[nt.CHEAP[0]]) + profile = nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + payload = record("h2", profile, status="budget_stop", cost=0) + payload["attempts"] = [] + nt.write_json(final(plan), payload) + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.dry_run() == 1 + assert "h2:blocked stopped at attempt 1" in resumed.out.getvalue() From 060b28793eae03e768090ed05a9746c8bc342527 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 02:30:20 -0700 Subject: [PATCH 31/64] Confirmatory runner: wait for host-log samples in two timing tests Two host-log tests slept a fixed time and expected the 0.02 s sampler to have run in it. On a loaded host it did not: test_the_host_log_samples_and_closes got three samples where it expects four (09:26Z run at 047c8682), and test_pressure_near_an_attempt_is_host_trouble_once_settled found no sample during its 0.05 s of pressure (09:39Z run). Neither failure was a fault in the runner. A helper, wait_for_samples, now polls the log's complete lines for up to 5 seconds until they hold what each test needs: four samples, or one sample under pressure. The assertions are unchanged. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index d4e3622f..246b2533 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -350,6 +350,17 @@ def host(self, folder): ) +def wait_for_samples(log: Path, enough, seconds: float = 5.0) -> None: + """Wait until the host log's complete samples satisfy `enough`. A loaded + host can stretch the 0.02 s sampler past any fixed sleep.""" + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + text = log.read_text() if log.exists() else "" + if enough([json.loads(x) for x in text.split("\n")[:-1]]): + return + time.sleep(0.01) + + @pytest.fixture def fast(monkeypatch): """Short evidence windows, with room for the shared host's scheduler. @@ -371,7 +382,7 @@ def fast(monkeypatch): def test_the_host_log_samples_and_closes(tmp_path, fast): readers = Readers() host = readers.host(tmp_path) - time.sleep(0.1) + wait_for_samples(tmp_path / "host-log.jsonl", lambda samples: len(samples) >= 4) host.close() meta = json.loads((tmp_path / "host.json").read_text()) samples = [ @@ -390,7 +401,10 @@ def test_pressure_near_an_attempt_is_host_trouble_once_settled(tmp_path, fast): host = readers.host(tmp_path) start = time.time() readers.pressure = 2 - time.sleep(0.05) + wait_for_samples( + tmp_path / "host-log.jsonl", + lambda samples: any(x["pressure"] == 2 for x in samples), + ) readers.pressure = 1 end = time.time() host.wait_settled(end) From 08d7b186448af15aa3bc42dd03708b4034ef6305 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 03:33:43 -0700 Subject: [PATCH 32/64] Confirmatory runner: coordinate no-tools shutdown and freeze inputs Install and restore run-scoped SIGINT and SIGTERM handlers. The first signal lets active attempts save, prevents further archives and attempts, and returns a nonzero interrupted state. A second signal warns that the active attempt and its spend are lost before immediate exit. Hold an exclusive nonblocking stage-root lock through initialization, history scanning and execution, including dry runs. Require the baseline exclusions file to exist and match its declared sha256 before startup. Provide a board-free stage validator that requires every cheap-model drawn final to stand and rejects invalid histories in every model folder. Exercise archive household identity, rerun eligibility, third-attempt harness stops, both blocked dry-run classes, and the real harness turn check after another model crosses the shared cap. Regression evidence records seven expected failures before these fixes and eleven passing focused cases afterwards; the expanded no-tools file passes all 33 tests. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 134 ++++++++++++++++- tests/test_confirmatory_no_tools.py | 221 +++++++++++++++++++++++++++- 2 files changed, 348 insertions(+), 7 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index c48f06ac..d165a950 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -11,10 +11,12 @@ from __future__ import annotations import argparse +import fcntl import json import math import os import re +import signal import sys import threading from pathlib import Path @@ -125,6 +127,42 @@ def history(folder: Path) -> tuple[dict[str, dict], float]: return states, spent +def stage_problems(design: dict) -> list[str]: + """Read stage histories and standing finals without a board or provider.""" + problems = [] + try: + spec = design["no_tools"] + root = Path(spec["runs"]) + folder, stage = component(spec["folder"]), component(spec["stage"]) + draw_order = [component(sid) for sid in design["draw_order"]] + models = set(CHEAP) + if root.is_dir(): + models.update(path.name for path in root.iterdir() if path.is_dir()) + except (OSError, KeyError, TypeError, ValueError) as error: + return [f"no-tools stage: {error}"] + for model in sorted(models): + try: + states, _ = history(root / model / folder / stage / "households") + if model not in CHEAP: + continue + for sid in draw_order: + state = states.get(sid) + if state is None or state["final"] is None: + problems.append(f"no-tools {model}/{sid}: missing standing final") + continue + outcome = action( + classify(state["final"], 0.0, 0.0, None)[0], + len(state["entries"]) + 1, + ) + if outcome != "stands": + problems.append( + f"no-tools {model}/{sid}: final action is {outcome}" + ) + except (OSError, TypeError, ValueError, AttributeError, KeyError) as error: + problems.append(f"no-tools {model}: {error}") + return problems + + class Spend: """The stage's recorded spend, shared by every model and every invocation.""" @@ -184,12 +222,37 @@ def __init__( self.root = Path(spec["runs"]) self.folder, self.stage = component(spec["folder"]), component(spec["stage"]) self.out = sys.stdout if out is None else out - self._say_lock = threading.Lock() + self._say_lock = threading.RLock() self.stopped = threading.Event() + self.interrupted = threading.Event() self.completion_for = completion_for self.run_household = ( run_household if run_household is not None else pilot.run_household ) + self._lock_file = None + try: + baseline = design["baseline_exclusions"] + if sha256_file(Path(baseline["path"])) != baseline["sha256"]: + raise StateError("baseline exclusions do not hash as declared") + except (OSError, KeyError, TypeError) as error: + raise StateError(f"baseline exclusions: {error}") from error + self.root.mkdir(parents=True, exist_ok=True) + self._lock_file = (self.root / ".confirmatory-no-tools.lock").open("a") + try: + fcntl.flock(self._lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + self.close() + raise StateError( + f"another no-tools invocation holds the lock in {self.root}" + ) from error + try: + self._load_state(board) + except BaseException: + self.close() + raise + + def _load_state(self, board) -> None: + design = self.design pilot.set_stated_engine(design["engine"].split()[-1]) if board is None: board_path = Path(design["board"]) @@ -229,6 +292,22 @@ def __init__( self.spend = Spend(spent, self.stopped) self.states = {model: "pending" for model in self.models} + def close(self) -> None: + """Release the invocation lock, including after initialization failure.""" + lock = getattr(self, "_lock_file", None) + if lock is not None: + lock.close() + self._lock_file = None + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __del__(self): + self.close() + def households(self, model: str) -> Path: return self.root / model / self.folder / self.stage / "households" @@ -248,6 +327,12 @@ def standing(self, model: str, sid: str) -> bool: ) def dry_run(self) -> int: + try: + return self._dry_run() + finally: + self.close() + + def _dry_run(self) -> int: for model in self.models: done = [sid for sid in self.draw_order if self.standing(model, sid)] owed = [] @@ -302,10 +387,16 @@ def run_model(self, model: str) -> None: completion = None completion_loaded = False for sid in self.draw_order: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return state = self.histories[model].setdefault( sid, {"entries": [], "final": None} ) while True: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return if state["final"] is not None: cls, _ = classify(state["final"], 0.0, 0.0, None) outcome = action(cls, len(state["entries"]) + 1) @@ -320,6 +411,9 @@ def run_model(self, model: str) -> None: self.states[model] = "budget-stopped" return if state["final"] is not None: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return self.archive(model, sid) if not completion_loaded: completion = self.completion_for(profile) @@ -327,6 +421,9 @@ def run_model(self, model: str) -> None: if self.spend.exhausted(): self.states[model] = "budget-stopped" return + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return spend = AttemptSpend(self.spend) def no_tool(arguments): @@ -355,6 +452,9 @@ def no_tool(arguments): self.say( f"{model} {sid} {n} {cls} {outcome} ${attempt_cost(record):.6f}" ) + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return if outcome == "pause-rerun": self.states[model] = "operator-paused" return @@ -373,6 +473,29 @@ def no_tool(arguments): self.say(f"{model}: {type(error).__name__}: {error}") def execute(self) -> int: + previous = {} + + def interrupt(signum, frame): + if self.interrupted.is_set(): + self.say( + "Forced interruption: an attempt under way, and its spend, " + "are then lost." + ) + os._exit(128 + signum) + self.interrupted.set() + self.say("Interrupted: attempts under way will finish and save.") + + try: + if threading.current_thread() is threading.main_thread(): + for sig in (signal.SIGINT, signal.SIGTERM): + previous[sig] = signal.signal(sig, interrupt) + return self._execute() + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + self.close() + + def _execute(self) -> int: threads = [ threading.Thread( target=self.run_model, args=(model,), name=f"no-tools-{model}" @@ -387,7 +510,10 @@ def execute(self) -> int: self.say(f"{model}: {self.states[model]}") self.say(f"stage spend ${self.spend.spent_usd:.6f} of ${NO_TOOLS_CAP:.2f}") return ( - 0 if all(self.states[model] == "complete" for model in self.models) else 1 + 0 + if not self.interrupted.is_set() + and all(self.states[model] == "complete" for model in self.models) + else 1 ) @@ -400,8 +526,8 @@ def main(argv=None) -> int: run.add_argument("--dry-run", action="store_true") args = parser.parse_args(argv) try: - stage = Stage(json.loads(args.design.read_text()), models=args.model) - return stage.dry_run() if args.dry_run else stage.execute() + with Stage(json.loads(args.design.read_text()), models=args.model) as stage: + return stage.dry_run() if args.dry_run else stage.execute() except (StateError, OSError, ValueError, KeyError, TypeError) as error: print(f"The no-tools stage cannot start: {error}") return 1 diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 1a2b388e..7c77f876 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -6,6 +6,7 @@ import io import json import os +import signal import sys import threading from pathlib import Path @@ -21,11 +22,17 @@ @pytest.fixture def setup(tmp_path): draw = ["h2", "h1"] + baseline = tmp_path / "baseline-exclusions.json" + baseline.write_text("[]\n") return { "design": { "board": "synthetic.csv", "engine": "policyengine-us 2.60.4", "draw_order": draw, + "baseline_exclusions": { + "path": str(baseline), + "sha256": nt.sha256_file(baseline), + }, "no_tools": { "runs": str(tmp_path / "no-tools"), "folder": "none", @@ -413,14 +420,222 @@ def fake(scenario, variables, tool_call, completion, spend, *, profile): assert plan.spend.spent_usd == pytest.approx(0.1) -def test_dry_run_marks_harness_and_budget_finals_as_blocked(setup): +@pytest.mark.parametrize( + "status,error,expected", + [ + ("budget_stop", None, "stopped"), + ("provider_error", "BadRequestError: synthetic", "stop"), + ], +) +def test_dry_run_marks_harness_and_budget_finals_as_blocked( + setup, status, error, expected +): plan = stage(setup, models=[nt.CHEAP[0]]) profile = nt.pilot.model_profile( nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION ) - payload = record("h2", profile, status="budget_stop", cost=0) + payload = record("h2", profile, status=status, error=error, cost=0) + payload["turns"] = [] payload["attempts"] = [] nt.write_json(final(plan), payload) + if hasattr(plan, "close"): + plan.close() resumed = stage(setup, models=[nt.CHEAP[0]]) assert resumed.dry_run() == 1 - assert "h2:blocked stopped at attempt 1" in resumed.out.getvalue() + assert f"h2:blocked {expected} at attempt 1" in resumed.out.getvalue() + + +def test_sigint_finishes_saves_and_stops_before_the_next_household(setup): + calls = [] + saved_handlers = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + continued = threading.Event() + + def interrupted(scenario, variables, tool_call, completion, spend, *, profile): + calls.append(scenario.id) + spend.add(0.17) + if scenario.id == "h2": + os.kill(os.getpid(), signal.SIGINT) + continued.wait(0.05) + return record(scenario.id, profile, cost=0.17) + + plan = stage(setup, interrupted, models=[nt.CHEAP[0]]) + try: + result = plan.execute() + except KeyboardInterrupt: + # Let the old implementation's uncoordinated worker finish before the + # regression assertion, so a red run leaves no worker behind. + continued.set() + for worker in threading.enumerate(): + if worker.name.startswith("no-tools-"): + worker.join(timeout=30) + result = 0 + assert result != 0, "SIGINT must give a coordinated nonzero exit" + assert calls == ["h2"] + assert load(final(plan))["totals"]["cost_usd"] == 0.17 + assert plan.states[nt.CHEAP[0]] == "interrupted" + assert not plan.stopped.is_set(), "interrupts must preserve the budget event" + assert not final(plan, sid="h1").exists() + assert all( + signal.getsignal(sig) == handler for sig, handler in saved_handlers.items() + ) + assert "finish and save" in plan.out.getvalue() + + +def test_stage_lock_refuses_a_second_invocation_and_releases_after_dry_run(setup): + first = stage(setup, models=[nt.CHEAP[0]]) + with pytest.raises(nt.StateError, match="another no-tools invocation"): + stage(setup, models=[nt.CHEAP[1]]) + assert first.dry_run() == 1 + second = stage(setup, models=[nt.CHEAP[1]]) + assert second.dry_run() == 1 + + +@pytest.mark.parametrize("fault", ["missing", "hash"]) +def test_baseline_exclusions_must_exist_and_hash_before_startup(setup, fault): + spec = setup["design"]["baseline_exclusions"] + if fault == "missing": + Path(spec["path"]).unlink() + else: + spec["sha256"] = "0" * 64 + with pytest.raises(nt.StateError, match="baseline exclusions"): + stage(setup) + + +def test_archive_identity_and_rerun_eligibility_are_checked_directly(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + path = final(first).parent / "attempts/h2-1.json" + payload = load(path) + payload["scenario_id"] = "h1" + path.write_text(json.dumps(payload)) + with pytest.raises(nt.StateError, match="household differs from archive name h2"): + stage(setup) + payload.update(scenario_id="h2", status="complete", error=None) + path.write_text(json.dumps(payload)) + with pytest.raises(nt.StateError, match="archived attempt was ineligible"): + stage(setup) + + +def test_two_provider_failures_then_harness_failure_has_no_standing_final(setup): + provider = Provider( + { + **plant(setup, "provider_error", "APIConnectionError", 1), + **plant(setup, "provider_error", "APIConnectionError", 2), + **plant(setup, "provider_error", "BadRequestError", 3), + } + ) + + def failure(*args, **kwargs): + result = provider(*args, **kwargs) + if result["error"] == "BadRequestError": + result["turns"] = [] + return result + + plan = stage(setup, failure, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert plan.states[nt.CHEAP[0]] == "harness-stopped" + assert [n for _, _, n in provider.calls] == [1, 2, 3] + assert len(load(final(plan))["attempts"]) == 2 + assert not plan.standing(nt.CHEAP[0], "h2") + assert not final(plan, sid="h1").exists() + + +@pytest.mark.parametrize( + "status,error,expected", + [ + ("provider_error", "BadRequestError", "stop"), + ("budget_stop", None, "stopped"), + ], +) +def test_stage_problems_requires_every_cheap_model_final_to_stand( + setup, status, error, expected, monkeypatch +): + plan = stage(setup) + assert plan.execute() == 0 + path = final(plan, model=nt.CHEAP[-1], sid="h1") + payload = load(path) + payload.update(status=status, error=error, turns=[]) + path.write_text(json.dumps(payload)) + monkeypatch.setattr( + nt.pilot, "load_board", lambda _: pytest.fail("helper loads board") + ) + problems = nt.stage_problems(setup["design"]) + assert any(nt.CHEAP[-1] in p and "h1" in p and expected in p for p in problems) + + +def test_stage_problems_reports_missing_finals_and_invalid_optional_history(setup): + plan = stage(setup) + assert plan.execute() == 0 + assert nt.stage_problems(setup["design"]) == [] + final(plan, sid="h1").unlink() + optional = plan.households("gpt-6-sol") / "attempts/h2-2.json" + optional.parent.mkdir(parents=True) + optional.write_text("{}") + problems = nt.stage_problems(setup["design"]) + assert any(nt.CHEAP[0] in p and "h1" in p and "missing" in p for p in problems) + assert any("attempt-number gap" in p and "gpt-6-sol" in p for p in problems) + + +def test_real_harness_checks_other_models_spend_before_the_next_turn( + setup, monkeypatch +): + from policybench import eval_no_tools + + monkeypatch.setattr(nt, "NO_TOOLS_CAP", 0.1) + for scenario in setup["board"]["scenarios"].values(): + scenario.country = "us" + + def prompt(scenario, variables, profile): + return "Synthetic household", { + "_board_prompt_sha256": "synthetic", + "_sent_board_prompt_sha256": "synthetic", + "_budget_ceiling": 100, + "_answer_tool": nt.pilot.answer_tool_for( + profile, variables, scenario.country + ), + "max_tokens": 100, + } + + monkeypatch.setattr(nt.pilot, "build_initial_request", prompt) + monkeypatch.setattr( + eval_no_tools, + "_extract_usage_metadata", + lambda response, *args: {"total_cost_usd": response.synthetic_cost}, + ) + entered = threading.Barrier(2) + requests = [] + models = [nt.CHEAP[0], nt.CHEAP[2]] + first_id = nt.pilot.model_profile(models[0], arm="none").model_id + + def completion(**kwargs): + requests.append(kwargs["model"]) + entered.wait(timeout=30) + if kwargs["model"] != first_id: + assert plan.stopped.wait(timeout=30), "other model must cross shared cap" + message = SimpleNamespace( + role="assistant", content="", tool_calls=None, function_call=None + ) + return SimpleNamespace( + choices=[SimpleNamespace(message=message, finish_reason="stop")], + synthetic_cost=0.11 if kwargs["model"] == first_id else 0.0, + id="synthetic-response", + ) + + plan = nt.Stage( + setup["design"], + models=models, + board=setup["board"], + run_household=nt.pilot.run_household, + completion_for=lambda profile: completion, + out=io.StringIO(), + ) + assert plan.execute() == 1 + assert len(requests) == 2 + for model in models: + payload = load(final(plan, model=model)) + assert payload["status"] == "budget_stop", payload.get("error") + assert len(payload["turns"]) == 1 + assert not final(plan, model=model, sid="h1").exists() + assert plan.spend.spent_usd == pytest.approx(0.11) From 5a9735b0e4984ca84013dffa49b305ad273f8f32 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 03:33:52 -0700 Subject: [PATCH 33/64] Confirmatory runner: require standing no-tools finals before calculation Calculator preflight calls the no-tools history validator and refuses an unfinished stage. Every drawn household must have a standing final for every required cheap model, and every saved history must pass the writer rules. Synthetic completed no-tools histories keep existing preflight tests meaningful. Require the calculator and no-tools runs roots to be disjoint after resolving their paths. Equal roots and either direction of containment are refused. Changes to the calculator runner are confined to preflight. All seven new preflight regressions failed before the change. The complete calculator test file passes 230 tests after the change, including those regressions. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 12 ++++ tests/test_confirmatory_run.py | 103 ++++++++++++++++++++++++++++----- 2 files changed, 102 insertions(+), 13 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index d4dd254b..3949f239 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2162,6 +2162,18 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: if missing: return [f"the design has no {', '.join(missing)}"] problems = [] + from confirmatory_no_tools import stage_problems + + problems.extend(stage_problems(design)) + try: + no_tools_root = Path(design["no_tools"]["runs"]).resolve() + calculator_root = Path(design["runs"]).resolve() + if no_tools_root.is_relative_to( + calculator_root + ) or calculator_root.is_relative_to(no_tools_root): + problems.append("the no-tools and calculator runs roots overlap") + except (KeyError, TypeError, OSError, ValueError) as error: + problems.append(f"the no-tools runs root is invalid: {error}") command = design["anchor_command"] if ( not isinstance(command, list) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 246b2533..0b39c5a6 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2173,10 +2173,8 @@ def test_the_preflight_names_a_missing_field(): assert cr.preflight({}, cr.CHEAP) == [f"the design has no {', '.join(cr.REQUIRED)}"] -def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( - world, tmp_path, monkeypatch -): - w = world(models=("gpt-6-luna",)) +def complete_preflight_design(w, tmp_path): + """Each calculator preflight starts after synthetic standing no-tools finals.""" pool = tmp_path / "pool.json" pool.write_text(json.dumps([{"id": s} for s in POOL])) design = dict( @@ -2184,8 +2182,36 @@ def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, treatment_harness=str(cr.ROOT), run_commit="0" * 40, - draw_order=list(reversed(w.draw_order)), + no_tools={ + "runs": str(tmp_path / "no-tools"), + "folder": "none", + "stage": "confirm", + }, ) + for model in cr.CHEAP: + for sid in design["draw_order"]: + cr.write_json( + Path(design["no_tools"]["runs"]) + / model + / "none/confirm/households" + / f"{sid}.json", + { + "scenario_id": sid, + "status": "complete", + "turns": [{}], + "totals": {"cost_usd": 0.0}, + "attempts": [], + }, + ) + return design + + +def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( + world, tmp_path, monkeypatch +): + w = world(models=("gpt-6-luna",)) + design = complete_preflight_design(w, tmp_path) + design["draw_order"] = list(reversed(w.draw_order)) monkeypatch.setattr(cr, "host_power", lambda: (True, False)) problems = cr.preflight(design, ("gpt-6-luna",)) text = "\n".join(problems) @@ -2194,22 +2220,73 @@ def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( assert "lid open False" in text design["draw_order"] = w.draw_order assert "the draw" not in "\n".join(cr.preflight(design, ("gpt-6-luna",))) + assert not any( + p.startswith("no-tools") for p in cr.preflight(design, ("gpt-6-luna",)) + ) def test_the_preflight_refuses_a_runs_folder_with_records(world, tmp_path): w = world(models=("gpt-6-luna",)).execute() - pool = tmp_path / "pool.json" - pool.write_text(json.dumps([{"id": s} for s in POOL])) - design = dict( - w.design, - eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, - treatment_harness=str(cr.ROOT), - run_commit="0" * 40, - ) + design = complete_preflight_design(w, tmp_path) text = "\n".join(cr.preflight(design, ("gpt-6-luna",))) assert "already holds records" in text and "already holds a host log" in text +@pytest.mark.parametrize( + "fault,expected", + [ + ("missing", "missing"), + ("harness", "stop"), + ("ceiling", "stopped"), + ("history", "attempts list"), + ], +) +def test_preflight_refuses_an_unfinished_no_tools_stage( + world, tmp_path, fault, expected +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + model, sid = cr.CHEAP[-1], w.draw_order[-1] + path = ( + Path(design["no_tools"]["runs"]) + / model + / "none/confirm/households" + / f"{sid}.json" + ) + payload = json.loads(path.read_text()) + if fault == "missing": + path.unlink() + else: + if fault == "history": + payload.pop("attempts") + else: + payload.update( + status="budget_stop" if fault == "ceiling" else "provider_error", + error="BadRequestError" if fault == "harness" else None, + turns=[], + ) + path.write_text(json.dumps(payload)) + text = "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert "no-tools" in text and model in text and sid in text and expected in text + + +@pytest.mark.parametrize("relationship", ["equal", "inside", "contains"]) +def test_preflight_refuses_overlapping_no_tools_and_calculator_roots( + world, tmp_path, relationship +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + calculator = Path(design["runs"]) + design["no_tools"]["runs"] = str( + { + "equal": calculator, + "inside": calculator / "no-tools", + "contains": calculator.parent, + }[relationship] + ) + assert any("roots overlap" in p for p in cr.preflight(design, (cr.CHEAP[0],))) + + def test_resume_writes_the_operator_s_note(tmp_path): design = tmp_path / "design.json" design.write_text(json.dumps({"runs": str(tmp_path / "runs")})) From c60086b05f77a8aaa49fa732033e9884e1d34988 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 05:21:39 -0700 Subject: [PATCH 34/64] Confirmatory runner: coordinate interruption and worker cleanup Archive moves and attempt admissions share a short lock with the first signal handler, so every admission after that handler returns is refused. Providers run outside the admission lock, and admitted attempts finish and save. Exceptional shutdown interrupts and joins every started worker before restoring signal handlers or releasing the invocation lock. A second signal writes a nonblocking stderr message and exits immediately without taking shared locks; every active unsaved attempt and its spend are lost, while saved finals and archives survive. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 74 ++++++++--- tests/test_confirmatory_no_tools.py | 197 ++++++++++++++++++++++++++++ 2 files changed, 253 insertions(+), 18 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index d165a950..ffaa42e5 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -6,6 +6,14 @@ its rerun; the final links every archive by number, hash, status and reason. Re-invocation resumes solely from those files and is an operator's approval to rerun a paused account failure. Native harness resume is never used. +An archive move or attempt is admitted when it takes the admission lock and +commits before the first signal handler finishes; everything else is refused. +The lock covers only an interruption check and a rename or an admission state +change. Admitted attempts finish and save outside the admission lock. +Exceptional shutdown interrupts and joins all started workers before restoring +signal handlers or releasing the invocation lock. A second signal immediately +uses a nonblocking stderr write and exits, losing every active unsaved attempt +and its spend; saved finals and archives survive. """ from __future__ import annotations @@ -225,6 +233,9 @@ def __init__( self._say_lock = threading.RLock() self.stopped = threading.Event() self.interrupted = threading.Event() + self._admission_lock = threading.Lock() + self._admitted_attempts = {} + self._threads = [] self.completion_for = completion_for self.run_household = ( run_household if run_household is not None else pilot.run_household @@ -360,8 +371,8 @@ def _dry_run(self) -> int: else 1 ) - def archive(self, model: str, sid: str) -> None: - """Move an eligible final unchanged, then extend its reconstructed links.""" + def archive(self, model: str, sid: str) -> bool: + """Atomically admit a prepared archive move, then extend its links.""" state = self.histories[model][sid] record, entries = state["final"], state["entries"] n = len(entries) + 1 @@ -369,13 +380,20 @@ def archive(self, model: str, sid: str) -> None: if action(cls, n) not in ("rerun", "pause-rerun"): raise StateError(f"{model}/{sid}: attempt {n} is ineligible for rerun") folder = self.households(model) + source = folder / f"{sid}.json" target = folder / "attempts" / f"{sid}-{n}.json" target.parent.mkdir(parents=True, exist_ok=True) if target.exists(): raise StateError(f"{target}: would overwrite an archived attempt") - os.replace(folder / f"{sid}.json", target) - entries.append(entry(target, record, n)) - state["final"] = None + link = entry(source, record, n) + link["file"] = f"attempts/{target.name}" + with self._admission_lock: + if self.interrupted.is_set(): + return False + os.replace(source, target) + state["final"] = None + entries.append(link) + return True def run_model(self, model: str) -> None: try: @@ -414,17 +432,21 @@ def run_model(self, model: str) -> None: if self.interrupted.is_set(): self.states[model] = "interrupted" return - self.archive(model, sid) + if not self.archive(model, sid): + self.states[model] = "interrupted" + return if not completion_loaded: completion = self.completion_for(profile) completion_loaded = True if self.spend.exhausted(): self.states[model] = "budget-stopped" return - if self.interrupted.is_set(): - self.states[model] = "interrupted" - return spend = AttemptSpend(self.spend) + with self._admission_lock: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return + self._admitted_attempts[model, sid] = len(state["entries"]) + 1 def no_tool(arguments): raise RuntimeError("the no-tools arm cannot call a tool") @@ -474,15 +496,24 @@ def no_tool(arguments): def execute(self) -> int: previous = {} + first_signal_seen = False def interrupt(signum, frame): - if self.interrupted.is_set(): - self.say( - "Forced interruption: an attempt under way, and its spend, " - "are then lost." - ) + nonlocal first_signal_seen + if first_signal_seen: + try: + os.set_blocking(2, False) + os.write( + 2, + b"Forced interruption: every attempt under way, and its spend, " + b"are lost; saved finals and archives survive.\n", + ) + except OSError: + pass os._exit(128 + signum) - self.interrupted.set() + first_signal_seen = True + with self._admission_lock: + self.interrupted.set() self.say("Interrupted: attempts under way will finish and save.") try: @@ -490,21 +521,28 @@ def interrupt(signum, frame): for sig in (signal.SIGINT, signal.SIGTERM): previous[sig] = signal.signal(sig, interrupt) return self._execute() + except BaseException: + with self._admission_lock: + self.interrupted.set() + for thread in self._threads: + if thread.ident is not None: + thread.join() + raise finally: for sig, handler in previous.items(): signal.signal(sig, handler) self.close() def _execute(self) -> int: - threads = [ + self._threads = [ threading.Thread( target=self.run_model, args=(model,), name=f"no-tools-{model}" ) for model in self.models ] - for thread in threads: + for thread in self._threads: thread.start() - for thread in threads: + for thread in self._threads: thread.join() for model in self.models: self.say(f"{model}: {self.states[model]}") diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 7c77f876..55e3d0c5 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -639,3 +639,200 @@ def completion(**kwargs): assert len(payload["turns"]) == 1 assert not final(plan, model=model, sid="h1").exists() assert plan.spend.spent_usd == pytest.approx(0.11) + + +def track_interrupt_return(monkeypatch): + """Observe the actual installed first-signal handler returning.""" + returned = threading.Event() + register = signal.signal + + def install(signum, handler): + if callable(handler) and handler.__name__ == "interrupt": + + def tracked(sig, frame): + handler(sig, frame) + returned.set() + + return register(signum, tracked) + return register(signum, handler) + + monkeypatch.setattr(signal, "signal", install) + return returned + + +@pytest.mark.parametrize("boundary", ["archive", "attempt"]) +def test_signal_between_check_and_action_refuses_new_admission( + setup, monkeypatch, boundary +): + """R1: a real SIGINT returns before the proposed action reaches admission.""" + if boundary == "archive": + paused(setup) + provider = Provider() + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + returned = track_interrupt_return(monkeypatch) + + def signal_and_wait(): + os.kill(os.getpid(), signal.SIGINT) + assert returned.wait(timeout=10), "the signal handler must finish" + + if boundary == "archive": + original = plan.archive + + def archive(*args): + signal_and_wait() + return original(*args) + + monkeypatch.setattr(plan, "archive", archive) + else: + original = nt.AttemptSpend + + def attempt_spend(*args): + signal_and_wait() + return original(*args) + + monkeypatch.setattr(nt, "AttemptSpend", attempt_spend) + assert plan.execute() == 1 + assert returned.is_set() + assert provider.calls == [], "R1: no provider attempt begins after interruption" + archives = plan.households(nt.CHEAP[0]) / "attempts" + assert not list(archives.glob("*.json")), "R1: no archive moves after interruption" + + +def test_failed_second_thread_start_keeps_invocation_lock_until_worker_finishes( + setup, monkeypatch +): + """R2: exceptional shutdown waits for every worker before closing its lock.""" + entered, release, failed_start, closed = (threading.Event() for _ in range(4)) + original_start = threading.Thread.start + original_close = nt.Stage.close + observed = {} + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + entered.set() + assert release.wait(timeout=10) + return record(scenario.id, profile) + + plan = stage(setup, provider) + + def start(thread): + if thread.name == f"no-tools-{nt.CHEAP[1]}": + assert entered.wait(timeout=10) + failed_start.set() + raise RuntimeError("synthetic second thread start failure") + return original_start(thread) + + def close(current): + original_close(current) + if current is plan: + closed.set() + + def observe(): + assert failed_start.wait(timeout=10) + observed["closed_while_live"] = closed.wait(timeout=0.2) + lock = (plan.root / ".confirmatory-no-tools.lock").open("a") + try: + import fcntl + + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + observed["held"] = True + else: + observed["held"] = False + finally: + lock.close() + release.set() + + watcher = threading.Thread(target=observe) + watcher.start() + monkeypatch.setattr(threading.Thread, "start", start) + monkeypatch.setattr(nt.Stage, "close", close) + try: + with pytest.raises(RuntimeError, match="second thread start"): + plan.execute() + finally: + release.set() + watcher.join(timeout=10) + for worker in threading.enumerate(): + if worker.name.startswith("no-tools-"): + worker.join(timeout=10) + assert observed["held"], "R2: a live worker must retain the invocation lock" + assert not observed["closed_while_live"], "R2: cleanup joins before unlocking" + assert plan.interrupted.is_set(), "R2: exceptional shutdown refuses further work" + assert plan._lock_file is None + + +def test_second_signal_exits_while_output_lock_is_held(setup, tmp_path): + """R4: forced termination is independent of the worker-shared output lock.""" + import select + import subprocess + import textwrap + + probe = tmp_path / "forced_exit.py" + probe.write_text( + textwrap.dedent("""\ + import io + import json + import os + import signal + import sys + import threading + import time + from types import SimpleNamespace + import confirmatory_no_tools as nt + + design = json.loads(sys.argv[1]) + draw = design['draw_order'] + board = { + 'scenarios': {sid: SimpleNamespace(id=sid) for sid in draw}, + 'variables': {sid: ['snap'] for sid in draw}, + } + held = threading.Event() + plan = nt.Stage(design, models=[nt.CHEAP[0]], board=board, + completion_for=lambda profile: None, out=io.StringIO()) + + def worker(model): + with plan._say_lock: + held.set() + time.sleep(30) + + plan.run_model = worker + + def signals(): + assert held.wait(10) + os.write(1, b'ready\\n') + os.kill(os.getpid(), signal.SIGINT) + time.sleep(0.1) + os.kill(os.getpid(), signal.SIGTERM) + + threading.Thread(target=signals, daemon=True).start() + plan.execute() + raise AssertionError('forced exit returned') + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + process = subprocess.Popen( + [sys.executable, str(probe), json.dumps(setup["design"])], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + try: + ready, _, _ = select.select([process.stdout], [], [], 90) + assert ready, "R4: subprocess must reach output contention before signals" + assert process.stdout.readline() == b"ready\n" + stdout, stderr = process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + pytest.fail("R4: the second signal blocked on the held output lock") + finally: + if process.poll() is None: + process.kill() + process.communicate() + assert process.returncode == 128 + signal.SIGTERM, (stdout, stderr) + assert b"every attempt under way" in stderr + assert b"saved finals and archives survive" in stderr From 248e777abddd83ff62637b8efc79eeb50a76d65d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 05:21:44 -0700 Subject: [PATCH 35/64] Confirmatory runner: validate every saved final and baseline inventory The shared history reader classifies every saved final, including optional models and households outside the draw, before initialization and calculator preflight accept its history. Preflight then requires every drawn cheap-model final to stand. Startup validates the hash-bound baseline as an object containing an exclusions list, with unique household and output cells identified by nonempty strings, before any provider call. The empty baseline fixture uses the report schema. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 33 ++++++++++++++++++++-- tests/test_confirmatory_no_tools.py | 43 ++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index ffaa42e5..ac189818 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -130,6 +130,12 @@ def history(folder: Path) -> tuple[dict[str, dict], float]: raise StateError(f"{path}: household differs from final name {sid}") if final["attempts"] != entries: raise StateError(f"{path}: attempts disagree with archived files") + try: + classify(final, 0.0, 0.0, None) + except (TypeError, ValueError, AttributeError, KeyError) as error: + raise StateError( + f"{path}: cannot classify saved final: {error}" + ) from error spent += attempt_cost(final) states[sid] = {"entries": entries, "final": final} return states, spent @@ -243,9 +249,32 @@ def __init__( self._lock_file = None try: baseline = design["baseline_exclusions"] - if sha256_file(Path(baseline["path"])) != baseline["sha256"]: + if not isinstance(baseline, dict) or not isinstance( + baseline.get("path"), str + ): + raise StateError("baseline exclusions need path and sha256") + baseline_path = Path(baseline["path"]) + if sha256_file(baseline_path) != baseline.get("sha256"): raise StateError("baseline exclusions do not hash as declared") - except (OSError, KeyError, TypeError) as error: + payload = json.loads(baseline_path.read_text()) + exclusions = ( + payload.get("exclusions") if isinstance(payload, dict) else None + ) + if not isinstance(exclusions, list): + raise StateError("baseline exclusions must contain an exclusions list") + cells = [] + for exclusion in exclusions: + if not isinstance(exclusion, dict) or any( + not isinstance(exclusion.get(key), str) or not exclusion[key] + for key in ("scenario_id", "variable") + ): + raise StateError( + "baseline exclusion needs household and output strings" + ) + cells.append((exclusion["scenario_id"], exclusion["variable"])) + if len(cells) != len(set(cells)): + raise StateError("baseline exclusions repeat a cell") + except (OSError, KeyError, TypeError, ValueError) as error: raise StateError(f"baseline exclusions: {error}") from error self.root.mkdir(parents=True, exist_ok=True) self._lock_file = (self.root / ".confirmatory-no-tools.lock").open("a") diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 55e3d0c5..e034d000 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -23,7 +23,7 @@ def setup(tmp_path): draw = ["h2", "h1"] baseline = tmp_path / "baseline-exclusions.json" - baseline.write_text("[]\n") + baseline.write_text('{"exclusions": []}\n') return { "design": { "board": "synthetic.csv", @@ -836,3 +836,44 @@ def signals(): assert process.returncode == 128 + signal.SIGTERM, (stdout, stderr) assert b"every attempt under way" in stderr assert b"saved finals and archives survive" in stderr + + +@pytest.mark.parametrize("category", ["optional", "outside-draw"]) +def test_stage_problems_classifies_every_saved_final(setup, category): + """R3: optional and off-draw finals use the initialization classifier.""" + plan = stage(setup) + assert plan.execute() == 0 + model = "gpt-6-sol" if category == "optional" else nt.CHEAP[0] + sid = "h2" if category == "optional" else "outside-draw" + profile = nt.pilot.model_profile( + model, arm="none", prompt_version=nt.PROMPT_VERSION + ) + payload = record(sid, profile) + payload.update(attempts=[], turns=[None]) + path = plan.households(model) / f"{sid}.json" + nt.write_json(path, payload) + with pytest.raises((nt.StateError, AttributeError), match="get"): + stage(setup) + problems = nt.stage_problems(setup["design"]) + assert any(model in problem and "get" in problem for problem in problems), ( + "R3: every final refused by initialization must fail stage preflight", + problems, + ) + + +@pytest.mark.parametrize( + "payload", ["[]", "{invalid json", '{"exclusions": [{"scenario_id": "h2"}]}'] +) +def test_baseline_schema_is_usable_before_any_provider_call(setup, payload): + """R5: correctly hashed malformed inventories refuse without provider calls.""" + spec = setup["design"]["baseline_exclusions"] + path = Path(spec["path"]) + path.write_text(payload) + spec["sha256"] = nt.sha256_file(path) + provider = Provider() + try: + with pytest.raises(nt.StateError, match="baseline exclusion"): + plan = stage(setup, provider) + plan.close() + finally: + assert provider.calls == [], "R5: schema checks precede provider calls" From 4ce7f8ef93209bb23c9c8f4eba8f8a2bbcf3ab9f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 05:23:44 -0700 Subject: [PATCH 36/64] Confirmatory runner: keep exceptional cleanup signal-safe Exceptional cleanup sets interruption directly before joining every started worker. Keeping the main thread outside the admission lock lets a first signal safely acquire that lock if it interrupts cleanup. The invocation lock remains held until every worker finishes and saves. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index ac189818..dcd52029 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -551,8 +551,9 @@ def interrupt(signum, frame): previous[sig] = signal.signal(sig, interrupt) return self._execute() except BaseException: - with self._admission_lock: - self.interrupted.set() + # The main thread must remain outside admission here: a first signal + # can interrupt cleanup and its handler must take that same lock. + self.interrupted.set() for thread in self._threads: if thread.ident is not None: thread.join() From 2f795eae44a9dd65514f00ccf5b60fddbeb052c3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 23:54:32 -0400 Subject: [PATCH 37/64] Confirmatory runner: defer signals and track worker completion Record signals and wake a pipe watcher with a handler that takes no lock and raises nothing. The watcher sets interruption under admission and announces admitted work after releasing the lock. Archive moves and attempts commit under the same lock before interruption; admitted attempts finish and save outside it. Track worker began and done events independently of Thread identity and join bookkeeping. Escaping exceptions retain handlers and the invocation lock through completion, allow ten seconds for uncertain startup, tolerate repeated wait exceptions and preserve the original exception. Late targets return before file access. Apply completion tracking and safe late entry to the watcher as well. A caught model exception marks that model error and leaves global interruption unset. A second signal exits with 128 plus its number and preserves saved finals, archives and their recorded spend. Every unsaved attempt then under way is lost with its spend. A signal before handler restoration requires graceful exit 1 even when all workers completed. Run signal and cleanup regressions in bounded subprocesses, synchronize through events and use a thirty-second forced-exit deadline. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 199 +++- tests/test_confirmatory_no_tools.py | 1484 ++++++++++++++++++++++++--- 2 files changed, 1519 insertions(+), 164 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index dcd52029..0f19a730 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -6,14 +6,22 @@ its rerun; the final links every archive by number, hash, status and reason. Re-invocation resumes solely from those files and is an operator's approval to rerun a paused account failure. Native harness resume is never used. -An archive move or attempt is admitted when it takes the admission lock and -commits before the first signal handler finishes; everything else is refused. -The lock covers only an interruption check and a rename or an admission state -change. Admitted attempts finish and save outside the admission lock. -Exceptional shutdown interrupts and joins all started workers before restoring -signal handlers or releasing the invocation lock. A second signal immediately -uses a nonblocking stderr write and exits, losing every active unsaved attempt -and its spend; saved finals and archives survive. +An archive move or attempt is admitted when its interruption check and commit +under the admission lock precede the watcher setting the interrupt under that +same lock. After that interrupt is set, no archive move or attempt is admitted. +The lock covers only the check and rename or admission state change; admitted +attempts finish and save outside it. The first SIGINT or SIGTERM only records +its number and wakes the watcher through a nonblocking pipe. The handler takes +no lock and raises nothing. The watcher sets the interrupt, releases admission, +and then announces that attempts already admitted will finish and save. +An exception escaping execute interrupts and waits for script-owned worker +completion events before restoring handlers or releasing the invocation lock. +An exception caught in a model thread instead marks that model error without +setting the interrupt. A second signal immediately uses a nonblocking stderr +write and exits, losing every unsaved attempt then under way with its spend; +saved finals, archives and their recorded spend survive. Any signal before +handler restoration requires exit 1 on graceful completion, even if every +worker completed. """ from __future__ import annotations @@ -27,6 +35,7 @@ import signal import sys import threading +import time from pathlib import Path # confirmatory_run pins the harness and policybench to this checkout. @@ -44,6 +53,7 @@ ) NO_TOOLS_CAP = 5.0 +START_GRACE_SECONDS = 10 class StateError(ValueError): @@ -525,54 +535,189 @@ def no_tool(arguments): def execute(self) -> int: previous = {} - first_signal_seen = False + signals = [] + closing = False + watcher = None + watcher_done = threading.Event() + watcher_began = threading.Event() + watcher_started = False + failure = None + read_fd = write_fd = None + self._started_workers = [] + self._starting_worker = None + self._worker_began = {} + self._worker_done = {} + + def wake_watcher(): + try: + os.write(write_fd, b"!") + except OSError: + pass def interrupt(signum, frame): - nonlocal first_signal_seen - if first_signal_seen: + signals.append(signum) + if len(signals) >= 2: try: os.set_blocking(2, False) os.write( 2, - b"Forced interruption: every attempt under way, and its spend, " - b"are lost; saved finals and archives survive.\n", + b"Forced interruption: every unsaved attempt then under way " + b"is lost, with its spend; saved finals, archives and the " + b"spend they record survive.\n", ) except OSError: pass os._exit(128 + signum) - first_signal_seen = True - with self._admission_lock: - self.interrupted.set() - self.say("Interrupted: attempts under way will finish and save.") + wake_watcher() + + def stop_admission(): + while True: + try: + with self._admission_lock: + self.interrupted.set() + return + except BaseException: + continue + + def watch_signals(): + watcher_began.set() + notified = False + try: + if closing: + return + while True: + os.read(read_fd, 1) + if closing: + return + if signals and not notified: + with self._admission_lock: + self.interrupted.set() + notified = True + self.say( + "Interrupted: attempts already admitted " + "will finish and save." + ) + finally: + watcher_done.set() try: + read_fd, write_fd = os.pipe() + os.set_blocking(write_fd, False) if threading.current_thread() is threading.main_thread(): for sig in (signal.SIGINT, signal.SIGTERM): previous[sig] = signal.signal(sig, interrupt) - return self._execute() - except BaseException: - # The main thread must remain outside admission here: a first signal - # can interrupt cleanup and its handler must take that same lock. - self.interrupted.set() - for thread in self._threads: - if thread.ident is not None: - thread.join() + watcher = threading.Thread( + target=watch_signals, name="no-tools-signal-watcher" + ) + watcher.start() + watcher_started = True + result = self._execute() + except BaseException as error: + failure = error + # SIGINT/SIGTERM handlers never raise, so production waits are not + # interrupted by them. Tolerant waits defend against exceptions from + # elsewhere (for example a test SIGUSR1 handler raising KeyboardInterrupt). + stop_admission() + self._wait_for_workers() raise finally: + closing = True + if write_fd is not None: + wake_watcher() + shutdown_error = None + if watcher is not None: + # start() can raise after native startup, even before ident is set. + # A watcher beginning after grace sees closing before reading a pipe. + deadline = time.monotonic() + START_GRACE_SECONDS + while not watcher_started and not watcher_began.is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + watcher_began.wait(remaining) + except BaseException as error: + if shutdown_error is None: + shutdown_error = error + if watcher_started or watcher_began.is_set(): + while True: + try: + watcher_done.wait() + watcher.join() + break + except BaseException as error: + if shutdown_error is None: + shutdown_error = error + if failure is None and shutdown_error is not None: + stop_admission() + for fd in (read_fd, write_fd): + if fd is not None: + os.close(fd) for sig, handler in previous.items(): signal.signal(sig, handler) self.close() + if failure is None and shutdown_error is not None: + raise shutdown_error + return 1 if signals else result + + def _wait_for_workers(self) -> None: + """Retain protection through completion despite exceptional wait interruptions. + + The installed SIGINT and SIGTERM handlers never raise. These tolerant + waits defend against an exception from elsewhere, such as a test-only + SIGUSR1 handler that raises KeyboardInterrupt, and preserve the original + exception escaping execute. + """ + workers = list(self._started_workers) + starting = self._starting_worker + if starting is not None and starting not in workers: + model = self.models[self._threads.index(starting)] + deadline = time.monotonic() + START_GRACE_SECONDS + while not self._worker_began[model].is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + self._worker_began[model].wait(remaining) + except BaseException: + continue + if self._worker_began[model].is_set(): + workers.append(starting) + for thread in workers: + model = self.models[self._threads.index(thread)] + while True: + try: + self._worker_done[model].wait() + thread.join() + break + except BaseException: + continue + + def _run_worker(self, model: str) -> None: + self._worker_began[model].set() + try: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return + self.run_model(model) + finally: + self._worker_done[model].set() def _execute(self) -> int: + self._worker_began = {model: threading.Event() for model in self.models} + self._worker_done = {model: threading.Event() for model in self.models} self._threads = [ threading.Thread( - target=self.run_model, args=(model,), name=f"no-tools-{model}" + target=self._run_worker, args=(model,), name=f"no-tools-{model}" ) for model in self.models ] for thread in self._threads: + self._starting_worker = thread thread.start() - for thread in self._threads: + self._started_workers.append(thread) + self._starting_worker = None + for model, thread in zip(self.models, self._threads): + self._worker_done[model].wait() thread.join() for model in self.models: self.say(f"{model}: {self.states[model]}") diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index e034d000..08620921 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -445,42 +445,48 @@ def test_dry_run_marks_harness_and_budget_finals_as_blocked( assert f"h2:blocked {expected} at attempt 1" in resumed.out.getvalue() -def test_sigint_finishes_saves_and_stops_before_the_next_household(setup): - calls = [] - saved_handlers = { - sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) - } - continued = threading.Event() +def test_sigint_finishes_saves_and_stops_before_the_next_household(setup, tmp_path): + run_no_tools_probe( + setup, + tmp_path, + "finish_admitted_attempt", + r""" + calls = [] + saved_handlers = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } - def interrupted(scenario, variables, tool_call, completion, spend, *, profile): - calls.append(scenario.id) - spend.add(0.17) - if scenario.id == "h2": - os.kill(os.getpid(), signal.SIGINT) - continued.wait(0.05) - return record(scenario.id, profile, cost=0.17) - plan = stage(setup, interrupted, models=[nt.CHEAP[0]]) - try: + def interrupted( + scenario, variables, tool_call, completion, spend, *, profile + ): + calls.append(scenario.id) + spend.add(0.17) + if scenario.id == "h2": + os.kill(os.getpid(), signal.SIGINT) + assert plan.interrupted.wait(30), ( + "SIGINT must set the interruption event" + ) + return record(scenario.id, profile, cost=0.17) + + + plan = stage(setup, interrupted, models=[nt.CHEAP[0]]) result = plan.execute() - except KeyboardInterrupt: - # Let the old implementation's uncoordinated worker finish before the - # regression assertion, so a red run leaves no worker behind. - continued.set() - for worker in threading.enumerate(): - if worker.name.startswith("no-tools-"): - worker.join(timeout=30) - result = 0 - assert result != 0, "SIGINT must give a coordinated nonzero exit" - assert calls == ["h2"] - assert load(final(plan))["totals"]["cost_usd"] == 0.17 - assert plan.states[nt.CHEAP[0]] == "interrupted" - assert not plan.stopped.is_set(), "interrupts must preserve the budget event" - assert not final(plan, sid="h1").exists() - assert all( - signal.getsignal(sig) == handler for sig, handler in saved_handlers.items() + assert result != 0, "SIGINT must give a coordinated nonzero exit" + assert calls == ["h2"] + assert load(final(plan))["totals"]["cost_usd"] == 0.17 + assert plan.states[nt.CHEAP[0]] == "interrupted" + assert not plan.stopped.is_set(), ( + "interrupts must preserve the budget event" + ) + assert not final(plan, sid="h1").exists() + assert all( + signal.getsignal(sig) == handler + for sig, handler in saved_handlers.items() + ) + assert "finish and save" in plan.out.getvalue() + """, ) - assert "finish and save" in plan.out.getvalue() def test_stage_lock_refuses_a_second_invocation_and_releases_after_dry_run(setup): @@ -662,110 +668,146 @@ def tracked(sig, frame): @pytest.mark.parametrize("boundary", ["archive", "attempt"]) def test_signal_between_check_and_action_refuses_new_admission( - setup, monkeypatch, boundary + setup, tmp_path, boundary ): - """R1: a real SIGINT returns before the proposed action reaches admission.""" - if boundary == "archive": - paused(setup) - provider = Provider() - plan = stage(setup, provider, models=[nt.CHEAP[0]]) - returned = track_interrupt_return(monkeypatch) + """R1: the watcher interrupts before the proposed action reaches admission.""" + run_no_tools_probe( + setup, + tmp_path, + f"refuse_{boundary}_admission", + f" boundary = {boundary!r}\n" + + r""" + if boundary == "archive": + paused(setup) + provider = Provider() + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + returned = track_interrupt_return(monkeypatch) + + + def signal_and_wait(): + os.kill(os.getpid(), signal.SIGINT) + assert returned.wait(30), "the signal handler must finish" + assert plan.interrupted.wait(30), ( + "the watcher must set the interrupt before admission" + ) - def signal_and_wait(): - os.kill(os.getpid(), signal.SIGINT) - assert returned.wait(timeout=10), "the signal handler must finish" - if boundary == "archive": - original = plan.archive + if boundary == "archive": + original = plan.archive - def archive(*args): - signal_and_wait() - return original(*args) + def archive(*args): + signal_and_wait() + return original(*args) - monkeypatch.setattr(plan, "archive", archive) - else: - original = nt.AttemptSpend + monkeypatch.setattr(plan, "archive", archive) + else: + original = nt.AttemptSpend - def attempt_spend(*args): - signal_and_wait() - return original(*args) + def attempt_spend(*args): + signal_and_wait() + return original(*args) - monkeypatch.setattr(nt, "AttemptSpend", attempt_spend) - assert plan.execute() == 1 - assert returned.is_set() - assert provider.calls == [], "R1: no provider attempt begins after interruption" - archives = plan.households(nt.CHEAP[0]) / "attempts" - assert not list(archives.glob("*.json")), "R1: no archive moves after interruption" + monkeypatch.setattr(nt, "AttemptSpend", attempt_spend) + assert plan.execute() == 1 + assert returned.is_set() + assert provider.calls == [], ( + "R1: no provider attempt begins after interruption" + ) + archives = plan.households(nt.CHEAP[0]) / "attempts" + assert not list(archives.glob("*.json")), ( + "R1: no archive moves after interruption" + ) + """, + ) def test_failed_second_thread_start_keeps_invocation_lock_until_worker_finishes( - setup, monkeypatch + setup, tmp_path ): """R2: exceptional shutdown waits for every worker before closing its lock.""" - entered, release, failed_start, closed = (threading.Event() for _ in range(4)) - original_start = threading.Thread.start - original_close = nt.Stage.close - observed = {} - - def provider(scenario, variables, tool_call, completion, spend, *, profile): - entered.set() - assert release.wait(timeout=10) - return record(scenario.id, profile) - - plan = stage(setup, provider) - - def start(thread): - if thread.name == f"no-tools-{nt.CHEAP[1]}": - assert entered.wait(timeout=10) - failed_start.set() - raise RuntimeError("synthetic second thread start failure") - return original_start(thread) - - def close(current): - original_close(current) - if current is plan: - closed.set() - - def observe(): - assert failed_start.wait(timeout=10) - observed["closed_while_live"] = closed.wait(timeout=0.2) - lock = (plan.root / ".confirmatory-no-tools.lock").open("a") - try: - import fcntl + run_no_tools_probe( + setup, + tmp_path, + "failed_second_start", + r""" + import fcntl + + entered, release, failed_start, closed = ( + threading.Event() for _ in range(4) + ) + original_start = threading.Thread.start + original_close = nt.Stage.close + observed = {} + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + entered.set() + assert release.wait(timeout=30) + return record(scenario.id, profile) + + + plan = stage(setup, provider) + monkeypatch.setattr(nt, "START_GRACE_SECONDS", 0.05, raising=False) + + + def start(thread): + if thread.name == f"no-tools-{nt.CHEAP[1]}": + assert entered.wait(timeout=30) + failed_start.set() + raise RuntimeError("synthetic second thread start failure") + return original_start(thread) + + def close(current): + original_close(current) + if current is plan: + closed.set() + + + def observe(): + assert failed_start.wait(timeout=30) + observed["closed_while_live"] = closed.wait(timeout=0.2) + lock = (plan.root / ".confirmatory-no-tools.lock").open("a") try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError: - observed["held"] = True - else: - observed["held"] = False + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + observed["held"] = True + else: + observed["held"] = False + finally: + lock.close() + release.set() + + + watcher = threading.Thread(target=observe) + watcher.start() + monkeypatch.setattr(threading.Thread, "start", start) + monkeypatch.setattr(nt.Stage, "close", close) + try: + with pytest.raises(RuntimeError, match="second thread start"): + plan.execute() finally: - lock.close() release.set() - - watcher = threading.Thread(target=observe) - watcher.start() - monkeypatch.setattr(threading.Thread, "start", start) - monkeypatch.setattr(nt.Stage, "close", close) - try: - with pytest.raises(RuntimeError, match="second thread start"): - plan.execute() - finally: - release.set() - watcher.join(timeout=10) - for worker in threading.enumerate(): - if worker.name.startswith("no-tools-"): - worker.join(timeout=10) - assert observed["held"], "R2: a live worker must retain the invocation lock" - assert not observed["closed_while_live"], "R2: cleanup joins before unlocking" - assert plan.interrupted.is_set(), "R2: exceptional shutdown refuses further work" - assert plan._lock_file is None + watcher.join(timeout=30) + assert not watcher.is_alive(), "R2: independent lock probe finishes" + assert observed["held"], "R2: a live worker must retain the invocation lock" + assert not observed["closed_while_live"], ( + "R2: cleanup joins before unlocking" + ) + assert plan.interrupted.is_set(), ( + "R2: exceptional shutdown refuses further work" + ) + assert plan._lock_file is None + assert load(final(plan))["status"] == "complete", ( + "R2: admitted worker saves before cleanup returns" + ) + """, + ) def test_second_signal_exits_while_output_lock_is_held(setup, tmp_path): """R4: forced termination is independent of the worker-shared output lock.""" - import select - import subprocess import textwrap probe = tmp_path / "forced_exit.py" @@ -802,7 +844,7 @@ def signals(): assert held.wait(10) os.write(1, b'ready\\n') os.kill(os.getpid(), signal.SIGINT) - time.sleep(0.1) + assert plan.interrupted.wait(30) os.kill(os.getpid(), signal.SIGTERM) threading.Thread(target=signals, daemon=True).start() @@ -814,28 +856,17 @@ def signals(): env["PYTHONPATH"] = os.pathsep.join( [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] ) - process = subprocess.Popen( - [sys.executable, str(probe), json.dumps(setup["design"])], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"])], env=env, + ready_marker=b"ready\n", ) - try: - ready, _, _ = select.select([process.stdout], [], [], 90) - assert ready, "R4: subprocess must reach output contention before signals" - assert process.stdout.readline() == b"ready\n" - stdout, stderr = process.communicate(timeout=5) - except subprocess.TimeoutExpired: - process.kill() - stdout, stderr = process.communicate() - pytest.fail("R4: the second signal blocked on the held output lock") - finally: - if process.poll() is None: - process.kill() - process.communicate() - assert process.returncode == 128 + signal.SIGTERM, (stdout, stderr) - assert b"every attempt under way" in stderr - assert b"saved finals and archives survive" in stderr + stdout, stderr = result.stdout, result.stderr + assert b"ready\n" in stdout, (stdout, stderr) + assert result.returncode == 128 + signal.SIGTERM, (stdout, stderr) + assert b"every unsaved attempt then under way is lost, with its spend" in stderr + assert b"saved finals, archives and the spend they record survive" in stderr @pytest.mark.parametrize("category", ["optional", "outside-draw"]) @@ -877,3 +908,1182 @@ def test_baseline_schema_is_usable_before_any_provider_call(setup, payload): plan.close() finally: assert provider.calls == [], "R5: schema checks precede provider calls" + + +def run_no_tools_probe(setup, tmp_path, name, body): + """Bound a signal or worker-shutdown regression to its own process.""" + import textwrap + + prelude = r""" + import io + import json + import os + import runpy + import signal + import sys + import threading + from types import SimpleNamespace + import pytest + + helpers = runpy.run_path(sys.argv[2]) + for helper in ( + "nt", + "record", + "Provider", + "stage", + "final", + "load", + "paused", + "track_interrupt_return", + ): + globals()[helper] = helpers[helper] + design = json.loads(sys.argv[1]) + setup = { + "design": design, + "board": { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + }, + } + monkeypatch = pytest.MonkeyPatch() + """ + probe = tmp_path / f"{name}.py" + probe.write_text(textwrap.dedent(prelude) + "\n" + textwrap.dedent(body)) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), str(Path(__file__))], + env=env, + ) + assert result.returncode == 0, (result.stdout, result.stderr) + + +def _run_cleanup_regression(setup, tmp_path, case): + """Run every exceptional-cleanup signal probe behind a process deadline.""" + import textwrap + + probe = tmp_path / "cleanup_regression.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import subprocess + import sys + import threading + import time + from types import SimpleNamespace + import confirmatory_no_tools as nt + + nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + print("cleanup-regression-ready", flush=True) + design, case = json.loads(sys.argv[1]), sys.argv[2] + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + model = nt.CHEAP[0] + entered, release, saved, worker_returned = ( + threading.Event() for _ in range(4) + ) + cleanup_entered, closed = threading.Event(), threading.Event() + handler_returned = threading.Event() + order, observed, calls = [], {}, [] + original_handlers = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_start = threading.Thread.start + original_signal = signal.signal + original_close = nt.Stage.close + original_write = nt.write_json + main_ident = threading.get_ident() + nt.START_GRACE_SECONDS = 0.05 + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + calls.append((profile.name, scenario.id)) + entered.set() + assert release.wait(20), "cleanup rule: release the admitted worker" + spend.add(0.1) + return { + "scenario_id": scenario.id, + "model": profile.name, + "arm": profile.arm, + "status": "complete", + "error": None, + "turns": [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": 0.1}, + } + + + models = list(nt.CHEAP[:2]) if case == "condition-signal" else [model] + plan = nt.Stage( + design, + models=models, + board=board, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) + original_model = plan.run_model + + + def run_model(name): + try: + return original_model(name) + finally: + worker_returned.set() + + + plan.run_model = run_model + + + def write(path, payload, *args, **kwargs): + result = original_write(path, payload, *args, **kwargs) + if path == plan.households(model) / "h2.json": + order.append("saved") + saved.set() + return result + + + nt.write_json = write + + + def close(current): + if current is plan and current._lock_file is not None: + order.append("close") + closed.set() + return original_close(current) + + + nt.Stage.close = close + + + def register(sig, handler): + if sig in original_handlers and handler == original_handlers[sig]: + order.append("restore:" + signal.Signals(sig).name) + elif sig in original_handlers and callable(handler): + original = handler + + def tracked(signum, frame): + original(signum, frame) + handler_returned.set() + + handler = tracked + return original_signal(sig, handler) + + + signal.signal = register + + + def lock_held(): + # A separate process opens the lock independently of the worker. + probe = ( + "import fcntl,sys; f=open(sys.argv[1],'a'); " + "\ntry: fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB)" + "\nexcept BlockingIOError: print('held')" + "\nelse: print('released')" + ) + result = subprocess.run( + [ + sys.executable, + "-c", + probe, + str(plan.root / ".confirmatory-no-tools.lock"), + ], + capture_output=True, + text=True, + timeout=10, + check=True, + ) + return result.stdout.strip() == "held" + + + if case == "condition-signal": + + class InterruptInsideSet(threading.Event): + def set(self): + if threading.get_ident() == main_ident and not hasattr( + self, "injected" + ): + self.injected = True + with self._cond: + cleanup_entered.set() + os.kill(os.getpid(), signal.SIGINT) + assert handler_returned.wait(10), ( + "cleanup rule: handler returns inside Event.set" + ) + self._flag = True + self._cond.notify_all() + return + super().set() + + plan.interrupted = InterruptInsideSet() + + def start(thread): + if thread.name == "no-tools-" + nt.CHEAP[1]: + assert entered.wait(10) + raise RuntimeError("synthetic second start failure") + return original_start(thread) + + def observe(): + assert cleanup_entered.wait(10) + assert handler_returned.wait(10) + observed["held"] = lock_held() + observed["closed_while_live"] = closed.is_set() + release.set() + + actor = threading.Thread(target=observe, daemon=True) + actor.start() + threading.Thread.start = start + try: + plan.execute() + except RuntimeError as error: + assert str(error) == "synthetic second start failure" + observed["propagated"] = True + propagated_error = error + else: + raise AssertionError( + "cleanup rule: original start exception propagates" + ) + actor.join(10) + assert handler_returned.is_set(), ( + "cleanup rule: SIGINT handler returns inside Event.set" + ) + assert observed.get("held"), ( + "cleanup rule: invocation lock survives Event.set signal" + ) + assert not observed.get("closed_while_live"), ( + "cleanup rule: admitted worker saves before close" + ) + assert plan.interrupted.is_set() + + elif case in ("join-once", "join-twice"): + raised = [] + original_set = plan.interrupted.set + + def interrupt_cleanup(): + cleanup_entered.set() + return original_set() + + plan.interrupted.set = interrupt_cleanup + + def injected(sig, frame): + raised.append(len(raised) + 1) + raise KeyboardInterrupt( + "synthetic wait interruption " + str(len(raised)) + ) + + signal.signal(signal.SIGUSR1, injected) + + def main_in_worker_wait(): + frame = sys._current_frames().get(main_ident) + while frame is not None: + if frame.f_code.co_name == "_wait_for_tstate_lock": + return True + if frame.f_code.co_name == "wait": + event = frame.f_locals.get("self") + if event in getattr(plan, "_worker_done", {}).values(): + return True + frame = frame.f_back + return False + + def await_wait(cleanup=False): + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if closed.is_set(): + return False + if ( + not cleanup or cleanup_entered.is_set() + ) and main_in_worker_wait(): + return True + time.sleep(0.001) + return False + + def observe(): + assert entered.wait(10) + assert await_wait(), ( + "cleanup rule: main thread reaches a worker wait" + ) + os.kill(os.getpid(), signal.SIGUSR1) + if case == "join-twice": + if await_wait(cleanup=True): + os.kill(os.getpid(), signal.SIGUSR1) + # The original implementation closes early after poisoning Thread's + # tstate lock. Give it that opportunity while the provider is held. + observed["closed_while_live"] = closed.wait(0.1) + observed["held"] = lock_held() + release.set() + + actor = threading.Thread(target=observe, daemon=True) + actor.start() + try: + plan.execute() + except KeyboardInterrupt as error: + observed["exception"] = str(error) + order.append("propagated") + else: + raise AssertionError("cleanup rule: wait exception must propagate") + finally: + release.set() + actor.join(15) + assert worker_returned.wait(10), ( + "cleanup rule: worker finishes after injected wait exception" + ) + assert observed.get("exception") == "synthetic wait interruption 1", ( + "cleanup rule: preserve original exception" + ) + expected = 2 if case == "join-twice" else 1 + assert len(raised) == expected, ( + "cleanup rule: repeat interruption exercises cleanup wait" + ) + assert not observed.get("closed_while_live"), ( + "cleanup rule: close waits for worker done" + ) + assert observed.get("held"), ( + "cleanup rule: invocation lock stays held through worker done" + ) + + elif case in ("start-after", "start-before"): + + def start(thread): + if thread.name == "no-tools-" + model: + if case == "start-after": + original_start(thread) + assert entered.wait(10) + raise RuntimeError("synthetic start interruption") + return original_start(thread) + + def observe(): + assert entered.wait(10) + observed["closed_while_live"] = closed.wait(0.1) + observed["held"] = lock_held() + release.set() + + actor = None + if case == "start-after": + actor = threading.Thread(target=observe, daemon=True) + actor.start() + threading.Thread.start = start + began = time.monotonic() + try: + plan.execute() + except RuntimeError as error: + observed["exception"] = str(error) + order.append("propagated") + else: + raise AssertionError( + "cleanup rule: failed start exception must propagate" + ) + if actor is not None: + actor.join(15) + assert not observed.get("closed_while_live"), ( + "cleanup rule: partially started worker saves before close" + ) + assert observed.get("held"), ( + "cleanup rule: partially started worker retains invocation lock" + ) + else: + elapsed = time.monotonic() - began + assert elapsed >= nt.START_GRACE_SECONDS * 0.9, ( + "cleanup rule: unstarted worker receives start grace" + ) + assert not calls and not entered.is_set(), ( + "cleanup rule: failed native start touches no worker files" + ) + assert not list( + plan.root.glob("*/none/confirmatory/households/*.json") + ) + assert observed.get("exception") == "synthetic start interruption" + + else: + raise AssertionError(case) + + if case != "start-before": + assert saved.is_set(), "cleanup rule: admitted worker saves its final" + assert order.index("saved") < order.index("close"), ( + "cleanup rule: save precedes close" + ) + for sig in (signal.SIGINT, signal.SIGTERM): + assert order.index("saved") < order.index( + "restore:" + signal.Signals(sig).name + ), "cleanup rule: save precedes handler restoration" + assert plan._lock_file is None, ( + "cleanup rule: invocation lock eventually closes" + ) + assert all( + signal.getsignal(sig) == old for sig, old in original_handlers.items() + ), "cleanup rule: handlers eventually restore" + print(json.dumps({"case": case, "order": order, "observed": observed})) + if case == "condition-signal": + raise propagated_error + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), case], + env=env, + ready_marker=b"cleanup-regression-ready\n", + text=True, + ) + assert "cleanup-regression-ready\n" in result.stdout, ( + "cleanup rule: subprocess reaches readiness", + result.stdout, + result.stderr, + ) + assert result.returncode == (1 if case == "condition-signal" else 0), ( + result.stdout, + result.stderr, + ) + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def test_signal_inside_cleanup_event_condition_keeps_worker_protected(setup, tmp_path): + """R10.1: a real signal inside Event.set cannot reenter its condition lock.""" + result = _run_cleanup_regression(setup, tmp_path, "condition-signal") + assert result["observed"]["propagated"] + + +@pytest.mark.parametrize("case", ["join-once", "join-twice"]) +def test_wait_exceptions_preserve_worker_completion_and_original_exception( + setup, tmp_path, case +): + """R10.2: real SIGUSR1 exceptions cannot release protection before done.""" + _run_cleanup_regression(setup, tmp_path, case) + + +@pytest.mark.parametrize("case", ["start-after", "start-before"]) +def test_raised_start_tracks_native_worker_or_waits_start_grace(setup, tmp_path, case): + """R10.2: a raised start has a completion event and bounded began grace.""" + _run_cleanup_regression(setup, tmp_path, case) + + +def admission_probe(setup, tmp_path, boundary, force): + """Exercise real signals behind a paused admission in a bounded subprocess.""" + import textwrap + + if boundary == "archive": + paused(setup) + probe = tmp_path / "admission_probe.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import sys + import threading + import traceback + from pathlib import Path + from types import SimpleNamespace + import confirmatory_no_tools as nt + + design = json.loads(sys.argv[1]) + boundary, force = sys.argv[2], sys.argv[3] == "force" + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + entered, release, returned, watcher_ready, committed = ( + threading.Event() for _ in range(5) + ) + errors, calls, order = [], [], [] + plan = None + + + class AdmissionLock: + def __init__(self): + self.lock = threading.Lock() + + def __enter__(self): + if threading.current_thread().name == "no-tools-signal-watcher": + watcher_ready.set() + self.lock.acquire() + return self + + def __exit__(self, *exc): + self.lock.release() + + def locked(self): + return self.lock.locked() + + + class InterruptEvent(threading.Event): + def set(self): + order.append("interrupt") + super().set() + watcher_ready.set() + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + calls.append(scenario.id) + assert plan.interrupted.wait(20), "R3: watcher eventually interrupts" + spend.add(0.1) + return { + "scenario_id": scenario.id, + "model": profile.name, + "arm": profile.arm, + "status": "complete", + "error": None, + "turns": [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": 0.1}, + } + + + def completion(profile): + if boundary == "archive": + assert plan.interrupted.wait(20), ( + "R3: archive finishes before another attempt" + ) + return None + + + plan = nt.Stage( + design, + models=[nt.CHEAP[0]], + board=board, + run_household=provider, + completion_for=completion, + out=io.StringIO(), + ) + plan.interrupted = InterruptEvent() + plan._admission_lock = AdmissionLock() + original_say = plan.say + + + def say(text): + if text.startswith("Interrupted:"): + assert plan.interrupted.is_set(), "R3: message follows interruption" + assert committed.is_set(), ( + "R3: admitted commit precedes interruption message" + ) + order.append("message") + original_say(text) + + + plan.say = say + original_signal = signal.signal + + + def install(signum, handler): + if callable(handler) and handler.__name__ == "interrupt": + + def tracked(sig, frame): + handler(sig, frame) + returned.set() + + return original_signal(signum, tracked) + return original_signal(signum, handler) + + + signal.signal = install + + + def pause_commit(): + entered.set() + os.write(1, b"ready\n") + assert release.wait(20), "R3: paused admitted commit is released" + + + if boundary == "archive": + original_replace = nt.os.replace + + def replace(source, target): + if Path(target).parent.name == "attempts": + pause_commit() + original_replace(source, target) + order.append("commit") + committed.set() + return + return original_replace(source, target) + + nt.os.replace = replace + else: + + class Admitted(dict): + def __setitem__(self, key, value): + pause_commit() + super().__setitem__(key, value) + order.append("commit") + committed.set() + + plan._admitted_attempts = Admitted() + + + def control(): + try: + assert entered.wait(20), "R3: worker reaches the admission commit" + held_at_commit = plan._admission_lock.locked() + os.kill(os.getpid(), signal.SIGINT) + assert returned.wait(5), ( + "R3: first handler returns while worker holds admission lock" + ) + assert held_at_commit, ( + "R3: worker holds the admission lock at its commit" + ) + assert watcher_ready.wait(5), ( + "R3: watcher reaches the admission boundary" + ) + assert not plan.interrupted.is_set(), ( + "R3: interrupt cannot precede admitted commit" + ) + assert not committed.is_set(), ( + "R3: commit remains paused during first handler" + ) + if force: + os.write(1, b"handler-returned-watcher-waiting\n") + os.kill(os.getpid(), signal.SIGINT) + raise AssertionError("R3: forced exit returned") + except BaseException: + errors.append(traceback.format_exc()) + if force: + os.write(2, errors[-1].encode()) + os._exit(99) + finally: + if not force: + release.set() + + + controller = threading.Thread(target=control, daemon=True) + controller.start() + result = plan.execute() + controller.join(20) + assert not errors, "\n".join(errors) + assert not controller.is_alive(), "R3: controller finishes" + assert result == 1, "R3: signalled run exits nonzero" + assert ( + order.index("commit") + < order.index("interrupt") + < order.index("message") + ), order + assert plan.interrupted.is_set(), ( + "R3: watcher sets interrupt after admitted commit" + ) + households = plan.households(nt.CHEAP[0]) + assert not (households / "h1.json").exists(), ( + "R3: no later household is admitted" + ) + if boundary == "archive": + assert calls == [], ( + "R3: no attempt admitted after the watcher interrupts" + ) + assert (households / "attempts/h2-1.json").exists(), ( + "R3: admitted archive is saved" + ) + assert not (households / "h2.json").exists(), ( + "R3: refused next attempt has no final" + ) + assert plan._admitted_attempts == {}, ( + "R3: no subsequent attempt is admitted" + ) + else: + assert calls == ["h2"], "R3: exactly the admitted attempt runs" + saved = json.loads((households / "h2.json").read_text()) + assert saved["status"] == "complete", ( + "R3: admitted attempt saves its final" + ) + assert saved["totals"]["cost_usd"] == 0.1, ( + "R3: admitted attempt saves spend" + ) + assert len(plan._admitted_attempts) == 1, "R3: nothing else is admitted" + print(json.dumps({"order": order, "calls": calls})) + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), boundary, "force" if force else "finish"], + env=env, + ready_marker=b"ready\n", + ) + assert b"ready\n" in result.stdout, (result.stdout, result.stderr) + return result.returncode, result.stdout, result.stderr + + +@pytest.mark.parametrize("boundary", ["archive", "attempt"]) +def test_signal_waits_for_an_admission_commit_without_blocking_handler( + setup, tmp_path, boundary +): + """R3: watcher interruption waits for commits admitted under the worker lock.""" + code, stdout, stderr = admission_probe(setup, tmp_path, boundary, force=False) + assert code == 0, (stdout, stderr) + + +@pytest.mark.parametrize("boundary", ["archive", "attempt"]) +def test_second_signal_forces_exit_while_watcher_waits_for_admission( + setup, tmp_path, boundary +): + """R3: the forced path remains available while the watcher waits for admission.""" + code, stdout, stderr = admission_probe(setup, tmp_path, boundary, force=True) + assert code == 128 + signal.SIGINT, (stdout, stderr) + assert b"handler-returned-watcher-waiting\n" in stdout + assert ( + b"Forced interruption: every unsaved attempt then under way is lost, " + b"with its spend; saved finals, archives and the spend they record survive." + ) in stderr + + +def test_signal_after_all_workers_complete_still_exits_one(setup, tmp_path): + """R4: a signal before handler restoration makes even a complete run fail.""" + import textwrap + + probe = tmp_path / "signal_after_workers.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import sys + from types import SimpleNamespace + import confirmatory_no_tools as nt + + design = json.loads(sys.argv[1]) + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + plan = nt.Stage( + design, + models=[nt.CHEAP[0]], + board=board, + completion_for=lambda profile: None, + out=io.StringIO(), + ) + + + def complete(model): + plan.states[model] = "complete" + + + plan.run_model = complete + original_signal = signal.signal + previous = signal.getsignal(signal.SIGINT) + sent = False + + + def restore(sig, handler): + global sent + if not sent and sig == signal.SIGINT and handler == previous: + sent = True + os.kill(os.getpid(), signal.SIGINT) + return original_signal(sig, handler) + + + signal.signal = restore + assert plan.execute() == 1, ( + "R4: signal arrival before restoration requires exit 1" + ) + assert all(state == "complete" for state in plan.states.values()) + assert sent, "R4: signal arrives after every worker completed" + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"])], + env=env, + ) + assert result.returncode == 0, (result.stdout, result.stderr) + + +@pytest.mark.parametrize("during_grace", [True, False]) +def test_native_worker_without_ident_obeys_start_grace_and_entry_interrupt( + setup, tmp_path, during_grace +): + """R10.2: native startup can precede ident and must never outlive file safety.""" + import textwrap + + probe = tmp_path / "native_start_regression.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import subprocess + import sys + import threading + import time + from types import SimpleNamespace + import confirmatory_no_tools as nt + + nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + print("native-start-regression-ready", flush=True) + design = json.loads(sys.argv[1]) + during_grace = sys.argv[2] == "True" + model = nt.CHEAP[0] + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + native_entered, bootstrap_release = threading.Event(), threading.Event() + cleanup_entered, closed, native_finished = ( + threading.Event() for _ in range(3) + ) + done_held, release_done = threading.Event(), threading.Event() + original_bootstrap = threading.Thread._bootstrap_inner + original_run = threading.Thread.run + original_close = nt.Stage.close + main_ident = threading.get_ident() + worker, calls, order, observed = [], [], [], {} + nt.START_GRACE_SECONDS = 10 if during_grace else 0.05 + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + calls.append(scenario.id) + return { + "scenario_id": scenario.id, + "model": profile.name, + "arm": profile.arm, + "status": "complete", + "error": None, + "turns": [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": 0.0}, + } + + + plan = nt.Stage( + design, + models=[model], + board=board, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) + original_set = plan.interrupted.set + + + def interrupt_cleanup(): + result = original_set() + cleanup_entered.set() + return result + + + plan.interrupted.set = interrupt_cleanup + + + def bootstrap(thread): + if thread.name == "no-tools-" + model: + worker.append(thread) + native_entered.set() + assert bootstrap_release.wait(20), ( + "start grace rule: release native startup" + ) + return original_bootstrap(thread) + + + def run(thread): + try: + return original_run(thread) + finally: + if thread.name == "no-tools-" + model: + order.append("native-finished") + native_finished.set() + + + def close(current): + if current is plan and current._lock_file is not None: + order.append("close") + closed.set() + return original_close(current) + + + threading.Thread._bootstrap_inner = bootstrap + threading.Thread.run = run + nt.Stage.close = close + + + def injected(sig, frame): + raise KeyboardInterrupt("synthetic startup wait interruption") + + + signal.signal(signal.SIGUSR1, injected) + + + def main_in_start_wait(): + frame = sys._current_frames().get(main_ident) + while frame is not None: + if ( + frame.f_code.co_name == "wait" + and frame.f_locals.get("self") is worker[0]._started + ): + return True + frame = frame.f_back + return False + + + def lock_held(): + probe = ( + "import fcntl,sys; f=open(sys.argv[1],'a'); " + "\ntry: fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB)" + "\nexcept BlockingIOError: print('held')" + "\nelse: print('released')" + ) + result = subprocess.run( + [ + sys.executable, + "-c", + probe, + str(plan.root / ".confirmatory-no-tools.lock"), + ], + capture_output=True, + text=True, + timeout=10, + check=True, + ) + return result.stdout.strip() == "held" + + + def observe(): + assert native_entered.wait(10) + assert worker[0].ident is None, ( + "start grace rule: native worker has no ident" + ) + deadline = time.monotonic() + 10 + while not main_in_start_wait(): + assert time.monotonic() < deadline, ( + "start grace rule: main reaches _started.wait" + ) + time.sleep(0.001) + os.kill(os.getpid(), signal.SIGUSR1) + assert cleanup_entered.wait(10) + if during_grace: + # Let the native target begin immediately within grace, then + # hold its completion independently of process-startup load. + done = getattr(plan, "_worker_done", {}).get(model) + if done is not None: + original_done_set = done.set + + def set_done(): + done_held.set() + assert release_done.wait(20), ( + "start grace rule: release begun worker completion" + ) + return original_done_set() + + done.set = set_done + bootstrap_release.set() + if done is not None: + assert done_held.wait(10), ( + "start grace rule: late target reaches completion" + ) + try: + observed["held_at_cleanup"] = lock_held() + finally: + release_done.set() + else: + assert closed.wait(10), ( + "start grace rule: close after bounded grace" + ) + bootstrap_release.set() + + + actor = threading.Thread(target=observe, daemon=True) + actor.start() + try: + plan.execute() + except KeyboardInterrupt as error: + assert str(error) == "synthetic startup wait interruption" + else: + raise AssertionError("start grace rule: original exception propagates") + finally: + if closed.is_set() and not during_grace: + bootstrap_release.set() + actor.join(15) + assert native_finished.wait(10), ( + "start grace rule: native worker eventually completes" + ) + assert not calls, ( + "start grace rule: interrupted late worker makes no provider call" + ) + assert not list(plan.root.glob("*/none/confirmatory/households/*.json")), ( + "start grace rule: interrupted late worker writes no files" + ) + if during_grace: + assert observed.get("held_at_cleanup"), ( + "start grace rule: invocation lock stays held during grace" + ) + assert order.index("native-finished") < order.index("close"), ( + "start grace rule: worker begun during grace completes before close" + ) + else: + assert order.index("close") < order.index("native-finished"), ( + "start grace rule: worker beyond grace exits safely after close" + ) + assert plan._lock_file is None + print(json.dumps({"during_grace": during_grace, "order": order})) + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), str(during_grace)], + env=env, + ready_marker=b"native-start-regression-ready\n", + ) + assert b"native-start-regression-ready\n" in result.stdout, ( + "start grace rule: subprocess reaches readiness", + result.stdout, + result.stderr, + ) + assert result.returncode == 0, (result.stdout, result.stderr) + + +_NO_TOOLS_FORK_WARMED = False + + +def run_no_tools_fork_probe( + probe, + args=(), + *, + env=None, + timeout=30, + ready_marker=None, + ready_timeout=120, + text=False, +): + """Run actual probe code in a forked subprocess with bounded cleanup. + + The parent warms the provider-free profile imports once. A fresh child + executes the probe with its own signals, stdout, stderr and process exit. + Readiness has a separate deadline; the execution deadline begins when the + child reaches its actual readiness marker, or after subprocess setup. + """ + import multiprocessing + import runpy + import subprocess + import time + import traceback + + global _NO_TOOLS_FORK_WARMED + if not _NO_TOOLS_FORK_WARMED: + nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + _NO_TOOLS_FORK_WARMED = True + probe = Path(probe).resolve() + stdout_path = probe.with_suffix(".stdout") + stderr_path = probe.with_suffix(".stderr") + command = [sys.executable, str(probe), *map(str, args)] + context = multiprocessing.get_context("fork") + startup_ready = context.Event() + + # Flush existing parent streams before fork so the child cannot duplicate + # any buffered pytest output when its streams are replaced. + sys.stdout.flush() + sys.stderr.flush() + with stdout_path.open("wb") as stdout_file, stderr_path.open("wb") as stderr_file: + + def child(): + code = 0 + try: + os.dup2(stdout_file.fileno(), 1) + os.dup2(stderr_file.fileno(), 2) + sys.stdout = os.fdopen(os.dup(1), "w", buffering=1) + sys.stderr = os.fdopen(os.dup(2), "w", buffering=1) + if env is not None: + os.environ.clear() + os.environ.update(env) + sys.argv = command[1:] + signal.signal(signal.SIGINT, signal.default_int_handler) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + startup_ready.set() + runpy.run_path(str(probe), run_name="__main__") + except SystemExit as error: + if error.code is None: + code = 0 + elif isinstance(error.code, int): + code = error.code + else: + print(error.code, file=sys.stderr) + code = 1 + except BaseException: + traceback.print_exc() + code = 1 + finally: + sys.stdout.flush() + sys.stderr.flush() + os._exit(code) + + process = context.Process(target=child, name=f"no-tools-probe-{probe.stem}") + process.start() + initial_deadline = time.monotonic() + ready_timeout + + def output(): + stdout = stdout_path.read_bytes() + stderr = stderr_path.read_bytes() + if text: + return stdout.decode(errors="replace"), stderr.decode(errors="replace") + return stdout, stderr + + def expired(limit): + process.kill() + process.join(timeout=30) + stdout, stderr = output() + raise subprocess.TimeoutExpired( + command, limit, output=stdout, stderr=stderr + ) + + try: + while not startup_ready.is_set() and process.is_alive(): + remaining = initial_deadline - time.monotonic() + if remaining <= 0: + expired(ready_timeout) + startup_ready.wait(timeout=min(remaining, 0.1)) + if ready_marker is not None: + marker = ( + ready_marker.encode() + if isinstance(ready_marker, str) + else ready_marker + ) + while marker not in stdout_path.read_bytes() and process.is_alive(): + if time.monotonic() >= initial_deadline: + expired(ready_timeout) + time.sleep(0.01) + process.join(timeout=timeout) + if process.is_alive(): + expired(timeout) + stdout, stderr = output() + return subprocess.CompletedProcess( + command, process.exitcode, stdout, stderr + ) + finally: + if process.is_alive(): + process.kill() + process.join(timeout=30) + process.close() From 6597adb467090bccb29f8e67e6691662cef3dc47 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 23:55:09 -0400 Subject: [PATCH 38/64] Confirmatory runner: restore signal handlers before closing the wake pipe Until the handlers are restored, a signal still writes to the watcher pipe; closing it first could let that write land in a file that reuses the descriptor. Note that an exception another handler raises before the exceptional wait begins cannot be shielded. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 0f19a730..b4b195cf 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -617,6 +617,8 @@ def watch_signals(): # SIGINT/SIGTERM handlers never raise, so production waits are not # interrupted by them. Tolerant waits defend against exceptions from # elsewhere (for example a test SIGUSR1 handler raising KeyboardInterrupt). + # Python cannot shield the instructions before these calls: another + # handler that raises there escapes without the wait. stop_admission() self._wait_for_workers() raise @@ -649,11 +651,13 @@ def watch_signals(): shutdown_error = error if failure is None and shutdown_error is not None: stop_admission() + # Restore the handlers first: until then a signal still writes to + # write_fd, whose number a closed pipe could hand to another file. + for sig, handler in previous.items(): + signal.signal(sig, handler) for fd in (read_fd, write_fd): if fd is not None: os.close(fd) - for sig, handler in previous.items(): - signal.signal(sig, handler) self.close() if failure is None and shutdown_error is not None: raise shutdown_error From 9acc9205cc717cf6176d4c72aad673b108f38cf5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 00:10:32 -0400 Subject: [PATCH 39/64] Confirmatory runner: let the forced-exit probe wait for the handler The admission probe's control thread sent the second SIGINT and then at once raised "forced exit returned". The handler runs on the main thread, so on a loaded host the control thread could reach its own os._exit(99) first; one of three suite runs at 6597adb4 failed that way, with the forced message printed after the control thread's traceback. The thread now sleeps 20 s, inside the probe's 30 s execution deadline, before it reports a forced exit that never came. Test only. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_no_tools.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 08620921..37c2ad60 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -1388,6 +1388,7 @@ def admission_probe(setup, tmp_path, boundary, force): import signal import sys import threading + import time import traceback from pathlib import Path from types import SimpleNamespace @@ -1551,6 +1552,11 @@ def control(): if force: os.write(1, b"handler-returned-watcher-waiting\n") os.kill(os.getpid(), signal.SIGINT) + # The handler runs on the main thread. Sleep (releasing + # the GIL) so it can exit the process before this thread + # reports that it did not; 20 s stays inside the probe's + # 30 s execution deadline. + time.sleep(20) raise AssertionError("R3: forced exit returned") except BaseException: errors.append(traceback.format_exc()) From a1114fe95d3170afbd71425cfcf89ae7fc6799b3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 02:31:13 -0400 Subject: [PATCH 40/64] Confirmatory runner: cover watcher startup and pipe teardown Exercise real signals after descriptor reuse and watcher shutdown. Require both handlers to restore before either descriptor closes. Pin late first SIGINT to exit 1 with completed finals, an unset interrupt and no announcement. Exercise watcher start failures before and after native startup and target entry beyond startup grace. Require the original start error to propagate, both handlers and descriptors to be released, and delayed targets to avoid closed descriptors and finish. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_no_tools.py | 443 ++++++++++++++++++++++++++++ 1 file changed, 443 insertions(+) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 37c2ad60..961d37c8 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -2093,3 +2093,446 @@ def expired(limit): process.kill() process.join(timeout=30) process.close() + + +def test_signal_handlers_restore_before_either_watcher_descriptor_closes( + setup, tmp_path +): + """R11: every handler that can write is removed before either pipe end closes.""" + run_no_tools_probe( + setup, + tmp_path, + "watcher_descriptor_order", + r""" + plan = stage(setup, models=[nt.CHEAP[0]]) + previous = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_pipe, original_close = os.pipe, os.close + original_signal = signal.signal + watcher_pipe, events = [], [] + + + def pipe(): + descriptors = original_pipe() + assert not watcher_pipe, "R11: execute creates one watcher pipe" + watcher_pipe.extend(descriptors) + return descriptors + + + def register(sig, handler): + old = original_signal(sig, handler) + if sig in previous and handler == previous[sig]: + events.append(("restore", sig)) + return old + + + def close(fd): + original_close(fd) + if fd in watcher_pipe: + events.append(("close", fd)) + + + monkeypatch.setattr(os, "pipe", pipe) + monkeypatch.setattr(os, "close", close) + monkeypatch.setattr(signal, "signal", register) + assert plan.execute() == 0 + restored, closed = set(), [] + for kind, value in events: + if kind == "restore": + restored.add(value) + else: + assert restored == set(previous), ( + "R11: both handlers restore before either pipe descriptor " + f"closes: {events}" + ) + closed.append(value) + assert len(watcher_pipe) == 2 + assert closed == watcher_pipe, "R11: both watcher descriptors close" + assert all( + signal.getsignal(sig) == handler for sig, handler in previous.items() + ), "R11: both prior handlers remain installed" + """, + ) + + +def test_signal_after_watcher_descriptor_reuse_cannot_write_to_replacement( + setup, tmp_path +): + """R11: a real signal after close cannot write into a reused descriptor.""" + run_no_tools_probe( + setup, + tmp_path, + "watcher_descriptor_reuse", + r""" + plan = stage(setup, models=[nt.CHEAP[0]]) + original_pipe, original_close = os.pipe, os.close + original_signal = signal.signal + previous_interrupts = [] + + + def previous_interrupt(sig, frame): + previous_interrupts.append(sig) + + + original_signal(signal.SIGINT, previous_interrupt) + watcher_pipe, replacement = [], [] + sent = False + + + def pipe(): + descriptors = original_pipe() + assert not watcher_pipe, "R11: execute creates one watcher pipe" + watcher_pipe.extend(descriptors) + return descriptors + + + def close(fd): + global sent + original_close(fd) + if fd == watcher_pipe[1] and not replacement: + read_fd, write_fd = original_pipe() + # Preserve the read end if it acquired the descriptor that must + # become the replacement write end. + if read_fd == fd: + copied_read_fd = os.dup(read_fd) + original_close(read_fd) + read_fd = copied_read_fd + if write_fd != fd: + os.dup2(write_fd, fd) + original_close(write_fd) + write_fd = fd + replacement.extend((read_fd, write_fd)) + os.set_blocking(read_fd, False) + sent = True + # This executes on the main thread immediately after the old + # write descriptor closes and its number is safely reused. + os.kill(os.getpid(), signal.SIGINT) + + + monkeypatch.setattr(os, "pipe", pipe) + monkeypatch.setattr(os, "close", close) + try: + result = plan.execute() + assert sent and len(replacement) == 2 + try: + received = os.read(replacement[0], 1) + except BlockingIOError: + received = b"" + assert received == b"", ( + "R11: a restored handler cannot write into the replacement " + f"descriptor; received {received!r}" + ) + assert result == 0 + assert previous_interrupts == [signal.SIGINT], ( + "R11: the real signal reaches the restored prior handler" + ) + finally: + for fd in replacement: + original_close(fd) + """, + ) + + +@pytest.mark.parametrize("case", ["before-native", "after-native", "after-grace"]) +def test_watcher_start_failure_restores_resources_and_late_entry_is_safe( + setup, tmp_path, case +): + """R11: failed watcher startup preserves cleanup and the late-entry guard.""" + run_no_tools_probe( + setup, + tmp_path, + "watcher_start_" + case, + " case = " + + repr(case) + + r""" + import errno + import time + + provider = Provider() + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + original_handlers = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_start = threading.Thread.start + original_run = threading.Thread.run + original_pipe = os.pipe + original_read = os.read + original_fd_close = os.close + original_close = plan.close + native_entered, reading, release_late, target_finished = ( + threading.Event() for _ in range(4) + ) + watchers, pipe_fds, closed_fds, reads, target_errors, order = ( + [] for _ in range(6) + ) + start_error = RuntimeError("synthetic watcher start " + case) + nt.START_GRACE_SECONDS = 0.05 + + + def pipe(): + descriptors = original_pipe() + assert not pipe_fds, "watcher lifecycle: execute creates one pipe" + pipe_fds.extend(descriptors) + return descriptors + + + def read(fd, count): + if fd in pipe_fds: + assert fd not in closed_fds, ( + "watcher lifecycle: a late watcher cannot read a closed pipe" + ) + reads.append(fd) + reading.set() + return original_read(fd, count) + + + def fd_close(fd): + result = original_fd_close(fd) + if fd in pipe_fds: + closed_fds.append(fd) + order.append("descriptor-closed") + return result + + + def close(): + result = original_close() + order.append("stage-closed") + return result + + + def run(thread): + if thread.name != "no-tools-signal-watcher": + return original_run(thread) + native_entered.set() + try: + if case == "after-grace": + assert release_late.wait(20), ( + "watcher lifecycle: release the delayed native target" + ) + assert set(closed_fds) == set(pipe_fds), ( + "watcher lifecycle: delayed target begins after pipe close" + ) + order.append("late-target-entered") + return original_run(thread) + except BaseException as error: + target_errors.append(error) + raise + finally: + target_finished.set() + + + def start(thread): + if thread.name != "no-tools-signal-watcher": + return original_start(thread) + watchers.append(thread) + if case != "before-native": + original_start(thread) + assert native_entered.wait(10), ( + "watcher lifecycle: the native thread starts before failure" + ) + if case == "after-native": + assert reading.wait(10), ( + "watcher lifecycle: the begun watcher reaches its pipe read" + ) + raise start_error + + + monkeypatch.setattr(os, "pipe", pipe) + monkeypatch.setattr(os, "read", read) + monkeypatch.setattr(os, "close", fd_close) + monkeypatch.setattr(threading.Thread, "run", run) + monkeypatch.setattr(threading.Thread, "start", start) + monkeypatch.setattr(plan, "close", close) + started_at = time.monotonic() + try: + plan.execute() + except RuntimeError as error: + assert error is start_error, ( + "watcher lifecycle: propagate the original start exception" + ) + else: + raise AssertionError("watcher lifecycle: startup failure must propagate") + elapsed = time.monotonic() - started_at + + assert len(watchers) == 1 and len(pipe_fds) == 2 + assert sorted(closed_fds) == sorted(pipe_fds), ( + "watcher lifecycle: close both descriptors exactly once" + ) + assert order.count("stage-closed") == 1 + assert all( + signal.getsignal(sig) == handler + for sig, handler in original_handlers.items() + ), "watcher lifecycle: restore both installed handlers" + for fd in pipe_fds: + try: + os.fstat(fd) + except OSError as error: + assert error.errno == errno.EBADF + else: + raise AssertionError("watcher lifecycle: descriptor remains open") + assert plan.interrupted.is_set(), ( + "watcher lifecycle: failed startup stops admission" + ) + assert provider.calls == [] and plan._started_workers == [], ( + "watcher lifecycle: no worker is admitted after startup failure" + ) + assert plan._lock_file is None, ( + "watcher lifecycle: release the stage invocation lock" + ) + if case == "before-native": + assert elapsed >= nt.START_GRACE_SECONDS * 0.9, ( + "watcher lifecycle: allow bounded grace for a possible native start" + ) + assert not native_entered.is_set() + assert not target_finished.is_set() + assert not reads + elif case == "after-native": + assert reads == [pipe_fds[0]] + assert target_finished.is_set(), ( + "watcher lifecycle: join the watcher before cleanup returns" + ) + else: + assert elapsed >= nt.START_GRACE_SECONDS * 0.9, ( + "watcher lifecycle: delayed target outlasts the bounded start grace" + ) + assert not target_finished.is_set() + release_late.set() + watchers[0].join(timeout=10) + assert target_finished.is_set() + assert not reads, ( + "watcher lifecycle: late entry checks closing before reading" + ) + assert order.index("stage-closed") < order.index("late-target-entered") + assert not target_errors, target_errors + assert not watchers[0].is_alive(), ( + "watcher lifecycle: no watcher remains after probe cleanup" + ) + assert not any( + thread.name == "no-tools-signal-watcher" + for thread in threading.enumerate() + ) + monkeypatch.undo() + """, + ) + + +def test_first_sigint_after_watcher_shutdown_preserves_completed_finals( + setup, tmp_path +): + """R11.2: late recording forces exit 1 without watcher interruption or output.""" + import textwrap + + probe = tmp_path / "late_first_sigint.py" + probe.write_text( + textwrap.dedent(r""" + import json + import os + import runpy + import signal + import sys + import threading + from types import SimpleNamespace + + helpers = runpy.run_path(sys.argv[2]) + nt, stage, final, load = ( + helpers[name] for name in ("nt", "stage", "final", "load") + ) + design = json.loads(sys.argv[1]) + setup = { + "design": design, + "board": { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + }, + } + plan = stage(setup) + previous = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_signal = signal.signal + original_start = threading.Thread.start + watcher = [] + restoring, recorded = threading.Event(), threading.Event() + handler_returned = threading.Event() + errors = [] + + + def start(thread): + if thread.name == "no-tools-signal-watcher": + watcher.append(thread) + return original_start(thread) + + + def register(sig, handler): + if sig in previous and handler == previous[sig]: + if sig == signal.SIGINT: + assert len(watcher) == 1 and not watcher[0].is_alive(), ( + "late signal: watcher has finished before restoration" + ) + restoring.set() + assert recorded.wait(20), "late signal: controller finishes" + elif sig in previous: + original_handler = handler + + def tracked(signum, frame): + original_handler(signum, frame) + handler_returned.set() + + handler = tracked + return original_signal(sig, handler) + + + def send(): + try: + assert restoring.wait(20), "late signal: restoration pauses" + os.kill(os.getpid(), signal.SIGINT) + assert handler_returned.wait(20), "late signal: handler returns" + except BaseException as error: + errors.append(error) + finally: + recorded.set() + + + actor = threading.Thread(target=send, daemon=True) + actor.start() + threading.Thread.start = start + signal.signal = register + result = plan.execute() + actor.join(20) + assert not actor.is_alive() and not errors, errors + assert result == 1, "late signal: recorded SIGINT requires exit 1" + assert not plan.interrupted.is_set(), ( + "late signal: a finished watcher cannot set interruption" + ) + assert all(state == "complete" for state in plan.states.values()) + for model in plan.models: + for sid in design["draw_order"]: + assert load(final(plan, model, sid))["status"] == "complete", ( + "late signal: every worker's final is saved" + ) + assert "Interrupted" not in plan.out.getvalue(), ( + "late signal: a finished watcher prints no announcement" + ) + assert all(signal.getsignal(sig) == old for sig, old in previous.items()) + assert not watcher[0].is_alive() and plan._lock_file is None + print("late-first-signal-verified", flush=True) + sys.exit(result) + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), str(Path(__file__))], + env=env, + ) + assert result.returncode == 1, (result.stdout, result.stderr) + assert b"late-first-signal-verified\n" in result.stdout, ( + result.stdout, + result.stderr, + ) From 89dd9464ecd0a443dbb60ed61f2340cc3d33c510 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 02:34:30 -0400 Subject: [PATCH 41/64] Confirmatory runner: qualify signal and cleanup guarantees Describe the handler recording a signal and waking the watcher, with allocation failure as its exception limit. Describe the watcher setting interruption while running and a first signal recorded after shutdown forcing exit 1 without setting interruption or announcing it. State that guarded waits retry exceptions raised within them. Explain that another handler raising in an unguarded instruction before a guarded call, in cleanup bookkeeping or in finally can replace the original exception and skip waits. Keep executable behavior unchanged. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 39 +++++++++++++++++++------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index b4b195cf..458179e2 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -10,12 +10,17 @@ under the admission lock precede the watcher setting the interrupt under that same lock. After that interrupt is set, no archive move or attempt is admitted. The lock covers only the check and rename or admission state change; admitted -attempts finish and save outside it. The first SIGINT or SIGTERM only records -its number and wakes the watcher through a nonblocking pipe. The handler takes -no lock and raises nothing. The watcher sets the interrupt, releases admission, -and then announces that attempts already admitted will finish and save. -An exception escaping execute interrupts and waits for script-owned worker -completion events before restoring handlers or releasing the invocation lock. +attempts finish and save outside it. The first SIGINT or SIGTERM records its +number and wakes the watcher through a nonblocking pipe. The handler takes no +lock and raises nothing beyond allocation failure. While running, the watcher +sets the interrupt, releases admission, and then announces that attempts already +admitted will finish and save. A first signal recorded after watcher shutdown +still forces exit 1 without setting the interrupt or printing that announcement. +The guarded cleanup path sets the interrupt and waits for script-owned worker +completion before restoring handlers or releasing the invocation lock. +Guarded waits retry exceptions raised within them and preserve the original +exception. Another handler raising in an unguarded instruction, before a guarded +call, in cleanup bookkeeping or in finally, can replace it and skip waits. An exception caught in a model thread instead marks that model error without setting the interrupt. A second signal immediately uses a nonblocking stderr write and exits, losing every unsaved attempt then under way with its spend; @@ -614,11 +619,12 @@ def watch_signals(): result = self._execute() except BaseException as error: failure = error - # SIGINT/SIGTERM handlers never raise, so production waits are not - # interrupted by them. Tolerant waits defend against exceptions from - # elsewhere (for example a test SIGUSR1 handler raising KeyboardInterrupt). - # Python cannot shield the instructions before these calls: another - # handler that raises there escapes without the wait. + # SIGINT/SIGTERM handlers raise nothing beyond allocation failure. + # Guarded waits retry exceptions raised within them, including those + # from another handler (for example SIGUSR1 raising KeyboardInterrupt). + # Another handler raising in an unguarded instruction, before a + # guarded call, in cleanup bookkeeping or in finally, can replace + # the original exception and skip waits. stop_admission() self._wait_for_workers() raise @@ -664,12 +670,13 @@ def watch_signals(): return 1 if signals else result def _wait_for_workers(self) -> None: - """Retain protection through completion despite exceptional wait interruptions. + """Retry exceptions raised within guarded worker-completion waits. - The installed SIGINT and SIGTERM handlers never raise. These tolerant - waits defend against an exception from elsewhere, such as a test-only - SIGUSR1 handler that raises KeyboardInterrupt, and preserve the original - exception escaping execute. + The installed SIGINT and SIGTERM handlers raise nothing beyond allocation + failure. Guarded waits retry exceptions from elsewhere, such as a test-only + SIGUSR1 handler raising KeyboardInterrupt, preserving the original exception. + Another handler raising in unguarded instructions before a guarded call, + in cleanup bookkeeping or in finally can replace it and skip waits. """ workers = list(self._started_workers) starting = self._starting_worker From 4dafb833ea3e34c679ca24725234126003095356 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 09:03:50 -0400 Subject: [PATCH 42/64] Confirmatory runner: refuse undecodable files with their paths Catch recursion failures while reading saved finals, archived attempts, baseline exclusions and the CLI design so deeply nested JSON becomes a refusal naming the file. Include the baseline path in its validation errors, and require the CLI design to be a JSON object before stage initialization. Valid stage decisions stay the same. Saved records and baseline exclusions must refuse decoder recursion failures with a path-bearing StateError. The CLI must refuse unreadable, undecodable and non-object designs with their path and exit 1 before initializing a stage. Invalid UTF-8 must remain a decoding refusal. Synthetic regressions exercise every reader in a subprocess with a timeout and require the path before provider calls; final and archive cases also check preflight. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 18 ++++++-- tests/test_confirmatory_no_tools.py | 68 +++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 458179e2..85a8038c 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -89,7 +89,7 @@ def read_record(path: Path) -> dict: if not math.isfinite(cost) or cost < 0: raise ValueError("attempt cost must be finite and nonnegative") return record - except (OSError, ValueError, TypeError, AttributeError) as error: + except (OSError, ValueError, TypeError, AttributeError, RecursionError) as error: raise StateError(f"{path}: {error}") from error @@ -262,6 +262,7 @@ def __init__( run_household if run_household is not None else pilot.run_household ) self._lock_file = None + baseline_path = None try: baseline = design["baseline_exclusions"] if not isinstance(baseline, dict) or not isinstance( @@ -289,8 +290,11 @@ def __init__( cells.append((exclusion["scenario_id"], exclusion["variable"])) if len(cells) != len(set(cells)): raise StateError("baseline exclusions repeat a cell") - except (OSError, KeyError, TypeError, ValueError) as error: - raise StateError(f"baseline exclusions: {error}") from error + except (OSError, KeyError, TypeError, ValueError, RecursionError) as error: + at = "baseline exclusions" + if baseline_path is not None: + at += f" ({baseline_path})" + raise StateError(f"{at}: {error}") from error self.root.mkdir(parents=True, exist_ok=True) self._lock_file = (self.root / ".confirmatory-no-tools.lock").open("a") try: @@ -750,7 +754,13 @@ def main(argv=None) -> int: run.add_argument("--dry-run", action="store_true") args = parser.parse_args(argv) try: - with Stage(json.loads(args.design.read_text()), models=args.model) as stage: + try: + design = json.loads(args.design.read_text()) + if not isinstance(design, dict): + raise ValueError("design is not a JSON object") + except (OSError, ValueError, RecursionError) as error: + raise StateError(f"{args.design}: {error}") from error + with Stage(design, models=args.model) as stage: return stage.dry_run() if args.dry_run else stage.execute() except (StateError, OSError, ValueError, KeyError, TypeError) as error: print(f"The no-tools stage cannot start: {error}") diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 961d37c8..81fed2e3 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -910,6 +910,74 @@ def test_baseline_schema_is_usable_before_any_provider_call(setup, payload): assert provider.calls == [], "R5: schema checks precede provider calls" +@pytest.mark.parametrize("reader", ["final", "archive", "baseline", "design"]) +@pytest.mark.parametrize("fault", ["nested", "non-object", "invalid-utf8"]) +def test_undecodable_stage_file_refuses_with_path(setup, tmp_path, reader, fault): + """Every file reader names its refusal; deep decoding stays in a child.""" + run_no_tools_probe( + setup, + tmp_path, + f"decode_{reader}_{fault}", + f" reader, fault = {reader!r}, {fault!r}\n" + + r""" + from contextlib import redirect_stdout + from pathlib import Path + + if fault == "nested": + payload = ( + b'{"attempts": [], "exclusions": [], "nested": ' + + b"[" * 20000 + b"0" + b"]" * 20000 + b"}" + ) + elif fault == "non-object": + payload = b"[]" + else: + payload = ( + b'{"scenario_id": "h2", "status": "provider_error", ' + b'"error": "APIConnectionError", "attempts": [], ' + b'"exclusions": [], "note": "\xff"}' + ) + + provider = Provider() + root = Path(setup["design"]["no_tools"]["runs"]) + if reader in ("final", "archive"): + path = root / nt.CHEAP[0] / "none/confirmatory/households" + path /= "h2.json" if reader == "final" else "attempts/h2-1.json" + elif reader == "baseline": + path = Path(setup["design"]["baseline_exclusions"]["path"]) + else: + path = root.parent / "design.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + + if reader == "design": + def unexpected_stage(*args, **kwargs): + pytest.fail("an undecodable design must refuse before Stage") + + monkeypatch.setattr(nt, "Stage", unexpected_stage) + output = io.StringIO() + with redirect_stdout(output): + assert nt.main(["run", str(path), "--dry-run"]) == 1 + assert "The no-tools stage cannot start:" in output.getvalue() + assert str(path) in output.getvalue() + else: + if reader == "baseline": + setup["design"]["baseline_exclusions"]["sha256"] = ( + nt.sha256_file(path) + ) + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("an undecodable file must refuse before execution") + assert str(path) in str(caught.value) + if fault == "invalid-utf8": + assert isinstance(caught.value.__cause__, UnicodeDecodeError) + if reader in ("final", "archive"): + problems = nt.stage_problems(setup["design"]) + assert any(str(path) in problem for problem in problems), problems + assert provider.calls == [] + """, + ) + + def run_no_tools_probe(setup, tmp_path, name, body): """Bound a signal or worker-shutdown regression to its own process.""" import textwrap From 040d206437d1cbf3dbcb43b8e39a2923bc136931 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 12:47:16 -0400 Subject: [PATCH 43/64] Confirmatory runner: name board and cost refusals Check board inputs as strict UTF-8 and required JSON objects before delegating to the pinned harness. Attribute metadata engine checks, CSV parsing, exclusion validation and scenario reconstruction failures to their source files. Preserve absent optional exclusions and return the harness's loaded board for usable inputs. Saved finals and archived attempts must refuse numeric cost overflow with a path-bearing StateError at startup and preflight. Nested fixtures must control the child's recursion limit, derive a deeper payload and require the decoder's RecursionError as the refusal's cause. Allocation exhaustion continues to propagate. Provider-free regressions cover each board decoder, both cost boundaries and valid board results. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 87 +++++++++++++- tests/test_confirmatory_no_tools.py | 179 +++++++++++++++++++++++++++- 2 files changed, 257 insertions(+), 9 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 85a8038c..b92d360d 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -89,10 +89,89 @@ def read_record(path: Path) -> dict: if not math.isfinite(cost) or cost < 0: raise ValueError("attempt cost must be finite and nonnegative") return record - except (OSError, ValueError, TypeError, AttributeError, RecursionError) as error: + except ( + OSError, + ValueError, + TypeError, + AttributeError, + RecursionError, + OverflowError, + ) as error: raise StateError(f"{path}: {error}") from error +def _board_input(path: Path, read): + """Keep a delegated board reader's refusal attached to its input file.""" + try: + return read() + except ( + OSError, + ValueError, + TypeError, + AttributeError, + KeyError, + RecursionError, + OverflowError, + SystemExit, + ) as error: + raise StateError(f"{path}: {error}") from error + + +def load_board(path: Path, engine: str) -> dict: + """Check board inputs locally, then use the harness's unchanged loader.""" + from policybench.reference_exclusions import ( + load_reference_exclusions, + split_reference, + ) + from policybench.scenarios import scenario_from_dict + from policybench.spec import expand_programs_for_scenario + + def object_json(text): + payload = json.loads(text) + if not isinstance(payload, dict): + raise ValueError("board input is not a JSON object") + return payload + + def csv_frame(source): + # pandas also reads the original path in load_board. Check UTF-8 here + # so every decode/parser failure has its concrete source filename. + with source.open(encoding="utf-8") as stream: + return pilot.pd.read_csv(stream) + + metadata = path / "reference_outputs.csv.meta.json" + meta = _board_input( + metadata, lambda: object_json(metadata.read_text(encoding="utf-8")) + ) + + def require_engine(): + if f"policyengine-us {pilot.board_engine(path)}" != engine: + raise StateError("the board's engine differs from the design") + pilot._require_engine(path) + + _board_input(metadata, require_engine) + reference_path = path / "reference_outputs.csv" + reference = _board_input(reference_path, lambda: csv_frame(reference_path)) + exclusions_path = path / "reference_exclusions.json" + if exclusions_path.exists(): + _board_input( + exclusions_path, + lambda: object_json(exclusions_path.read_text(encoding="utf-8")), + ) + exclusions = _board_input(exclusions_path, lambda: load_reference_exclusions(path)) + _board_input(exclusions_path, lambda: split_reference(reference, exclusions)) + scenarios_path = path / "scenarios.csv" + scenarios = _board_input(scenarios_path, lambda: csv_frame(scenarios_path)) + for _, row in scenarios.iterrows(): + scenario = _board_input( + scenarios_path, + lambda: scenario_from_dict(object_json(row["scenario_json"])), + ) + _board_input( + metadata, lambda: expand_programs_for_scenario(meta["programs"], scenario) + ) + return pilot.load_board(path) + + def entry(path: Path, record: dict, n: int) -> dict: cls, reason = classify(record, 0.0, 0.0, None) return { @@ -314,11 +393,7 @@ def _load_state(self, board) -> None: design = self.design pilot.set_stated_engine(design["engine"].split()[-1]) if board is None: - board_path = Path(design["board"]) - if f"policyengine-us {pilot.board_engine(board_path)}" != design["engine"]: - raise StateError("the board's engine differs from the design") - pilot._require_engine(board_path) - board = pilot.load_board(board_path) + board = load_board(Path(design["board"]), design["engine"]) self.board = board missing = set(self.draw_order) - set(board["scenarios"]) missing |= set(self.draw_order) - set(board["variables"]) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 81fed2e3..10b53b1c 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -924,9 +924,11 @@ def test_undecodable_stage_file_refuses_with_path(setup, tmp_path, reader, fault from pathlib import Path if fault == "nested": + sys.setrecursionlimit(1000) + depth = sys.getrecursionlimit() * 20 payload = ( b'{"attempts": [], "exclusions": [], "nested": ' - + b"[" * 20000 + b"0" + b"]" * 20000 + b"}" + + b"[" * depth + b"0" + b"]" * depth + b"}" ) elif fault == "non-object": payload = b"[]" @@ -950,15 +952,25 @@ def test_undecodable_stage_file_refuses_with_path(setup, tmp_path, reader, fault path.write_bytes(payload) if reader == "design": + refusals = [] + + class CapturedStateError(nt.StateError): + def __init__(self, *args): + super().__init__(*args) + refusals.append(self) + def unexpected_stage(*args, **kwargs): pytest.fail("an undecodable design must refuse before Stage") + monkeypatch.setattr(nt, "StateError", CapturedStateError) monkeypatch.setattr(nt, "Stage", unexpected_stage) output = io.StringIO() with redirect_stdout(output): assert nt.main(["run", str(path), "--dry-run"]) == 1 assert "The no-tools stage cannot start:" in output.getvalue() assert str(path) in output.getvalue() + assert len(refusals) == 1 + caught = SimpleNamespace(value=refusals[0]) else: if reader == "baseline": setup["design"]["baseline_exclusions"]["sha256"] = ( @@ -968,16 +980,176 @@ def unexpected_stage(*args, **kwargs): with stage(setup, provider): pytest.fail("an undecodable file must refuse before execution") assert str(path) in str(caught.value) - if fault == "invalid-utf8": - assert isinstance(caught.value.__cause__, UnicodeDecodeError) if reader in ("final", "archive"): problems = nt.stage_problems(setup["design"]) assert any(str(path) in problem for problem in problems), problems + if fault == "nested": + assert isinstance(caught.value.__cause__, RecursionError) + trace = caught.value.__cause__.__traceback__ + frames = [] + while trace is not None: + frames.append(trace.tb_frame.f_code.co_filename) + trace = trace.tb_next + assert any(name.endswith("json/decoder.py") for name in frames), frames + elif fault == "invalid-utf8": + assert isinstance(caught.value.__cause__, UnicodeDecodeError) assert provider.calls == [] """, ) +def write_board_fixture(setup, directory): + """Write a small board consumed by the unmodified harness readers.""" + import csv + + directory.mkdir() + engine = nt.pilot._versions()["policyengine-us"] + setup["design"]["engine"] = f"policyengine-us {engine}" + setup["design"]["board"] = str(directory) + meta = { + "programs": ["snap"], + "revisions": [ + {"kind": "engine_upgrade", "engine_version": f"policyengine-us {engine}"} + ], + } + (directory / "reference_outputs.csv.meta.json").write_text(json.dumps(meta)) + (directory / "reference_exclusions.json").write_text('{"exclusions": []}') + with (directory / "reference_outputs.csv").open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_id", "variable", "value"]) + writer.writerows((sid, "snap", 1) for sid in setup["design"]["draw_order"]) + with (directory / "scenarios.csv").open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_json"]) + writer.writerows( + [json.dumps({"id": sid, "state": "CA", "adults": []})] + for sid in setup["design"]["draw_order"] + ) + + +@pytest.mark.parametrize( + "reader,fault", + [ + (reader, fault) + for reader in ("metadata", "exclusions", "scenarios") + for fault in ("nested", "non-object", "invalid-utf8") + ] + + [("reference", "invalid-utf8")], +) +def test_undecodable_board_file_refuses_with_path(setup, tmp_path, reader, fault): + run_no_tools_probe( + setup, + tmp_path, + f"board_{reader}_{fault}", + f" reader, fault = {reader!r}, {fault!r}\n" + + r""" + import csv + from pathlib import Path + + directory = Path(setup["design"]["no_tools"]["runs"]).parent / "board" + write_board_fixture(setup, directory) + names = { + "metadata": "reference_outputs.csv.meta.json", + "exclusions": "reference_exclusions.json", + "scenarios": "scenarios.csv", + "reference": "reference_outputs.csv", + } + path = directory / names[reader] + if fault == "nested": + sys.setrecursionlimit(1000) + depth = sys.getrecursionlimit() * 20 + payload = b'{"nested":' + b"[" * depth + b"0" + b"]" * depth + b"}" + elif fault == "non-object": + payload = b"[]" + else: + payload = b'{"note":"\xff"}' + if reader == "scenarios": + # Quote the JSON as a CSV field, retaining the corrupt UTF-8 byte. + text = io.StringIO(newline="") + writer = csv.writer(text) + writer.writerow(["scenario_json"]) + writer.writerow([payload.decode("latin1")]) + payload = text.getvalue().encode("latin1") + elif reader == "reference": + payload = b"scenario_id,variable,value\nh2,\xff,1\n" + path.write_bytes(payload) + provider = Provider() + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], board=None, run_household=provider, + completion_for=lambda profile: None, out=io.StringIO(), + ): + pytest.fail("an unusable board must refuse before execution") + assert str(path) in str(caught.value) + if fault == "nested": + assert isinstance(caught.value.__cause__, RecursionError) + trace = caught.value.__cause__.__traceback__ + frames = [] + while trace is not None: + frames.append(trace.tb_frame.f_code.co_filename) + trace = trace.tb_next + assert any(name.endswith("json/decoder.py") for name in frames), frames + elif fault == "invalid-utf8": + assert isinstance(caught.value.__cause__, UnicodeDecodeError) + assert provider.calls == [] + """, + ) + + +@pytest.mark.parametrize("with_exclusions", [True, False]) +def test_valid_board_loads_through_harness(setup, tmp_path, with_exclusions): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + if not with_exclusions: + (directory / "reference_exclusions.json").unlink() + expected = nt.pilot.load_board(directory) + provider = Provider() + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.board.keys() == expected.keys() + for key in ("reference", "scored", "excluded", "scenarios_frame"): + assert plan.board[key].equals(expected[key]) + for key in ("meta", "scenarios", "variables"): + assert plan.board[key] == expected[key] + assert plan.spend.spent_usd == 0 + assert plan.histories == {model: {} for model in nt.CHEAP} + assert provider.calls == [] + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_oversized_record_cost_refuses_with_path(setup, reader, boundary): + root = Path(setup["design"]["no_tools"]["runs"]) + path = root / nt.CHEAP[0] / "none/confirmatory/households" + path /= "h2.json" if reader == "final" else "attempts/h2-1.json" + payload = record( + "h2", + SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic"), + status="provider_error", + error="APIConnectionError", + cost=10**400, + ) + payload["attempts"] = [] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload)) + provider = Provider() + if boundary == "startup": + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("an oversized cost must refuse before execution") + assert str(path) in str(caught.value) + assert isinstance(caught.value.__cause__, OverflowError) + else: + problems = nt.stage_problems(setup["design"]) + assert any(str(path) in problem for problem in problems), problems + assert provider.calls == [] + + def run_no_tools_probe(setup, tmp_path, name, body): """Bound a signal or worker-shutdown regression to its own process.""" import textwrap @@ -1003,6 +1175,7 @@ def run_no_tools_probe(setup, tmp_path, name, body): "load", "paused", "track_interrupt_return", + "write_board_fixture", ): globals()[helper] = helpers[helper] design = json.loads(sys.argv[1]) From 6928b06997bbada1ffad0860d74832d36f1b1d3f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 13:00:02 -0400 Subject: [PATCH 44/64] Confirmatory runner: check the CLI with a real board Give the CLI option test a real small board so it exercises the runner's input boundary and unchanged harness loader. Reference agreement failures must name both the reference CSV and exclusions JSON because either input can make that validation unusable. Preserve the checkpoint branch and its existing commits. The delivery branch retains two commits while including this fixture update and the paired reference paths. The CLI dry run must parse all model options and name its owed work before provider calls. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 7 +++++-- tests/test_confirmatory_no_tools.py | 8 ++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index b92d360d..9f4a06cb 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -100,7 +100,7 @@ def read_record(path: Path) -> dict: raise StateError(f"{path}: {error}") from error -def _board_input(path: Path, read): +def _board_input(path: Path | str, read): """Keep a delegated board reader's refusal attached to its input file.""" try: return read() @@ -158,7 +158,10 @@ def require_engine(): lambda: object_json(exclusions_path.read_text(encoding="utf-8")), ) exclusions = _board_input(exclusions_path, lambda: load_reference_exclusions(path)) - _board_input(exclusions_path, lambda: split_reference(reference, exclusions)) + _board_input( + f"{reference_path} and {exclusions_path}", + lambda: split_reference(reference, exclusions), + ) scenarios_path = path / "scenarios.csv" scenarios = _board_input(scenarios_path, lambda: csv_frame(scenarios_path)) for _, row in scenarios.iterrows(): diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 10b53b1c..9ea28ac0 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -371,14 +371,10 @@ def test_registered_cap_matches_the_report_when_available(): assert nt.NO_TOOLS_CAP == cap == 5.0 -def test_main_supports_multiple_model_options_and_dry_run( - setup, tmp_path, monkeypatch, capsys -): +def test_main_supports_multiple_model_options_and_dry_run(setup, tmp_path, capsys): + write_board_fixture(setup, tmp_path / "board") path = tmp_path / "design.json" path.write_text(json.dumps(setup["design"])) - monkeypatch.setattr(nt.pilot, "board_engine", lambda path: "2.60.4") - monkeypatch.setattr(nt.pilot, "_require_engine", lambda path: None) - monkeypatch.setattr(nt.pilot, "load_board", lambda path: setup["board"]) assert ( nt.main( [ From 8b6c8f70b080c31daae4ad28576bc5f89711f7f4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 17:17:25 -0400 Subject: [PATCH 45/64] Confirmatory runner: guard board rereads and strict saved costs Hash the bytes used by every board input check and guard the unchanged harness loader. Exclusions existence failures must name the exclusions file. Delegated I/O failures must name their filename or the board folder. Other delegated failures must name files whose checked bytes changed, or the board folder when no change is established, while preserving the original cause. Saved finals and archives must contain an object totals with a present numeric cost_usd. Booleans and other nonnumbers are refused. Costs must convert to finite nonnegative floats. Startup and calculator preflight use this reader; calculator attempt_cost and valid spend arithmetic stay unchanged. Provider-free regressions record 65 failures and 49 passes against 6928b069 before these fixes; final verification follows. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 74 ++++-- tests/test_confirmatory_no_tools.py | 335 ++++++++++++++++++++++++++++ 2 files changed, 393 insertions(+), 16 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 9f4a06cb..412b61a4 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -33,6 +33,8 @@ import argparse import fcntl +import hashlib +import io import json import math import os @@ -85,7 +87,13 @@ def read_record(path: Path) -> dict: raise ValueError("native prior_attempts cannot preserve full attempts") if not isinstance(record.get("attempts"), list): raise ValueError("record lacks an attempts list") - cost = attempt_cost(record) + totals = record.get("totals") + if not isinstance(totals, dict): + raise ValueError("record totals must be an object with cost_usd") + raw_cost = totals.get("cost_usd") + if isinstance(raw_cost, bool) or not isinstance(raw_cost, (int, float)): + raise ValueError("record cost_usd must be a present number") + cost = float(raw_cost) if not math.isfinite(cost) or cost < 0: raise ValueError("attempt cost must be finite and nonnegative") return record @@ -100,20 +108,23 @@ def read_record(path: Path) -> dict: raise StateError(f"{path}: {error}") from error +BOARD_INPUT_ERRORS = ( + OSError, + ValueError, + TypeError, + AttributeError, + KeyError, + RecursionError, + OverflowError, + SystemExit, +) + + def _board_input(path: Path | str, read): """Keep a delegated board reader's refusal attached to its input file.""" try: return read() - except ( - OSError, - ValueError, - TypeError, - AttributeError, - KeyError, - RecursionError, - OverflowError, - SystemExit, - ) as error: + except BOARD_INPUT_ERRORS as error: raise StateError(f"{path}: {error}") from error @@ -126,6 +137,13 @@ def load_board(path: Path, engine: str) -> dict: from policybench.scenarios import scenario_from_dict from policybench.spec import expand_programs_for_scenario + checked = {} + + def checked_bytes(source): + data = source.read_bytes() + checked[source] = hashlib.sha256(data).hexdigest() + return data + def object_json(text): payload = json.loads(text) if not isinstance(payload, dict): @@ -135,12 +153,14 @@ def object_json(text): def csv_frame(source): # pandas also reads the original path in load_board. Check UTF-8 here # so every decode/parser failure has its concrete source filename. - with source.open(encoding="utf-8") as stream: + with io.TextIOWrapper( + io.BytesIO(checked_bytes(source)), encoding="utf-8" + ) as stream: return pilot.pd.read_csv(stream) metadata = path / "reference_outputs.csv.meta.json" meta = _board_input( - metadata, lambda: object_json(metadata.read_text(encoding="utf-8")) + metadata, lambda: object_json(checked_bytes(metadata).decode("utf-8")) ) def require_engine(): @@ -152,11 +172,13 @@ def require_engine(): reference_path = path / "reference_outputs.csv" reference = _board_input(reference_path, lambda: csv_frame(reference_path)) exclusions_path = path / "reference_exclusions.json" - if exclusions_path.exists(): + if _board_input(exclusions_path, exclusions_path.exists): _board_input( exclusions_path, - lambda: object_json(exclusions_path.read_text(encoding="utf-8")), + lambda: object_json(checked_bytes(exclusions_path).decode("utf-8")), ) + else: + checked[exclusions_path] = None exclusions = _board_input(exclusions_path, lambda: load_reference_exclusions(path)) _board_input( f"{reference_path} and {exclusions_path}", @@ -172,7 +194,27 @@ def require_engine(): _board_input( metadata, lambda: expand_programs_for_scenario(meta["programs"], scenario) ) - return pilot.load_board(path) + try: + return pilot.load_board(path) + except BOARD_INPUT_ERRORS as error: + if isinstance(error, OSError): + raise StateError(f"{error.filename or path}: {error}") from error + changed, unreadable = [], [] + for source, digest in checked.items(): + try: + current = sha256_file(source) + except FileNotFoundError: + current = None + except BOARD_INPUT_ERRORS as reread_error: + unreadable.append(f"{source}: cannot recheck: {reread_error}") + continue + if current != digest: + changed.append(str(source)) + at = " and ".join(changed) if changed else str(path) + detail = "; changed after it was checked" if changed else "" + if unreadable: + detail += "; " + "; ".join(unreadable) + raise StateError(f"{at}{detail}: {error}") from error def entry(path: Path, record: dict, n: int) -> dict: diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 9ea28ac0..d5b253e1 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -1117,6 +1117,224 @@ def test_valid_board_loads_through_harness(setup, tmp_path, with_exclusions): assert provider.calls == [] +@pytest.mark.parametrize("with_filename", [True, False]) +def test_delegated_board_oserror_refuses_with_path( + setup, tmp_path, monkeypatch, with_filename +): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + reference = directory / "reference_outputs.csv" + error = ( + OSError(5, "synthetic reread failure", str(reference)) + if with_filename + else OSError("synthetic reread failure") + ) + read_csv = nt.pilot.pd.read_csv + rereads = [] + + def fail_reference_reread(source, *args, **kwargs): + # Prechecks pass an opened stream; the unchanged harness passes a path. + if isinstance(source, Path) and source == reference: + rereads.append(source) + raise error + return read_csv(source, *args, **kwargs) + + monkeypatch.setattr(nt.pilot.pd, "read_csv", fail_reference_reread) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("a failed board reread must refuse before execution") + assert rereads == [reference] + assert caught.value.__cause__ is error + assert str(reference if with_filename else directory) in str(caught.value) + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize("reader", ["metadata", "scenarios"]) +def test_board_changed_to_nested_json_after_check_refuses_with_path( + setup, tmp_path, reader +): + run_no_tools_probe( + setup, + tmp_path, + f"board_changed_{reader}", + f" reader = {reader!r}\n" + + r""" + import csv + from pathlib import Path + + sys.setrecursionlimit(1000) + depth = sys.getrecursionlimit() * 20 + payload = '{"nested":' + "[" * depth + "0" + "]" * depth + "}" + directory = Path(setup["design"]["no_tools"]["runs"]).parent / "board" + write_board_fixture(setup, directory) + filename = ( + "reference_outputs.csv.meta.json" if reader == "metadata" + else "scenarios.csv" + ) + path = directory / filename + load_board = nt.pilot.load_board + delegates = [] + + def change_then_load(source): + delegates.append(source) + if reader == "metadata": + path.write_text(payload) + else: + with path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_json"]) + writer.writerow([payload]) + return load_board(source) + + monkeypatch.setattr(nt.pilot, "load_board", change_then_load) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], board=None, run_household=provider, + completion_for=lambda profile: None, out=io.StringIO(), + ): + pytest.fail("a changed board must refuse before execution") + assert delegates == [directory], "all prechecks must finish first" + assert str(path) in str(caught.value) + assert "changed after" in str(caught.value) + assert isinstance(caught.value.__cause__, RecursionError) + trace = caught.value.__cause__.__traceback__ + frames = [] + while trace is not None: + frames.append(trace.tb_frame.f_code.co_filename) + trace = trace.tb_next + assert any(name.endswith("json/decoder.py") for name in frames), frames + finally: + assert provider.calls == [] + """, + ) + + +def test_board_exclusions_exists_failure_refuses_with_path( + setup, tmp_path, monkeypatch +): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + exclusions = directory / "reference_exclusions.json" + error = PermissionError(13, "synthetic unreadable board parent", str(exclusions)) + exists = Path.exists + checks = [] + + def fail_exclusions_exists(path): + if path == exclusions: + checks.append(path) + raise error + return exists(path) + + monkeypatch.setattr(Path, "exists", fail_exclusions_exists) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("an unreadable exclusions path must refuse") + assert checks == [exclusions] + assert caught.value.__cause__ is error + assert str(exclusions) in str(caught.value) + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize( + "error_type", + [ + ValueError, + TypeError, + AttributeError, + KeyError, + RecursionError, + OverflowError, + SystemExit, + ], +) +def test_unchanged_board_delegate_failure_names_folder( + setup, tmp_path, monkeypatch, error_type +): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + error = error_type("synthetic delegated failure") + delegates = [] + + def fail_load(source): + delegates.append(source) + raise error + + monkeypatch.setattr(nt.pilot, "load_board", fail_load) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("a failed board delegate must refuse before execution") + assert delegates == [directory] + assert caught.value.__cause__ is error + assert str(directory) in str(caught.value) + assert "changed after" not in str(caught.value) + finally: + assert provider.calls == [] + + +def test_board_delegate_failure_names_each_changed_file(setup, tmp_path, monkeypatch): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + paths = [ + directory / name + for name in ("reference_outputs.csv.meta.json", "scenarios.csv") + ] + error = ValueError("synthetic delegated failure") + delegates = [] + + def change_then_fail(source): + delegates.append(source) + for path in paths: + path.write_bytes(path.read_bytes() + b"\n") + raise error + + monkeypatch.setattr(nt.pilot, "load_board", change_then_fail) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("a changed board must refuse before execution") + assert delegates == [directory] + assert caught.value.__cause__ is error + assert all(str(path) in str(caught.value) for path in paths) + assert "changed after" in str(caught.value) + finally: + assert provider.calls == [] + + @pytest.mark.parametrize("reader", ["final", "archive"]) @pytest.mark.parametrize("boundary", ["startup", "preflight"]) def test_oversized_record_cost_refuses_with_path(setup, reader, boundary): @@ -1146,6 +1364,123 @@ def test_oversized_record_cost_refuses_with_path(setup, reader, boundary): assert provider.calls == [] +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize( + "fault", + [ + "missing-totals", + "null-totals", + "false-totals", + "empty-list-totals", + "empty-string-totals", + "true-totals", + "number-totals", + "list-totals", + "missing-cost", + "null", + "true", + "false", + "numeric-string", + "empty-string", + "invalid-string", + "empty-list", + "list", + "empty-object", + "object", + "nan", + "infinity", + "negative-infinity", + "negative", + "overflow", + ], +) +def test_strict_saved_cost_refuses_with_path(setup, reader, boundary, fault): + root = Path(setup["design"]["no_tools"]["runs"]) + path = root / nt.CHEAP[0] / "none/confirmatory/households" + path /= "h2.json" if reader == "final" else "attempts/h2-1.json" + payload = record( + "h2", + SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic"), + status="provider_error", + error="APIConnectionError", + ) + payload["attempts"] = [] + totals = { + "null-totals": None, + "false-totals": False, + "empty-list-totals": [], + "empty-string-totals": "", + "true-totals": True, + "number-totals": 1, + "list-totals": [1], + } + costs = { + "null": None, + "true": True, + "false": False, + "numeric-string": "0.25", + "empty-string": "", + "invalid-string": "invalid", + "empty-list": [], + "list": [0.25], + "empty-object": {}, + "object": {"value": 0.25}, + "nan": float("nan"), + "infinity": float("inf"), + "negative-infinity": float("-inf"), + "negative": -0.25, + "overflow": 10**400, + } + if fault == "missing-totals": + del payload["totals"] + elif fault in totals: + payload["totals"] = totals[fault] + elif fault == "missing-cost": + payload["totals"] = {} + else: + payload["totals"]["cost_usd"] = costs[fault] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload)) + provider = Provider() + try: + if boundary == "startup": + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("an invalid saved cost must refuse before execution") + assert str(path) in str(caught.value) + else: + problems = nt.stage_problems(setup["design"]) + assert any(str(path) in problem for problem in problems), problems + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize("cost", [0, 0.0, -0.0, 0.25, 2]) +def test_valid_saved_numeric_cost_preserves_spend(setup, cost): + root = Path(setup["design"]["no_tools"]["runs"]) + folder = root / nt.CHEAP[0] / "none/confirmatory/households" + folder.mkdir(parents=True) + profile = SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic") + archive = folder / "attempts/h2-1.json" + archive.parent.mkdir() + prior = record( + "h2", profile, status="provider_error", error="APIConnectionError", cost=cost + ) + prior["attempts"] = [] + archive.write_text(json.dumps(prior)) + saved = record("h2", profile, cost=cost) + saved["attempts"] = [nt.entry(archive, prior, 1)] + (folder / "h2.json").write_text(json.dumps(saved)) + provider = Provider() + with stage(setup, provider) as plan: + assert plan.spend.spent_usd == 2 * float(cost) + assert not any( + str(folder) in problem for problem in nt.stage_problems(setup["design"]) + ) + assert provider.calls == [] + + def run_no_tools_probe(setup, tmp_path, name, body): """Bound a signal or worker-shutdown regression to its own process.""" import textwrap From 0300257c18b032032f525802963e32dc26a29580 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 18:10:47 -0400 Subject: [PATCH 46/64] Confirmatory runner: align saved treatment and link checks Check every saved final and archive against the pinned none-arm treatment before providers or calculator admission. Model, arm, prompt version, wording, notice, explicit null calculator fields, board outputs and provider turns must match. A completed reply or nonnull answer requires its contract, sent prompt and rebuilt hashes. Empty failed attempts may omit that evidence; any evidence present must match. Cheap finals must be in the draw and retain a predictions object. Preflight loads the checked board lazily to apply these same rules. Final link numbers must be consecutive integers excluding booleans. Archived prefixes retain the report equality rule. Known link fields must match while extra keys and an absent status matching null remain permitted, and earlier prefixes are preserved. Archive classification failures must name the archive. Valid numeric spend, rerun eligibility, interrupted-archive recovery, locking and worker execution keep their existing rules. Keep the minimal state-machine fixtures explicit about their injected treatment validator. Add real-board fixtures with harness-built evidence, conditional failure controls and exact archive/spend resume checks. Record 164 agreement failures and 32 controls against the original reader before these fixes. The final selection passes 315 cases, including legacy assertions updated for the new refusal reasons. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 220 +++++++++- tests/test_confirmatory_no_tools.py | 13 +- tests/test_confirmatory_no_tools_records.py | 454 ++++++++++++++++++++ 3 files changed, 674 insertions(+), 13 deletions(-) create mode 100644 tests/test_confirmatory_no_tools_records.py diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 412b61a4..acdf0f88 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -53,6 +53,7 @@ action, attempt_cost, classify, + completed_turns, default_completion, pilot, sha256_file, @@ -61,6 +62,9 @@ NO_TOOLS_CAP = 5.0 START_GRACE_SECONDS = 10 +NO_TOOLS_WORDING_SHA256 = ( + "43b13e4e64463f4ee1654e9546672899f80ca5e1ce18e93cf2eb04091b3a2699" +) class StateError(ValueError): @@ -217,6 +221,150 @@ def require_engine(): raise StateError(f"{at}{detail}: {error}") from error +def _no_tools_answered(record: dict) -> bool: + predictions = record.get("predictions") + return isinstance(predictions, dict) and any( + value is not None for value in predictions.values() + ) + + +def _saved_treatment_problems(record: dict, model: str, board: dict) -> list[str]: + """Report treatment_problems specialized to the fixed none arm. + + Keep the report's conditional evidence requirements and conversions. + In particular, empty failed attempts may omit prompt evidence, variables + use the report's list conversion, and unexamined fields remain unexamined. + """ + problems = [] + if record.get("model") != model: + return [f"record names model {record.get('model')!r}"] + if record.get("arm") != "none": + return [f"arm {record.get('arm')!r} differs from 'none'"] + + profile = pilot.model_profile(model, arm="none", prompt_version=PROMPT_VERSION) + treatment = pilot.profile_treatment(profile) + if treatment["tool_features"] is not None: + raise ValueError( + f"none: the harness gives features {treatment['tool_features']}" + ) + + answered = _no_tools_answered(record) + owed = ( + isinstance(record.get("turns"), list) and bool(completed_turns(record)) + ) or answered + keys = ( + "prompt_version", + "prompt_wording_sha256", + "tool_notice_sha256", + "tool_features", + "calculate_tool_sha256", + "variables", + "turns", + ) + if owed: + keys += ( + "contract", + "messages", + "board_prompt_sha256", + "sent_board_prompt_sha256", + ) + for key in keys: + if key not in record: + problems.append(f"missing no-tools field {key!r}") + if "predictions" in record and not isinstance(record["predictions"], dict): + problems.append("no-tools predictions must be an object from run_household") + + contract = record.get("contract") + if (owed or "contract" in record) and contract != profile.contract: + problems.append( + f"answer contract {contract!r} differs from {profile.contract!r}" + ) + if record.get("prompt_version") != PROMPT_VERSION: + problems.append(f"prompt version {record.get('prompt_version')!r}") + wording = pilot.prompt_wording_sha256(profile) + if ( + wording != NO_TOOLS_WORDING_SHA256 + or record.get("prompt_wording_sha256") != NO_TOOLS_WORDING_SHA256 + ): + problems.append( + f"prompt wording {str(record.get('prompt_wording_sha256'))[:12]}, " + f"harness {str(wording)[:12]}" + ) + notice = hashlib.sha256(profile.notice.encode()).hexdigest() + if record.get("tool_notice_sha256") != notice: + problems.append("the notice hash differs from the arm's") + for key in ("tool_features", "calculate_tool_sha256"): + if record.get(key) != treatment[key]: + problems.append(f"{key} differs from the none arm's") + + sid = record.get("scenario_id") + if sid not in board["scenarios"]: + return problems + ["no such household on the board"] + if list(record.get("variables") or []) != list(board["variables"][sid]): + problems.append("its outputs differ from the board's") + if not isinstance(record.get("turns"), list): + return problems + ["no-tools turns must be an array from run_household"] + if answered and not completed_turns(record): + problems.append("a no-tools answer has no completed turn") + for number, turn in enumerate(record.get("turns") or [], start=1): + served = turn.get("provider_resolved_model") + if not served: + problems.append(f"turn {number} reports no provider's model") + elif served != profile.model_id.split("/")[-1]: + problems.append( + f"turn {number}: providers reported {served!r}, " + f"expected {profile.model_id}" + ) + + messages = record.get("messages") + if (owed or "messages" in record) and not isinstance(messages, list): + return problems + [ + "no-tools messages must be an array from conversation.transcript" + ] + if not messages: + if owed: + problems.append("no no-tools sent prompt in messages") + if completed_turns(record): + problems.append("no sent prompt in messages") + if owed: + return problems + + built, settings = pilot.build_initial_request( + board["scenarios"][sid], board["variables"][sid], profile + ) + if messages: + first = messages[0] if isinstance(messages[0], dict) else {} + text = ( + first.get("input") + if isinstance(first.get("input"), str) + else first.get("content") + ) + if text != built: + problems.append("the sent prompt differs from the harness's rebuilt prompt") + for key, description in ( + ("sent_board_prompt_sha256", "sent board prompt"), + ("board_prompt_sha256", "board prompt"), + ): + if (messages or key in record) and record.get(key) != settings[f"_{key}"]: + problems.append(f"the {description} hash differs from the rebuilt one") + return problems + + +def validate_treatment(path, record, model, board, engine): + """Check each saved record against the report's pinned none-arm treatment.""" + try: + with pilot.stating_engine(engine.split()[-1]): + problems = _saved_treatment_problems( + record, model, board() if callable(board) else board + ) + except BOARD_INPUT_ERRORS as error: + raise StateError( + f"{path}: cannot check saved no-tools treatment: {error}" + ) from error + if problems: + raise StateError(f"{path}: " + "; ".join(problems)) + + def entry(path: Path, record: dict, n: int) -> dict: cls, reason = classify(record, 0.0, 0.0, None) return { @@ -229,7 +377,17 @@ def entry(path: Path, record: dict, n: int) -> dict: } -def history(folder: Path) -> tuple[dict[str, dict], float]: +def _attempts_agree(listed: list, entries: list) -> bool: + """Match known link fields and retain each archive's exact earlier prefix.""" + if len(listed) != len(entries) or listed[:-1] != entries[:-1]: + return False + return not entries or ( + isinstance(listed[-1], dict) + and all(listed[-1].get(key) == value for key, value in entries[-1].items()) + ) + + +def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], float]: """Validate a model's histories, including an interrupted archive, and sum spend.""" archives: dict[str, list[tuple[int, Path]]] = {} for path in sorted((folder / "attempts").glob("*.json")): @@ -254,34 +412,58 @@ def history(folder: Path) -> tuple[dict[str, dict], float]: record = read_record(path) if record.get("scenario_id") != sid: raise StateError(f"{path}: household differs from archive name {sid}") - if record["attempts"] != entries: + if validate is not None: + validate(path, record) + if not _attempts_agree(record["attempts"], entries): raise StateError(f"{path}: attempts disagree with earlier archives") - cls, _ = classify(record, 0.0, 0.0, None) + try: + cls, _ = classify(record, 0.0, 0.0, None) + except BOARD_INPUT_ERRORS as error: + raise StateError( + f"{path}: cannot classify saved archive: {error}" + ) from error if action(cls, n) not in ("rerun", "pause-rerun"): raise StateError(f"{path}: archived attempt was ineligible for rerun") - entries.append(entry(path, record, n)) + entries = record["attempts"] + [entry(path, record, n)] spent += attempt_cost(record) final = None if sid in finals: path = finals[sid] final = read_record(path) + if drawn is not None: + if sid not in drawn: + raise StateError(f"{path}: cheap-model final is outside the draw") + if not isinstance(final.get("predictions"), dict): + raise StateError( + f"{path}: final no-tools predictions must be an object" + ) if final.get("scenario_id") != sid: raise StateError(f"{path}: household differs from final name {sid}") - if final["attempts"] != entries: + for n, link in enumerate(final["attempts"], 1): + if ( + not isinstance(link, dict) + or type(link.get("n")) is not int + or link["n"] != n + ): + raise StateError(f"{path}: attempt link n must be integer {n}") + if validate is not None: + validate(path, final) + if not _attempts_agree(final["attempts"], entries): raise StateError(f"{path}: attempts disagree with archived files") try: classify(final, 0.0, 0.0, None) - except (TypeError, ValueError, AttributeError, KeyError) as error: + except BOARD_INPUT_ERRORS as error: raise StateError( f"{path}: cannot classify saved final: {error}" ) from error + entries = final["attempts"] spent += attempt_cost(final) states[sid] = {"entries": entries, "final": final} return states, spent def stage_problems(design: dict) -> list[str]: - """Read stage histories and standing finals without a board or provider.""" + """Check saved histories, treatment and standing without a provider.""" problems = [] try: spec = design["no_tools"] @@ -293,9 +475,23 @@ def stage_problems(design: dict) -> list[str]: models.update(path.name for path in root.iterdir() if path.is_dir()) except (OSError, KeyError, TypeError, ValueError) as error: return [f"no-tools stage: {error}"] + board = None + + def checked_board(): + nonlocal board + if board is None: + board = load_board(Path(design["board"]), design["engine"]) + return board + for model in sorted(models): try: - states, _ = history(root / model / folder / stage / "households") + states, _ = history( + root / model / folder / stage / "households", + validate=lambda path, record: validate_treatment( + path, record, model, checked_board, design["engine"] + ), + drawn=set(draw_order) if model in CHEAP else None, + ) if model not in CHEAP: continue for sid in draw_order: @@ -454,7 +650,13 @@ def _load_state(self, board) -> None: else set() ) for model in sorted(folders | set(self.models)): - states, cost = history(self.households(model)) + states, cost = history( + self.households(model), + validate=lambda path, record: validate_treatment( + path, record, model, self.board, design["engine"] + ), + drawn=set(self.draw_order) if model in CHEAP else None, + ) self.histories[model] = states spent += cost # A saved ceiling failure is a stage-wide stop on every invocation. diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index d5b253e1..7bead2a2 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -20,7 +20,10 @@ @pytest.fixture -def setup(tmp_path): +def setup(tmp_path, monkeypatch): + # These state-machine fixtures inject a minimal board and fake records. + # Real treatment pins and prompts are covered in the saved-record suite. + monkeypatch.setattr(nt, "validate_treatment", lambda *args: None, raising=False) draw = ["h2", "h1"] baseline = tmp_path / "baseline-exclusions.json" baseline.write_text('{"exclusions": []}\n') @@ -309,7 +312,8 @@ def test_refuse_a_final_with_corrupted_attempt_links(setup, field, value): payload = load(path) payload["attempts"][0][field] = value path.write_text(json.dumps(payload)) - with pytest.raises(nt.StateError, match="attempts disagree"): + match = "attempt link n" if field == "n" else "attempts disagree" + with pytest.raises(nt.StateError, match=match): stage(setup) @@ -879,10 +883,11 @@ def test_stage_problems_classifies_every_saved_final(setup, category): payload.update(attempts=[], turns=[None]) path = plan.households(model) / f"{sid}.json" nt.write_json(path, payload) - with pytest.raises((nt.StateError, AttributeError), match="get"): + match = "outside the draw" if category == "outside-draw" else "get" + with pytest.raises(nt.StateError, match=match): stage(setup) problems = nt.stage_problems(setup["design"]) - assert any(model in problem and "get" in problem for problem in problems), ( + assert any(model in problem and match in problem for problem in problems), ( "R3: every final refused by initialization must fail stage preflight", problems, ) diff --git a/tests/test_confirmatory_no_tools_records.py b/tests/test_confirmatory_no_tools_records.py new file mode 100644 index 00000000..c145012c --- /dev/null +++ b/tests/test_confirmatory_no_tools_records.py @@ -0,0 +1,454 @@ +"""Saved no-tools evidence must agree with the report before any provider call.""" + +from __future__ import annotations + +import hashlib +import io +import json +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests.test_confirmatory_no_tools import Provider, nt, write_board_fixture + +OPTIONAL_EVIDENCE = ( + "contract", + "messages", + "board_prompt_sha256", + "sent_board_prompt_sha256", +) +REQUIRED_FIELDS = ( + "model", + "arm", + "prompt_version", + "prompt_wording_sha256", + "tool_notice_sha256", + "tool_features", + "calculate_tool_sha256", + "variables", + "turns", + *OPTIONAL_EVIDENCE, +) + + +def full_record(board, model, engine, sid="h2"): + """Rebuild the exact treatment pins and prompt through the real harness.""" + profile = nt.pilot.model_profile( + model, arm="none", prompt_version=nt.PROMPT_VERSION + ) + with nt.pilot.stating_engine(engine.split()[-1]): + prompt, settings = nt.pilot.build_initial_request( + board["scenarios"][sid], board["variables"][sid], profile + ) + return { + "scenario_id": sid, + "model": model, + "arm": "none", + "status": "complete", + "error": None, + "prompt_version": nt.PROMPT_VERSION, + "prompt_wording_sha256": nt.pilot.prompt_wording_sha256(profile), + "tool_notice_sha256": hashlib.sha256(profile.notice.encode()).hexdigest(), + **nt.pilot.profile_treatment(profile), + "variables": list(board["variables"][sid]), + "turns": [{"provider_resolved_model": profile.model_id.split("/")[-1]}], + "predictions": {"snap": 1}, + "contract": profile.contract, + "messages": [{"role": "user", "content": prompt}], + "board_prompt_sha256": settings["_board_prompt_sha256"], + "sent_board_prompt_sha256": settings["_sent_board_prompt_sha256"], + "totals": {"cost_usd": 0.1}, + "attempts": [], + } + + +def write_fixture_json(path, payload): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload)) + + +def save_records(saved, change_link=None): + """Keep downstream byte hashes and link prefixes valid after one mutation.""" + + def save(path, payload): + text = json.dumps(payload) + if saved.serialized.get(path) != text: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + saved.serialized[path] = text + + links = [] + for n, payload in enumerate(saved.archives, 1): + payload["attempts"] = deepcopy(links) + path = saved.folder / "attempts" / f"h2-{n}.json" + save(path, payload) + try: + link = nt.entry(path, payload, n) + except (AttributeError, TypeError, ValueError, KeyError): + # Malformed turn evidence must reach the startup reader, rather + # than fail while this fixture refreshes otherwise valid links. + link = deepcopy(saved.original_links[n - 1]) + link["sha256"] = nt.sha256_file(path) + if change_link is not None: + change_link(link) + links.append(link) + for model, payload in saved.finals.items(): + payload["attempts"] = deepcopy(links) if model == nt.CHEAP[0] else [] + path = saved.root / model / "none/confirmatory/households/h2.json" + save(path, payload) + saved.original_links = links + + +@pytest.fixture(scope="module") +def treatment_templates(tmp_path_factory): + """Build actual treatment templates once and retain the installed versions.""" + directory = tmp_path_factory.mktemp("saved-record-board") + versions = nt.pilot._versions() + baseline = directory / "baseline-exclusions.json" + baseline.write_text('{"exclusions": []}\n') + design = { + "draw_order": ["h2", "outside"], + "baseline_exclusions": { + "path": str(baseline), + "sha256": nt.sha256_file(baseline), + }, + "no_tools": { + "runs": "", + "folder": "none", + "stage": "confirmatory", + }, + } + with pytest.MonkeyPatch.context() as patch: + patch.setattr(nt.pilot, "_versions", lambda: versions) + write_board_fixture({"design": design}, directory / "board") + design["draw_order"] = ["h2"] + board = nt.pilot.load_board(Path(design["board"])) + finals = { + model: full_record(board, model, design["engine"]) for model in nt.CHEAP + } + yield design, finals + + +@pytest.fixture +def saved_records(tmp_path, treatment_templates): + template, template_finals = treatment_templates + design, finals = deepcopy(template), deepcopy(template_finals) + design["no_tools"]["runs"] = str(tmp_path / "no-tools") + archives = [deepcopy(finals[nt.CHEAP[0]]) for _ in range(2)] + for payload in archives: + payload.update(status="provider_error", error="APIConnectionError") + root = Path(design["no_tools"]["runs"]) + folder = root / nt.CHEAP[0] / "none/confirmatory/households" + saved = SimpleNamespace( + design=design, + root=root, + folder=folder, + finals=finals, + archives=archives, + original_links=[], + serialized={}, + paths={"final": folder / "h2.json", "archive": folder / "attempts/h2-1.json"}, + ) + save_records(saved) + return saved + + +def target_record(saved, reader): + return saved.finals[nt.CHEAP[0]] if reader == "final" else saved.archives[0] + + +def no_reply(payload, truncated=False): + payload["predictions"] = {"snap": None} + payload["turns"] = ( + [{**payload["turns"][0], "truncated_and_retried": True}] if truncated else [] + ) + for key in OPTIONAL_EVIDENCE: + payload.pop(key, None) + + +BAD_VALUES = { + "model": "synthetic-other-model", + "arm": "pe", + "prompt_version": "v1", + "prompt_wording_sha256": "wrong", + "tool_notice_sha256": "wrong", + "tool_features": [], + "calculate_tool_sha256": "wrong", + "variables": ["other"], + "contract": "synthetic-other-contract", + "board_prompt_sha256": "wrong", + "sent_board_prompt_sha256": "wrong", + "messages": {"content": "wrong"}, + "predictions": [], + "turns": None, +} +MUTATIONS = ( + [f"missing:{field}" for field in REQUIRED_FIELDS] + + [f"wrong:{field}" for field in BAD_VALUES] + + [ + "malformed-turn", + "missing-provider-model", + "wrong-provider-model", + "answer-without-completed-turn", + "empty-messages", + "malformed-message", + "wrong-sent-prompt", + ] + + [f"fallback:{field}" for field in OPTIONAL_EVIDENCE] +) + + +def mutate_record(payload, mutation): + kind, _, field = mutation.partition(":") + if kind == "missing": + payload.pop(field) + elif kind == "wrong": + payload[field] = deepcopy(BAD_VALUES[field]) + elif kind == "fallback": + no_reply(payload) + payload[field] = [] if field == "messages" else "wrong" + if field == "messages": + payload[field] = {"content": "wrong"} + elif mutation == "malformed-turn": + payload["turns"] = [None] + elif mutation == "missing-provider-model": + payload["turns"] = [{}] + elif mutation == "wrong-provider-model": + payload["turns"][0]["provider_resolved_model"] = "synthetic-other-model" + elif mutation == "answer-without-completed-turn": + payload["turns"] = [] + elif mutation == "empty-messages": + payload["messages"] = [] + elif mutation == "malformed-message": + payload["messages"] = [None] + else: + assert mutation == "wrong-sent-prompt" + payload["messages"][0]["content"] = "synthetic other household" + + +def assert_boundary(saved, boundary, refuses, expected_spend=0.5): + provider = Provider() + try: + if boundary == "startup": + if refuses is not None: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + saved.design, + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail( + "unusable saved evidence must refuse before execution" + ) + assert str(refuses) in str(caught.value) + else: + with nt.Stage( + saved.design, + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.spend.spent_usd == pytest.approx(expected_spend) + assert all(plan.standing(model, "h2") for model in nt.CHEAP) + else: + problems = nt.stage_problems(saved.design) + if refuses is not None: + assert any(str(refuses) in problem for problem in problems), problems + else: + assert problems == [] + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("mutation", MUTATIONS) +def test_saved_treatment_refuses_with_path(saved_records, reader, boundary, mutation): + saved = saved_records + mutate_record(target_record(saved, reader), mutation) + save_records(saved) + assert_boundary(saved, boundary, saved.paths[reader]) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("number", [True, 1.0], ids=["boolean", "float"]) +def test_saved_attempt_link_number_requires_integer(saved_records, boundary, number): + saved = saved_records + + def replace_number(link): + if link["n"] == 1: + link["n"] = number + + save_records(saved, replace_number) + assert_boundary(saved, boundary, saved.paths["final"]) + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize( + "form", ["full", "fallback", "truncated-only", "input-message"] +) +def test_valid_saved_treatment_preserves_spend(saved_records, reader, boundary, form): + saved = saved_records + payload = target_record(saved, reader) + if form in ("fallback", "truncated-only"): + no_reply(payload, truncated=form == "truncated-only") + elif form == "input-message": + first = payload["messages"][0] + first["input"] = first.pop("content") + save_records(saved) + assert_boundary(saved, boundary, None) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("form", ["extra-key", "absent-null-status"]) +def test_report_compatible_attempt_links_are_accepted(saved_records, boundary, form): + saved = saved_records + if form == "absent-null-status": + no_reply(saved.archives[0]) + saved.archives[0].pop("status") + + def change_link(link): + if form == "extra-key": + link["synthetic_extra"] = True + elif link["n"] == 1: + link.pop("status") + + save_records(saved, change_link) + assert_boundary(saved, boundary, None) + + +def add_board_household(saved, sid): + board = nt.pilot.load_board(Path(saved.design["board"])) + assert sid in board["scenarios"] and sid not in saved.design["draw_order"] + return board + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_cheap_final_outside_draw_refuses_with_path(saved_records, boundary): + saved = saved_records + board = add_board_household(saved, "outside") + payload = full_record(board, nt.CHEAP[0], saved.design["engine"], "outside") + path = saved.folder / "outside.json" + write_fixture_json(path, payload) + assert_boundary(saved, boundary, path) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_cheap_final_missing_predictions_refuses_with_path(saved_records, boundary): + saved = saved_records + payload = target_record(saved, "final") + no_reply(payload) + payload.pop("predictions") + save_records(saved) + assert_boundary(saved, boundary, saved.paths["final"]) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("reader", ["archive", "optional"]) +def test_absent_predictions_without_reply_remain_accepted( + saved_records, boundary, reader +): + saved = saved_records + if reader == "optional": + board = nt.pilot.load_board(Path(saved.design["board"])) + payload = full_record(board, "gpt-6-sol", saved.design["engine"]) + saved.finals["gpt-6-sol"] = payload + else: + payload = target_record(saved, "archive") + no_reply(payload) + payload.pop("predictions") + save_records(saved) + assert_boundary( + saved, boundary, None, expected_spend=0.6 if reader == "optional" else 0.5 + ) + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_saved_call_classification_failure_refuses_with_path( + saved_records, reader, boundary +): + saved = saved_records + payload = target_record(saved, reader) + payload["status"] = "complete" + payload["turns"][0]["calls"] = [None] + save_records(saved) + assert_boundary(saved, boundary, saved.paths[reader]) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("number", [True, 1.0], ids=["boolean", "float"]) +def test_archive_prefix_numeric_equality_remains_accepted( + saved_records, boundary, number +): + saved = saved_records + second = saved.archives[1] + second["attempts"][0]["n"] = number + path = saved.folder / "attempts/h2-2.json" + write_fixture_json(path, second) + final = target_record(saved, "final") + final["attempts"][1]["sha256"] = nt.sha256_file(path) + write_fixture_json(saved.paths["final"], final) + assert_boundary(saved, boundary, None) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_nonobject_archive_prefix_refuses_with_path(saved_records, boundary): + saved = saved_records + second = saved.archives[1] + second["attempts"][0] = None + path = saved.folder / "attempts/h2-2.json" + write_fixture_json(path, second) + final = target_record(saved, "final") + final["attempts"][1]["sha256"] = nt.sha256_file(path) + write_fixture_json(saved.paths["final"], final) + assert_boundary(saved, boundary, path) + + +@pytest.mark.parametrize("history", ["interrupted-archive", "rerunnable-final"]) +def test_real_treatment_resume_preserves_attempts_and_spend(saved_records, history): + saved = saved_records + first = saved.paths["archive"] + second = saved.folder / "attempts/h2-2.json" + final = saved.paths["final"] + prior_bytes = first.read_bytes() + second.unlink() + final.unlink() + if history == "rerunnable-final": + first.unlink() + final.write_bytes(prior_bytes) + board = nt.pilot.load_board(Path(saved.design["board"])) + calls = [] + + def fake_household( + scenario, variables, tool_call, completion_fn, spend, *, profile + ): + calls.append((profile.name, scenario.id)) + assert variables == board["variables"][scenario.id] + spend.add(0.1) + return full_record(board, profile.name, saved.design["engine"], scenario.id) + + with nt.Stage( + saved.design, + board=None, + models=[nt.CHEAP[0]], + run_household=fake_household, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.spend.spent_usd == pytest.approx(0.3) + assert plan.execute() == 0 + assert plan.spend.spent_usd == pytest.approx(0.4) + assert calls == [(nt.CHEAP[0], "h2")] + assert first.read_bytes() == prior_bytes + assert not second.exists() + payload = json.loads(final.read_text()) + assert payload["status"] == "complete" + assert payload["attempts"] == [nt.entry(first, saved.archives[0], 1)] + assert nt.stage_problems(saved.design) == [] From 4fb80d9b74194739c7e3f9d2127f86daa32bcd08 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 02:59:55 -0400 Subject: [PATCH 47/64] Confirmatory runner tests: build preflight finals the treatment checks accept 0300257c checks each saved no-tools record against the pinned none arm, and a cheap model's final must hold a predictions object. complete_preflight_design still wrote bare finals, so every preflight refused each model's first household and five tests never reached their planted fault. The fixture now builds each final through the harness with the records tests' full_record and writes the board files the no-tools loader checks. The harness and ceiling faults use the form a failed call saves: no answer and no prompt evidence. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 44 +++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 0b39c5a6..10847f52 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2174,7 +2174,32 @@ def test_the_preflight_names_a_missing_field(): def complete_preflight_design(w, tmp_path): - """Each calculator preflight starts after synthetic standing no-tools finals.""" + """Each calculator preflight starts after standing no-tools finals that pass + the no-tools script's treatment checks: built through the harness from the + fake board, whose files that script's loader checks before it calls the + patched harness loader.""" + import csv + + from policybench.scenarios import scenario_to_dict + from tests.test_confirmatory_no_tools_records import full_record + + board = Path(w.design["board"]) + (board / "reference_outputs.csv.meta.json").write_text( + json.dumps( + { + "programs": ["snap"], + "revisions": [ + {"kind": "engine_upgrade", "engine_version": w.design["engine"]} + ], + } + ) + ) + with (board / "scenarios.csv").open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_json"]) + writer.writerows( + [json.dumps(scenario_to_dict(w.board["scenarios"][sid]))] for sid in POOL + ) pool = tmp_path / "pool.json" pool.write_text(json.dumps([{"id": s} for s in POOL])) design = dict( @@ -2190,18 +2215,15 @@ def complete_preflight_design(w, tmp_path): ) for model in cr.CHEAP: for sid in design["draw_order"]: + final = full_record(w.board, model, design["engine"], sid) + final["predictions"] = {v: 1.0 for v in w.board["variables"][sid]} + final["totals"] = {"cost_usd": 0.0} cr.write_json( Path(design["no_tools"]["runs"]) / model / "none/confirm/households" / f"{sid}.json", - { - "scenario_id": sid, - "status": "complete", - "turns": [{}], - "totals": {"cost_usd": 0.0}, - "attempts": [], - }, + final, ) return design @@ -2260,9 +2282,15 @@ def test_preflight_refuses_an_unfinished_no_tools_stage( if fault == "history": payload.pop("attempts") else: + # A failed call leaves no reply: no answer and no prompt evidence. + from tests.test_confirmatory_no_tools_records import OPTIONAL_EVIDENCE + + for key in OPTIONAL_EVIDENCE: + payload.pop(key) payload.update( status="budget_stop" if fault == "ceiling" else "provider_error", error="BadRequestError" if fault == "harness" else None, + predictions={v: None for v in payload["variables"]}, turns=[], ) path.write_text(json.dumps(payload)) From 5e334d9a516b228c964e480845dcfb961aeaf295 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 06:52:38 -0400 Subject: [PATCH 48/64] Confirmatory runner: reproduce unreadable evidence refusals Record provider-free regressions for denied history enumeration, saved-evidence metadata failures, hash and decoder buffer disagreement, and board engine failures through calculator preflight and main. Each unusable file or folder must produce a refusal naming its path before providers. Baseline and eligible-pool hashes must bind the bytes decoded, and main must print all collected problems and return 1. The original production code fails all 38 regression cases. Two controls establish that runner startup, its stage check, and the separate report accept object-form variables with the same saved spend. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_r15_baseline.py | 162 +++++++++++++++ tests/test_confirmatory_r15_engine.py | 90 +++++++++ tests/test_confirmatory_r15_files.py | 249 ++++++++++++++++++++++++ 3 files changed, 501 insertions(+) create mode 100644 tests/test_confirmatory_r15_baseline.py create mode 100644 tests/test_confirmatory_r15_engine.py create mode 100644 tests/test_confirmatory_r15_files.py diff --git a/tests/test_confirmatory_r15_baseline.py b/tests/test_confirmatory_r15_baseline.py new file mode 100644 index 00000000..8d4aceb9 --- /dev/null +++ b/tests/test_confirmatory_r15_baseline.py @@ -0,0 +1,162 @@ +"""Hash-bound inputs and report-compatible variables stay provider-free.""" + +from __future__ import annotations + +import importlib.util +import io +import os +import sys +from pathlib import Path + +import pytest + +from tests.test_confirmatory_no_tools import Provider, nt +from tests.test_confirmatory_no_tools import setup as _setup_fixture +from tests.test_confirmatory_no_tools_records import ( + assert_boundary, + save_records, +) +from tests.test_confirmatory_no_tools_records import ( + saved_records as _saved_records_fixture, +) +from tests.test_confirmatory_no_tools_records import ( + treatment_templates as _treatment_templates_fixture, +) +from tests.test_confirmatory_run import N_DRAW, World, complete_preflight_design, cr + +setup = _setup_fixture +saved_records = _saved_records_fixture +treatment_templates = _treatment_templates_fixture + + +def replace_text_read(monkeypatch, path, replacement): + """Simulate replacement after the first byte read, without changing disk.""" + original_bytes, original_text = Path.read_bytes, Path.read_text + reads = [] + + def read_bytes(source): + data = original_bytes(source) + if source == path: + reads.append(("bytes", data)) + return data + + def read_text(source, *args, **kwargs): + if source == path: + reads.append(("text", replacement)) + return replacement + return original_text(source, *args, **kwargs) + + monkeypatch.setattr(Path, "read_bytes", read_bytes) + monkeypatch.setattr(Path, "read_text", read_text) + return reads + + +@pytest.mark.parametrize("data", [b"not JSON", b"\xff"], ids=["json", "utf8"]) +def test_baseline_replacement_cannot_supply_unhashed_decoded_bytes( + setup, monkeypatch, data +): + path = Path(setup["design"]["baseline_exclusions"]["path"]) + path.write_bytes(data) + setup["design"]["baseline_exclusions"]["sha256"] = nt.sha256_file(path) + reads = replace_text_read(monkeypatch, path, '{"exclusions": []}') + provider = Provider() + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=setup["board"], + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("the declared baseline bytes must be decoded") + assert str(path) in str(caught.value) + assert reads == [("bytes", data)] + assert provider.calls == [] + + +def test_baseline_hash_and_decoder_share_one_valid_buffer(setup, monkeypatch): + path = Path(setup["design"]["baseline_exclusions"]["path"]) + data = path.read_bytes() + reads = replace_text_read(monkeypatch, path, "not JSON") + provider = Provider() + with nt.Stage( + setup["design"], + board=setup["board"], + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.spend.spent_usd == 0 + assert reads == [("bytes", data)] + assert provider.calls == [] + + +def pool_design(tmp_path, monkeypatch): + monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + world = World(tmp_path, monkeypatch, models=(cr.CHEAP[0],)) + return world, complete_preflight_design(world, tmp_path) + + +@pytest.mark.parametrize("data", [b"not JSON", b"\xff"], ids=["json", "utf8"]) +def test_preflight_pool_replacement_cannot_supply_unhashed_decoded_bytes( + tmp_path, monkeypatch, data +): + world, design = pool_design(tmp_path, monkeypatch) + path = Path(design["eligible_pool"]["path"]) + replacement = path.read_text() + path.write_bytes(data) + design["eligible_pool"]["sha256"] = cr.sha256_file(path) + reads = replace_text_read(monkeypatch, path, replacement) + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(str(path) in problem for problem in problems), problems + assert reads == [("bytes", data)] + assert world.provider.count == {} + + +def test_preflight_pool_hash_and_decoder_share_one_valid_buffer(tmp_path, monkeypatch): + world, design = pool_design(tmp_path, monkeypatch) + path = Path(design["eligible_pool"]["path"]) + data = path.read_bytes() + reads = replace_text_read(monkeypatch, path, "[]") + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert not any("the draw, recomputed" in problem for problem in problems), problems + assert not any(str(path) in problem for problem in problems), problems + assert reads == [("bytes", data)] + assert world.provider.count == {} + + +@pytest.fixture(scope="module") +def report_module(): + configured = os.environ.get("CONFIRMATORY_REPORT") + if not configured: + pytest.skip( + "set CONFIRMATORY_REPORT for the separate report's acceptance control" + ) + path = Path(configured) + name = "confirmatory_report_r15_variables_control" + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + with pytest.MonkeyPatch.context() as patch: + patch.syspath_prepend(str(path.parent)) + patch.setitem(sys.modules, name, module) + spec.loader.exec_module(module) + yield module + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +def test_variables_object_is_accepted_by_runner_and_report( + saved_records, report_module, monkeypatch, reader +): + saved = saved_records + payload = saved.finals[nt.CHEAP[0]] if reader == "final" else saved.archives[0] + payload["variables"] = {"snap": {"synthetic_metadata": True}} + save_records(saved) + assert_boundary(saved, "startup", None) + assert_boundary(saved, "preflight", None) + saved.design["treatment_harness"] = "synthetic-harness" + monkeypatch.setattr(report_module, "harness_module", lambda path: nt.pilot) + problems = [] + costs = report_module.no_tools_costs(saved.design, problems) + assert problems == [] + assert costs["registered_usd"] + costs["other_usd"] == pytest.approx(0.5) diff --git a/tests/test_confirmatory_r15_engine.py b/tests/test_confirmatory_r15_engine.py new file mode 100644 index 00000000..11910e52 --- /dev/null +++ b/tests/test_confirmatory_r15_engine.py @@ -0,0 +1,90 @@ +"""Engine sidecars must refuse a calculator start before any provider call.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tests.test_confirmatory_run import ( + ENGINE, + complete_preflight_design, + cr, + pilot, +) +from tests.test_confirmatory_run import ( + fast as _fast_fixture, +) +from tests.test_confirmatory_run import ( + world as _world_fixture, +) + +world = _world_fixture +fast = _fast_fixture + + +REAL_BOARD_ENGINE = pilot.board_engine + + +def engine_fault(world, tmp_path, monkeypatch, fault): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + metadata = Path(design["board"]) / "reference_outputs.csv.meta.json" + monkeypatch.setattr(pilot, "board_engine", REAL_BOARD_ENGINE) + monkeypatch.setattr(pilot, "_versions", lambda: {"policyengine-us": ENGINE}) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + + def no_run(*args, **kwargs): + pytest.fail("a refused preflight must not construct or call a provider") + + monkeypatch.setattr(cr, "Run", no_run) + monkeypatch.setattr(pilot, "run_household", no_run) + if fault == "malformed": + metadata.write_text("{") + else: + read_text = Path.read_text + + def denied(path, *args, **kwargs): + if path == metadata: + raise PermissionError("engine metadata read denied") + return read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", denied) + return w, design, metadata + + +@pytest.mark.parametrize("fault", ["malformed", "unreadable"]) +@pytest.mark.parametrize("entrypoint", ["preflight", "main"]) +def test_engine_metadata_refuses_full_calculator_start( + world, tmp_path, monkeypatch, capsys, fault, entrypoint +): + w, design, metadata = engine_fault(world, tmp_path, monkeypatch, fault) + if entrypoint == "preflight": + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert str(metadata) in "\n".join(problems) + else: + design_path = tmp_path / "design.json" + design_path.write_text(json.dumps(design)) + assert cr.main(["run", str(design_path)]) == 1 + output = capsys.readouterr().out + assert "The run cannot start" in output + assert str(metadata) in output + assert not w.provider.count + + +@pytest.mark.parametrize("fault", ["malformed", "unreadable"]) +def test_main_names_engine_reread_failure_and_preserves_all_problems( + world, tmp_path, monkeypatch, capsys, fault +): + w, design, metadata = engine_fault(world, tmp_path, monkeypatch, fault) + # Isolate main's final engine reread, which must retain preflight's problems. + monkeypatch.setattr(cr, "preflight", lambda *_: ["earlier preflight problem"]) + design_path = tmp_path / "design.json" + design_path.write_text(json.dumps(design)) + assert cr.main(["run", str(design_path)]) == 1 + output = capsys.readouterr().out + assert "The run cannot start" in output + assert "earlier preflight problem" in output + assert str(metadata) in output + assert not w.provider.count diff --git a/tests/test_confirmatory_r15_files.py b/tests/test_confirmatory_r15_files.py new file mode 100644 index 00000000..18a9d75a --- /dev/null +++ b/tests/test_confirmatory_r15_files.py @@ -0,0 +1,249 @@ +"""File and folder failures must remain named refusals before providers.""" + +import errno +import io +import json +import os +from pathlib import Path + +import pytest + +from tests.test_confirmatory_no_tools import ( + Provider, + nt, + stage, + write_board_fixture, +) +from tests.test_confirmatory_no_tools import ( + setup as _setup_fixture, +) +from tests.test_confirmatory_run import ( + complete_preflight_design, + cr, +) +from tests.test_confirmatory_run import ( + fast as _fast_fixture, +) +from tests.test_confirmatory_run import ( + world as _world_fixture, +) + +setup = _setup_fixture +world = _world_fixture +fast = _fast_fixture + + +def deny_listing(monkeypatch, target): + for name in ("scandir", "listdir"): + original = getattr(os, name) + + def denied(path, *args, _original=original, **kwargs): + if not isinstance(path, int) and Path(path) == target: + raise PermissionError("synthetic denied listing") + return _original(path, *args, **kwargs) + + monkeypatch.setattr(os, name, denied) + + +def deny_stat(monkeypatch, target): + original = os.stat + + def denied(path, *args, **kwargs): + if not isinstance(path, int) and Path(path) == target: + # Path predicates suppress ELOOP on the pinned Python 3.12. + raise OSError(errno.ELOOP, "synthetic inaccessible metadata") + return original(path, *args, **kwargs) + + monkeypatch.setattr(os, "stat", denied) + + +@pytest.mark.parametrize("which", ["finals", "attempts"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_unlistable_optional_history_refuses( + setup, world, tmp_path, monkeypatch, which, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + spec = design["no_tools"] + folder = ( + Path(spec["runs"]) / "gpt-6-sol" / spec["folder"] / spec["stage"] / "households" + ) + target = folder if which == "finals" else folder / "attempts" + target.mkdir(parents=True) + deny_listing(monkeypatch, target) + if boundary == "startup": + with pytest.raises(nt.StateError, match=str(target)): + with stage(setup, provider): + pass + else: + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + assert provider.calls == [] + + +@pytest.mark.parametrize("which", ["root-list", "root-stat", "model-stat"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_model_inventory_failure_refuses( + setup, world, tmp_path, monkeypatch, which, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + root = Path(design["no_tools"]["runs"]) + (root / "gpt-6-sol").mkdir(parents=True, exist_ok=True) + target = root / "gpt-6-sol" if which == "model-stat" else root + (deny_listing if which == "root-list" else deny_stat)(monkeypatch, target) + if boundary == "startup": + with state_for_loading(setup) as plan: + with pytest.raises(nt.StateError, match=str(target)): + plan._load_state(setup["board"]) + else: + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + assert provider.calls == [] + + +def state_for_loading(setup): + # Isolate _load_state from the constructor's mkdir and lock operations. + import threading + + plan = object.__new__(nt.Stage) + plan.design = setup["design"] + plan.draw_order = setup["design"]["draw_order"] + plan.root = Path(setup["design"]["no_tools"]["runs"]) + plan.folder, plan.stage = "none", "confirmatory" + plan.models = nt.CHEAP + plan.stopped = threading.Event() + plan._lock_file = None + return plan + + +@pytest.mark.parametrize("which", ["folder-list", "folder-stat", "log-stat"]) +def test_calculator_summary_inventory_failure_refuses(world, monkeypatch, which): + w = world(models=(cr.CHEAP[0],)) + lanes = w.run.lanes[cr.CHEAP[0]] + target = ( + next(iter(lanes.households_dir.values())) + if which != "log-stat" + else lanes.folder / "run-log.jsonl" + ) + target.parent.mkdir(parents=True, exist_ok=True) + if which == "folder-list": + target.mkdir(exist_ok=True) + (deny_listing if which == "folder-list" else deny_stat)(monkeypatch, target) + with pytest.raises(nt.StateError, match=str(target)): + lanes.summary() + assert not w.provider.count + + +@pytest.mark.parametrize( + "which", ["model", "arm", "stage", "households", "run-log", "host-log"] +) +def test_calculator_preflight_inventory_failure_refuses( + world, tmp_path, monkeypatch, which +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + folder = Path(design["runs"]) / cr.CHEAP[0] + targets = { + "model": folder, + "arm": folder / "pe3", + "stage": folder / "pe3/confirm", + "households": folder / "pe3/confirm/households", + "run-log": folder / "run-log.jsonl", + "host-log": Path(design["runs"]) / "host/host-log.jsonl", + } + target = targets[which] + if which.endswith("log"): + target.parent.mkdir(parents=True, exist_ok=True) + deny_stat(monkeypatch, target) + else: + target.mkdir(parents=True, exist_ok=True) + deny_listing(monkeypatch, target) + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + + +@pytest.mark.parametrize("which", ["model", "arm", "stage", "households"]) +def test_calculator_preflight_directory_stat_failure_refuses( + world, tmp_path, monkeypatch, which +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + folder = Path(design["runs"]) / cr.CHEAP[0] + target = ( + folder + / { + "model": "", + "arm": "pe3", + "stage": "pe3/confirm", + "households": "pe3/confirm/households", + }[which] + ) + target.mkdir(parents=True, exist_ok=True) + deny_stat(monkeypatch, target) + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + + +def test_preflight_unreadable_preregistration_refuses(world, tmp_path, monkeypatch): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + target = Path(design["preregistration"]) + original = Path.read_bytes + + def denied(path): + if path == target: + raise PermissionError("synthetic denied read") + return original(path) + + monkeypatch.setattr(Path, "read_bytes", denied) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + + +def test_archive_target_stat_failure_refuses(setup, monkeypatch): + provider = Provider() + with stage(setup, provider) as plan: + model, sid = nt.CHEAP[0], "h2" + folder = plan.households(model) + folder.mkdir(parents=True) + payload = { + "scenario_id": sid, + "status": "provider_error", + "error": "APIConnectionError", + "turns": [], + "attempts": [], + "totals": {"cost_usd": 0.1}, + } + (folder / f"{sid}.json").write_text(json.dumps(payload)) + plan.histories[model][sid] = {"final": payload, "entries": []} + target = folder / "attempts" / f"{sid}-1.json" + deny_stat(monkeypatch, target) + with pytest.raises(nt.StateError, match=str(target)): + plan.archive(model, sid) + assert (folder / f"{sid}.json").read_text() == json.dumps(payload) + assert provider.calls == [] + + +def test_board_exclusions_stat_failure_refuses(setup, tmp_path, monkeypatch): + write_board_fixture(setup, tmp_path / "board") + target = Path(setup["design"]["board"]) / "reference_exclusions.json" + deny_stat(monkeypatch, target) + provider = Provider() + with pytest.raises(nt.StateError, match=str(target)): + with nt.Stage(setup["design"], run_household=provider, out=io.StringIO()): + pass + assert provider.calls == [] From d31c1d7deb71e996900caabc26a616cde51384c7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 06:55:34 -0400 Subject: [PATCH 49/64] Confirmatory runner: bind decoded inputs and name evidence failures Enumerate saved histories and calculator evidence explicitly. Missing optional folders retain empty inventories; covered listing and stat failures must name their paths. Preserve sorted JSON histories, saved spend and attempt decisions. Guard exclusions metadata, model discovery, archive overwrite checks, summaries, calculator records and host logs, and the preregistration hash read. Read baseline exclusions and the eligible pool once each. Their hashes must bind the exact buffer decoded with strict UTF-8. Guard both calculator engine reads with the metadata sidecar path. Preflight retains its collected problems, and main prints every problem and returns 1. Valid board engine checks and provider execution retain their existing paths. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 68 +++++++++++++--- scripts/confirmatory_run.py | 120 +++++++++++++++++++++------- tests/test_confirmatory_no_tools.py | 8 +- 3 files changed, 148 insertions(+), 48 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index acdf0f88..cf67e653 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -40,6 +40,7 @@ import os import re import signal +import stat import sys import threading import time @@ -132,6 +133,51 @@ def _board_input(path: Path | str, read): raise StateError(f"{path}: {error}") from error +def saved_stat(path: Path): + """Only a missing path means absent evidence; other stat failures refuse.""" + + def read(): + try: + return path.stat() + except FileNotFoundError: + return None + + return _board_input(path, read) + + +def saved_exists(path: Path) -> bool: + return saved_stat(path) is not None + + +def saved_is_dir(path: Path) -> bool: + metadata = saved_stat(path) + return metadata is not None and stat.S_ISDIR(metadata.st_mode) + + +def saved_children(folder: Path) -> list[Path]: + """Enumerate explicitly so a failed scan cannot silently erase evidence.""" + metadata = saved_stat(folder) + if metadata is None: + return [] + if not stat.S_ISDIR(metadata.st_mode): + raise StateError(f"{folder}: evidence folder is not a directory") + + def read(): + try: + with os.scandir(folder) as entries: + return [folder / entry.name for entry in entries] + except FileNotFoundError: + return [] + + return _board_input(folder, read) + + +def saved_json_paths(folder: Path) -> list[Path]: + return sorted( + path for path in saved_children(folder) if path.name.endswith(".json") + ) + + def load_board(path: Path, engine: str) -> dict: """Check board inputs locally, then use the harness's unchanged loader.""" from policybench.reference_exclusions import ( @@ -176,7 +222,7 @@ def require_engine(): reference_path = path / "reference_outputs.csv" reference = _board_input(reference_path, lambda: csv_frame(reference_path)) exclusions_path = path / "reference_exclusions.json" - if _board_input(exclusions_path, exclusions_path.exists): + if saved_exists(exclusions_path): _board_input( exclusions_path, lambda: object_json(checked_bytes(exclusions_path).decode("utf-8")), @@ -390,12 +436,12 @@ def _attempts_agree(listed: list, entries: list) -> bool: def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], float]: """Validate a model's histories, including an interrupted archive, and sum spend.""" archives: dict[str, list[tuple[int, Path]]] = {} - for path in sorted((folder / "attempts").glob("*.json")): + for path in saved_json_paths(folder / "attempts"): match = re.fullmatch(r"(.+)-([1-9][0-9]*)\.json", path.name) if not match: raise StateError(f"{path}: invalid attempt number") archives.setdefault(match[1], []).append((int(match[2]), path)) - finals = {path.stem: path for path in sorted(folder.glob("*.json"))} + finals = {path.stem: path for path in saved_json_paths(folder)} states, spent = {}, 0.0 for sid in sorted(set(finals) | set(archives)): paths = sorted(archives.get(sid, [])) @@ -471,8 +517,7 @@ def stage_problems(design: dict) -> list[str]: folder, stage = component(spec["folder"]), component(spec["stage"]) draw_order = [component(sid) for sid in design["draw_order"]] models = set(CHEAP) - if root.is_dir(): - models.update(path.name for path in root.iterdir() if path.is_dir()) + models.update(path.name for path in saved_children(root) if saved_is_dir(path)) except (OSError, KeyError, TypeError, ValueError) as error: return [f"no-tools stage: {error}"] board = None @@ -590,9 +635,10 @@ def __init__( ): raise StateError("baseline exclusions need path and sha256") baseline_path = Path(baseline["path"]) - if sha256_file(baseline_path) != baseline.get("sha256"): + baseline_bytes = baseline_path.read_bytes() + if hashlib.sha256(baseline_bytes).hexdigest() != baseline.get("sha256"): raise StateError("baseline exclusions do not hash as declared") - payload = json.loads(baseline_path.read_text()) + payload = json.loads(baseline_bytes.decode("utf-8")) exclusions = ( payload.get("exclusions") if isinstance(payload, dict) else None ) @@ -644,11 +690,7 @@ def _load_state(self, board) -> None: ) self.histories = {} spent = 0.0 - folders = ( - {p.name for p in self.root.iterdir() if p.is_dir()} - if self.root.is_dir() - else set() - ) + folders = {p.name for p in saved_children(self.root) if saved_is_dir(p)} for model in sorted(folders | set(self.models)): states, cost = history( self.households(model), @@ -753,7 +795,7 @@ def archive(self, model: str, sid: str) -> bool: source = folder / f"{sid}.json" target = folder / "attempts" / f"{sid}-{n}.json" target.parent.mkdir(parents=True, exist_ok=True) - if target.exists(): + if saved_exists(target): raise StateError(f"{target}: would overwrite an archived attempt") link = entry(source, record, n) link["file"] = f"attempts/{target.name}" diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 3949f239..c6daacd1 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -1558,14 +1558,21 @@ def timeout(profile): } def summary(self) -> dict: + from confirmatory_no_tools import _board_input, saved_exists, saved_json_paths + records = [] for label, folder in self.households_dir.items(): - for path in sorted(folder.glob("*.json")) if folder.is_dir() else []: - records.append(json.loads(path.read_text())) + for path in saved_json_paths(folder): + records.append(_board_input(path, lambda: json.loads(path.read_text()))) lines = [] path = self.folder / "run-log.jsonl" - if path.exists(): - lines = [json.loads(x) for x in path.read_text().splitlines() if x.strip()] + if saved_exists(path): + lines = _board_input( + path, + lambda: [ + json.loads(x) for x in path.read_text().splitlines() if x.strip() + ], + ) attempts = [x for x in lines if "event" not in x] count: dict[str, int] = {} for x in attempts: @@ -2162,7 +2169,15 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: if missing: return [f"the design has no {', '.join(missing)}"] problems = [] - from confirmatory_no_tools import stage_problems + from confirmatory_no_tools import ( + BOARD_INPUT_ERRORS, + _board_input, + saved_children, + saved_exists, + saved_is_dir, + saved_json_paths, + stage_problems, + ) problems.extend(stage_problems(design)) try: @@ -2172,7 +2187,7 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: calculator_root ) or calculator_root.is_relative_to(no_tools_root): problems.append("the no-tools and calculator runs roots overlap") - except (KeyError, TypeError, OSError, ValueError) as error: + except (KeyError, TypeError, OSError, ValueError, RuntimeError) as error: problems.append(f"the no-tools runs root is invalid: {error}") command = design["anchor_command"] if ( @@ -2183,19 +2198,31 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: problems.append(f"the anchor command {command!r} is not a command here") if design.get("stated_engine_overrides"): problems.append("the confirmatory run states the board's engine: no overrides") - if sha256_file(Path(design["preregistration"])) != design["frozen_sha256"]: - problems.append("the frozen copy does not hash to frozen_sha256") - pool = design["eligible_pool"] - if sha256_file(Path(pool["path"])) != pool["sha256"]: - problems.append("the eligible pool does not hash as frozen") - else: - ids = [e["id"] for e in json.loads(Path(pool["path"]).read_text())] - if draw(master_seed(design["frozen_sha256"]), ids) != list( - design["draw_order"] + preregistration = Path(design["preregistration"]) + try: + if ( + _board_input(preregistration, lambda: sha256_file(preregistration)) + != design["frozen_sha256"] ): - problems.append( - "the draw, recomputed from the stream 'draw', is not the design's" - ) + problems.append("the frozen copy does not hash to frozen_sha256") + except ValueError as error: + problems.append(str(error)) + pool = design["eligible_pool"] + pool_path = Path(pool["path"]) + try: + pool_bytes = pool_path.read_bytes() + if sha256_bytes(pool_bytes) != pool["sha256"]: + problems.append(f"{pool_path}: the eligible pool does not hash as frozen") + else: + ids = [e["id"] for e in json.loads(pool_bytes.decode("utf-8"))] + if draw(master_seed(design["frozen_sha256"]), ids) != list( + design["draw_order"] + ): + problems.append( + "the draw, recomputed from the stream 'draw', is not the design's" + ) + except BOARD_INPUT_ERRORS as error: + problems.append(f"{pool_path}: cannot read the eligible pool: {error}") head, dirty = pilot._git("rev-parse", "HEAD"), pilot._git("status", "--porcelain") if head != design["run_commit"] or dirty: problems.append( @@ -2235,14 +2262,21 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: f"not {TREATMENT_COMMIT[:12]}" ) board = Path(design["board"]) - engine = pilot.board_engine(board) - if f"policyengine-us {engine}" != design["engine"]: + try: + engine = pilot.board_engine(board) + except BOARD_INPUT_ERRORS as error: problems.append( - f"the board's engine {engine} is not the design's {design['engine']}" + f"{board / 'reference_outputs.csv.meta.json'}: " + f"cannot read the board's engine: {error}" ) - installed = pilot._versions()["policyengine-us"] - if installed != engine: - problems.append(f"policyengine-us {installed} is installed, not {engine}") + else: + if f"policyengine-us {engine}" != design["engine"]: + problems.append( + f"the board's engine {engine} is not the design's {design['engine']}" + ) + installed = pilot._versions()["policyengine-us"] + if installed != engine: + problems.append(f"policyengine-us {installed} is installed, not {engine}") if set(design["arms"]) != set(ARMS) or set(design["frontier_arms"]) != set( FRONTIER_ARMS ): @@ -2262,12 +2296,27 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: if not os.environ.get(key): problems.append(f"{model}: {key} is not set") folder = Path(design["runs"]) / model - if (folder / "run-log.jsonl").exists() or any( - folder.glob("*/*/households/*.json") - ): - problems.append(f"{folder} already holds records; a run is never resumed") - if (Path(design["runs"]) / "host" / "host-log.jsonl").exists(): - problems.append("the runs folder already holds a host log") + try: + holds_records = saved_exists(folder / "run-log.jsonl") + if not holds_records: + holds_records = any( + saved_json_paths(stage / "households") + for arm in saved_children(folder) + if saved_is_dir(arm) + for stage in saved_children(arm) + if saved_is_dir(stage) + ) + if holds_records: + problems.append( + f"{folder} already holds records; a run is never resumed" + ) + except ValueError as error: + problems.append(str(error)) + try: + if saved_exists(Path(design["runs"]) / "host" / "host-log.jsonl"): + problems.append("the runs folder already holds a host log") + except ValueError as error: + problems.append(str(error)) ac, lid = host_power() if not (ac and lid): problems.append( @@ -2304,7 +2353,16 @@ def main(argv=None) -> int: return 1 models = CHEAP + FRONTIER problems = preflight(design, models) - pilot._require_engine(Path(design["board"])) + from confirmatory_no_tools import BOARD_INPUT_ERRORS + + board = Path(design["board"]) + try: + pilot._require_engine(board) + except BOARD_INPUT_ERRORS as error: + problems.append( + f"{board / 'reference_outputs.csv.meta.json'}: " + f"the board's engine check failed: {error}" + ) if problems: print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) return 1 diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 7bead2a2..b7da9487 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -1232,16 +1232,16 @@ def test_board_exclusions_exists_failure_refuses_with_path( write_board_fixture(setup, directory) exclusions = directory / "reference_exclusions.json" error = PermissionError(13, "synthetic unreadable board parent", str(exclusions)) - exists = Path.exists + stat = Path.stat checks = [] - def fail_exclusions_exists(path): + def fail_exclusions_stat(path, *args, **kwargs): if path == exclusions: checks.append(path) raise error - return exists(path) + return stat(path, *args, **kwargs) - monkeypatch.setattr(Path, "exists", fail_exclusions_exists) + monkeypatch.setattr(Path, "stat", fail_exclusions_stat) provider = Provider() try: with pytest.raises(nt.StateError) as caught: From cccc3817d030e7d7afeaf8583b73c8313391c70f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 07:05:12 -0400 Subject: [PATCH 50/64] Confirmatory runner: guard intermediate stage inventories Explicitly list each candidate calculator stage before checking its literal households child. A covered failure listing that parent must become a preflight problem naming the stage folder. Preserve the same household path, record filter and early refusal once saved calculator records are found. The final regression run exposed this missing parent listing after the first implementation: 39 cases passed and the denied-stage-listing case still failed. Keep the original failing regression and verify the corrected inventory and preflight paths. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index c6daacd1..9b8b6e08 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2284,6 +2284,12 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: for name in ("arms", "frontier_arms"): if len(set(design[name].values())) != len(design[name]): problems.append(f"two {name} share a folder") + + def household_paths(stage): + # Check the parent inventory even when its next component is literal. + saved_children(stage) + return saved_json_paths(stage / "households") + for model in models: if model not in design["caps"] or model not in design["reserves"]: problems.append(f"{model}: no cap or reserve") @@ -2300,7 +2306,7 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: holds_records = saved_exists(folder / "run-log.jsonl") if not holds_records: holds_records = any( - saved_json_paths(stage / "households") + household_paths(stage) for arm in saved_children(folder) if saved_is_dir(arm) for stage in saved_children(arm) From 4d93b4067e2a2fb1f9e4f36e3e9a1a4cab0ed5fb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 07:44:06 -0400 Subject: [PATCH 51/64] Confirmatory runner: name the new test modules for what they check The three test modules added for this round take descriptive names: test_confirmatory_unreadable_evidence.py holds the folder, listing and stat refusals; test_confirmatory_input_reads.py holds the single-buffer hash and decode checks for the baseline exclusions and the eligible pool, and the saved-variables control shared with the report; test_confirmatory_board_engine.py holds the board-engine refusals through preflight and main. The control's imported report module drops the round number from its name too. No test or production code changes. Co-Authored-By: Claude Opus 5.5 --- ...irmatory_r15_engine.py => test_confirmatory_board_engine.py} | 0 ...rmatory_r15_baseline.py => test_confirmatory_input_reads.py} | 2 +- ...ry_r15_files.py => test_confirmatory_unreadable_evidence.py} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename tests/{test_confirmatory_r15_engine.py => test_confirmatory_board_engine.py} (100%) rename tests/{test_confirmatory_r15_baseline.py => test_confirmatory_input_reads.py} (99%) rename tests/{test_confirmatory_r15_files.py => test_confirmatory_unreadable_evidence.py} (100%) diff --git a/tests/test_confirmatory_r15_engine.py b/tests/test_confirmatory_board_engine.py similarity index 100% rename from tests/test_confirmatory_r15_engine.py rename to tests/test_confirmatory_board_engine.py diff --git a/tests/test_confirmatory_r15_baseline.py b/tests/test_confirmatory_input_reads.py similarity index 99% rename from tests/test_confirmatory_r15_baseline.py rename to tests/test_confirmatory_input_reads.py index 8d4aceb9..10be2c16 100644 --- a/tests/test_confirmatory_r15_baseline.py +++ b/tests/test_confirmatory_input_reads.py @@ -134,7 +134,7 @@ def report_module(): "set CONFIRMATORY_REPORT for the separate report's acceptance control" ) path = Path(configured) - name = "confirmatory_report_r15_variables_control" + name = "confirmatory_report_variables_control" spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) with pytest.MonkeyPatch.context() as patch: diff --git a/tests/test_confirmatory_r15_files.py b/tests/test_confirmatory_unreadable_evidence.py similarity index 100% rename from tests/test_confirmatory_r15_files.py rename to tests/test_confirmatory_unreadable_evidence.py From 0ae6df56750bc6c720b8ec164085446cae99a8f4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 12:53:45 -0400 Subject: [PATCH 52/64] Confirmatory runner: refuse listed entries without metadata; one read per record Runner review r16 found two gaps. A no-tools model folder, a calculator arm or a calculator stage that a listing returned was dropped when its stat failed with ENOENT: a dangling link (an optional model's folder on an unmounted volume) or an entry removed after the listing. The report refuses such an entry. listed_is_dir now refuses every stat failure of a listed entry, at startup, in stage_problems and in the preflight. A missing root, households folder or attempts folder, which the code names itself, still reads as empty. history decoded each saved record with one read and hashed it with another, and Stage.archive hashed the runner's final again before the move. read_record now returns the digest of the bytes it decoded; an archive's link carries that digest, and the history keeps each final's digest (the bytes read at startup, or the bytes write_json wrote). Before the move, archive refuses a final whose bytes no longer hash to it. history now lists a model's finals before its attempts folder, as the report does, so a households folder that cannot be listed or searched is the folder both programs name (report review r15 finding 4). New tests: a dangling or vanished model entry (startup and preflight), a dangling or vanished calculator arm or stage (preflight), a file at or a stat failure on an optional model's households or attempts folder (startup and preflight), a second read of an archive or a final that differs from the hashed bytes, an archive link equal to the digest its final was read from, and finals changed after the runner read or wrote them. The board engine test now requires the preflight guard's own line and every saved-treatment line beside it. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 48 ++++---- scripts/confirmatory_run.py | 6 +- tests/test_confirmatory_board_engine.py | 11 +- tests/test_confirmatory_input_reads.py | 112 +++++++++++++++++- tests/test_confirmatory_no_tools.py | 2 +- tests/test_confirmatory_no_tools_records.py | 6 +- .../test_confirmatory_unreadable_evidence.py | 109 ++++++++++++++++- 7 files changed, 265 insertions(+), 29 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index cf67e653..b0877b17 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -83,9 +83,11 @@ def component(value: str) -> str: return value -def read_record(path: Path) -> dict: +def read_record(path: Path) -> tuple[dict, str]: + """Decode a saved record and hash the same bytes, so its link names them.""" try: - record = json.loads(path.read_text()) + data = path.read_bytes() + record = json.loads(data.decode("utf-8")) if not isinstance(record, dict): raise ValueError("record is not a JSON object") if "prior_attempts" in record: @@ -101,7 +103,7 @@ def read_record(path: Path) -> dict: cost = float(raw_cost) if not math.isfinite(cost) or cost < 0: raise ValueError("attempt cost must be finite and nonnegative") - return record + return record, hashlib.sha256(data).hexdigest() except ( OSError, ValueError, @@ -149,9 +151,10 @@ def saved_exists(path: Path) -> bool: return saved_stat(path) is not None -def saved_is_dir(path: Path) -> bool: - metadata = saved_stat(path) - return metadata is not None and stat.S_ISDIR(metadata.st_mode) +def listed_is_dir(path: Path) -> bool: + """A listing returned this entry, so every stat failure refuses, even ENOENT + (a dangling link, or an entry removed since the listing).""" + return stat.S_ISDIR(_board_input(path, path.stat).st_mode) def saved_children(folder: Path) -> list[Path]: @@ -411,12 +414,13 @@ def validate_treatment(path, record, model, board, engine): raise StateError(f"{path}: " + "; ".join(problems)) -def entry(path: Path, record: dict, n: int) -> dict: +def entry(path: Path, record: dict, n: int, sha256: str) -> dict: + """Link an archive by the digest of the bytes its record came from.""" cls, reason = classify(record, 0.0, 0.0, None) return { "n": n, "file": f"attempts/{path.name}", - "sha256": sha256_file(path), + "sha256": sha256, "status": record.get("status"), "class": cls, "reason": reason, @@ -435,13 +439,15 @@ def _attempts_agree(listed: list, entries: list) -> bool: def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], float]: """Validate a model's histories, including an interrupted archive, and sum spend.""" + # Finals first, as the report lists them: an unlistable households folder + # is named before the attempts folder inside it. + finals = {path.stem: path for path in saved_json_paths(folder)} archives: dict[str, list[tuple[int, Path]]] = {} for path in saved_json_paths(folder / "attempts"): match = re.fullmatch(r"(.+)-([1-9][0-9]*)\.json", path.name) if not match: raise StateError(f"{path}: invalid attempt number") archives.setdefault(match[1], []).append((int(match[2]), path)) - finals = {path.stem: path for path in saved_json_paths(folder)} states, spent = {}, 0.0 for sid in sorted(set(finals) | set(archives)): paths = sorted(archives.get(sid, [])) @@ -455,7 +461,7 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f raise StateError(f"{folder}: {sid} exceeds {MAX_RERUNS} reruns") entries = [] for n, path in paths: - record = read_record(path) + record, digest = read_record(path) if record.get("scenario_id") != sid: raise StateError(f"{path}: household differs from archive name {sid}") if validate is not None: @@ -470,12 +476,12 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f ) from error if action(cls, n) not in ("rerun", "pause-rerun"): raise StateError(f"{path}: archived attempt was ineligible for rerun") - entries = record["attempts"] + [entry(path, record, n)] + entries = record["attempts"] + [entry(path, record, n, digest)] spent += attempt_cost(record) - final = None + final = final_sha256 = None if sid in finals: path = finals[sid] - final = read_record(path) + final, final_sha256 = read_record(path) if drawn is not None: if sid not in drawn: raise StateError(f"{path}: cheap-model final is outside the draw") @@ -504,7 +510,7 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f ) from error entries = final["attempts"] spent += attempt_cost(final) - states[sid] = {"entries": entries, "final": final} + states[sid] = {"entries": entries, "final": final, "final_sha256": final_sha256} return states, spent @@ -517,7 +523,7 @@ def stage_problems(design: dict) -> list[str]: folder, stage = component(spec["folder"]), component(spec["stage"]) draw_order = [component(sid) for sid in design["draw_order"]] models = set(CHEAP) - models.update(path.name for path in saved_children(root) if saved_is_dir(path)) + models.update(path.name for path in saved_children(root) if listed_is_dir(path)) except (OSError, KeyError, TypeError, ValueError) as error: return [f"no-tools stage: {error}"] board = None @@ -690,7 +696,7 @@ def _load_state(self, board) -> None: ) self.histories = {} spent = 0.0 - folders = {p.name for p in saved_children(self.root) if saved_is_dir(p)} + folders = {p.name for p in saved_children(self.root) if listed_is_dir(p)} for model in sorted(folders | set(self.models)): states, cost = history( self.households(model), @@ -797,7 +803,9 @@ def archive(self, model: str, sid: str) -> bool: target.parent.mkdir(parents=True, exist_ok=True) if saved_exists(target): raise StateError(f"{target}: would overwrite an archived attempt") - link = entry(source, record, n) + if _board_input(source, lambda: sha256_file(source)) != state["final_sha256"]: + raise StateError(f"{source}: final changed after it was read or written") + link = entry(source, record, n, state["final_sha256"]) link["file"] = f"attempts/{target.name}" with self._admission_lock: if self.interrupted.is_set(): @@ -821,7 +829,7 @@ def run_model(self, model: str) -> None: self.states[model] = "interrupted" return state = self.histories[model].setdefault( - sid, {"entries": [], "final": None} + sid, {"entries": [], "final": None, "final_sha256": None} ) while True: if self.interrupted.is_set(): @@ -878,8 +886,8 @@ def no_tool(arguments): ) record["totals"]["cost_usd"] = spend.spent_usd record["attempts"] = list(state["entries"]) - write_json(self.households(model) / f"{sid}.json", record) - state["final"] = record + digest = write_json(self.households(model) / f"{sid}.json", record) + state["final"], state["final_sha256"] = record, digest n = len(state["entries"]) + 1 cls, _ = classify(record, 0.0, 0.0, None) outcome = action(cls, n) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 9b8b6e08..bfad199f 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2172,9 +2172,9 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: from confirmatory_no_tools import ( BOARD_INPUT_ERRORS, _board_input, + listed_is_dir, saved_children, saved_exists, - saved_is_dir, saved_json_paths, stage_problems, ) @@ -2308,9 +2308,9 @@ def household_paths(stage): holds_records = any( household_paths(stage) for arm in saved_children(folder) - if saved_is_dir(arm) + if listed_is_dir(arm) for stage in saved_children(arm) - if saved_is_dir(stage) + if listed_is_dir(stage) ) if holds_records: problems.append( diff --git a/tests/test_confirmatory_board_engine.py b/tests/test_confirmatory_board_engine.py index 11910e52..99dc3faf 100644 --- a/tests/test_confirmatory_board_engine.py +++ b/tests/test_confirmatory_board_engine.py @@ -62,7 +62,16 @@ def test_engine_metadata_refuses_full_calculator_start( w, design, metadata = engine_fault(world, tmp_path, monkeypatch, fault) if entrypoint == "preflight": problems = cr.preflight(design, (cr.CHEAP[0],)) - assert str(metadata) in "\n".join(problems) + # The preflight's own guard, beside the saved-treatment lines it keeps. + guard = f"{metadata}: cannot read the board's engine: " + assert sum(problem.startswith(guard) for problem in problems) == 1, problems + for model in cr.CHEAP: + assert any( + problem.startswith(f"no-tools {model}: ") + and "cannot check saved no-tools treatment" in problem + and str(metadata) in problem + for problem in problems + ), problems else: design_path = tmp_path / "design.json" design_path.write_text(json.dumps(design)) diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index 10be2c16..9236daad 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -2,15 +2,19 @@ from __future__ import annotations +import hashlib import importlib.util import io +import json import os +import re import sys from pathlib import Path +from types import SimpleNamespace import pytest -from tests.test_confirmatory_no_tools import Provider, nt +from tests.test_confirmatory_no_tools import Provider, nt, record, stage from tests.test_confirmatory_no_tools import setup as _setup_fixture from tests.test_confirmatory_no_tools_records import ( assert_boundary, @@ -91,6 +95,112 @@ def test_baseline_hash_and_decoder_share_one_valid_buffer(setup, monkeypatch): assert provider.calls == [] +def saved_history(setup, archive_cost=0.3, final_cost=0.1): + """Save one rerun-eligible archive and the final that links it.""" + root = Path(setup["design"]["no_tools"]["runs"]) + folder = root / nt.CHEAP[0] / "none/confirmatory/households" + profile = SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic") + archive = folder / "attempts/h2-1.json" + archive.parent.mkdir(parents=True) + prior = record( + "h2", + profile, + status="provider_error", + error="APIConnectionError", + cost=archive_cost, + ) + prior["attempts"] = [] + archive.write_text(json.dumps(prior)) + saved = record("h2", profile, cost=final_cost) + saved["attempts"] = [ + { + "n": 1, + "file": "attempts/h2-1.json", + "sha256": nt.sha256_file(archive), + "status": "provider_error", + "class": "provider", + "reason": "APIConnectionError", + } + ] + (folder / "h2.json").write_text(json.dumps(saved)) + return {"archive": archive, "final": folder / "h2.json"}, prior, saved + + +@pytest.mark.parametrize("which", ["archive", "final"]) +def test_saved_record_hash_and_decoder_share_one_buffer(setup, monkeypatch, which): + paths, prior, saved = saved_history(setup) + path = paths[which] + data = path.read_bytes() + # A second read would see a cheaper record than the hashed bytes hold. + replaced = dict(prior if which == "archive" else saved, totals={"cost_usd": 0.2}) + reads = replace_text_read(monkeypatch, path, json.dumps(replaced)) + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + assert plan.spend.spent_usd == pytest.approx(0.4) + state = plan.histories[nt.CHEAP[0]]["h2"] + assert state["entries"][0]["sha256"] == nt.sha256_file(paths["archive"]) + assert state["final_sha256"] == nt.sha256_file(paths["final"]) + assert reads == [("bytes", data)] + assert provider.calls == [] + + +def test_archive_links_the_digest_its_final_was_read_from(setup): + paths, _, saved = saved_history(setup) + saved["status"], saved["error"] = "provider_error", "APIConnectionError" + paths["final"].write_text(json.dumps(saved)) + data = paths["final"].read_bytes() + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + assert plan.archive(nt.CHEAP[0], "h2") + link = plan.histories[nt.CHEAP[0]]["h2"]["entries"][-1] + assert link["n"] == 2 and link["file"] == "attempts/h2-2.json" + assert link["sha256"] == hashlib.sha256(data).hexdigest() + assert (paths["archive"].parent / "h2-2.json").read_bytes() == data + assert provider.calls == [] + + +def test_archive_refuses_a_final_changed_after_it_was_read(setup): + paths, _, saved = saved_history(setup) + saved["status"], saved["error"] = "provider_error", "APIConnectionError" + paths["final"].write_text(json.dumps(saved)) + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + changed = dict(saved, totals={"cost_usd": 0.2}) + paths["final"].write_text(json.dumps(changed)) + with pytest.raises(nt.StateError, match=re.escape(str(paths["final"]))): + plan.archive(nt.CHEAP[0], "h2") + assert len(plan.histories[nt.CHEAP[0]]["h2"]["entries"]) == 1 + assert json.loads(paths["final"].read_text()) == changed + assert not (paths["archive"].parent / "h2-2.json").exists() + assert provider.calls == [] + + +def test_archive_refuses_a_final_changed_after_the_runner_wrote_it(setup, monkeypatch): + provider = Provider( + { + (nt.CHEAP[0], "h2", 1): { + "status": "provider_error", + "error": "APIConnectionError", + } + } + ) + original = nt.write_json + + def write_then_change(path, payload, *args, **kwargs): + digest = original(path, payload, *args, **kwargs) + path.write_bytes(path.read_bytes() + b" ") + return digest + + monkeypatch.setattr(nt, "write_json", write_then_change) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert plan.states[nt.CHEAP[0]] == "error" + assert provider.calls == [(nt.CHEAP[0], "h2", 1)] + folder = plan.households(nt.CHEAP[0]) + assert json.loads((folder / "h2.json").read_text())["status"] == "provider_error" + assert not (folder / "attempts").exists() + + def pool_design(tmp_path, monkeypatch): monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) monkeypatch.setattr(cr, "host_power", lambda: (True, True)) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index b7da9487..85ff9ed0 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -1475,7 +1475,7 @@ def test_valid_saved_numeric_cost_preserves_spend(setup, cost): prior["attempts"] = [] archive.write_text(json.dumps(prior)) saved = record("h2", profile, cost=cost) - saved["attempts"] = [nt.entry(archive, prior, 1)] + saved["attempts"] = [nt.entry(archive, prior, 1, nt.sha256_file(archive))] (folder / "h2.json").write_text(json.dumps(saved)) provider = Provider() with stage(setup, provider) as plan: diff --git a/tests/test_confirmatory_no_tools_records.py b/tests/test_confirmatory_no_tools_records.py index c145012c..d9d7ac37 100644 --- a/tests/test_confirmatory_no_tools_records.py +++ b/tests/test_confirmatory_no_tools_records.py @@ -85,7 +85,7 @@ def save(path, payload): path = saved.folder / "attempts" / f"h2-{n}.json" save(path, payload) try: - link = nt.entry(path, payload, n) + link = nt.entry(path, payload, n, nt.sha256_file(path)) except (AttributeError, TypeError, ValueError, KeyError): # Malformed turn evidence must reach the startup reader, rather # than fail while this fixture refreshes otherwise valid links. @@ -450,5 +450,7 @@ def fake_household( assert not second.exists() payload = json.loads(final.read_text()) assert payload["status"] == "complete" - assert payload["attempts"] == [nt.entry(first, saved.archives[0], 1)] + assert payload["attempts"] == [ + nt.entry(first, saved.archives[0], 1, nt.sha256_file(first)) + ] assert nt.stage_problems(saved.design) == [] diff --git a/tests/test_confirmatory_unreadable_evidence.py b/tests/test_confirmatory_unreadable_evidence.py index 18a9d75a..bead81aa 100644 --- a/tests/test_confirmatory_unreadable_evidence.py +++ b/tests/test_confirmatory_unreadable_evidence.py @@ -4,6 +4,7 @@ import io import json import os +import re from pathlib import Path import pytest @@ -112,6 +113,108 @@ def test_model_inventory_failure_refuses( assert provider.calls == [] +def vanish(monkeypatch, target): + """A listing returned target, but its metadata read finds it gone.""" + original = os.stat + + def gone(path, *args, **kwargs): + if not isinstance(path, int) and Path(path) == target: + raise FileNotFoundError(errno.ENOENT, "synthetic vanished entry", str(path)) + return original(path, *args, **kwargs) + + monkeypatch.setattr(os, "stat", gone) + + +def plant_listed_entry(monkeypatch, target, fault, tmp_path): + if fault == "dangling": + # A model folder symlinked to an unmounted volume. + target.symlink_to(tmp_path / "unmounted-volume") + else: + target.mkdir() + vanish(monkeypatch, target) + + +@pytest.mark.parametrize("fault", ["dangling", "vanished"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_listed_model_entry_without_metadata_refuses( + setup, world, tmp_path, monkeypatch, fault, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + root = Path(design["no_tools"]["runs"]) + root.mkdir(parents=True, exist_ok=True) + target = root / "ghost-model" + plant_listed_entry(monkeypatch, target, fault, tmp_path) + if boundary == "startup": + with pytest.raises(nt.StateError, match=re.escape(str(target))): + with stage(setup, provider): + pass + else: + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any( + problem.startswith(f"no-tools stage: {target}:") for problem in problems + ), problems + assert not w.provider.count + assert provider.calls == [] + + +@pytest.mark.parametrize("fault", ["dangling", "vanished"]) +@pytest.mark.parametrize("level", ["arm", "stage"]) +def test_preflight_listed_calculator_entry_without_metadata_refuses( + world, tmp_path, monkeypatch, fault, level +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + folder = Path(design["runs"]) / cr.CHEAP[0] + parent = folder if level == "arm" else folder / "pe" + parent.mkdir(parents=True, exist_ok=True) + target = parent / "ghost" + plant_listed_entry(monkeypatch, target, fault, tmp_path) + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(problem.startswith(f"{target}:") for problem in problems), problems + assert not w.provider.count + + +@pytest.mark.parametrize("fault", ["file", "stat"]) +@pytest.mark.parametrize("which", ["finals", "attempts"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_broken_optional_history_folder_refuses( + setup, world, tmp_path, monkeypatch, fault, which, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + spec = design["no_tools"] + folder = ( + Path(spec["runs"]) / "gpt-6-sol" / spec["folder"] / spec["stage"] / "households" + ) + target = folder if which == "finals" else folder / "attempts" + target.parent.mkdir(parents=True, exist_ok=True) + if fault == "file": + target.write_text("{}") + else: + target.mkdir() + deny_stat(monkeypatch, target) + if boundary == "startup": + with pytest.raises(nt.StateError, match=re.escape(str(target))): + with stage(setup, provider): + pass + else: + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + assert provider.calls == [] + + def state_for_loading(setup): # Isolate _load_state from the constructor's mkdir and lock operations. import threading @@ -229,7 +332,11 @@ def test_archive_target_stat_failure_refuses(setup, monkeypatch): "totals": {"cost_usd": 0.1}, } (folder / f"{sid}.json").write_text(json.dumps(payload)) - plan.histories[model][sid] = {"final": payload, "entries": []} + plan.histories[model][sid] = { + "final": payload, + "entries": [], + "final_sha256": nt.sha256_file(folder / f"{sid}.json"), + } target = folder / "attempts" / f"{sid}-1.json" deny_stat(monkeypatch, target) with pytest.raises(nt.StateError, match=str(target)): From c2ca2dc92440c102ddf6cb23472e5a245ac0e86c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 22:14:22 -0400 Subject: [PATCH 53/64] Confirmatory runner: refuse an archive move before creating its folder In suites r10, three of 0ae6df56's new cases failed in all three runs: - test_archive_refuses_a_final_changed_after_the_runner_wrote_it: Stage.archive created the attempts folder before it checked the target and the final's digest, so a refused move left an empty attempts folder in a history that had none. Both refusals now come first, and the folder exists only when the move proceeds. - test_saved_record_hash_and_decoder_share_one_buffer[archive] and [final]: the test hashed the files itself after installing the read counter, so its own read counted as a second runner read. It now hashes them before counting. The runner read each record once. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 4 +++- tests/test_confirmatory_input_reads.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index b0877b17..b27baccd 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -800,11 +800,13 @@ def archive(self, model: str, sid: str) -> bool: folder = self.households(model) source = folder / f"{sid}.json" target = folder / "attempts" / f"{sid}-{n}.json" - target.parent.mkdir(parents=True, exist_ok=True) + # Both refusals come before the folder exists, so a refused move + # leaves the history as it was. if saved_exists(target): raise StateError(f"{target}: would overwrite an archived attempt") if _board_input(source, lambda: sha256_file(source)) != state["final_sha256"]: raise StateError(f"{source}: final changed after it was read or written") + target.parent.mkdir(parents=True, exist_ok=True) link = entry(source, record, n, state["final_sha256"]) link["file"] = f"attempts/{target.name}" with self._admission_lock: diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index 9236daad..798f6d3b 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -131,6 +131,9 @@ def test_saved_record_hash_and_decoder_share_one_buffer(setup, monkeypatch, whic paths, prior, saved = saved_history(setup) path = paths[which] data = path.read_bytes() + # Hashed before the reads are counted, so the count is the runner's alone. + archive_sha256 = nt.sha256_file(paths["archive"]) + final_sha256 = nt.sha256_file(paths["final"]) # A second read would see a cheaper record than the hashed bytes hold. replaced = dict(prior if which == "archive" else saved, totals={"cost_usd": 0.2}) reads = replace_text_read(monkeypatch, path, json.dumps(replaced)) @@ -138,8 +141,8 @@ def test_saved_record_hash_and_decoder_share_one_buffer(setup, monkeypatch, whic with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: assert plan.spend.spent_usd == pytest.approx(0.4) state = plan.histories[nt.CHEAP[0]]["h2"] - assert state["entries"][0]["sha256"] == nt.sha256_file(paths["archive"]) - assert state["final_sha256"] == nt.sha256_file(paths["final"]) + assert state["entries"][0]["sha256"] == archive_sha256 + assert state["final_sha256"] == final_sha256 assert reads == [("bytes", data)] assert provider.calls == [] From 274e1c7ea8d85c7a098fb02a41f9f88192560aa0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 01:10:45 -0400 Subject: [PATCH 54/64] Confirmatory runner: refuse unfinished optional histories and special files Review runner-r17 (sol61, REQUEST_CHANGES) found two gaps between the runner and the report and three smaller items. This commit fixes each. - The calculator preflight now refuses what the report refuses. stage_problems refuses any model's archive that no final lists, and any final whose action is rerun or pause-rerun. That covers optional models, and cheap-model archives outside the draw. Startup recovery still resumes both states. - Saved records now go through the report's descriptor guard. read_regular opens with O_NONBLOCK and refuses a descriptor that is not a regular file, then reads through that descriptor. Before this, a FIFO with no writer could hang startup while it held the invocation lock. The archive move's digest checks use the same reader. - The archive move hashes the moved bytes again after the rename. If a write landed between the first hash and the rename, it moves the bytes back and refuses. The docstring states the limit that remains: the runner cannot see a write to the same inode during either hash or after the second. Every later reader hashes each archive against its link. - New tests: - a FIFO or device in place of a final or an archive, at startup and preflight, bounded by a writer that releases a blocked open after 5 s; - unfinished optional and outside-draw histories; - a write between the digest check and the rename; - a target collision that keeps both files; - a digest refusal that leaves no attempts folder, and the missing folder after a target-stat refusal; - the changed-written-final case across every cheap model, which pins the other models' continuation. - The optional no-reply control now expects the preflight to refuse its final, which owes a rerun. - The read-counting helper also counts read_regular. It patches that reader only where it exists, so the same cases run against c2ca2dc9. Against c2ca2dc9, 14 of the new cases fail: 8 special-file, 1 move-back and 5 unfinished-history. The collision, folder and continuation cases pin existing behaviour. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 76 ++++++++++-- tests/test_confirmatory_input_reads.py | 57 ++++++++- tests/test_confirmatory_no_tools_records.py | 48 ++++++++ .../test_confirmatory_unreadable_evidence.py | 113 ++++++++++++++++++ 4 files changed, 279 insertions(+), 15 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index b27baccd..4be1f98a 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -83,10 +83,37 @@ def component(value: str) -> str: return value +def read_regular(path: Path) -> bytes: + """Read saved bytes as the report does: open with O_NONBLOCK, refuse a + descriptor that is not a regular file, then read through that descriptor. + A FIFO or device refuses at once, before any read can block under the + invocation lock. It follows file symlinks; a link to nothing refuses.""" + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + kind = { + stat.S_IFIFO: "FIFO", + stat.S_IFDIR: "directory", + stat.S_IFSOCK: "socket", + stat.S_IFCHR: "character device", + stat.S_IFBLK: "block device", + stat.S_IFLNK: "symlink", + }.get(stat.S_IFMT(metadata.st_mode), "unknown file type") + raise ValueError(f"expected a regular file, found {kind}") + os.set_blocking(descriptor, True) + chunks = [] + while chunk := os.read(descriptor, 1024 * 1024): + chunks.append(chunk) + return b"".join(chunks) + finally: + os.close(descriptor) + + def read_record(path: Path) -> tuple[dict, str]: """Decode a saved record and hash the same bytes, so its link names them.""" try: - data = path.read_bytes() + data = read_regular(path) record = json.loads(data.decode("utf-8")) if not isinstance(record, dict): raise ValueError("record is not a JSON object") @@ -151,6 +178,11 @@ def saved_exists(path: Path) -> bool: return saved_stat(path) is not None +def saved_sha256(path: Path) -> str: + """Hash a saved file through the same guard, naming it on failure.""" + return _board_input(path, lambda: hashlib.sha256(read_regular(path)).hexdigest()) + + def listed_is_dir(path: Path) -> bool: """A listing returned this entry, so every stat failure refuses, even ENOENT (a dangling link, or an entry removed since the listing).""" @@ -515,7 +547,12 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f def stage_problems(design: dict) -> list[str]: - """Check saved histories, treatment and standing without a provider.""" + """Check saved histories, treatment and standing without a provider. + + As the report does, refuse any model's archive that no final lists and any + final that owes a rerun, optional models included; each drawn cheap-model + household also needs a final that stands. Startup recovery still resumes + both unfinished states.""" problems = [] try: spec = design["no_tools"] @@ -543,18 +580,23 @@ def checked_board(): ), drawn=set(draw_order) if model in CHEAP else None, ) - if model not in CHEAP: - continue - for sid in draw_order: + drawn = draw_order if model in CHEAP else [] + for sid in drawn + sorted(set(states) - set(drawn)): state = states.get(sid) if state is None or state["final"] is None: - problems.append(f"no-tools {model}/{sid}: missing standing final") + problems.append( + f"no-tools {model}/{sid}: missing standing final" + if sid in drawn + else f"no-tools {model}/{sid}: archived attempts lack a final" + ) continue outcome = action( classify(state["final"], 0.0, 0.0, None)[0], len(state["entries"]) + 1, ) - if outcome != "stands": + if outcome in ("rerun", "pause-rerun") or ( + sid in drawn and outcome != "stands" + ): problems.append( f"no-tools {model}/{sid}: final action is {outcome}" ) @@ -790,7 +832,14 @@ def _dry_run(self) -> int: ) def archive(self, model: str, sid: str) -> bool: - """Atomically admit a prepared archive move, then extend its links.""" + """Atomically admit a prepared archive move, then extend its links. + + The target-exists and digest checks precede folder creation; either + refusal leaves the disk history unchanged. After the rename it hashes + the moved bytes again; if a write landed between the first hash and + the rename, it moves them back and refuses. It cannot see a write to + the same inode during either hash or after the second; every later + reader hashes each archive against its link and refuses a mismatch.""" state = self.histories[model][sid] record, entries = state["final"], state["entries"] n = len(entries) + 1 @@ -800,11 +849,9 @@ def archive(self, model: str, sid: str) -> bool: folder = self.households(model) source = folder / f"{sid}.json" target = folder / "attempts" / f"{sid}-{n}.json" - # Both refusals come before the folder exists, so a refused move - # leaves the history as it was. if saved_exists(target): raise StateError(f"{target}: would overwrite an archived attempt") - if _board_input(source, lambda: sha256_file(source)) != state["final_sha256"]: + if saved_sha256(source) != state["final_sha256"]: raise StateError(f"{source}: final changed after it was read or written") target.parent.mkdir(parents=True, exist_ok=True) link = entry(source, record, n, state["final_sha256"]) @@ -814,6 +861,13 @@ def archive(self, model: str, sid: str) -> bool: return False os.replace(source, target) state["final"] = None + if saved_sha256(target) != state["final_sha256"]: + os.replace(target, source) + state["final"] = record + raise StateError( + f"{source}: final changed during its archive move; " + f"the runner moved it back from {target}" + ) entries.append(link) return True diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index 798f6d3b..c4247ab1 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -34,8 +34,12 @@ def replace_text_read(monkeypatch, path, replacement): - """Simulate replacement after the first byte read, without changing disk.""" + """Simulate replacement after the first byte read, without changing disk. + Count both byte readers: Path.read_bytes, and the descriptor-guarded + read_regular that saved records go through (when the runner has it, so the + same cases run against an older runner).""" original_bytes, original_text = Path.read_bytes, Path.read_text + original_regular = getattr(nt, "read_regular", None) reads = [] def read_bytes(source): @@ -44,6 +48,15 @@ def read_bytes(source): reads.append(("bytes", data)) return data + def read_regular(source): + data = original_regular(source) + if Path(source) == path: + reads.append(("bytes", data)) + return data + + if original_regular is not None: + monkeypatch.setattr(nt, "read_regular", read_regular) + def read_text(source, *args, **kwargs): if source == path: reads.append(("text", replacement)) @@ -178,6 +191,34 @@ def test_archive_refuses_a_final_changed_after_it_was_read(setup): assert provider.calls == [] +def test_archive_moves_back_a_final_changed_before_its_rename(setup, monkeypatch): + paths, _, saved = saved_history(setup) + saved["status"], saved["error"] = "provider_error", "APIConnectionError" + paths["final"].write_text(json.dumps(saved)) + changed = dict(saved, totals={"cost_usd": 0.2}) + original = nt.os.replace + + def change_then_rename(source, target): + # The write lands after the digest check and before the rename. + if Path(source) == paths["final"]: + Path(source).write_text(json.dumps(changed)) + original(source, target) + + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + monkeypatch.setattr(nt.os, "replace", change_then_rename) + with pytest.raises( + nt.StateError, + match=re.escape(f"{paths['final']}: final changed during its archive move"), + ): + plan.archive(nt.CHEAP[0], "h2") + state = plan.histories[nt.CHEAP[0]]["h2"] + assert len(state["entries"]) == 1 and state["final"] == saved + assert json.loads(paths["final"].read_text()) == changed + assert sorted(p.name for p in paths["archive"].parent.iterdir()) == ["h2-1.json"] + assert provider.calls == [] + + def test_archive_refuses_a_final_changed_after_the_runner_wrote_it(setup, monkeypatch): provider = Provider( { @@ -191,14 +232,22 @@ def test_archive_refuses_a_final_changed_after_the_runner_wrote_it(setup, monkey def write_then_change(path, payload, *args, **kwargs): digest = original(path, payload, *args, **kwargs) - path.write_bytes(path.read_bytes() + b" ") + if nt.CHEAP[0] in Path(path).parts: + path.write_bytes(path.read_bytes() + b" ") return digest monkeypatch.setattr(nt, "write_json", write_then_change) - plan = stage(setup, provider, models=[nt.CHEAP[0]]) + plan = stage(setup, provider) assert plan.execute() == 1 assert plan.states[nt.CHEAP[0]] == "error" - assert provider.calls == [(nt.CHEAP[0], "h2", 1)] + # The other models continue through both households. + assert all(plan.states[model] == "complete" for model in nt.CHEAP[1:]) + assert all( + plan.standing(model, sid) for model in nt.CHEAP[1:] for sid in ("h2", "h1") + ) + assert [call for call in provider.calls if call[0] == nt.CHEAP[0]] == [ + (nt.CHEAP[0], "h2", 1) + ] folder = plan.households(nt.CHEAP[0]) assert json.loads((folder / "h2.json").read_text())["status"] == "provider_error" assert not (folder / "attempts").exists() diff --git a/tests/test_confirmatory_no_tools_records.py b/tests/test_confirmatory_no_tools_records.py index d9d7ac37..45d9681b 100644 --- a/tests/test_confirmatory_no_tools_records.py +++ b/tests/test_confirmatory_no_tools_records.py @@ -364,11 +364,59 @@ def test_absent_predictions_without_reply_remain_accepted( no_reply(payload) payload.pop("predictions") save_records(saved) + if reader == "optional" and boundary == "preflight": + # Absent predictions raise nothing; the unanswered final still owes a + # rerun, which the report refuses for every model. + assert nt.stage_problems(saved.design) == [ + "no-tools gpt-6-sol/h2: final action is rerun" + ] + return assert_boundary( saved, boundary, None, expected_spend=0.6 if reader == "optional" else 0.5 ) +UNFINISHED = { + "optional-rerun": ("gpt-6-sol", "h2", "final action is rerun"), + "optional-pause": ("gpt-6-sol", "h2", "final action is pause-rerun"), + "optional-archive": ("gpt-6-sol", "h2", "archived attempts lack a final"), + "cheap-archive-outside-draw": ( + nt.CHEAP[1], + "outside", + "archived attempts lack a final", + ), +} + + +@pytest.mark.parametrize("unfinished", UNFINISHED) +def test_preflight_refuses_unfinished_histories_that_startup_resumes( + saved_records, unfinished +): + saved = saved_records + model, sid, needle = UNFINISHED[unfinished] + board = ( + add_board_household(saved, sid) + if sid == "outside" + else nt.pilot.load_board(Path(saved.design["board"])) + ) + payload = full_record(board, model, saved.design["engine"], sid) + error = ( + "AuthenticationError" + if unfinished == "optional-pause" + else "APIConnectionError" + ) + payload.update(status="provider_error", error=error) + households = saved.root / model / "none/confirmatory/households" + path = households / ( + f"{sid}.json" + if unfinished.endswith(("rerun", "pause")) + else f"attempts/{sid}-1.json" + ) + write_fixture_json(path, payload) + assert nt.stage_problems(saved.design) == [f"no-tools {model}/{sid}: {needle}"] + assert_boundary(saved, "startup", None, expected_spend=0.6) + + @pytest.mark.parametrize("reader", ["final", "archive"]) @pytest.mark.parametrize("boundary", ["startup", "preflight"]) def test_saved_call_classification_failure_refuses_with_path( diff --git a/tests/test_confirmatory_unreadable_evidence.py b/tests/test_confirmatory_unreadable_evidence.py index bead81aa..75b225ab 100644 --- a/tests/test_confirmatory_unreadable_evidence.py +++ b/tests/test_confirmatory_unreadable_evidence.py @@ -1,10 +1,13 @@ """File and folder failures must remain named refusals before providers.""" +import contextlib import errno import io import json import os import re +import threading +import time from pathlib import Path import pytest @@ -317,6 +320,115 @@ def denied(path): assert not w.provider.count +@contextlib.contextmanager +def fifo_released_after(path, seconds=5.0): + """Bound a reader that blocks on the FIFO: after `seconds`, open and close a + writer every 50 ms, so each blocked open sees end of file instead of hanging.""" + done = threading.Event() + + def release(): + if done.wait(seconds): + return + while not done.is_set(): + with contextlib.suppress(OSError): # ENXIO while no reader waits + os.close(os.open(path, os.O_WRONLY | os.O_NONBLOCK)) + done.wait(0.05) + + thread = threading.Thread(target=release, daemon=True) + thread.start() + try: + yield + finally: + done.set() + thread.join() + + +@pytest.mark.parametrize("kind", ["fifo", "device"]) +@pytest.mark.parametrize("which", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_saved_record_that_is_not_a_regular_file_refuses_at_once( + setup, boundary, which, kind +): + spec = setup["design"]["no_tools"] + folder = ( + Path(spec["runs"]) / nt.CHEAP[0] / spec["folder"] / spec["stage"] / "households" + ) + target = folder / ("h2.json" if which == "final" else "attempts/h2-1.json") + target.parent.mkdir(parents=True) + if kind == "fifo": + os.mkfifo(target) + else: + target.symlink_to("/dev/null") + provider = Provider() + start = time.monotonic() + with fifo_released_after(target) if kind == "fifo" else contextlib.nullcontext(): + if boundary == "startup": + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("a special file must refuse before execution") + message = str(caught.value) + else: + message = "\n".join(nt.stage_problems(setup["design"])) + found = "FIFO" if kind == "fifo" else "character device" + assert str(target) in message + assert f"expected a regular file, found {found}" in message + assert time.monotonic() - start < 5, "no read may block on the special file" + assert provider.calls == [] + + +def prepared_final(plan, model=nt.CHEAP[0], sid="h2"): + """Save a rerun-eligible final and its history, with no attempts folder.""" + folder = plan.households(model) + folder.mkdir(parents=True) + payload = { + "scenario_id": sid, + "status": "provider_error", + "error": "APIConnectionError", + "turns": [], + "attempts": [], + "totals": {"cost_usd": 0.1}, + } + source = folder / f"{sid}.json" + source.write_text(json.dumps(payload)) + plan.histories[model][sid] = { + "final": payload, + "entries": [], + "final_sha256": nt.sha256_file(source), + } + return folder, source, payload + + +def test_archive_refuses_an_existing_target_and_keeps_both_files(setup): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, payload = prepared_final(plan) + target = folder / "attempts" / "h2-1.json" + target.parent.mkdir() + target.write_text('{"synthetic": "an earlier archive"}') + with pytest.raises( + nt.StateError, match=re.escape(f"{target}: would overwrite") + ): + plan.archive(nt.CHEAP[0], "h2") + assert source.read_text() == json.dumps(payload) + assert target.read_text() == '{"synthetic": "an earlier archive"}' + assert sorted(p.name for p in target.parent.iterdir()) == ["h2-1.json"] + assert plan.histories[nt.CHEAP[0]]["h2"]["entries"] == [] + assert provider.calls == [] + + +def test_archive_digest_refusal_leaves_no_attempts_folder(setup): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, payload = prepared_final(plan) + source.write_text(json.dumps(dict(payload, totals={"cost_usd": 0.2}))) + with pytest.raises(nt.StateError, match=re.escape(f"{source}: final changed")): + plan.archive(nt.CHEAP[0], "h2") + assert json.loads(source.read_text())["totals"] == {"cost_usd": 0.2} + assert not (folder / "attempts").exists() + assert plan.histories[nt.CHEAP[0]]["h2"]["entries"] == [] + assert provider.calls == [] + + def test_archive_target_stat_failure_refuses(setup, monkeypatch): provider = Provider() with stage(setup, provider) as plan: @@ -342,6 +454,7 @@ def test_archive_target_stat_failure_refuses(setup, monkeypatch): with pytest.raises(nt.StateError, match=str(target)): plan.archive(model, sid) assert (folder / f"{sid}.json").read_text() == json.dumps(payload) + assert not (folder / "attempts").exists() assert provider.calls == [] From d3673c3ced995dec9252122076b0245e807864cf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 01:13:25 -0400 Subject: [PATCH 55/64] Confirmatory runner: format to CI's ruff 0.15.2 and wrap long lines CI runs `uv run ruff check .` and `ruff format --check .` with the locked ruff 0.15.2. CI has not run on this draft. Run locally on c2ca2dc9's tree, both fail on files this PR adds: four files needed formatting, and four lines ran past 88 characters. This commit formats those four files and rewraps the one string the formatter could not split. Every changed file parses to the same AST as before. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 4 +++- scripts/confirmatory_run.py | 10 +++++----- tests/test_confirmatory_anchor.py | 12 +++++++----- tests/test_confirmatory_run.py | 14 +++++++------- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py index 8cd02865..18a56701 100644 --- a/scripts/confirmatory_anchor.py +++ b/scripts/confirmatory_anchor.py @@ -61,7 +61,9 @@ def main(argv=None) -> int: """Push a whole snapshot only while exclusively extending the clone's file.""" parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--repo", type=Path, required=True, help="the anchors clone") - parser.add_argument("anchors", type=Path, help="a snapshot of the run's anchors.jsonl") + parser.add_argument( + "anchors", type=Path, help="a snapshot of the run's anchors.jsonl" + ) args = parser.parse_args(argv) repo = args.repo.expanduser() target = repo / "anchors.jsonl" diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index bfad199f..7a9a5a0b 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -1771,8 +1771,10 @@ def _join_push(self) -> None: "ended": clock(), "outcome": "error", "returncode": None, - "output": "the anchor push thread did not end after its time limit; " - "its command may still be running", + "output": ( + "the anchor push thread did not end after its " + "time limit; its command may still be running" + ), } ) self._check_push_error() @@ -1788,9 +1790,7 @@ def _record_push(self, line: dict) -> None: self._push_pending = None self._last_push_outcome = line["outcome"] if line["outcome"] != "success": - self._next_anchor = min( - self._next_anchor, line["ended"] + ANCHOR_RETRY - ) + self._next_anchor = min(self._next_anchor, line["ended"] + ANCHOR_RETRY) try: path = Path(self.design["runs"]) / "anchor-pushes.jsonl" with path.open("a") as handle: diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index fbc5a367..3af7be87 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -987,7 +987,9 @@ def finish(): for line in (tmp_path / "anchor-pushes.jsonl").read_text().splitlines() ] assert len(rows) == 1 and rows[0]["outcome"] == "timeout" - assert "pipes" in rows[0]["output"] and "held open after the kill" in rows[0]["output"] + assert ( + "pipes" in rows[0]["output"] and "held open after the kill" in rows[0]["output"] + ) def test_finish_records_a_stuck_push_once_even_if_its_thread_later_ends( @@ -1051,7 +1053,9 @@ def short_join(timeout=None): live = (tmp_path / "anchors.jsonl").read_bytes() assert len(copies[Path(calls[0][-1])].splitlines()) == 1 assert len(copies[Path(calls[1][-1])].splitlines()) == 2 - assert all(copy.endswith(b"\n") and live.startswith(copy) for copy in copies.values()) + assert all( + copy.endswith(b"\n") and live.startswith(copy) for copy in copies.values() + ) def test_a_retired_command_cannot_shrink_the_remote_after_the_final_push( @@ -1074,9 +1078,7 @@ def test_a_retired_command_cannot_shrink_the_remote_after_the_final_push( ready_read, ready_write = os.pipe() release_read, release_write = os.pipe() real_popen, children, waits = subprocess.Popen, [], [] - run = bare_finish_run( - folder, [sys.executable, str(SCRIPT), "--repo", str(repo)] - ) + run = bare_finish_run(folder, [sys.executable, str(SCRIPT), "--repo", str(repo)]) def popen(command, **kwargs): """Delay only the first real helper, before it can acquire the clone lock.""" diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 10847f52..5ad1f156 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2511,9 +2511,9 @@ def tool(arguments): assert record["api"] == treatment["api"] == api assert record["contract"] == treatment["contract"] == contract assert record["status"] == treatment["status"] == "complete" - assert record["predictions"] == treatment["predictions"] == { - v: 0.0 for v in VARIABLES - } + assert ( + record["predictions"] == treatment["predictions"] == {v: 0.0 for v in VARIABLES} + ) assert len(sent) == len(requests) == 2 assert sent == requests and called == calculations and len(called) == 1 assert record["messages"] == treatment["messages"] @@ -3151,8 +3151,7 @@ def communicate(timeout): assert [p["outcome"] for p in pushes] == ["failed"] * 3 assert [p["ended"] - p["started"] for p in pushes] == [7.0, 11.0, 5.0] assert all( - b["started"] >= a["ended"] + cr.ANCHOR_RETRY - for a, b in zip(pushes, pushes[1:]) + b["started"] >= a["ended"] + cr.ANCHOR_RETRY for a, b in zip(pushes, pushes[1:]) ) @@ -3182,8 +3181,9 @@ def during_retry(seconds): assert [row["outcome"] for row in pushes] == ["failed", "success", "success"] for model in w.models: assert [row["cause"] for row in w.events(model, "stop")] == ["interrupt"] - assert anchors[-1]["models"][model]["run-log.jsonl"] == ( - w.run.lanes[model].heads()["run-log.jsonl"] + assert ( + anchors[-1]["models"][model]["run-log.jsonl"] + == (w.run.lanes[model].heads()["run-log.jsonl"]) ) assert anchors[-2]["models"][model]["run-log.jsonl"]["lines"] == ( len(w.lines(model)) - 1 From 0193811081a18be7e051757004a4047d904b3bae Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 01:15:57 -0400 Subject: [PATCH 56/64] Confirmatory runner: pin the archive move's descriptor guard A mutant check of 274e1c7e found one fault that the tests missed. Its digest read in Stage.archive used the blocking sha256_file. The new case replaces a prepared final with a FIFO after startup. The archive move must refuse it at once by name, within the 5 s bound, and leave no attempts folder. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_unreadable_evidence.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_confirmatory_unreadable_evidence.py b/tests/test_confirmatory_unreadable_evidence.py index 75b225ab..e3dbd174 100644 --- a/tests/test_confirmatory_unreadable_evidence.py +++ b/tests/test_confirmatory_unreadable_evidence.py @@ -429,6 +429,23 @@ def test_archive_digest_refusal_leaves_no_attempts_folder(setup): assert provider.calls == [] +def test_archive_refuses_a_final_replaced_by_a_fifo_at_once(setup): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, _ = prepared_final(plan) + source.unlink() + os.mkfifo(source) + start = time.monotonic() + with fifo_released_after(source): + with pytest.raises( + nt.StateError, match="expected a regular file, found FIFO" + ): + plan.archive(nt.CHEAP[0], "h2") + assert time.monotonic() - start < 5, "no digest read may block on a FIFO" + assert not (folder / "attempts").exists() + assert provider.calls == [] + + def test_archive_target_stat_failure_refuses(setup, monkeypatch): provider = Provider() with stage(setup, provider) as plan: From a8226ddb2aecaf0eed1b56b7b58089067dc0400a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 11:27:59 -0400 Subject: [PATCH 57/64] Confirmatory runner tests: hold the deadline test's second turn for the abandon suites-r12's first runner pass at 01938110 failed test_an_attempt_past_its_deadline_is_abandoned_and_stops_the_model with 0.02 == 0.01, and the next two passes held 730 passed. The test gave each turn 0.2 s against a 0.3 s deadline, so its second turn spent at 0.4 s, and it expected the waiting thread to abandon the attempt inside that 0.1 s. A wake 0.1 s late put the second turn's cost in the attempt's record, which is the spend so far that the runner's rule asks for. The runner was right and the test's expectation assumed a wake time. A plugin that delays the abandon by 0.25 s reproduces the same assertion at the same line. The first turn now spends at once and the second waits for the abandon itself, through a new `hold` key in FakeProvider. The deadline rises to 1 s and the other model's attempts to 0.2 s, which leaves 0.8 s of room against that model reaching its own deadline. The assertions are unchanged. The test passed 25 of 25 runs alone and 25 of 25 with the 0.25 s delay. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 5ad1f156..070bda02 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -649,7 +649,8 @@ def close(self): class FakeProvider: """Conversations a test plants. specs[(model, label, household, n)] holds a record's `status` and `error`, `turns` (default 1), `cost` per turn, - `sleep` per turn, `calls` (tool arguments, one call each, per turn), + `sleep` per turn, `hold` (an event each turn after the first waits for), + `calls` (tool arguments, one call each, per turn), `pressure` (high pressure during the attempt), `stall` (a host-log stall, seconds), `guard` (a guard pause over the attempt), `resume_after` (seconds after which the operator resumes), `raise` (the conversation @@ -694,6 +695,8 @@ def __call__(self, lanes, sid, label, tool_call, spend): rec["status"] = "budget_stop" break started = time.time() + if i and spec.get("hold"): + assert spec["hold"].wait(30) time.sleep(spec.get("sleep", 0.0)) completed = time.time() cost = spec.get("cost", COST) @@ -1905,16 +1908,26 @@ def test_an_attempt_past_its_deadline_is_abandoned_and_stops_the_model( """The conversation runs on in its own thread; at the deadline its record is a harness error with the spend so far, the model stops, and the conversation ends `budget_stop` at its next turn. Its spend after the - deadline is a late-cost event.""" - monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 0.3) + deadline is a late-cost event. The second turn waits for the abandon, so + the split does not depend on when the waiting thread wakes.""" + monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 1.0) + abandoned = threading.Event() + abandon = cr.AttemptSpend.abandon + + def abandon_then_release(spend): + spent = abandon(spend) + abandoned.set() + return spent + + monkeypatch.setattr(cr.AttemptSpend, "abandon", abandon_then_release) model = "gpt-6-luna" w = world(models=(model, "claude-haiku-5.5")) sid = w.draw_order[0] first = w.run.lanes[model].orders[sid][0] w.provider.specs = { - (model, first, sid, 1): {"turns": 4, "cost": 0.01, "sleep": 0.2}, + (model, first, sid, 1): {"turns": 4, "cost": 0.01, "hold": abandoned}, **{ - ("claude-haiku-5.5", label, s, 1): {"sleep": 0.05} + ("claude-haiku-5.5", label, s, 1): {"sleep": 0.2} for label in cr.ARMS for s in w.draw_order[:2] }, From 4441364256d1824dc82fb8c558bb3bd579b06da8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 18:28:18 -0400 Subject: [PATCH 58/64] Confirmatory runner: hold the preflight to the report's spend and baseline rules Answers review runner-r18. stage_problems dropped the cost history returned and never compared it with spent_before, and the preflight never read the baseline. Calculator spending could therefore start from an understated ceiling, or on a baseline the report refuses, and the report would then withhold the results. stage_problems now sums every model's validated cost and refuses a spent_before below that sum by more than MONEY_TOLERANCE, the report's value. It refuses an unusable value by the report's money rule: finite and nonnegative, with numeric text and booleans counting as numbers there too. check_baseline is the startup check moved unchanged, so startup and the preflight share one rule. Tests: a full preflight against spend in a final, an archive and an optional model; each unusable spent_before; and each baseline fault. The second hash after a rename gets its own FIFO case, because the earlier case attacks the first hash. Two execution cases fail the second hash and the move back, then check that the other models finish, the lock is released and startup recovers the archive. The deadline test now records the join's timeout: its hold on the second turn hides when the abandon came, so a half deadline passed it. The ceiling test waits for the crossing where it slept 0.3 s against an unbounded competitor. The archive docstring now limits later digest checks to surviving links: after an interrupt, startup rebuilds the last archive's digest from the bytes then present. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 114 +++++++++------ tests/test_confirmatory_input_reads.py | 132 ++++++++++++++++++ tests/test_confirmatory_no_tools.py | 1 + tests/test_confirmatory_no_tools_records.py | 1 + tests/test_confirmatory_run.py | 21 ++- .../test_confirmatory_unreadable_evidence.py | 80 +++++++++++ 6 files changed, 303 insertions(+), 46 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 4be1f98a..7ab908f1 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -62,6 +62,7 @@ ) NO_TOOLS_CAP = 5.0 +MONEY_TOLERANCE = 1e-6 # dollars, as the report compares spent_before with records START_GRACE_SECONDS = 10 NO_TOOLS_WORDING_SHA256 = ( "43b13e4e64463f4ee1654e9546672899f80ca5e1ce18e93cf2eb04091b3a2699" @@ -546,14 +547,54 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f return states, spent -def stage_problems(design: dict) -> list[str]: - """Check saved histories, treatment and standing without a provider. +def check_baseline(design: dict) -> None: + """Refuse a baseline inventory that fails its hash or schema, from one read.""" + baseline_path = None + try: + baseline = design["baseline_exclusions"] + if not isinstance(baseline, dict) or not isinstance(baseline.get("path"), str): + raise StateError("baseline exclusions need path and sha256") + baseline_path = Path(baseline["path"]) + baseline_bytes = baseline_path.read_bytes() + if hashlib.sha256(baseline_bytes).hexdigest() != baseline.get("sha256"): + raise StateError("baseline exclusions do not hash as declared") + payload = json.loads(baseline_bytes.decode("utf-8")) + exclusions = payload.get("exclusions") if isinstance(payload, dict) else None + if not isinstance(exclusions, list): + raise StateError("baseline exclusions must contain an exclusions list") + cells = [] + for exclusion in exclusions: + if not isinstance(exclusion, dict) or any( + not isinstance(exclusion.get(key), str) or not exclusion[key] + for key in ("scenario_id", "variable") + ): + raise StateError( + "baseline exclusion needs household and output strings" + ) + cells.append((exclusion["scenario_id"], exclusion["variable"])) + if len(cells) != len(set(cells)): + raise StateError("baseline exclusions repeat a cell") + except (OSError, KeyError, TypeError, ValueError, RecursionError) as error: + at = "baseline exclusions" + if baseline_path is not None: + at += f" ({baseline_path})" + raise StateError(f"{at}: {error}") from error + - As the report does, refuse any model's archive that no final lists and any - final that owes a rerun, optional models included; each drawn cheap-model - household also needs a final that stands. Startup recovery still resumes - both unfinished states.""" +def stage_problems(design: dict) -> list[str]: + """Check the baseline, saved histories, treatment, standing and spend + without a provider. + + As the report does, refuse a baseline that fails its hash or schema, any + model's archive that no final lists, any final that owes a rerun, optional + models included, and a spent_before below the saved records' spend; each + drawn cheap-model household also needs a final that stands. Startup + recovery still accepts both unfinished states.""" problems = [] + try: + check_baseline(design) + except StateError as error: + problems.append(str(error)) try: spec = design["no_tools"] root = Path(spec["runs"]) @@ -562,8 +603,9 @@ def stage_problems(design: dict) -> list[str]: models = set(CHEAP) models.update(path.name for path in saved_children(root) if listed_is_dir(path)) except (OSError, KeyError, TypeError, ValueError) as error: - return [f"no-tools stage: {error}"] + return [*problems, f"no-tools stage: {error}"] board = None + spent = 0.0 def checked_board(): nonlocal board @@ -573,13 +615,14 @@ def checked_board(): for model in sorted(models): try: - states, _ = history( + states, cost = history( root / model / folder / stage / "households", validate=lambda path, record: validate_treatment( path, record, model, checked_board, design["engine"] ), drawn=set(draw_order) if model in CHEAP else None, ) + spent += cost drawn = draw_order if model in CHEAP else [] for sid in drawn + sorted(set(states) - set(drawn)): state = states.get(sid) @@ -602,6 +645,20 @@ def checked_board(): ) except (OSError, TypeError, ValueError, AttributeError, KeyError) as error: problems.append(f"no-tools {model}: {error}") + try: + declared = float(design.get("spent_before")) + except (TypeError, ValueError, OverflowError): + declared = math.nan + if not 0 <= declared < math.inf: + problems.append( + "no-tools spend: spent_before must be a finite, nonnegative number, " + f"not {design.get('spent_before')!r}" + ) + elif declared < spent - MONEY_TOLERANCE: + problems.append( + f"no-tools spend: spent_before ${declared:.4f} is less than " + f"the saved records' ${spent:.4f}" + ) return problems @@ -675,40 +732,7 @@ def __init__( run_household if run_household is not None else pilot.run_household ) self._lock_file = None - baseline_path = None - try: - baseline = design["baseline_exclusions"] - if not isinstance(baseline, dict) or not isinstance( - baseline.get("path"), str - ): - raise StateError("baseline exclusions need path and sha256") - baseline_path = Path(baseline["path"]) - baseline_bytes = baseline_path.read_bytes() - if hashlib.sha256(baseline_bytes).hexdigest() != baseline.get("sha256"): - raise StateError("baseline exclusions do not hash as declared") - payload = json.loads(baseline_bytes.decode("utf-8")) - exclusions = ( - payload.get("exclusions") if isinstance(payload, dict) else None - ) - if not isinstance(exclusions, list): - raise StateError("baseline exclusions must contain an exclusions list") - cells = [] - for exclusion in exclusions: - if not isinstance(exclusion, dict) or any( - not isinstance(exclusion.get(key), str) or not exclusion[key] - for key in ("scenario_id", "variable") - ): - raise StateError( - "baseline exclusion needs household and output strings" - ) - cells.append((exclusion["scenario_id"], exclusion["variable"])) - if len(cells) != len(set(cells)): - raise StateError("baseline exclusions repeat a cell") - except (OSError, KeyError, TypeError, ValueError, RecursionError) as error: - at = "baseline exclusions" - if baseline_path is not None: - at += f" ({baseline_path})" - raise StateError(f"{at}: {error}") from error + check_baseline(design) self.root.mkdir(parents=True, exist_ok=True) self._lock_file = (self.root / ".confirmatory-no-tools.lock").open("a") try: @@ -838,8 +862,10 @@ def archive(self, model: str, sid: str) -> bool: refusal leaves the disk history unchanged. After the rename it hashes the moved bytes again; if a write landed between the first hash and the rename, it moves them back and refuses. It cannot see a write to - the same inode during either hash or after the second; every later - reader hashes each archive against its link and refuses a mismatch.""" + the same inode during either hash or after the second. Later readers + refuse an archive that differs from a surviving link; a saved record + holds that link only once the next attempt's final lands, so startup + rebuilds the last archive's digest from its bytes after an interrupt.""" state = self.histories[model][sid] record, entries = state["final"], state["entries"] n = len(entries) + 1 diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index c4247ab1..080296ed 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -17,7 +17,9 @@ from tests.test_confirmatory_no_tools import Provider, nt, record, stage from tests.test_confirmatory_no_tools import setup as _setup_fixture from tests.test_confirmatory_no_tools_records import ( + OPTIONAL_EVIDENCE, assert_boundary, + full_record, save_records, ) from tests.test_confirmatory_no_tools_records import ( @@ -288,6 +290,136 @@ def test_preflight_pool_hash_and_decoder_share_one_valid_buffer(tmp_path, monkey assert world.provider.count == {} +def no_tools_households(design, model): + spec = design["no_tools"] + return Path(spec["runs"]) / model / spec["folder"] / spec["stage"] / "households" + + +def saved_spend(world, design): + """Save $0.20 in a cheap model's final, $0.10 in the archive it replaced and + $0.05 in an optional model's final. Return their sum.""" + model, sid = cr.CHEAP[0], design["draw_order"][0] + folder = no_tools_households(design, model) + final = json.loads((folder / f"{sid}.json").read_text()) + failed = { + key: value for key, value in final.items() if key not in OPTIONAL_EVIDENCE + } + failed.update( + status="provider_error", + error="APIConnectionError", + predictions={variable: None for variable in final["variables"]}, + turns=[], + totals={"cost_usd": 0.1}, + ) + archive = folder / "attempts" / f"{sid}-1.json" + digest = cr.write_json(archive, failed) + final.update( + attempts=[nt.entry(archive, failed, 1, digest)], totals={"cost_usd": 0.2} + ) + cr.write_json(folder / f"{sid}.json", final) + optional = full_record(world.board, "gpt-6-sol", design["engine"], sid) + optional["totals"] = {"cost_usd": 0.05} + cr.write_json(no_tools_households(design, "gpt-6-sol") / f"{sid}.json", optional) + return 0.35 + + +@pytest.mark.parametrize( + "margin, refused", + [ + (1.0, False), + (0.0, False), + (-nt.MONEY_TOLERANCE / 2, False), + (-nt.MONEY_TOLERANCE * 2, True), + (-0.05, True), + (-0.35, True), + ], + ids=[ + "above", + "equal", + "inside-tolerance", + "past-tolerance", + "lacks-optional", + "zero", + ], +) +def test_preflight_compares_spent_before_with_all_saved_spend( + tmp_path, monkeypatch, margin, refused +): + world, design = pool_design(tmp_path, monkeypatch) + spent = saved_spend(world, design) + design["spent_before"] = spent + margin + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert [problem for problem in problems if problem.startswith("no-tools")] == ( + [ + f"no-tools spend: spent_before ${spent + margin:.4f} is less than " + f"the saved records' ${spent:.4f}" + ] + if refused + else [] + ) + assert world.provider.count == {} + + +@pytest.mark.parametrize( + "value", + [float("nan"), float("inf"), None, "unknown", -1e-7, 10**400], + ids=["nan", "infinite", "null", "text", "negative", "overflow"], +) +def test_preflight_refuses_an_unusable_spent_before(tmp_path, monkeypatch, value): + world, design = pool_design(tmp_path, monkeypatch) + design["spent_before"] = value + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any("finite, nonnegative" in problem for problem in problems), problems + assert world.provider.count == {} + + +@pytest.mark.parametrize( + "fault, needle", + [ + ("valid", None), + ("undeclared", "baseline_exclusions"), + ("missing", "No such file"), + ("changed", "do not hash as declared"), + ("malformed", "Expecting"), + ("schema", "must contain an exclusions list"), + ("repeated", "repeat a cell"), + ], +) +def test_preflight_applies_startup_s_baseline_check( + tmp_path, monkeypatch, fault, needle +): + world, design = pool_design(tmp_path, monkeypatch) + spec = design["baseline_exclusions"] + path = Path(spec["path"]) + cell = {"scenario_id": "h", "variable": "snap"} + if fault == "undeclared": + del design["baseline_exclusions"] + elif fault == "missing": + path.unlink() + elif fault == "changed": + path.write_text('{"exclusions": [], "changed": true}') + elif fault != "valid": + path.write_text( + { + "malformed": "{", + "schema": "{}", + "repeated": json.dumps({"exclusions": [cell, cell]}), + }[fault] + ) + spec["sha256"] = nt.sha256_file(path) + found = [ + problem + for problem in cr.preflight(design, (cr.CHEAP[0],)) + if "baseline" in problem + ] + if needle is None: + assert found == [] + else: + assert len(found) == 1 and needle in found[0], found + assert fault == "undeclared" or str(path) in found[0] + assert world.provider.count == {} + + @pytest.fixture(scope="module") def report_module(): configured = os.environ.get("CONFIRMATORY_REPORT") diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 85ff9ed0..2c56bb4a 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -32,6 +32,7 @@ def setup(tmp_path, monkeypatch): "board": "synthetic.csv", "engine": "policyengine-us 2.60.4", "draw_order": draw, + "spent_before": nt.NO_TOOLS_CAP, "baseline_exclusions": { "path": str(baseline), "sha256": nt.sha256_file(baseline), diff --git a/tests/test_confirmatory_no_tools_records.py b/tests/test_confirmatory_no_tools_records.py index 45d9681b..5c1b8ce3 100644 --- a/tests/test_confirmatory_no_tools_records.py +++ b/tests/test_confirmatory_no_tools_records.py @@ -110,6 +110,7 @@ def treatment_templates(tmp_path_factory): baseline.write_text('{"exclusions": []}\n') design = { "draw_order": ["h2", "outside"], + "spent_before": nt.NO_TOOLS_CAP, "baseline_exclusions": { "path": str(baseline), "sha256": nt.sha256_file(baseline), diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 070bda02..0616463b 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -1561,7 +1561,11 @@ def test_a_model_that_decides_nothing_still_has_both_logs(world): admit = held.admit def late(sid, lane): - time.sleep(0.3) + # A sleep here would race haiku's attempt on a loaded host. + deadline = time.monotonic() + 30 + while not w.run.ceiling.exhausted(): + assert time.monotonic() < deadline, "haiku never crossed the ceiling" + time.sleep(0.01) return admit(sid, lane) held.admit = late @@ -1909,10 +1913,19 @@ def test_an_attempt_past_its_deadline_is_abandoned_and_stops_the_model( is a harness error with the spend so far, the model stops, and the conversation ends `budget_stop` at its next turn. Its spend after the deadline is a late-cost event. The second turn waits for the abandon, so - the split does not depend on when the waiting thread wakes.""" + the split does not depend on when the waiting thread wakes, and the join's + own timeout pins the deadline, because that wait hides when the abandon came.""" monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 1.0) abandoned = threading.Event() abandon = cr.AttemptSpend.abandon + join = threading.Thread.join + waits = {} + + def recorded_join(thread, timeout=None): + waits.setdefault(thread.name, timeout) + return join(thread, timeout) + + monkeypatch.setattr(threading.Thread, "join", recorded_join) def abandon_then_release(spend): spent = abandon(spend) @@ -1933,6 +1946,7 @@ def abandon_then_release(spend): }, } w.execute() + assert waits[f"{model}-{sid}-{first}-1"] == 1.0 line = next( x for x in w.attempts(model) if x["household"] == sid and x["arm"] == first ) @@ -2215,9 +2229,12 @@ def complete_preflight_design(w, tmp_path): ) pool = tmp_path / "pool.json" pool.write_text(json.dumps([{"id": s} for s in POOL])) + baseline = tmp_path / "baseline-exclusions.json" + baseline.write_text('{"exclusions": []}\n') design = dict( w.design, eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, + baseline_exclusions={"path": str(baseline), "sha256": cr.sha256_file(baseline)}, treatment_harness=str(cr.ROOT), run_commit="0" * 40, no_tools={ diff --git a/tests/test_confirmatory_unreadable_evidence.py b/tests/test_confirmatory_unreadable_evidence.py index e3dbd174..f48be754 100644 --- a/tests/test_confirmatory_unreadable_evidence.py +++ b/tests/test_confirmatory_unreadable_evidence.py @@ -446,6 +446,86 @@ def test_archive_refuses_a_final_replaced_by_a_fifo_at_once(setup): assert provider.calls == [] +def test_archive_refuses_a_target_replaced_by_a_fifo_after_its_rename( + setup, monkeypatch +): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, _ = prepared_final(plan) + target = folder / "attempts" / "h2-1.json" + original = nt.os.replace + + def rename_then_replace(old, new): + original(old, new) + Path(new).unlink() + os.mkfifo(new) + + monkeypatch.setattr(nt.os, "replace", rename_then_replace) + start = time.monotonic() + with fifo_released_after(target): + with pytest.raises( + nt.StateError, match="expected a regular file, found FIFO" + ): + plan.archive(nt.CHEAP[0], "h2") + assert time.monotonic() - start < 5, "the second hash may not block on a FIFO" + assert not source.exists() + assert plan.histories[nt.CHEAP[0]]["h2"]["entries"] == [] + assert provider.calls == [] + + +@pytest.mark.parametrize("fault", ["second-read", "move-back"]) +def test_a_failed_archive_stops_one_model_and_startup_recovers_it(setup, fault): + """The second hash or the move back fails: that model errors, the others + finish, the invocation releases its lock, and the archive stays with no + final and no link. Startup then rebuilds the link from the bytes present, + changed or not, which is the limit the draft states.""" + model = nt.CHEAP[0] + provider = Provider( + {(model, "h2", 1): {"status": "provider_error", "error": "APIConnectionError"}} + ) + plan = stage(setup, provider) + folder = plan.households(model) + source, target = folder / "h2.json", folder / "attempts" / "h2-1.json" + with pytest.MonkeyPatch.context() as patch: + if fault == "second-read": + read = nt.read_regular + + def unreadable(path): + if Path(path) == target: + raise PermissionError(errno.EACCES, "synthetic denied", str(path)) + return read(path) + + patch.setattr(nt, "read_regular", unreadable) + else: + replace = os.replace + + def failed_move_back(old, new): + if Path(old) == source: + Path(old).write_bytes(Path(old).read_bytes() + b" ") + elif Path(old) == target: + raise OSError(errno.EIO, "synthetic failed move", str(old)) + replace(old, new) + + patch.setattr(os, "replace", failed_move_back) + assert plan.execute() == 1 + assert plan.states[model] == "error" + assert str(target) in plan.out.getvalue() + assert all(plan.states[other] == "complete" for other in nt.CHEAP[1:]) + assert all( + plan.standing(other, sid) for other in nt.CHEAP[1:] for sid in ("h2", "h1") + ) + assert [call for call in provider.calls if call[0] == model] == [(model, "h2", 1)] + assert target.exists() and not source.exists() + digest = nt.sha256_file(target) + again = Provider() + resumed = stage(setup, again, models=[model]) # raises if the lock survived + assert resumed.histories[model]["h2"]["entries"][0]["sha256"] == digest + assert resumed.execute() == 0 + assert again.calls == [(model, "h2", 1), (model, "h1", 1)] + assert json.loads(source.read_text())["attempts"][0]["sha256"] == digest + assert nt.stage_problems(setup["design"]) == [] + + def test_archive_target_stat_failure_refuses(setup, monkeypatch): provider = Provider() with stage(setup, provider) as plan: From ceae93f4f095fd687512a4eb4a791a286b603b11 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 20:14:27 -0400 Subject: [PATCH 59/64] Confirmatory runner: sum saved spend exactly, as the report does Answers review runner-r19 and item 3 of confirmatory-report-r18. The preflight added each cost to a running total, and the report grouped its total by model, so the two programs rounded differently. A spent_before at the tolerance passed one and failed the other, and $2 of saved spend could round away beside a $1e16 cost. history now lists each saved cost, and exact_spend takes their correctly rounded sum, the same in any order and grouping. The report takes the same sum. An intermediate overflow means a total past every float, which the comparison refuses. Startup's spend uses the same sum. Tests: every order of two sets of costs at the tolerance, costs a running total rounds away, and a total past every float; a spent_before given as numeric text or a boolean, which counts as its number in both programs; and a design without spent_before. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 44 +++++++++----- tests/test_confirmatory_input_reads.py | 83 +++++++++++++++++++++++--- 2 files changed, 105 insertions(+), 22 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 7ab908f1..af728dad 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -470,8 +470,11 @@ def _attempts_agree(listed: list, entries: list) -> bool: ) -def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], float]: - """Validate a model's histories, including an interrupted archive, and sum spend.""" +def history( + folder: Path, validate=None, drawn=None +) -> tuple[dict[str, dict], list[float]]: + """Validate a model's histories, including an interrupted archive, and list + each saved cost.""" # Finals first, as the report lists them: an unlistable households folder # is named before the attempts folder inside it. finals = {path.stem: path for path in saved_json_paths(folder)} @@ -481,7 +484,7 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f if not match: raise StateError(f"{path}: invalid attempt number") archives.setdefault(match[1], []).append((int(match[2]), path)) - states, spent = {}, 0.0 + states, costs = {}, [] for sid in sorted(set(finals) | set(archives)): paths = sorted(archives.get(sid, [])) numbers = [n for n, _ in paths] @@ -510,7 +513,7 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f if action(cls, n) not in ("rerun", "pause-rerun"): raise StateError(f"{path}: archived attempt was ineligible for rerun") entries = record["attempts"] + [entry(path, record, n, digest)] - spent += attempt_cost(record) + costs.append(attempt_cost(record)) final = final_sha256 = None if sid in finals: path = finals[sid] @@ -542,9 +545,19 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f f"{path}: cannot classify saved final: {error}" ) from error entries = final["attempts"] - spent += attempt_cost(final) + costs.append(attempt_cost(final)) states[sid] = {"entries": entries, "final": final, "final_sha256": final_sha256} - return states, spent + return states, costs + + +def exact_spend(costs: list[float]) -> float: + """The correctly rounded sum of saved costs, the same in any order and + grouping, so that this reaches the report's total. Its intermediate overflow + means a total past every float.""" + try: + return math.fsum(costs) + except OverflowError: + return math.inf def check_baseline(design: dict) -> None: @@ -587,8 +600,8 @@ def stage_problems(design: dict) -> list[str]: As the report does, refuse a baseline that fails its hash or schema, any model's archive that no final lists, any final that owes a rerun, optional - models included, and a spent_before below the saved records' spend; each - drawn cheap-model household also needs a final that stands. Startup + models included, and a spent_before below the saved records' exact total; + each drawn cheap-model household also needs a final that stands. Startup recovery still accepts both unfinished states.""" problems = [] try: @@ -605,7 +618,7 @@ def stage_problems(design: dict) -> list[str]: except (OSError, KeyError, TypeError, ValueError) as error: return [*problems, f"no-tools stage: {error}"] board = None - spent = 0.0 + costs = [] def checked_board(): nonlocal board @@ -615,14 +628,14 @@ def checked_board(): for model in sorted(models): try: - states, cost = history( + states, saved = history( root / model / folder / stage / "households", validate=lambda path, record: validate_treatment( path, record, model, checked_board, design["engine"] ), drawn=set(draw_order) if model in CHEAP else None, ) - spent += cost + costs += saved drawn = draw_order if model in CHEAP else [] for sid in drawn + sorted(set(states) - set(drawn)): state = states.get(sid) @@ -645,6 +658,7 @@ def checked_board(): ) except (OSError, TypeError, ValueError, AttributeError, KeyError) as error: problems.append(f"no-tools {model}: {error}") + spent = exact_spend(costs) try: declared = float(design.get("spent_before")) except (TypeError, ValueError, OverflowError): @@ -761,10 +775,10 @@ def _load_state(self, board) -> None: f"drawn households missing from the board: {sorted(missing)}" ) self.histories = {} - spent = 0.0 + costs = [] folders = {p.name for p in saved_children(self.root) if listed_is_dir(p)} for model in sorted(folders | set(self.models)): - states, cost = history( + states, saved = history( self.households(model), validate=lambda path, record: validate_treatment( path, record, model, self.board, design["engine"] @@ -772,7 +786,7 @@ def _load_state(self, board) -> None: drawn=set(self.draw_order) if model in CHEAP else None, ) self.histories[model] = states - spent += cost + costs += saved # A saved ceiling failure is a stage-wide stop on every invocation. if any( state["final"] is not None @@ -784,7 +798,7 @@ def _load_state(self, board) -> None: for state in states.values() ): self.stopped.set() - self.spend = Spend(spent, self.stopped) + self.spend = Spend(exact_spend(costs), self.stopped) self.states = {model: "pending" for model in self.models} def close(self) -> None: diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index 080296ed..31894d88 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -5,7 +5,9 @@ import hashlib import importlib.util import io +import itertools import json +import math import os import re import sys @@ -295,9 +297,9 @@ def no_tools_households(design, model): return Path(spec["runs"]) / model / spec["folder"] / spec["stage"] / "households" -def saved_spend(world, design): - """Save $0.20 in a cheap model's final, $0.10 in the archive it replaced and - $0.05 in an optional model's final. Return their sum.""" +def saved_spend(world, design, final_cost=0.2, archive_cost=0.1, optional_cost=0.05): + """Save a cost in a cheap model's final, one in the archive it replaced and + one in an optional model's final. Return their sum.""" model, sid = cr.CHEAP[0], design["draw_order"][0] folder = no_tools_households(design, model) final = json.loads((folder / f"{sid}.json").read_text()) @@ -309,18 +311,19 @@ def saved_spend(world, design): error="APIConnectionError", predictions={variable: None for variable in final["variables"]}, turns=[], - totals={"cost_usd": 0.1}, + totals={"cost_usd": archive_cost}, ) archive = folder / "attempts" / f"{sid}-1.json" digest = cr.write_json(archive, failed) final.update( - attempts=[nt.entry(archive, failed, 1, digest)], totals={"cost_usd": 0.2} + attempts=[nt.entry(archive, failed, 1, digest)], + totals={"cost_usd": final_cost}, ) cr.write_json(folder / f"{sid}.json", final) optional = full_record(world.board, "gpt-6-sol", design["engine"], sid) - optional["totals"] = {"cost_usd": 0.05} + optional["totals"] = {"cost_usd": optional_cost} cr.write_json(no_tools_households(design, "gpt-6-sol") / f"{sid}.json", optional) - return 0.35 + return final_cost + archive_cost + optional_cost @pytest.mark.parametrize( @@ -360,6 +363,47 @@ def test_preflight_compares_spent_before_with_all_saved_spend( assert world.provider.count == {} +@pytest.mark.parametrize( + "costs", + sorted( + { + *itertools.permutations((0.1, 0.1, 0.4)), + *itertools.permutations((0.1, 0.2, 0.3)), + } + ), +) +def test_preflight_sums_saved_spend_exactly_in_any_order(tmp_path, monkeypatch, costs): + """Added one at a time, these costs land on either side of the tolerance + with their order; the report's exact total does not move.""" + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design, *costs) + design["spent_before"] = 0.599999 + problems = cr.preflight(design, (cr.CHEAP[0],)) + refused = 0.599999 < math.fsum(costs) - nt.MONEY_TOLERANCE + assert any(p.startswith("no-tools spend") for p in problems) is refused, problems + + +@pytest.mark.parametrize( + "declared, refused", [(1e16, True), (1e16 + 2, False)], ids=["short", "exact"] +) +def test_preflight_keeps_costs_a_running_total_rounds_away( + tmp_path, monkeypatch, declared, refused +): + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design, 1e16, 1.0, 1.0) + design["spent_before"] = declared + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(p.startswith("no-tools spend") for p in problems) is refused, problems + + +def test_preflight_refuses_a_saved_total_past_every_float(tmp_path, monkeypatch): + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design, 1.5e308, 0.0, 1.5e308) + design["spent_before"] = 1e308 + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any("the saved records' $inf" in problem for problem in problems), problems + + @pytest.mark.parametrize( "value", [float("nan"), float("inf"), None, "unknown", -1e-7, 10**400], @@ -373,6 +417,31 @@ def test_preflight_refuses_an_unusable_spent_before(tmp_path, monkeypatch, value assert world.provider.count == {} +def test_preflight_refuses_a_design_without_spent_before(tmp_path, monkeypatch): + world, design = pool_design(tmp_path, monkeypatch) + del design["spent_before"] + assert "the design has no spent_before" in cr.preflight(design, (cr.CHEAP[0],)) + assert any("finite, nonnegative" in p for p in nt.stage_problems(design)) + assert world.provider.count == {} + + +@pytest.mark.parametrize( + "declared, refused", + [("1", False), ("0.1", True), (True, False), (False, True)], + ids=["text-enough", "text-short", "true", "false"], +) +def test_preflight_counts_numeric_text_and_booleans_as_numbers( + tmp_path, monkeypatch, declared, refused +): + """As the report does: float() decides what a spent_before is worth.""" + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design) + design["spent_before"] = declared + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(p.startswith("no-tools spend") for p in problems) is refused, problems + assert not any("finite, nonnegative" in p for p in problems), problems + + @pytest.mark.parametrize( "fault, needle", [ From 3c80377ebe18b8b162c58b8a07933931f8859817 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 23:16:38 -0400 Subject: [PATCH 60/64] Confirmatory no-tools: pin startup's exact spend at the stage cap Review runner-r20 found that restoring a running total at startup alone would pass every test. A new case saves costs whose running total stays under the cap and whose exact sum reaches it, and asserts that no owed attempt starts and that the invocation lock comes free. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 4 ++-- tests/test_confirmatory_no_tools.py | 30 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index af728dad..ac39aab8 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -552,8 +552,8 @@ def history( def exact_spend(costs: list[float]) -> float: """The correctly rounded sum of saved costs, the same in any order and - grouping, so that this reaches the report's total. Its intermediate overflow - means a total past every float.""" + grouping, so that this reaches the report's total. An OverflowError gives + infinity, which every finite `spent_before` falls short of.""" try: return math.fsum(costs) except OverflowError: diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 2c56bb4a..b7d184f8 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -3,8 +3,11 @@ from __future__ import annotations import ast +import functools import io import json +import math +import operator import os import signal import sys @@ -1487,6 +1490,33 @@ def test_valid_saved_numeric_cost_preserves_spend(setup, cost): assert provider.calls == [] +def test_startup_spend_takes_the_exact_sum_at_the_stage_cap(setup): + """A running total of these costs stays under the cap and their exact sum + reaches it, so no owed attempt starts and the invocation lock comes free.""" + costs = [math.nextafter(nt.NO_TOOLS_CAP, 0.0), 2**-52, 2**-52, 2**-52] + assert functools.reduce(operator.add, costs) < nt.NO_TOOLS_CAP + root = Path(setup["design"]["no_tools"]["runs"]) + saved = [ + (nt.CHEAP[0], "h2"), + (nt.CHEAP[0], "h1"), + (nt.CHEAP[1], "h2"), + (nt.CHEAP[2], "h2"), + ] + for (model, sid), cost in zip(saved, costs): + folder = root / model / "none/confirmatory/households" + folder.mkdir(parents=True, exist_ok=True) + profile = SimpleNamespace(name=model, arm="none", model_id="synthetic") + payload = dict(record(sid, profile, cost=cost), attempts=[]) + (folder / f"{sid}.json").write_text(json.dumps(payload)) + provider = Provider() + plan = stage(setup, provider) + assert plan.spend.spent_usd == nt.NO_TOOLS_CAP + assert plan.stopped.is_set() + assert plan.execute() == 1 + assert provider.calls == [] + assert plan._lock_file is None + + def run_no_tools_probe(setup, tmp_path, name, body): """Bound a signal or worker-shutdown regression to its own process.""" import textwrap From 58ba64be0241ff93be4c7fb3a7e3f4aa50645b75 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 00:54:05 -0400 Subject: [PATCH 61/64] No-tools startup: test the sum across models at the cap and the lock's release Review runner-r21 found two gaps in the startup test. Rounding each model's costs first escaped it, so a second test saves costs whose subtotals fall one float under the cap and whose exact sum reaches it. The test's lock assertion read bookkeeping, so both tests now take the lock from a second open. `exact_spend`'s docstring limits the claim to a flat list of costs, in any order, and to histories that pass validation. Production code changes only in that docstring. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 7 ++-- tests/test_confirmatory_no_tools.py | 54 ++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index ac39aab8..d3891a8e 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -551,9 +551,10 @@ def history( def exact_spend(costs: list[float]) -> float: - """The correctly rounded sum of saved costs, the same in any order and - grouping, so that this reaches the report's total. An OverflowError gives - infinity, which every finite `spent_before` falls short of.""" + """The correctly rounded sum of a flat list of saved costs, the same in any + order. For histories that pass validation this reaches the report's total. + An OverflowError gives infinity, which every finite `spent_before` falls + short of.""" try: return math.fsum(costs) except OverflowError: diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index b7d184f8..e53abf47 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import fcntl import functools import io import json @@ -1490,19 +1491,11 @@ def test_valid_saved_numeric_cost_preserves_spend(setup, cost): assert provider.calls == [] -def test_startup_spend_takes_the_exact_sum_at_the_stage_cap(setup): - """A running total of these costs stays under the cap and their exact sum - reaches it, so no owed attempt starts and the invocation lock comes free.""" - costs = [math.nextafter(nt.NO_TOOLS_CAP, 0.0), 2**-52, 2**-52, 2**-52] - assert functools.reduce(operator.add, costs) < nt.NO_TOOLS_CAP +def require_a_stop_at_the_cap(setup, saved): + """Save these (model, household, cost) finals, and require that startup's sum + reaches the cap, no owed attempt starts, and another open takes the lock.""" root = Path(setup["design"]["no_tools"]["runs"]) - saved = [ - (nt.CHEAP[0], "h2"), - (nt.CHEAP[0], "h1"), - (nt.CHEAP[1], "h2"), - (nt.CHEAP[2], "h2"), - ] - for (model, sid), cost in zip(saved, costs): + for model, sid, cost in saved: folder = root / model / "none/confirmatory/households" folder.mkdir(parents=True, exist_ok=True) profile = SimpleNamespace(name=model, arm="none", model_id="synthetic") @@ -1514,7 +1507,42 @@ def test_startup_spend_takes_the_exact_sum_at_the_stage_cap(setup): assert plan.stopped.is_set() assert plan.execute() == 1 assert provider.calls == [] - assert plan._lock_file is None + with (root / ".confirmatory-no-tools.lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + pytest.fail(f"the invocation lock stayed held: {error}") + + +def test_startup_spend_takes_the_exact_sum_at_the_stage_cap(setup): + """A running total of these costs stays under the cap and their exact sum + reaches it.""" + costs = [math.nextafter(nt.NO_TOOLS_CAP, 0.0), 2**-52, 2**-52, 2**-52] + assert functools.reduce(operator.add, costs) < nt.NO_TOOLS_CAP + places = [ + (nt.CHEAP[0], "h2"), + (nt.CHEAP[0], "h1"), + (nt.CHEAP[1], "h2"), + (nt.CHEAP[2], "h2"), + ] + require_a_stop_at_the_cap(setup, [(*p, c) for p, c in zip(places, costs)]) + + +def test_startup_spend_sums_every_model_at_once_at_the_stage_cap(setup): + """Rounding each model's costs first falls one float under the cap, and the + exact sum of all three reaches it.""" + first = [3 * 2**-53, math.nextafter(nt.NO_TOOLS_CAP, 0.0)] + second = [2**-52] + subtotals = [nt.exact_spend(first), nt.exact_spend(second)] + assert nt.exact_spend(subtotals) < nt.NO_TOOLS_CAP + require_a_stop_at_the_cap( + setup, + [ + (nt.CHEAP[0], "h1", first[0]), + (nt.CHEAP[0], "h2", first[1]), + (nt.CHEAP[1], "h2", second[0]), + ], + ) def run_no_tools_probe(setup, tmp_path, name, body): From 384666600fb35be1e6b13720716739f19f10ae2c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 01:50:40 -0400 Subject: [PATCH 62/64] No-tools startup: test costs whose sum overflows Review runner-r22 found that startup's overflow rule had no test. Replacing `exact_spend(costs)` with `math.fsum(costs)` at startup alone would have raised a raw OverflowError and passed every test, because the existing overflow cases reach the preflight or refuse one invalid cost. A new test saves two valid 1.5e308 finals and requires that startup takes infinity for the total, stops every owed attempt and frees the lock. The helper takes the expected total as an argument. Production code doesn't change. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_no_tools.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index e53abf47..14c38fd9 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -1491,9 +1491,9 @@ def test_valid_saved_numeric_cost_preserves_spend(setup, cost): assert provider.calls == [] -def require_a_stop_at_the_cap(setup, saved): +def require_a_stop_at_the_cap(setup, saved, spent=nt.NO_TOOLS_CAP): """Save these (model, household, cost) finals, and require that startup's sum - reaches the cap, no owed attempt starts, and another open takes the lock.""" + reaches `spent`, no owed attempt starts, and another open takes the lock.""" root = Path(setup["design"]["no_tools"]["runs"]) for model, sid, cost in saved: folder = root / model / "none/confirmatory/households" @@ -1503,7 +1503,7 @@ def require_a_stop_at_the_cap(setup, saved): (folder / f"{sid}.json").write_text(json.dumps(payload)) provider = Provider() plan = stage(setup, provider) - assert plan.spend.spent_usd == nt.NO_TOOLS_CAP + assert plan.spend.spent_usd == spent assert plan.stopped.is_set() assert plan.execute() == 1 assert provider.calls == [] @@ -1545,6 +1545,19 @@ def test_startup_spend_sums_every_model_at_once_at_the_stage_cap(setup): ) +def test_startup_spend_stops_on_costs_whose_sum_overflows(setup): + """Each cost fits a float and `math.fsum` overflows on the pair, so startup + takes infinity for the total and stops.""" + costs = [1.5e308, 1.5e308] + with pytest.raises(OverflowError): + math.fsum(costs) + require_a_stop_at_the_cap( + setup, + [(nt.CHEAP[0], "h1", costs[0]), (nt.CHEAP[1], "h1", costs[1])], + spent=math.inf, + ) + + def run_no_tools_probe(setup, tmp_path, name, body): """Bound a signal or worker-shutdown regression to its own process.""" import textwrap From 2ee8acfc0bc9d66ec191077b6869dad620e764c9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 03:06:15 -0400 Subject: [PATCH 63/64] Stub the host facts in the confirmatory tests' World CI's test job failed 22 confirmatory tests on Linux at 38466660; they had passed on the Mac that ran every earlier suite. Fourteen reached a preflight check that compares the installed policyengine-us with the board's engine, and the lockfile installs 2.41.0 where the fixture board names 2.60.4. Eight reached host_power, which calls macOS's pmset. World already fakes the board's engine, so it now fakes the installed version and the power reading too. Its tests no longer read either from the host. A test that wants a different reading still sets it after World, as the lid-closed test does. The eight stubs in the unreadable-evidence tests and the ones in pool_design and engine_fault repeat World's and go. Production code does not change. A pytest plugin that hides pmset and reports 2.41.0 reproduced CI's 22 failures on this Mac at 38466660, by name. With this commit all 772 runner tests pass under that plugin. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_board_engine.py | 3 --- tests/test_confirmatory_input_reads.py | 1 - tests/test_confirmatory_run.py | 3 +++ tests/test_confirmatory_unreadable_evidence.py | 8 -------- 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/tests/test_confirmatory_board_engine.py b/tests/test_confirmatory_board_engine.py index 99dc3faf..e8dc7924 100644 --- a/tests/test_confirmatory_board_engine.py +++ b/tests/test_confirmatory_board_engine.py @@ -8,7 +8,6 @@ import pytest from tests.test_confirmatory_run import ( - ENGINE, complete_preflight_design, cr, pilot, @@ -32,8 +31,6 @@ def engine_fault(world, tmp_path, monkeypatch, fault): design = complete_preflight_design(w, tmp_path) metadata = Path(design["board"]) / "reference_outputs.csv.meta.json" monkeypatch.setattr(pilot, "board_engine", REAL_BOARD_ENGINE) - monkeypatch.setattr(pilot, "_versions", lambda: {"policyengine-us": ENGINE}) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) def no_run(*args, **kwargs): pytest.fail("a refused preflight must not construct or call a provider") diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index 31894d88..e85e7562 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -259,7 +259,6 @@ def write_then_change(path, payload, *args, **kwargs): def pool_design(tmp_path, monkeypatch): monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) world = World(tmp_path, monkeypatch, models=(cr.CHEAP[0],)) return world, complete_preflight_design(world, tmp_path) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 0616463b..44068cd8 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -789,6 +789,9 @@ def __init__( } monkeypatch.setattr(pilot, "load_board", lambda path: self.board) monkeypatch.setattr(pilot, "board_engine", lambda path=None: ENGINE) + installed = {**pilot._versions(), "policyengine-us": ENGINE} + monkeypatch.setattr(pilot, "_versions", lambda: installed) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) monkeypatch.setitem( pilot._STATED_ENGINE, "version", pilot._STATED_ENGINE["version"] ) diff --git a/tests/test_confirmatory_unreadable_evidence.py b/tests/test_confirmatory_unreadable_evidence.py index f48be754..cac7af15 100644 --- a/tests/test_confirmatory_unreadable_evidence.py +++ b/tests/test_confirmatory_unreadable_evidence.py @@ -72,7 +72,6 @@ def test_unlistable_optional_history_refuses( else: w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) spec = design["no_tools"] folder = ( Path(spec["runs"]) / "gpt-6-sol" / spec["folder"] / spec["stage"] / "households" @@ -101,7 +100,6 @@ def test_model_inventory_failure_refuses( else: w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) root = Path(design["no_tools"]["runs"]) (root / "gpt-6-sol").mkdir(parents=True, exist_ok=True) target = root / "gpt-6-sol" if which == "model-stat" else root @@ -148,7 +146,6 @@ def test_listed_model_entry_without_metadata_refuses( else: w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) root = Path(design["no_tools"]["runs"]) root.mkdir(parents=True, exist_ok=True) target = root / "ghost-model" @@ -173,7 +170,6 @@ def test_preflight_listed_calculator_entry_without_metadata_refuses( ): w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) folder = Path(design["runs"]) / cr.CHEAP[0] parent = folder if level == "arm" else folder / "pe" parent.mkdir(parents=True, exist_ok=True) @@ -196,7 +192,6 @@ def test_broken_optional_history_folder_refuses( else: w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) spec = design["no_tools"] folder = ( Path(spec["runs"]) / "gpt-6-sol" / spec["folder"] / spec["stage"] / "households" @@ -259,7 +254,6 @@ def test_calculator_preflight_inventory_failure_refuses( ): w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) folder = Path(design["runs"]) / cr.CHEAP[0] targets = { "model": folder, @@ -286,7 +280,6 @@ def test_calculator_preflight_directory_stat_failure_refuses( ): w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) folder = Path(design["runs"]) / cr.CHEAP[0] target = ( folder @@ -315,7 +308,6 @@ def denied(path): return original(path) monkeypatch.setattr(Path, "read_bytes", denied) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) assert not w.provider.count From 0ab2b1e25a1bdf3564d6ad2085fc774b80b98c63 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 03:10:45 -0400 Subject: [PATCH 64/64] Test the host facts that World stubs World now reports the installed engine and the power reading, so no test could see the checks that read them, and no earlier test had held those checks with the real values either. An installed engine other than the board's must now refuse in the preflight twice, in its own check and through the no-tools treatment check, and again in main's final check, which must keep the preflight's problems. Each test requires that no provider is called. host_power gets its first test, on the four mixes of power source and lid state, with the text pmset and ioreg print on this Mac. Production code does not change. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_board_engine.py | 46 +++++++++++++++++++++++++ tests/test_confirmatory_run.py | 22 ++++++++++++ 2 files changed, 68 insertions(+) diff --git a/tests/test_confirmatory_board_engine.py b/tests/test_confirmatory_board_engine.py index e8dc7924..c7361703 100644 --- a/tests/test_confirmatory_board_engine.py +++ b/tests/test_confirmatory_board_engine.py @@ -8,6 +8,7 @@ import pytest from tests.test_confirmatory_run import ( + ENGINE, complete_preflight_design, cr, pilot, @@ -94,3 +95,48 @@ def test_main_names_engine_reread_failure_and_preserves_all_problems( assert "earlier preflight problem" in output assert str(metadata) in output assert not w.provider.count + + +def installed_elsewhere(world, tmp_path, monkeypatch): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(pilot, "_versions", lambda: {"policyengine-us": "2.41.0"}) + + def no_run(*args, **kwargs): + pytest.fail("a refused preflight must not construct or call a provider") + + monkeypatch.setattr(cr, "Run", no_run) + monkeypatch.setattr(pilot, "run_household", no_run) + return w, design + + +def test_preflight_refuses_an_installed_engine_other_than_the_boards( + world, tmp_path, monkeypatch +): + w, design = installed_elsewhere(world, tmp_path, monkeypatch) + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert f"policyengine-us 2.41.0 is installed, not {ENGINE}" in problems + for model in cr.CHEAP: + assert any( + problem.startswith(f"no-tools {model}: ") + and f"policyengine-us 2.41.0 is installed; the pilot pins {ENGINE}" + in problem + for problem in problems + ), problems + assert not w.provider.count + + +def test_main_rechecks_the_installed_engine_and_preserves_all_problems( + world, tmp_path, monkeypatch, capsys +): + w, design = installed_elsewhere(world, tmp_path, monkeypatch) + # Isolate main's final engine check, which must retain preflight's problems. + monkeypatch.setattr(cr, "preflight", lambda *_: ["earlier preflight problem"]) + design_path = tmp_path / "design.json" + design_path.write_text(json.dumps(design)) + assert cr.main(["run", str(design_path)]) == 1 + output = capsys.readouterr().out + assert "The run cannot start" in output + assert "earlier preflight problem" in output + assert "the board's engine check failed: policyengine-us 2.41.0" in output + assert not w.provider.count diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 44068cd8..e2e924f1 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2280,6 +2280,28 @@ def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( ) +@pytest.mark.parametrize( + "drawing, clamshell, expected", + [ + ("AC Power", "No", (True, True)), + ("Battery Power", "No", (False, True)), + ("AC Power", "Yes", (True, False)), + ("Battery Power", "Yes", (False, False)), + ], +) +def test_host_power_reads_pmset_and_ioreg(monkeypatch, drawing, clamshell, expected): + output = { + "pmset": f"Now drawing from '{drawing}'\n", + "ioreg": f' "AppleClamshellState" = {clamshell}\n', + } + + def run(command, **kwargs): + return subprocess.CompletedProcess(command, 0, stdout=output[command[0]]) + + monkeypatch.setattr(cr.subprocess, "run", run) + assert cr.host_power() == expected + + def test_the_preflight_refuses_a_runs_folder_with_records(world, tmp_path): w = world(models=("gpt-6-luna",)).execute() design = complete_preflight_design(w, tmp_path)