diff --git a/policybench/reference_adversary.py b/policybench/reference_adversary.py index 5dfd5740..301a2d7e 100644 --- a/policybench/reference_adversary.py +++ b/policybench/reference_adversary.py @@ -73,6 +73,11 @@ REFERENCE_FREEZES = {"us": REFERENCE_FREEZE, "uk": "2026-10-10"} UK_FISCAL_YEAR = f"{TAX_YEAR}-{str(TAX_YEAR + 1)[-2:]}" +# How the stage headers say the consensus models answered. A payload's +# ``answerCondition`` replaces it, for clusters of tool-using arms (the tool +# pilot's web arms); the default renders the headers byte for byte as before. +DEFAULT_ANSWER_CONDITION = "answering from memory, without tools" + VERDICTS = ( "reference_holds", "reference_wrong", @@ -457,6 +462,7 @@ class AdversaryCase: metric_type: str = "amount" state: str = "" trigger: tuple[str, ...] = () + answer_condition: str = DEFAULT_ANSWER_CONDITION def to_manifest_row(self) -> dict: """The case as a ``cases.jsonl`` row, without the derivation.""" @@ -465,6 +471,8 @@ def to_manifest_row(self) -> dict: row["consensus"] = [dict(cluster) for cluster in self.consensus] row["trigger"] = list(self.trigger) row["derivation_sha256"] = sha256_text(derivation) if derivation else None + if self.answer_condition == DEFAULT_ANSWER_CONDITION: + row.pop("answer_condition") return row @classmethod @@ -486,6 +494,9 @@ def from_manifest_row(cls, row: dict, derivation: str) -> AdversaryCase: metric_type=str(row.get("metric_type", "amount")), state=str(row.get("state", "") or ""), trigger=tuple(row.get("trigger", ())), + answer_condition=str( + row.get("answer_condition") or DEFAULT_ANSWER_CONDITION + ), ) @@ -600,6 +611,10 @@ def build_adversary_cases( country = str(payload.get("country") or "us") scenarios = payload.get("scenarios") or {} predictions = payload.get("scenarioPredictions") or {} + # A payload may state the definitions its models saw (the tool pilot's + # v2-pilot wording) and the condition they answered under. + definitions = payload.get("outputDefinitions") or {} + condition = answer_condition_of(payload) cases: list[AdversaryCase] = [] seen: set[tuple[str, str]] = set() for flag in flags: @@ -673,7 +688,8 @@ def build_adversary_cases( country=country, scenario_id=scenario_id, variable=variable, - definition=get_variable_description(variable, country=country), + definition=_clean(definitions.get(variable)) + or get_variable_description(variable, country=country), household_prompt=household_prompt, reference_value=reference, consensus=tuple(clusters), @@ -684,11 +700,33 @@ def build_adversary_cases( metric_type=metric_type_for_output(variable), state=_clean(flag.get("state") or scenario.get("state")), trigger=tuple(str(t) for t in flag.get("trigger", ())), + answer_condition=condition, ) ) return cases +def answer_condition_of(payload: dict) -> str: + """The payload's ``answerCondition``, checked, or the default.""" + condition = payload.get("answerCondition") + if condition is None: + return DEFAULT_ANSWER_CONDITION + if not isinstance(condition, str) or not condition.startswith("answering "): + raise ValueError( + f"answerCondition must be a phrase starting 'answering ': {condition!r}" + ) + return condition + + +def _with_condition(header: str, case: AdversaryCase) -> str: + """A stage header stating the case's answer condition.""" + if case.answer_condition == DEFAULT_ANSWER_CONDITION: + return header + if header.count(DEFAULT_ANSWER_CONDITION) != 1: + raise ValueError(f"{case.country} header does not state the answer condition") + return header.replace(DEFAULT_ANSWER_CONDITION, case.answer_condition) + + # --- Prompts ------------------------------------------------------------------- _DOMAINS = ", ".join(BLOCKED_DOMAINS) @@ -1022,7 +1060,7 @@ def _case_sections(case: AdversaryCase) -> list[str]: def render_stage1_prompt(case: AdversaryCase) -> str: """The law-first prompt: everything about the case except the derivation.""" - header = _country_template(_STAGE1_HEADERS, case.country) + header = _with_condition(_country_template(_STAGE1_HEADERS, case.country), case) return "\n".join([header, "", *_case_sections(case)]) @@ -1036,7 +1074,7 @@ def render_stage2_prompt(case: AdversaryCase, stage1: dict, stage1_sha256: str) derivation = case.derivation or "(no derivation was recorded for this case)" return "\n".join( [ - _country_template(_STAGE2_HEADERS, case.country), + _with_condition(_country_template(_STAGE2_HEADERS, case.country), case), "", *_case_sections(case), "", @@ -1646,6 +1684,24 @@ def _markers_in(text: str) -> list[str]: return [marker for marker in DERIVATION_MARKERS if marker in lowered] +# An exclusion of a blocked domain in a search query ("-site:policyengine.org"): +# how a judge whose search tool takes no blocked_domains (Codex) follows the +# prompt's rule to exclude those domains on every search. It keeps a domain +# out; it does not search for it. +_EXCLUDED_DOMAIN = re.compile( + r"(? str: + """A search query without its exclusions of blocked domains: what the + judge searched for, for the contamination check.""" + return _EXCLUDED_DOMAIN.sub(" ", str(query or "")) + + def codex_events_audit(events_text: str) -> tuple[list[str], dict]: """Audit a ``codex exec --json`` event log for contamination. @@ -1714,7 +1770,7 @@ def codex_events_audit(events_text: str) -> tuple[list[str], dict]: ] url = str(action.get("url") or "") for text in [str(item.get("query") or ""), *searched]: - for marker in _markers_in(text): + for marker in _markers_in(searched_for(text)): problems.append(f"line {number}: a web search names {marker!r}") for marker in _markers_in(str(action.get("pattern") or "")): problems.append(f"line {number}: an in-page search names {marker!r}") @@ -1819,7 +1875,7 @@ def claude_transcript_audit( query = str(payload.get("query") or "") activity["searches"].append(query) searches.append((str(part.get("id")), query)) - if any(n in query.lower() for n in BLOCKED_NAMES): + if any(n in searched_for(query).lower() for n in BLOCKED_NAMES): problems.append(f"the judge searched for the engine: {query}") else: problems.append(f"the judge called a tool it was not given: {name}") @@ -2064,6 +2120,7 @@ def check_login( declared: str, config_dir: str, desktop_opt_in: bool, + api_key_secret: str = "", ) -> dict: """The Claude login a runner may judge with, or ``ValueError`` saying why not. @@ -2073,6 +2130,12 @@ def check_login( in which case it must be exactly the desktop's own claude.ai login. Unlike that script, it also refuses a token lane whose declared account (AUDIT_ACCOUNT) is the desktop login's. + + With ``api_key_secret`` (the runner's JUDGE_API_KEY_SECRET, Max + 2026-10-22) the judges bill that API key instead: the CLI must report a + first-party ``api_key`` login from ``ANTHROPIC_API_KEY`` in exactly + ``config_dir`` (the runner's fresh, empty one), with no lane token and no + desktop opt-in. """ def parse(text: str) -> dict: @@ -2092,6 +2155,31 @@ def parse(text: str) -> dict: f"the login is through {status.get('apiProvider')!r}, not a " "first-party subscription" ) + if api_key_secret: + if token or desktop_opt_in: + raise ValueError( + "an API key judge takes no lane token and no desktop opt-in" + ) + if method != "api_key" or status.get("apiKeySource") != "ANTHROPIC_API_KEY": + raise ValueError( + f"JUDGE_API_KEY_SECRET is set but the CLI logs in by {method!r} " + f"from {status.get('apiKeySource')!r}, not ANTHROPIC_API_KEY" + ) + if not ( + isinstance(reported, str) + and os.path.isdir(reported) + and os.path.samefile(reported, config_dir) + ): + raise ValueError( + f"the CLI reports config directory {reported!r}, not the run's " + f"own {config_dir}" + ) + return { + "method": "api_key", + "account": None, + "org": None, + "secret": api_key_secret, + } if method not in ("oauth_token", "claude.ai"): raise ValueError(f"the login method {method!r} is not a subscription login") same_account = bool( @@ -2188,6 +2276,7 @@ def main(argv: list[str] | None = None) -> int: login.add_argument("--declared", default="") login.add_argument("--config-dir", required=True) login.add_argument("--desktop-opt-in", choices=["0", "1"], default="0") + login.add_argument("--api-key-secret", default="") args = parser.parse_args(argv) if args.command == "render-stage2": @@ -2226,6 +2315,7 @@ def main(argv: list[str] | None = None) -> int: declared=args.declared, config_dir=args.config_dir, desktop_opt_in=args.desktop_opt_in == "1", + api_key_secret=args.api_key_secret, ) except ValueError as exc: print(str(exc), file=sys.stderr) diff --git a/policybench/scenarios.py b/policybench/scenarios.py index 35e88d00..4e8ae379 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -918,19 +918,29 @@ def scenario_from_dict(data: dict[str, Any]) -> Scenario: ) -def load_certified_us_person_frame() -> tuple[pd.DataFrame, int, str]: +def load_certified_us_person_frame( + dataset: str | None = None, dataset_label: str | None = None +) -> tuple[pd.DataFrame, int, str]: """Load a person-level frame from the certified US microsimulation dataset. Returns ``(person_df, dataset_year, dataset_label)``. The label is read from the live simulation's PolicyEngine bundle (``default_dataset``), so it reflects the dataset the run actually materialized -- the certified populace build (``populace_us_2024``) -- rather than a hardcoded name. + + ``dataset`` (a local H5 path) replaces the certified dataset, for test + household sets only; ``dataset_label`` then names it. """ from policybench.policyengine_runtime import make_us_microsimulation - sim = make_us_microsimulation() + if dataset is None: + sim = make_us_microsimulation() + dataset_label = str(sim.policyengine_bundle["default_dataset"]) + else: + if not dataset_label: + raise ValueError("an explicit dataset needs a dataset_label") + sim = make_us_microsimulation(dataset=dataset) dataset_year = sim.default_input_period - dataset_label = str(sim.policyengine_bundle["default_dataset"]) input_specs = get_promptable_input_specs() values = {} @@ -1822,10 +1832,22 @@ def generate_scenarios( seed: int = SEED, excluded_household_ids: set[int] | None = None, country: str = DEFAULT_COUNTRY, + dataset: str | None = None, + dataset_label: str | None = None, ) -> list[Scenario]: - """Generate benchmark scenarios for a country.""" + """Generate benchmark scenarios for a country. + + ``dataset`` and ``dataset_label`` replace the certified US dataset (see + :func:`load_certified_us_person_frame`); test household sets only. + """ + if dataset is not None and country != "us": + raise ValueError("an explicit dataset is supported for the US only") if country == "us": - person_df, dataset_year, dataset_label = load_certified_us_person_frame() + person_df, dataset_year, dataset_label = ( + load_certified_us_person_frame() + if dataset is None + else load_certified_us_person_frame(dataset, dataset_label) + ) return scenarios_from_cps_frame( person_df, n=n, diff --git a/scripts/batch_judge.py b/scripts/batch_judge.py new file mode 100644 index 00000000..17e33045 --- /dev/null +++ b/scripts/batch_judge.py @@ -0,0 +1,328 @@ +"""AI judges on each batch of a household set, before the next batch runs. + +Max, 2026-10-22: "we should go slowly since the new microdata could reveal +different pe bugs - we should do ai judges in small batches or real time". + +Each batch of a household set (scripts/household_set.py) runs every model in +every arm of the tool pilot (scripts/tool_pilot_argon.py --board). This script +then hands the batch to PolicyBench's reference adversary +(policybench/reference_adversary.py), arm by arm: + + batch_judge.py payload --board DIR --runs ROOT --stage S --arm none \\ + --batch K --out payload.json + policybench consensus-flags --payload payload.json --output flags.json \\ + --min-models 2 --top-k 3 --min-top 2 --zero-cluster-min-models 2 + batch_judge.py derivations --board DIR --payload payload.json \\ + --flags flags.json --out-dir annotations/ + policybench adversary-prepare --payload payload.json --flags flags.json \\ + --adversary-dir judge/ --annotations-dir annotations/ + scripts/run_reference_adversary_claude.sh judge/ + policybench adversary-collect --adversary-dir claude=judge/ --output-dir out/ + +``payload`` writes one arm's answers as a dashboard payload, each model a +"model" of the board, so the consensus trigger and the adversary run on it +unchanged. Only the no-tools and web arms are judged: a calculator arm answers +with the reference engine itself, and its explanations name it, which the +adversary's blinding refuses. The payload carries what the models saw: the +v2-pilot household prompt (the tool contract's, checked against each record's +sent prompt hash), the v2-pilot output definitions, and the condition they +answered under (from memory, or with web search). + +The trigger's thresholds are the board's, scaled to three models: a cell is +flagged when two of the three models agree on one answer other than the +reference (``min_models`` 2, a zero cluster too), or when two of the three +(all are "top") do. + +``derivations`` writes the engine derivation of each flagged reference: the +pruned computation trace on the household set's own reference system (its +engine and fix modules), in the frozen annotations' CSV format. A trace is the +engine's own record; no model narrates it. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path + +import pandas as pd + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "scripts")) + +import tool_pilot_argon as pilot # noqa: E402 + +MODELS = ("gpt-6-sol", "claude-opus-5.5", "gemini-4-argon") +JUDGED_ARMS = { + "none": "answering from memory, without tools", + "web": "answering with web search", +} +# The tool contract's household prompt is the dashboard payload's +# ``prompt.tool``; this model is on the tool contract. +PROMPT_MODEL = "gpt-6-sol" +TRACE_MAX_LINES = 400 +YEAR = 2026 + + +def arm_folder(arm: str) -> str: + return arm.replace("+", "-") + + +def load_batch(board: Path, batch: int) -> list[str]: + batches = json.loads((board / "batches.json").read_text())["batches"] + if not 0 <= batch < len(batches): + raise SystemExit(f"{board} has batches 0 to {len(batches) - 1}") + return list(batches[batch]) + + +def prompt_model_for(models) -> str: + """The model whose household prompt a payload shows: the first on the tool + contract (the dashboard's ``prompt.tool``), else the first model.""" + for model in models: + if pilot.model_profile(model, "none").contract == "tool": + return model + return models[0] + + +def household_prompt( + scenario, variables: list[str], prompt_version: str, model: str = PROMPT_MODEL +) -> str: + """The household prompt ``model`` saw, without the arm's notice: the + board's prompt in the run's prompt version.""" + profile = pilot.model_profile(model, "none", prompt_version, False) + prompt, kwargs = pilot.build_initial_request(scenario, variables, profile) + return prompt[: len(prompt) - len(profile.notice)] if profile.notice else prompt + + +def output_definitions(variables, prompt_version: str) -> dict[str, str]: + from policybench.prompts import get_variable_description + + definitions = {} + for variable in sorted(set(variables)): + text = get_variable_description(variable, country="us") + if prompt_version == "v2-pilot": + text = pilot.v2_pilot_text(text) + definitions[variable] = text + return definitions + + +def build_payload( + board_dir: Path, + runs: Path, + stage: str, + arm: str, + batch: int | None = None, + scenarios: list[str] | None = None, + models: tuple[str, ...] = MODELS, +) -> dict: + """One arm's answers on one batch (or on named households), as a + dashboard payload, from ``models`` (the frontier three by default; cheap + models for development, Max 2026-10-23).""" + models = tuple(models) + if len(models) < 2: + raise SystemExit("a payload needs at least two models") + prompt_model = prompt_model_for(models) + if arm not in JUDGED_ARMS: + raise SystemExit(f"only {sorted(JUDGED_ARMS)} are judged, not {arm}") + if (batch is None) == (not scenarios): + raise SystemExit("name a batch or households, not both") + board = pilot.load_board(board_dir) + ids = list(scenarios) if scenarios else load_batch(board_dir, batch) + unknown = sorted(set(ids) - set(board["scenarios"])) + if unknown: + raise SystemExit(f"{board_dir} has no households {unknown}") + scored = set(zip(board["scored"].scenario_id, board["scored"].variable)) + reference = board["reference"].set_index(["scenario_id", "variable"])["value"] + records = {} + for model in models: + folder = runs / model / arm_folder(arm) / stage / "households" + found = {} + for sid in ids: + path = folder / f"{sid}.json" + if not path.exists(): + raise SystemExit(f"{path} is missing: the batch has not finished") + found[sid] = json.loads(path.read_text()) + if pilot.records_model(found) != model or pilot.records_arm(found) != arm: + raise SystemExit(f"{folder} does not hold {model} {arm} records") + records[model] = found + versions = {pilot.records_prompt(r)[0] for r in records.values()} + if len(versions) != 1: + raise SystemExit(f"the models ran different prompt versions: {versions}") + prompt_version = versions.pop() + + labels = {model: f"{model}+{arm}" for model in models} + scenarios, predictions, variables_seen = {}, {}, set() + for sid in ids: + scenario = board["scenarios"][sid] + variables = board["variables"][sid] + prompt = household_prompt(scenario, variables, prompt_version, prompt_model) + sent = records[prompt_model][sid].get("sent_board_prompt_sha256") + if sent != hashlib.sha256(prompt.encode()).hexdigest(): + raise SystemExit( + f"{sid}: the rebuilt prompt is not the one {prompt_model} saw" + ) + scenarios[sid] = {"prompt": {"tool": prompt}, "state": scenario.state} + cells = {} + for variable in variables: + variables_seen.add(variable) + cell = {} + for model, label in labels.items(): + record = records[model][sid] + cell[label] = { + "prediction": record["predictions"].get(variable), + "explanation": record["explanations"].get(variable), + "groundTruth": float(reference[(sid, variable)]), + "scored": (sid, variable) in scored, + } + cells[variable] = cell + predictions[sid] = cells + return { + "country": "us", + "source": { + "board": str(board_dir), + "board_reference_sha256": board["meta"].get("reference_csv_sha256"), + "runs": str(runs), + "stage": stage, + "arm": arm, + "batch": batch, + "households": ids, + "prompt_version": prompt_version, + }, + "answerCondition": JUDGED_ARMS[arm], + "models": list(models), + "outputDefinitions": output_definitions(variables_seen, prompt_version), + "modelStats": [{"model": labels[model]} for model in models], + "scenarios": scenarios, + "scenarioPredictions": predictions, + } + + +def render_trace(system, scenario, variable: str) -> str: + """The pruned computation trace of one output on ``system``.""" + import household_set + from policyengine_us import Simulation + + from policybench.case_reference_explanations import ( + _find_target_tree, + _render_trace, + ) + from policybench.ground_truth import ( + _extract_person_value, + _pe_variable_for_output, + _person_index_for_output, + ) + + pe_variable = _pe_variable_for_output(variable, "us") + simulation = Simulation( + tax_benefit_system=system, situation=household_set.situation_for(scenario) + ) + simulation.trace = True + result = simulation.calculate(pe_variable, YEAR) + tree = _find_target_tree(simulation.tracer.trees, pe_variable) + if tree is None: + raise RuntimeError(f"no trace for {pe_variable}") + value = float(_extract_person_value(result, scenario, variable)) + lines = _render_trace(tree) + cut = len(lines) > TRACE_MAX_LINES + person = _person_index_for_output(scenario, variable) + header = [ + f"Output {variable} = {value:g}, computed as the engine variable " + f"{pe_variable} for {YEAR}" + + (f", read for person index {person}." if person is not None else "."), + "Each line is a variable and its value; a variable's inputs are indented " + "beneath it. Zero-valued branches below the first level are omitted.", + "", + ] + footer = [f"... ({len(lines) - TRACE_MAX_LINES} more lines cut)"] if cut else [] + return "\n".join(header + lines[:TRACE_MAX_LINES] + footer) + + +def command_payload(args) -> None: + payload = build_payload( + Path(args.board), + Path(args.runs), + args.stage, + args.arm, + batch=args.batch, + scenarios=args.scenarios, + models=tuple(args.models), + ) + Path(args.out).write_text(json.dumps(payload, indent=1) + "\n") + cells = sum(len(v) for v in payload["scenarioPredictions"].values()) + print(f"{len(payload['scenarios'])} households, {cells} cells, arm {args.arm}") + + +def command_derivations(args) -> None: + board_dir = Path(args.board) + pilot._require_engine(board_dir) + payload = json.loads(Path(args.payload).read_text()) + if payload["source"]["board"] != str(board_dir): + raise SystemExit("the payload is from another board") + flags = json.loads(Path(args.flags).read_text())["flags"] + board = pilot.load_board(board_dir) + system = pilot.build_tool_system(reform=True, snapshot=board_dir) + rows = [] + for flag in flags: + sid, variable = flag["scenario_id"], flag["variable"] + rows.append( + { + "country": "us", + "scenario_id": sid, + "variable": variable, + "reference_value": flag["reference"], + "trace_lines": None, + "explanation": render_trace(system, board["scenarios"][sid], variable), + "error": None, + } + ) + for row in rows: + row["trace_lines"] = row["explanation"].count("\n") + 1 + out = Path(args.out_dir) + out.mkdir(parents=True, exist_ok=True) + pd.DataFrame( + rows, + columns=[ + "country", + "scenario_id", + "variable", + "reference_value", + "trace_lines", + "explanation", + "error", + ], + ).to_csv(out / "us_case_reference_explanations.csv", index=False) + print(f"{len(rows)} derivations in {out}") + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + payload = sub.add_parser("payload", help="one arm's batch as a dashboard payload") + payload.add_argument("--board", required=True) + payload.add_argument("--runs", required=True) + payload.add_argument("--stage", required=True) + payload.add_argument("--arm", required=True, choices=sorted(JUDGED_ARMS)) + payload.add_argument("--batch", type=int) + payload.add_argument( + "--scenarios", nargs="*", help="households, for a board without batches" + ) + payload.add_argument("--out", required=True) + payload.add_argument( + "--models", nargs="+", default=list(MODELS), help="the models to judge" + ) + payload.set_defaults(func=command_payload) + derivations = sub.add_parser("derivations", help="traces of flagged references") + derivations.add_argument("--board", required=True) + derivations.add_argument("--payload", required=True) + derivations.add_argument("--flags", required=True) + derivations.add_argument("--out-dir", required=True) + derivations.set_defaults(func=command_derivations) + args = parser.parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py new file mode 100644 index 00000000..18a56701 --- /dev/null +++ b/scripts/confirmatory_anchor.py @@ -0,0 +1,108 @@ +"""Push the confirmatory run's anchors outside the run. + +Usage: + python scripts/confirmatory_anchor.py --repo DIR ANCHORS + +The runner calls this every 15 minutes and at its end (with up to three final +tries), with a snapshot of its `anchors.jsonl`: each line holds every log's +line count and chain head at a time. The snapshot is a copy, not a record. +This copies it to `anchors.jsonl` in DIR, a clone checked out on a branch +of its own, commits it and pushes the branch to its remote. The remote's record +of each push is the anchor's time from outside the run: a log edited after an +anchor was pushed no longer matches that anchor's head. + +An exclusive non-blocking lock in the clone's `.git` directory covers every +read, copy, commit and push. Every Git command inherits its descriptor, so a +surviving command keeps the clone busy even if this helper alone is killed. +Ordinary local hooks and remote helpers inherit it too. The per-command +`core.fsmonitor=false` disables normal index-triggered fsmonitor startup. +A configured hook can still run `git fsmonitor--daemon start`, whose child +closes inherited descriptors; this falls under the configured-command limit: +configured commands must retain the descriptor without unlocking it in any +background clone writer. This advisory lock does not cover independent +writers or replacement of the lock file. The lock is local: processes on an +external remote server never hold it. The remote's safety rests on the byte-prefix +check and ordinary non-force pushes. + +A credential-cache daemon or any other descriptor holder can keep the clone +busy up to the holder's lifetime (the credential cache defaults to 900 +seconds), indefinitely if it never exits or closes the descriptor. Every +later try, including all three final tries, then fails as busy and is reported +as a failed push. A busy clone fails without changing its records. Under the +lock, the clone's current file must be a byte prefix of the snapshot (equality +is allowed): a delayed older command cannot replace newer evidence. This +checks bytes, not JSON or line boundaries; the runner supplies complete, +synced lines. Git's own index.lock is left alone; a git failure is a failed +push. Never force-push. +""" + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import subprocess +import sys +from pathlib import Path + + +def git(repo: Path, lock_fd: int, *args: str) -> str: + """Run Git with the clone lock inherited, retaining failures for the caller.""" + return subprocess.run( + ["git", "-c", "core.fsmonitor=false", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + pass_fds=(lock_fd,), + ).stdout + + +def main(argv=None) -> int: + """Push a whole snapshot only while exclusively extending the clone's file.""" + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--repo", type=Path, required=True, help="the anchors clone") + parser.add_argument( + "anchors", type=Path, help="a snapshot of the run's anchors.jsonl" + ) + args = parser.parse_args(argv) + repo = args.repo.expanduser() + target = repo / "anchors.jsonl" + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print( + "anchor clone busy: another helper or Git process holds its lock", + file=sys.stderr, + ) + return 1 + snapshot = args.anchors.read_bytes() + current = target.read_bytes() if target.exists() else b"" + if not snapshot.startswith(current): + print( + "refusing anchor snapshot: the clone's anchors.jsonl is not a byte " + "prefix of the snapshot", + file=sys.stderr, + ) + return 1 + if snapshot != current or not target.exists(): + target.write_bytes(snapshot) + lines = snapshot.decode().splitlines() + git(repo, lock.fileno(), "add", "anchors.jsonl") + if git(repo, lock.fileno(), "status", "--porcelain", "--", "anchors.jsonl"): + head = hashlib.sha256(lines[-1].encode()).hexdigest() if lines else "none" + git( + repo, + lock.fileno(), + "commit", + "-q", + "-m", + f"anchor {len(lines)}: {head}", + ) + git(repo, lock.fileno(), "push", "-q", "origin", "HEAD") + print(f"pushed {len(lines)} anchors") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py new file mode 100644 index 00000000..d3891a8e --- /dev/null +++ b/scripts/confirmatory_no_tools.py @@ -0,0 +1,1240 @@ +#!/usr/bin/env python3 +"""Run the confirmatory no-tools stage with complete, recoverable attempts. + +Every model runs in its own thread, in draw order, under a shared $5 ceiling. +A replaced final is moved byte for byte into ``households/attempts`` before +its rerun; the final links every archive by number, hash, status and reason. +Re-invocation resumes solely from those files and is an operator's approval +to rerun a paused account failure. Native harness resume is never used. +An archive move or attempt is admitted when its interruption check and commit +under the admission lock precede the watcher setting the interrupt under that +same lock. After that interrupt is set, no archive move or attempt is admitted. +The lock covers only the check and rename or admission state change; admitted +attempts finish and save outside it. The first SIGINT or SIGTERM records its +number and wakes the watcher through a nonblocking pipe. The handler takes no +lock and raises nothing beyond allocation failure. While running, the watcher +sets the interrupt, releases admission, and then announces that attempts already +admitted will finish and save. A first signal recorded after watcher shutdown +still forces exit 1 without setting the interrupt or printing that announcement. +The guarded cleanup path sets the interrupt and waits for script-owned worker +completion before restoring handlers or releasing the invocation lock. +Guarded waits retry exceptions raised within them and preserve the original +exception. Another handler raising in an unguarded instruction, before a guarded +call, in cleanup bookkeeping or in finally, can replace it and skip waits. +An exception caught in a model thread instead marks that model error without +setting the interrupt. A second signal immediately uses a nonblocking stderr +write and exits, losing every unsaved attempt then under way with its spend; +saved finals, archives and their recorded spend survive. Any signal before +handler restoration requires exit 1 on graceful completion, even if every +worker completed. +""" + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import io +import json +import math +import os +import re +import signal +import stat +import sys +import threading +import time +from pathlib import Path + +# confirmatory_run pins the harness and policybench to this checkout. +from confirmatory_run import ( + CHEAP, + MAX_RERUNS, + PROMPT_VERSION, + action, + attempt_cost, + classify, + completed_turns, + default_completion, + pilot, + sha256_file, + write_json, +) + +NO_TOOLS_CAP = 5.0 +MONEY_TOLERANCE = 1e-6 # dollars, as the report compares spent_before with records +START_GRACE_SECONDS = 10 +NO_TOOLS_WORDING_SHA256 = ( + "43b13e4e64463f4ee1654e9546672899f80ca5e1ce18e93cf2eb04091b3a2699" +) + + +class StateError(ValueError): + """The files cannot describe an intact no-tools attempt history.""" + + +def component(value: str) -> str: + """A record/model name must name one ordinary path component.""" + if ( + not isinstance(value, str) + or value in ("", ".", "..") + or Path(value).name != value + ): + raise StateError(f"invalid path component {value!r}") + return value + + +def read_regular(path: Path) -> bytes: + """Read saved bytes as the report does: open with O_NONBLOCK, refuse a + descriptor that is not a regular file, then read through that descriptor. + A FIFO or device refuses at once, before any read can block under the + invocation lock. It follows file symlinks; a link to nothing refuses.""" + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + kind = { + stat.S_IFIFO: "FIFO", + stat.S_IFDIR: "directory", + stat.S_IFSOCK: "socket", + stat.S_IFCHR: "character device", + stat.S_IFBLK: "block device", + stat.S_IFLNK: "symlink", + }.get(stat.S_IFMT(metadata.st_mode), "unknown file type") + raise ValueError(f"expected a regular file, found {kind}") + os.set_blocking(descriptor, True) + chunks = [] + while chunk := os.read(descriptor, 1024 * 1024): + chunks.append(chunk) + return b"".join(chunks) + finally: + os.close(descriptor) + + +def read_record(path: Path) -> tuple[dict, str]: + """Decode a saved record and hash the same bytes, so its link names them.""" + try: + data = read_regular(path) + record = json.loads(data.decode("utf-8")) + if not isinstance(record, dict): + raise ValueError("record is not a JSON object") + if "prior_attempts" in record: + raise ValueError("native prior_attempts cannot preserve full attempts") + if not isinstance(record.get("attempts"), list): + raise ValueError("record lacks an attempts list") + totals = record.get("totals") + if not isinstance(totals, dict): + raise ValueError("record totals must be an object with cost_usd") + raw_cost = totals.get("cost_usd") + if isinstance(raw_cost, bool) or not isinstance(raw_cost, (int, float)): + raise ValueError("record cost_usd must be a present number") + cost = float(raw_cost) + if not math.isfinite(cost) or cost < 0: + raise ValueError("attempt cost must be finite and nonnegative") + return record, hashlib.sha256(data).hexdigest() + except ( + OSError, + ValueError, + TypeError, + AttributeError, + RecursionError, + OverflowError, + ) as error: + raise StateError(f"{path}: {error}") from error + + +BOARD_INPUT_ERRORS = ( + OSError, + ValueError, + TypeError, + AttributeError, + KeyError, + RecursionError, + OverflowError, + SystemExit, +) + + +def _board_input(path: Path | str, read): + """Keep a delegated board reader's refusal attached to its input file.""" + try: + return read() + except BOARD_INPUT_ERRORS as error: + raise StateError(f"{path}: {error}") from error + + +def saved_stat(path: Path): + """Only a missing path means absent evidence; other stat failures refuse.""" + + def read(): + try: + return path.stat() + except FileNotFoundError: + return None + + return _board_input(path, read) + + +def saved_exists(path: Path) -> bool: + return saved_stat(path) is not None + + +def saved_sha256(path: Path) -> str: + """Hash a saved file through the same guard, naming it on failure.""" + return _board_input(path, lambda: hashlib.sha256(read_regular(path)).hexdigest()) + + +def listed_is_dir(path: Path) -> bool: + """A listing returned this entry, so every stat failure refuses, even ENOENT + (a dangling link, or an entry removed since the listing).""" + return stat.S_ISDIR(_board_input(path, path.stat).st_mode) + + +def saved_children(folder: Path) -> list[Path]: + """Enumerate explicitly so a failed scan cannot silently erase evidence.""" + metadata = saved_stat(folder) + if metadata is None: + return [] + if not stat.S_ISDIR(metadata.st_mode): + raise StateError(f"{folder}: evidence folder is not a directory") + + def read(): + try: + with os.scandir(folder) as entries: + return [folder / entry.name for entry in entries] + except FileNotFoundError: + return [] + + return _board_input(folder, read) + + +def saved_json_paths(folder: Path) -> list[Path]: + return sorted( + path for path in saved_children(folder) if path.name.endswith(".json") + ) + + +def load_board(path: Path, engine: str) -> dict: + """Check board inputs locally, then use the harness's unchanged loader.""" + from policybench.reference_exclusions import ( + load_reference_exclusions, + split_reference, + ) + from policybench.scenarios import scenario_from_dict + from policybench.spec import expand_programs_for_scenario + + checked = {} + + def checked_bytes(source): + data = source.read_bytes() + checked[source] = hashlib.sha256(data).hexdigest() + return data + + def object_json(text): + payload = json.loads(text) + if not isinstance(payload, dict): + raise ValueError("board input is not a JSON object") + return payload + + def csv_frame(source): + # pandas also reads the original path in load_board. Check UTF-8 here + # so every decode/parser failure has its concrete source filename. + with io.TextIOWrapper( + io.BytesIO(checked_bytes(source)), encoding="utf-8" + ) as stream: + return pilot.pd.read_csv(stream) + + metadata = path / "reference_outputs.csv.meta.json" + meta = _board_input( + metadata, lambda: object_json(checked_bytes(metadata).decode("utf-8")) + ) + + def require_engine(): + if f"policyengine-us {pilot.board_engine(path)}" != engine: + raise StateError("the board's engine differs from the design") + pilot._require_engine(path) + + _board_input(metadata, require_engine) + reference_path = path / "reference_outputs.csv" + reference = _board_input(reference_path, lambda: csv_frame(reference_path)) + exclusions_path = path / "reference_exclusions.json" + if saved_exists(exclusions_path): + _board_input( + exclusions_path, + lambda: object_json(checked_bytes(exclusions_path).decode("utf-8")), + ) + else: + checked[exclusions_path] = None + exclusions = _board_input(exclusions_path, lambda: load_reference_exclusions(path)) + _board_input( + f"{reference_path} and {exclusions_path}", + lambda: split_reference(reference, exclusions), + ) + scenarios_path = path / "scenarios.csv" + scenarios = _board_input(scenarios_path, lambda: csv_frame(scenarios_path)) + for _, row in scenarios.iterrows(): + scenario = _board_input( + scenarios_path, + lambda: scenario_from_dict(object_json(row["scenario_json"])), + ) + _board_input( + metadata, lambda: expand_programs_for_scenario(meta["programs"], scenario) + ) + try: + return pilot.load_board(path) + except BOARD_INPUT_ERRORS as error: + if isinstance(error, OSError): + raise StateError(f"{error.filename or path}: {error}") from error + changed, unreadable = [], [] + for source, digest in checked.items(): + try: + current = sha256_file(source) + except FileNotFoundError: + current = None + except BOARD_INPUT_ERRORS as reread_error: + unreadable.append(f"{source}: cannot recheck: {reread_error}") + continue + if current != digest: + changed.append(str(source)) + at = " and ".join(changed) if changed else str(path) + detail = "; changed after it was checked" if changed else "" + if unreadable: + detail += "; " + "; ".join(unreadable) + raise StateError(f"{at}{detail}: {error}") from error + + +def _no_tools_answered(record: dict) -> bool: + predictions = record.get("predictions") + return isinstance(predictions, dict) and any( + value is not None for value in predictions.values() + ) + + +def _saved_treatment_problems(record: dict, model: str, board: dict) -> list[str]: + """Report treatment_problems specialized to the fixed none arm. + + Keep the report's conditional evidence requirements and conversions. + In particular, empty failed attempts may omit prompt evidence, variables + use the report's list conversion, and unexamined fields remain unexamined. + """ + problems = [] + if record.get("model") != model: + return [f"record names model {record.get('model')!r}"] + if record.get("arm") != "none": + return [f"arm {record.get('arm')!r} differs from 'none'"] + + profile = pilot.model_profile(model, arm="none", prompt_version=PROMPT_VERSION) + treatment = pilot.profile_treatment(profile) + if treatment["tool_features"] is not None: + raise ValueError( + f"none: the harness gives features {treatment['tool_features']}" + ) + + answered = _no_tools_answered(record) + owed = ( + isinstance(record.get("turns"), list) and bool(completed_turns(record)) + ) or answered + keys = ( + "prompt_version", + "prompt_wording_sha256", + "tool_notice_sha256", + "tool_features", + "calculate_tool_sha256", + "variables", + "turns", + ) + if owed: + keys += ( + "contract", + "messages", + "board_prompt_sha256", + "sent_board_prompt_sha256", + ) + for key in keys: + if key not in record: + problems.append(f"missing no-tools field {key!r}") + if "predictions" in record and not isinstance(record["predictions"], dict): + problems.append("no-tools predictions must be an object from run_household") + + contract = record.get("contract") + if (owed or "contract" in record) and contract != profile.contract: + problems.append( + f"answer contract {contract!r} differs from {profile.contract!r}" + ) + if record.get("prompt_version") != PROMPT_VERSION: + problems.append(f"prompt version {record.get('prompt_version')!r}") + wording = pilot.prompt_wording_sha256(profile) + if ( + wording != NO_TOOLS_WORDING_SHA256 + or record.get("prompt_wording_sha256") != NO_TOOLS_WORDING_SHA256 + ): + problems.append( + f"prompt wording {str(record.get('prompt_wording_sha256'))[:12]}, " + f"harness {str(wording)[:12]}" + ) + notice = hashlib.sha256(profile.notice.encode()).hexdigest() + if record.get("tool_notice_sha256") != notice: + problems.append("the notice hash differs from the arm's") + for key in ("tool_features", "calculate_tool_sha256"): + if record.get(key) != treatment[key]: + problems.append(f"{key} differs from the none arm's") + + sid = record.get("scenario_id") + if sid not in board["scenarios"]: + return problems + ["no such household on the board"] + if list(record.get("variables") or []) != list(board["variables"][sid]): + problems.append("its outputs differ from the board's") + if not isinstance(record.get("turns"), list): + return problems + ["no-tools turns must be an array from run_household"] + if answered and not completed_turns(record): + problems.append("a no-tools answer has no completed turn") + for number, turn in enumerate(record.get("turns") or [], start=1): + served = turn.get("provider_resolved_model") + if not served: + problems.append(f"turn {number} reports no provider's model") + elif served != profile.model_id.split("/")[-1]: + problems.append( + f"turn {number}: providers reported {served!r}, " + f"expected {profile.model_id}" + ) + + messages = record.get("messages") + if (owed or "messages" in record) and not isinstance(messages, list): + return problems + [ + "no-tools messages must be an array from conversation.transcript" + ] + if not messages: + if owed: + problems.append("no no-tools sent prompt in messages") + if completed_turns(record): + problems.append("no sent prompt in messages") + if owed: + return problems + + built, settings = pilot.build_initial_request( + board["scenarios"][sid], board["variables"][sid], profile + ) + if messages: + first = messages[0] if isinstance(messages[0], dict) else {} + text = ( + first.get("input") + if isinstance(first.get("input"), str) + else first.get("content") + ) + if text != built: + problems.append("the sent prompt differs from the harness's rebuilt prompt") + for key, description in ( + ("sent_board_prompt_sha256", "sent board prompt"), + ("board_prompt_sha256", "board prompt"), + ): + if (messages or key in record) and record.get(key) != settings[f"_{key}"]: + problems.append(f"the {description} hash differs from the rebuilt one") + return problems + + +def validate_treatment(path, record, model, board, engine): + """Check each saved record against the report's pinned none-arm treatment.""" + try: + with pilot.stating_engine(engine.split()[-1]): + problems = _saved_treatment_problems( + record, model, board() if callable(board) else board + ) + except BOARD_INPUT_ERRORS as error: + raise StateError( + f"{path}: cannot check saved no-tools treatment: {error}" + ) from error + if problems: + raise StateError(f"{path}: " + "; ".join(problems)) + + +def entry(path: Path, record: dict, n: int, sha256: str) -> dict: + """Link an archive by the digest of the bytes its record came from.""" + cls, reason = classify(record, 0.0, 0.0, None) + return { + "n": n, + "file": f"attempts/{path.name}", + "sha256": sha256, + "status": record.get("status"), + "class": cls, + "reason": reason, + } + + +def _attempts_agree(listed: list, entries: list) -> bool: + """Match known link fields and retain each archive's exact earlier prefix.""" + if len(listed) != len(entries) or listed[:-1] != entries[:-1]: + return False + return not entries or ( + isinstance(listed[-1], dict) + and all(listed[-1].get(key) == value for key, value in entries[-1].items()) + ) + + +def history( + folder: Path, validate=None, drawn=None +) -> tuple[dict[str, dict], list[float]]: + """Validate a model's histories, including an interrupted archive, and list + each saved cost.""" + # Finals first, as the report lists them: an unlistable households folder + # is named before the attempts folder inside it. + finals = {path.stem: path for path in saved_json_paths(folder)} + archives: dict[str, list[tuple[int, Path]]] = {} + for path in saved_json_paths(folder / "attempts"): + match = re.fullmatch(r"(.+)-([1-9][0-9]*)\.json", path.name) + if not match: + raise StateError(f"{path}: invalid attempt number") + archives.setdefault(match[1], []).append((int(match[2]), path)) + states, costs = {}, [] + for sid in sorted(set(finals) | set(archives)): + paths = sorted(archives.get(sid, [])) + numbers = [n for n, _ in paths] + if numbers != list(range(1, len(paths) + 1)): + raise StateError( + f"{folder}: attempt-number gap for {sid}: " + + ", ".join(path.name for _, path in paths) + ) + if len(paths) > MAX_RERUNS: + raise StateError(f"{folder}: {sid} exceeds {MAX_RERUNS} reruns") + entries = [] + for n, path in paths: + record, digest = read_record(path) + if record.get("scenario_id") != sid: + raise StateError(f"{path}: household differs from archive name {sid}") + if validate is not None: + validate(path, record) + if not _attempts_agree(record["attempts"], entries): + raise StateError(f"{path}: attempts disagree with earlier archives") + try: + cls, _ = classify(record, 0.0, 0.0, None) + except BOARD_INPUT_ERRORS as error: + raise StateError( + f"{path}: cannot classify saved archive: {error}" + ) from error + if action(cls, n) not in ("rerun", "pause-rerun"): + raise StateError(f"{path}: archived attempt was ineligible for rerun") + entries = record["attempts"] + [entry(path, record, n, digest)] + costs.append(attempt_cost(record)) + final = final_sha256 = None + if sid in finals: + path = finals[sid] + final, final_sha256 = read_record(path) + if drawn is not None: + if sid not in drawn: + raise StateError(f"{path}: cheap-model final is outside the draw") + if not isinstance(final.get("predictions"), dict): + raise StateError( + f"{path}: final no-tools predictions must be an object" + ) + if final.get("scenario_id") != sid: + raise StateError(f"{path}: household differs from final name {sid}") + for n, link in enumerate(final["attempts"], 1): + if ( + not isinstance(link, dict) + or type(link.get("n")) is not int + or link["n"] != n + ): + raise StateError(f"{path}: attempt link n must be integer {n}") + if validate is not None: + validate(path, final) + if not _attempts_agree(final["attempts"], entries): + raise StateError(f"{path}: attempts disagree with archived files") + try: + classify(final, 0.0, 0.0, None) + except BOARD_INPUT_ERRORS as error: + raise StateError( + f"{path}: cannot classify saved final: {error}" + ) from error + entries = final["attempts"] + costs.append(attempt_cost(final)) + states[sid] = {"entries": entries, "final": final, "final_sha256": final_sha256} + return states, costs + + +def exact_spend(costs: list[float]) -> float: + """The correctly rounded sum of a flat list of saved costs, the same in any + order. For histories that pass validation this reaches the report's total. + An OverflowError gives infinity, which every finite `spent_before` falls + short of.""" + try: + return math.fsum(costs) + except OverflowError: + return math.inf + + +def check_baseline(design: dict) -> None: + """Refuse a baseline inventory that fails its hash or schema, from one read.""" + baseline_path = None + try: + baseline = design["baseline_exclusions"] + if not isinstance(baseline, dict) or not isinstance(baseline.get("path"), str): + raise StateError("baseline exclusions need path and sha256") + baseline_path = Path(baseline["path"]) + baseline_bytes = baseline_path.read_bytes() + if hashlib.sha256(baseline_bytes).hexdigest() != baseline.get("sha256"): + raise StateError("baseline exclusions do not hash as declared") + payload = json.loads(baseline_bytes.decode("utf-8")) + exclusions = payload.get("exclusions") if isinstance(payload, dict) else None + if not isinstance(exclusions, list): + raise StateError("baseline exclusions must contain an exclusions list") + cells = [] + for exclusion in exclusions: + if not isinstance(exclusion, dict) or any( + not isinstance(exclusion.get(key), str) or not exclusion[key] + for key in ("scenario_id", "variable") + ): + raise StateError( + "baseline exclusion needs household and output strings" + ) + cells.append((exclusion["scenario_id"], exclusion["variable"])) + if len(cells) != len(set(cells)): + raise StateError("baseline exclusions repeat a cell") + except (OSError, KeyError, TypeError, ValueError, RecursionError) as error: + at = "baseline exclusions" + if baseline_path is not None: + at += f" ({baseline_path})" + raise StateError(f"{at}: {error}") from error + + +def stage_problems(design: dict) -> list[str]: + """Check the baseline, saved histories, treatment, standing and spend + without a provider. + + As the report does, refuse a baseline that fails its hash or schema, any + model's archive that no final lists, any final that owes a rerun, optional + models included, and a spent_before below the saved records' exact total; + each drawn cheap-model household also needs a final that stands. Startup + recovery still accepts both unfinished states.""" + problems = [] + try: + check_baseline(design) + except StateError as error: + problems.append(str(error)) + try: + spec = design["no_tools"] + root = Path(spec["runs"]) + folder, stage = component(spec["folder"]), component(spec["stage"]) + draw_order = [component(sid) for sid in design["draw_order"]] + models = set(CHEAP) + models.update(path.name for path in saved_children(root) if listed_is_dir(path)) + except (OSError, KeyError, TypeError, ValueError) as error: + return [*problems, f"no-tools stage: {error}"] + board = None + costs = [] + + def checked_board(): + nonlocal board + if board is None: + board = load_board(Path(design["board"]), design["engine"]) + return board + + for model in sorted(models): + try: + states, saved = history( + root / model / folder / stage / "households", + validate=lambda path, record: validate_treatment( + path, record, model, checked_board, design["engine"] + ), + drawn=set(draw_order) if model in CHEAP else None, + ) + costs += saved + drawn = draw_order if model in CHEAP else [] + for sid in drawn + sorted(set(states) - set(drawn)): + state = states.get(sid) + if state is None or state["final"] is None: + problems.append( + f"no-tools {model}/{sid}: missing standing final" + if sid in drawn + else f"no-tools {model}/{sid}: archived attempts lack a final" + ) + continue + outcome = action( + classify(state["final"], 0.0, 0.0, None)[0], + len(state["entries"]) + 1, + ) + if outcome in ("rerun", "pause-rerun") or ( + sid in drawn and outcome != "stands" + ): + problems.append( + f"no-tools {model}/{sid}: final action is {outcome}" + ) + except (OSError, TypeError, ValueError, AttributeError, KeyError) as error: + problems.append(f"no-tools {model}: {error}") + spent = exact_spend(costs) + try: + declared = float(design.get("spent_before")) + except (TypeError, ValueError, OverflowError): + declared = math.nan + if not 0 <= declared < math.inf: + problems.append( + "no-tools spend: spent_before must be a finite, nonnegative number, " + f"not {design.get('spent_before')!r}" + ) + elif declared < spent - MONEY_TOLERANCE: + problems.append( + f"no-tools spend: spent_before ${declared:.4f} is less than " + f"the saved records' ${spent:.4f}" + ) + return problems + + +class Spend: + """The stage's recorded spend, shared by every model and every invocation.""" + + def __init__(self, spent: float, stopped: threading.Event): + self.spent_usd, self.budget_usd = spent, NO_TOOLS_CAP + self.lock, self.stopped = threading.Lock(), stopped + if spent >= self.budget_usd: + stopped.set() + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + with self.lock: + return ( + self.stopped.is_set() or self.spent_usd + reserve_usd >= self.budget_usd + ) + + +class AttemptSpend: + """Count a request in both the stage and its own exception fallback record.""" + + def __init__(self, stage: Spend): + self.stage, self.spent_usd = stage, 0.0 + + def add(self, usd: float | None) -> None: + cost = float(usd or 0.0) + with self.stage.lock: + self.spent_usd += cost + self.stage.spent_usd += cost + if self.stage.spent_usd >= self.stage.budget_usd: + self.stage.stopped.set() + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + return self.stage.exhausted(reserve_usd) + + +class Stage: + """One stage, with injectable providers/board for a provider-free rehearsal.""" + + def __init__( + self, + design: dict, + models=None, + run_household=None, + completion_for=default_completion, + board=None, + out=None, + ): + self.design = design + self.models = tuple(dict.fromkeys(models if models is not None else CHEAP)) + if not self.models: + raise StateError("at least one model is required") + for model in self.models: + component(model) + self.draw_order = [component(sid) for sid in design["draw_order"]] + if len(self.draw_order) != len(set(self.draw_order)): + raise StateError("draw_order repeats a household") + spec = design["no_tools"] + self.root = Path(spec["runs"]) + self.folder, self.stage = component(spec["folder"]), component(spec["stage"]) + self.out = sys.stdout if out is None else out + self._say_lock = threading.RLock() + self.stopped = threading.Event() + self.interrupted = threading.Event() + self._admission_lock = threading.Lock() + self._admitted_attempts = {} + self._threads = [] + self.completion_for = completion_for + self.run_household = ( + run_household if run_household is not None else pilot.run_household + ) + self._lock_file = None + check_baseline(design) + self.root.mkdir(parents=True, exist_ok=True) + self._lock_file = (self.root / ".confirmatory-no-tools.lock").open("a") + try: + fcntl.flock(self._lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + self.close() + raise StateError( + f"another no-tools invocation holds the lock in {self.root}" + ) from error + try: + self._load_state(board) + except BaseException: + self.close() + raise + + def _load_state(self, board) -> None: + design = self.design + pilot.set_stated_engine(design["engine"].split()[-1]) + if board is None: + board = load_board(Path(design["board"]), design["engine"]) + self.board = board + missing = set(self.draw_order) - set(board["scenarios"]) + missing |= set(self.draw_order) - set(board["variables"]) + if missing: + raise StateError( + f"drawn households missing from the board: {sorted(missing)}" + ) + self.histories = {} + costs = [] + folders = {p.name for p in saved_children(self.root) if listed_is_dir(p)} + for model in sorted(folders | set(self.models)): + states, saved = history( + self.households(model), + validate=lambda path, record: validate_treatment( + path, record, model, self.board, design["engine"] + ), + drawn=set(self.draw_order) if model in CHEAP else None, + ) + self.histories[model] = states + costs += saved + # A saved ceiling failure is a stage-wide stop on every invocation. + if any( + state["final"] is not None + and action( + classify(state["final"], 0.0, 0.0, None)[0], + len(state["entries"]) + 1, + ) + == "stopped" + for state in states.values() + ): + self.stopped.set() + self.spend = Spend(exact_spend(costs), self.stopped) + self.states = {model: "pending" for model in self.models} + + def close(self) -> None: + """Release the invocation lock, including after initialization failure.""" + lock = getattr(self, "_lock_file", None) + if lock is not None: + lock.close() + self._lock_file = None + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __del__(self): + self.close() + + def households(self, model: str) -> Path: + return self.root / model / self.folder / self.stage / "households" + + def say(self, text: str) -> None: + with self._say_lock: + print(text, file=self.out, flush=True) + + def standing(self, model: str, sid: str) -> bool: + state = self.histories[model].get(sid) + return bool( + state + and state["final"] is not None + and action( + classify(state["final"], 0.0, 0.0, None)[0], len(state["entries"]) + 1 + ) + == "stands" + ) + + def dry_run(self) -> int: + try: + return self._dry_run() + finally: + self.close() + + def _dry_run(self) -> int: + for model in self.models: + done = [sid for sid in self.draw_order if self.standing(model, sid)] + owed = [] + for sid in self.draw_order: + if sid in done: + continue + state = self.histories[model].get(sid, {"entries": [], "final": None}) + n = len(state["entries"]) + 1 + if state["final"] is not None: + outcome = action(classify(state["final"], 0.0, 0.0, None)[0], n) + if outcome in ("stop", "stopped"): + owed.append(f"{sid}:blocked {outcome} at attempt {n}") + continue + n += 1 + owed.append(f"{sid}:{n}") + self.say(f"{model}: done {done}; owed (household:next attempt) {owed}") + self.say(f"stage spend ${self.spend.spent_usd:.6f} of ${NO_TOOLS_CAP:.2f}") + return ( + 0 + if all( + self.standing(model, sid) + for model in self.models + for sid in self.draw_order + ) + else 1 + ) + + def archive(self, model: str, sid: str) -> bool: + """Atomically admit a prepared archive move, then extend its links. + + The target-exists and digest checks precede folder creation; either + refusal leaves the disk history unchanged. After the rename it hashes + the moved bytes again; if a write landed between the first hash and + the rename, it moves them back and refuses. It cannot see a write to + the same inode during either hash or after the second. Later readers + refuse an archive that differs from a surviving link; a saved record + holds that link only once the next attempt's final lands, so startup + rebuilds the last archive's digest from its bytes after an interrupt.""" + state = self.histories[model][sid] + record, entries = state["final"], state["entries"] + n = len(entries) + 1 + cls, _ = classify(record, 0.0, 0.0, None) + if action(cls, n) not in ("rerun", "pause-rerun"): + raise StateError(f"{model}/{sid}: attempt {n} is ineligible for rerun") + folder = self.households(model) + source = folder / f"{sid}.json" + target = folder / "attempts" / f"{sid}-{n}.json" + if saved_exists(target): + raise StateError(f"{target}: would overwrite an archived attempt") + if saved_sha256(source) != state["final_sha256"]: + raise StateError(f"{source}: final changed after it was read or written") + target.parent.mkdir(parents=True, exist_ok=True) + link = entry(source, record, n, state["final_sha256"]) + link["file"] = f"attempts/{target.name}" + with self._admission_lock: + if self.interrupted.is_set(): + return False + os.replace(source, target) + state["final"] = None + if saved_sha256(target) != state["final_sha256"]: + os.replace(target, source) + state["final"] = record + raise StateError( + f"{source}: final changed during its archive move; " + f"the runner moved it back from {target}" + ) + entries.append(link) + return True + + def run_model(self, model: str) -> None: + try: + profile = pilot.model_profile( + model, arm="none", prompt_version=PROMPT_VERSION + ) + if profile.uses_calculator or profile.uses_web: + raise StateError(f"{model}: the none profile offers a tool") + completion = None + completion_loaded = False + for sid in self.draw_order: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return + state = self.histories[model].setdefault( + sid, {"entries": [], "final": None, "final_sha256": None} + ) + while True: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return + if state["final"] is not None: + cls, _ = classify(state["final"], 0.0, 0.0, None) + outcome = action(cls, len(state["entries"]) + 1) + if outcome == "stands": + break + if outcome == "stop": + self.states[model] = "harness-stopped" + return + if outcome == "stopped": + self.stopped.set() + if self.spend.exhausted(): + self.states[model] = "budget-stopped" + return + if state["final"] is not None: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return + if not self.archive(model, sid): + self.states[model] = "interrupted" + return + if not completion_loaded: + completion = self.completion_for(profile) + completion_loaded = True + if self.spend.exhausted(): + self.states[model] = "budget-stopped" + return + spend = AttemptSpend(self.spend) + with self._admission_lock: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return + self._admitted_attempts[model, sid] = len(state["entries"]) + 1 + + def no_tool(arguments): + raise RuntimeError("the no-tools arm cannot call a tool") + + try: + record = self.run_household( + self.board["scenarios"][sid], + self.board["variables"][sid], + no_tool, + completion, + spend, + profile=profile, + ) + except Exception as error: + record = pilot.harness_error_record( + sid, self.board["variables"][sid], profile, error + ) + record["totals"]["cost_usd"] = spend.spent_usd + record["attempts"] = list(state["entries"]) + digest = write_json(self.households(model) / f"{sid}.json", record) + state["final"], state["final_sha256"] = record, digest + n = len(state["entries"]) + 1 + cls, _ = classify(record, 0.0, 0.0, None) + outcome = action(cls, n) + self.say( + f"{model} {sid} {n} {cls} {outcome} ${attempt_cost(record):.6f}" + ) + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return + if outcome == "pause-rerun": + self.states[model] = "operator-paused" + return + if outcome == "stop": + self.states[model] = "harness-stopped" + return + if outcome == "stopped": + self.stopped.set() + self.states[model] = "budget-stopped" + return + if outcome == "stands": + break + self.states[model] = "complete" + except Exception as error: + self.states[model] = "error" + self.say(f"{model}: {type(error).__name__}: {error}") + + def execute(self) -> int: + previous = {} + signals = [] + closing = False + watcher = None + watcher_done = threading.Event() + watcher_began = threading.Event() + watcher_started = False + failure = None + read_fd = write_fd = None + self._started_workers = [] + self._starting_worker = None + self._worker_began = {} + self._worker_done = {} + + def wake_watcher(): + try: + os.write(write_fd, b"!") + except OSError: + pass + + def interrupt(signum, frame): + signals.append(signum) + if len(signals) >= 2: + try: + os.set_blocking(2, False) + os.write( + 2, + b"Forced interruption: every unsaved attempt then under way " + b"is lost, with its spend; saved finals, archives and the " + b"spend they record survive.\n", + ) + except OSError: + pass + os._exit(128 + signum) + wake_watcher() + + def stop_admission(): + while True: + try: + with self._admission_lock: + self.interrupted.set() + return + except BaseException: + continue + + def watch_signals(): + watcher_began.set() + notified = False + try: + if closing: + return + while True: + os.read(read_fd, 1) + if closing: + return + if signals and not notified: + with self._admission_lock: + self.interrupted.set() + notified = True + self.say( + "Interrupted: attempts already admitted " + "will finish and save." + ) + finally: + watcher_done.set() + + try: + read_fd, write_fd = os.pipe() + os.set_blocking(write_fd, False) + if threading.current_thread() is threading.main_thread(): + for sig in (signal.SIGINT, signal.SIGTERM): + previous[sig] = signal.signal(sig, interrupt) + watcher = threading.Thread( + target=watch_signals, name="no-tools-signal-watcher" + ) + watcher.start() + watcher_started = True + result = self._execute() + except BaseException as error: + failure = error + # SIGINT/SIGTERM handlers raise nothing beyond allocation failure. + # Guarded waits retry exceptions raised within them, including those + # from another handler (for example SIGUSR1 raising KeyboardInterrupt). + # Another handler raising in an unguarded instruction, before a + # guarded call, in cleanup bookkeeping or in finally, can replace + # the original exception and skip waits. + stop_admission() + self._wait_for_workers() + raise + finally: + closing = True + if write_fd is not None: + wake_watcher() + shutdown_error = None + if watcher is not None: + # start() can raise after native startup, even before ident is set. + # A watcher beginning after grace sees closing before reading a pipe. + deadline = time.monotonic() + START_GRACE_SECONDS + while not watcher_started and not watcher_began.is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + watcher_began.wait(remaining) + except BaseException as error: + if shutdown_error is None: + shutdown_error = error + if watcher_started or watcher_began.is_set(): + while True: + try: + watcher_done.wait() + watcher.join() + break + except BaseException as error: + if shutdown_error is None: + shutdown_error = error + if failure is None and shutdown_error is not None: + stop_admission() + # Restore the handlers first: until then a signal still writes to + # write_fd, whose number a closed pipe could hand to another file. + for sig, handler in previous.items(): + signal.signal(sig, handler) + for fd in (read_fd, write_fd): + if fd is not None: + os.close(fd) + self.close() + if failure is None and shutdown_error is not None: + raise shutdown_error + return 1 if signals else result + + def _wait_for_workers(self) -> None: + """Retry exceptions raised within guarded worker-completion waits. + + The installed SIGINT and SIGTERM handlers raise nothing beyond allocation + failure. Guarded waits retry exceptions from elsewhere, such as a test-only + SIGUSR1 handler raising KeyboardInterrupt, preserving the original exception. + Another handler raising in unguarded instructions before a guarded call, + in cleanup bookkeeping or in finally can replace it and skip waits. + """ + workers = list(self._started_workers) + starting = self._starting_worker + if starting is not None and starting not in workers: + model = self.models[self._threads.index(starting)] + deadline = time.monotonic() + START_GRACE_SECONDS + while not self._worker_began[model].is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + self._worker_began[model].wait(remaining) + except BaseException: + continue + if self._worker_began[model].is_set(): + workers.append(starting) + for thread in workers: + model = self.models[self._threads.index(thread)] + while True: + try: + self._worker_done[model].wait() + thread.join() + break + except BaseException: + continue + + def _run_worker(self, model: str) -> None: + self._worker_began[model].set() + try: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return + self.run_model(model) + finally: + self._worker_done[model].set() + + def _execute(self) -> int: + self._worker_began = {model: threading.Event() for model in self.models} + self._worker_done = {model: threading.Event() for model in self.models} + self._threads = [ + threading.Thread( + target=self._run_worker, args=(model,), name=f"no-tools-{model}" + ) + for model in self.models + ] + for thread in self._threads: + self._starting_worker = thread + thread.start() + self._started_workers.append(thread) + self._starting_worker = None + for model, thread in zip(self.models, self._threads): + self._worker_done[model].wait() + thread.join() + for model in self.models: + self.say(f"{model}: {self.states[model]}") + self.say(f"stage spend ${self.spend.spent_usd:.6f} of ${NO_TOOLS_CAP:.2f}") + return ( + 0 + if not self.interrupted.is_set() + and all(self.states[model] == "complete" for model in self.models) + else 1 + ) + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run") + run.add_argument("design", type=Path) + run.add_argument("--model", action="extend", nargs="+") + run.add_argument("--dry-run", action="store_true") + args = parser.parse_args(argv) + try: + try: + design = json.loads(args.design.read_text()) + if not isinstance(design, dict): + raise ValueError("design is not a JSON object") + except (OSError, ValueError, RecursionError) as error: + raise StateError(f"{args.design}: {error}") from error + with Stage(design, models=args.model) as stage: + return stage.dry_run() if args.dry_run else stage.execute() + except (StateError, OSError, ValueError, KeyError, TypeError) as error: + print(f"The no-tools stage cannot start: {error}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py new file mode 100644 index 00000000..7a9a5a0b --- /dev/null +++ b/scripts/confirmatory_run.py @@ -0,0 +1,2397 @@ +"""The confirmatory calculator-interface run. + +Usage: + python scripts/confirmatory_run.py run DESIGN [--workers N] [--dry-run] + python scripts/confirmatory_run.py resume DESIGN MODEL [--note TEXT] + +Runs the frozen pre-registration's "Run procedure" and writes its "Run +records", which the confirmatory report checks before it scores anything. +DESIGN is the JSON file written at the freeze, the file the report reads. + +- **Lanes.** Each model has two lanes. Households alternate between them in + draw order, the first drawn to lane 1. A lane runs one household's arms one + after another, in the order drawn for that household and model (the stream + ``order/``), then its reruns in passes (second attempts before + third, each pass in the household's order), and only then decides its next + household. The six models run at once. +- **One engine pool per model**, started once, with no features of its own. + Each call carries its arm's features in a call frame (``tool_call_frame``), + the message a pool started with those features would send. ``setup-start`` + and ``setup-end`` events bracket startup and pool restarts, so setup time + can be distinguished from an idle lane. +- **Admission.** A lane admits its next household when the spend of the + model's finished households (every attempt, replaced ones included) plus + the reserve for each household under way, this one included, is at most + the model's cap plus $1e-9. A household is finished when its last attempt + has ended. After one refusal every later decision is a refusal. A decision + waits while an operator pause holds, and while any of the model's + households has ended a pass whose reruns are undecided, so that whether it + has finished is known. +- **Spend.** No conversation is stopped for spend, except at the hard + ceiling: the overall cap plus $25, over the spend before the calculator + stage and every model's spend since. At the crossing, no attempt starts, + the conversations under way end ``budget_stop`` at their next turn, and + every model's run log gets a ``ceiling`` event at the crossing's time. + The decision to admit a household and the crossing share one lock, so no + household is decided after the crossing. +- **The attempt deadline.** A conversation still running an hour after it + started is abandoned: its attempt gets a harness-error record + (``AttemptDeadline``) holding the spend so far, the model's lanes stop, + and the conversation ends ``budget_stop`` at its next turn. Its spend after + the deadline still counts toward the ceiling, and goes to the run log as + ``late-cost`` events. +- **Failures.** Each attempt's class follows the pre-registration's table, in + the precedence the report's ``classify`` uses, from its record and the host + evidence for its start and end; its action follows from its class and its + attempt number. An operator failure pauses the model's lanes until + ``resume``, which is accepted only if written after the pause began. A + harness failure stops them, and so does one inconclusive error that ends + runs in more than five of the model's households. A replaced attempt is + saved, before its rerun, as ``households/attempts/-.json``. +- **Stops.** Every ``stop`` event names its cause: ``harness``, + ``inconclusive``, ``deadline``, ``pool`` (the engine pool could not be + started again), ``runner-error`` (an exception in the runner itself, which + stops every model's lanes after writing what had ended), ``interrupt`` or + ``sampler``. The ceiling's events are ``ceiling`` events of cause + ``ceiling``. +- **The host log.** Every 30 seconds: the time, the monotonic clock, the + load average and the kernel's memory-pressure level (None when it could not + be read, which is no sample). At the end, the guard's and the power log's + lines from a day before the run's start to its end. A read of those logs is + tried five times; one that still fails during the run is a runner error, + and at the end leaves ``host-error.txt`` in place of the excerpts. An + attempt's class waits for its host evidence to + be complete: a good sample more than 60 seconds after the pass's last end, + or no good sample for more than 90 seconds. A sampler whose samples have + failed for five minutes, or whose thread has ended, stops every model. +- **Chains and anchors.** Every line of the run logs, the admission logs and + the host log carries ``prev``, the sha256 of the line before it. The first + line's is a seed: for a model's logs, the sha256 of the frozen hash, the + model and the file's name; for the host log, of its name. Every 15 + minutes, and at the end, ``anchors.jsonl`` gets a line with every log's + line count and head and every manifest's sha256, itself chained, and the + design's ``anchor_command`` is run on a synced, atomic copy in the + background. Each ``anchors..push-snapshot.jsonl`` is a copy, not a + record, and is never replaced by a later push's snapshot. Each push's + result goes to ``anchor-pushes.jsonl``. ``confirmatory_anchor.py`` is that + command: under an exclusive, non-blocking clone lock, it accepts only a + snapshot extending the clone's anchors, commits it and pushes it, so the + remote holds each anchor's time from outside the run. Retiring a worker + does not ensure its command has ended. If an old helper or its Git child + still holds the lock, later pushes fail as busy; a final segment left local + is a deviation. + Final pushes get up to three tries per set of heads, waiting a minute + after each failed try ends before starting the next. + +The records' format is the confirmatory report's (its module docstring). The +report recomputes every class, action, order and admission decision from the +records, with its own copy of these rules; the study's differential test runs +this runner on fakes and checks that its records pass the report. + +The runner runs the treatment harness's code. It refuses to start unless its +``tool_pilot_argon.py``, ``policybench`` package and fix module are the +treatment commit's (6b00a6f0), its tree is clean at the design's run commit, +the machine is on AC power with the lid open, and the runs folder holds no +records: a run is never resumed. ``--workers`` must equal the design's +engine workers per model. An interrupt (SIGINT or SIGTERM) only notes the +signal; the main loop then stops every model's lanes with cause +``interrupt`` and lets the conversations under way end. A second interrupt +exits at once until cleanup ignores signals before its final check. Signals +during any re-anchoring after that check are ignored, including a second +interrupt. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import shutil +import signal +import subprocess +import sys +import threading +import time +import traceback +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Callable +from zoneinfo import ZoneInfo + +import numpy as np + +# The harness and policybench come from this checkout, whatever else is on the +# path: the preflight refuses any other. +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT), str(ROOT / "scripts")] + +import tool_pilot_argon as pilot # noqa: E402 + +TREATMENT_COMMIT = "6b00a6f0c4997e73b9968b412b57646957cad631" +# The treatment's code, which must be the treatment commit's. +TREATMENT_PATHS = ( + "scripts/tool_pilot_argon.py", + "policybench", + "reference_audit/2026-09-28/fixes", +) +PROMPT_VERSION = "v2-pilot" +CHEAP = ("claude-haiku-5.5", "gpt-6-luna", "gemini-3.5-flash-lite") +FRONTIER = ("gpt-6-sol", "claude-opus-5.5", "gemini-4-argon") +ARMS = ( + "pe", + "pe3", + "pe3-b", + "loo-labels", + "loo-names", + "loo-flat", + "loo-labels-flat", +) +FRONTIER_ARMS = ("pe", "pe3") +# The features each label drops from pe3; None for the tool v1 arm. +DROPPED = { + "pe": None, + "pe3": (), + "pe3-b": (), + "loo-labels": ("labels",), + "loo-names": ("names",), + "loo-flat": ("flat",), + "loo-labels-flat": ("labels", "flat"), +} +HOUSEHOLDS = 120 +MAX_RERUNS = 2 +INCONCLUSIVE_STOP = 5 # households of a model one inconclusive error may end +PRESSURE_WINDOW = 60.0 # seconds +HOST_LOG_GAP = 90.0 # seconds +SAMPLE_SECONDS = 30.0 +CEILING_MARGIN = 25.0 # dollars above the overall cap +ADMISSION_SLACK = 1e-9 # dollars +EXCERPT_LEAD = 86_400.0 # host logs are kept from a day before the run's start +WAIT_SECONDS = 1.0 # how often a waiting lane looks again +ATTEMPT_DEADLINE = 3600.0 # seconds; the longest development attempt took 714 +SAMPLER_SILENCE = 300.0 # seconds without a host-log sample that stop the run +READ_TRIES = 5 # reads of the guard's and the power log before a runner error +READ_WAIT = 10.0 # seconds between them +ANCHOR_SECONDS = 900.0 # how often the logs' chain heads are anchored +ANCHOR_RETRY = 60.0 # seconds to the next anchor after a failed push +ANCHOR_PUSH_TIMEOUT = 300.0 # seconds; then the command's process group is killed +KILL_GRACE = 10.0 # seconds to drain pipes after killing the process group +FINAL_PUSH_TRIES = 3 # includes the first final push +LATE_JOIN = 60.0 # seconds the run's end waits, in all, for abandoned attempts +# What a stop event names as its cause. +STOP_CAUSES = frozenset( + { + "harness", + "inconclusive", + "pool", + "deadline", + "runner-error", + "interrupt", + "sampler", + } +) + +# The pre-registration's error classes (its "Failures" table): the names of +# litellm's exception classes under the harness's retryable classes (transport) +# and under its model-fatal ones (request and account). A record keeps only +# the raised class's name. +TRANSPORT = frozenset( + { + "APIConnectionError", + "APIError", + "APIResponseValidationError", + "BadGatewayError", + "InternalServerError", + "JSONSchemaValidationError", + "MidStreamFallbackError", + "RateLimitError", + "ServiceUnavailableError", + "Timeout", + "RequestWallTimeoutError", + } +) +REQUEST = frozenset( + { + "BadRequestError", + "ContentPolicyViolationError", + "ContextWindowExceededError", + "ImageFetchError", + "InvalidRequestError", + "LiteLLMUnknownProvider", + "RejectedRequestError", + "UnprocessableEntityError", + "UnsupportedParamsError", + } +) +ACCOUNT = frozenset({"AuthenticationError", "PermissionDeniedError", "NotFoundError"}) +# A timeout on the escalated request, the one asking for the escalation +# ceiling after a truncated turn, is the model's long generation timing out. +TIMEOUTS = frozenset({"Timeout", "RequestWallTimeoutError"}) +HOST_ENGINE = frozenset({"MemoryError", "OSError"}) +RERUN = frozenset({"provider", "host", "none", "operator"}) +FINISHED = frozenset({"complete", "incomplete", "turn_limit"}) +# Classes decided without the host evidence: they come before host trouble. +EARLY = frozenset({"operator", "harness", "ceiling", "none"}) + +REQUIRED = ( + "preregistration", + "frozen_sha256", + "runs", + "stage", + "board", + "engine", + "eligible_pool", + "draw_order", + "arms", + "frontier_arms", + "treatment_harness", + "run_commit", + "caps", + "reserves", + "overall_cap", + "spent_before", + "workers", + "anchor_command", +) + + +# ---------------------------------------------------------------- streams + + +def master_seed(frozen_sha256: str) -> int: + if not re.fullmatch(r"[0-9a-f]{64}", frozen_sha256 or ""): + raise ValueError(f"not a sha256: {frozen_sha256!r}") + return int(frozen_sha256[:16], 16) + + +def stream(master: int, name: str) -> np.random.Generator: + key = int.from_bytes(hashlib.sha256(name.encode()).digest()[:8], "big") + return np.random.default_rng(np.random.SeedSequence(master, spawn_key=(key,))) + + +def draw(master: int, pool_ids: list[str], n: int | None = None) -> list[str]: + ids = np.array(sorted(pool_ids)) + size = HOUSEHOLDS if n is None else n + return [ + str(s) for s in stream(master, "draw").choice(ids, size=size, replace=False) + ] + + +def orders( + master: int, model: str, draw_order: list[str], arms: tuple[str, ...] +) -> dict[str, list[str]]: + """Each drawn household's arm order for a model, drawn for every household + in draw order, so no order depends on which households run.""" + generator = stream(master, f"order/{model}") + return { + sid: [str(a) for a in generator.permutation(list(arms))] for sid in draw_order + } + + +# ---------------------------------------------------------------- arms + + +def arm_string(label: str) -> str: + """The harness's arm string for a design label (`TOOL_FEATURES` order).""" + dropped = DROPPED[label] + if dropped is None: + return "pe" + if not dropped: + return "pe3" + return "pe:" + "+".join(f for f in pilot.TOOL_FEATURES if f not in dropped) + + +def arm_profile(model: str, label: str) -> pilot.ModelProfile: + """The treatment harness's profile for a model and design label; refused + unless its calculator features are the ones the label registers.""" + profile = pilot.model_profile( + model, arm=arm_string(label), prompt_version=PROMPT_VERSION + ) + dropped = DROPPED[label] + want = set() if dropped is None else set(pilot.TOOL_FEATURES) - set(dropped) + if not profile.uses_calculator or set(profile.tool_features) != want: + raise ValueError( + f"{model}/{label}: the harness gives features " + f"{sorted(profile.tool_features)}, not {sorted(want)}" + ) + return profile + + +# ---------------------------------------------------------------- classes + + +def error_class(record: dict) -> str: + return str(record.get("error") or "").split(":", 1)[0].strip() + + +def completed_turns(record: dict) -> list: + return [t for t in record.get("turns") or [] if not t.get("truncated_and_retried")] + + +def call_failures(record: dict) -> tuple[list[str], list[str]]: + """(the inconclusive tool failures, the model's engine exceptions) in its + calls: a call's own error, and the errors in its results and in its + explanations (the explain feature's trace of a variable).""" + host, model = [], [] + for turn in record.get("turns") or []: + for call in turn.get("calls") or []: + result = call.get("result") + if not isinstance(result, dict): + continue + error, message = result.get("error"), str(result.get("message") or "") + name = message.split(":", 1)[0] + if error == "timeout": + host.append("tool timeout") + elif error == "engine" and message.startswith("tool worker failed:"): + host.append("tool worker failure") + elif error == "engine": + (host if name in HOST_ENGINE else model).append(name) + for group in ("results", "explanations"): + values = result.get(group) + for value in values.values() if isinstance(values, dict) else (): + if isinstance(value, dict) and value.get("error"): + name = str(value["error"]).split(":", 1)[0] + (host if name in HOST_ENGINE else model).append(name) + return host, model + + +def escalation_timed_out(record: dict) -> bool: + """Whether the attempt ended on a timeout of its escalated request: the + harness records a truncated turn, then asks again at the escalation + ceiling, and an error there leaves the truncated turn last.""" + turns = record.get("turns") or [] + return ( + record.get("status") == "provider_error" + and error_class(record) in TIMEOUTS + and bool(turns) + and bool(turns[-1].get("truncated_and_retried")) + ) + + +def classify(record: dict, start: float, end: float, host) -> tuple[str, str]: + """(class, detail) of one attempt. In order of precedence: operator (an + account class or insufficient_quota); harness (a request class with no + completed model turn yet); ceiling (budget_stop); none (no turn recorded: + no model request returned, truncated or not); host (host trouble during + the attempt); model (a timeout of the escalated request); provider (a + transport class); model (a request class); inconclusive (any other + provider error, any harness error, a tool timeout, a worker failure, + MemoryError or OSError); otherwise ok, or model when the engine raised on + a household the model built. `host` is None to decide only the classes + before host.""" + status, error = record.get("status"), str(record.get("error") or "") + name = error_class(record) + returned = bool(record.get("turns")) + completed = bool(completed_turns(record)) + if status == "provider_error" and ( + name in ACCOUNT or "insufficient_quota" in error + ): + return "operator", name + if status == "provider_error" and name in REQUEST and not completed: + return "harness", name + if status == "budget_stop": + return "ceiling", status + if not returned: + return "none", f"{status}: {name}" if name else str(status) + trouble = host.trouble(start, end) if host is not None else None + if trouble: + return "host", trouble + if status == "provider_error": + if escalation_timed_out(record): + return "model", f"{name} on the escalated request" + if name in TRANSPORT: + return "provider", name + if name in REQUEST: + return "model", name + return "inconclusive", name or "provider_error" + if status == "harness_error": + return "inconclusive", error or "harness_error" + tool, engine = call_failures(record) + if tool: + return "inconclusive", tool[0] + if status not in FINISHED: + return "inconclusive", f"status {status}" + return ("model", engine[0]) if engine else ("ok", str(status)) + + +def action(cls: str, attempt: int) -> str: + if cls == "harness": + return "stop" + if cls == "ceiling": + return "stopped" + if cls in RERUN and attempt <= MAX_RERUNS: + return "pause-rerun" if cls == "operator" else "rerun" + return "stands" + + +def attempt_cost(record: dict) -> float: + return float((record.get("totals") or {}).get("cost_usd") or 0.0) + + +_clock_lock = threading.Lock() +_clock_last = 0.0 + + +def clock() -> float: + """`time.time()`, never earlier than a time it gave before. The wall clock + can step back by about 0.1 ms, even within a thread, and the run records order + attempts, stops, admission decisions and the ceiling's crossing by the + times this gives; each is taken under the lock that orders it.""" + global _clock_last + with _clock_lock: + _clock_last = max(time.time(), _clock_last) + return _clock_last + + +# ---------------------------------------------------------------- host evidence + + +GUARD_LINE = re.compile(r"^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d) \[[^\]]*\] (.*)$") +POWER_LINE = re.compile(r"^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d [+-]\d{4}) ([^\t]*?)\s*\t") +GUARD_LOG = Path.home() / "Library/Logs/clamshell-guard.log" + + +def guard_times(line: str, zone: ZoneInfo) -> tuple[float, float] | None: + """The earliest and latest instants a guard line's local time can be. They + differ only within an hour that a change of offset repeats or skips: on + 2026-11-01, New York's 01:00-02:00 comes twice, and the guard's lines + carry no offset.""" + match = GUARD_LINE.match(line) + if not match: + return None + stamp = datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S") + both = [stamp.replace(tzinfo=zone, fold=fold).timestamp() for fold in (0, 1)] + return min(both), max(both) + + +def power_times(line: str) -> tuple[float, float] | None: + """A power-log line's instant, twice (its time carries its offset).""" + match = POWER_LINE.match(line) + if not match: + return None + t = datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S %z").timestamp() + return t, t + + +def guard_spans(text: str, timezone: str, end: float) -> list: + """Pauses: from a 'paused pids:' line to the next 'resumed pids:' line (or + `end`), each from the earliest reading of its pause to the latest of its + resume.""" + zone, spans, opened = ZoneInfo(timezone), [], None + for line in text.splitlines(): + times = guard_times(line, zone) + if times is None: + continue + what = GUARD_LINE.match(line).group(2) + if "paused pids:" in what and opened is None: + opened = times[0] + elif "resumed pids:" in what and opened is not None: + spans.append((opened, times[1], "guard pause")) + opened = None + if opened is not None: + spans.append((opened, end, "guard pause")) + return spans + + +def power_spans(text: str, end: float) -> list: + """Sleeps: from a Sleep line to the next Wake or DarkWake line (or `end`).""" + spans, opened = [], None + for line in text.splitlines(): + times = power_times(line) + if times is None: + continue + t = times[0] + kind = POWER_LINE.match(line).group(2).strip() + if kind == "Sleep" and opened is None: + opened = t + elif kind in ("Wake", "DarkWake") and opened is not None: + spans.append((opened, t, "sleep")) + opened = None + if opened is not None: + spans.append((opened, end, "sleep")) + return spans + + +def excerpt( + text: str, + when: Callable[[str], tuple[float, float] | None], + since: float, + until: float, +): + """The lines of a log that one reading of its time puts within [since, + until].""" + keep = [] + for line in text.splitlines(): + times = when(line) + if times is not None and times[0] <= until and times[1] >= since: + keep.append(line + "\n") + return "".join(keep) + + +def memory_pressure() -> int | None: + """`sysctl kern.memorystatus_vm_pressure_level`: 1 when normal on this host.""" + try: + out = subprocess.run( + ["sysctl", "-n", "kern.memorystatus_vm_pressure_level"], + capture_output=True, + text=True, + timeout=10, + check=True, + ) + return int(out.stdout.strip()) + except (OSError, subprocess.SubprocessError, ValueError): + return None + + +def read_guard_log() -> str: + """The guard's log; empty when there is none (the guard never paused + anything). Any other error is raised, for the reader to try again.""" + try: + return GUARD_LOG.read_text(errors="replace") + except FileNotFoundError: + return "" + + +def read_power_log() -> str: + out = subprocess.run( + ["pmset", "-g", "log"], capture_output=True, text=True, timeout=120, check=True + ) + return out.stdout + + +def local_timezone() -> str: + """The IANA name of the host's zone, which the guard log's times are in.""" + target = os.path.realpath("/etc/localtime") + if "zoneinfo/" not in target: + raise SystemExit(f"cannot tell the host's time zone from {target}") + return target.split("zoneinfo/", 1)[1] + + +# The `prev` of the host log's first line. +HOST_SEED = hashlib.sha256(b"host-log.jsonl").hexdigest() + + +class HostEvidenceError(RuntimeError): + """The guard's or the power log could not be read: a runner error.""" + + +class HostLog: + """The host log, written every `interval` seconds, and the host evidence an + attempt's class reads: the guard's pauses, the power log's sleeps, the + host log's gaps and its memory-pressure samples. A sample whose pressure + could not be read is no sample: the silence around it is a gap.""" + + def __init__( + self, + folder: Path, + timezone: str | None = None, + interval: float = SAMPLE_SECONDS, + read_load: Callable[[], tuple] = os.getloadavg, + read_pressure: Callable[[], int | None] = memory_pressure, + read_guard: Callable[[], str] = read_guard_log, + read_power: Callable[[], str] = read_power_log, + ): + self.folder = Path(folder) + self.folder.mkdir(parents=True, exist_ok=True) + self.timezone = timezone or local_timezone() + ZoneInfo(self.timezone) # refused here, not at the first pause + self.interval = interval + self._read_load, self._read_pressure = read_load, read_pressure + self._read_guard, self._read_power = read_guard, read_power + self._cond = threading.Condition() + self._samples: list[tuple[float, int | None]] = [] + self._spans: list = [] # guard pauses and sleeps at the last refresh + self._read_at = -math.inf + self._refresh_lock = threading.Lock() + self._stop = threading.Event() + self.failure: str | None = None # the sampler's last error + self.failing_since: float | None = None # failed samples since then + self._head, self._lines = HOST_SEED, 0 # the log's hash chain + self.start = time.time() + self.written = self.start # the last sample written + self.end: float | None = None + self._write_meta() + self.sample() + self._thread = threading.Thread(target=self._loop, name="host-log", daemon=True) + self._thread.start() + + def _write_meta(self) -> None: + meta = {"timezone": self.timezone, "start": self.start} + if self.end is not None: + meta["end"] = self.end + write_json(self.folder / "host.json", meta) + + def sample(self) -> None: + """One sample. A load or pressure the host would not give is written + as null; a sample that cannot be written is the sampler's failure, + and the silence it leaves is a gap.""" + try: + load1, load5, load15 = self._read_load() + except OSError: + load1 = load5 = load15 = None + pressure = self._read_pressure() + with self._cond: + t = time.time() + line = { + "time": t, + "monotonic": time.monotonic(), + "load1": load1, + "load5": load5, + "load15": load15, + "pressure": pressure, + } + text = json.dumps({**line, "prev": self._head}) + try: + with (self.folder / "host-log.jsonl").open("a") as handle: + handle.write(text + "\n") + except OSError as error: + self.failure = f"{type(error).__name__}: {error}" + self.failing_since = self.failing_since or t + return + self._head, self._lines = line_sha(text), self._lines + 1 + self.written = t + if pressure is None: + self.failure = "no memory-pressure reading" + self.failing_since = self.failing_since or t + else: + self.failure = None + self.failing_since = None + self._samples.append((t, pressure)) + self._cond.notify_all() + + def heads(self) -> dict: + """The host log's line count and last line's sha256, for an anchor.""" + with self._cond: + return {"lines": self._lines, "sha256": self._head} + + def dead(self) -> str | None: + """Why the sampler is taken for dead: no good sample for + SAMPLER_SILENCE seconds while its samples fail, or its thread gone. + A host that slept or was paused is not this: its sampler failed + nothing.""" + if self.end is None and not self._thread.is_alive(): + return "the sampler's thread ended" + since = self.failing_since + if since is not None and time.time() - since > SAMPLER_SILENCE: + return f"samples failing since {since:.1f} ({self.failure})" + return None + + def _loop(self) -> None: + k = 1 + while not self._stop.wait( + max(0.0, self.start + k * self.interval - time.time()) + ): + try: + self.sample() + except Exception as error: # the sampler runs on; watch() sees the silence + self.failure = f"{type(error).__name__}: {error}" + self.failing_since = self.failing_since or time.time() + # After a stall, the next sample is the next tick, with no burst. + k = max(k + 1, math.ceil((time.time() - self.start) / self.interval)) + + def _good(self) -> list[float]: + """The times of the samples with a pressure reading.""" + return [t for t, p in self._samples if p is not None] + + def settled(self, end: float) -> bool: + """Whether every sample that could bear on an attempt ending at `end` is + in: one later than the pressure window, or a silence longer than a + gap (which is itself host trouble when it covers the attempt).""" + with self._cond: + good = self._good() + latest = good[-1] if good else self.start + return latest > end + PRESSURE_WINDOW or time.time() - latest > HOST_LOG_GAP + + def wait_settled(self, end: float) -> None: + with self._cond: + while not self.settled(end): + self._cond.wait(timeout=min(self.interval, 5.0)) + + def _read_logs(self) -> tuple[str, str]: + """The guard's and the power log, read up to READ_TRIES times.""" + for attempt in range(READ_TRIES): + try: + return self._read_guard(), self._read_power() + except (OSError, subprocess.SubprocessError) as error: + failure = f"{type(error).__name__}: {error}" + if attempt < READ_TRIES - 1: + time.sleep(READ_WAIT) + raise HostEvidenceError(f"the guard's or the power log: {failure}") + + def refresh(self, after: float) -> None: + """Read the guard's and the power log again unless read since `after`. + A failed read keeps the last spans and tries again; HostEvidenceError + after READ_TRIES.""" + with self._refresh_lock: + if self._read_at > after: + return + now = time.time() + since = self.start - EXCERPT_LEAD + zone = ZoneInfo(self.timezone) + guard_text, power_text = self._read_logs() + guard = excerpt(guard_text, lambda x: guard_times(x, zone), since, now) + power = excerpt(power_text, power_times, since, now) + spans = guard_spans(guard, self.timezone, now) + power_spans(power, now) + with self._cond: + self._spans, self._read_at = spans, now + + def _evidence(self) -> tuple[list, list[float]]: + """(spans, the times of samples above 1), as the report builds them + from the archived logs, with a silence so far as an open gap.""" + with self._cond: + spans = list(self._spans) + times = self._good() + pressure = [t for t, p in self._samples if p is not None and int(p) > 1] + for a, b in zip([self.start] + times, times): + if b - a > HOST_LOG_GAP: + spans.append((a, b, "host log gap")) + now = time.time() if self.end is None else self.end + last = times[-1] if times else self.start + if now - last > HOST_LOG_GAP: + spans.append((last, now, "host log gap")) + return spans, pressure + + def trouble(self, start: float, end: float) -> str | None: + spans, pressure = self._evidence() + for a, b, what in spans: + if a <= end and b >= start: + return what + for t in pressure: + if start - PRESSURE_WINDOW <= t <= end + PRESSURE_WINDOW: + return "memory pressure" + return None + + def describe(self, start: float, end: float) -> str: + """The host evidence behind an attempt's host class, for its rerun's + reason.""" + spans, _ = self._evidence() + parts = [ + f"{what} {a:.1f}-{b:.1f}" for a, b, what in spans if a <= end and b >= start + ] + with self._cond: + parts += [ + f"memory pressure {p} at {t:.1f}" + for t, p in self._samples + if int(p or 0) > 1 + and start - PRESSURE_WINDOW <= t <= end + PRESSURE_WINDOW + ] + return "; ".join(parts) + + def close(self) -> None: + """A last sample, the run's end, and the guard's and power log's lines + from a day before the run's start to its end. If the logs cannot be + read, `host-error.txt` says why and the excerpts are not written: the + report then has no host evidence to check against.""" + self._stop.set() + self._thread.join(timeout=self.interval + 10) + try: + self.sample() + except Exception as error: + self.failure = f"{type(error).__name__}: {error}" + with self._cond: + self.end = time.time() + self._write_meta() + since, zone = self.start - EXCERPT_LEAD, ZoneInfo(self.timezone) + try: + guard_text, power_text = self._read_logs() + except HostEvidenceError as error: + (self.folder / "host-error.txt").write_text(f"{error}\n") + return + guard = excerpt(guard_text, lambda x: guard_times(x, zone), since, self.end) + (self.folder / "clamshell-guard.log").write_text(guard) + power = excerpt(power_text, power_times, since, self.end) + (self.folder / "pmset.log").write_text(power) + + +# ---------------------------------------------------------------- spend + + +class Ceiling(pilot.Spend): + """The hard ceiling over every model's spend: once the total reaches it, + `exhausted` is true for good, which ends each conversation under way at + its next turn and lets no attempt start.""" + + def __init__(self, ceiling_usd: float, spent_before: float): + super().__init__(ceiling_usd, spent_before) + self.crossed_at: float | None = None + self.closed = False # the records are final: spend is no longer counted + self.unrecorded_usd = 0.0 # spend after that, in no record + + @property + def lock(self) -> threading.Lock: + return self._lock + + def charge(self, usd: float) -> bool: + """Count `usd`, with the lock held; False, and the spend unrecorded, + once the ledger has closed.""" + if self.closed: + self.unrecorded_usd += usd + return False + self.spent_usd += usd + if self.crossed_at is None and self.spent_usd >= self.budget_usd: + self.crossed_at = clock() + return True + + def add(self, usd: float | None) -> None: + with self._lock: + self.charge(float(usd or 0.0)) + + def close(self, pending: list[dict] | None = None) -> list[dict]: + """Close the ledger at the run's end: what the ceiling has counted is + then what the records hold. Snapshot unresolved requests under the + same lock: one finishing afterward cannot disappear from recovery.""" + with self._lock: + self.closed = True + return [item for item in pending or [] if item["thread"].is_alive()] + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + with self._lock: + return self.crossed_at is not None + + +class AttemptSpend: + """The hard ceiling as one attempt's conversation sees it. Every cost goes + to the ceiling, and the attempt keeps its own total, so an attempt that + raises still records what it spent. Once the attempt is abandoned at its + deadline, the conversation ends `budget_stop` at its next turn, and what + it spends after is kept apart as late spend. The attempt's totals and the + ceiling's change under the ceiling's lock, so they always agree; once + the ledger has closed, neither counts more.""" + + def __init__(self, ceiling: Ceiling): + self.ceiling = ceiling + self.spent_usd = 0.0 + self.late_usd = 0.0 + self.abandoned = False + + def add(self, usd: float | None) -> None: + usd = float(usd or 0.0) + with self.ceiling.lock: + if not self.ceiling.charge(usd): + return + if self.abandoned: + self.late_usd += usd + else: + self.spent_usd += usd + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + return self.abandoned or self.ceiling.exhausted() + + def abandon(self) -> float: + """Abandon the attempt; what it had spent.""" + with self.ceiling.lock: + self.abandoned = True + return self.spent_usd + + +# ---------------------------------------------------------------- files + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + return sha256_bytes(Path(path).read_bytes()) + + +def write_json(path: Path, payload, indent: int | None = 1) -> str: + """Write a file whole (to a temporary name, synced, then renamed); its + sha256.""" + data = json.dumps(payload, indent=indent, default=str).encode() + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.tmp") + with tmp.open("wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + return sha256_bytes(data) + + +def chain_seed(frozen_sha256: str, model: str, name: str) -> str: + """The `prev` of a log's first line: the log's place in the design.""" + return sha256_bytes(f"{frozen_sha256}/{model}/{name}".encode()) + + +def line_sha(text: str) -> str: + """A log line's hash, the next line's `prev`: its JSON, no newline.""" + return sha256_bytes(text.encode()) + + +# ---------------------------------------------------------------- one model + + +@dataclass +class Attempt: + label: str + n: int + start: float + end: float + record: dict + cls: str = "" + detail: str = "" + reason: str = "" + settled: bool = False # classed with its pass's host evidence + logged: bool = False # its run-log line written + + +class AttemptDeadline(RuntimeError): + """An attempt ran past ATTEMPT_DEADLINE and was abandoned.""" + + +def converse_with_harness( + lanes: "Lanes", sid: str, label: str, tool_call, spend: AttemptSpend +) -> dict: + """One attempt: the treatment harness's conversation for the arm, under the + hard ceiling.""" + board = lanes.run.board + return pilot.run_household( + board["scenarios"][sid], + board["variables"][sid], + tool_call, + lanes.completion, + spend, + profile=lanes.profiles[label], + ) + + +class Lanes: + """One model's two lanes: its admission, its attempts and its records.""" + + def __init__(self, run: "Run", model: str, labels: tuple[str, ...], folders: dict): + self.run, self.model, self.labels = run, model, labels + design = run.design + self.folder = Path(design["runs"]) / model + self.households_dir = { + label: self.folder / folders[label] / design["stage"] / "households" + for label in labels + } + self.profiles = {label: arm_profile(model, label) for label in labels} + self.cap = float(design["caps"][model]) + self.reserve = float(design["reserves"][model]) + self.orders = orders(run.master, model, run.draw_order, labels) + self.cond = threading.Condition() + self._log_lock = threading.Lock() + self.state: dict[str, str] = {} # household -> running, settling, finished + self.spent: dict[str, float] = {} # household -> its attempts' cost so far + self.refused = False + self.paused = False + self.paused_at: float | None = None + self.stopped = False + self.inconclusive: dict[str, list[str]] = {} + self.pool = None + self._pool_lock = threading.Lock() + self.completion = None + self.manifest_sha256: str | None = None + # Each log is a hash chain: a line's `prev` is the line before's sha256. + self._heads = { + name: chain_seed(design["frozen_sha256"], model, name) + for name in ("run-log.jsonl", "admission-log.jsonl") + } + self._counts = {name: 0 for name in self._heads} + # Attempts abandoned at their deadline, whose spend after it is logged. + self.abandoned: list[dict] = [] + + # The records. + + def households(self, lane: int) -> list[str]: + return [ + sid + for i, sid in enumerate(self.run.draw_order) + if i % 2 == lane - 1 and sid not in self.run.no_output + ] + + def write_line(self, line: dict, name: str = "run-log.jsonl") -> None: + with self._log_lock: + text = json.dumps({**line, "prev": self._heads[name]}) + self.folder.mkdir(parents=True, exist_ok=True) + with (self.folder / name).open("a") as handle: + handle.write(text + "\n") + handle.flush() + os.fsync(handle.fileno()) + self._heads[name] = line_sha(text) + self._counts[name] += 1 + + def heads(self) -> dict: + """Each log's line count and last line's sha256, for an anchor.""" + with self._log_lock: + return { + name: {"lines": self._counts[name], "sha256": self._heads[name]} + for name in self._heads + } + + def event( + self, kind: str, reason: str, at: float | None = None, cause=None, **extra + ) -> None: + line = {"event": kind, "time": clock() if at is None else at, "reason": reason} + if cause is not None: + line["cause"] = cause + self.write_line({**line, **extra}) + self.run.say(f"{self.model}: {kind} ({reason})") + + def attempt_line(self, sid: str, lane: int, a: Attempt, sha: str) -> None: + self.write_line( + { + "household": sid, + "arm": a.label, + "lane": lane, + "attempt": a.n, + "start": a.start, + "end": a.end, + "status": a.record.get("status"), + "class": a.cls, + "action": action(a.cls, a.n), + "sha256": sha, + } + ) + a.logged = True + + # Stops and pauses. + + def halted(self) -> bool: + return self.stopped or self.run._failed.is_set() or self.run.ceiling.exhausted() + + def stop( + self, reason: str, cause: str, kind: str = "stop", at: float | None = None + ) -> None: + """Stop the lanes. Every rule that stops them writes its own event, + naming its cause.""" + if (kind, cause) != ("ceiling", "ceiling") and cause not in STOP_CAUSES: + raise ValueError(f"no stop has the cause {cause!r}") + with self.cond: + self.stopped = True + self.cond.notify_all() + self.event(kind, reason, at, cause=cause) + + def resume(self, note: str, noted_at: float) -> bool: + """A note may end only a pause it was written during; another lane's + later operator failure cannot be cleared by the older note.""" + with self.cond: + if not (self.paused and noted_at >= self.paused_at): + return False + self.paused = False + self.cond.notify_all() + self.event("resume", note) + return True + + # One household. + + def admit(self, sid: str, lane: int) -> bool: + with self.cond: + while not self.halted() and ( + self.paused or "settling" in self.state.values() + ): + self.cond.wait(timeout=WAIT_SECONDS) + # Taken under the ceiling's lock, so no decision follows the crossing. + with self.run.ceiling.lock: + if ( + self.stopped + or self.run._failed.is_set() + or self.run.ceiling.crossed_at is not None + ): + return False + now = clock() + finished = [h for h, s in self.state.items() if s == "finished"] + spend = sum(self.spent[h] for h in finished) + under_way = len(self.state) - len(finished) + 1 + ok = ( + not self.refused + and spend + under_way * self.reserve <= self.cap + ADMISSION_SLACK + ) + self.write_line( + { + "household": sid, + "lane": lane, + "time": now, + "decision": "admit" if ok else "refuse", + "finished_spend": spend, + "under_way": under_way, + "reserve": self.reserve, + "cap": self.cap, + }, + "admission-log.jsonl", + ) + if ok: + self.state[sid], self.spent[sid] = "running", 0.0 + else: + self.refused = True + self.cond.notify_all() + return ok + + def start_attempt(self) -> float | None: + """The attempt's start, or None when the lanes have stopped. Waits out a + pause. The start is taken under the ceiling's lock, so no attempt + starts after the crossing.""" + with self.cond: + while self.paused and not self.halted(): + self.cond.wait(timeout=WAIT_SECONDS) + with self.run.ceiling.lock: + if ( + self.stopped + or self.run._failed.is_set() + or self.run.ceiling.crossed_at is not None + ): + return None + return clock() + + def end_attempt( + self, + sid: str, + label: str, + n: int, + start: float, + record: dict, + last: bool, + deadline: bool = False, + retained: list[Attempt] | None = None, + ) -> Attempt: + """The attempt's end. A class decided without the host evidence acts at + once: an operator failure pauses the lanes, a harness failure stops + them, before any other attempt can start; so does an attempt + abandoned at its deadline.""" + with self.cond: + end = clock() + self.spent[sid] += attempt_cost(record) + attempt = Attempt(label, n, start, end, record) + if retained is not None: + retained.append(attempt) # recovery owns it before any event I/O + cls, detail = classify(record, start, end, None) + if cls in EARLY: + attempt.cls, attempt.detail = cls, detail + if cls == "operator": + self.paused, self.paused_at = True, clock() + if cls == "harness" or deadline: + self.stopped = True + if last: + self.state[sid] = "settling" + self.cond.notify_all() + if cls == "operator": + self.event( + "pause", f"{label}/{sid} attempt {n}: {detail}", at=self.paused_at + ) + if cls == "harness": + self.event( + "stop", + f"{label}/{sid}: a request class on the first request ({detail})", + cause="harness", + ) + if deadline: + self.event( + "stop", + f"{label}/{sid} attempt {n} ran past the attempt deadline " + f"of {ATTEMPT_DEADLINE:.0f} s", + cause="deadline", + ) + return attempt + + def settle(self, sid: str, done: list[Attempt]) -> None: + """Each attempt's class once the host evidence for the pass is in.""" + last = max(a.end for a in done) + host = self.run.host + host.wait_settled(last) + host.refresh(after=last) + stop = None + for a in done: + cls, detail = classify(a.record, a.start, a.end, host) + if a.cls and a.cls != cls: + raise RuntimeError(f"{a.label}/{sid}: class {a.cls} became {cls}") + a.cls, a.detail, a.settled = cls, detail, True + a.reason = ( + f"{detail}: {host.describe(a.start, a.end)}" + if cls == "host" + else detail + ) + status = a.record.get("status") + if cls == "inconclusive" and status in ("provider_error", "harness_error"): + key = f"{status}: {error_class(a.record) or a.record.get('error')}" + with self.cond: + seen = self.inconclusive.setdefault(key, []) + if sid not in seen: + seen.append(sid) + if len(seen) > INCONCLUSIVE_STOP and stop is None: + stop = f"{key} ended runs in {len(seen)} households" + if stop: + self.stop(stop, cause="inconclusive") + + def converse(self, sid: str, label: str, n: int) -> tuple[dict, bool]: + """(the attempt's record, whether it was abandoned). The conversation + runs in a thread of its own. One that raises leaves a harness-error + record with the spend it counted; one still running at + ATTEMPT_DEADLINE is abandoned with a harness-error record holding + the spend so far, ends `budget_stop` at its next turn, and writes + nothing.""" + profile = self.profiles[label] + variables = self.run.board["variables"][sid] + features = frozenset(profile.tool_features) + spend = AttemptSpend(self.run.ceiling) + + def tool_call(arguments): + message = ( + pilot.tool_call_frame(features, arguments) if features else arguments + ) + return self.pool.call(message) + + box: dict = {} + + def work(): + try: + box["record"] = self.run.converse(self, sid, label, tool_call, spend) + except Exception as error: # an attempt that left no record of its own + box["error"] = error + + thread = threading.Thread( + target=work, name=f"{self.model}-{sid}-{label}-{n}", daemon=True + ) + thread.start() + thread.join(timeout=ATTEMPT_DEADLINE) + if thread.is_alive(): + spent = spend.abandon() + with self.cond: + self.abandoned.append( + { + "household": sid, + "arm": label, + "attempt": n, + "spend": spend, + "thread": thread, + "logged": 0.0, + } + ) + error = AttemptDeadline( + f"past the attempt deadline of {ATTEMPT_DEADLINE:.0f} s" + ) + record = pilot.harness_error_record(sid, variables, profile, error) + record["totals"]["cost_usd"] = spent + return record, True + if "error" in box: + record = pilot.harness_error_record(sid, variables, profile, box["error"]) + record["totals"]["cost_usd"] = spend.spent_usd + return record, False + return box["record"], False + + def log_late(self, final: bool = False) -> None: + """A `late-cost` event for each abandoned attempt that has ended (or, + at the end of the run, for each one still running) with spend after + its deadline not yet logged.""" + with self.cond: + pending = list(self.abandoned) + for item in pending: + ended = not item["thread"].is_alive() + late = item["spend"].late_usd + if (ended or final) and late > item["logged"]: + self.event( + "late-cost", + f"{item['arm']}/{item['household']} attempt {item['attempt']}: " + "spend after its deadline", + household=item["household"], + arm=item["arm"], + attempt=item["attempt"], + cost_usd=late - item["logged"], + ) + item["logged"] = late + if ended: + with self.cond: + self.abandoned.remove(item) + + def ensure_pool(self, startup: bool = False) -> bool: + """The model's pool, started again if it has lost every worker; False + if it cannot be.""" + with self._pool_lock: + if self.halted(): + return False + if ( + self.pool is not None + and self.pool.counters()["lost"] < self.run.workers + ): + return True + old = self.pool + start, outcome = clock(), "failed" + if not startup: + self.event( + "setup-start", + "engine pool restart", + at=start, + phase="restart", + start=start, + ) + try: + try: + self.pool = self.run.pool_factory(self) + except RuntimeError as error: + self.run.say( + f"{self.model}: the engine pool did not start: {error}" + ) + return False + if old is not None: + self.run.say( + f"{self.model}: the engine pool lost every worker; " + "started again" + ) + old.close() + outcome = "stopped" if self.halted() else "ready" + return outcome == "ready" + finally: + if not startup: + self.event( + "setup-end", + "engine pool restart", + phase="restart", + start=start, + outcome=outcome, + ) + + def replace(self, sid: str, lane: int, previous: Attempt, entries: list) -> None: + """Save the attempt a rerun replaces, before the rerun.""" + folder = self.households_dir[previous.label] / "attempts" + name = f"{sid}-{previous.n}.json" + sha = write_json(folder / name, previous.record) + entries.append( + { + "n": previous.n, + "file": f"attempts/{name}", + "sha256": sha, + "status": previous.record.get("status"), + "class": previous.cls, + "reason": previous.reason, + } + ) + self.attempt_line(sid, lane, previous, sha) + + def passes( + self, sid: str, lane: int, attempts: dict, entries: dict, in_flight: Path + ) -> None: + """The household's arms in its drawn order, then its reruns in passes, + each pass classed once its host evidence is in.""" + todo = [(1, label) for label in self.orders[sid]] + while todo: + done: list[Attempt] = [] + for i, (n, label) in enumerate(todo): + if not self.ensure_pool(): + if not self.halted(): + self.stop("the engine pool could not be started again", "pool") + break + start = self.start_attempt() + if start is None: + break + if n > 1: + self.replace(sid, lane, attempts[label][-1], entries[label]) + record, abandoned = self.converse(sid, label, n) + a = self.end_attempt( + sid, + label, + n, + start, + record, + i == len(todo) - 1, + abandoned, + retained=attempts[label], + ) + done.append(a) + write_json(in_flight / f"{label}-{n}.json", record) + self.run.say( + f"{self.model} lane {lane} {sid} {label} #{n}: " + f"{record.get('status')} ${attempt_cost(record):.4f} " + f"(total ${self.run.ceiling.spent_usd:.2f})" + ) + if not done: + break + self.settle(sid, done) + cut = len(done) < len(todo) + todo = ( + [] + if cut or self.halted() + else [ + (a.n + 1, a.label) + for a in done + if a.n <= MAX_RERUNS and a.cls in RERUN + ] + ) + with self.cond: + self.state[sid] = "running" if todo else "finished" + self.cond.notify_all() + + def run_household(self, sid: str, lane: int) -> bool: + """One household; False when the runner failed part way. Then every + model stops, and the attempts that ended are classed (as `unsettled` + when their host evidence cannot be read) and written before the lane + ends.""" + attempts: dict[str, list[Attempt]] = {label: [] for label in self.orders[sid]} + entries: dict[str, list[dict]] = {label: [] for label in self.orders[sid]} + in_flight = self.folder / "in-flight" / sid + failure = None + try: + self.passes(sid, lane, attempts, entries, in_flight) + except Exception as error: + self.run._failed.set() # block starts while records are recovered + failure = ( + f"runner error on {self.model} lane {lane} ({sid}): " + f"{type(error).__name__}: {error}" + ) + self.run.say(traceback.format_exc()) + pending = [a for mine in attempts.values() for a in mine if not a.settled] + if pending: + try: + self.settle(sid, pending) + except Exception as again: + self.run.say(traceback.format_exc()) + for a in pending: + a.cls, a.settled = "unsettled", True + a.detail = a.reason = ( + f"unsettled: {type(again).__name__}: {again}" + ) + for label, mine in attempts.items(): + if mine and not mine[-1].logged: + final = mine[-1] + try: + sha = write_json( + self.households_dir[label] / f"{sid}.json", + { + **final.record, + "attempts": [ + entry + for entry in entries[label] + if entry["n"] < final.n + ], + }, + ) + self.attempt_line(sid, lane, final, sha) + except Exception as error: # the other arms still get written + self.run._failed.set() + failure = failure or ( + f"runner error writing {self.model} lane {lane} ({sid}): " + f"{type(error).__name__}: {error}" + ) + self.run.say(traceback.format_exc()) + if failure is not None: + self.run.stop_all(failure, cause="runner-error") + with self.cond: + self.state[sid] = "finished" + self.cond.notify_all() + if failure is None: + shutil.rmtree(in_flight, ignore_errors=True) + return failure is None + + def lane(self, lane: int) -> None: + try: + for sid in self.households(lane): + if not self.admit(sid, lane) or not self.run_household(sid, lane): + return + except Exception as error: # a runner defect: every model stops + self.run.say(traceback.format_exc()) + self.run.stop_all( + f"runner error on {self.model} lane {lane}: " + f"{type(error).__name__}: {error}", + cause="runner-error", + ) + + def manifest(self) -> dict: + board, design = self.run.board, self.run.design + pe3 = arm_profile(self.model, "pe3") + first = sorted(board["scenarios"])[0] + + def timeout(profile): + built = pilot.build_initial_request( + board["scenarios"][first], board["variables"][first], profile + ) + return built[1].get("timeout") + + board_path = Path(design["board"]) + return { + "started_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "git_head": pilot._git("rev-parse", "HEAD"), + "git_dirty": bool(pilot._git("status", "--porcelain")), + "script_sha256": sha256_file(Path(__file__)), + "harness_sha256": sha256_file(Path(pilot.__file__)), + "treatment_commit": TREATMENT_COMMIT, + "versions": pilot._versions(), + "fix_module": str(pilot.FIX_MODULE.relative_to(pilot.ROOT)), + "fix_module_sha256": sha256_file(pilot.FIX_MODULE), + "board": str(board_path), + "board_engine": f"policyengine-us {pilot.board_engine(board_path)}", + "board_reference_sha256": sha256_file(board_path / "reference_outputs.csv"), + "model": self.model, + "model_id": pe3.model_id, + "api": pe3.api, + "answer_contract": pe3.contract, + "prompt_version": PROMPT_VERSION, + "prompt_wording_sha256": pilot.prompt_wording_sha256(pe3), + "request_timeout_seconds": timeout(pe3), + "turn_cap": pilot.TOOL_TURN_CAP, + "max_calculations_per_turn": pilot.MAX_CALCULATIONS_PER_TURN, + "tool_timeout_seconds": pilot.TOOL_TIMEOUT_SECONDS, + "arms": { + label: { + "arm": arm_string(label), + "features": sorted(self.profiles[label].tool_features), + } + for label in self.labels + }, + "arm_settings": { + label: { + "tool_notice_sha256": hashlib.sha256( + profile.notice.encode() + ).hexdigest(), + **pilot.profile_treatment(profile), + "tool_version": profile.tool_version, + "request_timeout_seconds": timeout(profile), + } + for label, profile in self.profiles.items() + }, + "folders": { + label: str(path) for label, path in self.households_dir.items() + }, + "cap_usd": self.cap, + "reserve_usd": self.reserve, + "overall_cap_usd": float(design["overall_cap"]), + "ceiling_usd": self.run.ceiling.budget_usd, + "spent_before_usd": float(design["spent_before"]), + "lanes": {lane: self.households(lane) for lane in (1, 2)}, + "no_scored_output": sorted(self.run.no_output), + "workers": self.run.workers, + "attempt_deadline_seconds": ATTEMPT_DEADLINE, + "anchor_seconds": ANCHOR_SECONDS, + "anchor_command": list(design["anchor_command"]), + } + + def summary(self) -> dict: + from confirmatory_no_tools import _board_input, saved_exists, saved_json_paths + + records = [] + for label, folder in self.households_dir.items(): + for path in saved_json_paths(folder): + records.append(_board_input(path, lambda: json.loads(path.read_text()))) + lines = [] + path = self.folder / "run-log.jsonl" + if saved_exists(path): + lines = _board_input( + path, + lambda: [ + json.loads(x) for x in path.read_text().splitlines() if x.strip() + ], + ) + attempts = [x for x in lines if "event" not in x] + count: dict[str, int] = {} + for x in attempts: + count[x["class"]] = count.get(x["class"], 0) + 1 + return { + "finished_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "model": self.model, + "households_finished": sorted( + h for h, s in self.state.items() if s == "finished" + ), + "refused": self.refused, + "stopped": self.stopped, + "spent_usd": sum(self.spent.values()), + "final_records": len(records), + "attempts": len(attempts), + "classes": dict(sorted(count.items())), + "events": [x for x in lines if "event" in x], + "tool_pool": self.pool.counters() if self.pool is not None else None, + "abandoned_running": len(self.abandoned), + } + + +# ---------------------------------------------------------------- the run + + +def default_pool(lanes: Lanes): + return pilot.ToolPool( + size=lanes.run.workers, + board=Path(lanes.run.design["board"]), + tool_features=frozenset(), + ) + + +def default_completion(profile: pilot.ModelProfile): + from litellm import completion, responses + + return responses if profile.api == "responses" else completion + + +class Run: + """The six models' lanes, the host log and the hard ceiling.""" + + def __init__( + self, + design: dict, + workers: int | None = None, + models: tuple[str, ...] | None = None, + converse: Callable = converse_with_harness, + pool_factory: Callable = default_pool, + completion_for: Callable = default_completion, + host_factory: Callable | None = None, + out=sys.stdout, + ): + self.design, self.out = design, out + self.workers = int(design["workers"]) if workers is None else workers + self.converse, self.pool_factory = converse, pool_factory + self.completion_for = completion_for + self.host_factory = host_factory or (lambda folder: HostLog(folder)) + self._say_lock = threading.Lock() + self.master = master_seed(design["frozen_sha256"]) + self.draw_order = list(design["draw_order"]) + board_path = Path(design["board"]) + pilot.set_stated_engine(pilot.board_engine(board_path)) + self.board = pilot.load_board(board_path) + scored = {str(s) for s in self.board["scored"]["scenario_id"]} + self.no_output = {sid for sid in self.draw_order if sid not in scored} + self.ceiling = Ceiling( + float(design["overall_cap"]) + CEILING_MARGIN, float(design["spent_before"]) + ) + self.models = tuple(models or CHEAP + FRONTIER) + self.lanes = { + model: Lanes( + self, + model, + ARMS if model in CHEAP else FRONTIER_ARMS, + design["arms"] if model in CHEAP else design["frontier_arms"], + ) + for model in self.models + } + self.host: HostLog | None = None + self._ceiling_written = False + self.interrupts = 0 + self.signalled: int | None = None # set by the signal handler alone + self._interrupted = False + self._sampler_stopped = False + self._failed = threading.Event() # recovery blocks starts before stop I/O + self._anchor_head = chain_seed(design["frozen_sha256"], "", "anchors.jsonl") + self._anchored = -math.inf + self._pushing = threading.Lock() + self._next_anchor = -math.inf + self._push: threading.Thread | None = None + self._push_error: OSError | None = None + self._push_pending: dict | None = None + self._last_push_outcome: str | None = None + + def say(self, text: str) -> None: + with self._say_lock: + print(text, file=self.out, flush=True) + + def stop_all( + self, reason: str, cause: str, kind: str = "stop", at: float | None = None + ) -> None: + # Set every state before the first fallible write; notify even if a + # disk error prevents a model's event from being recorded. + for lanes in self.lanes.values(): + with lanes.cond: + lanes.stopped = True + lanes.cond.notify_all() + for lanes in self.lanes.values(): + try: + lanes.event(kind, reason, at, cause=cause) + except Exception: + self.say(traceback.format_exc()) + + # Anchors: the logs' chain heads, written and pushed outside the run. + + def _check_push_error(self) -> None: + with self._pushing: + error = self._push_error + if error is not None: + raise OSError( + f"anchor push outcome could not be recorded: {error}" + ) from error + + def anchor_due(self) -> bool: + self._check_push_error() + with self._pushing: + return clock() >= self._next_anchor + + def anchor(self, wait: bool = False, push: bool = True) -> None: + """Every log's chain head to `anchors.jsonl`, itself a chain, then the + design's anchor command on a synced snapshot, in the background (one + current worker at a time; an anchor whose push would wait is written + but not pushed until the next). A snapshot is a copy, not a record, + and its path is unique even if a push thread has to be abandoned. + Each push's result goes to `anchor-pushes.jsonl`.""" + runs = Path(self.design["runs"]) + line = { + "time": clock(), + "models": { + model: {**lanes.heads(), "manifest": lanes.manifest_sha256} + for model, lanes in self.lanes.items() + }, + "host-log.jsonl": self.host.heads() if self.host is not None else None, + "prev": self._anchor_head, + } + text = json.dumps(line) + with (runs / "anchors.jsonl").open("a") as handle: + handle.write(text + "\n") + handle.flush() + os.fsync(handle.fileno()) + self._anchor_head, self._anchored = line_sha(text), clock() + with self._pushing: + self._next_anchor = self._anchored + ANCHOR_SECONDS + if not push: + return # a push-log failure still leaves the final local heads + if self._push is not None and self._push.is_alive(): + if not wait: + return + self._join_push() + snapshot = runs / f"anchors.{self._anchor_head}.push-snapshot.jsonl" + tmp = snapshot.with_suffix(".tmp") + with tmp.open("wb") as handle: + handle.write((runs / "anchors.jsonl").read_bytes()) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, snapshot) + with self._pushing: + self._push_pending = { + "anchor": self._anchor_head, + "anchor_time": line["time"], + "started": clock(), + } + self._last_push_outcome = None + self._push = threading.Thread( + target=self._push_anchor, + args=(self._anchor_head, line["time"]), + daemon=True, + ) + self._push.start() + if wait: + self._join_push() + + def _join_push(self) -> None: + """Bound the thread join and retire its outcome if it cannot end. + Its command may survive; the helper's lock and prefix check protect + the clone. Waiting for _pushing still assumes a responsive filesystem.""" + self._push.join(timeout=ANCHOR_PUSH_TIMEOUT + KILL_GRACE + 30) + if self._push.is_alive(): + with self._pushing: + pending = self._push_pending + if pending is not None: + self._record_push( + { + **pending, + "ended": clock(), + "outcome": "error", + "returncode": None, + "output": ( + "the anchor push thread did not end after its " + "time limit; its command may still be running" + ), + } + ) + self._check_push_error() + + def _record_push(self, line: dict) -> None: + """Claim one outcome under the push lock; a late worker cannot repeat it. + The lock covers writing and fsync, so a wedged filesystem can delay + supervision and retirement beyond the thread join's bound.""" + with self._pushing: + pending = self._push_pending + if pending is None or pending["anchor"] != line["anchor"]: + return + self._push_pending = None + self._last_push_outcome = line["outcome"] + if line["outcome"] != "success": + self._next_anchor = min(self._next_anchor, line["ended"] + ANCHOR_RETRY) + try: + path = Path(self.design["runs"]) / "anchor-pushes.jsonl" + with path.open("a") as handle: + handle.write(json.dumps(line) + "\n") + handle.flush() + os.fsync(handle.fileno()) + except OSError as error: + # A daemon's exception must reach the supervisor or final join. + self._push_error = self._push_error or error + return + if line["outcome"] != "success": + self.say( + f"the anchor push failed ({line['outcome']}, " + f"{line['returncode']}): {line['output']}" + ) + + def _push_anchor(self, head: str, anchor_time: float) -> None: + """Push one immutable snapshot and record its bounded command's outcome.""" + runs = Path(self.design["runs"]) + command = [str(x) for x in self.design["anchor_command"]] + command.append(str(runs / f"anchors.{head}.push-snapshot.jsonl")) + with self._pushing: + pending = self._push_pending + if pending is None or pending["anchor"] != head: + return + started = pending["started"] = clock() + try: + child = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + stdout, stderr = child.communicate(timeout=ANCHOR_PUSH_TIMEOUT) + outcome = "success" if child.returncode == 0 else "failed" + except subprocess.TimeoutExpired: + # The command is a session leader: git and ssh share its group. + # Killing just the command leaves its push running after timeout. + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + stdout, stderr = child.communicate(timeout=KILL_GRACE) + except subprocess.TimeoutExpired as error: + for pipe in (child.stdout, child.stderr): + if pipe is not None: + pipe.close() + stdout, stderr = error.output or b"", error.stderr or b"" + if isinstance(stdout, bytes): + stdout = stdout.decode(errors="replace") + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + stderr += "\npipes were held open after the kill" + outcome = "timeout" + code, text = child.returncode, (stdout + stderr)[-2000:] + except (OSError, subprocess.SubprocessError) as error: + code, outcome, text = None, "error", f"{type(error).__name__}: {error}" + ended = clock() + line = { + "anchor": head, + "anchor_time": anchor_time, + "started": started, + "ended": ended, + "outcome": outcome, + "returncode": code, + "output": text, + } + self._record_push(line) + + # The main loop's work. + + def watch(self, anchor: bool = True) -> None: + """The ceiling's events, an interrupt, a dead sampler, resumes after + operator pauses, late spend and the anchors.""" + crossed = self.ceiling.crossed_at + if crossed is not None and not self._ceiling_written: + self._ceiling_written = True + reason = ( + f"total spend ${self.ceiling.spent_usd:.2f} reached the hard ceiling " + f"${self.ceiling.budget_usd:.2f}" + ) + self.stop_all(reason, "ceiling", kind="ceiling", at=crossed) + if self.signalled is not None and not self._interrupted: + self._interrupted = True + self.say( + f"signal {self.signalled}: stopping every model's lanes; " + "the conversations under way finish" + ) + self.stop_all(f"operator stop (signal {self.signalled})", "interrupt") + dead = self.host.dead() if self.host is not None else None + if dead and not self._sampler_stopped: + self._sampler_stopped = True + self.stop_all(f"the host log stopped: {dead}", "sampler") + for lanes in self.lanes.values(): + path = lanes.folder / "RESUME.json" + if not path.exists(): + continue + try: + note = json.loads(path.read_text()) + except (OSError, ValueError): + continue # still being written + stamp = f"{time.time():.3f}" + if lanes.resume( + note.get("note") or "resumed", float(note.get("time") or 0.0) + ): + path.rename(lanes.folder / f"resume-{stamp}.json") + else: + # Written before the pause it would end, or with no pause. + path.rename(lanes.folder / f"resume-ignored-{stamp}.json") + self.say(f"{lanes.model}: a resume written before any pause, ignored") + for lanes in self.lanes.values(): + lanes.log_late() + if anchor and self.anchor_due(): + self.anchor() + + def execute(self) -> None: + runs = Path(self.design["runs"]) + starting, threads = {}, [] + setup_open = set() + try: + self.host = self.host_factory(runs / "host") + ready, failures = {}, {} + startup = clock() + for lanes in self.lanes.values(): + lanes.event( + "setup-start", + "run startup", + at=startup, + phase="startup", + start=startup, + ) + setup_open.add(lanes.model) + for model, lanes in self.lanes.items(): + self.watch(anchor=False) + lanes.completion = self.completion_for(lanes.profiles["pe3"]) + stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) + lanes.manifest_sha256 = write_json( + lanes.folder / f"manifest-{stamp}.json", lanes.manifest(), indent=2 + ) + # A model whose lanes never decide still has both logs. + for name in ("run-log.jsonl", "admission-log.jsonl"): + (lanes.folder / name).touch() + + def start_pool(model, lanes): + try: + ready[model] = lanes.ensure_pool(startup=True) + except Exception as error: + failures[model] = error + self._failed.set() + + for model, lanes in self.lanes.items(): + self.watch() + if lanes.halted(): + continue + starting[model] = threading.Thread( + target=start_pool, args=(model, lanes), daemon=True + ) + starting[model].start() + while any(t.is_alive() for t in starting.values()): + self.watch() + for thread in starting.values(): + thread.join(timeout=0.5 / len(starting)) + self.watch() + for model, lanes in self.lanes.items(): + outcome = ( + "failed" + if model in failures + else "stopped" + if lanes.halted() + else "ready" + if ready.get(model) + else "failed" + ) + lanes.event( + "setup-end", + "run startup", + phase="startup", + start=startup, + outcome=outcome, + ) + setup_open.discard(model) + if outcome == "failed" and model not in failures: + lanes.stop("the engine pool could not be started", "pool") + if failures: + raise next(iter(failures.values())) + threads = [] + for model, lanes in self.lanes.items(): + for lane in (1, 2): + self.watch() + with lanes.cond: + if lanes.halted() or self.signalled is not None: + continue + thread = threading.Thread( + target=lanes.lane, + args=(lane,), + name=f"{model}-{lane}", + daemon=True, + ) + thread.start() + threads.append(thread) + while any(t.is_alive() for t in threads): + self.watch() + for t in threads: + t.join(timeout=0.5 / len(threads)) + self.watch() + except Exception as error: + self._failed.set() + # Supervisors finish their bounded attempts and recovery before + # the ledger, host and final anchor can close underneath them. + for lanes in self.lanes.values(): + with lanes.cond: + lanes.cond.notify_all() + for thread in [*starting.values(), *threads]: + while thread.is_alive(): + thread.join(timeout=WAIT_SECONDS) + try: + self.watch(anchor=False) + except Exception: + self.say(traceback.format_exc()) + for model in setup_open: + try: + self.lanes[model].event( + "setup-end", + "run startup", + phase="startup", + start=startup, + outcome="failed", + ) + except Exception: + self.say(traceback.format_exc()) + self.stop_all( + f"runner error: {type(error).__name__}: {error}", "runner-error" + ) + raise + finally: + self.finish() + + def finish(self) -> None: + """One join deadline, then close pools and host before closing the + ledger. Its final late costs, crossings and signals precede the last + anchor. An unresolved request makes future spend unknowable: record + it explicitly, with a runner-error stop, rather than claim it settled. + Retry the final push up to three times per set of heads, waiting + ANCHOR_RETRY after each failed try ends. Then ignore signals and check + once more for a first signal needing coverage. Signals during the + resulting re-anchoring are ignored, including a second interrupt.""" + errors = [] + + def attempt(work): + try: + work() + return True + except Exception as error: + errors.append(error) + self._failed.set() + self.say(traceback.format_exc()) + self.stop_all( + f"runner error during cleanup: {type(error).__name__}: {error}", + "runner-error", + ) + return False + + until = time.monotonic() + LATE_JOIN + for lanes in self.lanes.values(): + for item in list(lanes.abandoned): + while item["thread"].is_alive() and time.monotonic() < until: + item["thread"].join( + timeout=min(WAIT_SECONDS, max(0.0, until - time.monotonic())) + ) + attempt(lambda: self.watch(anchor=False)) + for lanes in self.lanes.values(): + if lanes.pool is not None: + attempt(lanes.pool.close) + if self.host is not None: + attempt(self.host.close) + pending = [] + for lanes in self.lanes.values(): + with lanes.cond: + pending.extend({**item, "lanes": lanes} for item in lanes.abandoned) + unresolved = self.ceiling.close(pending) + for lanes in self.lanes.values(): + attempt(lambda: lanes.log_late(final=True)) + for item in unresolved: + attempt( + lambda: item["lanes"].event( + "late-unresolved", + "request unresolved when the ledger closed; " + "future spend is unrecorded", + household=item["household"], + arm=item["arm"], + attempt=item["attempt"], + outcome="unresolved-at-ledger-close", + counted_usd=item["spend"].spent_usd + item["spend"].late_usd, + ) + ) + if unresolved: + self._failed.set() + self.stop_all( + f"{len(unresolved)} abandoned requests unresolved at ledger close; " + "future spend is unrecorded", + "runner-error", + ) + # A first signal can also arrive while the final push is running. + while True: + attempt(lambda: self.watch(anchor=False)) + for lanes in self.lanes.values(): + stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) + attempt( + lambda: write_json( + lanes.folder / f"run-summary-{stamp}.json", + lanes.summary(), + indent=2, + ) + ) + for push_try in range(FINAL_PUSH_TRIES): + if push_try: + time.sleep(ANCHOR_RETRY) + if not attempt(lambda: self.anchor(wait=True)): + attempt(lambda: self.watch(anchor=False)) + # The error handler appended stops after that snapshot. Keep + # them covered locally even if pushing cannot be recorded. + try: + self.anchor(push=False) + except Exception as error: + errors.append(error) + self.say(traceback.format_exc()) + break + if self._last_push_outcome == "success": + break + if self.signalled is not None and not self._interrupted: + continue + self._ignore_signals() + if self.signalled is not None and not self._interrupted: + continue + break + if errors: + raise errors[0] + + def _ignore_signals(self) -> None: + """Close signal delivery before the last check; the process is ending.""" + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + + def interrupt(self, signum, frame) -> None: + """The signal handler. It only notes the signal; the main loop stops + every model's lanes. A second signal exits at once while this handler + is installed; cleanup ignores signals before its final check.""" + self.interrupts += 1 + if self.interrupts > 1: + os._exit(130) + self.signalled = signum + + +# ---------------------------------------------------------------- before a run + + +def host_power() -> tuple[bool, bool]: + """(on AC power, lid open).""" + batt = subprocess.run( + ["pmset", "-g", "batt"], capture_output=True, text=True + ).stdout + lid = subprocess.run( + ["ioreg", "-r", "-k", "AppleClamshellState", "-d", "1"], + capture_output=True, + text=True, + ).stdout + return "'AC Power'" in batt, '"AppleClamshellState" = No' in lid + + +def preflight(design: dict, models: tuple[str, ...]) -> list[str]: + """What stops the run from starting.""" + missing = [k for k in REQUIRED if k not in design] + if missing: + return [f"the design has no {', '.join(missing)}"] + problems = [] + from confirmatory_no_tools import ( + BOARD_INPUT_ERRORS, + _board_input, + listed_is_dir, + saved_children, + saved_exists, + saved_json_paths, + stage_problems, + ) + + problems.extend(stage_problems(design)) + try: + no_tools_root = Path(design["no_tools"]["runs"]).resolve() + calculator_root = Path(design["runs"]).resolve() + if no_tools_root.is_relative_to( + calculator_root + ) or calculator_root.is_relative_to(no_tools_root): + problems.append("the no-tools and calculator runs roots overlap") + except (KeyError, TypeError, OSError, ValueError, RuntimeError) as error: + problems.append(f"the no-tools runs root is invalid: {error}") + command = design["anchor_command"] + if ( + not isinstance(command, list) + or not command + or shutil.which(str(command[0])) is None + ): + problems.append(f"the anchor command {command!r} is not a command here") + if design.get("stated_engine_overrides"): + problems.append("the confirmatory run states the board's engine: no overrides") + preregistration = Path(design["preregistration"]) + try: + if ( + _board_input(preregistration, lambda: sha256_file(preregistration)) + != design["frozen_sha256"] + ): + problems.append("the frozen copy does not hash to frozen_sha256") + except ValueError as error: + problems.append(str(error)) + pool = design["eligible_pool"] + pool_path = Path(pool["path"]) + try: + pool_bytes = pool_path.read_bytes() + if sha256_bytes(pool_bytes) != pool["sha256"]: + problems.append(f"{pool_path}: the eligible pool does not hash as frozen") + else: + ids = [e["id"] for e in json.loads(pool_bytes.decode("utf-8"))] + if draw(master_seed(design["frozen_sha256"]), ids) != list( + design["draw_order"] + ): + problems.append( + "the draw, recomputed from the stream 'draw', is not the design's" + ) + except BOARD_INPUT_ERRORS as error: + problems.append(f"{pool_path}: cannot read the eligible pool: {error}") + head, dirty = pilot._git("rev-parse", "HEAD"), pilot._git("status", "--porcelain") + if head != design["run_commit"] or dirty: + problems.append( + f"the runner is at {head[:12]} (dirty {bool(dirty)}), " + "not the run commit, clean" + ) + changed = subprocess.run( + [ + "git", + "-C", + str(pilot.ROOT), + "diff", + "--quiet", + TREATMENT_COMMIT, + "HEAD", + "--", + *TREATMENT_PATHS, + ] + ).returncode + if changed: + problems.append(f"the treatment's code differs from {TREATMENT_COMMIT[:12]}'s") + import policybench + + places = [Path(p).resolve() for p in policybench.__path__] + if pilot.ROOT != ROOT or not all(p.is_relative_to(ROOT) for p in places): + problems.append( + f"the harness ({pilot.__file__}) or policybench ({places}) is not " + "this checkout's" + ) + harness = design["treatment_harness"] + harness_head = subprocess.run( + ["git", "-C", harness, "rev-parse", "HEAD"], capture_output=True, text=True + ).stdout.strip() + if harness_head != TREATMENT_COMMIT: + problems.append( + f"the treatment harness is at {harness_head[:12]}, " + f"not {TREATMENT_COMMIT[:12]}" + ) + board = Path(design["board"]) + try: + engine = pilot.board_engine(board) + except BOARD_INPUT_ERRORS as error: + problems.append( + f"{board / 'reference_outputs.csv.meta.json'}: " + f"cannot read the board's engine: {error}" + ) + else: + if f"policyengine-us {engine}" != design["engine"]: + problems.append( + f"the board's engine {engine} is not the design's {design['engine']}" + ) + installed = pilot._versions()["policyengine-us"] + if installed != engine: + problems.append(f"policyengine-us {installed} is installed, not {engine}") + if set(design["arms"]) != set(ARMS) or set(design["frontier_arms"]) != set( + FRONTIER_ARMS + ): + problems.append("the design's arms are not the registered ones") + for name in ("arms", "frontier_arms"): + if len(set(design[name].values())) != len(design[name]): + problems.append(f"two {name} share a folder") + + def household_paths(stage): + # Check the parent inventory even when its next component is literal. + saved_children(stage) + return saved_json_paths(stage / "households") + + for model in models: + if model not in design["caps"] or model not in design["reserves"]: + problems.append(f"{model}: no cap or reserve") + for label in ARMS if model in CHEAP else FRONTIER_ARMS: + try: + arm_profile(model, label) + except (ValueError, SystemExit) as error: + problems.append(str(error)) + key = pilot.provider_key(pilot.model_profile(model).model_id) + if not os.environ.get(key): + problems.append(f"{model}: {key} is not set") + folder = Path(design["runs"]) / model + try: + holds_records = saved_exists(folder / "run-log.jsonl") + if not holds_records: + holds_records = any( + household_paths(stage) + for arm in saved_children(folder) + if listed_is_dir(arm) + for stage in saved_children(arm) + if listed_is_dir(stage) + ) + if holds_records: + problems.append( + f"{folder} already holds records; a run is never resumed" + ) + except ValueError as error: + problems.append(str(error)) + try: + if saved_exists(Path(design["runs"]) / "host" / "host-log.jsonl"): + problems.append("the runs folder already holds a host log") + except ValueError as error: + problems.append(str(error)) + ac, lid = host_power() + if not (ac and lid): + problems.append( + f"the host is not on AC power with the lid open (AC {ac}, lid open {lid})" + ) + return problems + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run", help="run the confirmatory stage") + run.add_argument("design", type=Path) + run.add_argument( + "--workers", type=int, help="engine workers per model: the design's alone" + ) + run.add_argument("--dry-run", action="store_true", help="check and print the plan") + resume = sub.add_parser("resume", help="end an operator pause") + resume.add_argument("design", type=Path) + resume.add_argument("model") + resume.add_argument("--note", default="the account was fixed") + args = parser.parse_args(argv) + design = json.loads(args.design.read_text()) + if args.command == "resume": + folder = Path(design["runs"]) / args.model + write_json(folder / "RESUME.json", {"time": time.time(), "note": args.note}) + print(f"{args.model}: resume requested; the runner resumes within a second") + return 0 + if args.workers is not None and args.workers != design.get("workers"): + print( + f"The run cannot start: --workers {args.workers}, and the design " + f"runs {design.get('workers')} engine workers per model." + ) + return 1 + models = CHEAP + FRONTIER + problems = preflight(design, models) + from confirmatory_no_tools import BOARD_INPUT_ERRORS + + board = Path(design["board"]) + try: + pilot._require_engine(board) + except BOARD_INPUT_ERRORS as error: + problems.append( + f"{board / 'reference_outputs.csv.meta.json'}: " + f"the board's engine check failed: {error}" + ) + if problems: + print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) + return 1 + plan = Run(design) + if args.dry_run: + for model, lanes in plan.lanes.items(): + sizes = [len(lanes.households(lane)) for lane in (1, 2)] + print( + f"{model}: arms {', '.join(lanes.labels)}; lanes {sizes[0]} and " + f"{sizes[1]} households; cap ${lanes.cap:.2f}, " + f"reserve ${lanes.reserve:.2f}" + ) + print( + f"no scored output: {sorted(plan.no_output)}; hard ceiling " + f"${plan.ceiling.budget_usd:.2f} " + f"(${plan.ceiling.spent_usd:.2f} spent before)" + ) + return 0 + signal.signal(signal.SIGINT, plan.interrupt) + signal.signal(signal.SIGTERM, plan.interrupt) + plan.execute() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/household_set.py b/scripts/household_set.py new file mode 100644 index 00000000..ff385040 --- /dev/null +++ b/scripts/household_set.py @@ -0,0 +1,560 @@ +"""Build a fresh US household set for PolicyBench, in the frozen board's layout. + +Max, 2026-10-22: v2's head-to-head runs on fresh households from the latest +Microcosm build, with references from the latest policyengine-us, served as +one certified pair by the installed policyengine.py bundle. The run goes +slowly, in small batches with AI judges between them, because new microdata +can reach engine paths the frozen 100 households never did. + + household_set.py build --n 100 --seed 20261022 --batch-size 10 --out DIR + household_set.py references --board DIR --out FILE + household_set.py check-frozen --out FILE + +``build`` samples households from the dataset the installed policyengine.py +bundle certifies (:func:`policybench.scenarios.generate_scenarios`, the board's +own sampler and eligibility rules), renames them ``mc_000`` onward so they +never share an id with the frozen board, computes every output on the +installed policyengine-us with the reference conventions +(reference_audit/2026-09-28/fixes/latest_final.py), and writes a board +directory the tool pilot reads with ``--board``: + + scenarios.csv (+ .meta.json) the households, as the frozen board + reference_outputs.csv (+ .meta.json) value and impact_weight per output + reference_exclusions.json an empty exclusion record + batches.json the households in batches, in draw order + +The reference sidecar has the frozen board's shape: one ``engine_upgrade`` +revision naming the engine and pinning every fix module by sha256, so the +pilot's pin check reads either board the same way. + +``references`` recomputes a board's references and reports any output that +differs (the determinism check). ``check-frozen`` is the differential check +of the reference path: on the frozen board's own households it must reproduce +every scored reference, which holds only on the engine those references were +built with (policyengine-us 2.41.0). + +The set is not public. It stays outside the repository until a release +publishes it, so a web-search arm cannot find its answers. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import datetime +import hashlib +import importlib.util +import json +import sys +from pathlib import Path + +import pandas as pd + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +FROZEN_BOARD = ( + ROOT + / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +FIX_MODULE = ROOT / "reference_audit/2026-09-28/fixes/latest_final.py" +YEAR = 2026 +# The v1.1 reference run renamed this input for policyengine-us 1.755.4; the +# frozen households still carry the old name (reference_audit/2026-09-28/ +# scripts/sweep.py build_situation). Harmless for a household without it. +RENAME = {"partnership_se_income": "partnership_self_employment_net_earnings"} +EXCLUSIONS_RULE = ( + "An output is excluded from scoring for every model when review finds its " + "reference wrong in law or undetermined by the stated facts. A fresh household " + "set starts with no exclusions; its batch judges propose them." +) + + +def sha256_bytes(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def sha256_file(path: Path) -> str: + return sha256_bytes(Path(path).read_bytes()) + + +def installed_engine() -> str: + from importlib import metadata + + return f"policyengine-us {metadata.version('policyengine-us')}" + + +def frozen_programs() -> list[str]: + meta = json.loads((FROZEN_BOARD / "reference_outputs.csv.meta.json").read_text()) + return list(meta["programs"]) + + +def frozen_fix_pins() -> list[dict]: + """The fix modules the frozen 2.41.0 references were built with.""" + meta = json.loads((FROZEN_BOARD / "reference_outputs.csv.meta.json").read_text()) + revisions = [ + r + for r in meta["revisions"] + if r.get("kind") == "engine_upgrade" + and r.get("engine_version") == "policyengine-us 2.41.0" + ] + if len(revisions) != 1: + raise SystemExit("the frozen board has no single 2.41.0 engine_upgrade") + return revisions[0]["fix_modules"] + + +def fix_module_pins(fix_module: Path = FIX_MODULE) -> list[dict]: + """Each module the reform composes, at its current bytes. + + The set of modules is the frozen board's; each must exist and match the + frozen pin, so a household set's references follow exactly the frozen + conventions. Changing a convention for a new engine is an engine-upgrade + audit, which pins its own modules. + """ + pins, problems = [], [] + for pin in frozen_fix_pins(): + path = ROOT / pin["path"] + if not path.exists(): + problems.append(f"{pin['path']} is missing") + continue + digest = sha256_file(path) + if digest != pin["sha256"]: + problems.append(f"{pin['path']} is not the frozen {pin['sha256'][:12]}") + pins.append({"module": pin["module"], "path": pin["path"], "sha256": digest}) + if problems: + raise SystemExit("; ".join(problems)) + return pins + + +def load_reform(fix_module: Path = FIX_MODULE): + spec = importlib.util.spec_from_file_location("household_set_fixes", fix_module) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module.reform + + +def reference_system(fix_module: Path = FIX_MODULE): + from policyengine_us import CountryTaxBenefitSystem + + return CountryTaxBenefitSystem(reform=load_reform(fix_module)) + + +def situation_for(scenario) -> dict: + situation = scenario.to_pe_household() + for person in situation["people"].values(): + for old, new in RENAME.items(): + if old in person: + person[new] = person.pop(old) + return situation + + +def compute_references(scenarios, programs: list[str], system) -> pd.DataFrame: + """Every output of every household on ``system``: the ground-truth path + (policybench.ground_truth._calculate_ground_truth_us_scalar) with the + reference conventions.""" + from policyengine_us import Simulation + + from policybench.ground_truth import ( + _extract_person_impact_weight, + _extract_person_value, + _impact_weight_variable_for_output, + _pe_variable_for_output, + ) + from policybench.spec import expand_programs_for_scenario + + rows = [] + for scenario in scenarios: + simulation = Simulation( + tax_benefit_system=system, situation=situation_for(scenario) + ) + for variable in expand_programs_for_scenario(programs, scenario): + result = simulation.calculate(_pe_variable_for_output(variable, "us"), YEAR) + weight_variable = _impact_weight_variable_for_output(variable, "us") + weight = None + if weight_variable is not None: + weight = _extract_person_impact_weight( + result, + simulation.calculate(weight_variable, YEAR), + scenario, + variable, + ) + rows.append( + { + "scenario_id": scenario.id, + "variable": variable, + "value": float(_extract_person_value(result, scenario, variable)), + "impact_weight": weight, + } + ) + return pd.DataFrame( + rows, columns=["scenario_id", "variable", "value", "impact_weight"] + ) + + +def rename_scenarios(scenarios, prefix: str) -> list: + """The sampled households under fresh ids, in draw order.""" + return [ + dataclasses.replace(scenario, id=f"{prefix}_{index:03d}") + for index, scenario in enumerate(scenarios) + ] + + +def split_batches(ids: list[str], size: int) -> list[list[str]]: + if size < 1: + raise ValueError("batch size must be at least 1") + return [ids[start : start + size] for start in range(0, len(ids), size)] + + +def load_scenarios(board: Path) -> list: + from policybench.scenarios import scenario_from_dict + + frame = pd.read_csv(Path(board) / "scenarios.csv") + return [scenario_from_dict(json.loads(text)) for text in frame["scenario_json"]] + + +def _utc_now() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def explicit_dataset(repo: str, filename: str, revision: str) -> tuple[Path, dict]: + """A published dataset named by repo, file and tag (not the bundle's), and + what its release manifest says about the engine it was built for.""" + from huggingface_hub import hf_hub_download + from huggingface_hub.errors import EntryNotFoundError + + path = Path(hf_hub_download(repo, filename, revision=revision, repo_type="dataset")) + record = { + "uri": f"hf://datasets/{repo}/{filename}@{revision}", + "release_manifest": None, + } + try: + manifest_path = hf_hub_download( + repo, + f"releases/{revision}/release_manifest.json", + revision=revision, + repo_type="dataset", + ) + except EntryNotFoundError: + return path, record + manifest = json.loads(Path(manifest_path).read_text()) + build = manifest.get("build") or {} + record["release_manifest"] = { + "sha256": sha256_file(Path(manifest_path)), + "built_with_model_package": build.get("built_with_model_package"), + "built_with_core_package": build.get("built_with_core_package"), + "compatible_model_packages": manifest.get("compatible_model_packages"), + } + return path, record + + +# How a dataset's stated filing status reads as a scenario's. +STATED_FILING_STATUS = { + "SINGLE": "single", + "JOINT": "joint", + "HEAD_OF_HOUSEHOLD": "head_of_household", + "SEPARATE": "separate", + "SURVIVING_SPOUSE": "surviving_spouse", +} + + +def stated_filing_status(path: Path) -> dict[int, str] | None: + """Each tax unit's filing_status_input from a Populace H5 (pandas table + format), or None when the file states none. Engines after 2.2.1 no longer + read the column; they derive filing status from the household.""" + import h5py + + if not h5py.is_hdf5(path): + return None + with h5py.File(path, "r") as handle: + if "tax_unit" not in handle or "table" not in handle["tax_unit"]: + return None + table = handle["tax_unit"]["table"][:] + columns = {} + for field in table.dtype.names: + sub = table.dtype[field] + for name in sub.names or (field,): + columns[name] = table[field][name] if sub.names else table[field] + if "filing_status_input" not in columns or "tax_unit_id" not in columns: + return None + return { + int(unit): (value.decode() if isinstance(value, bytes) else str(value)) + for unit, value in zip(columns["tax_unit_id"], columns["filing_status_input"]) + } + + +def filing_status_mismatches(scenarios, stated: dict[int, str] | None) -> list[dict]: + """Households whose scenario filing status (the engine's) is not the one + the dataset states.""" + if stated is None: + return [] + found = [] + for scenario in scenarios: + unit = scenario.metadata.get("tax_unit_id") + said = stated.get(int(unit)) if unit is not None else None + if said is None: + continue + if STATED_FILING_STATUS.get(said, said.lower()) != scenario.filing_status: + found.append( + { + "scenario_id": scenario.id, + "tax_unit_id": int(unit), + "stated": said, + "engine": scenario.filing_status, + } + ) + return found + + +def command_build(args) -> None: + from policybench.policyengine_runtime import ( + materialize_certified_us_dataset, + policyengine_bundles_for_countries, + ) + from policybench.scenarios import generate_scenarios, scenario_manifest + + out = Path(args.out) + if out.exists() and any(out.iterdir()): + raise SystemExit(f"{out} is not empty; a household set is written once") + named = [args.dataset_repo, args.dataset_file, args.dataset_revision] + if any(named) and not all(named): + raise SystemExit( + "name a dataset by --dataset-repo, --dataset-file and --dataset-revision" + ) + engine = installed_engine() + pins = fix_module_pins(Path(args.fix_module)) + programs = frozen_programs() + bundles = policyengine_bundles_for_countries(["us"]) + bundle = bundles["us"] + if all(named): + # Max, 2026-10-22: "we can still test before that but i'd like v2 to be + # clean". A dataset other than the bundle's makes a test set only. + dataset_path, dataset_record = explicit_dataset(*named) + dataset_label = f"{Path(args.dataset_file).stem}@{args.dataset_revision}" + certified_pair = False + else: + dataset_path = Path(materialize_certified_us_dataset()) + dataset_record = {"uri": bundle.get("default_dataset_uri")} + dataset_label = None + certified = bundle.get("certified_data_artifact_sha256") + if certified and certified != sha256_file(dataset_path): + raise SystemExit( + f"{dataset_path} hashes to {sha256_file(dataset_path)}, not the " + f"bundle's {certified}" + ) + # The bundle certifies its data for its own model pin; the pair is + # certified only when that pin is the installed engine. + certified_pair = bundle.get("bundled_model_version") == bundle.get( + "model_version" + ) + if not certified_pair and args.purpose != "test": + raise SystemExit( + "this dataset and engine are not a certified pair: build it with " + "--purpose test, or install a bundle that certifies the engine" + ) + dataset_sha256 = sha256_file(dataset_path) + + sampled = generate_scenarios( + n=args.n, + seed=args.seed, + country="us", + dataset=str(dataset_path) if all(named) else None, + dataset_label=dataset_label, + ) + scenarios = rename_scenarios(sampled, args.id_prefix) + mismatches = filing_status_mismatches(scenarios, stated_filing_status(dataset_path)) + references = compute_references( + scenarios, programs, reference_system(Path(args.fix_module)) + ) + + out.mkdir(parents=True, exist_ok=True) + manifest = scenario_manifest(scenarios) + manifest.to_csv(out / "scenarios.csv", index=False) + references.to_csv(out / "reference_outputs.csv", index=False) + batches = split_batches([s.id for s in scenarios], args.batch_size) + (out / "batches.json").write_text( + json.dumps({"batch_size": args.batch_size, "batches": batches}, indent=1) + "\n" + ) + (out / "reference_exclusions.json").write_text( + json.dumps( + { + "schema_version": 1, + "rule": EXCLUSIONS_RULE, + "derivation": f"Built {_utc_now()[:10]} with no exclusions.", + "exclusions": [], + }, + indent=1, + ) + + "\n" + ) + generated = _utc_now() + common = { + "country": "us", + "metadata_version": 1, + "board_kind": "household_set", + "purpose": args.purpose, + "certified_pair": certified_pair, + "generated_at_utc": generated, + "num_scenarios": len(scenarios), + "requested_num_scenarios": args.n, + "seed": args.seed, + "id_prefix": args.id_prefix, + "program_set": "headline", + "programs": programs, + "policyengine_bundles": bundles, + "dataset": { + **dataset_record, + "path_name": dataset_path.name, + "sha256": dataset_sha256, + "bundle_default_dataset_uri": bundle.get("default_dataset_uri"), + }, + # The dataset states a filing status the engine no longer reads; these + # households' scenarios carry the engine's, as their references do. + "filing_status_mismatches": mismatches, + "source_households": [ + {"scenario_id": s.id, **{k: s.metadata[k] for k in sorted(s.metadata)}} + for s in scenarios + ], + } + (out / "scenarios.csv.meta.json").write_text( + json.dumps( + { + **common, + "task": "scenario_manifest", + "scenarios_csv_sha256": sha256_file(out / "scenarios.csv"), + }, + indent=1, + ) + + "\n" + ) + (out / "reference_outputs.csv.meta.json").write_text( + json.dumps( + { + **common, + "task": "reference_outputs", + "reference_csv_sha256": sha256_file(out / "reference_outputs.csv"), + "scenarios_csv_sha256": sha256_file(out / "scenarios.csv"), + "revisions": [ + { + "kind": "engine_upgrade", + "engine_version": engine, + "date": generated[:10], + "builder": "scripts/household_set.py build", + "fix_module": { + "path": str( + Path(args.fix_module).resolve().relative_to(ROOT) + ), + "sha256": sha256_file(Path(args.fix_module)), + }, + "fix_modules": pins, + "outputs": len(references), + } + ], + }, + indent=1, + ) + + "\n" + ) + print( + f"{len(scenarios)} households, {len(references)} outputs on {engine}; " + f"{len(batches)} batches of up to {args.batch_size} in {out}; purpose " + f"{args.purpose}, certified pair {certified_pair}; " + f"{len(mismatches)} filing status mismatches" + ) + + +def differences( + expected: pd.DataFrame, computed: pd.DataFrame, tolerance: float +) -> pd.DataFrame: + """Outputs whose computed value is missing or differs beyond ``tolerance``.""" + merged = expected.merge( + computed[["scenario_id", "variable", "value"]], + on=["scenario_id", "variable"], + how="left", + suffixes=("", "_computed"), + ) + merged["delta"] = (merged["value_computed"] - merged["value"]).abs() + return merged[merged["value_computed"].isna() | (merged["delta"] > tolerance)] + + +def command_references(args) -> None: + board = Path(args.board) + expected = pd.read_csv(board / "reference_outputs.csv") + meta = json.loads((board / "reference_outputs.csv.meta.json").read_text()) + computed = compute_references( + load_scenarios(board), meta["programs"], reference_system(Path(args.fix_module)) + ) + diff = differences(expected, computed, args.tolerance) + computed.to_csv(args.out, index=False) + print( + f"{len(computed)} outputs recomputed on {installed_engine()}; " + f"{len(diff)} differ" + ) + if len(diff): + print(diff.to_string(index=False)) + raise SystemExit(1) + + +def command_check_frozen(args) -> None: + """The reference path on the frozen board must reproduce every scored output.""" + from policybench.reference_exclusions import ( + load_reference_exclusions, + split_reference, + ) + + engine = installed_engine() + if engine != "policyengine-us 2.41.0": + raise SystemExit(f"{engine} is installed; the frozen references are 2.41.0's") + fix_module_pins(FIX_MODULE) + reference = pd.read_csv(FROZEN_BOARD / "reference_outputs.csv") + scored, _ = split_reference(reference, load_reference_exclusions(FROZEN_BOARD)) + computed = compute_references( + load_scenarios(FROZEN_BOARD), frozen_programs(), reference_system(FIX_MODULE) + ) + computed.to_csv(args.out, index=False) + diff = differences(scored, computed, args.tolerance) + print( + f"{len(scored)} scored outputs; {len(scored) - len(diff)} reproduced within " + f"{args.tolerance} on {engine}" + ) + if len(diff): + print(diff.to_string(index=False)) + raise SystemExit(1) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + build = sub.add_parser("build", help="sample a household set and its references") + build.add_argument("--n", type=int, required=True) + build.add_argument("--seed", type=int, required=True) + build.add_argument("--batch-size", type=int, default=10) + build.add_argument("--id-prefix", default="mc") + build.add_argument("--fix-module", default=str(FIX_MODULE)) + build.add_argument("--out", required=True) + build.add_argument( + "--purpose", + choices=["v2", "test"], + default="v2", + help="v2 needs a certified dataset-engine pair; test takes any", + ) + build.add_argument("--dataset-repo", help="a published dataset, for a test set") + build.add_argument("--dataset-file") + build.add_argument("--dataset-revision") + build.set_defaults(func=command_build) + refs = sub.add_parser("references", help="recompute a board's references") + refs.add_argument("--board", required=True) + refs.add_argument("--fix-module", default=str(FIX_MODULE)) + refs.add_argument("--tolerance", type=float, default=1e-6) + refs.add_argument("--out", required=True) + refs.set_defaults(func=command_references) + frozen = sub.add_parser("check-frozen", help="reproduce the frozen references") + frozen.add_argument("--tolerance", type=float, default=1e-6) + frozen.add_argument("--out", required=True) + frozen.set_defaults(func=command_check_frozen) + args = parser.parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_reference_adversary_claude.sh b/scripts/run_reference_adversary_claude.sh index 69818c67..5c680040 100755 --- a/scripts/run_reference_adversary_claude.sh +++ b/scripts/run_reference_adversary_claude.sh @@ -58,6 +58,17 @@ # judge runs. A call the API refuses with 401, 403 or 429 stops the run: no # further judge starts, and the run resumes where it stopped. # +# API key (Max, 2026-10-22: "in case youre running any claude api stuff please +# use the key on my clipboard"): JUDGE_API_KEY_SECRET names an agent-secret +# entry (account JUDGE_API_KEY_ACCOUNT, default $USER; AUDIT_SECRET_BIN +# replaces agent-secret) holding an Anthropic API key. The judges then bill +# that key and nothing else: it cannot be combined with a lane, a lane token +# or the desktop login; every claude call runs in a fresh empty config +# directory, removed when the run ends; the key joins only the claude calls' +# allowlisted environment; and `claude auth status` must report a first-party +# api_key login in that directory. The sidecars record the mode and the +# secret's name, never the key. +# # Provenance: beside stage1.json and verdict.json the runner writes # stage1.meta.json and verdict.meta.json with the runner, the stage, the model # requested and the models the CLI reports, the CLI version, the effort, the @@ -165,7 +176,34 @@ for base in {home, pwd.getpwuid(os.getuid()).pw_dir}: print("other") PY } -if [ "$DESKTOP_OPT_IN" = 1 ]; then +API_SECRET="${JUDGE_API_KEY_SECRET:-}" +API_KEY="" +if [ -n "$API_SECRET" ]; then + [ "$DESKTOP_OPT_IN" = 0 ] || { + echo "JUDGE_API_KEY_SECRET and JUDGE_ALLOW_DESKTOP_LOGIN=1 are exclusive; refusing to start" >&2 + exit 1 + } + [ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] || { + echo "JUDGE_API_KEY_SECRET takes no lane token: unset CLAUDE_CODE_OAUTH_TOKEN; refusing to start" >&2 + exit 1 + } + [ -z "${AUDIT_ACCOUNT:-}" ] || { + echo "JUDGE_API_KEY_SECRET bills a key, not an account: unset AUDIT_ACCOUNT; refusing to start" >&2 + exit 1 + } + [ -z "${CLAUDE_CONFIG_DIR:-}" ] || { + echo "JUDGE_API_KEY_SECRET runs in its own empty config directory: unset CLAUDE_CONFIG_DIR; refusing to start" >&2 + exit 1 + } + API_KEY=$("${AUDIT_SECRET_BIN:-agent-secret}" get "$API_SECRET" "${JUDGE_API_KEY_ACCOUNT:-$USER}" 2>/dev/null) + [ -n "$API_KEY" ] || { + echo "could not read the API key $API_SECRET; refusing to start" >&2 + exit 1 + } + CONFIG_DIR=$(mktemp -d "${TMPDIR:-/tmp}/pb-adversary-config.XXXXXX") || exit 1 + CONFIG_DIR="$(cd "$CONFIG_DIR" && pwd -P)" + trap 'rm -rf "$CONFIG_DIR"' EXIT +elif [ "$DESKTOP_OPT_IN" = 1 ]; then [ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] || { echo "JUDGE_ALLOW_DESKTOP_LOGIN=1 takes no lane token: unset CLAUDE_CODE_OAUTH_TOKEN; refusing to start" >&2 exit 1 @@ -218,6 +256,7 @@ while IFS= read -r line; do [ -n "$line" ] && CHILD_ENV+=("$line"); done <&2 + --desktop-opt-in "$DESKTOP_OPT_IN" --api-key-secret "$API_SECRET") || { + echo "refusing to start: the judges must bill a lane's subscription login, the opted-in desktop login or the declared API key" >&2 exit 1 } DECLARED="${AUDIT_ACCOUNT:-}" [ "$DESKTOP_OPT_IN" = 1 ] && DECLARED="$DESKTOP_DECLARED" +[ -n "$API_SECRET" ] && DECLARED="API key $API_SECRET (JUDGE_API_KEY_SECRET, Max 2026-10-22)" CLI_VERSION=$(cd / && claude_child --version /dev/null | head -n 1) valid() { [ -s "$2" ] && adversary validate --schema "$1" --file "$2" >/dev/null 2>&1; } diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 97d880e7..f09e0b6f 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -48,9 +48,11 @@ import argparse import concurrent.futures +import contextlib import copy import decimal import difflib +import functools import hashlib import importlib.metadata import importlib.util @@ -152,13 +154,215 @@ # with no tools), so a prompt version other than the board's has its own # no-tools baseline. Repairs, retries and truncation then follow this pilot's # loop, not the board's; they matter only when a first answer does not parse. -ARMS = ("none", "pe", "web", "web+pe") +ARMS = ("none", "pe", "web", "web+pe", "pe2", "pe3") ARM_SUFFIX = { "none": "+no-tools", "pe": "+pe-tool", "web": "+web", "web+pe": "+web+pe-tool", + "pe2": "+pe-tool-v2", + "pe3": "+pe-tool-v3", } +# The calculator's features, each on or off, so a study can change one at a +# time (Max, 2026-10-23: "measuring this as an experiment could be +# publishable"; "do all these tests with a cheap model"): +# - names: a name written with spaces, or a benchmark output key, is read as +# its engine variable; +# - labels: a fact written under the label the prompt gives it (a prompt +# labels some facts in words, e.g. employment_income as "gross wages and +# salaries"; policybench.prompts.INPUT_LABEL_OVERRIDES) is read as its +# variable, and a fact the reference also sets under another name +# (policybench.scenarios.PE_INPUT_ALIASES: usual weekly hours) is set there +# too; +# - placement: an input set on another entity moves to the one unit holding +# its people; +# - overrides: a realized contribution is read as its _desired input, and an +# input that replaces a formula is flagged; +# - explain: `explain` traces up to three variables; +# - concise: an explanation is brief unless `detail` is "full"; +# - flat: the household is described as people with roles and facts +# (`household`), and the tool builds its units as the benchmark does. +# "pe2" is tool version 2 (names, placement, overrides and explain); "pe3" is +# all seven. Any other set +# runs as an arm named "pe:" plus its features joined by "+", e.g. +# "pe:names+flat". +NAMES, LABELS, PLACEMENT, OVERRIDES, EXPLAIN, CONCISE, FLAT = ( + "names", + "labels", + "placement", + "overrides", + "explain", + "concise", + "flat", +) +TOOL_FEATURES = (NAMES, LABELS, PLACEMENT, OVERRIDES, EXPLAIN, CONCISE, FLAT) +V2_FEATURES = frozenset({NAMES, PLACEMENT, OVERRIDES, EXPLAIN}) +FEATURES_BY_ARM = { + "pe": frozenset(), + "web+pe": frozenset(), + "pe2": V2_FEATURES, + "pe3": frozenset(TOOL_FEATURES), +} +_FEATURE_ARM = re.compile(r"^pe:([a-z]+(?:\+[a-z]+)*)$") + + +def arm_features(arm: str) -> frozenset | None: + """The calculator features an arm offers, or None for an arm without the + calculator. Refuses an unknown feature, or concise without explain.""" + if arm in FEATURES_BY_ARM: + return FEATURES_BY_ARM[arm] + match = _FEATURE_ARM.match(arm or "") + if match is None: + return None + features = frozenset(match.group(1).split("+")) + unknown = sorted(features - set(TOOL_FEATURES)) + if unknown: + raise SystemExit( + f"unknown tool features {unknown}; the features are {TOOL_FEATURES}" + ) + if CONCISE in features and EXPLAIN not in features: + raise SystemExit("the concise feature needs explain") + return features + + +def arm_suffix(arm: str) -> str: + if arm in ARM_SUFFIX: + return ARM_SUFFIX[arm] + features = arm_features(arm) + return f"+pe-tool[{'+'.join(f for f in TOOL_FEATURES if f in features)}]" + + +TOOL_NOTICE_V2 = ( + "\n\nYou also have a `calculate_household` function. It runs PolicyEngine US " + "(policyengine-us 2.41.0), an open-source rules engine for US federal and " + "state taxes and benefits, on a household situation you build from the facts " + "above. The facts above are labeled with its variable names, written with " + "spaces; it accepts them as written, and it accepts the output keys you are " + "asked for. Set every listed fact. Its result says which names it read, which " + "inputs it moved to the entity they belong to, and which inputs replace a " + "formula; ask it to `explain` up to three variables to see how it computed " + "them, with the parameters and laws it used. You may call it in up to 8 " + "turns. Then call `submit_outputs` exactly once with your final answers, as " + "instructed above." +) +JSON_TOOL_NOTICE_V2 = TOOL_NOTICE_V2.replace( + "Then call `submit_outputs` exactly once with your final answers, as " + "instructed above.", + "Then give your final answers as the single JSON object described above.", +) + + +def _listing(items: list[str]) -> str: + if len(items) <= 2: + return " and ".join(items) + return ", ".join(items[:-1]) + ", and " + items[-1] + + +def compose_tool_notice(features: frozenset, tool_contract: bool) -> str: + """The calculator notice for a set of features: the opening, a sentence + per feature, then the turn limit and the answer instruction. No feature + gives TOOL_NOTICE and the version 2 features TOOL_NOTICE_V2, as those arms + ran. TOOL_NOTICE_V2 says the facts are labeled with the engine's variable + names, which holds for most facts but not those the prompt labels in words + ("gross wages and salaries"); every other set says only what it reads.""" + if not features: + return TOOL_NOTICE if tool_contract else JSON_TOOL_NOTICE + if features == V2_FEATURES: + return TOOL_NOTICE_V2 if tool_contract else JSON_TOOL_NOTICE_V2 + what = ( + "household you describe" + if FLAT in features + else "household situation you build" + ) + text = ( + "\n\nYou also have a `calculate_household` function. It runs PolicyEngine US " + "(policyengine-us 2.41.0), an open-source rules engine for US federal and " + f"state taxes and benefits, on a {what} from the facts above." + ) + if FLAT in features: + text += ( + " Describe it as `household`: each person with their role (head, spouse " + "or dependent) and facts, and the facts about the household as a " + "whole; it builds the household's units as the facts imply." + ) + if LABELS in features and NAMES in features: + text += ( + " It accepts the facts above under the labels they have there, its " + "variable names written with spaces, and the output keys you are asked " + "for. Set every listed fact." + ) + elif LABELS in features: + text += ( + " It accepts the facts above under the labels they have there. Set " + "every listed fact." + ) + elif NAMES in features: + text += ( + " It accepts its variable names written with spaces, and the output " + "keys you are asked for. Set every listed fact." + ) + notes = [] + if NAMES in features or LABELS in features: + notes.append("which names it read") + if PLACEMENT in features: + notes.append("which inputs it moved to the entity they belong to") + if OVERRIDES in features: + notes.append("which inputs replace a formula") + explain = "" + if EXPLAIN in features: + explain = ( + "ask it to `explain` up to three variables to see in brief how it " + "computed them, with the parameters and laws it used, or the fuller " + 'trace with `detail` set to "full"' + if CONCISE in features + else "ask it to `explain` up to three variables to see how it computed " + "them, with the parameters and laws it used" + ) + if notes and explain: + text += f" Its result says {_listing(notes)}; {explain}." + elif notes: + text += f" Its result says {_listing(notes)}." + elif explain: + text += f" You may {explain}." + text += " You may call it in up to 8 turns. " + text += ( + "Then call `submit_outputs` exactly once with your final answers, as " + "instructed above." + if tool_contract + else "Then give your final answers as the single JSON object described above." + ) + return text + + +# The engine version the notices and the tool description state: the board's +# (set by command_run), so a household set on a newer engine says so. The +# frozen board's 2.41.0 leaves every text byte for byte as before. +_STATED_ENGINE = {"version": REFERENCE_ENGINE} + + +def set_stated_engine(version: str) -> None: + _STATED_ENGINE["version"] = version + + +@contextlib.contextmanager +def stating_engine(version: str): + """The texts as a run on `version` sent them, for a check that rebuilds a + notice; the stated engine is restored afterwards.""" + before = _STATED_ENGINE["version"] + set_stated_engine(version) + try: + yield + finally: + set_stated_engine(before) + + +def engine_text(text: str) -> str: + return text.replace( + f"policyengine-us {REFERENCE_ENGINE}", + f"policyengine-us {_STATED_ENGINE['version']}", + ) + + # A web arm's request timeout. A search-heavy turn can outlast the board's # (300 s for Gemini 4 Argon, the harness default for its card; its web-only arm # never finished scenario_005 within it in six attempts on 2026-10-21), so the @@ -452,12 +656,130 @@ def refuse_policyengine(label: str, value) -> None: ) -def calculate_tool_schema() -> dict: +CALCULATE_DESCRIPTION_V2 = CALCULATE_DESCRIPTION.replace( + "An unknown name, a variable set on the wrong entity, or a malformed " + "situation returns an error with suggestions instead of results.", + "A name may also be written with spaces (as the facts are labeled) or be one " + "of the output keys you are asked for; the result's `notes` say how each was " + "read. An input set on the wrong entity is moved to the one unit it belongs " + "to, and the notes say so; where there is no single such unit, or a name is " + "unknown, or the situation is malformed, it returns an error with " + "suggestions instead of results. Setting a variable the engine computes " + "replaces its formula, and the notes say so; a contribution set as the " + "amount contributed is read as the amount asked for before limits " + "(`..._desired`). `explain` lists up to 3 variables to trace: each comes " + "back with the steps that produced it and the parameters, with their " + "values and sources, that it used.", +) +MAX_EXPLAIN = 3 + + +FLAT_DESCRIPTION = ( + "Run PolicyEngine US (policyengine-us 2.41.0) on one household and return " + "the requested variables. `household` is a JSON object: `state` (the " + "two-letter state code), `people` (a list, each with `role` -- head, " + 'spouse or dependent --, `facts`, for example {"age": 40, ' + '"employment_income": 30000}, and optionally `name`, by default the role, ' + "numbered for dependents), and `household_facts` (facts about the " + "household as a whole: its tax unit, family, SPM unit or home). The head " + "and spouse are one couple and everyone else is on their own; everyone " + "shares one tax unit, family, SPM unit and household. A fact is a value " + 'for `period`, or {period: value} with a period such as "2026" or ' + '"2026-01". Facts you do not give take the engine\'s defaults. `variables` ' + 'lists the variables to compute for `period` (default "2026"). Each ' + "result gives the variable's entity, label and value for each member of " + "that entity; a monthly variable computed for a year is summed over its " + "months. An unknown name or a malformed household returns an error with " + "suggestions instead of results." +) +# The version 1 description's last sentence, which placement amends. +_ENTITY_ERROR_SENTENCE = ( + "An unknown name, a variable set on the wrong entity, or a malformed " + "situation returns an error with suggestions instead of results." +) + + +def calculate_tool_schema(version: int = 1, features: frozenset | None = None) -> dict: + if features is None: + features = V2_FEATURES if version == 2 else frozenset() + if features: + schema = calculate_tool_schema(1) + function = schema["function"] + properties = function["parameters"]["properties"] + if features == V2_FEATURES: + function["description"] = engine_text(CALCULATE_DESCRIPTION_V2) + else: + description = ( + FLAT_DESCRIPTION if FLAT in features else CALCULATE_DESCRIPTION + ) + if PLACEMENT in features and FLAT not in features: + description = description.replace( + _ENTITY_ERROR_SENTENCE, + "An unknown name or a malformed situation returns an error " + "with suggestions instead of results.", + ) + notes = [] + if LABELS in features: + notes.append( + "A fact may be written under the label it has in the facts " + "you were given." + ) + if NAMES in features: + notes.append( + "A name may also be written with spaces or be one of the " + "output keys you are asked for." + ) + if PLACEMENT in features and FLAT not in features: + notes.append( + "An input set on the wrong entity is moved to the one unit " + "holding its people; where no single unit holds them, it " + "returns an error." + ) + if OVERRIDES in features: + notes.append( + "Setting a variable the engine computes replaces its formula; a " + "contribution set as the amount contributed is read as the " + "amount asked for before limits (`..._desired`)." + ) + if notes: + notes.append("The result's `notes` say what it did.") + function["description"] = engine_text(" ".join([description, *notes])) + if FLAT in features: + del properties["situation"] + properties["household"] = { + "type": "string", + "description": ( + "The household as a JSON object (state, people with role and " + "facts, household_facts), written as a JSON string." + ), + } + function["parameters"]["required"] = ["household", "variables"] + if EXPLAIN in features: + properties["explain"] = { + "type": "array", + "items": {"type": "string"}, + "description": ( + f"Up to {MAX_EXPLAIN} variables to explain: how the engine " + "computed them." + ), + } + if CONCISE in features: + properties["detail"] = { + "type": "string", + "enum": ["brief", "full"], + "description": ( + 'How much an explanation shows: "brief" (the default: the ' + 'value, what it came from, and the main parameters) or "full" ' + f"(the trace, up to {EXPLAIN_MAX_LINES} steps and " + f"{EXPLAIN_MAX_PARAMETERS} parameters)." + ), + } + return schema return { "type": "function", "function": { "name": TOOL_NAME, - "description": CALCULATE_DESCRIPTION, + "description": engine_text(CALCULATE_DESCRIPTION), "parameters": { "type": "object", "properties": { @@ -499,9 +821,49 @@ class VariableInfo: label: str | None = None definition_period: str | None = None possible_values: tuple[str, ...] | None = None + # Whether the engine computes the variable (a formula or a sum of other + # variables) and no household fact the benchmark states sets it: setting + # it replaces what the engine would compute. + computed: bool = False + +# Inputs the benchmark's reference situations set besides its promptable +# facts (policybench.scenarios.Scenario.to_pe_household), and the convention +# switches (take-up, filing) any household may set. +BASE_STATED_INPUTS = frozenset( + { + "age", + "employment_income", + "self_employment_income", + "long_term_capital_gains", + "partnership_s_corp_income", + "is_tax_unit_head", + "is_tax_unit_spouse", + "state_code", + "weekly_hours_worked_before_lsr", + } +) +CONVENTION_PREFIXES = ("takes_up_", "would_") -def build_registry(system) -> dict[str, VariableInfo]: + +def stated_inputs() -> frozenset[str]: + """Every variable a household fact the benchmark states can set.""" + from policybench.scenarios import PE_INPUT_ALIASES, PE_INPUT_RENAMES + + names = {spec.source_name for spec in _promptable_specs()} + names |= BASE_STATED_INPUTS + names |= set(PE_INPUT_ALIASES.values()) | set(PE_INPUT_RENAMES.values()) + return frozenset(names) + + +def build_registry( + system, stated: frozenset[str] | None = None +) -> dict[str, VariableInfo]: + """The engine's variables. ``stated`` (default :func:`stated_inputs`) marks + which ones a stated fact sets; the rest that the engine computes are + ``computed``.""" + if stated is None: + stated = stated_inputs() registry = {} for name, variable in system.variables.items(): value_type = getattr(variable.value_type, "__name__", str(variable.value_type)) @@ -514,6 +876,13 @@ def build_registry(system) -> dict[str, VariableInfo]: label=variable.label, definition_period=str(variable.definition_period), possible_values=possible, + computed=bool( + getattr(variable, "formulas", None) + or getattr(variable, "adds", None) + or getattr(variable, "subtracts", None) + ) + and name not in stated + and not name.startswith(CONVENTION_PREFIXES), ) return registry @@ -555,6 +924,44 @@ def _type_problem(value, info: VariableInfo) -> str | None: return None +def _normalize_value( + where: str, + name: str, + value, + info: VariableInfo, + period: str, + problems: list[str], +) -> dict | None: + """One input's {period: value}, or None after recording why not.""" + if isinstance(value, dict): + if not value: + problems.append( + f"{where}: {name} has no values; set it as {{period: value}}." + ) + return None + periods = value + else: + periods = {period: value} + normalized = {} + for key, item in periods.items(): + if not isinstance(key, str) or not PERIOD_RE.match(key): + problems.append( + f"{where}: {name} has period {key!r}; use a year such as " + '"2026" or a month such as "2026-01".' + ) + continue + scalar = _scalar_problem(item) + if scalar is not None: + problems.append(f"{where}: {name} for {key} {scalar}.") + continue + typed = _type_problem(item, info) + if typed is not None: + problems.append(f"{where}: {name} {typed}.") + continue + normalized[key] = item + return normalized + + def _normalize_inputs( where: str, entity: str, @@ -563,11 +970,23 @@ def _normalize_inputs( period: str, problems: list[str], allow_members: bool, + v2: dict | None = None, ) -> dict: + """Each input under its engine name. ``v2`` (tool version 2) collects the + names it read, the inputs it redirected or flagged, and the inputs set on + another entity, which normalize_request then places.""" out: dict[str, Any] = {} for name, value in data.items(): if allow_members and name == "members": continue + if v2 is not None and isinstance(name, str) and name not in registry: + resolved, note = read_fact_name(name, registry, v2["features"]) + if resolved is not None: + v2["notes"].append(f"{where}: {note}") + name = resolved + elif (field := dropped_fact(name, registry, v2["features"])) is not None: + v2["notes"].append(_dropped_note(where, name, field)) + continue if not isinstance(name, str) or name not in registry: problems.append( f"{where}: unknown variable {name!r}." @@ -575,52 +994,62 @@ def _normalize_inputs( ) continue info = registry[name] + if v2 is not None and OVERRIDES in v2["features"]: + desired = f"{name}{DESIRED_SUFFIX}" + if ( + info.computed + and desired in registry + and registry[desired].entity == info.entity + ): + v2["notes"].append( + f"{where}: set {desired} (the amount asked for, before " + f"limits) for {name}, which the engine computes from it" + ) + name, info = desired, registry[desired] + elif info.computed: + v2["notes"].append( + f"{where}: {name} is computed by the engine; the value set " + "replaces its formula" + ) if info.entity != entity: + if v2 is not None and PLACEMENT in v2["features"]: + v2["misplaced"].append((where, entity, name, value)) + continue problems.append( f"{where}: {name} is a {info.entity} variable; set it under " f"{ENTITY_GROUP.get(info.entity, info.entity)}." ) continue - if isinstance(value, dict): - if not value: - problems.append( - f"{where}: {name} has no values; set it as {{period: value}}." - ) - continue - periods = value - else: - periods = {period: value} - normalized = {} - for key, item in periods.items(): - if not isinstance(key, str) or not PERIOD_RE.match(key): - problems.append( - f"{where}: {name} has period {key!r}; use a year such as " - '"2026" or a month such as "2026-01".' - ) - continue - scalar = _scalar_problem(item) - if scalar is not None: - problems.append(f"{where}: {name} for {key} {scalar}.") - continue - typed = _type_problem(item, info) - if typed is not None: - problems.append(f"{where}: {name} {typed}.") - continue - normalized[key] = item + normalized = _normalize_value(where, name, value, info, period, problems) + if normalized is None: + continue + if v2 is not None and name in out and out[name] != normalized: + problems.append(f"{where}: {name} is set twice with different values") + continue out[name] = normalized return out def normalize_request( - args, registry: Mapping[str, VariableInfo] + args, + registry: Mapping[str, VariableInfo], + version: int = 1, + features: frozenset | None = None, ) -> tuple[dict | None, dict | None]: """Check a calculate_household call; return (request, None) or (None, error). Accepts only plain JSON data. Every input and output name must be a variable of the engine, on its own entity; nothing else in the situation - (axes, reforms, parameters) is accepted. + (axes, reforms, parameters) is accepted. Tool version 2 also reads a name + written with spaces or a benchmark output key, redirects a contribution to + its ``_desired`` input, flags an input that replaces a formula, places an + input set on another entity where exactly one unit holds it, and takes + ``explain``; the request's ``notes`` say what it did. """ problems: list[str] = [] + if features is None: + features = V2_FEATURES if version == 2 else frozenset() + v2 = {"notes": [], "misplaced": [], "features": features} if features else None def fail() -> tuple[None, dict]: return None, { @@ -643,7 +1072,11 @@ def fail() -> tuple[None, dict]: if not isinstance(args, dict): problems.append("arguments must be an object") return fail() - unknown = sorted(set(args) - {"situation", "variables", "period"}) + allowed = {"variables", "period"} + allowed |= {"household"} if FLAT in features else {"situation"} + allowed |= {"explain"} if EXPLAIN in features else set() + allowed |= {"detail"} if CONCISE in features else set() + unknown = sorted(set(args) - allowed) if unknown: problems.append(f"unknown arguments: {', '.join(map(str, unknown))}") @@ -658,6 +1091,13 @@ def fail() -> tuple[None, dict]: ) period = str(YEAR) + if FLAT in features: + args = dict(args) + args["situation"] = flat_situation( + args.get("household"), registry, problems, v2, period + ) + if args["situation"] is None: + return fail() situation = args.get("situation") if isinstance(situation, str): if len(situation) > MAX_SITUATION_CHARS: @@ -702,7 +1142,7 @@ def fail() -> tuple[None, dict]: problems.append(f"people.{name} must be an object of inputs") continue normalized_people[name] = _normalize_inputs( - f"people.{name}", "person", data, registry, period, problems, False + f"people.{name}", "person", data, registry, period, problems, False, v2 ) normalized["people"] = normalized_people @@ -746,7 +1186,9 @@ def fail() -> tuple[None, dict]: seen[member] = unit_name normalized_unit = {"members": list(members)} normalized_unit.update( - _normalize_inputs(where, entity, unit, registry, period, problems, True) + _normalize_inputs( + where, entity, unit, registry, period, problems, True, v2 + ) ) normalized_units[unit_name] = normalized_unit normalized[group] = normalized_units @@ -763,18 +1205,556 @@ def fail() -> tuple[None, dict]: variables = [] elif len(variables) > MAX_VARIABLES: problems.append(f"at most {MAX_VARIABLES} variables per call") + if v2 is not None and PLACEMENT in features: + _place_misplaced(v2, normalized, situation, registry, period, problems) + if v2 is not None and LABELS in features: + _set_aliases(v2, normalized["people"], registry) requested = [] for name in variables: + if ( + v2 is not None + and NAMES in features + and isinstance(name, str) + and name not in registry + ): + resolved, note = resolve_name(name, registry, output_names=True) + if resolved is not None: + v2["notes"].append(f"variables: {note}") + name = resolved if name not in registry: problems.append( f"variables: unknown variable {name!r}." + _suggest(name, registry) ) elif name not in requested: requested.append(name) + explain = [] + if EXPLAIN in features and args.get("explain") is not None: + names = args["explain"] + if isinstance(names, str): + names = [names] + if not isinstance(names, list) or not all(isinstance(n, str) for n in names): + problems.append("explain must be a list of variable names") + names = [] + elif len(names) > MAX_EXPLAIN: + problems.append(f"explain at most {MAX_EXPLAIN} variables per call") + for name in names: + if NAMES in features and name not in registry: + resolved, note = resolve_name(name, registry, output_names=True) + if resolved is not None: + v2["notes"].append(f"explain: {note}") + name = resolved + if name not in registry: + problems.append( + f"explain: unknown variable {name!r}." + _suggest(name, registry) + ) + elif name not in explain: + explain.append(name) + + detail = None + if CONCISE in features: + detail = args.get("detail") or "brief" + if detail not in ("brief", "full"): + problems.append('detail must be "brief" or "full"') if problems: return fail() - return {"situation": normalized, "variables": requested, "period": period}, None + request = {"situation": normalized, "variables": requested, "period": period} + if v2 is not None: + request["explain"] = explain + request["notes"] = v2["notes"] + if detail is not None: + request["detail"] = detail + return request, None + + +FLAT_ROLES = ("head", "spouse", "dependent") +FLAT_PERSON_KEYS = ("name", "role", "facts") +# The relationship inputs a role sets (as Scenario.to_pe_household's people +# carry them). +FLAT_RELATIONS = {"is_tax_unit_head": "head", "is_tax_unit_spouse": "spouse"} + + +def flat_names(people) -> list: + """The name each listed person goes by: the one given, or, for a person + given none, the role (head, spouse, dependent_1, ...: dependents are + numbered by their place among the dependents listed).""" + names, dependents = [], 0 + for person in people if isinstance(people, list) else []: + if not isinstance(person, dict): + names.append(None) + continue + role = person.get("role") + if role == "dependent": + dependents += 1 + name = person.get("name") + if name is None and role in FLAT_ROLES: + name = role if role != "dependent" else f"dependent_{dependents}" + names.append(name) + return names + + +def _same_value( + registry, + name: str, + a, + b, + period: str, + problems: list[str] | None = None, + where: str = "", +) -> bool | None: + """Whether two values of an input are the same once normalized, or None + when either is malformed. ``a`` stays in the situation, so normalization + reports it later; a caller that drops ``b`` passes ``problems`` so a + malformed ``b`` is reported, not lost.""" + info = registry[name] + first_problems: list[str] = [] + second_problems: list[str] = [] + first = _normalize_value(where, name, a, info, period, first_problems) + second = _normalize_value(where, name, b, info, period, second_problems) + if problems is not None: + problems.extend(second_problems) + if first_problems or second_problems: + return None + return first == second + + +def flat_situation( + household, + registry, + problems: list[str], + v2: dict | None, + period: str = str(YEAR), +): + """The flat feature's household -- state, people with role and facts, and + household facts -- as a situation, with the units the benchmark's own + reference situations have (policybench.scenarios.Scenario.to_pe_household): + the head and spouse one marital unit and everyone else alone, and one tax + unit, SPM unit, family and household holding everyone. Roles set the + relationship inputs; a stated one that contradicts its role is refused. + A household fact goes to the unit of its variable's entity, under the + name it was written with, so normalization reads and compares it as any + other input. A person given no name is named by role (flat_names). + Returns None after recording problems.""" + features = v2["features"] if v2 is not None else frozenset() + + def engine_name(name): + if isinstance(name, str) and name in registry: + return name + if isinstance(name, str): + return read_fact_name(name, registry, features)[0] + return None + + if isinstance(household, str): + if len(household) > MAX_SITUATION_CHARS: + problems.append("household is too large") + return None + try: + household = json.loads(household) + except (ValueError, RecursionError) as error: + problems.append(f"household is not valid JSON: {error}"[:MAX_MESSAGE_CHARS]) + return None + if not isinstance(household, dict): + problems.append( + "household must be a JSON object with state, people and household_facts" + ) + return None + extra = sorted(set(household) - {"state", "people", "household_facts"}) + if extra: + problems.append( + "household accepts state, people and household_facts; remove " + + ", ".join(map(str, extra)) + ) + people = household.get("people") + if not isinstance(people, list) or not people: + problems.append("household.people must be a non-empty list of people") + return None + if len(people) > MAX_PEOPLE: + problems.append(f"at most {MAX_PEOPLE} people") + return None + situation: dict[str, Any] = {"people": {}} + roles: dict[str, str] = {} + for index, (person, name) in enumerate(zip(people, flat_names(people))): + if not isinstance(person, dict): + problems.append( + f"people[{index}] must be an object with name, role and facts" + ) + continue + stray = sorted(map(str, set(person) - set(FLAT_PERSON_KEYS))) + if stray: + problems.append( + f"people[{index}] takes only name, role and facts; give " + f"{', '.join(stray)} under facts" + ) + continue + role, facts = person.get("role"), person.get("facts", {}) + if role not in FLAT_ROLES: + problems.append( + f"people[{index}].role must be one of {', '.join(FLAT_ROLES)}" + ) + continue + if person.get("name") is None and v2 is not None: + v2["notes"].append(f"people[{index}]: named {name}") + if not isinstance(name, str) or not name or len(name) > 64: + problems.append(f"people[{index}].name must be a short string") + continue + if name in roles: + problems.append(f"people[{index}]: {name!r} is listed twice") + continue + if not isinstance(facts, dict): + problems.append(f"people[{index}].facts must be an object") + continue + roles[name] = role + situation["people"][name] = dict(facts) + heads = [n for n, r in roles.items() if r == "head"] + spouses = [n for n, r in roles.items() if r == "spouse"] + if len(heads) != 1: + problems.append("the household needs exactly one head") + if len(spouses) > 1: + problems.append("the household has at most one spouse") + if problems: + return None + for name, role in roles.items(): + facts = situation["people"][name] + for relation, relation_role in FLAT_RELATIONS.items(): + if relation not in registry: + continue + implied = role == relation_role + stated = [k for k in facts if engine_name(k) == relation] + if not stated: + facts[relation] = implied + elif any( + _same_value(registry, relation, facts[k], implied, period) is False + for k in stated + ): + problems.append( + f"people.{name}: {relation} contradicts the role {role}; the " + "role sets it" + ) + everyone = list(situation["people"]) + couple = heads + spouses + groups = [couple] + [[n] for n in everyone if n not in couple] + situation["marital_units"] = { + f"marital_unit_{i}": {"members": members} + for i, members in enumerate(groups, start=1) + } + situation["tax_units"] = {"tax_unit": {"members": list(everyone)}} + situation["spm_units"] = {"spm_unit": {"members": list(everyone)}} + situation["families"] = {"family": {"members": list(everyone)}} + situation["households"] = {"household": {"members": list(everyone)}} + state = household.get("state") + if state is not None: + situation["households"]["household"]["state_code"] = state + unit_of = { + "tax_unit": situation["tax_units"]["tax_unit"], + "spm_unit": situation["spm_units"]["spm_unit"], + "family": situation["families"]["family"], + "household": situation["households"]["household"], + "marital_unit": situation["marital_units"]["marital_unit_1"], + } + facts = household.get("household_facts") or {} + if not isinstance(facts, dict): + problems.append("household_facts must be an object") + return None + for name, value in facts.items(): + resolved = engine_name(name) + if resolved is None: + field = ( + dropped_fact(name, registry, features) + if isinstance(name, str) + else None + ) + if field is not None: + v2["notes"].append(_dropped_note("household_facts", name, field)) + continue + problems.append( + f"household_facts: unknown variable {name!r}." + + (_suggest(name, registry) if isinstance(name, str) else "") + ) + continue + entity = registry[resolved].entity + if entity == "person": + # As placement moves a person variable off a one-person unit: only + # a household of one says whose fact it is. + if v2 is None or PLACEMENT not in features or len(everyone) != 1: + problems.append( + f"household_facts: {resolved} is a person's fact; give it " + "under the person whose facts list it" + ) + continue + target = situation["people"][everyone[0]] + v2["notes"].append( + f"household_facts: moved {resolved}, a person's fact, to {everyone[0]}" + ) + else: + target = unit_of[entity] + if name in target: + # The repeat is dropped, so only one validated as equal is kept quiet. + same = _same_value( + registry, + resolved, + target[name], + value, + period, + problems, + "household_facts", + ) + if same is False: + problems.append( + f"household_facts: {resolved} is set twice with different values" + ) + continue + target[name] = value + return None if problems else situation + + +DESIRED_SUFFIX = "_desired" +_LABEL_SEPARATORS = re.compile(r"[\s\-]+") + + +def output_reading(name: str) -> tuple[str, str] | None: + """A benchmark output key's engine variable and how the output reads it, + or None when the name is not an output key.""" + from policybench.ground_truth import ( + _aggregation_for_output, + _pe_variable_for_output, + ) + from policybench.spec import find_output_spec, parse_person_output + + person = parse_person_output(name) + if person is None and find_output_spec(name, country="us") is None: + return None + engine = _pe_variable_for_output(name, "us") + if engine == name: + return None + aggregation = _aggregation_for_output(name, "us") + if person is not None: + how = f"{engine} for the person the key names ({person[0]})" + elif aggregation == "any_positive": + how = f"1 when {engine} is above 0, else 0" + else: + how = f"{engine}, summed over the household's units" + return engine, how + + +def _label_key(text: str) -> str: + """A label as compared: lower case, letters and digits only, so "Roth + 401(k) contributions" and "roth 401k contributions" read alike.""" + return re.sub(r"[^0-9a-z]", "", text.casefold()) + + +# The prompt's household block gives the state as "state: CT". +PROMPT_EXTRA_LABELS = {"state": "state_code"} + + +@functools.lru_cache(maxsize=1) +def _promptable_specs(): + from policybench.scenarios import get_promptable_input_specs + + return get_promptable_input_specs() + + +def _sample_value(value_type: str): + if value_type == "bool": + return True + if value_type in ("str", "Enum"): + return "x" + return 1.0 + + +def rendered_labels(field: str, value_type: str = "float") -> tuple[str, ...]: + """The labels a fact is shown under: in the board's prompt + (policybench.prompts._format_input_line, special renderers included) and + in the v2-pilot prompt (v2_pilot_text applied to that line).""" + from policybench.prompts import _format_input_line, _humanize_input_label + + line = _format_input_line(field, _sample_value(value_type)) + label = _humanize_input_label(field) + if not (line == f"- {label}" or line.startswith(f"- {label}: ")): + label = line[2:].split(": ", 1)[0] + labels = [label] + suffix = line[2 + len(label) :] + rewritten = v2_pilot_text(line) + if rewritten != line and rewritten.startswith("- ") and rewritten.endswith(suffix): + labels.append(rewritten[2 : len(rewritten) - len(suffix)]) + return tuple(labels) + + +@functools.lru_cache(maxsize=1) +def prompt_labels() -> dict[str, frozenset[tuple[str, bool]]]: + """Every label the prompts show a fact under, keyed by _label_key, each + with the facts it may name as (engine name, dropped): the promptable + inputs, every labeled input (policybench.prompts.INPUT_LABEL_OVERRIDES), + the inputs the reference sets besides them (BASE_STATED_INPUTS), inputs + the engine has renamed and inputs the reference leaves out + (policybench.scenarios.PE_INPUT_RENAMES, PE_DROPPED_INPUTS), each in the + board's and the v2-pilot prompt's wording. Dropped facts are shown but are + not engine inputs. A label may name facts of another country; a lookup + reads only the one its engine has.""" + from policybench.prompts import INPUT_LABEL_OVERRIDES + from policybench.scenarios import PE_DROPPED_INPUTS, PE_INPUT_RENAMES + + fields = {spec.output_name: spec.value_type for spec in _promptable_specs()} + dropped = {f for names in PE_DROPPED_INPUTS.values() for f in names} + for source in ( + "age", + *BASE_STATED_INPUTS, + *INPUT_LABEL_OVERRIDES, + *PE_INPUT_RENAMES, + *dropped, + ): + fields.setdefault(source, "int" if source == "age" else "float") + labels: dict[str, set[tuple[str, bool]]] = {} + for source, value_type in fields.items(): + target = (PE_INPUT_RENAMES.get(source, source), source in dropped) + keys = {_label_key(label) for label in rendered_labels(source, value_type)} + if source in PE_INPUT_RENAMES or source in dropped: + keys.add(_label_key(source)) + for key in keys: + labels.setdefault(key, set()).add(target) + for label, name in PROMPT_EXTRA_LABELS.items(): + labels.setdefault(_label_key(label), set()).add((name, False)) + return {key: frozenset(targets) for key, targets in labels.items()} + + +def read_fact_name( + name: str, registry: Mapping[str, VariableInfo], features: frozenset +) -> tuple[str | None, str | None]: + """A fact's engine variable under the labels feature (any label the + prompt shows it under, when exactly one fact it may name is the engine's) + and then the names feature (spaces for underscores). Returns (engine + name, note) or (None, None).""" + if LABELS in features: + found = [ + engine + for engine, dropped in prompt_labels().get(_label_key(name), ()) + if not dropped and engine in registry + ] + if len(found) == 1: + return found[0], f"read {name!r} as {found[0]}" + if NAMES in features: + return resolve_name(name, registry, output_names=False) + return None, None + + +def dropped_fact( + name: str, registry: Mapping[str, VariableInfo], features: frozenset +) -> str | None: + """Under the labels feature, the input a fact names when the prompt shows + it but the reference leaves it out (policybench.scenarios.PE_DROPPED_INPUTS: + no longer an engine input); otherwise None.""" + if LABELS not in features: + return None + found = [ + engine + for engine, dropped in prompt_labels().get(_label_key(name), ()) + if dropped + ] + return found[0] if len(found) == 1 else None + + +def _dropped_note(where: str, name: str, field: str) -> str: + return ( + f"{where}: {name!r} ({field}) is not an input of this engine; the " + "benchmark's reference leaves it out too, so it was ignored" + ) + + +def _set_aliases(v2: dict, people: dict, registry: Mapping[str, VariableInfo]): + """Set each fact the reference also sets under another name there too + (policybench.scenarios.PE_INPUT_ALIASES: the SNAP work rules read stated + usual weekly hours as weekly_hours_worked_before_lsr), unless it is set.""" + from policybench.scenarios import PE_INPUT_ALIASES + + for person, inputs in people.items(): + for source, target in PE_INPUT_ALIASES.items(): + if source in inputs and target in registry and target not in inputs: + inputs[target] = dict(inputs[source]) + v2["notes"].append( + f"people.{person}: also set {target} from {source}, as the " + "benchmark's reference does" + ) + + +def resolve_name( + name: str, registry: Mapping[str, VariableInfo], output_names: bool +) -> tuple[str | None, str | None]: + """Tool version 2's reading of a name the engine does not know: the name + with spaces or hyphens as underscores, or (for outputs) a benchmark output + key. Returns (engine name, note) or (None, None).""" + if name in registry: + return name, None + written = _LABEL_SEPARATORS.sub("_", name.strip().lower()) + if written in registry: + return written, f"read {name!r} as {written}" + if output_names: + reading = output_reading(name) + if reading is not None and reading[0] in registry: + return reading[0], f"{name} is {reading[1]}; computed {reading[0]}" + return None, None + + +def _place_misplaced( + v2: dict, + normalized: dict, + situation: dict, + registry: Mapping[str, VariableInfo], + period: str, + problems: list[str], +) -> None: + """Move each input set on another entity to the one unit that holds the + people it was set for: a group variable set on a person goes to that + person's unit (a group left out of the situation becomes one unit of + everyone, as the engine reads it), a person variable set on a one-person + unit goes to that person, and a group variable set on another group goes + to the unit holding all of its members. Anything else stays an error.""" + for where, source_entity, name, value in v2["misplaced"]: + info = registry[name] + if source_entity == "person": + persons = [where.split(".", 1)[1]] + else: + group, unit = where.split(".", 1) + persons = list(normalized.get(group, {}).get(unit, {}).get("members", [])) + target_group = ENTITY_GROUP.get(info.entity, info.entity) + if info.entity == "person": + if len(persons) != 1: + problems.append( + f"{where}: {name} is a person variable; set it on one person " + "under people." + ) + continue + destination = normalized["people"].get(persons[0]) + to = f"people.{persons[0]}" + else: + if target_group not in situation and target_group not in normalized: + normalized[target_group] = { + "all": {"members": list(normalized["people"])} + } + units = normalized.get(target_group, {}) + holding = [ + unit + for unit, data in units.items() + if set(persons) <= set(data.get("members", [])) + ] + if len(holding) != 1: + problems.append( + f"{where}: {name} is a {info.entity} variable; set it under " + f"{target_group} (no single unit there holds " + f"{', '.join(persons) or 'its members'})." + ) + continue + destination = units[holding[0]] + to = f"{target_group}.{holding[0]}" + if destination is None: + problems.append(f"{where}: {name} has no place to go") + continue + normalized_value = _normalize_value(to, name, value, info, period, problems) + if normalized_value is None: + continue + if name in destination and destination[name] != normalized_value: + problems.append(f"{to}: {name} is set twice with different values") + continue + destination[name] = normalized_value + v2["notes"].append(f"moved {name} from {where} to {to}") def _jsonable(value): @@ -823,47 +1803,212 @@ def execute_request(system, registry, request: dict, simulation_cls=None) -> dic "period": request["period"], "values": {i: _jsonable(v) for i, v in zip(ids, list(array))}, } - return {"results": results} + out = {"results": results} + if "notes" in request: + out["notes"] = list(request["notes"]) + if request.get("explain"): + out["explanations"] = { + name: ( + brief_explanation( + explain_variable(system, request, name, simulation_cls) + ) + if request.get("detail") == "brief" + else explain_variable(system, request, name, simulation_cls) + ) + for name in request["explain"] + } + return out + + +BRIEF_INPUTS = 8 +BRIEF_PARAMETERS = 5 + + +def brief_explanation(full: dict) -> dict: + """The concise feature's explanation: the value, the variables it came + from directly (one level down, the first BRIEF_INPUTS), and the first + parameters it read with their sources.""" + if "error" in full: + return full + steps = full["steps"] + direct = [ + s.strip() for s in steps[1:] if s.startswith(" ") and not s.startswith(" ") + ] + return { + "value": steps[0].strip() if steps else "", + "from": direct[:BRIEF_INPUTS], + "parameters": full["parameters"][:BRIEF_PARAMETERS], + "more": 'set detail to "full" for the trace', + } + + +EXPLAIN_MAX_LINES = 60 +EXPLAIN_MAX_DEPTH = 6 +EXPLAIN_MAX_PARAMETERS = 15 +# Switches every variable has; never part of a reason. +_QUIET_PARAMETERS = ("gov.abolitions.",) + + +def _trace_value(value) -> Any: + if hasattr(value, "decode_to_str"): + value = value.decode_to_str() + try: + items = list(value) + except TypeError: + return _jsonable(value) + items = [_jsonable(v) for v in items] + return items[0] if len(items) == 1 else items + + +def _all_zero(value) -> bool: + try: + return all((not v) if not isinstance(v, str) else False for v in list(value)) + except TypeError: + return not value + + +def parameter_sources(system, name: str, limit: int = 2) -> list[dict]: + """A parameter's cited sources: its own ``reference`` metadata, or the + nearest ancestor's.""" + node = system.parameters + chain = [node] + for part in name.split("."): + children = getattr(node, "children", None) + if not isinstance(children, dict) or part not in children: + break + node = children[part] + chain.append(node) + for node in reversed(chain): + refs = (getattr(node, "metadata", None) or {}).get("reference") + if refs: + refs = refs if isinstance(refs, list) else [refs] + out = [] + for ref in refs[:limit]: + if isinstance(ref, dict): + out.append({k: ref[k] for k in ("title", "href") if ref.get(k)}) + else: + out.append({"href": str(ref)}) + return out + return [] + + +def explain_variable(system, request: dict, name: str, simulation_cls=None) -> dict: + """How the engine computed ``name`` for the request's household: the + computation steps (each variable, its period and value, with its inputs + indented beneath it; zero-valued branches below the first level left out), + then the parameters it read, each with its value and cited sources.""" + if simulation_cls is None: + from policyengine_us import Simulation as simulation_cls + try: + simulation = simulation_cls( + tax_benefit_system=system, situation=copy.deepcopy(request["situation"]) + ) + simulation.trace = True + simulation.calculate(name, request["period"]) + except Exception as error: + return {"error": f"{type(error).__name__}: {error}"[:MAX_MESSAGE_CHARS]} + + def find(nodes): + for node in nodes: + if node.name == name: + return node + hit = find(node.children) + if hit is not None: + return hit + return None + + root = find(simulation.tracer.trees) + if root is None: + return {"error": f"no trace for {name}"} + steps: list[str] = [] + parameters: dict[str, Any] = {} + + def walk(node, depth): + if len(steps) >= EXPLAIN_MAX_LINES or depth > EXPLAIN_MAX_DEPTH: + return + if depth > 1 and _all_zero(node.value): + return + steps.append( + " " * depth + f"{node.name} ({node.period}) = {_trace_value(node.value)}" + ) + for parameter in node.parameters: + if parameter.name.startswith(_QUIET_PARAMETERS): + continue + parameters.setdefault(parameter.name, _trace_value(parameter.value)) + for child in node.children: + walk(child, depth + 1) + + walk(root, 0) + return { + "steps": steps, + "parameters": [ + {"name": key, "value": value, "sources": parameter_sources(system, key)} + for key, value in list(parameters.items())[:EXPLAIN_MAX_PARAMETERS] + ], + } + + +def _board_upgrade(snapshot: Path) -> dict: + """A board's last engine_upgrade revision: the engine its references are + on and the fix modules it pins. The frozen board's is 2.41.0's; a household + set (scripts/household_set.py) records one.""" + meta = json.loads((snapshot / "reference_outputs.csv.meta.json").read_text()) + revisions = [r for r in meta["revisions"] if r.get("kind") == "engine_upgrade"] + if not revisions: + raise SystemExit(f"{snapshot} records no engine_upgrade revision") + return revisions[-1] + + +def board_engine(snapshot: Path = SNAPSHOT) -> str: + """The policyengine-us version a board's references were computed on.""" + return _board_upgrade(snapshot)["engine_version"].removeprefix("policyengine-us ") + + +def is_household_set(snapshot: Path) -> bool: + meta = json.loads((snapshot / "reference_outputs.csv.meta.json").read_text()) + return meta.get("board_kind") == "household_set" def pinned_fix_modules(snapshot: Path = SNAPSHOT) -> list[dict]: - """The fix modules the 2.41.0 references were built with, as the sidecar's + """The fix modules a board's references were built with, as its sidecar's engine_upgrade revision pins them (path and sha256).""" - meta = json.loads((snapshot / "reference_outputs.csv.meta.json").read_text()) - revisions = [ - r - for r in meta["revisions"] - if r.get("kind") == "engine_upgrade" - and r.get("engine_version") == f"policyengine-us {REFERENCE_ENGINE}" - ] - if len(revisions) != 1: - raise SystemExit(f"expected one {REFERENCE_ENGINE} engine_upgrade revision") - return revisions[0]["fix_modules"] + return _board_upgrade(snapshot)["fix_modules"] + + +def board_fix_module_sha256(snapshot: Path = SNAPSHOT) -> str: + """The pinned latest_final.py: the constant for the frozen board, the + sidecar's record for a household set.""" + if not is_household_set(snapshot): + return FIX_MODULE_SHA256 + return _board_upgrade(snapshot)["fix_module"]["sha256"] -def verify_fix_pins(fixes_dir: Path = FIX_MODULE.parent) -> None: +def verify_fix_pins( + fixes_dir: Path = FIX_MODULE.parent, snapshot: Path = SNAPSHOT +) -> None: """Every module latest_final.py composes must be the pinned bytes. Each pin is checked at its recorded path; a module the reform reads from its own directory (the sales tax tables) is also checked there. """ problems = [] - for pin in pinned_fix_modules(): + for pin in pinned_fix_modules(snapshot): candidates = {ROOT / pin["path"], fixes_dir / pin["module"]} for path in sorted(candidates): if path.exists() and _sha256(path) != pin["sha256"]: problems.append(f"{path} does not match the pinned {pin['sha256']}") if not any(path.exists() for path in candidates): problems.append(f"{pin['module']} is missing") - if _sha256(FIX_MODULE) != FIX_MODULE_SHA256: - problems.append(f"{FIX_MODULE} does not match {FIX_MODULE_SHA256}") + pinned = board_fix_module_sha256(snapshot) + if _sha256(FIX_MODULE) != pinned: + problems.append(f"{FIX_MODULE} does not match {pinned}") if problems: raise SystemExit("; ".join(problems)) -def load_fix_reform(path: Path = FIX_MODULE): +def load_fix_reform(path: Path = FIX_MODULE, snapshot: Path = SNAPSHOT): if path == FIX_MODULE: - verify_fix_pins() + verify_fix_pins(snapshot=snapshot) spec = importlib.util.spec_from_file_location("tool_pilot_latest_final", path) module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module @@ -871,18 +2016,61 @@ def load_fix_reform(path: Path = FIX_MODULE): return module.reform -def build_tool_system(reform: bool = True, path: Path = FIX_MODULE): +def build_tool_system( + reform: bool = True, path: Path = FIX_MODULE, snapshot: Path = SNAPSHOT +): from policyengine_us import CountryTaxBenefitSystem if not reform: return CountryTaxBenefitSystem() - return CountryTaxBenefitSystem(reform=load_fix_reform(path)) + return CountryTaxBenefitSystem(reform=load_fix_reform(path, snapshot)) + + +# How a pool tells its worker the calculator's features: a tuple, which a +# model's JSON arguments can never be, so a model cannot turn on a feature its +# arm does not offer. A pool without features sends the arguments as they are. +CALL_FRAME = "calculate" + +def tool_call_frame(features: frozenset, arguments) -> tuple: + return (CALL_FRAME, tuple(sorted(features)), arguments) -def _worker_main(conn, fix_path: str, reform: bool) -> None: + +def frame_features(names) -> frozenset: + """A frame's features, refused (ValueError) unless they are a tuple of + known feature names with concise only beside explain.""" + if not isinstance(names, tuple) or not all(isinstance(n, str) for n in names): + raise ValueError("the features must be a tuple of names") + features = frozenset(names) + if features - set(TOOL_FEATURES): + raise ValueError( + f"unknown tool features {sorted(features - set(TOOL_FEATURES))}" + ) + if CONCISE in features and EXPLAIN not in features: + raise ValueError("the concise feature needs explain") + return features + + +def worker_request(message, registry) -> tuple[dict | None, dict | None]: + """A worker's reading of one pool message: a call frame's arguments under + its features, or anything else as a model's arguments with no features + (keys such as "__tool_features__" or "__tool_version__" in them are the + model's, and refused as unknown arguments).""" + features = frozenset() + if isinstance(message, tuple) and len(message) == 3 and message[0] == CALL_FRAME: + try: + features = frame_features(message[1]) + except ValueError as error: + return None, {"error": "configuration", "message": str(error)} + message = message[2] + return normalize_request(message, registry, features=features) + + +def _worker_main(conn, fix_path: str, reform: bool, board: str | None = None) -> None: for name in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS"): os.environ[name] = "1" - system = build_tool_system(reform=reform, path=Path(fix_path)) + snapshot = Path(board) if board else SNAPSHOT + system = build_tool_system(reform=reform, path=Path(fix_path), snapshot=snapshot) registry = build_registry(system) conn.send(("ready", None)) while True: @@ -892,7 +2080,7 @@ def _worker_main(conn, fix_path: str, reform: bool) -> None: return if args is None: return - request, error = normalize_request(args, registry) + request, error = worker_request(args, registry) if error is not None: conn.send(("ok", error)) continue @@ -910,9 +2098,21 @@ def __init__( reform: bool = True, worker_target: Callable | None = None, acquire_timeout: float | None = None, + board: Path | None = None, + tool_version: int = 1, + tool_features: frozenset | None = None, ): self._ctx = mp.get_context("spawn") + self._tool_version = tool_version + self._tool_features = ( + tool_features + if tool_features is not None + else (V2_FEATURES if tool_version == 2 else frozenset()) + ) self._fix_path = str(fix_path) + # A worker gets the board only when it is not the frozen one, so a + # three-argument worker target still starts on the default. + self._board = None if board is None or board == SNAPSHOT else str(board) self._reform = reform self._target = worker_target or _worker_main self.timeout = timeout @@ -932,11 +2132,10 @@ def __init__( def _start(self): parent, child = self._ctx.Pipe() - process = self._ctx.Process( - target=self._target, - args=(child, self._fix_path, self._reform), - daemon=True, - ) + args = (child, self._fix_path, self._reform) + if self._board is not None: + args += (self._board,) + process = self._ctx.Process(target=self._target, args=args, daemon=True) process.start() return process, parent @@ -976,6 +2175,8 @@ def call(self, args) -> tuple[dict, float]: time.perf_counter() - started, ) process, conn = worker + if self._tool_features: + args = tool_call_frame(self._tool_features, args) try: conn.send(args) if conn.poll(self.timeout): @@ -1131,13 +2332,27 @@ def notice(self) -> str: notice = WEB_NOTICE if tool else JSON_WEB_NOTICE elif self.arm == "web+pe": notice = WEB_TOOL_NOTICE if tool else JSON_WEB_TOOL_NOTICE + elif self.arm == "pe2": + return engine_text(TOOL_NOTICE_V2 if tool else JSON_TOOL_NOTICE_V2) + elif self.arm != "pe" and arm_features(self.arm) is not None: + return engine_text(compose_tool_notice(arm_features(self.arm), tool)) else: - return TOOL_NOTICE if tool else JSON_TOOL_NOTICE - return notice + (BENCHMARK_SEARCH_RULE if self.benchmark_search_rule else "") + return engine_text(TOOL_NOTICE if tool else JSON_TOOL_NOTICE) + return engine_text(notice) + ( + BENCHMARK_SEARCH_RULE if self.benchmark_search_rule else "" + ) @property def uses_calculator(self) -> bool: - return self.arm in ("pe", "web+pe") + return arm_features(self.arm) is not None + + @property + def tool_features(self) -> frozenset: + return arm_features(self.arm) or frozenset() + + @property + def tool_version(self) -> int: + return 2 if self.tool_features == V2_FEATURES else 1 @property def uses_web(self) -> bool: @@ -1155,7 +2370,7 @@ def arm1_label(self) -> str: def arm2_label(self) -> str: version = "" if self.prompt_version == "v1" else f"@{self.prompt_version}" rule = "+no-benchmark-search" if self.benchmark_search_rule else "" - return f"{self.name}{ARM_SUFFIX[self.arm]}{version}{rule}" + return f"{self.name}{arm_suffix(self.arm)}{version}{rule}" def provider_family(model_id: str) -> str: @@ -1181,8 +2396,11 @@ def model_profile( _uses_responses_api, ) - if arm not in ARMS: - raise SystemExit(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}") + if arm not in ARMS and arm_features(arm) is None: + raise SystemExit( + f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}, or pe: plus " + f"calculator features joined by + ({', '.join(TOOL_FEATURES)})" + ) if prompt_version not in PROMPT_VERSIONS: raise SystemExit( f"unknown prompt {prompt_version!r}; the prompts are " @@ -1816,7 +3034,7 @@ def run_household( sent_board_prompt_sha256 = base.pop("_sent_board_prompt_sha256") ceiling = base.pop("_budget_ceiling") answer_tool = base.pop("_answer_tool") - calculate_tool = calculate_tool_schema() + calculate_tool = calculate_tool_schema(features=profile.tool_features) offered = ([web_search_tool(profile)] if profile.uses_web else []) + ( [calculate_tool] if profile.uses_calculator else [] ) @@ -1857,6 +3075,7 @@ def run_household( "benchmark_search_rule": profile.benchmark_search_rule, "sent_board_prompt_sha256": sent_board_prompt_sha256, "tool_notice_sha256": hashlib.sha256(profile.notice.encode()).hexdigest(), + **profile_treatment(profile), "turns": [], "status": "running", } @@ -2366,28 +3585,38 @@ def _versions() -> dict: return out -def _require_engine() -> None: +def board_path(args) -> Path: + """The board a command reads: --board, or the frozen snapshot.""" + return Path(getattr(args, "board", None) or SNAPSHOT) + + +def _require_engine(board: Path = SNAPSHOT) -> None: version = _versions()["policyengine-us"] - if version != REFERENCE_ENGINE: + pinned = board_engine(board) + if version != pinned: + where = ( + f"the reference engine of household set {board}" + if is_household_set(board) + else "the reference engine of release dashboard-data-20261014" + ) raise SystemExit( - f"policyengine-us {version} is installed; the pilot pins " - f"{REFERENCE_ENGINE}, the reference engine of release " - "dashboard-data-20261014." + f"policyengine-us {version} is installed; the pilot pins {pinned}, {where}." ) def command_oracle(args) -> None: """Reference situations through the tool engine: every scored reference must come back within the board's tolerance.""" - _require_engine() + snapshot = board_path(args) + _require_engine(snapshot) from policybench.ground_truth import _extract_person_value, _pe_variable_for_output - board = load_board() + board = load_board(snapshot) scored_keys = set(zip(board["scored"].scenario_id, board["scored"].variable)) values = board["reference"].set_index(["scenario_id", "variable"])["value"] rows = [] for label, reform in (("reference_engine", True), ("plain_engine", False)): - system = build_tool_system(reform=reform) + system = build_tool_system(reform=reform, snapshot=snapshot) registry = build_registry(system) for sid, scenario in board["scenarios"].items(): situation = scenario.to_pe_household() @@ -2555,6 +3784,115 @@ def records_wording(records: Mapping[str, dict]) -> str | None: return hashes.pop() if hashes else None +# Arms whose calculator features never changed, so a record written before +# households recorded their features has these. +LEGACY_TOOL_ARMS = {"pe": frozenset(), "web+pe": frozenset(), "pe2": V2_FEATURES} + + +def profile_treatment(profile: "ModelProfile") -> dict: + """What a household record keeps of the treatment it ran under, beyond + the notice hash: the calculator's features and its schema's hash.""" + if not profile.uses_calculator: + return {"tool_features": None, "calculate_tool_sha256": None} + schema = calculate_tool_schema(features=profile.tool_features) + return { + "tool_features": sorted(profile.tool_features), + "calculate_tool_sha256": hashlib.sha256( + json.dumps(schema, sort_keys=True).encode() + ).hexdigest(), + } + + +def record_treatment_problem(record: dict, profile: "ModelProfile") -> str | None: + """Why a household record ran under another treatment than the profile's + (features, notice or calculator schema), or None. A record written before + a field existed counts as matching only for an arm whose treatment never + changed (none, web, and LEGACY_TOOL_ARMS); otherwise it is unrecorded.""" + sid = record.get("scenario_id") + want = profile_treatment(profile) + legacy = profile.arm in ("none", "web") or profile.arm in LEGACY_TOOL_ARMS + if "tool_features" in record: + if record["tool_features"] != want["tool_features"]: + return f"{sid} ran with tool features {record['tool_features']}" + elif not legacy: + return f"{sid} has no recorded tool features" + notice = record.get("tool_notice_sha256") + if notice is None and not legacy: + return f"{sid} has no recorded tool notice" + if ( + notice is not None + and notice != hashlib.sha256(profile.notice.encode()).hexdigest() + ): + return f"{sid} ran under another tool notice" + schema = record.get("calculate_tool_sha256") + if schema is not None and schema != want["calculate_tool_sha256"]: + return f"{sid} ran with another calculator schema" + return None + + +def treatment_problems( + records: Mapping[str, dict], profile: "ModelProfile" +) -> list[str]: + """record_treatment_problem for every record in a run directory, a + rerunnable one (RERUNNABLE_STATUSES) too. A rerun is not sure to replace + it: a budget stop, lost workers or a narrower --scenarios leave it in + place, and scoring reads whatever is there (review r3). So a resume and + a score both refuse a directory holding any record from another treatment.""" + return [ + problem + for record in records.values() + if (problem := record_treatment_problem(record, profile)) is not None + ] + + +def scored_treatment_problems( + records: Mapping[str, dict], profile: "ModelProfile", engine: str +) -> tuple[list[str], str | None]: + """treatment_problems as score applies it, with the note score prints. + The records' notices state the board's engine, as command_run set it. + Runs before the texts stated the board's engine (6e1578fc) named the + reference engine whatever the board's: a directory written wholly so is + one treatment and scores as it ran. A mix of the two texts does not. A + resume stays strict, since its new records would state the board's.""" + with stating_engine(engine): + problems = treatment_problems(records, profile) + if not problems or engine == REFERENCE_ENGINE: + return problems, None + with stating_engine(REFERENCE_ENGINE): + if treatment_problems(records, profile): + return problems, None + return [], ( + f"NOTE: these records' notices name policyengine-us {REFERENCE_ENGINE}, " + f"as runs before 6e1578fc stated it; the board's engine is {engine}" + ) + + +def harness_error_record( + sid: str, variables: list[str], profile: "ModelProfile", error: Exception +) -> dict: + """The record of a household the harness itself failed on: rerunnable, + with every setup and treatment key a resume and score check, so a folder + holding it still resumes and scores.""" + record = { + "scenario_id": sid, + "model": profile.name, + "arm": profile.arm, + "prompt_version": profile.prompt_version, + "prompt_wording_sha256": prompt_wording_sha256(profile), + "benchmark_search_rule": profile.benchmark_search_rule, + "tool_notice_sha256": hashlib.sha256(profile.notice.encode()).hexdigest(), + **profile_treatment(profile), + "variables": variables, + "status": "harness_error", + "error": f"{type(error).__name__}: {error}"[:MAX_MESSAGE_CHARS], + "turns": [], + "predictions": {v: None for v in variables}, + "explanations": {v: None for v in variables}, + } + record["totals"] = household_totals(record) + return record + + def records_arm(records: Mapping[str, dict]) -> str: """The one arm a run directory's records belong to; records written before the web arms are the PE tool's.""" @@ -2565,7 +3903,10 @@ def records_arm(records: Mapping[str, dict]) -> str: def command_run(args) -> None: - _require_engine() + snapshot = board_path(args) + _require_engine(snapshot) + # The notices and the tool description state the board's engine. + set_stated_engine(board_engine(snapshot)) profile = model_profile( args.model, args.arm, args.prompt, args.forbid_benchmark_search ) @@ -2579,7 +3920,7 @@ def command_run(args) -> None: from litellm import completion, responses completion_fn = responses if profile.api == "responses" else completion - board = load_board() + board = load_board(snapshot) out = Path(args.out_dir) households_dir = out / "households" households_dir.mkdir(parents=True, exist_ok=True) @@ -2610,6 +3951,13 @@ def command_run(args) -> None: f"{households_dir} holds records sent under other v2-pilot wording " f"({records_wording(previous)}); use a fresh --out-dir" ) + mixed = treatment_problems(previous, profile) + if mixed: + raise SystemExit( + f"{households_dir} holds records from another treatment (" + + "; ".join(mixed[:3]) + + "); use a fresh --out-dir" + ) spend = Spend(args.budget_usd, spent_before) todo = [sid for sid in ids if sid not in done] manifest = { @@ -2620,6 +3968,9 @@ def command_run(args) -> None: "versions": _versions(), "fix_module": str(FIX_MODULE.relative_to(ROOT)), "fix_module_sha256": _sha256(FIX_MODULE), + "board": str(snapshot), + "board_engine": f"policyengine-us {board_engine(snapshot)}", + "board_reference_sha256": board["meta"].get("reference_csv_sha256"), "model": profile.name, "model_id": profile.model_id, "arm": profile.arm, @@ -2635,7 +3986,13 @@ def command_run(args) -> None: "api": profile.api, "answer_contract": profile.contract, "tool_notice": profile.notice, - "calculate_tool": calculate_tool_schema() if profile.uses_calculator else None, + "calculate_tool": calculate_tool_schema(features=profile.tool_features) + if profile.uses_calculator + else None, + "tool_features": sorted(profile.tool_features) + if profile.uses_calculator + else None, + "tool_version": profile.tool_version if profile.uses_calculator else None, "web_search_tool": web_search_tool(profile) if profile.uses_web else None, # The timeout actually sent: the board's for the model, or the web arms'. "request_timeout_seconds": build_initial_request( @@ -2662,7 +4019,16 @@ def command_run(args) -> None: if not todo: return # Arms without the calculator start no engine workers. - pool = ToolPool(size=args.workers) if profile.uses_calculator else None + pool = ( + ToolPool( + size=args.workers, + board=snapshot, + tool_version=profile.tool_version, + tool_features=profile.tool_features, + ) + if profile.uses_calculator + else None + ) def no_calculator(arguments) -> tuple[dict, float]: return {"error": "not_run", "message": "no calculator in this arm"}, 0.0 @@ -2690,23 +4056,9 @@ def one(sid: str) -> dict: profile=profile, ) except Exception as error: # one household must not end the run - record = { - "scenario_id": sid, - "model": profile.name, - # The setup keys a resume and score check, so a folder with - # this record still resumes and scores. - "arm": profile.arm, - "prompt_version": profile.prompt_version, - "prompt_wording_sha256": prompt_wording_sha256(profile), - "benchmark_search_rule": profile.benchmark_search_rule, - "variables": board["variables"][sid], - "status": "harness_error", - "error": f"{type(error).__name__}: {error}"[:MAX_MESSAGE_CHARS], - "turns": [], - "predictions": {v: None for v in board["variables"][sid]}, - "explanations": {v: None for v in board["variables"][sid]}, - } - record["totals"] = household_totals(record) + record = harness_error_record( + sid, board["variables"][sid], profile, error + ) if sid in previous: prior = previous[sid] record["prior_attempts"] = prior.get("prior_attempts", []) + [ @@ -2797,16 +4149,50 @@ def arm1_predictions( return rows +def baseline_predictions( + records: Mapping[str, dict], label: str = ARM1_LABEL +) -> pd.DataFrame: + """A no-tools run's records as the board's prediction frame, with each + household's usage on its first row so per-household sums are its totals.""" + frame = arm2_predictions(records, label) + usage = { + "prompt_tokens": "prompt_tokens", + "completion_tokens": "completion_tokens", + "reasoning_tokens": "reasoning_tokens", + "total_cost_usd": "cost_usd", + "elapsed_seconds": "model_seconds", + "cached_prompt_tokens": "cached_prompt_tokens", + } + first = ~frame.duplicated("scenario_id") + for column, total in usage.items(): + frame[column] = [ + float(records[sid]["totals"].get(total) or 0.0) if is_first else 0.0 + for sid, is_first in zip(frame.scenario_id, first) + ] + frame["call_id"] = frame.scenario_id + return frame + + def command_score(args) -> None: from policybench.analysis import analyze_no_tools from policybench.spec import output_group_id - board = load_board() + snapshot = board_path(args) + board = load_board(snapshot) out = Path(args.out_dir) records = load_records(out) profile = model_profile( records_model(records), records_arm(records), *records_prompt(records) ) + mixed, note = scored_treatment_problems(records, profile, board_engine(snapshot)) + if note: + print(note) + if mixed: + raise SystemExit( + f"{out / 'households'} holds records from another treatment (" + + "; ".join(mixed[:3]) + + "); scoring would mix them" + ) arm1_label, arm2_label = profile.arm1_label, profile.arm2_label complete = sorted( sid for sid, r in records.items() if r.get("status") == "complete" @@ -2825,7 +4211,28 @@ def command_score(args) -> None: f"failure and score as misses until a resume reruns them: {leftover}" ) scored = board["scored"][board["scored"].scenario_id.isin(households)] - arm1 = arm1_predictions(label=arm1_label) + if is_household_set(snapshot): + # No board run exists for a fresh household set: the baseline is the + # same model's no-tools arm on it, run with the same prompt setup. + baseline_dir = getattr(args, "baseline_dir", None) + if not baseline_dir: + raise SystemExit("a household set is scored against --baseline-dir") + baseline = load_records(Path(baseline_dir)) + if records_model(baseline) != profile.name or records_arm(baseline) != "none": + raise SystemExit( + f"--baseline-dir holds {records_model(baseline)} " + f"{records_arm(baseline)} records, not {profile.name} none" + ) + if records_prompt(baseline)[0] != profile.prompt_version: + raise SystemExit("--baseline-dir was run on another prompt version") + missing = sorted(set(households) - set(baseline)) + if missing: + raise SystemExit(f"--baseline-dir lacks households {missing}") + arm1 = baseline_predictions( + {sid: baseline[sid] for sid in households}, arm1_label + ) + else: + arm1 = arm1_predictions(label=arm1_label) arm1 = arm1[arm1.scenario_id.isin(households)] arm2 = arm2_predictions({sid: records[sid] for sid in households}, arm2_label) scenarios = board["scenarios_frame"][ @@ -3328,6 +4735,12 @@ def _successful_calls(record: dict) -> list[dict]: def _person_names(arguments) -> list[str]: try: args = json.loads(arguments) if isinstance(arguments, str) else arguments + if "household" in args: # the flat feature's people list + household = args["household"] + household = ( + json.loads(household) if isinstance(household, str) else household + ) + return [name or "" for name in flat_names(household["people"])] situation = args["situation"] situation = json.loads(situation) if isinstance(situation, str) else situation return list(situation["people"]) @@ -3934,7 +5347,9 @@ def main_call(record: dict, scenario, outputs: list[str]) -> dict | None: return main_call_and_count(record, scenario, outputs)[0] -def situation_fidelity(records, board, system, registry, defaults, out: Path) -> None: +def situation_fidelity( + records, board, system, registry, defaults, out: Path, features=frozenset() +) -> None: """How each household's main situation differs from the reference's. The main situation is ``main_call``'s: the successful call that computed @@ -3955,7 +5370,7 @@ def situation_fidelity(records, board, system, registry, defaults, out: Path) -> if call is None: rows.append({"scenario_id": sid, "differences": None}) continue - request, _ = normalize_request(call["arguments"], registry) + request, _ = normalize_request(call["arguments"], registry, features=features) if request is None: rows.append({"scenario_id": sid, "differences": None}) continue @@ -4003,9 +5418,12 @@ def situation_fidelity(records, board, system, registry, defaults, out: Path) -> def command_diagnose(args) -> None: from policybench.spec import metric_type_for_output - board = load_board() + snapshot = board_path(args) + board = load_board(snapshot) out = Path(args.out_dir) records = load_records(out) + # A run's calls are read as its own tool version read them. + features = arm_features(records_arm(records)) or frozenset() rows_path = out / "scored_rows.csv" if not rows_path.exists(): raise SystemExit("run score first") @@ -4014,14 +5432,14 @@ def command_diagnose(args) -> None: registry = None defaults = {} if not args.no_engine_defaults: - _require_engine() - system = build_tool_system(reform=True) + _require_engine(snapshot) + system = build_tool_system(reform=True, snapshot=snapshot) registry = build_registry(system) defaults = { name: system.variables[name].default_value for name in system.variables } if registry is not None: - situation_fidelity(records, board, system, registry, defaults, out) + situation_fidelity(records, board, system, registry, defaults, out, features) out_rows = [] if misses.empty: print("no arm-2 misses") @@ -4039,7 +5457,7 @@ def command_diagnose(args) -> None: if registry is not None: def board_value_for(arguments, scenario=scenario, output=row.variable): - request, _ = normalize_request(arguments, registry) + request, _ = normalize_request(arguments, registry, features=features) if request is None: return None return output_value(system, request["situation"], scenario, output) @@ -4063,7 +5481,7 @@ def board_value_for(arguments, scenario=scenario, output=row.variable): attribution = {} arguments = verdict.get("call_arguments") if arguments is not None and registry is not None: - request, _ = normalize_request(arguments, registry) + request, _ = normalize_request(arguments, registry, features=features) if request is not None: reference_situation = scenario.to_pe_household() diff = situation_diff( @@ -4146,6 +5564,12 @@ def main(argv: list[str] | None = None) -> None: for name in ("oracle", "run", "score", "diagnose"): p = sub.add_parser(name) p.add_argument("--out-dir", required=True) + p.add_argument( + "--board", + default=str(SNAPSHOT), + help="the frozen board (default) or a household set from " + "scripts/household_set.py", + ) if name == "run": p.add_argument( "--model", @@ -4154,9 +5578,10 @@ def main(argv: list[str] | None = None) -> None: ) p.add_argument( "--arm", - choices=ARMS, default="pe", - help="the tools: none, PolicyEngine (pe), web search (web), or both", + help="the tools: none, PolicyEngine (pe, pe2, pe3, or pe: plus " + "calculator features joined by +), web search (web), or both " + "(web+pe)", ) p.add_argument( "--prompt", @@ -4186,6 +5611,11 @@ def main(argv: list[str] | None = None) -> None: action="store_true", help="score only households whose conversation completed", ) + p.add_argument( + "--baseline-dir", + help="on a household set, the same model's no-tools run: the " + "baseline each arm is compared with", + ) if name == "diagnose": p.add_argument("--no-engine-defaults", action="store_true") args = parser.parse_args(argv) diff --git a/tests/conftest.py b/tests/conftest.py index 0802969c..27797a13 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,3 +57,58 @@ def low_income_scenario(): def sample_scenarios(simple_single_scenario, family_scenario, low_income_scenario): """A small set of test scenarios.""" return [simple_single_scenario, family_scenario, low_income_scenario] + + +# A three-household set built from the frozen board's first households, with +# sampling and the dataset download faked (tests/test_household_set.py, +# tests/test_batch_judge.py). Its references must be the frozen ones on the +# frozen engine, policyengine-us 2.41.0; on any other engine it is skipped. +@pytest.fixture(scope="session") +def household_board(tmp_path_factory): + """A three-household set built from the frozen board's first households.""" + import importlib.metadata + import sys + from pathlib import Path + + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + import household_set as hs + + if importlib.metadata.version("policyengine-us") != "2.41.0": + pytest.skip("the build test reproduces the 2.41.0 references") + import policybench.policyengine_runtime as runtime + import policybench.scenarios as scenarios_module + + root = tmp_path_factory.mktemp("set") + dataset = root / "populace_us_2024.h5" + dataset.write_bytes(b"not a real dataset") + digest = hs.sha256_file(dataset) + sampled = hs.load_scenarios(hs.FROZEN_BOARD)[:3] + patch = pytest.MonkeyPatch() + patch.setattr( + scenarios_module, + "generate_scenarios", + lambda n, seed, country, **kwargs: sampled[:n], + ) + patch.setattr(runtime, "materialize_certified_us_dataset", lambda: str(dataset)) + patch.setattr( + runtime, + "policyengine_bundles_for_countries", + lambda countries: { + "us": { + "bundle_id": "us-test", + "certified_data_artifact_sha256": digest, + "default_dataset_uri": "hf://policyengine/populace-us/x.h5@test", + # The bundle certifies its data for the installed engine. + "bundled_model_version": "2.41.0", + "model_version": "2.41.0", + } + }, + ) + out = root / "board" + try: + hs.main( + ["build", "--n", "3", "--seed", "7", "--batch-size", "2", "--out", str(out)] + ) + finally: + patch.undo() + return out, sampled diff --git a/tests/fake_anchor.py b/tests/fake_anchor.py new file mode 100644 index 00000000..73b5c32f --- /dev/null +++ b/tests/fake_anchor.py @@ -0,0 +1,13 @@ +"""The confirmatory runner's anchor command in its tests: a copy of the run's +anchors.jsonl to FOLDER/anchored.jsonl. + +Usage: python fake_anchor.py FOLDER ANCHORS +""" + +import shutil +import sys +from pathlib import Path + +folder, anchors = Path(sys.argv[1]), Path(sys.argv[2]) +shutil.copyfile(anchors, folder / "anchored.jsonl") +print("anchored") diff --git a/tests/test_batch_judge.py b/tests/test_batch_judge.py new file mode 100644 index 00000000..f9e4f7d2 --- /dev/null +++ b/tests/test_batch_judge.py @@ -0,0 +1,284 @@ +"""The batch judge's payload and derivations, and the adversary hooks it uses. + +The payload tests run on the three-household set from tests/conftest.py, so +they need the frozen engine; the adversary hook tests need no engine. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import batch_judge as bj # noqa: E402 +import tool_pilot_argon as pilot # noqa: E402 + +from policybench import reference_adversary as ra # noqa: E402 +from policybench.consensus import ConsensusParams, consensus_flags # noqa: E402 + +THREE = ConsensusParams(min_models=2, top_k=3, min_top=2, zero_cluster_min_models=2) + + +def _case(**overrides): + case = dict( + case_id="us__s__snap", + country="us", + scenario_id="s", + variable="snap", + definition="annual SNAP benefit", + household_prompt="A household.", + reference_value=100.0, + consensus=( + { + "answer": 0.0, + "n_models": 2, + "n_top": 2, + "top_models": ["a", "b"], + "members": [ + {"model": "a", "prediction": 0.0, "explanation": "x"}, + {"model": "b", "prediction": 0.0, "explanation": "y"}, + ], + }, + ), + models_answered=3, + models_exact=1, + derivation="snap = 100", + ) + case.update(overrides) + return ra.AdversaryCase(**case) + + +def test_the_default_condition_leaves_both_headers_unchanged(): + case = _case() + assert ra._STAGE1_HEADER in ra.render_stage1_prompt(case) + stage2 = ra.render_stage2_prompt(case, {"answer": 1}, "0" * 64) + assert stage2.startswith(ra._STAGE2_HEADER) + assert "answer_condition" not in case.to_manifest_row() + + +def test_a_web_condition_restates_both_headers_and_survives_the_manifest(): + case = _case(answer_condition="answering with web search") + stage1 = ra.render_stage1_prompt(case) + stage2 = ra.render_stage2_prompt(case, {"answer": 1}, "0" * 64) + for text in (stage1, stage2): + assert "AI models answering with web search, agreed" in text + assert ra.DEFAULT_ANSWER_CONDITION not in text + row = case.to_manifest_row() + assert row["answer_condition"] == "answering with web search" + again = ra.AdversaryCase.from_manifest_row(row, case.derivation) + assert again == case + default = _case() + assert ( + ra.AdversaryCase.from_manifest_row( + default.to_manifest_row(), default.derivation + ) + == default + ) + + +@pytest.mark.parametrize("condition", ["with web search", 3, ""]) +def test_a_malformed_condition_is_refused(condition): + with pytest.raises(ValueError, match="answerCondition"): + ra.answer_condition_of({"answerCondition": condition}) + + +def _payload(predictions, reference=100.0, definitions=None, condition=None): + payload = { + "country": "us", + "modelStats": [{"model": m} for m in ("a", "b", "c")], + "scenarios": {"s": {"prompt": {"tool": "A household."}, "state": "CA"}}, + "scenarioPredictions": { + "s": { + "snap": { + model: { + "prediction": value, + "explanation": f"{model} says {value}", + "groundTruth": reference, + "scored": True, + } + for model, value in predictions.items() + } + } + }, + } + if definitions is not None: + payload["outputDefinitions"] = definitions + if condition is not None: + payload["answerCondition"] = condition + return payload + + +def test_two_of_three_agreeing_off_the_reference_flag_a_cell_and_one_does_not(): + flagged = consensus_flags(_payload({"a": 0.0, "b": 0.0, "c": 100.0}), THREE) + assert [(f["scenario_id"], f["variable"]) for f in flagged] == [("s", "snap")] + assert flagged[0]["clusters"][0]["models"] == ["a", "b"] + alone = consensus_flags(_payload({"a": 0.0, "b": 50.0, "c": 100.0}), THREE) + assert alone == [] + exact = consensus_flags(_payload({"a": 100.4, "b": 99.6, "c": 100.0}), THREE) + assert exact == [] + + +def test_cases_take_the_payload_definition_and_condition(): + payload = _payload( + {"a": 0.0, "b": 0.0, "c": 100.0}, + definitions={"snap": "v2 SNAP definition"}, + condition="answering with web search", + ) + flags = consensus_flags(payload, THREE) + (case,) = ra.build_adversary_cases(payload, flags) + assert case.definition == "v2 SNAP definition" + assert case.answer_condition == "answering with web search" + payload.pop("outputDefinitions") + payload.pop("answerCondition") + (case,) = ra.build_adversary_cases(payload, flags) + assert case.answer_condition == ra.DEFAULT_ANSWER_CONDITION + assert case.definition != "v2 SNAP definition" + + +# --- On a household set ---------------------------------------------------- + + +def _write_arm(runs, board, model, arm, stage, wrong=None): + """Records for one model and arm whose sent prompt is the v2-pilot one.""" + reference = board["reference"].set_index(["scenario_id", "variable"])["value"] + folder = runs / model / bj.arm_folder(arm) / stage / "households" + folder.mkdir(parents=True) + for sid in sorted(board["scenarios"]): + scenario, variables = board["scenarios"][sid], board["variables"][sid] + prompt = bj.household_prompt(scenario, variables, "v2-pilot", model) + predictions = {v: float(reference[(sid, v)]) for v in variables} + if wrong and sid == wrong[0]: + predictions[wrong[1]] = wrong[2] + record = { + "scenario_id": sid, + "model": model, + "arm": arm, + "prompt_version": "v2-pilot", + "benchmark_search_rule": arm == "web", + "variables": variables, + "predictions": predictions, + "explanations": {v: f"{model} on {v}" for v in variables}, + "status": "complete", + "sent_board_prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + } + (folder / f"{sid}.json").write_text(json.dumps(record)) + + +@pytest.mark.slow +def test_a_batch_payload_flags_what_two_models_get_wrong_and_judges_it_blind( + household_board, tmp_path +): + out, _ = household_board + board = pilot.load_board(out) + variable = next( + v + for v in board["variables"]["mc_000"] + if v == "federal_income_tax_before_refundable_credits" + ) + runs = tmp_path / "runs" + for model in bj.MODELS: + wrong = ("mc_000", variable, 123456.0) if model != "gemini-4-argon" else None + _write_arm(runs, board, model, "web", "b0", wrong=wrong) + payload = bj.build_payload(out, runs, "b0", "web", batch=0) + assert sorted(payload["scenarios"]) == ["mc_000", "mc_001"] + assert payload["answerCondition"] == "answering with web search" + assert [m["model"] for m in payload["modelStats"]] == [ + f"{m}+web" for m in bj.MODELS + ] + medicaid = payload["outputDefinitions"].get("head_medicaid_eligible") + if medicaid is not None: + assert "PolicyEngine" not in medicaid + flags = consensus_flags(payload, THREE) + assert [(f["scenario_id"], f["variable"]) for f in flags] == [("mc_000", variable)] + (case,) = ra.build_adversary_cases(payload, flags) + assert case.household_prompt == payload["scenarios"]["mc_000"]["prompt"]["tool"] + assert "with web search" in ra.render_stage1_prompt(case) + + flags_path = tmp_path / "flags.json" + flags_path.write_text(json.dumps({"flags": flags})) + payload_path = tmp_path / "payload.json" + payload_path.write_text(json.dumps(payload)) + bj.main( + [ + "derivations", + "--board", + str(out), + "--payload", + str(payload_path), + "--flags", + str(flags_path), + "--out-dir", + str(tmp_path / "annotations"), + ] + ) + derivations = ra.load_derivations(tmp_path / "annotations") + text = derivations[("mc_000", variable)] + reference = board["reference"].set_index(["scenario_id", "variable"])["value"] + assert text.startswith( + f"Output {variable} = {float(reference[('mc_000', variable)]):g}" + ) + assert "income_tax_before_refundable_credits" in text + + +@pytest.mark.slow +def test_a_payload_refuses_an_unfinished_batch_a_calculator_arm_and_another_prompt( + household_board, tmp_path +): + out, _ = household_board + board = pilot.load_board(out) + runs = tmp_path / "runs" + with pytest.raises(SystemExit, match="only"): + bj.build_payload(out, runs, "b0", "pe", batch=0) + for model in bj.MODELS[:2]: + _write_arm(runs, board, model, "none", "b0") + with pytest.raises(SystemExit, match="has not finished"): + bj.build_payload(out, runs, "b0", "none", batch=0) + _write_arm(runs, board, bj.MODELS[2], "none", "b0") + record_path = runs / bj.PROMPT_MODEL / "none" / "b0" / "households" / "mc_000.json" + record = json.loads(record_path.read_text()) + record["sent_board_prompt_sha256"] = "0" * 64 + record_path.write_text(json.dumps(record)) + with pytest.raises(SystemExit, match="rebuilt prompt"): + bj.build_payload(out, runs, "b0", "none", batch=0) + with pytest.raises(SystemExit, match="batches 0 to 1"): + bj.build_payload(out, runs, "b0", "none", batch=2) + with pytest.raises(SystemExit, match="not both"): + bj.build_payload(out, runs, "b0", "none", batch=0, scenarios=["mc_000"]) + + +CHEAP = ("claude-haiku-5.5", "gpt-6-luna", "gemini-3.1-flash-lite-preview") + + +def test_the_prompt_model_is_the_first_on_the_tool_contract(): + assert bj.prompt_model_for(bj.MODELS) == "gpt-6-sol" + assert bj.prompt_model_for(CHEAP) == "claude-haiku-5.5" + assert bj.prompt_model_for(("gemini-3.1-flash-lite-preview", "gpt-6-luna")) == ( + "gpt-6-luna" + ) + + +@pytest.mark.slow +def test_a_payload_judges_any_model_set(household_board, tmp_path): + out, _ = household_board + board = pilot.load_board(out) + runs = tmp_path / "runs" + variable = "federal_income_tax_before_refundable_credits" + for model in CHEAP: + wrong = ("mc_000", variable, 999.0) if model != "gpt-6-luna" else None + _write_arm(runs, board, model, "none", "b0", wrong=wrong) + payload = bj.build_payload(out, runs, "b0", "none", batch=0, models=CHEAP) + assert payload["models"] == list(CHEAP) + assert [m["model"] for m in payload["modelStats"]] == [f"{m}+none" for m in CHEAP] + flags = consensus_flags(payload, THREE) + assert [(f["scenario_id"], f["variable"]) for f in flags] == [("mc_000", variable)] + assert flags[0]["clusters"][0]["models"] == [ + "claude-haiku-5.5+none", + "gemini-3.1-flash-lite-preview+none", + ] + with pytest.raises(SystemExit, match="at least two models"): + bj.build_payload(out, runs, "b0", "none", batch=0, models=CHEAP[:1]) diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py new file mode 100644 index 00000000..3af7be87 --- /dev/null +++ b/tests/test_confirmatory_anchor.py @@ -0,0 +1,1149 @@ +"""The anchor command: each call commits the run's anchors and pushes them.""" + +import fcntl +import hashlib +import io +import json +import math +import os +import select +import signal +import subprocess +import sys +import threading +import time +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import confirmatory_anchor # noqa: E402 +import confirmatory_run as cr # noqa: E402 + +SCRIPT = Path(confirmatory_anchor.__file__) + + +@pytest.fixture +def clone(tmp_path, monkeypatch): + """A clone of an empty bare remote, with git's own settings only.""" + empty = tmp_path / "gitconfig" + empty.write_text("") + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(empty)) + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + for role in ("AUTHOR", "COMMITTER"): + monkeypatch.setenv(f"GIT_{role}_NAME", "anchor test") + monkeypatch.setenv(f"GIT_{role}_EMAIL", "anchor@example.invalid") + remote = tmp_path / "remote.git" + subprocess.run(["git", "init", "-q", "--bare", str(remote)], check=True) + repo = tmp_path / "anchors" + subprocess.run(["git", "clone", "-q", str(remote), str(repo)], check=True) + return remote, repo + + +def remote_log(remote: Path) -> list[str]: + out = subprocess.run( + ["git", "-C", str(remote), "log", "--format=%s", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + return out.stdout.splitlines() + + +def remote_file(remote: Path) -> str: + out = subprocess.run( + ["git", "-C", str(remote), "show", "HEAD:anchors.jsonl"], + check=True, + capture_output=True, + text=True, + ) + return out.stdout + + +def anchor_command(repo: Path, anchors: Path): + """Run the real helper and retain its failure message and exit status.""" + return subprocess.run( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + capture_output=True, + text=True, + ) + + +@pytest.mark.parametrize("name", ["anchors.jsonl", "anchors.push-snapshot.jsonl"]) +def test_each_anchor_is_committed_and_pushed(clone, tmp_path, name): + """A snapshot's local name never changes the remote record's name.""" + remote, repo = clone + anchors = tmp_path / name + lines = ['{"time": 1.0, "prev": "a"}', '{"time": 2.0, "prev": "b"}'] + shas = [hashlib.sha256(line.encode()).hexdigest() for line in lines] + anchors.write_text(lines[0] + "\n") + assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 + anchors.write_text("\n".join(lines) + "\n") + assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 + assert remote_log(remote) == [f"anchor 2: {shas[1]}", f"anchor 1: {shas[0]}"] + assert remote_file(remote) == anchors.read_text() + assert sorted(path.name for path in repo.iterdir() if path.name != ".git") == [ + "anchors.jsonl" + ] + + +def test_an_unchanged_file_makes_no_commit(clone, tmp_path): + """An equal snapshot succeeds without changing the file or remote history.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + anchors.write_text('{"time": 1.0}\n') + for _ in range(2): + assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 + assert len(remote_log(remote)) == 1 + assert remote_file(remote) == anchors.read_text() + + +def test_an_older_snapshot_cannot_replace_a_newer_remote_file(clone, tmp_path): + """A delayed helper cannot commit or push a prefix after the newer snapshot.""" + remote, repo = clone + older, newer = tmp_path / "older.jsonl", tmp_path / "newer.jsonl" + older.write_bytes(b'{"time": 1.0}\n') + newer.write_bytes(older.read_bytes() + b'{"time": 2.0}\n') + assert anchor_command(repo, newer).returncode == 0 + history, index = remote_log(remote), (repo / ".git" / "index").read_bytes() + out = anchor_command(repo, older) + assert out.returncode != 0 and "not a byte prefix" in out.stderr + assert (repo / "anchors.jsonl").read_bytes() == newer.read_bytes() + assert (repo / ".git" / "index").read_bytes() == index + assert remote_file(remote).encode() == newer.read_bytes() + assert remote_log(remote) == history + + +def test_a_busy_clone_is_refused_without_changing_its_records(clone, tmp_path): + """A held flock prevents target, index, commit and remote writes.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + history, index = remote_log(remote), (repo / ".git" / "index").read_bytes() + anchors.write_bytes(first + b'{"time": 2.0}\n') + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + out = anchor_command(repo, anchors) + assert out.returncode != 0 and "busy" in out.stderr + assert (repo / "anchors.jsonl").read_bytes() == first + assert (repo / ".git" / "index").read_bytes() == index + assert remote_file(remote).encode() == first + assert remote_log(remote) == history + + +@pytest.mark.parametrize("hook_name", ["pre-commit", "pre-push", "pre-receive"]) +def test_killing_only_the_helper_keeps_its_git_writer_locked( + clone, tmp_path, hook_name +): + """A Git child already using the clone retains the lock after helper death.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + older = first + b'{"time": 2.0}\n' + snapshots = [older + b'{"time": 3.0}\n', older + b'{"time": 3.0}\n{"time": 4.0}\n'] + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + ready, release = tmp_path / "ready", tmp_path / "release" + for path in (ready, release): + os.mkfifo(path) + hook_lock = tmp_path / "hook-lock.json" + hooks = remote / "hooks" if hook_name == "pre-receive" else repo / ".git" / "hooks" + hook = hooks / hook_name + hook.write_text( + f"#!{sys.executable}\n" + "import json, os\n" + "from pathlib import Path\n" + "if os.environ.get('ANCHOR_TEST_BLOCK') == '1':\n" + f" lock = Path({str(repo / '.git' / 'confirmatory-anchor.lock')!r}).stat()\n" + " inherited = []\n" + " for fd in range(3, 256):\n" + " try:\n" + " info = os.fstat(fd)\n" + " except OSError:\n" + " continue\n" + " if (info.st_dev, info.st_ino) == (lock.st_dev, lock.st_ino):\n" + " inherited.append(os.get_inheritable(fd))\n" + f" Path({str(hook_lock)!r}).write_text(json.dumps(inherited))\n" + f" with open({str(ready)!r}, 'wb', buffering=0) as pipe:\n" + " pipe.write(b'1')\n" + f" with open({str(release)!r}, 'rb', buffering=0) as pipe:\n" + " pipe.read(1)\n" + ) + hook.chmod(0o755) + + def records(): + """Read the clone's file, index, HEAD and the remote without writing.""" + objects = [ + subprocess.run( + ["git", "-C", str(repo), "show", obj], check=True, capture_output=True + ).stdout + for obj in (":anchors.jsonl", "HEAD:anchors.jsonl") + ] + return ( + (repo / "anchors.jsonl").read_bytes(), + (repo / ".git" / "index").read_bytes(), + *objects, + remote_file(remote).encode(), + remote_log(remote), + ) + + ready_fd, release_fd = (os.open(path, os.O_RDWR) for path in (ready, release)) + child = None + try: + anchors.write_bytes(older) + child = subprocess.Popen( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + env={**os.environ, "ANCHOR_TEST_BLOCK": "1"}, + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert select.select([ready_fd], [], [], 15.0)[0], "Git did not reach its hook" + assert os.read(ready_fd, 1) == b"1" + before = records() + assert before[0] == before[2] == older + child.kill() + assert child.wait(timeout=15.0) == -signal.SIGKILL + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 1 and "busy" in out.stderr, ( + "a surviving Git writer lost the helper's clone lock: " + out.stderr + ) + assert records() == before + assert json.loads(hook_lock.read_text()) == [True] + os.write(release_fd, b"1") + child.communicate(timeout=15.0) + # Pipe closure need not mean every inherited lock holder has ended. + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + deadline = time.monotonic() + 15.0 + while True: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, "a Git child did not finish" + threading.Event().wait(0.01) + after = records() + assert after[0] == after[2] == after[3] == older + assert after[4] in (first, older) + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + state = records() + assert state[0] == state[2] == state[3] == state[4] == snapshot + history = subprocess.run( + ["git", "-C", str(remote), "rev-list", "--reverse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + previous = b"" + for commit in history: + content = subprocess.run( + ["git", "-C", str(remote), "show", f"{commit}:anchors.jsonl"], + check=True, + capture_output=True, + ).stdout + assert content.startswith(previous) + previous = content + assert previous == snapshots[-1] + finally: + if child is not None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.communicate(timeout=15.0) + for fd in (ready_fd, release_fd): + os.close(fd) + + +@pytest.mark.parametrize("command", ["add", "status"]) +def test_killing_only_the_helper_during_add_or_status_keeps_the_clone_locked( + clone, tmp_path, command +): + """A clean filter pins each real Git command across helper-only death.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + older = first + b'{"time": 2.0}\n' + snapshots = [older + b'{"time": 3.0}\n', older + b'{"time": 3.0}\n{"time": 4.0}\n'] + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + ready, release = tmp_path / "ready", tmp_path / "release" + for path in (ready, release): + os.mkfifo(path) + invocation = tmp_path / "filter-invocation" + filter_lock = tmp_path / "filter-lock.json" + clean = tmp_path / "clean-filter" + block_at = 1 if command == "add" else 2 + clean.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "content = sys.stdin.buffer.read()\n" + "if os.environ.get('ANCHOR_TEST_BLOCK') == '1':\n" + f" invocation = Path({str(invocation)!r})\n" + " count = int(invocation.read_text()) + 1 if invocation.exists() else 1\n" + " invocation.write_text(str(count))\n" + f" if count == {block_at}:\n" + f" lock = Path({str(repo / '.git' / 'confirmatory-anchor.lock')!r})" + ".stat()\n" + " inherited = []\n" + " for fd in range(3, 256):\n" + " try:\n" + " info = os.fstat(fd)\n" + " except OSError:\n" + " continue\n" + " if (info.st_dev, info.st_ino) == (lock.st_dev, lock.st_ino):\n" + " inherited.append(os.get_inheritable(fd))\n" + f" Path({str(filter_lock)!r}).write_text(json.dumps(inherited))\n" + f" with open({str(ready)!r}, 'wb', buffering=0) as pipe:\n" + " pipe.write(b'1')\n" + f" with open({str(release)!r}, 'rb', buffering=0) as pipe:\n" + " pipe.read(1)\n" + f" elif count == 1 and {command == 'status'!r}:\n" + f" target = Path({str(repo / 'anchors.jsonl')!r})\n" + " info = target.stat()\n" + " os.utime(target, ns=(info.st_atime_ns, " + "info.st_mtime_ns - 2000000000))\n" + "sys.stdout.buffer.write(content)\n" + ) + clean.chmod(0o755) + (repo / ".git" / "info" / "attributes").write_text("anchors.jsonl filter=anchor\n") + for name, value in (("clean", str(clean)), ("required", "true")): + subprocess.run( + ["git", "-C", str(repo), "config", f"filter.anchor.{name}", value], + check=True, + ) + + def records(): + """Read file, raw index, staged and committed bytes and remote history.""" + objects = [ + subprocess.run( + ["git", "-C", str(repo), "show", obj], check=True, capture_output=True + ).stdout + for obj in (":anchors.jsonl", "HEAD:anchors.jsonl") + ] + return ( + (repo / "anchors.jsonl").read_bytes(), + (repo / ".git" / "index").read_bytes(), + *objects, + remote_file(remote).encode(), + remote_log(remote), + ) + + ready_fd, release_fd = (os.open(path, os.O_RDWR) for path in (ready, release)) + child = None + try: + anchors.write_bytes(older) + child = subprocess.Popen( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + env={**os.environ, "ANCHOR_TEST_BLOCK": "1"}, + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert select.select([ready_fd], [], [], 15.0)[0], ( + f"Git {command} did not reach its clean filter" + ) + assert os.read(ready_fd, 1) == b"1" + # The first conversion runs in add; changing its stat forces status's second. + assert invocation.read_text() == str(block_at) + before = records() + assert before[0] == older + assert before[2] == (first if command == "add" else older) + assert before[3] == before[4] == first + child.kill() + assert child.wait(timeout=15.0) == -signal.SIGKILL + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 1 and "busy" in out.stderr, ( + f"a surviving Git {command} lost the helper's clone lock: " + out.stderr + ) + assert records() == before + assert json.loads(filter_lock.read_text()) == [True] + os.write(release_fd, b"1") + child.communicate(timeout=15.0) + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + deadline = time.monotonic() + 15.0 + while True: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, f"Git {command} did not finish" + threading.Event().wait(0.01) + after = records() + assert after[0] == after[2] == older + assert after[3] == after[4] == first + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + state = records() + assert state[0] == state[2] == state[3] == state[4] == snapshot + history = subprocess.run( + ["git", "-C", str(remote), "rev-list", "--reverse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + previous = b"" + for commit in history: + content = subprocess.run( + ["git", "-C", str(remote), "show", f"{commit}:anchors.jsonl"], + check=True, + capture_output=True, + ).stdout + assert content.startswith(previous) + previous = content + assert previous == snapshots[-1] + finally: + if child is not None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.communicate(timeout=15.0) + for fd in (ready_fd, release_fd): + os.close(fd) + + +def test_a_detached_hook_descendant_keeps_the_clone_locked(clone, tmp_path): + """A double-forked holder outlives its hook and helper until explicitly released.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + older = first + b'{"time": 2.0}\n' + snapshots = [older + b'{"time": 3.0}\n', older + b'{"time": 3.0}\n{"time": 4.0}\n'] + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + ready, release = tmp_path / "ready", tmp_path / "release" + for path in (ready, release): + os.mkfifo(path) + holder = tmp_path / "detached-holder.json" + hook = repo / ".git" / "hooks" / "pre-push" + hook.write_text( + f"#!{sys.executable}\n" + "import json, os\n" + "from pathlib import Path\n" + "if os.environ.get('ANCHOR_TEST_DETACH') == '1':\n" + f" ready = os.open({str(ready)!r}, os.O_WRONLY)\n" + f" release = os.open({str(release)!r}, os.O_RDONLY)\n" + " hook_pid = os.getpid()\n" + " child = os.fork()\n" + " if child:\n" + " os.waitpid(child, 0)\n" + " raise SystemExit(0)\n" + " os.setsid()\n" + " if os.fork():\n" + " os._exit(0)\n" + " with open(os.devnull, 'r+b', buffering=0) as null:\n" + " for fd in range(3):\n" + " os.dup2(null.fileno(), fd)\n" + f" lock = Path({str(repo / '.git' / 'confirmatory-anchor.lock')!r}).stat()\n" + " inherited = []\n" + " for fd in range(3, 256):\n" + " try:\n" + " info = os.fstat(fd)\n" + " except OSError:\n" + " continue\n" + " if (info.st_dev, info.st_ino) == (lock.st_dev, lock.st_ino):\n" + " inherited.append(os.get_inheritable(fd))\n" + " state = dict(pid=os.getpid(), sid=os.getsid(0), pgid=os.getpgrp(),\n" + " hook_pid=hook_pid, inherited=inherited)\n" + f" Path({str(holder)!r}).write_text(json.dumps(state))\n" + " os.write(ready, b'1')\n" + " os.read(release, 1)\n" + " os._exit(0)\n" + ) + hook.chmod(0o755) + + def records(): + """Read the clone's file, index, HEAD and remote without writing.""" + objects = [ + subprocess.run( + ["git", "-C", str(repo), "show", obj], check=True, capture_output=True + ).stdout + for obj in (":anchors.jsonl", "HEAD:anchors.jsonl") + ] + return ( + (repo / "anchors.jsonl").read_bytes(), + (repo / ".git" / "index").read_bytes(), + *objects, + remote_file(remote).encode(), + remote_log(remote), + ) + + ready_fd, release_fd = (os.open(path, os.O_RDWR) for path in (ready, release)) + child = None + descendant = None + try: + anchors.write_bytes(older) + child = subprocess.Popen( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + env={**os.environ, "ANCHOR_TEST_DETACH": "1"}, + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert select.select([ready_fd], [], [], 15.0)[0], ( + "hook did not detach its holder" + ) + assert os.read(ready_fd, 1) == b"1" + state = json.loads(holder.read_text()) + descendant = state["pid"] + stdout, stderr = child.communicate(timeout=15.0) + assert child.returncode == 0, stderr.decode() + assert stdout == b"pushed 2 anchors\n" + assert state["sid"] != child.pid and state["pgid"] != child.pid + assert descendant != state["sid"] + os.kill(descendant, 0) + with pytest.raises(ProcessLookupError): + os.kill(state["hook_pid"], 0) + before = records() + assert before[0] == before[2] == before[3] == before[4] == older + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 1 and "busy" in out.stderr, ( + "a detached hook descendant lost the helper's clone lock: " + out.stderr + ) + assert records() == before + assert state["inherited"] == [True] + os.write(release_fd, b"1") + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + deadline = time.monotonic() + 15.0 + while True: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, ( + "the detached holder did not exit" + ) + threading.Event().wait(0.01) + descendant = None + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + state = records() + assert state[0] == state[2] == state[3] == state[4] == snapshot + finally: + os.write(release_fd, b"1") + if descendant is not None: + try: + os.kill(descendant, signal.SIGKILL) + except ProcessLookupError: + pass + if child is not None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.communicate(timeout=15.0) + for fd in (ready_fd, release_fd): + os.close(fd) + + +def test_a_changed_middle_line_is_refused_without_writing(clone, tmp_path): + """A longer snapshot still fails when it rewrites an existing line.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n{"time": 2.0}\n{"time": 3.0}\n' + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + history, index = remote_log(remote), (repo / ".git" / "index").read_bytes() + anchors.write_bytes(first.replace(b"2.0", b"9.0") + b'{"time": 4.0}\n') + out = anchor_command(repo, anchors) + assert out.returncode != 0 and "not a byte prefix" in out.stderr + assert (repo / "anchors.jsonl").read_bytes() == first + assert (repo / ".git" / "index").read_bytes() == index + assert remote_file(remote).encode() == first + assert remote_log(remote) == history + + +def test_a_stale_git_index_lock_is_left_alone(clone, tmp_path): + """A failed git command never removes another command's index lock.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + history = remote_log(remote) + index_lock = repo / ".git" / "index.lock" + index_lock.write_bytes(b"stale lock kept for its owner\n") + anchors.write_bytes(first + b'{"time": 2.0}\n') + out = anchor_command(repo, anchors) + assert out.returncode != 0 and "CalledProcessError" in out.stderr + assert index_lock.read_bytes() == b"stale lock kept for its owner\n" + assert remote_file(remote).encode() == first + assert remote_log(remote) == history + + +def test_normal_index_fsmonitor_startup_is_disabled(clone, tmp_path): + """The per-command override suppresses a configured index-triggered monitor.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + anchors.write_bytes(b'{"time": 1.0}\n') + assert anchor_command(repo, anchors).returncode == 0 + invoked, monitor = tmp_path / "monitor-invoked", tmp_path / "fsmonitor" + monitor.write_text( + f"#!{sys.executable}\n" + "from pathlib import Path\n" + f"Path({str(invoked)!r}).touch()\n" + ) + monitor.chmod(0o755) + subprocess.run( + ["git", "-C", str(repo), "config", "core.fsmonitor", str(monitor)], check=True + ) + anchors.write_bytes(anchors.read_bytes() + b'{"time": 2.0}\n') + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + assert not invoked.exists() + assert remote_file(remote).encode() == anchors.read_bytes() + + +def test_a_failed_push_exits_nonzero(clone, tmp_path): + """The runner records the command's return code; a push that fails must + not look like one that landed.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + anchors.write_text('{"time": 1.0}\n') + subprocess.run( + ["git", "-C", str(repo), "remote", "set-url", "origin", str(tmp_path / "gone")], + check=True, + ) + out = subprocess.run( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + capture_output=True, + text=True, + ) + assert out.returncode != 0 + + +def bare_run(folder, command): + """Just the anchor writer, without the board, engines or providers.""" + run = object.__new__(cr.Run) + run.design = {"runs": str(folder), "anchor_command": command} + run.out, run._say_lock = io.StringIO(), threading.Lock() + run.lanes, run.host = {}, None + run._anchor_head = cr.chain_seed("a" * 64, "", "anchors.jsonl") + run._anchored = run._next_anchor = -math.inf + run._pushing, run._push = threading.Lock(), None + run._push_error = None + return run + + +def bare_finish_run(folder, command): + """An anchor writer with an empty, already closed run to finish.""" + + class Ledger: + def close(self, pending): + return [] + + run = bare_run(folder, command) + run.ceiling, run._failed = Ledger(), threading.Event() + run.signalled, run._interrupted = None, False + run.watch = lambda anchor=True: None + return run + + +def test_failed_pushes_retry_and_keep_every_outcome(tmp_path, monkeypatch): + """A failed remote never looks anchored; retry follows the failure's end, + then a successful push restores the ordinary cadence.""" + now, calls = [1000.0], [] + monkeypatch.setattr(cr.time, "time", lambda: now[0]) + monkeypatch.setattr(cr, "clock", lambda: now[0]) + monkeypatch.setattr(cr, "ANCHOR_SECONDS", 900.0) + monkeypatch.setattr(cr, "ANCHOR_RETRY", 60.0) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 123.0) + + class Push: + def __init__(self, command, **kwargs): + calls.append((command, kwargs)) + self.returncode = 1 if len(calls) < 4 else 0 + + def communicate(self, timeout): + assert timeout == cr.ANCHOR_PUSH_TIMEOUT + now[0] += 7.0 + return "", "remote unavailable" if self.returncode else "pushed" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + run = bare_run(tmp_path, ["push-anchors"]) + assert run.anchor_due() + for i in range(4): + started = now[0] + run.anchor(wait=True) + assert not run.anchor_due() + expected = now[0] + cr.ANCHOR_RETRY if i < 3 else started + cr.ANCHOR_SECONDS + assert run._next_anchor == expected + now[0] = expected + assert run.anchor_due() + anchors = [ + json.loads(x) for x in (tmp_path / "anchors.jsonl").read_text().splitlines() + ] + pushes = [ + json.loads(x) + for x in (tmp_path / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert [x["outcome"] for x in pushes] == ["failed"] * 3 + ["success"] + assert [x["returncode"] for x in pushes] == [1, 1, 1, 0] + assert [x["anchor_time"] for x in pushes] == [x["time"] for x in anchors] + assert all(x["started"] == x["anchor_time"] < x["ended"] for x in pushes) + assert all(x[1]["start_new_session"] for x in calls) + assert len({x["anchor"] for x in pushes}) == 4 + + +def test_an_anchor_command_that_cannot_start_is_recorded(tmp_path): + run = bare_run(tmp_path, [str(tmp_path / "missing-command")]) + run.anchor(wait=True) + push = json.loads((tmp_path / "anchor-pushes.jsonl").read_text()) + assert push["outcome"] == "error" and push["returncode"] is None + assert "FileNotFoundError" in push["output"] + assert run._next_anchor == push["ended"] + cr.ANCHOR_RETRY + + +@pytest.mark.parametrize("wait", [False, True]) +def test_push_log_io_failure_reaches_the_supervisor(tmp_path, monkeypatch, wait): + """A successful remote push whose outcome could not be synced is still a + runner error, both in supervision and in the final blocking anchor.""" + calls, fsync = [], os.fsync + + class Push: + returncode = 0 + + def __init__(self, *args, **kwargs): + pass + + def communicate(self, timeout): + return "pushed", "" + + def fail_push_sync(fd): + calls.append(fd) + if (tmp_path / "anchor-pushes.jsonl").exists(): + # The anchor and its copy synced; only the outcome's sync fails. + raise OSError("push-log fsync failed") + return fsync(fd) + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + monkeypatch.setattr(cr.os, "fsync", fail_push_sync) + run = bare_run(tmp_path, ["push-anchors"]) + if wait: + with pytest.raises(OSError, match="anchor push outcome could not be recorded"): + run.anchor(wait=True) + else: + run.anchor() + run._push.join() + with pytest.raises(OSError, match="push-log fsync failed"): + run.anchor_due() + assert not run._push.is_alive() + assert len(calls) == 3 + + +def test_a_running_push_reads_a_snapshot_while_the_final_anchor_is_appended( + tmp_path, monkeypatch +): + """A push released after the final append still sees its own whole prefix.""" + reading, release, joining = threading.Event(), threading.Event(), threading.Event() + calls, copies, errors = [], [], [] + + class Push: + returncode = 0 + + def __init__(self, command, **kwargs): + self.source = Path(command[-1]) + calls.append(self.source) + self.first = len(calls) == 1 + + def communicate(self, timeout): + if self.first: + reading.set() + assert release.wait(15.0), "the first push was never released" + copies.append(self.source.read_bytes()) + return "pushed", "" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + run = bare_run(tmp_path, ["push-anchors"]) + run.anchor() + assert reading.wait(15.0), "the first push did not start" + first, join = run._push, run._push.join + + def joining_push(timeout=None): + joining.set() + return join(timeout=timeout) + + def final_anchor(): + try: + run.anchor(wait=True) + except BaseException as error: + errors.append(error) + + monkeypatch.setattr(first, "join", joining_push) + final = threading.Thread(target=final_anchor, daemon=True) + final.start() + try: + assert joining.wait(15.0), "the final anchor did not join the blocked push" + live = (tmp_path / "anchors.jsonl").read_bytes() + assert len(live.splitlines()) == 2 + assert calls[0] != tmp_path / "anchors.jsonl" + assert calls[0].name.endswith("push-snapshot.jsonl") + assert len(calls[0].read_bytes().splitlines()) == 1 + finally: + release.set() + final.join(timeout=15.0) + assert not final.is_alive() and not errors + live = (tmp_path / "anchors.jsonl").read_bytes() + assert len(copies) == 2 and copies[-1] == live + assert all(copy.endswith(b"\n") and live.startswith(copy) for copy in copies) + assert len(set(calls)) == 2 + + +def test_push_times_use_the_ordered_clock_after_a_wall_clock_step_back( + tmp_path, monkeypatch +): + """The anchor, its cadence and its push never precede that anchor's time.""" + base = time.time() + 1000.0 + readings = iter([base, base - 0.0001]) + monkeypatch.setattr(cr, "_clock_last", 0.0) + monkeypatch.setattr(cr.time, "time", lambda: next(readings, base - 0.0001)) + + class Push: + returncode = 0 + + def __init__(self, *args, **kwargs): + pass + + def communicate(self, timeout): + return "pushed", "" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + run = bare_run(tmp_path, ["push-anchors"]) + run.anchor(wait=True) + anchor = json.loads((tmp_path / "anchors.jsonl").read_text()) + push = json.loads((tmp_path / "anchor-pushes.jsonl").read_text()) + assert anchor["time"] <= run._anchored <= push["started"] <= push["ended"] + assert push["anchor_time"] == anchor["time"] + + +def test_a_push_timeout_kills_its_children_and_grandchildren(tmp_path, monkeypatch): + """All descendants inherit the captured pipe: reaping the command cannot + finish until they close it. The timeout starts after both have launched.""" + script = tmp_path / "push.py" + script.write_text( + "import os, subprocess, sys, threading\n" + "from pathlib import Path\n" + "folder, level = Path(sys.argv[1]), int(sys.argv[2])\n" + "(folder / f'pid-{level}').write_text(str(os.getpid()))\n" + "if level < 2:\n" + " child = subprocess.Popen(\n" + " [sys.executable, __file__, str(folder), str(level + 1)])\n" + "threading.Event().wait()\n" + ) + run = bare_run(tmp_path, [sys.executable, str(script), str(tmp_path), "0"]) + real_popen, real_killpg = subprocess.Popen, os.killpg + children, kills, errors = [], [], [] + done = threading.Event() + + def popen(*args, **kwargs): + child = real_popen(*args, **kwargs) + children.append(child) + communicate = child.communicate + + def ready_communicate(timeout=None): + if timeout is not None: + deadline = time.monotonic() + 30.0 + ready = tmp_path / "pid-2" + while not ready.exists() or not ready.read_text().strip(): + assert time.monotonic() < deadline, "the descendants did not start" + done.wait(0.01) + return communicate(timeout=timeout) + + child.communicate = ready_communicate + return child + + def killpg(pid, signum): + kills.append((pid, signum)) + return real_killpg(pid, signum) + + def push(): + try: + run.anchor(wait=True) + except BaseException as error: + errors.append(error) + finally: + done.set() + + with monkeypatch.context() as patch: + patch.setattr(cr.subprocess, "Popen", popen) + patch.setattr(cr.os, "killpg", killpg) + patch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 0.05) + thread = threading.Thread(target=push, daemon=True) + thread.start() + finished = done.wait(45.0) + if not finished: + for child in children: + try: + real_killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + thread.join(timeout=5.0) + assert finished, "the timed-out command left descendants holding its pipes" + assert not errors + assert len(children) == 1 and kills == [(children[0].pid, signal.SIGKILL)] + push = json.loads((tmp_path / "anchor-pushes.jsonl").read_text()) + assert push["outcome"] == "timeout" and push["returncode"] == -signal.SIGKILL + assert push["started"] <= push["ended"] + for level in (1, 2): + assert int((tmp_path / f"pid-{level}").read_text()) > 0 + + +def test_finish_abandons_pipes_held_by_a_descendant_outside_the_push_group( + tmp_path, monkeypatch +): + """An escaped grandchild cannot hold finish open after the kill's grace.""" + script = tmp_path / "escaped-push.py" + script.write_text( + "import os, subprocess, sys, threading\n" + "from pathlib import Path\n" + "folder = Path(sys.argv[1])\n" + "if len(sys.argv) > 2 and sys.argv[2] == 'escaped':\n" + " (folder / 'escaped-pid').write_text(str(os.getpid()))\n" + "else:\n" + " subprocess.Popen(\n" + " [sys.executable, __file__, str(folder), 'escaped'],\n" + " start_new_session=True)\n" + "threading.Event().wait()\n" + ) + run = bare_finish_run(tmp_path, [sys.executable, str(script), str(tmp_path)]) + real_popen = subprocess.Popen + ready, done = threading.Event(), threading.Event() + children, errors = [], [] + + def popen(*args, **kwargs): + child = real_popen(*args, **kwargs) + children.append(child) + communicate = child.communicate + + def ready_communicate(timeout=None): + if not ready.is_set(): + deadline = time.monotonic() + 15.0 + pid = tmp_path / "escaped-pid" + while not pid.exists() or not pid.read_text().strip(): + assert time.monotonic() < deadline, "the grandchild did not start" + done.wait(0.01) + ready.set() + return communicate(timeout=timeout) + + child.communicate = ready_communicate + return child + + def finish(): + try: + run.finish() + except BaseException as error: + errors.append(error) + finally: + done.set() + + monkeypatch.setattr(cr.subprocess, "Popen", popen) + monkeypatch.setattr(cr.signal, "signal", lambda *args: None) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 0.05) + monkeypatch.setattr(cr, "KILL_GRACE", 0.05) + monkeypatch.setattr(cr, "FINAL_PUSH_TRIES", 1) + thread = threading.Thread(target=finish, daemon=True) + thread.start() + try: + assert ready.wait(15.0), "the push never launched its grandchild" + started = time.monotonic() + finished = done.wait(10.0) + elapsed = time.monotonic() - started + finally: + pid = tmp_path / "escaped-pid" + if pid.exists() and pid.read_text().strip(): + try: + os.kill(int(pid.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass + for child in children: + if child.poll() is None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + thread.join(timeout=15.0) + assert finished and elapsed < 10.0, "finish waited for the escaped child's pipe" + assert not errors and not thread.is_alive() + rows = [ + json.loads(line) + for line in (tmp_path / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert len(rows) == 1 and rows[0]["outcome"] == "timeout" + assert ( + "pipes" in rows[0]["output"] and "held open after the kill" in rows[0]["output"] + ) + + +def test_finish_records_a_stuck_push_once_even_if_its_thread_later_ends( + tmp_path, monkeypatch +): + """A bounded join retires the worker's right to append a second outcome.""" + started, release = threading.Event(), threading.Event() + calls, waits, copies = [], [], {} + + class Push: + returncode = 0 + + def __init__(self, command, **kwargs): + calls.append(command) + self.first = len(calls) == 1 + self.source = Path(command[-1]) + self.returncode = 1 if self.first else 0 + + def communicate(self, timeout): + if self.first: + started.set() + assert release.wait(15.0), "the abandoned worker was never released" + copies[self.source] = self.source.read_bytes() + return "pushed", "" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + monkeypatch.setattr(cr.signal, "signal", lambda *args: None) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 0.05) + monkeypatch.setattr(cr, "KILL_GRACE", 0.05) + run = bare_finish_run(tmp_path, ["push-anchors"]) + run.anchor() + assert started.wait(15.0), "the periodic push did not start" + worker, join = run._push, run._push.join + + def short_join(timeout=None): + waits.append(timeout) + return join(timeout=0.02) + + monkeypatch.setattr(worker, "join", short_join) + try: + began = time.monotonic() + run.finish() + assert time.monotonic() - began < 10.0 + path = tmp_path / "anchor-pushes.jsonl" + before = path.read_bytes() + rows = [json.loads(line) for line in before.splitlines()] + assert waits == [pytest.approx(cr.ANCHOR_PUSH_TIMEOUT + cr.KILL_GRACE + 30)] + assert len(rows) == 2 + assert rows[0]["outcome"] == "error" and rows[0]["returncode"] is None + assert "push thread did not end" in rows[0]["output"] + assert "command may still be running" in rows[0]["output"] + assert rows[1]["outcome"] == "success" + next_anchor = run._next_anchor + finally: + release.set() + join(timeout=15.0) + assert not worker.is_alive() + assert path.read_bytes() == before + assert len({row["anchor"] for row in rows}) == len(rows) + assert run._last_push_outcome == "success" and run._next_anchor == next_anchor + live = (tmp_path / "anchors.jsonl").read_bytes() + assert len(copies[Path(calls[0][-1])].splitlines()) == 1 + assert len(copies[Path(calls[1][-1])].splitlines()) == 2 + assert all( + copy.endswith(b"\n") and live.startswith(copy) for copy in copies.values() + ) + + +def test_a_retired_command_cannot_shrink_the_remote_after_the_final_push( + clone, tmp_path, monkeypatch +): + """A live old command released after finish cannot replace newer evidence.""" + remote, repo = clone + folder = tmp_path / "run" + folder.mkdir() + delayed = tmp_path / "delayed-helper.py" + delayed.write_text( + "import os, sys\n" + "ready, release = map(int, sys.argv[1:3])\n" + "os.write(ready, b'1')\n" + "os.close(ready)\n" + "os.read(release, 1)\n" + "os.close(release)\n" + "os.execv(sys.executable, [sys.executable, *sys.argv[3:]])\n" + ) + ready_read, ready_write = os.pipe() + release_read, release_write = os.pipe() + real_popen, children, waits = subprocess.Popen, [], [] + run = bare_finish_run(folder, [sys.executable, str(SCRIPT), "--repo", str(repo)]) + + def popen(command, **kwargs): + """Delay only the first real helper, before it can acquire the clone lock.""" + if len(command) > 1 and command[1] == str(SCRIPT): + if not children: + command = [ + sys.executable, + str(delayed), + str(ready_write), + str(release_read), + *command[1:], + ] + kwargs["pass_fds"] = (ready_write, release_read) + child = real_popen(command, **kwargs) + children.append(child) + return child + return real_popen(command, **kwargs) + + def short_join(timeout=None): + """Exercise retirement without waiting for the production join bound.""" + waits.append(timeout) + return join(timeout=0.02) + + monkeypatch.setattr(cr.subprocess, "Popen", popen) + monkeypatch.setattr(cr.signal, "signal", lambda *args: None) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 30.0) + worker = join = None + try: + run.anchor() + worker, join = run._push, run._push.join + assert select.select([ready_read], [], [], 15.0)[0], ( + "the old command did not start" + ) + assert os.read(ready_read, 1) == b"1" + monkeypatch.setattr(worker, "join", short_join) + run.finish() + assert children[0].poll() is None + newer = (folder / "anchors.jsonl").read_bytes() + assert len(newer.splitlines()) == 2 + assert remote_file(remote).encode() == newer + history = remote_log(remote) + path = folder / "anchor-pushes.jsonl" + before = path.read_bytes() + rows = [json.loads(line) for line in before.splitlines()] + assert waits == [pytest.approx(cr.ANCHOR_PUSH_TIMEOUT + cr.KILL_GRACE + 30)] + assert [row["outcome"] for row in rows] == ["error", "success"] + assert rows[0]["returncode"] is None + assert "command may still be running" in rows[0]["output"] + finally: + try: + os.write(release_write, b"1") + except BrokenPipeError: + pass + if join is not None: + join(timeout=45.0) + for child in children: + if child.poll() is None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + for fd in (ready_read, ready_write, release_read, release_write): + os.close(fd) + assert not worker.is_alive() and len(children) == 2 + assert children[0].returncode == 1 + assert path.read_bytes() == before + assert remote_file(remote).encode() == newer + assert remote_log(remote) == history diff --git a/tests/test_confirmatory_board_engine.py b/tests/test_confirmatory_board_engine.py new file mode 100644 index 00000000..c7361703 --- /dev/null +++ b/tests/test_confirmatory_board_engine.py @@ -0,0 +1,142 @@ +"""Engine sidecars must refuse a calculator start before any provider call.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tests.test_confirmatory_run import ( + ENGINE, + complete_preflight_design, + cr, + pilot, +) +from tests.test_confirmatory_run import ( + fast as _fast_fixture, +) +from tests.test_confirmatory_run import ( + world as _world_fixture, +) + +world = _world_fixture +fast = _fast_fixture + + +REAL_BOARD_ENGINE = pilot.board_engine + + +def engine_fault(world, tmp_path, monkeypatch, fault): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + metadata = Path(design["board"]) / "reference_outputs.csv.meta.json" + monkeypatch.setattr(pilot, "board_engine", REAL_BOARD_ENGINE) + + def no_run(*args, **kwargs): + pytest.fail("a refused preflight must not construct or call a provider") + + monkeypatch.setattr(cr, "Run", no_run) + monkeypatch.setattr(pilot, "run_household", no_run) + if fault == "malformed": + metadata.write_text("{") + else: + read_text = Path.read_text + + def denied(path, *args, **kwargs): + if path == metadata: + raise PermissionError("engine metadata read denied") + return read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", denied) + return w, design, metadata + + +@pytest.mark.parametrize("fault", ["malformed", "unreadable"]) +@pytest.mark.parametrize("entrypoint", ["preflight", "main"]) +def test_engine_metadata_refuses_full_calculator_start( + world, tmp_path, monkeypatch, capsys, fault, entrypoint +): + w, design, metadata = engine_fault(world, tmp_path, monkeypatch, fault) + if entrypoint == "preflight": + problems = cr.preflight(design, (cr.CHEAP[0],)) + # The preflight's own guard, beside the saved-treatment lines it keeps. + guard = f"{metadata}: cannot read the board's engine: " + assert sum(problem.startswith(guard) for problem in problems) == 1, problems + for model in cr.CHEAP: + assert any( + problem.startswith(f"no-tools {model}: ") + and "cannot check saved no-tools treatment" in problem + and str(metadata) in problem + for problem in problems + ), problems + else: + design_path = tmp_path / "design.json" + design_path.write_text(json.dumps(design)) + assert cr.main(["run", str(design_path)]) == 1 + output = capsys.readouterr().out + assert "The run cannot start" in output + assert str(metadata) in output + assert not w.provider.count + + +@pytest.mark.parametrize("fault", ["malformed", "unreadable"]) +def test_main_names_engine_reread_failure_and_preserves_all_problems( + world, tmp_path, monkeypatch, capsys, fault +): + w, design, metadata = engine_fault(world, tmp_path, monkeypatch, fault) + # Isolate main's final engine reread, which must retain preflight's problems. + monkeypatch.setattr(cr, "preflight", lambda *_: ["earlier preflight problem"]) + design_path = tmp_path / "design.json" + design_path.write_text(json.dumps(design)) + assert cr.main(["run", str(design_path)]) == 1 + output = capsys.readouterr().out + assert "The run cannot start" in output + assert "earlier preflight problem" in output + assert str(metadata) in output + assert not w.provider.count + + +def installed_elsewhere(world, tmp_path, monkeypatch): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(pilot, "_versions", lambda: {"policyengine-us": "2.41.0"}) + + def no_run(*args, **kwargs): + pytest.fail("a refused preflight must not construct or call a provider") + + monkeypatch.setattr(cr, "Run", no_run) + monkeypatch.setattr(pilot, "run_household", no_run) + return w, design + + +def test_preflight_refuses_an_installed_engine_other_than_the_boards( + world, tmp_path, monkeypatch +): + w, design = installed_elsewhere(world, tmp_path, monkeypatch) + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert f"policyengine-us 2.41.0 is installed, not {ENGINE}" in problems + for model in cr.CHEAP: + assert any( + problem.startswith(f"no-tools {model}: ") + and f"policyengine-us 2.41.0 is installed; the pilot pins {ENGINE}" + in problem + for problem in problems + ), problems + assert not w.provider.count + + +def test_main_rechecks_the_installed_engine_and_preserves_all_problems( + world, tmp_path, monkeypatch, capsys +): + w, design = installed_elsewhere(world, tmp_path, monkeypatch) + # Isolate main's final engine check, which must retain preflight's problems. + monkeypatch.setattr(cr, "preflight", lambda *_: ["earlier preflight problem"]) + design_path = tmp_path / "design.json" + design_path.write_text(json.dumps(design)) + assert cr.main(["run", str(design_path)]) == 1 + output = capsys.readouterr().out + assert "The run cannot start" in output + assert "earlier preflight problem" in output + assert "the board's engine check failed: policyengine-us 2.41.0" in output + assert not w.provider.count diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py new file mode 100644 index 00000000..e85e7562 --- /dev/null +++ b/tests/test_confirmatory_input_reads.py @@ -0,0 +1,524 @@ +"""Hash-bound inputs and report-compatible variables stay provider-free.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import io +import itertools +import json +import math +import os +import re +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests.test_confirmatory_no_tools import Provider, nt, record, stage +from tests.test_confirmatory_no_tools import setup as _setup_fixture +from tests.test_confirmatory_no_tools_records import ( + OPTIONAL_EVIDENCE, + assert_boundary, + full_record, + save_records, +) +from tests.test_confirmatory_no_tools_records import ( + saved_records as _saved_records_fixture, +) +from tests.test_confirmatory_no_tools_records import ( + treatment_templates as _treatment_templates_fixture, +) +from tests.test_confirmatory_run import N_DRAW, World, complete_preflight_design, cr + +setup = _setup_fixture +saved_records = _saved_records_fixture +treatment_templates = _treatment_templates_fixture + + +def replace_text_read(monkeypatch, path, replacement): + """Simulate replacement after the first byte read, without changing disk. + Count both byte readers: Path.read_bytes, and the descriptor-guarded + read_regular that saved records go through (when the runner has it, so the + same cases run against an older runner).""" + original_bytes, original_text = Path.read_bytes, Path.read_text + original_regular = getattr(nt, "read_regular", None) + reads = [] + + def read_bytes(source): + data = original_bytes(source) + if source == path: + reads.append(("bytes", data)) + return data + + def read_regular(source): + data = original_regular(source) + if Path(source) == path: + reads.append(("bytes", data)) + return data + + if original_regular is not None: + monkeypatch.setattr(nt, "read_regular", read_regular) + + def read_text(source, *args, **kwargs): + if source == path: + reads.append(("text", replacement)) + return replacement + return original_text(source, *args, **kwargs) + + monkeypatch.setattr(Path, "read_bytes", read_bytes) + monkeypatch.setattr(Path, "read_text", read_text) + return reads + + +@pytest.mark.parametrize("data", [b"not JSON", b"\xff"], ids=["json", "utf8"]) +def test_baseline_replacement_cannot_supply_unhashed_decoded_bytes( + setup, monkeypatch, data +): + path = Path(setup["design"]["baseline_exclusions"]["path"]) + path.write_bytes(data) + setup["design"]["baseline_exclusions"]["sha256"] = nt.sha256_file(path) + reads = replace_text_read(monkeypatch, path, '{"exclusions": []}') + provider = Provider() + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=setup["board"], + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("the declared baseline bytes must be decoded") + assert str(path) in str(caught.value) + assert reads == [("bytes", data)] + assert provider.calls == [] + + +def test_baseline_hash_and_decoder_share_one_valid_buffer(setup, monkeypatch): + path = Path(setup["design"]["baseline_exclusions"]["path"]) + data = path.read_bytes() + reads = replace_text_read(monkeypatch, path, "not JSON") + provider = Provider() + with nt.Stage( + setup["design"], + board=setup["board"], + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.spend.spent_usd == 0 + assert reads == [("bytes", data)] + assert provider.calls == [] + + +def saved_history(setup, archive_cost=0.3, final_cost=0.1): + """Save one rerun-eligible archive and the final that links it.""" + root = Path(setup["design"]["no_tools"]["runs"]) + folder = root / nt.CHEAP[0] / "none/confirmatory/households" + profile = SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic") + archive = folder / "attempts/h2-1.json" + archive.parent.mkdir(parents=True) + prior = record( + "h2", + profile, + status="provider_error", + error="APIConnectionError", + cost=archive_cost, + ) + prior["attempts"] = [] + archive.write_text(json.dumps(prior)) + saved = record("h2", profile, cost=final_cost) + saved["attempts"] = [ + { + "n": 1, + "file": "attempts/h2-1.json", + "sha256": nt.sha256_file(archive), + "status": "provider_error", + "class": "provider", + "reason": "APIConnectionError", + } + ] + (folder / "h2.json").write_text(json.dumps(saved)) + return {"archive": archive, "final": folder / "h2.json"}, prior, saved + + +@pytest.mark.parametrize("which", ["archive", "final"]) +def test_saved_record_hash_and_decoder_share_one_buffer(setup, monkeypatch, which): + paths, prior, saved = saved_history(setup) + path = paths[which] + data = path.read_bytes() + # Hashed before the reads are counted, so the count is the runner's alone. + archive_sha256 = nt.sha256_file(paths["archive"]) + final_sha256 = nt.sha256_file(paths["final"]) + # A second read would see a cheaper record than the hashed bytes hold. + replaced = dict(prior if which == "archive" else saved, totals={"cost_usd": 0.2}) + reads = replace_text_read(monkeypatch, path, json.dumps(replaced)) + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + assert plan.spend.spent_usd == pytest.approx(0.4) + state = plan.histories[nt.CHEAP[0]]["h2"] + assert state["entries"][0]["sha256"] == archive_sha256 + assert state["final_sha256"] == final_sha256 + assert reads == [("bytes", data)] + assert provider.calls == [] + + +def test_archive_links_the_digest_its_final_was_read_from(setup): + paths, _, saved = saved_history(setup) + saved["status"], saved["error"] = "provider_error", "APIConnectionError" + paths["final"].write_text(json.dumps(saved)) + data = paths["final"].read_bytes() + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + assert plan.archive(nt.CHEAP[0], "h2") + link = plan.histories[nt.CHEAP[0]]["h2"]["entries"][-1] + assert link["n"] == 2 and link["file"] == "attempts/h2-2.json" + assert link["sha256"] == hashlib.sha256(data).hexdigest() + assert (paths["archive"].parent / "h2-2.json").read_bytes() == data + assert provider.calls == [] + + +def test_archive_refuses_a_final_changed_after_it_was_read(setup): + paths, _, saved = saved_history(setup) + saved["status"], saved["error"] = "provider_error", "APIConnectionError" + paths["final"].write_text(json.dumps(saved)) + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + changed = dict(saved, totals={"cost_usd": 0.2}) + paths["final"].write_text(json.dumps(changed)) + with pytest.raises(nt.StateError, match=re.escape(str(paths["final"]))): + plan.archive(nt.CHEAP[0], "h2") + assert len(plan.histories[nt.CHEAP[0]]["h2"]["entries"]) == 1 + assert json.loads(paths["final"].read_text()) == changed + assert not (paths["archive"].parent / "h2-2.json").exists() + assert provider.calls == [] + + +def test_archive_moves_back_a_final_changed_before_its_rename(setup, monkeypatch): + paths, _, saved = saved_history(setup) + saved["status"], saved["error"] = "provider_error", "APIConnectionError" + paths["final"].write_text(json.dumps(saved)) + changed = dict(saved, totals={"cost_usd": 0.2}) + original = nt.os.replace + + def change_then_rename(source, target): + # The write lands after the digest check and before the rename. + if Path(source) == paths["final"]: + Path(source).write_text(json.dumps(changed)) + original(source, target) + + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + monkeypatch.setattr(nt.os, "replace", change_then_rename) + with pytest.raises( + nt.StateError, + match=re.escape(f"{paths['final']}: final changed during its archive move"), + ): + plan.archive(nt.CHEAP[0], "h2") + state = plan.histories[nt.CHEAP[0]]["h2"] + assert len(state["entries"]) == 1 and state["final"] == saved + assert json.loads(paths["final"].read_text()) == changed + assert sorted(p.name for p in paths["archive"].parent.iterdir()) == ["h2-1.json"] + assert provider.calls == [] + + +def test_archive_refuses_a_final_changed_after_the_runner_wrote_it(setup, monkeypatch): + provider = Provider( + { + (nt.CHEAP[0], "h2", 1): { + "status": "provider_error", + "error": "APIConnectionError", + } + } + ) + original = nt.write_json + + def write_then_change(path, payload, *args, **kwargs): + digest = original(path, payload, *args, **kwargs) + if nt.CHEAP[0] in Path(path).parts: + path.write_bytes(path.read_bytes() + b" ") + return digest + + monkeypatch.setattr(nt, "write_json", write_then_change) + plan = stage(setup, provider) + assert plan.execute() == 1 + assert plan.states[nt.CHEAP[0]] == "error" + # The other models continue through both households. + assert all(plan.states[model] == "complete" for model in nt.CHEAP[1:]) + assert all( + plan.standing(model, sid) for model in nt.CHEAP[1:] for sid in ("h2", "h1") + ) + assert [call for call in provider.calls if call[0] == nt.CHEAP[0]] == [ + (nt.CHEAP[0], "h2", 1) + ] + folder = plan.households(nt.CHEAP[0]) + assert json.loads((folder / "h2.json").read_text())["status"] == "provider_error" + assert not (folder / "attempts").exists() + + +def pool_design(tmp_path, monkeypatch): + monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) + world = World(tmp_path, monkeypatch, models=(cr.CHEAP[0],)) + return world, complete_preflight_design(world, tmp_path) + + +@pytest.mark.parametrize("data", [b"not JSON", b"\xff"], ids=["json", "utf8"]) +def test_preflight_pool_replacement_cannot_supply_unhashed_decoded_bytes( + tmp_path, monkeypatch, data +): + world, design = pool_design(tmp_path, monkeypatch) + path = Path(design["eligible_pool"]["path"]) + replacement = path.read_text() + path.write_bytes(data) + design["eligible_pool"]["sha256"] = cr.sha256_file(path) + reads = replace_text_read(monkeypatch, path, replacement) + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(str(path) in problem for problem in problems), problems + assert reads == [("bytes", data)] + assert world.provider.count == {} + + +def test_preflight_pool_hash_and_decoder_share_one_valid_buffer(tmp_path, monkeypatch): + world, design = pool_design(tmp_path, monkeypatch) + path = Path(design["eligible_pool"]["path"]) + data = path.read_bytes() + reads = replace_text_read(monkeypatch, path, "[]") + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert not any("the draw, recomputed" in problem for problem in problems), problems + assert not any(str(path) in problem for problem in problems), problems + assert reads == [("bytes", data)] + assert world.provider.count == {} + + +def no_tools_households(design, model): + spec = design["no_tools"] + return Path(spec["runs"]) / model / spec["folder"] / spec["stage"] / "households" + + +def saved_spend(world, design, final_cost=0.2, archive_cost=0.1, optional_cost=0.05): + """Save a cost in a cheap model's final, one in the archive it replaced and + one in an optional model's final. Return their sum.""" + model, sid = cr.CHEAP[0], design["draw_order"][0] + folder = no_tools_households(design, model) + final = json.loads((folder / f"{sid}.json").read_text()) + failed = { + key: value for key, value in final.items() if key not in OPTIONAL_EVIDENCE + } + failed.update( + status="provider_error", + error="APIConnectionError", + predictions={variable: None for variable in final["variables"]}, + turns=[], + totals={"cost_usd": archive_cost}, + ) + archive = folder / "attempts" / f"{sid}-1.json" + digest = cr.write_json(archive, failed) + final.update( + attempts=[nt.entry(archive, failed, 1, digest)], + totals={"cost_usd": final_cost}, + ) + cr.write_json(folder / f"{sid}.json", final) + optional = full_record(world.board, "gpt-6-sol", design["engine"], sid) + optional["totals"] = {"cost_usd": optional_cost} + cr.write_json(no_tools_households(design, "gpt-6-sol") / f"{sid}.json", optional) + return final_cost + archive_cost + optional_cost + + +@pytest.mark.parametrize( + "margin, refused", + [ + (1.0, False), + (0.0, False), + (-nt.MONEY_TOLERANCE / 2, False), + (-nt.MONEY_TOLERANCE * 2, True), + (-0.05, True), + (-0.35, True), + ], + ids=[ + "above", + "equal", + "inside-tolerance", + "past-tolerance", + "lacks-optional", + "zero", + ], +) +def test_preflight_compares_spent_before_with_all_saved_spend( + tmp_path, monkeypatch, margin, refused +): + world, design = pool_design(tmp_path, monkeypatch) + spent = saved_spend(world, design) + design["spent_before"] = spent + margin + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert [problem for problem in problems if problem.startswith("no-tools")] == ( + [ + f"no-tools spend: spent_before ${spent + margin:.4f} is less than " + f"the saved records' ${spent:.4f}" + ] + if refused + else [] + ) + assert world.provider.count == {} + + +@pytest.mark.parametrize( + "costs", + sorted( + { + *itertools.permutations((0.1, 0.1, 0.4)), + *itertools.permutations((0.1, 0.2, 0.3)), + } + ), +) +def test_preflight_sums_saved_spend_exactly_in_any_order(tmp_path, monkeypatch, costs): + """Added one at a time, these costs land on either side of the tolerance + with their order; the report's exact total does not move.""" + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design, *costs) + design["spent_before"] = 0.599999 + problems = cr.preflight(design, (cr.CHEAP[0],)) + refused = 0.599999 < math.fsum(costs) - nt.MONEY_TOLERANCE + assert any(p.startswith("no-tools spend") for p in problems) is refused, problems + + +@pytest.mark.parametrize( + "declared, refused", [(1e16, True), (1e16 + 2, False)], ids=["short", "exact"] +) +def test_preflight_keeps_costs_a_running_total_rounds_away( + tmp_path, monkeypatch, declared, refused +): + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design, 1e16, 1.0, 1.0) + design["spent_before"] = declared + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(p.startswith("no-tools spend") for p in problems) is refused, problems + + +def test_preflight_refuses_a_saved_total_past_every_float(tmp_path, monkeypatch): + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design, 1.5e308, 0.0, 1.5e308) + design["spent_before"] = 1e308 + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any("the saved records' $inf" in problem for problem in problems), problems + + +@pytest.mark.parametrize( + "value", + [float("nan"), float("inf"), None, "unknown", -1e-7, 10**400], + ids=["nan", "infinite", "null", "text", "negative", "overflow"], +) +def test_preflight_refuses_an_unusable_spent_before(tmp_path, monkeypatch, value): + world, design = pool_design(tmp_path, monkeypatch) + design["spent_before"] = value + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any("finite, nonnegative" in problem for problem in problems), problems + assert world.provider.count == {} + + +def test_preflight_refuses_a_design_without_spent_before(tmp_path, monkeypatch): + world, design = pool_design(tmp_path, monkeypatch) + del design["spent_before"] + assert "the design has no spent_before" in cr.preflight(design, (cr.CHEAP[0],)) + assert any("finite, nonnegative" in p for p in nt.stage_problems(design)) + assert world.provider.count == {} + + +@pytest.mark.parametrize( + "declared, refused", + [("1", False), ("0.1", True), (True, False), (False, True)], + ids=["text-enough", "text-short", "true", "false"], +) +def test_preflight_counts_numeric_text_and_booleans_as_numbers( + tmp_path, monkeypatch, declared, refused +): + """As the report does: float() decides what a spent_before is worth.""" + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design) + design["spent_before"] = declared + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(p.startswith("no-tools spend") for p in problems) is refused, problems + assert not any("finite, nonnegative" in p for p in problems), problems + + +@pytest.mark.parametrize( + "fault, needle", + [ + ("valid", None), + ("undeclared", "baseline_exclusions"), + ("missing", "No such file"), + ("changed", "do not hash as declared"), + ("malformed", "Expecting"), + ("schema", "must contain an exclusions list"), + ("repeated", "repeat a cell"), + ], +) +def test_preflight_applies_startup_s_baseline_check( + tmp_path, monkeypatch, fault, needle +): + world, design = pool_design(tmp_path, monkeypatch) + spec = design["baseline_exclusions"] + path = Path(spec["path"]) + cell = {"scenario_id": "h", "variable": "snap"} + if fault == "undeclared": + del design["baseline_exclusions"] + elif fault == "missing": + path.unlink() + elif fault == "changed": + path.write_text('{"exclusions": [], "changed": true}') + elif fault != "valid": + path.write_text( + { + "malformed": "{", + "schema": "{}", + "repeated": json.dumps({"exclusions": [cell, cell]}), + }[fault] + ) + spec["sha256"] = nt.sha256_file(path) + found = [ + problem + for problem in cr.preflight(design, (cr.CHEAP[0],)) + if "baseline" in problem + ] + if needle is None: + assert found == [] + else: + assert len(found) == 1 and needle in found[0], found + assert fault == "undeclared" or str(path) in found[0] + assert world.provider.count == {} + + +@pytest.fixture(scope="module") +def report_module(): + configured = os.environ.get("CONFIRMATORY_REPORT") + if not configured: + pytest.skip( + "set CONFIRMATORY_REPORT for the separate report's acceptance control" + ) + path = Path(configured) + name = "confirmatory_report_variables_control" + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + with pytest.MonkeyPatch.context() as patch: + patch.syspath_prepend(str(path.parent)) + patch.setitem(sys.modules, name, module) + spec.loader.exec_module(module) + yield module + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +def test_variables_object_is_accepted_by_runner_and_report( + saved_records, report_module, monkeypatch, reader +): + saved = saved_records + payload = saved.finals[nt.CHEAP[0]] if reader == "final" else saved.archives[0] + payload["variables"] = {"snap": {"synthetic_metadata": True}} + save_records(saved) + assert_boundary(saved, "startup", None) + assert_boundary(saved, "preflight", None) + saved.design["treatment_harness"] = "synthetic-harness" + monkeypatch.setattr(report_module, "harness_module", lambda path: nt.pilot) + problems = [] + costs = report_module.no_tools_costs(saved.design, problems) + assert problems == [] + assert costs["registered_usd"] + costs["other_usd"] == pytest.approx(0.5) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py new file mode 100644 index 00000000..14c38fd9 --- /dev/null +++ b/tests/test_confirmatory_no_tools.py @@ -0,0 +1,3187 @@ +"""Provider-free tests of the no-tools stage's files, retries and shared spend.""" + +from __future__ import annotations + +import ast +import fcntl +import functools +import io +import json +import math +import operator +import os +import signal +import sys +import threading +from pathlib import Path +from types import SimpleNamespace + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import confirmatory_no_tools as nt # noqa: E402 + + +@pytest.fixture +def setup(tmp_path, monkeypatch): + # These state-machine fixtures inject a minimal board and fake records. + # Real treatment pins and prompts are covered in the saved-record suite. + monkeypatch.setattr(nt, "validate_treatment", lambda *args: None, raising=False) + draw = ["h2", "h1"] + baseline = tmp_path / "baseline-exclusions.json" + baseline.write_text('{"exclusions": []}\n') + return { + "design": { + "board": "synthetic.csv", + "engine": "policyengine-us 2.60.4", + "draw_order": draw, + "spent_before": nt.NO_TOOLS_CAP, + "baseline_exclusions": { + "path": str(baseline), + "sha256": nt.sha256_file(baseline), + }, + "no_tools": { + "runs": str(tmp_path / "no-tools"), + "folder": "none", + "stage": "confirmatory", + }, + }, + "board": { + "scenarios": {sid: SimpleNamespace(id=sid) for sid in draw}, + "variables": {sid: ["snap"] for sid in draw}, + }, + } + + +def record(sid, profile, status="complete", error=None, cost=0.1): + return { + "scenario_id": sid, + "model": profile.name, + "arm": profile.arm, + "status": status, + "error": error, + "turns": [] if status == "harness_error" else [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": cost}, + } + + +class Provider: + def __init__(self, planted=None): + self.planted, self.calls = planted or {}, [] + self.lock = threading.Lock() + + def __call__( + self, scenario, variables, tool_call, completion_fn, spend, *, profile + ): + key = profile.name, scenario.id + with self.lock: + n = 1 + sum(call[:2] == key for call in self.calls) + self.calls.append((*key, n)) + spec = self.planted.get((*key, n), {}) + cost = spec.get("cost", 0.1) + spend.add(cost) + if spec.get("raise"): + raise RuntimeError("synthetic harness failure") + return record( + scenario.id, + profile, + cost=cost, + status=spec.get("status", "complete"), + error=spec.get("error"), + ) + + +def stage(setup, provider=None, models=None, **kwargs): + return nt.Stage( + setup["design"], + models=models, + board=setup["board"], + run_household=provider or Provider(), + completion_for=lambda profile: None, + out=io.StringIO(), + **kwargs, + ) + + +def final(plan, model=nt.CHEAP[0], sid="h2"): + return plan.households(model) / f"{sid}.json" + + +def load(path): + return json.loads(path.read_text()) + + +def plant(setup, status, error, n=1, **extra): + return {(nt.CHEAP[0], "h2", n): {"status": status, "error": error, **extra}} + + +def test_every_household_ok_in_draw_order_and_each_model_has_its_thread(setup): + provider, threads = Provider(), {} + + def fake(*args, **kwargs): + threads.setdefault(kwargs["profile"].name, threading.current_thread().name) + return provider(*args, **kwargs) + + plan = stage(setup, fake) + assert plan.execute() == 0 + assert all(plan.states[m] == "complete" for m in nt.CHEAP) + assert len(set(threads.values())) == len(nt.CHEAP) + for model in nt.CHEAP: + assert [call[1] for call in provider.calls if call[0] == model] == ["h2", "h1"] + for sid in setup["design"]["draw_order"]: + assert load(final(plan, model, sid))["attempts"] == [] + assert plan.spend.spent_usd == pytest.approx(0.6) + + +def test_provider_rerun_links_the_unchanged_archived_bytes(setup): + provider = Provider(plant(setup, "provider_error", "APIConnectionError: synthetic")) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + written = {} + original = nt.write_json + + def capture(path, payload, *args, **kwargs): + result = original(path, payload, *args, **kwargs) + written.setdefault(path.name, path.read_bytes()) + return result + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(nt, "write_json", capture) + assert plan.execute() == 0 + path = final(plan) + archive = path.parent / "attempts/h2-1.json" + assert archive.read_bytes() == written["h2.json"] + assert load(path)["attempts"] == [ + { + "n": 1, + "file": "attempts/h2-1.json", + "sha256": nt.sha256_file(archive), + "status": "provider_error", + "class": "provider", + "reason": "APIConnectionError", + } + ] + assert stage(setup, models=[nt.CHEAP[0]]).spend.spent_usd == pytest.approx(0.3) + + +def test_third_provider_failure_stands_and_has_only_two_archives(setup): + planted = { + **plant(setup, "provider_error", "APIConnectionError", 1), + **plant(setup, "provider_error", "APIConnectionError", 2), + **plant(setup, "provider_error", "APIConnectionError", 3), + } + provider = Provider(planted) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.execute() == 0 + assert [n for m, sid, n in provider.calls if sid == "h2"] == [1, 2, 3] + assert len(load(final(plan))["attempts"]) == nt.MAX_RERUNS + assert load(final(plan))["status"] == "provider_error" + + +def test_operator_pause_stops_its_model_and_reinvocation_resumes(setup): + provider = Provider( + plant(setup, "provider_error", "AuthenticationError: synthetic") + ) + first = stage(setup, provider) + assert first.execute() == 1 + assert first.states[nt.CHEAP[0]] == "operator-paused" + assert not final(first, sid="h1").exists() + assert first.states[nt.CHEAP[1]] == "complete" + before = final(first).read_bytes() + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.execute() == 0 + assert (final(resumed).parent / "attempts/h2-1.json").read_bytes() == before + assert load(final(resumed))["attempts"][0]["class"] == "operator" + + +def test_harness_stop_prevents_later_households_and_persists(setup): + provider = Provider(plant(setup, "provider_error", "BadRequestError: synthetic")) + + # A request error is a harness class only before a completed turn. + def early_failure(*args, **kwargs): + result = provider(*args, **kwargs) + if result["status"] == "provider_error": + result["turns"] = [] + return result + + plan = stage(setup, early_failure, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert plan.states[nt.CHEAP[0]] == "harness-stopped" + assert not final(plan, sid="h1").exists() + resumed_provider = Provider() + resumed = stage(setup, resumed_provider, models=[nt.CHEAP[0]]) + assert resumed.execute() == 1 + assert resumed_provider.calls == [] + + +def paused(setup): + plan = stage( + setup, + Provider(plant(setup, "provider_error", "AuthenticationError")), + models=[nt.CHEAP[0]], + ) + assert plan.execute() == 1 + return plan + + +def test_crash_after_archive_resumes_at_the_next_attempt_number(setup, monkeypatch): + first = paused(setup) + original = nt.os.replace + + def interrupted(source, target): + original(source, target) + if Path(target).parent.name == "attempts": + raise RuntimeError("synthetic crash after archive") + + monkeypatch.setattr(nt.os, "replace", interrupted) + with pytest.raises(RuntimeError, match="synthetic crash"): + first.archive(nt.CHEAP[0], "h2") + monkeypatch.setattr(nt.os, "replace", original) + assert not final(first).exists() + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.execute() == 0 + assert load(final(resumed))["attempts"][0]["n"] == 1 + assert " h2 2 ok stands " in resumed.out.getvalue() + + +def test_spend_in_archives_and_optional_models_carries_across_invocations(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + optional = stage(setup, models=["gpt-6-sol"]) + assert optional.execute() == 0 + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.spend.spent_usd == pytest.approx(0.3) + assert resumed.execute() == 0 + assert resumed.spend.spent_usd == pytest.approx(0.5) + + +def test_reaching_the_cap_prevents_every_new_attempt_across_invocations( + setup, monkeypatch +): + monkeypatch.setattr(nt, "NO_TOOLS_CAP", 0.1) + plan = stage(setup, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert len(list(plan.households(nt.CHEAP[0]).glob("*.json"))) == 1 + provider = Provider() + resumed = stage(setup, provider, models=[nt.CHEAP[1]]) + assert resumed.execute() == 1 + assert provider.calls == [] + assert resumed.spend.spent_usd == pytest.approx(0.1) + + +def test_budget_stop_is_shared_by_all_models(setup): + provider = Provider(plant(setup, "budget_stop", None)) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert plan.stopped.is_set() + resumed = stage(setup, Provider(), models=[nt.CHEAP[1]]) + assert resumed.execute() == 1 + assert resumed.states[nt.CHEAP[1]] == "budget-stopped" + + +@pytest.mark.parametrize( + "mutation,match", + [ + (lambda r: r.update(prior_attempts=[]), "prior_attempts"), + (lambda r: r.pop("attempts"), "attempts list"), + ], +) +def test_refuse_a_final_not_written_by_this_script(setup, mutation, match): + first = paused(setup) + path = final(first) + payload = load(path) + mutation(payload) + path.write_text(json.dumps(payload)) + with pytest.raises(nt.StateError, match=match): + stage(setup) + + +def test_refuse_an_attempt_number_gap_and_name_its_files(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + archive = final(first).parent / "attempts/h2-1.json" + archive.rename(archive.with_name("h2-2.json")) + with pytest.raises(nt.StateError, match="attempt-number gap.*h2-2.json"): + stage(setup) + + +@pytest.mark.parametrize( + "field,value", [("sha256", "wrong"), ("n", 2), ("reason", "edited")] +) +def test_refuse_a_final_with_corrupted_attempt_links(setup, field, value): + paused(setup) + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.execute() == 0 + path = final(resumed) + payload = load(path) + payload["attempts"][0][field] = value + path.write_text(json.dumps(payload)) + match = "attempt link n" if field == "n" else "attempts disagree" + with pytest.raises(nt.StateError, match=match): + stage(setup) + + +def test_an_attempt_that_raises_keeps_its_spend(setup): + provider = Provider(plant(setup, "complete", None, **{"raise": True, "cost": 0.27})) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.execute() == 0 + archive = final(plan).parent / "attempts/h2-1.json" + assert load(archive)["status"] == "harness_error" + assert load(archive)["totals"]["cost_usd"] == 0.27 + assert plan.spend.spent_usd == pytest.approx(0.47) + assert stage(setup, models=[nt.CHEAP[0]]).spend.spent_usd == pytest.approx(0.47) + + +def test_dry_run_lists_done_households_and_next_owed_attempt_without_providers(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + provider = Provider() + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.dry_run() == 1 + text = plan.out.getvalue() + assert "done []" in text and "h2:2" in text and "h1:1" in text + assert "$0.100000" in text + assert provider.calls == [] + + +def test_each_attempt_uses_none_profile_and_refuses_a_tool_call(setup): + def fake(scenario, variables, tool_call, completion, spend, *, profile): + assert profile.arm == "none" and profile.prompt_version == nt.PROMPT_VERSION + assert variables == ["snap"] + with pytest.raises(RuntimeError, match="cannot call a tool"): + tool_call({}) + return record(scenario.id, profile, cost=0) + + assert stage(setup, fake, models=[nt.CHEAP[0]]).execute() == 0 + + +def test_registered_cap_matches_the_report_when_available(): + """The local report is separate from this public repository's dependency tree.""" + configured = os.environ.get("CONFIRMATORY_REPORT") + candidates = ( + [Path(configured)] + if configured + else [Path(__file__).resolve().parents[2] / "report" / "confirmatory_report.py"] + ) + path = next((p for p in candidates if p.is_file()), None) + if path is None: + pytest.skip("set CONFIRMATORY_REPORT to compare the separate report checkout") + tree = ast.parse(path.read_text()) + cap = next( + ast.literal_eval(node.value) + for node in tree.body + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "NO_TOOLS_CAP" + for target in node.targets + ) + ) + assert nt.NO_TOOLS_CAP == cap == 5.0 + + +def test_main_supports_multiple_model_options_and_dry_run(setup, tmp_path, capsys): + write_board_fixture(setup, tmp_path / "board") + path = tmp_path / "design.json" + path.write_text(json.dumps(setup["design"])) + assert ( + nt.main( + [ + "run", + str(path), + "--model", + nt.CHEAP[0], + nt.CHEAP[1], + "--model", + nt.CHEAP[2], + "--dry-run", + ] + ) + == 1 + ) + output = capsys.readouterr().out + assert all(model in output for model in nt.CHEAP) + + +def test_in_flight_models_observe_the_shared_cap_before_their_next_turn( + setup, monkeypatch +): + monkeypatch.setattr(nt, "NO_TOOLS_CAP", 0.1) + ready, charged = threading.Barrier(3), threading.Barrier(3) + calls = [] + + def fake(scenario, variables, tool_call, completion, spend, *, profile): + calls.append((profile.name, scenario.id)) + ready.wait(timeout=30) + cost = 0.1 if profile.name == nt.CHEAP[0] else 0.0 + spend.add(cost) + charged.wait(timeout=30) + assert spend.exhausted() + return record(scenario.id, profile, cost=cost) + + plan = stage(setup, fake) + assert plan.execute() == 1 + assert len(calls) == 3 and all(sid == "h2" for _, sid in calls) + assert plan.spend.spent_usd == pytest.approx(0.1) + + +@pytest.mark.parametrize( + "status,error,expected", + [ + ("budget_stop", None, "stopped"), + ("provider_error", "BadRequestError: synthetic", "stop"), + ], +) +def test_dry_run_marks_harness_and_budget_finals_as_blocked( + setup, status, error, expected +): + plan = stage(setup, models=[nt.CHEAP[0]]) + profile = nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + payload = record("h2", profile, status=status, error=error, cost=0) + payload["turns"] = [] + payload["attempts"] = [] + nt.write_json(final(plan), payload) + if hasattr(plan, "close"): + plan.close() + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.dry_run() == 1 + assert f"h2:blocked {expected} at attempt 1" in resumed.out.getvalue() + + +def test_sigint_finishes_saves_and_stops_before_the_next_household(setup, tmp_path): + run_no_tools_probe( + setup, + tmp_path, + "finish_admitted_attempt", + r""" + calls = [] + saved_handlers = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + + + def interrupted( + scenario, variables, tool_call, completion, spend, *, profile + ): + calls.append(scenario.id) + spend.add(0.17) + if scenario.id == "h2": + os.kill(os.getpid(), signal.SIGINT) + assert plan.interrupted.wait(30), ( + "SIGINT must set the interruption event" + ) + return record(scenario.id, profile, cost=0.17) + + + plan = stage(setup, interrupted, models=[nt.CHEAP[0]]) + result = plan.execute() + assert result != 0, "SIGINT must give a coordinated nonzero exit" + assert calls == ["h2"] + assert load(final(plan))["totals"]["cost_usd"] == 0.17 + assert plan.states[nt.CHEAP[0]] == "interrupted" + assert not plan.stopped.is_set(), ( + "interrupts must preserve the budget event" + ) + assert not final(plan, sid="h1").exists() + assert all( + signal.getsignal(sig) == handler + for sig, handler in saved_handlers.items() + ) + assert "finish and save" in plan.out.getvalue() + """, + ) + + +def test_stage_lock_refuses_a_second_invocation_and_releases_after_dry_run(setup): + first = stage(setup, models=[nt.CHEAP[0]]) + with pytest.raises(nt.StateError, match="another no-tools invocation"): + stage(setup, models=[nt.CHEAP[1]]) + assert first.dry_run() == 1 + second = stage(setup, models=[nt.CHEAP[1]]) + assert second.dry_run() == 1 + + +@pytest.mark.parametrize("fault", ["missing", "hash"]) +def test_baseline_exclusions_must_exist_and_hash_before_startup(setup, fault): + spec = setup["design"]["baseline_exclusions"] + if fault == "missing": + Path(spec["path"]).unlink() + else: + spec["sha256"] = "0" * 64 + with pytest.raises(nt.StateError, match="baseline exclusions"): + stage(setup) + + +def test_archive_identity_and_rerun_eligibility_are_checked_directly(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + path = final(first).parent / "attempts/h2-1.json" + payload = load(path) + payload["scenario_id"] = "h1" + path.write_text(json.dumps(payload)) + with pytest.raises(nt.StateError, match="household differs from archive name h2"): + stage(setup) + payload.update(scenario_id="h2", status="complete", error=None) + path.write_text(json.dumps(payload)) + with pytest.raises(nt.StateError, match="archived attempt was ineligible"): + stage(setup) + + +def test_two_provider_failures_then_harness_failure_has_no_standing_final(setup): + provider = Provider( + { + **plant(setup, "provider_error", "APIConnectionError", 1), + **plant(setup, "provider_error", "APIConnectionError", 2), + **plant(setup, "provider_error", "BadRequestError", 3), + } + ) + + def failure(*args, **kwargs): + result = provider(*args, **kwargs) + if result["error"] == "BadRequestError": + result["turns"] = [] + return result + + plan = stage(setup, failure, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert plan.states[nt.CHEAP[0]] == "harness-stopped" + assert [n for _, _, n in provider.calls] == [1, 2, 3] + assert len(load(final(plan))["attempts"]) == 2 + assert not plan.standing(nt.CHEAP[0], "h2") + assert not final(plan, sid="h1").exists() + + +@pytest.mark.parametrize( + "status,error,expected", + [ + ("provider_error", "BadRequestError", "stop"), + ("budget_stop", None, "stopped"), + ], +) +def test_stage_problems_requires_every_cheap_model_final_to_stand( + setup, status, error, expected, monkeypatch +): + plan = stage(setup) + assert plan.execute() == 0 + path = final(plan, model=nt.CHEAP[-1], sid="h1") + payload = load(path) + payload.update(status=status, error=error, turns=[]) + path.write_text(json.dumps(payload)) + monkeypatch.setattr( + nt.pilot, "load_board", lambda _: pytest.fail("helper loads board") + ) + problems = nt.stage_problems(setup["design"]) + assert any(nt.CHEAP[-1] in p and "h1" in p and expected in p for p in problems) + + +def test_stage_problems_reports_missing_finals_and_invalid_optional_history(setup): + plan = stage(setup) + assert plan.execute() == 0 + assert nt.stage_problems(setup["design"]) == [] + final(plan, sid="h1").unlink() + optional = plan.households("gpt-6-sol") / "attempts/h2-2.json" + optional.parent.mkdir(parents=True) + optional.write_text("{}") + problems = nt.stage_problems(setup["design"]) + assert any(nt.CHEAP[0] in p and "h1" in p and "missing" in p for p in problems) + assert any("attempt-number gap" in p and "gpt-6-sol" in p for p in problems) + + +def test_real_harness_checks_other_models_spend_before_the_next_turn( + setup, monkeypatch +): + from policybench import eval_no_tools + + monkeypatch.setattr(nt, "NO_TOOLS_CAP", 0.1) + for scenario in setup["board"]["scenarios"].values(): + scenario.country = "us" + + def prompt(scenario, variables, profile): + return "Synthetic household", { + "_board_prompt_sha256": "synthetic", + "_sent_board_prompt_sha256": "synthetic", + "_budget_ceiling": 100, + "_answer_tool": nt.pilot.answer_tool_for( + profile, variables, scenario.country + ), + "max_tokens": 100, + } + + monkeypatch.setattr(nt.pilot, "build_initial_request", prompt) + monkeypatch.setattr( + eval_no_tools, + "_extract_usage_metadata", + lambda response, *args: {"total_cost_usd": response.synthetic_cost}, + ) + entered = threading.Barrier(2) + requests = [] + models = [nt.CHEAP[0], nt.CHEAP[2]] + first_id = nt.pilot.model_profile(models[0], arm="none").model_id + + def completion(**kwargs): + requests.append(kwargs["model"]) + entered.wait(timeout=30) + if kwargs["model"] != first_id: + assert plan.stopped.wait(timeout=30), "other model must cross shared cap" + message = SimpleNamespace( + role="assistant", content="", tool_calls=None, function_call=None + ) + return SimpleNamespace( + choices=[SimpleNamespace(message=message, finish_reason="stop")], + synthetic_cost=0.11 if kwargs["model"] == first_id else 0.0, + id="synthetic-response", + ) + + plan = nt.Stage( + setup["design"], + models=models, + board=setup["board"], + run_household=nt.pilot.run_household, + completion_for=lambda profile: completion, + out=io.StringIO(), + ) + assert plan.execute() == 1 + assert len(requests) == 2 + for model in models: + payload = load(final(plan, model=model)) + assert payload["status"] == "budget_stop", payload.get("error") + assert len(payload["turns"]) == 1 + assert not final(plan, model=model, sid="h1").exists() + assert plan.spend.spent_usd == pytest.approx(0.11) + + +def track_interrupt_return(monkeypatch): + """Observe the actual installed first-signal handler returning.""" + returned = threading.Event() + register = signal.signal + + def install(signum, handler): + if callable(handler) and handler.__name__ == "interrupt": + + def tracked(sig, frame): + handler(sig, frame) + returned.set() + + return register(signum, tracked) + return register(signum, handler) + + monkeypatch.setattr(signal, "signal", install) + return returned + + +@pytest.mark.parametrize("boundary", ["archive", "attempt"]) +def test_signal_between_check_and_action_refuses_new_admission( + setup, tmp_path, boundary +): + """R1: the watcher interrupts before the proposed action reaches admission.""" + run_no_tools_probe( + setup, + tmp_path, + f"refuse_{boundary}_admission", + f" boundary = {boundary!r}\n" + + r""" + if boundary == "archive": + paused(setup) + provider = Provider() + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + returned = track_interrupt_return(monkeypatch) + + + def signal_and_wait(): + os.kill(os.getpid(), signal.SIGINT) + assert returned.wait(30), "the signal handler must finish" + assert plan.interrupted.wait(30), ( + "the watcher must set the interrupt before admission" + ) + + + if boundary == "archive": + original = plan.archive + + def archive(*args): + signal_and_wait() + return original(*args) + + monkeypatch.setattr(plan, "archive", archive) + else: + original = nt.AttemptSpend + + def attempt_spend(*args): + signal_and_wait() + return original(*args) + + monkeypatch.setattr(nt, "AttemptSpend", attempt_spend) + assert plan.execute() == 1 + assert returned.is_set() + assert provider.calls == [], ( + "R1: no provider attempt begins after interruption" + ) + archives = plan.households(nt.CHEAP[0]) / "attempts" + assert not list(archives.glob("*.json")), ( + "R1: no archive moves after interruption" + ) + """, + ) + + +def test_failed_second_thread_start_keeps_invocation_lock_until_worker_finishes( + setup, tmp_path +): + """R2: exceptional shutdown waits for every worker before closing its lock.""" + run_no_tools_probe( + setup, + tmp_path, + "failed_second_start", + r""" + import fcntl + + entered, release, failed_start, closed = ( + threading.Event() for _ in range(4) + ) + original_start = threading.Thread.start + original_close = nt.Stage.close + observed = {} + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + entered.set() + assert release.wait(timeout=30) + return record(scenario.id, profile) + + + plan = stage(setup, provider) + monkeypatch.setattr(nt, "START_GRACE_SECONDS", 0.05, raising=False) + + + def start(thread): + if thread.name == f"no-tools-{nt.CHEAP[1]}": + assert entered.wait(timeout=30) + failed_start.set() + raise RuntimeError("synthetic second thread start failure") + return original_start(thread) + + + def close(current): + original_close(current) + if current is plan: + closed.set() + + + def observe(): + assert failed_start.wait(timeout=30) + observed["closed_while_live"] = closed.wait(timeout=0.2) + lock = (plan.root / ".confirmatory-no-tools.lock").open("a") + try: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + observed["held"] = True + else: + observed["held"] = False + finally: + lock.close() + release.set() + + + watcher = threading.Thread(target=observe) + watcher.start() + monkeypatch.setattr(threading.Thread, "start", start) + monkeypatch.setattr(nt.Stage, "close", close) + try: + with pytest.raises(RuntimeError, match="second thread start"): + plan.execute() + finally: + release.set() + watcher.join(timeout=30) + assert not watcher.is_alive(), "R2: independent lock probe finishes" + assert observed["held"], "R2: a live worker must retain the invocation lock" + assert not observed["closed_while_live"], ( + "R2: cleanup joins before unlocking" + ) + assert plan.interrupted.is_set(), ( + "R2: exceptional shutdown refuses further work" + ) + assert plan._lock_file is None + assert load(final(plan))["status"] == "complete", ( + "R2: admitted worker saves before cleanup returns" + ) + """, + ) + + +def test_second_signal_exits_while_output_lock_is_held(setup, tmp_path): + """R4: forced termination is independent of the worker-shared output lock.""" + import textwrap + + probe = tmp_path / "forced_exit.py" + probe.write_text( + textwrap.dedent("""\ + import io + import json + import os + import signal + import sys + import threading + import time + from types import SimpleNamespace + import confirmatory_no_tools as nt + + design = json.loads(sys.argv[1]) + draw = design['draw_order'] + board = { + 'scenarios': {sid: SimpleNamespace(id=sid) for sid in draw}, + 'variables': {sid: ['snap'] for sid in draw}, + } + held = threading.Event() + plan = nt.Stage(design, models=[nt.CHEAP[0]], board=board, + completion_for=lambda profile: None, out=io.StringIO()) + + def worker(model): + with plan._say_lock: + held.set() + time.sleep(30) + + plan.run_model = worker + + def signals(): + assert held.wait(10) + os.write(1, b'ready\\n') + os.kill(os.getpid(), signal.SIGINT) + assert plan.interrupted.wait(30) + os.kill(os.getpid(), signal.SIGTERM) + + threading.Thread(target=signals, daemon=True).start() + plan.execute() + raise AssertionError('forced exit returned') + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"])], + env=env, + ready_marker=b"ready\n", + ) + stdout, stderr = result.stdout, result.stderr + assert b"ready\n" in stdout, (stdout, stderr) + assert result.returncode == 128 + signal.SIGTERM, (stdout, stderr) + assert b"every unsaved attempt then under way is lost, with its spend" in stderr + assert b"saved finals, archives and the spend they record survive" in stderr + + +@pytest.mark.parametrize("category", ["optional", "outside-draw"]) +def test_stage_problems_classifies_every_saved_final(setup, category): + """R3: optional and off-draw finals use the initialization classifier.""" + plan = stage(setup) + assert plan.execute() == 0 + model = "gpt-6-sol" if category == "optional" else nt.CHEAP[0] + sid = "h2" if category == "optional" else "outside-draw" + profile = nt.pilot.model_profile( + model, arm="none", prompt_version=nt.PROMPT_VERSION + ) + payload = record(sid, profile) + payload.update(attempts=[], turns=[None]) + path = plan.households(model) / f"{sid}.json" + nt.write_json(path, payload) + match = "outside the draw" if category == "outside-draw" else "get" + with pytest.raises(nt.StateError, match=match): + stage(setup) + problems = nt.stage_problems(setup["design"]) + assert any(model in problem and match in problem for problem in problems), ( + "R3: every final refused by initialization must fail stage preflight", + problems, + ) + + +@pytest.mark.parametrize( + "payload", ["[]", "{invalid json", '{"exclusions": [{"scenario_id": "h2"}]}'] +) +def test_baseline_schema_is_usable_before_any_provider_call(setup, payload): + """R5: correctly hashed malformed inventories refuse without provider calls.""" + spec = setup["design"]["baseline_exclusions"] + path = Path(spec["path"]) + path.write_text(payload) + spec["sha256"] = nt.sha256_file(path) + provider = Provider() + try: + with pytest.raises(nt.StateError, match="baseline exclusion"): + plan = stage(setup, provider) + plan.close() + finally: + assert provider.calls == [], "R5: schema checks precede provider calls" + + +@pytest.mark.parametrize("reader", ["final", "archive", "baseline", "design"]) +@pytest.mark.parametrize("fault", ["nested", "non-object", "invalid-utf8"]) +def test_undecodable_stage_file_refuses_with_path(setup, tmp_path, reader, fault): + """Every file reader names its refusal; deep decoding stays in a child.""" + run_no_tools_probe( + setup, + tmp_path, + f"decode_{reader}_{fault}", + f" reader, fault = {reader!r}, {fault!r}\n" + + r""" + from contextlib import redirect_stdout + from pathlib import Path + + if fault == "nested": + sys.setrecursionlimit(1000) + depth = sys.getrecursionlimit() * 20 + payload = ( + b'{"attempts": [], "exclusions": [], "nested": ' + + b"[" * depth + b"0" + b"]" * depth + b"}" + ) + elif fault == "non-object": + payload = b"[]" + else: + payload = ( + b'{"scenario_id": "h2", "status": "provider_error", ' + b'"error": "APIConnectionError", "attempts": [], ' + b'"exclusions": [], "note": "\xff"}' + ) + + provider = Provider() + root = Path(setup["design"]["no_tools"]["runs"]) + if reader in ("final", "archive"): + path = root / nt.CHEAP[0] / "none/confirmatory/households" + path /= "h2.json" if reader == "final" else "attempts/h2-1.json" + elif reader == "baseline": + path = Path(setup["design"]["baseline_exclusions"]["path"]) + else: + path = root.parent / "design.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + + if reader == "design": + refusals = [] + + class CapturedStateError(nt.StateError): + def __init__(self, *args): + super().__init__(*args) + refusals.append(self) + + def unexpected_stage(*args, **kwargs): + pytest.fail("an undecodable design must refuse before Stage") + + monkeypatch.setattr(nt, "StateError", CapturedStateError) + monkeypatch.setattr(nt, "Stage", unexpected_stage) + output = io.StringIO() + with redirect_stdout(output): + assert nt.main(["run", str(path), "--dry-run"]) == 1 + assert "The no-tools stage cannot start:" in output.getvalue() + assert str(path) in output.getvalue() + assert len(refusals) == 1 + caught = SimpleNamespace(value=refusals[0]) + else: + if reader == "baseline": + setup["design"]["baseline_exclusions"]["sha256"] = ( + nt.sha256_file(path) + ) + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("an undecodable file must refuse before execution") + assert str(path) in str(caught.value) + if reader in ("final", "archive"): + problems = nt.stage_problems(setup["design"]) + assert any(str(path) in problem for problem in problems), problems + if fault == "nested": + assert isinstance(caught.value.__cause__, RecursionError) + trace = caught.value.__cause__.__traceback__ + frames = [] + while trace is not None: + frames.append(trace.tb_frame.f_code.co_filename) + trace = trace.tb_next + assert any(name.endswith("json/decoder.py") for name in frames), frames + elif fault == "invalid-utf8": + assert isinstance(caught.value.__cause__, UnicodeDecodeError) + assert provider.calls == [] + """, + ) + + +def write_board_fixture(setup, directory): + """Write a small board consumed by the unmodified harness readers.""" + import csv + + directory.mkdir() + engine = nt.pilot._versions()["policyengine-us"] + setup["design"]["engine"] = f"policyengine-us {engine}" + setup["design"]["board"] = str(directory) + meta = { + "programs": ["snap"], + "revisions": [ + {"kind": "engine_upgrade", "engine_version": f"policyengine-us {engine}"} + ], + } + (directory / "reference_outputs.csv.meta.json").write_text(json.dumps(meta)) + (directory / "reference_exclusions.json").write_text('{"exclusions": []}') + with (directory / "reference_outputs.csv").open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_id", "variable", "value"]) + writer.writerows((sid, "snap", 1) for sid in setup["design"]["draw_order"]) + with (directory / "scenarios.csv").open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_json"]) + writer.writerows( + [json.dumps({"id": sid, "state": "CA", "adults": []})] + for sid in setup["design"]["draw_order"] + ) + + +@pytest.mark.parametrize( + "reader,fault", + [ + (reader, fault) + for reader in ("metadata", "exclusions", "scenarios") + for fault in ("nested", "non-object", "invalid-utf8") + ] + + [("reference", "invalid-utf8")], +) +def test_undecodable_board_file_refuses_with_path(setup, tmp_path, reader, fault): + run_no_tools_probe( + setup, + tmp_path, + f"board_{reader}_{fault}", + f" reader, fault = {reader!r}, {fault!r}\n" + + r""" + import csv + from pathlib import Path + + directory = Path(setup["design"]["no_tools"]["runs"]).parent / "board" + write_board_fixture(setup, directory) + names = { + "metadata": "reference_outputs.csv.meta.json", + "exclusions": "reference_exclusions.json", + "scenarios": "scenarios.csv", + "reference": "reference_outputs.csv", + } + path = directory / names[reader] + if fault == "nested": + sys.setrecursionlimit(1000) + depth = sys.getrecursionlimit() * 20 + payload = b'{"nested":' + b"[" * depth + b"0" + b"]" * depth + b"}" + elif fault == "non-object": + payload = b"[]" + else: + payload = b'{"note":"\xff"}' + if reader == "scenarios": + # Quote the JSON as a CSV field, retaining the corrupt UTF-8 byte. + text = io.StringIO(newline="") + writer = csv.writer(text) + writer.writerow(["scenario_json"]) + writer.writerow([payload.decode("latin1")]) + payload = text.getvalue().encode("latin1") + elif reader == "reference": + payload = b"scenario_id,variable,value\nh2,\xff,1\n" + path.write_bytes(payload) + provider = Provider() + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], board=None, run_household=provider, + completion_for=lambda profile: None, out=io.StringIO(), + ): + pytest.fail("an unusable board must refuse before execution") + assert str(path) in str(caught.value) + if fault == "nested": + assert isinstance(caught.value.__cause__, RecursionError) + trace = caught.value.__cause__.__traceback__ + frames = [] + while trace is not None: + frames.append(trace.tb_frame.f_code.co_filename) + trace = trace.tb_next + assert any(name.endswith("json/decoder.py") for name in frames), frames + elif fault == "invalid-utf8": + assert isinstance(caught.value.__cause__, UnicodeDecodeError) + assert provider.calls == [] + """, + ) + + +@pytest.mark.parametrize("with_exclusions", [True, False]) +def test_valid_board_loads_through_harness(setup, tmp_path, with_exclusions): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + if not with_exclusions: + (directory / "reference_exclusions.json").unlink() + expected = nt.pilot.load_board(directory) + provider = Provider() + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.board.keys() == expected.keys() + for key in ("reference", "scored", "excluded", "scenarios_frame"): + assert plan.board[key].equals(expected[key]) + for key in ("meta", "scenarios", "variables"): + assert plan.board[key] == expected[key] + assert plan.spend.spent_usd == 0 + assert plan.histories == {model: {} for model in nt.CHEAP} + assert provider.calls == [] + + +@pytest.mark.parametrize("with_filename", [True, False]) +def test_delegated_board_oserror_refuses_with_path( + setup, tmp_path, monkeypatch, with_filename +): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + reference = directory / "reference_outputs.csv" + error = ( + OSError(5, "synthetic reread failure", str(reference)) + if with_filename + else OSError("synthetic reread failure") + ) + read_csv = nt.pilot.pd.read_csv + rereads = [] + + def fail_reference_reread(source, *args, **kwargs): + # Prechecks pass an opened stream; the unchanged harness passes a path. + if isinstance(source, Path) and source == reference: + rereads.append(source) + raise error + return read_csv(source, *args, **kwargs) + + monkeypatch.setattr(nt.pilot.pd, "read_csv", fail_reference_reread) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("a failed board reread must refuse before execution") + assert rereads == [reference] + assert caught.value.__cause__ is error + assert str(reference if with_filename else directory) in str(caught.value) + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize("reader", ["metadata", "scenarios"]) +def test_board_changed_to_nested_json_after_check_refuses_with_path( + setup, tmp_path, reader +): + run_no_tools_probe( + setup, + tmp_path, + f"board_changed_{reader}", + f" reader = {reader!r}\n" + + r""" + import csv + from pathlib import Path + + sys.setrecursionlimit(1000) + depth = sys.getrecursionlimit() * 20 + payload = '{"nested":' + "[" * depth + "0" + "]" * depth + "}" + directory = Path(setup["design"]["no_tools"]["runs"]).parent / "board" + write_board_fixture(setup, directory) + filename = ( + "reference_outputs.csv.meta.json" if reader == "metadata" + else "scenarios.csv" + ) + path = directory / filename + load_board = nt.pilot.load_board + delegates = [] + + def change_then_load(source): + delegates.append(source) + if reader == "metadata": + path.write_text(payload) + else: + with path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_json"]) + writer.writerow([payload]) + return load_board(source) + + monkeypatch.setattr(nt.pilot, "load_board", change_then_load) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], board=None, run_household=provider, + completion_for=lambda profile: None, out=io.StringIO(), + ): + pytest.fail("a changed board must refuse before execution") + assert delegates == [directory], "all prechecks must finish first" + assert str(path) in str(caught.value) + assert "changed after" in str(caught.value) + assert isinstance(caught.value.__cause__, RecursionError) + trace = caught.value.__cause__.__traceback__ + frames = [] + while trace is not None: + frames.append(trace.tb_frame.f_code.co_filename) + trace = trace.tb_next + assert any(name.endswith("json/decoder.py") for name in frames), frames + finally: + assert provider.calls == [] + """, + ) + + +def test_board_exclusions_exists_failure_refuses_with_path( + setup, tmp_path, monkeypatch +): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + exclusions = directory / "reference_exclusions.json" + error = PermissionError(13, "synthetic unreadable board parent", str(exclusions)) + stat = Path.stat + checks = [] + + def fail_exclusions_stat(path, *args, **kwargs): + if path == exclusions: + checks.append(path) + raise error + return stat(path, *args, **kwargs) + + monkeypatch.setattr(Path, "stat", fail_exclusions_stat) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("an unreadable exclusions path must refuse") + assert checks == [exclusions] + assert caught.value.__cause__ is error + assert str(exclusions) in str(caught.value) + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize( + "error_type", + [ + ValueError, + TypeError, + AttributeError, + KeyError, + RecursionError, + OverflowError, + SystemExit, + ], +) +def test_unchanged_board_delegate_failure_names_folder( + setup, tmp_path, monkeypatch, error_type +): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + error = error_type("synthetic delegated failure") + delegates = [] + + def fail_load(source): + delegates.append(source) + raise error + + monkeypatch.setattr(nt.pilot, "load_board", fail_load) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("a failed board delegate must refuse before execution") + assert delegates == [directory] + assert caught.value.__cause__ is error + assert str(directory) in str(caught.value) + assert "changed after" not in str(caught.value) + finally: + assert provider.calls == [] + + +def test_board_delegate_failure_names_each_changed_file(setup, tmp_path, monkeypatch): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + paths = [ + directory / name + for name in ("reference_outputs.csv.meta.json", "scenarios.csv") + ] + error = ValueError("synthetic delegated failure") + delegates = [] + + def change_then_fail(source): + delegates.append(source) + for path in paths: + path.write_bytes(path.read_bytes() + b"\n") + raise error + + monkeypatch.setattr(nt.pilot, "load_board", change_then_fail) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("a changed board must refuse before execution") + assert delegates == [directory] + assert caught.value.__cause__ is error + assert all(str(path) in str(caught.value) for path in paths) + assert "changed after" in str(caught.value) + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_oversized_record_cost_refuses_with_path(setup, reader, boundary): + root = Path(setup["design"]["no_tools"]["runs"]) + path = root / nt.CHEAP[0] / "none/confirmatory/households" + path /= "h2.json" if reader == "final" else "attempts/h2-1.json" + payload = record( + "h2", + SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic"), + status="provider_error", + error="APIConnectionError", + cost=10**400, + ) + payload["attempts"] = [] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload)) + provider = Provider() + if boundary == "startup": + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("an oversized cost must refuse before execution") + assert str(path) in str(caught.value) + assert isinstance(caught.value.__cause__, OverflowError) + else: + problems = nt.stage_problems(setup["design"]) + assert any(str(path) in problem for problem in problems), problems + assert provider.calls == [] + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize( + "fault", + [ + "missing-totals", + "null-totals", + "false-totals", + "empty-list-totals", + "empty-string-totals", + "true-totals", + "number-totals", + "list-totals", + "missing-cost", + "null", + "true", + "false", + "numeric-string", + "empty-string", + "invalid-string", + "empty-list", + "list", + "empty-object", + "object", + "nan", + "infinity", + "negative-infinity", + "negative", + "overflow", + ], +) +def test_strict_saved_cost_refuses_with_path(setup, reader, boundary, fault): + root = Path(setup["design"]["no_tools"]["runs"]) + path = root / nt.CHEAP[0] / "none/confirmatory/households" + path /= "h2.json" if reader == "final" else "attempts/h2-1.json" + payload = record( + "h2", + SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic"), + status="provider_error", + error="APIConnectionError", + ) + payload["attempts"] = [] + totals = { + "null-totals": None, + "false-totals": False, + "empty-list-totals": [], + "empty-string-totals": "", + "true-totals": True, + "number-totals": 1, + "list-totals": [1], + } + costs = { + "null": None, + "true": True, + "false": False, + "numeric-string": "0.25", + "empty-string": "", + "invalid-string": "invalid", + "empty-list": [], + "list": [0.25], + "empty-object": {}, + "object": {"value": 0.25}, + "nan": float("nan"), + "infinity": float("inf"), + "negative-infinity": float("-inf"), + "negative": -0.25, + "overflow": 10**400, + } + if fault == "missing-totals": + del payload["totals"] + elif fault in totals: + payload["totals"] = totals[fault] + elif fault == "missing-cost": + payload["totals"] = {} + else: + payload["totals"]["cost_usd"] = costs[fault] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload)) + provider = Provider() + try: + if boundary == "startup": + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("an invalid saved cost must refuse before execution") + assert str(path) in str(caught.value) + else: + problems = nt.stage_problems(setup["design"]) + assert any(str(path) in problem for problem in problems), problems + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize("cost", [0, 0.0, -0.0, 0.25, 2]) +def test_valid_saved_numeric_cost_preserves_spend(setup, cost): + root = Path(setup["design"]["no_tools"]["runs"]) + folder = root / nt.CHEAP[0] / "none/confirmatory/households" + folder.mkdir(parents=True) + profile = SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic") + archive = folder / "attempts/h2-1.json" + archive.parent.mkdir() + prior = record( + "h2", profile, status="provider_error", error="APIConnectionError", cost=cost + ) + prior["attempts"] = [] + archive.write_text(json.dumps(prior)) + saved = record("h2", profile, cost=cost) + saved["attempts"] = [nt.entry(archive, prior, 1, nt.sha256_file(archive))] + (folder / "h2.json").write_text(json.dumps(saved)) + provider = Provider() + with stage(setup, provider) as plan: + assert plan.spend.spent_usd == 2 * float(cost) + assert not any( + str(folder) in problem for problem in nt.stage_problems(setup["design"]) + ) + assert provider.calls == [] + + +def require_a_stop_at_the_cap(setup, saved, spent=nt.NO_TOOLS_CAP): + """Save these (model, household, cost) finals, and require that startup's sum + reaches `spent`, no owed attempt starts, and another open takes the lock.""" + root = Path(setup["design"]["no_tools"]["runs"]) + for model, sid, cost in saved: + folder = root / model / "none/confirmatory/households" + folder.mkdir(parents=True, exist_ok=True) + profile = SimpleNamespace(name=model, arm="none", model_id="synthetic") + payload = dict(record(sid, profile, cost=cost), attempts=[]) + (folder / f"{sid}.json").write_text(json.dumps(payload)) + provider = Provider() + plan = stage(setup, provider) + assert plan.spend.spent_usd == spent + assert plan.stopped.is_set() + assert plan.execute() == 1 + assert provider.calls == [] + with (root / ".confirmatory-no-tools.lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + pytest.fail(f"the invocation lock stayed held: {error}") + + +def test_startup_spend_takes_the_exact_sum_at_the_stage_cap(setup): + """A running total of these costs stays under the cap and their exact sum + reaches it.""" + costs = [math.nextafter(nt.NO_TOOLS_CAP, 0.0), 2**-52, 2**-52, 2**-52] + assert functools.reduce(operator.add, costs) < nt.NO_TOOLS_CAP + places = [ + (nt.CHEAP[0], "h2"), + (nt.CHEAP[0], "h1"), + (nt.CHEAP[1], "h2"), + (nt.CHEAP[2], "h2"), + ] + require_a_stop_at_the_cap(setup, [(*p, c) for p, c in zip(places, costs)]) + + +def test_startup_spend_sums_every_model_at_once_at_the_stage_cap(setup): + """Rounding each model's costs first falls one float under the cap, and the + exact sum of all three reaches it.""" + first = [3 * 2**-53, math.nextafter(nt.NO_TOOLS_CAP, 0.0)] + second = [2**-52] + subtotals = [nt.exact_spend(first), nt.exact_spend(second)] + assert nt.exact_spend(subtotals) < nt.NO_TOOLS_CAP + require_a_stop_at_the_cap( + setup, + [ + (nt.CHEAP[0], "h1", first[0]), + (nt.CHEAP[0], "h2", first[1]), + (nt.CHEAP[1], "h2", second[0]), + ], + ) + + +def test_startup_spend_stops_on_costs_whose_sum_overflows(setup): + """Each cost fits a float and `math.fsum` overflows on the pair, so startup + takes infinity for the total and stops.""" + costs = [1.5e308, 1.5e308] + with pytest.raises(OverflowError): + math.fsum(costs) + require_a_stop_at_the_cap( + setup, + [(nt.CHEAP[0], "h1", costs[0]), (nt.CHEAP[1], "h1", costs[1])], + spent=math.inf, + ) + + +def run_no_tools_probe(setup, tmp_path, name, body): + """Bound a signal or worker-shutdown regression to its own process.""" + import textwrap + + prelude = r""" + import io + import json + import os + import runpy + import signal + import sys + import threading + from types import SimpleNamespace + import pytest + + helpers = runpy.run_path(sys.argv[2]) + for helper in ( + "nt", + "record", + "Provider", + "stage", + "final", + "load", + "paused", + "track_interrupt_return", + "write_board_fixture", + ): + globals()[helper] = helpers[helper] + design = json.loads(sys.argv[1]) + setup = { + "design": design, + "board": { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + }, + } + monkeypatch = pytest.MonkeyPatch() + """ + probe = tmp_path / f"{name}.py" + probe.write_text(textwrap.dedent(prelude) + "\n" + textwrap.dedent(body)) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), str(Path(__file__))], + env=env, + ) + assert result.returncode == 0, (result.stdout, result.stderr) + + +def _run_cleanup_regression(setup, tmp_path, case): + """Run every exceptional-cleanup signal probe behind a process deadline.""" + import textwrap + + probe = tmp_path / "cleanup_regression.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import subprocess + import sys + import threading + import time + from types import SimpleNamespace + import confirmatory_no_tools as nt + + nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + print("cleanup-regression-ready", flush=True) + design, case = json.loads(sys.argv[1]), sys.argv[2] + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + model = nt.CHEAP[0] + entered, release, saved, worker_returned = ( + threading.Event() for _ in range(4) + ) + cleanup_entered, closed = threading.Event(), threading.Event() + handler_returned = threading.Event() + order, observed, calls = [], {}, [] + original_handlers = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_start = threading.Thread.start + original_signal = signal.signal + original_close = nt.Stage.close + original_write = nt.write_json + main_ident = threading.get_ident() + nt.START_GRACE_SECONDS = 0.05 + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + calls.append((profile.name, scenario.id)) + entered.set() + assert release.wait(20), "cleanup rule: release the admitted worker" + spend.add(0.1) + return { + "scenario_id": scenario.id, + "model": profile.name, + "arm": profile.arm, + "status": "complete", + "error": None, + "turns": [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": 0.1}, + } + + + models = list(nt.CHEAP[:2]) if case == "condition-signal" else [model] + plan = nt.Stage( + design, + models=models, + board=board, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) + original_model = plan.run_model + + + def run_model(name): + try: + return original_model(name) + finally: + worker_returned.set() + + + plan.run_model = run_model + + + def write(path, payload, *args, **kwargs): + result = original_write(path, payload, *args, **kwargs) + if path == plan.households(model) / "h2.json": + order.append("saved") + saved.set() + return result + + + nt.write_json = write + + + def close(current): + if current is plan and current._lock_file is not None: + order.append("close") + closed.set() + return original_close(current) + + + nt.Stage.close = close + + + def register(sig, handler): + if sig in original_handlers and handler == original_handlers[sig]: + order.append("restore:" + signal.Signals(sig).name) + elif sig in original_handlers and callable(handler): + original = handler + + def tracked(signum, frame): + original(signum, frame) + handler_returned.set() + + handler = tracked + return original_signal(sig, handler) + + + signal.signal = register + + + def lock_held(): + # A separate process opens the lock independently of the worker. + probe = ( + "import fcntl,sys; f=open(sys.argv[1],'a'); " + "\ntry: fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB)" + "\nexcept BlockingIOError: print('held')" + "\nelse: print('released')" + ) + result = subprocess.run( + [ + sys.executable, + "-c", + probe, + str(plan.root / ".confirmatory-no-tools.lock"), + ], + capture_output=True, + text=True, + timeout=10, + check=True, + ) + return result.stdout.strip() == "held" + + + if case == "condition-signal": + + class InterruptInsideSet(threading.Event): + def set(self): + if threading.get_ident() == main_ident and not hasattr( + self, "injected" + ): + self.injected = True + with self._cond: + cleanup_entered.set() + os.kill(os.getpid(), signal.SIGINT) + assert handler_returned.wait(10), ( + "cleanup rule: handler returns inside Event.set" + ) + self._flag = True + self._cond.notify_all() + return + super().set() + + plan.interrupted = InterruptInsideSet() + + def start(thread): + if thread.name == "no-tools-" + nt.CHEAP[1]: + assert entered.wait(10) + raise RuntimeError("synthetic second start failure") + return original_start(thread) + + def observe(): + assert cleanup_entered.wait(10) + assert handler_returned.wait(10) + observed["held"] = lock_held() + observed["closed_while_live"] = closed.is_set() + release.set() + + actor = threading.Thread(target=observe, daemon=True) + actor.start() + threading.Thread.start = start + try: + plan.execute() + except RuntimeError as error: + assert str(error) == "synthetic second start failure" + observed["propagated"] = True + propagated_error = error + else: + raise AssertionError( + "cleanup rule: original start exception propagates" + ) + actor.join(10) + assert handler_returned.is_set(), ( + "cleanup rule: SIGINT handler returns inside Event.set" + ) + assert observed.get("held"), ( + "cleanup rule: invocation lock survives Event.set signal" + ) + assert not observed.get("closed_while_live"), ( + "cleanup rule: admitted worker saves before close" + ) + assert plan.interrupted.is_set() + + elif case in ("join-once", "join-twice"): + raised = [] + original_set = plan.interrupted.set + + def interrupt_cleanup(): + cleanup_entered.set() + return original_set() + + plan.interrupted.set = interrupt_cleanup + + def injected(sig, frame): + raised.append(len(raised) + 1) + raise KeyboardInterrupt( + "synthetic wait interruption " + str(len(raised)) + ) + + signal.signal(signal.SIGUSR1, injected) + + def main_in_worker_wait(): + frame = sys._current_frames().get(main_ident) + while frame is not None: + if frame.f_code.co_name == "_wait_for_tstate_lock": + return True + if frame.f_code.co_name == "wait": + event = frame.f_locals.get("self") + if event in getattr(plan, "_worker_done", {}).values(): + return True + frame = frame.f_back + return False + + def await_wait(cleanup=False): + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if closed.is_set(): + return False + if ( + not cleanup or cleanup_entered.is_set() + ) and main_in_worker_wait(): + return True + time.sleep(0.001) + return False + + def observe(): + assert entered.wait(10) + assert await_wait(), ( + "cleanup rule: main thread reaches a worker wait" + ) + os.kill(os.getpid(), signal.SIGUSR1) + if case == "join-twice": + if await_wait(cleanup=True): + os.kill(os.getpid(), signal.SIGUSR1) + # The original implementation closes early after poisoning Thread's + # tstate lock. Give it that opportunity while the provider is held. + observed["closed_while_live"] = closed.wait(0.1) + observed["held"] = lock_held() + release.set() + + actor = threading.Thread(target=observe, daemon=True) + actor.start() + try: + plan.execute() + except KeyboardInterrupt as error: + observed["exception"] = str(error) + order.append("propagated") + else: + raise AssertionError("cleanup rule: wait exception must propagate") + finally: + release.set() + actor.join(15) + assert worker_returned.wait(10), ( + "cleanup rule: worker finishes after injected wait exception" + ) + assert observed.get("exception") == "synthetic wait interruption 1", ( + "cleanup rule: preserve original exception" + ) + expected = 2 if case == "join-twice" else 1 + assert len(raised) == expected, ( + "cleanup rule: repeat interruption exercises cleanup wait" + ) + assert not observed.get("closed_while_live"), ( + "cleanup rule: close waits for worker done" + ) + assert observed.get("held"), ( + "cleanup rule: invocation lock stays held through worker done" + ) + + elif case in ("start-after", "start-before"): + + def start(thread): + if thread.name == "no-tools-" + model: + if case == "start-after": + original_start(thread) + assert entered.wait(10) + raise RuntimeError("synthetic start interruption") + return original_start(thread) + + def observe(): + assert entered.wait(10) + observed["closed_while_live"] = closed.wait(0.1) + observed["held"] = lock_held() + release.set() + + actor = None + if case == "start-after": + actor = threading.Thread(target=observe, daemon=True) + actor.start() + threading.Thread.start = start + began = time.monotonic() + try: + plan.execute() + except RuntimeError as error: + observed["exception"] = str(error) + order.append("propagated") + else: + raise AssertionError( + "cleanup rule: failed start exception must propagate" + ) + if actor is not None: + actor.join(15) + assert not observed.get("closed_while_live"), ( + "cleanup rule: partially started worker saves before close" + ) + assert observed.get("held"), ( + "cleanup rule: partially started worker retains invocation lock" + ) + else: + elapsed = time.monotonic() - began + assert elapsed >= nt.START_GRACE_SECONDS * 0.9, ( + "cleanup rule: unstarted worker receives start grace" + ) + assert not calls and not entered.is_set(), ( + "cleanup rule: failed native start touches no worker files" + ) + assert not list( + plan.root.glob("*/none/confirmatory/households/*.json") + ) + assert observed.get("exception") == "synthetic start interruption" + + else: + raise AssertionError(case) + + if case != "start-before": + assert saved.is_set(), "cleanup rule: admitted worker saves its final" + assert order.index("saved") < order.index("close"), ( + "cleanup rule: save precedes close" + ) + for sig in (signal.SIGINT, signal.SIGTERM): + assert order.index("saved") < order.index( + "restore:" + signal.Signals(sig).name + ), "cleanup rule: save precedes handler restoration" + assert plan._lock_file is None, ( + "cleanup rule: invocation lock eventually closes" + ) + assert all( + signal.getsignal(sig) == old for sig, old in original_handlers.items() + ), "cleanup rule: handlers eventually restore" + print(json.dumps({"case": case, "order": order, "observed": observed})) + if case == "condition-signal": + raise propagated_error + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), case], + env=env, + ready_marker=b"cleanup-regression-ready\n", + text=True, + ) + assert "cleanup-regression-ready\n" in result.stdout, ( + "cleanup rule: subprocess reaches readiness", + result.stdout, + result.stderr, + ) + assert result.returncode == (1 if case == "condition-signal" else 0), ( + result.stdout, + result.stderr, + ) + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def test_signal_inside_cleanup_event_condition_keeps_worker_protected(setup, tmp_path): + """R10.1: a real signal inside Event.set cannot reenter its condition lock.""" + result = _run_cleanup_regression(setup, tmp_path, "condition-signal") + assert result["observed"]["propagated"] + + +@pytest.mark.parametrize("case", ["join-once", "join-twice"]) +def test_wait_exceptions_preserve_worker_completion_and_original_exception( + setup, tmp_path, case +): + """R10.2: real SIGUSR1 exceptions cannot release protection before done.""" + _run_cleanup_regression(setup, tmp_path, case) + + +@pytest.mark.parametrize("case", ["start-after", "start-before"]) +def test_raised_start_tracks_native_worker_or_waits_start_grace(setup, tmp_path, case): + """R10.2: a raised start has a completion event and bounded began grace.""" + _run_cleanup_regression(setup, tmp_path, case) + + +def admission_probe(setup, tmp_path, boundary, force): + """Exercise real signals behind a paused admission in a bounded subprocess.""" + import textwrap + + if boundary == "archive": + paused(setup) + probe = tmp_path / "admission_probe.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import sys + import threading + import time + import traceback + from pathlib import Path + from types import SimpleNamespace + import confirmatory_no_tools as nt + + design = json.loads(sys.argv[1]) + boundary, force = sys.argv[2], sys.argv[3] == "force" + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + entered, release, returned, watcher_ready, committed = ( + threading.Event() for _ in range(5) + ) + errors, calls, order = [], [], [] + plan = None + + + class AdmissionLock: + def __init__(self): + self.lock = threading.Lock() + + def __enter__(self): + if threading.current_thread().name == "no-tools-signal-watcher": + watcher_ready.set() + self.lock.acquire() + return self + + def __exit__(self, *exc): + self.lock.release() + + def locked(self): + return self.lock.locked() + + + class InterruptEvent(threading.Event): + def set(self): + order.append("interrupt") + super().set() + watcher_ready.set() + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + calls.append(scenario.id) + assert plan.interrupted.wait(20), "R3: watcher eventually interrupts" + spend.add(0.1) + return { + "scenario_id": scenario.id, + "model": profile.name, + "arm": profile.arm, + "status": "complete", + "error": None, + "turns": [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": 0.1}, + } + + + def completion(profile): + if boundary == "archive": + assert plan.interrupted.wait(20), ( + "R3: archive finishes before another attempt" + ) + return None + + + plan = nt.Stage( + design, + models=[nt.CHEAP[0]], + board=board, + run_household=provider, + completion_for=completion, + out=io.StringIO(), + ) + plan.interrupted = InterruptEvent() + plan._admission_lock = AdmissionLock() + original_say = plan.say + + + def say(text): + if text.startswith("Interrupted:"): + assert plan.interrupted.is_set(), "R3: message follows interruption" + assert committed.is_set(), ( + "R3: admitted commit precedes interruption message" + ) + order.append("message") + original_say(text) + + + plan.say = say + original_signal = signal.signal + + + def install(signum, handler): + if callable(handler) and handler.__name__ == "interrupt": + + def tracked(sig, frame): + handler(sig, frame) + returned.set() + + return original_signal(signum, tracked) + return original_signal(signum, handler) + + + signal.signal = install + + + def pause_commit(): + entered.set() + os.write(1, b"ready\n") + assert release.wait(20), "R3: paused admitted commit is released" + + + if boundary == "archive": + original_replace = nt.os.replace + + def replace(source, target): + if Path(target).parent.name == "attempts": + pause_commit() + original_replace(source, target) + order.append("commit") + committed.set() + return + return original_replace(source, target) + + nt.os.replace = replace + else: + + class Admitted(dict): + def __setitem__(self, key, value): + pause_commit() + super().__setitem__(key, value) + order.append("commit") + committed.set() + + plan._admitted_attempts = Admitted() + + + def control(): + try: + assert entered.wait(20), "R3: worker reaches the admission commit" + held_at_commit = plan._admission_lock.locked() + os.kill(os.getpid(), signal.SIGINT) + assert returned.wait(5), ( + "R3: first handler returns while worker holds admission lock" + ) + assert held_at_commit, ( + "R3: worker holds the admission lock at its commit" + ) + assert watcher_ready.wait(5), ( + "R3: watcher reaches the admission boundary" + ) + assert not plan.interrupted.is_set(), ( + "R3: interrupt cannot precede admitted commit" + ) + assert not committed.is_set(), ( + "R3: commit remains paused during first handler" + ) + if force: + os.write(1, b"handler-returned-watcher-waiting\n") + os.kill(os.getpid(), signal.SIGINT) + # The handler runs on the main thread. Sleep (releasing + # the GIL) so it can exit the process before this thread + # reports that it did not; 20 s stays inside the probe's + # 30 s execution deadline. + time.sleep(20) + raise AssertionError("R3: forced exit returned") + except BaseException: + errors.append(traceback.format_exc()) + if force: + os.write(2, errors[-1].encode()) + os._exit(99) + finally: + if not force: + release.set() + + + controller = threading.Thread(target=control, daemon=True) + controller.start() + result = plan.execute() + controller.join(20) + assert not errors, "\n".join(errors) + assert not controller.is_alive(), "R3: controller finishes" + assert result == 1, "R3: signalled run exits nonzero" + assert ( + order.index("commit") + < order.index("interrupt") + < order.index("message") + ), order + assert plan.interrupted.is_set(), ( + "R3: watcher sets interrupt after admitted commit" + ) + households = plan.households(nt.CHEAP[0]) + assert not (households / "h1.json").exists(), ( + "R3: no later household is admitted" + ) + if boundary == "archive": + assert calls == [], ( + "R3: no attempt admitted after the watcher interrupts" + ) + assert (households / "attempts/h2-1.json").exists(), ( + "R3: admitted archive is saved" + ) + assert not (households / "h2.json").exists(), ( + "R3: refused next attempt has no final" + ) + assert plan._admitted_attempts == {}, ( + "R3: no subsequent attempt is admitted" + ) + else: + assert calls == ["h2"], "R3: exactly the admitted attempt runs" + saved = json.loads((households / "h2.json").read_text()) + assert saved["status"] == "complete", ( + "R3: admitted attempt saves its final" + ) + assert saved["totals"]["cost_usd"] == 0.1, ( + "R3: admitted attempt saves spend" + ) + assert len(plan._admitted_attempts) == 1, "R3: nothing else is admitted" + print(json.dumps({"order": order, "calls": calls})) + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), boundary, "force" if force else "finish"], + env=env, + ready_marker=b"ready\n", + ) + assert b"ready\n" in result.stdout, (result.stdout, result.stderr) + return result.returncode, result.stdout, result.stderr + + +@pytest.mark.parametrize("boundary", ["archive", "attempt"]) +def test_signal_waits_for_an_admission_commit_without_blocking_handler( + setup, tmp_path, boundary +): + """R3: watcher interruption waits for commits admitted under the worker lock.""" + code, stdout, stderr = admission_probe(setup, tmp_path, boundary, force=False) + assert code == 0, (stdout, stderr) + + +@pytest.mark.parametrize("boundary", ["archive", "attempt"]) +def test_second_signal_forces_exit_while_watcher_waits_for_admission( + setup, tmp_path, boundary +): + """R3: the forced path remains available while the watcher waits for admission.""" + code, stdout, stderr = admission_probe(setup, tmp_path, boundary, force=True) + assert code == 128 + signal.SIGINT, (stdout, stderr) + assert b"handler-returned-watcher-waiting\n" in stdout + assert ( + b"Forced interruption: every unsaved attempt then under way is lost, " + b"with its spend; saved finals, archives and the spend they record survive." + ) in stderr + + +def test_signal_after_all_workers_complete_still_exits_one(setup, tmp_path): + """R4: a signal before handler restoration makes even a complete run fail.""" + import textwrap + + probe = tmp_path / "signal_after_workers.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import sys + from types import SimpleNamespace + import confirmatory_no_tools as nt + + design = json.loads(sys.argv[1]) + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + plan = nt.Stage( + design, + models=[nt.CHEAP[0]], + board=board, + completion_for=lambda profile: None, + out=io.StringIO(), + ) + + + def complete(model): + plan.states[model] = "complete" + + + plan.run_model = complete + original_signal = signal.signal + previous = signal.getsignal(signal.SIGINT) + sent = False + + + def restore(sig, handler): + global sent + if not sent and sig == signal.SIGINT and handler == previous: + sent = True + os.kill(os.getpid(), signal.SIGINT) + return original_signal(sig, handler) + + + signal.signal = restore + assert plan.execute() == 1, ( + "R4: signal arrival before restoration requires exit 1" + ) + assert all(state == "complete" for state in plan.states.values()) + assert sent, "R4: signal arrives after every worker completed" + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"])], + env=env, + ) + assert result.returncode == 0, (result.stdout, result.stderr) + + +@pytest.mark.parametrize("during_grace", [True, False]) +def test_native_worker_without_ident_obeys_start_grace_and_entry_interrupt( + setup, tmp_path, during_grace +): + """R10.2: native startup can precede ident and must never outlive file safety.""" + import textwrap + + probe = tmp_path / "native_start_regression.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import subprocess + import sys + import threading + import time + from types import SimpleNamespace + import confirmatory_no_tools as nt + + nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + print("native-start-regression-ready", flush=True) + design = json.loads(sys.argv[1]) + during_grace = sys.argv[2] == "True" + model = nt.CHEAP[0] + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + native_entered, bootstrap_release = threading.Event(), threading.Event() + cleanup_entered, closed, native_finished = ( + threading.Event() for _ in range(3) + ) + done_held, release_done = threading.Event(), threading.Event() + original_bootstrap = threading.Thread._bootstrap_inner + original_run = threading.Thread.run + original_close = nt.Stage.close + main_ident = threading.get_ident() + worker, calls, order, observed = [], [], [], {} + nt.START_GRACE_SECONDS = 10 if during_grace else 0.05 + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + calls.append(scenario.id) + return { + "scenario_id": scenario.id, + "model": profile.name, + "arm": profile.arm, + "status": "complete", + "error": None, + "turns": [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": 0.0}, + } + + + plan = nt.Stage( + design, + models=[model], + board=board, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) + original_set = plan.interrupted.set + + + def interrupt_cleanup(): + result = original_set() + cleanup_entered.set() + return result + + + plan.interrupted.set = interrupt_cleanup + + + def bootstrap(thread): + if thread.name == "no-tools-" + model: + worker.append(thread) + native_entered.set() + assert bootstrap_release.wait(20), ( + "start grace rule: release native startup" + ) + return original_bootstrap(thread) + + + def run(thread): + try: + return original_run(thread) + finally: + if thread.name == "no-tools-" + model: + order.append("native-finished") + native_finished.set() + + + def close(current): + if current is plan and current._lock_file is not None: + order.append("close") + closed.set() + return original_close(current) + + + threading.Thread._bootstrap_inner = bootstrap + threading.Thread.run = run + nt.Stage.close = close + + + def injected(sig, frame): + raise KeyboardInterrupt("synthetic startup wait interruption") + + + signal.signal(signal.SIGUSR1, injected) + + + def main_in_start_wait(): + frame = sys._current_frames().get(main_ident) + while frame is not None: + if ( + frame.f_code.co_name == "wait" + and frame.f_locals.get("self") is worker[0]._started + ): + return True + frame = frame.f_back + return False + + + def lock_held(): + probe = ( + "import fcntl,sys; f=open(sys.argv[1],'a'); " + "\ntry: fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB)" + "\nexcept BlockingIOError: print('held')" + "\nelse: print('released')" + ) + result = subprocess.run( + [ + sys.executable, + "-c", + probe, + str(plan.root / ".confirmatory-no-tools.lock"), + ], + capture_output=True, + text=True, + timeout=10, + check=True, + ) + return result.stdout.strip() == "held" + + + def observe(): + assert native_entered.wait(10) + assert worker[0].ident is None, ( + "start grace rule: native worker has no ident" + ) + deadline = time.monotonic() + 10 + while not main_in_start_wait(): + assert time.monotonic() < deadline, ( + "start grace rule: main reaches _started.wait" + ) + time.sleep(0.001) + os.kill(os.getpid(), signal.SIGUSR1) + assert cleanup_entered.wait(10) + if during_grace: + # Let the native target begin immediately within grace, then + # hold its completion independently of process-startup load. + done = getattr(plan, "_worker_done", {}).get(model) + if done is not None: + original_done_set = done.set + + def set_done(): + done_held.set() + assert release_done.wait(20), ( + "start grace rule: release begun worker completion" + ) + return original_done_set() + + done.set = set_done + bootstrap_release.set() + if done is not None: + assert done_held.wait(10), ( + "start grace rule: late target reaches completion" + ) + try: + observed["held_at_cleanup"] = lock_held() + finally: + release_done.set() + else: + assert closed.wait(10), ( + "start grace rule: close after bounded grace" + ) + bootstrap_release.set() + + + actor = threading.Thread(target=observe, daemon=True) + actor.start() + try: + plan.execute() + except KeyboardInterrupt as error: + assert str(error) == "synthetic startup wait interruption" + else: + raise AssertionError("start grace rule: original exception propagates") + finally: + if closed.is_set() and not during_grace: + bootstrap_release.set() + actor.join(15) + assert native_finished.wait(10), ( + "start grace rule: native worker eventually completes" + ) + assert not calls, ( + "start grace rule: interrupted late worker makes no provider call" + ) + assert not list(plan.root.glob("*/none/confirmatory/households/*.json")), ( + "start grace rule: interrupted late worker writes no files" + ) + if during_grace: + assert observed.get("held_at_cleanup"), ( + "start grace rule: invocation lock stays held during grace" + ) + assert order.index("native-finished") < order.index("close"), ( + "start grace rule: worker begun during grace completes before close" + ) + else: + assert order.index("close") < order.index("native-finished"), ( + "start grace rule: worker beyond grace exits safely after close" + ) + assert plan._lock_file is None + print(json.dumps({"during_grace": during_grace, "order": order})) + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), str(during_grace)], + env=env, + ready_marker=b"native-start-regression-ready\n", + ) + assert b"native-start-regression-ready\n" in result.stdout, ( + "start grace rule: subprocess reaches readiness", + result.stdout, + result.stderr, + ) + assert result.returncode == 0, (result.stdout, result.stderr) + + +_NO_TOOLS_FORK_WARMED = False + + +def run_no_tools_fork_probe( + probe, + args=(), + *, + env=None, + timeout=30, + ready_marker=None, + ready_timeout=120, + text=False, +): + """Run actual probe code in a forked subprocess with bounded cleanup. + + The parent warms the provider-free profile imports once. A fresh child + executes the probe with its own signals, stdout, stderr and process exit. + Readiness has a separate deadline; the execution deadline begins when the + child reaches its actual readiness marker, or after subprocess setup. + """ + import multiprocessing + import runpy + import subprocess + import time + import traceback + + global _NO_TOOLS_FORK_WARMED + if not _NO_TOOLS_FORK_WARMED: + nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + _NO_TOOLS_FORK_WARMED = True + probe = Path(probe).resolve() + stdout_path = probe.with_suffix(".stdout") + stderr_path = probe.with_suffix(".stderr") + command = [sys.executable, str(probe), *map(str, args)] + context = multiprocessing.get_context("fork") + startup_ready = context.Event() + + # Flush existing parent streams before fork so the child cannot duplicate + # any buffered pytest output when its streams are replaced. + sys.stdout.flush() + sys.stderr.flush() + with stdout_path.open("wb") as stdout_file, stderr_path.open("wb") as stderr_file: + + def child(): + code = 0 + try: + os.dup2(stdout_file.fileno(), 1) + os.dup2(stderr_file.fileno(), 2) + sys.stdout = os.fdopen(os.dup(1), "w", buffering=1) + sys.stderr = os.fdopen(os.dup(2), "w", buffering=1) + if env is not None: + os.environ.clear() + os.environ.update(env) + sys.argv = command[1:] + signal.signal(signal.SIGINT, signal.default_int_handler) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + startup_ready.set() + runpy.run_path(str(probe), run_name="__main__") + except SystemExit as error: + if error.code is None: + code = 0 + elif isinstance(error.code, int): + code = error.code + else: + print(error.code, file=sys.stderr) + code = 1 + except BaseException: + traceback.print_exc() + code = 1 + finally: + sys.stdout.flush() + sys.stderr.flush() + os._exit(code) + + process = context.Process(target=child, name=f"no-tools-probe-{probe.stem}") + process.start() + initial_deadline = time.monotonic() + ready_timeout + + def output(): + stdout = stdout_path.read_bytes() + stderr = stderr_path.read_bytes() + if text: + return stdout.decode(errors="replace"), stderr.decode(errors="replace") + return stdout, stderr + + def expired(limit): + process.kill() + process.join(timeout=30) + stdout, stderr = output() + raise subprocess.TimeoutExpired( + command, limit, output=stdout, stderr=stderr + ) + + try: + while not startup_ready.is_set() and process.is_alive(): + remaining = initial_deadline - time.monotonic() + if remaining <= 0: + expired(ready_timeout) + startup_ready.wait(timeout=min(remaining, 0.1)) + if ready_marker is not None: + marker = ( + ready_marker.encode() + if isinstance(ready_marker, str) + else ready_marker + ) + while marker not in stdout_path.read_bytes() and process.is_alive(): + if time.monotonic() >= initial_deadline: + expired(ready_timeout) + time.sleep(0.01) + process.join(timeout=timeout) + if process.is_alive(): + expired(timeout) + stdout, stderr = output() + return subprocess.CompletedProcess( + command, process.exitcode, stdout, stderr + ) + finally: + if process.is_alive(): + process.kill() + process.join(timeout=30) + process.close() + + +def test_signal_handlers_restore_before_either_watcher_descriptor_closes( + setup, tmp_path +): + """R11: every handler that can write is removed before either pipe end closes.""" + run_no_tools_probe( + setup, + tmp_path, + "watcher_descriptor_order", + r""" + plan = stage(setup, models=[nt.CHEAP[0]]) + previous = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_pipe, original_close = os.pipe, os.close + original_signal = signal.signal + watcher_pipe, events = [], [] + + + def pipe(): + descriptors = original_pipe() + assert not watcher_pipe, "R11: execute creates one watcher pipe" + watcher_pipe.extend(descriptors) + return descriptors + + + def register(sig, handler): + old = original_signal(sig, handler) + if sig in previous and handler == previous[sig]: + events.append(("restore", sig)) + return old + + + def close(fd): + original_close(fd) + if fd in watcher_pipe: + events.append(("close", fd)) + + + monkeypatch.setattr(os, "pipe", pipe) + monkeypatch.setattr(os, "close", close) + monkeypatch.setattr(signal, "signal", register) + assert plan.execute() == 0 + restored, closed = set(), [] + for kind, value in events: + if kind == "restore": + restored.add(value) + else: + assert restored == set(previous), ( + "R11: both handlers restore before either pipe descriptor " + f"closes: {events}" + ) + closed.append(value) + assert len(watcher_pipe) == 2 + assert closed == watcher_pipe, "R11: both watcher descriptors close" + assert all( + signal.getsignal(sig) == handler for sig, handler in previous.items() + ), "R11: both prior handlers remain installed" + """, + ) + + +def test_signal_after_watcher_descriptor_reuse_cannot_write_to_replacement( + setup, tmp_path +): + """R11: a real signal after close cannot write into a reused descriptor.""" + run_no_tools_probe( + setup, + tmp_path, + "watcher_descriptor_reuse", + r""" + plan = stage(setup, models=[nt.CHEAP[0]]) + original_pipe, original_close = os.pipe, os.close + original_signal = signal.signal + previous_interrupts = [] + + + def previous_interrupt(sig, frame): + previous_interrupts.append(sig) + + + original_signal(signal.SIGINT, previous_interrupt) + watcher_pipe, replacement = [], [] + sent = False + + + def pipe(): + descriptors = original_pipe() + assert not watcher_pipe, "R11: execute creates one watcher pipe" + watcher_pipe.extend(descriptors) + return descriptors + + + def close(fd): + global sent + original_close(fd) + if fd == watcher_pipe[1] and not replacement: + read_fd, write_fd = original_pipe() + # Preserve the read end if it acquired the descriptor that must + # become the replacement write end. + if read_fd == fd: + copied_read_fd = os.dup(read_fd) + original_close(read_fd) + read_fd = copied_read_fd + if write_fd != fd: + os.dup2(write_fd, fd) + original_close(write_fd) + write_fd = fd + replacement.extend((read_fd, write_fd)) + os.set_blocking(read_fd, False) + sent = True + # This executes on the main thread immediately after the old + # write descriptor closes and its number is safely reused. + os.kill(os.getpid(), signal.SIGINT) + + + monkeypatch.setattr(os, "pipe", pipe) + monkeypatch.setattr(os, "close", close) + try: + result = plan.execute() + assert sent and len(replacement) == 2 + try: + received = os.read(replacement[0], 1) + except BlockingIOError: + received = b"" + assert received == b"", ( + "R11: a restored handler cannot write into the replacement " + f"descriptor; received {received!r}" + ) + assert result == 0 + assert previous_interrupts == [signal.SIGINT], ( + "R11: the real signal reaches the restored prior handler" + ) + finally: + for fd in replacement: + original_close(fd) + """, + ) + + +@pytest.mark.parametrize("case", ["before-native", "after-native", "after-grace"]) +def test_watcher_start_failure_restores_resources_and_late_entry_is_safe( + setup, tmp_path, case +): + """R11: failed watcher startup preserves cleanup and the late-entry guard.""" + run_no_tools_probe( + setup, + tmp_path, + "watcher_start_" + case, + " case = " + + repr(case) + + r""" + import errno + import time + + provider = Provider() + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + original_handlers = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_start = threading.Thread.start + original_run = threading.Thread.run + original_pipe = os.pipe + original_read = os.read + original_fd_close = os.close + original_close = plan.close + native_entered, reading, release_late, target_finished = ( + threading.Event() for _ in range(4) + ) + watchers, pipe_fds, closed_fds, reads, target_errors, order = ( + [] for _ in range(6) + ) + start_error = RuntimeError("synthetic watcher start " + case) + nt.START_GRACE_SECONDS = 0.05 + + + def pipe(): + descriptors = original_pipe() + assert not pipe_fds, "watcher lifecycle: execute creates one pipe" + pipe_fds.extend(descriptors) + return descriptors + + + def read(fd, count): + if fd in pipe_fds: + assert fd not in closed_fds, ( + "watcher lifecycle: a late watcher cannot read a closed pipe" + ) + reads.append(fd) + reading.set() + return original_read(fd, count) + + + def fd_close(fd): + result = original_fd_close(fd) + if fd in pipe_fds: + closed_fds.append(fd) + order.append("descriptor-closed") + return result + + + def close(): + result = original_close() + order.append("stage-closed") + return result + + + def run(thread): + if thread.name != "no-tools-signal-watcher": + return original_run(thread) + native_entered.set() + try: + if case == "after-grace": + assert release_late.wait(20), ( + "watcher lifecycle: release the delayed native target" + ) + assert set(closed_fds) == set(pipe_fds), ( + "watcher lifecycle: delayed target begins after pipe close" + ) + order.append("late-target-entered") + return original_run(thread) + except BaseException as error: + target_errors.append(error) + raise + finally: + target_finished.set() + + + def start(thread): + if thread.name != "no-tools-signal-watcher": + return original_start(thread) + watchers.append(thread) + if case != "before-native": + original_start(thread) + assert native_entered.wait(10), ( + "watcher lifecycle: the native thread starts before failure" + ) + if case == "after-native": + assert reading.wait(10), ( + "watcher lifecycle: the begun watcher reaches its pipe read" + ) + raise start_error + + + monkeypatch.setattr(os, "pipe", pipe) + monkeypatch.setattr(os, "read", read) + monkeypatch.setattr(os, "close", fd_close) + monkeypatch.setattr(threading.Thread, "run", run) + monkeypatch.setattr(threading.Thread, "start", start) + monkeypatch.setattr(plan, "close", close) + started_at = time.monotonic() + try: + plan.execute() + except RuntimeError as error: + assert error is start_error, ( + "watcher lifecycle: propagate the original start exception" + ) + else: + raise AssertionError("watcher lifecycle: startup failure must propagate") + elapsed = time.monotonic() - started_at + + assert len(watchers) == 1 and len(pipe_fds) == 2 + assert sorted(closed_fds) == sorted(pipe_fds), ( + "watcher lifecycle: close both descriptors exactly once" + ) + assert order.count("stage-closed") == 1 + assert all( + signal.getsignal(sig) == handler + for sig, handler in original_handlers.items() + ), "watcher lifecycle: restore both installed handlers" + for fd in pipe_fds: + try: + os.fstat(fd) + except OSError as error: + assert error.errno == errno.EBADF + else: + raise AssertionError("watcher lifecycle: descriptor remains open") + assert plan.interrupted.is_set(), ( + "watcher lifecycle: failed startup stops admission" + ) + assert provider.calls == [] and plan._started_workers == [], ( + "watcher lifecycle: no worker is admitted after startup failure" + ) + assert plan._lock_file is None, ( + "watcher lifecycle: release the stage invocation lock" + ) + if case == "before-native": + assert elapsed >= nt.START_GRACE_SECONDS * 0.9, ( + "watcher lifecycle: allow bounded grace for a possible native start" + ) + assert not native_entered.is_set() + assert not target_finished.is_set() + assert not reads + elif case == "after-native": + assert reads == [pipe_fds[0]] + assert target_finished.is_set(), ( + "watcher lifecycle: join the watcher before cleanup returns" + ) + else: + assert elapsed >= nt.START_GRACE_SECONDS * 0.9, ( + "watcher lifecycle: delayed target outlasts the bounded start grace" + ) + assert not target_finished.is_set() + release_late.set() + watchers[0].join(timeout=10) + assert target_finished.is_set() + assert not reads, ( + "watcher lifecycle: late entry checks closing before reading" + ) + assert order.index("stage-closed") < order.index("late-target-entered") + assert not target_errors, target_errors + assert not watchers[0].is_alive(), ( + "watcher lifecycle: no watcher remains after probe cleanup" + ) + assert not any( + thread.name == "no-tools-signal-watcher" + for thread in threading.enumerate() + ) + monkeypatch.undo() + """, + ) + + +def test_first_sigint_after_watcher_shutdown_preserves_completed_finals( + setup, tmp_path +): + """R11.2: late recording forces exit 1 without watcher interruption or output.""" + import textwrap + + probe = tmp_path / "late_first_sigint.py" + probe.write_text( + textwrap.dedent(r""" + import json + import os + import runpy + import signal + import sys + import threading + from types import SimpleNamespace + + helpers = runpy.run_path(sys.argv[2]) + nt, stage, final, load = ( + helpers[name] for name in ("nt", "stage", "final", "load") + ) + design = json.loads(sys.argv[1]) + setup = { + "design": design, + "board": { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + }, + } + plan = stage(setup) + previous = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_signal = signal.signal + original_start = threading.Thread.start + watcher = [] + restoring, recorded = threading.Event(), threading.Event() + handler_returned = threading.Event() + errors = [] + + + def start(thread): + if thread.name == "no-tools-signal-watcher": + watcher.append(thread) + return original_start(thread) + + + def register(sig, handler): + if sig in previous and handler == previous[sig]: + if sig == signal.SIGINT: + assert len(watcher) == 1 and not watcher[0].is_alive(), ( + "late signal: watcher has finished before restoration" + ) + restoring.set() + assert recorded.wait(20), "late signal: controller finishes" + elif sig in previous: + original_handler = handler + + def tracked(signum, frame): + original_handler(signum, frame) + handler_returned.set() + + handler = tracked + return original_signal(sig, handler) + + + def send(): + try: + assert restoring.wait(20), "late signal: restoration pauses" + os.kill(os.getpid(), signal.SIGINT) + assert handler_returned.wait(20), "late signal: handler returns" + except BaseException as error: + errors.append(error) + finally: + recorded.set() + + + actor = threading.Thread(target=send, daemon=True) + actor.start() + threading.Thread.start = start + signal.signal = register + result = plan.execute() + actor.join(20) + assert not actor.is_alive() and not errors, errors + assert result == 1, "late signal: recorded SIGINT requires exit 1" + assert not plan.interrupted.is_set(), ( + "late signal: a finished watcher cannot set interruption" + ) + assert all(state == "complete" for state in plan.states.values()) + for model in plan.models: + for sid in design["draw_order"]: + assert load(final(plan, model, sid))["status"] == "complete", ( + "late signal: every worker's final is saved" + ) + assert "Interrupted" not in plan.out.getvalue(), ( + "late signal: a finished watcher prints no announcement" + ) + assert all(signal.getsignal(sig) == old for sig, old in previous.items()) + assert not watcher[0].is_alive() and plan._lock_file is None + print("late-first-signal-verified", flush=True) + sys.exit(result) + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), str(Path(__file__))], + env=env, + ) + assert result.returncode == 1, (result.stdout, result.stderr) + assert b"late-first-signal-verified\n" in result.stdout, ( + result.stdout, + result.stderr, + ) diff --git a/tests/test_confirmatory_no_tools_records.py b/tests/test_confirmatory_no_tools_records.py new file mode 100644 index 00000000..5c1b8ce3 --- /dev/null +++ b/tests/test_confirmatory_no_tools_records.py @@ -0,0 +1,505 @@ +"""Saved no-tools evidence must agree with the report before any provider call.""" + +from __future__ import annotations + +import hashlib +import io +import json +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests.test_confirmatory_no_tools import Provider, nt, write_board_fixture + +OPTIONAL_EVIDENCE = ( + "contract", + "messages", + "board_prompt_sha256", + "sent_board_prompt_sha256", +) +REQUIRED_FIELDS = ( + "model", + "arm", + "prompt_version", + "prompt_wording_sha256", + "tool_notice_sha256", + "tool_features", + "calculate_tool_sha256", + "variables", + "turns", + *OPTIONAL_EVIDENCE, +) + + +def full_record(board, model, engine, sid="h2"): + """Rebuild the exact treatment pins and prompt through the real harness.""" + profile = nt.pilot.model_profile( + model, arm="none", prompt_version=nt.PROMPT_VERSION + ) + with nt.pilot.stating_engine(engine.split()[-1]): + prompt, settings = nt.pilot.build_initial_request( + board["scenarios"][sid], board["variables"][sid], profile + ) + return { + "scenario_id": sid, + "model": model, + "arm": "none", + "status": "complete", + "error": None, + "prompt_version": nt.PROMPT_VERSION, + "prompt_wording_sha256": nt.pilot.prompt_wording_sha256(profile), + "tool_notice_sha256": hashlib.sha256(profile.notice.encode()).hexdigest(), + **nt.pilot.profile_treatment(profile), + "variables": list(board["variables"][sid]), + "turns": [{"provider_resolved_model": profile.model_id.split("/")[-1]}], + "predictions": {"snap": 1}, + "contract": profile.contract, + "messages": [{"role": "user", "content": prompt}], + "board_prompt_sha256": settings["_board_prompt_sha256"], + "sent_board_prompt_sha256": settings["_sent_board_prompt_sha256"], + "totals": {"cost_usd": 0.1}, + "attempts": [], + } + + +def write_fixture_json(path, payload): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload)) + + +def save_records(saved, change_link=None): + """Keep downstream byte hashes and link prefixes valid after one mutation.""" + + def save(path, payload): + text = json.dumps(payload) + if saved.serialized.get(path) != text: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + saved.serialized[path] = text + + links = [] + for n, payload in enumerate(saved.archives, 1): + payload["attempts"] = deepcopy(links) + path = saved.folder / "attempts" / f"h2-{n}.json" + save(path, payload) + try: + link = nt.entry(path, payload, n, nt.sha256_file(path)) + except (AttributeError, TypeError, ValueError, KeyError): + # Malformed turn evidence must reach the startup reader, rather + # than fail while this fixture refreshes otherwise valid links. + link = deepcopy(saved.original_links[n - 1]) + link["sha256"] = nt.sha256_file(path) + if change_link is not None: + change_link(link) + links.append(link) + for model, payload in saved.finals.items(): + payload["attempts"] = deepcopy(links) if model == nt.CHEAP[0] else [] + path = saved.root / model / "none/confirmatory/households/h2.json" + save(path, payload) + saved.original_links = links + + +@pytest.fixture(scope="module") +def treatment_templates(tmp_path_factory): + """Build actual treatment templates once and retain the installed versions.""" + directory = tmp_path_factory.mktemp("saved-record-board") + versions = nt.pilot._versions() + baseline = directory / "baseline-exclusions.json" + baseline.write_text('{"exclusions": []}\n') + design = { + "draw_order": ["h2", "outside"], + "spent_before": nt.NO_TOOLS_CAP, + "baseline_exclusions": { + "path": str(baseline), + "sha256": nt.sha256_file(baseline), + }, + "no_tools": { + "runs": "", + "folder": "none", + "stage": "confirmatory", + }, + } + with pytest.MonkeyPatch.context() as patch: + patch.setattr(nt.pilot, "_versions", lambda: versions) + write_board_fixture({"design": design}, directory / "board") + design["draw_order"] = ["h2"] + board = nt.pilot.load_board(Path(design["board"])) + finals = { + model: full_record(board, model, design["engine"]) for model in nt.CHEAP + } + yield design, finals + + +@pytest.fixture +def saved_records(tmp_path, treatment_templates): + template, template_finals = treatment_templates + design, finals = deepcopy(template), deepcopy(template_finals) + design["no_tools"]["runs"] = str(tmp_path / "no-tools") + archives = [deepcopy(finals[nt.CHEAP[0]]) for _ in range(2)] + for payload in archives: + payload.update(status="provider_error", error="APIConnectionError") + root = Path(design["no_tools"]["runs"]) + folder = root / nt.CHEAP[0] / "none/confirmatory/households" + saved = SimpleNamespace( + design=design, + root=root, + folder=folder, + finals=finals, + archives=archives, + original_links=[], + serialized={}, + paths={"final": folder / "h2.json", "archive": folder / "attempts/h2-1.json"}, + ) + save_records(saved) + return saved + + +def target_record(saved, reader): + return saved.finals[nt.CHEAP[0]] if reader == "final" else saved.archives[0] + + +def no_reply(payload, truncated=False): + payload["predictions"] = {"snap": None} + payload["turns"] = ( + [{**payload["turns"][0], "truncated_and_retried": True}] if truncated else [] + ) + for key in OPTIONAL_EVIDENCE: + payload.pop(key, None) + + +BAD_VALUES = { + "model": "synthetic-other-model", + "arm": "pe", + "prompt_version": "v1", + "prompt_wording_sha256": "wrong", + "tool_notice_sha256": "wrong", + "tool_features": [], + "calculate_tool_sha256": "wrong", + "variables": ["other"], + "contract": "synthetic-other-contract", + "board_prompt_sha256": "wrong", + "sent_board_prompt_sha256": "wrong", + "messages": {"content": "wrong"}, + "predictions": [], + "turns": None, +} +MUTATIONS = ( + [f"missing:{field}" for field in REQUIRED_FIELDS] + + [f"wrong:{field}" for field in BAD_VALUES] + + [ + "malformed-turn", + "missing-provider-model", + "wrong-provider-model", + "answer-without-completed-turn", + "empty-messages", + "malformed-message", + "wrong-sent-prompt", + ] + + [f"fallback:{field}" for field in OPTIONAL_EVIDENCE] +) + + +def mutate_record(payload, mutation): + kind, _, field = mutation.partition(":") + if kind == "missing": + payload.pop(field) + elif kind == "wrong": + payload[field] = deepcopy(BAD_VALUES[field]) + elif kind == "fallback": + no_reply(payload) + payload[field] = [] if field == "messages" else "wrong" + if field == "messages": + payload[field] = {"content": "wrong"} + elif mutation == "malformed-turn": + payload["turns"] = [None] + elif mutation == "missing-provider-model": + payload["turns"] = [{}] + elif mutation == "wrong-provider-model": + payload["turns"][0]["provider_resolved_model"] = "synthetic-other-model" + elif mutation == "answer-without-completed-turn": + payload["turns"] = [] + elif mutation == "empty-messages": + payload["messages"] = [] + elif mutation == "malformed-message": + payload["messages"] = [None] + else: + assert mutation == "wrong-sent-prompt" + payload["messages"][0]["content"] = "synthetic other household" + + +def assert_boundary(saved, boundary, refuses, expected_spend=0.5): + provider = Provider() + try: + if boundary == "startup": + if refuses is not None: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + saved.design, + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail( + "unusable saved evidence must refuse before execution" + ) + assert str(refuses) in str(caught.value) + else: + with nt.Stage( + saved.design, + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.spend.spent_usd == pytest.approx(expected_spend) + assert all(plan.standing(model, "h2") for model in nt.CHEAP) + else: + problems = nt.stage_problems(saved.design) + if refuses is not None: + assert any(str(refuses) in problem for problem in problems), problems + else: + assert problems == [] + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("mutation", MUTATIONS) +def test_saved_treatment_refuses_with_path(saved_records, reader, boundary, mutation): + saved = saved_records + mutate_record(target_record(saved, reader), mutation) + save_records(saved) + assert_boundary(saved, boundary, saved.paths[reader]) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("number", [True, 1.0], ids=["boolean", "float"]) +def test_saved_attempt_link_number_requires_integer(saved_records, boundary, number): + saved = saved_records + + def replace_number(link): + if link["n"] == 1: + link["n"] = number + + save_records(saved, replace_number) + assert_boundary(saved, boundary, saved.paths["final"]) + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize( + "form", ["full", "fallback", "truncated-only", "input-message"] +) +def test_valid_saved_treatment_preserves_spend(saved_records, reader, boundary, form): + saved = saved_records + payload = target_record(saved, reader) + if form in ("fallback", "truncated-only"): + no_reply(payload, truncated=form == "truncated-only") + elif form == "input-message": + first = payload["messages"][0] + first["input"] = first.pop("content") + save_records(saved) + assert_boundary(saved, boundary, None) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("form", ["extra-key", "absent-null-status"]) +def test_report_compatible_attempt_links_are_accepted(saved_records, boundary, form): + saved = saved_records + if form == "absent-null-status": + no_reply(saved.archives[0]) + saved.archives[0].pop("status") + + def change_link(link): + if form == "extra-key": + link["synthetic_extra"] = True + elif link["n"] == 1: + link.pop("status") + + save_records(saved, change_link) + assert_boundary(saved, boundary, None) + + +def add_board_household(saved, sid): + board = nt.pilot.load_board(Path(saved.design["board"])) + assert sid in board["scenarios"] and sid not in saved.design["draw_order"] + return board + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_cheap_final_outside_draw_refuses_with_path(saved_records, boundary): + saved = saved_records + board = add_board_household(saved, "outside") + payload = full_record(board, nt.CHEAP[0], saved.design["engine"], "outside") + path = saved.folder / "outside.json" + write_fixture_json(path, payload) + assert_boundary(saved, boundary, path) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_cheap_final_missing_predictions_refuses_with_path(saved_records, boundary): + saved = saved_records + payload = target_record(saved, "final") + no_reply(payload) + payload.pop("predictions") + save_records(saved) + assert_boundary(saved, boundary, saved.paths["final"]) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("reader", ["archive", "optional"]) +def test_absent_predictions_without_reply_remain_accepted( + saved_records, boundary, reader +): + saved = saved_records + if reader == "optional": + board = nt.pilot.load_board(Path(saved.design["board"])) + payload = full_record(board, "gpt-6-sol", saved.design["engine"]) + saved.finals["gpt-6-sol"] = payload + else: + payload = target_record(saved, "archive") + no_reply(payload) + payload.pop("predictions") + save_records(saved) + if reader == "optional" and boundary == "preflight": + # Absent predictions raise nothing; the unanswered final still owes a + # rerun, which the report refuses for every model. + assert nt.stage_problems(saved.design) == [ + "no-tools gpt-6-sol/h2: final action is rerun" + ] + return + assert_boundary( + saved, boundary, None, expected_spend=0.6 if reader == "optional" else 0.5 + ) + + +UNFINISHED = { + "optional-rerun": ("gpt-6-sol", "h2", "final action is rerun"), + "optional-pause": ("gpt-6-sol", "h2", "final action is pause-rerun"), + "optional-archive": ("gpt-6-sol", "h2", "archived attempts lack a final"), + "cheap-archive-outside-draw": ( + nt.CHEAP[1], + "outside", + "archived attempts lack a final", + ), +} + + +@pytest.mark.parametrize("unfinished", UNFINISHED) +def test_preflight_refuses_unfinished_histories_that_startup_resumes( + saved_records, unfinished +): + saved = saved_records + model, sid, needle = UNFINISHED[unfinished] + board = ( + add_board_household(saved, sid) + if sid == "outside" + else nt.pilot.load_board(Path(saved.design["board"])) + ) + payload = full_record(board, model, saved.design["engine"], sid) + error = ( + "AuthenticationError" + if unfinished == "optional-pause" + else "APIConnectionError" + ) + payload.update(status="provider_error", error=error) + households = saved.root / model / "none/confirmatory/households" + path = households / ( + f"{sid}.json" + if unfinished.endswith(("rerun", "pause")) + else f"attempts/{sid}-1.json" + ) + write_fixture_json(path, payload) + assert nt.stage_problems(saved.design) == [f"no-tools {model}/{sid}: {needle}"] + assert_boundary(saved, "startup", None, expected_spend=0.6) + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_saved_call_classification_failure_refuses_with_path( + saved_records, reader, boundary +): + saved = saved_records + payload = target_record(saved, reader) + payload["status"] = "complete" + payload["turns"][0]["calls"] = [None] + save_records(saved) + assert_boundary(saved, boundary, saved.paths[reader]) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("number", [True, 1.0], ids=["boolean", "float"]) +def test_archive_prefix_numeric_equality_remains_accepted( + saved_records, boundary, number +): + saved = saved_records + second = saved.archives[1] + second["attempts"][0]["n"] = number + path = saved.folder / "attempts/h2-2.json" + write_fixture_json(path, second) + final = target_record(saved, "final") + final["attempts"][1]["sha256"] = nt.sha256_file(path) + write_fixture_json(saved.paths["final"], final) + assert_boundary(saved, boundary, None) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_nonobject_archive_prefix_refuses_with_path(saved_records, boundary): + saved = saved_records + second = saved.archives[1] + second["attempts"][0] = None + path = saved.folder / "attempts/h2-2.json" + write_fixture_json(path, second) + final = target_record(saved, "final") + final["attempts"][1]["sha256"] = nt.sha256_file(path) + write_fixture_json(saved.paths["final"], final) + assert_boundary(saved, boundary, path) + + +@pytest.mark.parametrize("history", ["interrupted-archive", "rerunnable-final"]) +def test_real_treatment_resume_preserves_attempts_and_spend(saved_records, history): + saved = saved_records + first = saved.paths["archive"] + second = saved.folder / "attempts/h2-2.json" + final = saved.paths["final"] + prior_bytes = first.read_bytes() + second.unlink() + final.unlink() + if history == "rerunnable-final": + first.unlink() + final.write_bytes(prior_bytes) + board = nt.pilot.load_board(Path(saved.design["board"])) + calls = [] + + def fake_household( + scenario, variables, tool_call, completion_fn, spend, *, profile + ): + calls.append((profile.name, scenario.id)) + assert variables == board["variables"][scenario.id] + spend.add(0.1) + return full_record(board, profile.name, saved.design["engine"], scenario.id) + + with nt.Stage( + saved.design, + board=None, + models=[nt.CHEAP[0]], + run_household=fake_household, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.spend.spent_usd == pytest.approx(0.3) + assert plan.execute() == 0 + assert plan.spend.spent_usd == pytest.approx(0.4) + assert calls == [(nt.CHEAP[0], "h2")] + assert first.read_bytes() == prior_bytes + assert not second.exists() + payload = json.loads(final.read_text()) + assert payload["status"] == "complete" + assert payload["attempts"] == [ + nt.entry(first, saved.archives[0], 1, nt.sha256_file(first)) + ] + assert nt.stage_problems(saved.design) == [] diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py new file mode 100644 index 00000000..e2e924f1 --- /dev/null +++ b/tests/test_confirmatory_run.py @@ -0,0 +1,3323 @@ +"""The confirmatory runner (scripts/confirmatory_run.py). + +The runs here are fakes: a fake provider whose conversations a test plants, +a fake engine pool and a host log sampled every 20 ms, on a fake board. Each +run is checked by `run_problems`, which recomputes, from the files the runner +wrote and nothing else, every rule the run procedure states: the records and +their hashes, each attempt's class and action, the lanes and the schedule, +admission, pauses, stops and the ceiling. The study's differential test runs +the runner and checks its records with the confirmatory report itself. +""" + +from __future__ import annotations + +import copy +import hashlib +import io +import json +import math +import os +import signal +import subprocess +import sys +import threading +import time +from datetime import datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +import numpy as np +import pandas as pd +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import confirmatory_run as cr # noqa: E402 +import tool_pilot_argon as pilot # noqa: E402 + +from policybench.scenarios import Person, Scenario # noqa: E402 + +VARIABLES = ["snap", "federal_income_tax_before_refundable_credits"] +POOL = [f"h{i:02d}" for i in range(12)] +N_DRAW = 6 +COST = 0.01 +TZ = "America/New_York" +PREREG = "A frozen pre-registration for a test.\n" +ENGINE = "2.60.4" +# The harness times requests with the raw wall clock, which steps back by +# about 0.1 ms on this host, even within a thread; the report allows a second, as +# here, which also covers a small correction of the clock. +CLOCK_SLACK = 1.0 +# The anchor command in a test: a copy of the anchors to the test's folder. +ANCHOR_SCRIPT = Path(__file__).resolve().parent / "fake_anchor.py" +# How far apart anchors and their triggers' stops may be in a test, on a +# loaded host. +ANCHOR_SLACK = 30.0 +TRIGGER_BOUND = 5.0 + + +# ---------------------------------------------------------------- streams + + +def test_the_master_seed_is_the_first_sixteen_hex_digits(): + sha = hashlib.sha256(b"x").hexdigest() + assert cr.master_seed(sha) == int(sha[:16], 16) + with pytest.raises(ValueError): + cr.master_seed("not a sha") + + +def test_a_stream_is_keyed_by_its_name_alone(): + a = cr.stream(7, "order/m").integers(0, 1 << 30, 5) + b = cr.stream(7, "order/m").integers(0, 1 << 30, 5) + c = cr.stream(7, "order/n").integers(0, 1 << 30, 5) + assert list(a) == list(b) and list(a) != list(c) + key = int.from_bytes(hashlib.sha256(b"draw").digest()[:8], "big") + want = np.random.default_rng(np.random.SeedSequence(7, spawn_key=(key,))) + assert list(cr.stream(7, "draw").integers(0, 100, 8)) == list( + want.integers(0, 100, 8) + ) + + +@settings(deadline=None) +@given(st.integers(0, (1 << 64) - 1), st.integers(1, 12)) +def test_the_draw_is_seeded_and_without_replacement(master, n): + got = cr.draw(master, list(reversed(POOL)), n=n) + assert got == cr.draw(master, POOL, n=n) # the pool is sorted first + assert len(set(got)) == n and set(got) <= set(POOL) + + +def test_orders_are_permutations_drawn_for_every_household_in_draw_order(): + got = cr.orders(9, "m", POOL, cr.ARMS) + assert list(got) == POOL + assert all(sorted(o) == sorted(cr.ARMS) for o in got.values()) + generator = cr.stream(9, "order/m") + assert got[POOL[0]] == [str(a) for a in generator.permutation(list(cr.ARMS))] + # The first households' orders do not depend on the later ones. + assert cr.orders(9, "m", POOL[:3], cr.ARMS) == {s: got[s] for s in POOL[:3]} + + +# ---------------------------------------------------------------- arms + + +def test_arm_strings_follow_the_harness_feature_order(): + assert cr.arm_string("pe") == "pe" + assert cr.arm_string("pe3") == cr.arm_string("pe3-b") == "pe3" + for label, dropped in cr.DROPPED.items(): + if dropped: + kept = [f for f in pilot.TOOL_FEATURES if f not in dropped] + assert cr.arm_string(label) == "pe:" + "+".join(kept) + + +@pytest.mark.parametrize("model", cr.CHEAP + cr.FRONTIER) +def test_each_arm_has_its_registered_features(model): + for label in cr.ARMS if model in cr.CHEAP else cr.FRONTIER_ARMS: + profile = cr.arm_profile(model, label) + dropped = cr.DROPPED[label] + want = set() if dropped is None else set(pilot.TOOL_FEATURES) - set(dropped) + assert profile.uses_calculator and set(profile.tool_features) == want + assert profile.prompt_version == cr.PROMPT_VERSION + + +def test_an_arm_whose_features_are_not_registered_is_refused(monkeypatch): + # The harness builds an arm's features from its arm string: a string that + # does not give the label's registered features is refused. + monkeypatch.setattr(cr, "arm_string", lambda label: "pe:names+labels") + with pytest.raises(ValueError, match="features"): + cr.arm_profile("gpt-6-luna", "loo-flat") + + +def litellm_names(groups) -> set[str]: + """The names of litellm's exception classes under any of `groups`.""" + import inspect + + import litellm + + found = set() + for module in (litellm, litellm.exceptions): + for obj in vars(module).values(): + if inspect.isclass(obj) and issubclass(obj, BaseException): + if issubclass(obj, groups): + found.add(obj.__name__) + return found + + +def test_the_error_classes_are_policybench_s(): + """A record keeps the raised class's name, and the harness groups classes + by inheritance: so each list holds every litellm class under the + harness's group.""" + from policybench import eval_no_tools as e + + fatal = litellm_names(e.MODEL_FATAL_ERRORS) + retryable = litellm_names(e.RETRYABLE_PROVIDER_ERRORS) - fatal + assert cr.REQUEST | cr.ACCOUNT == fatal + assert cr.TRANSPORT - {"RequestWallTimeoutError"} == retryable + assert "RequestWallTimeoutError" in { + c.__name__ for c in e.RETRYABLE_PROVIDER_ERRORS + } + assert {"ContentPolicyViolationError", "RejectedRequestError"} <= cr.REQUEST + assert not cr.REQUEST & cr.ACCOUNT + assert not cr.TRANSPORT & (cr.REQUEST | cr.ACCOUNT) + assert cr.TIMEOUTS <= cr.TRANSPORT + + +# ---------------------------------------------------------------- classes + + +def record(status="complete", error=None, turns=1, calls=(), truncated=False): + """`truncated`: the record's last turn is a truncated one, its escalated + request then failing (with `turns` - 1 completed turns before it).""" + turn = {"calls": [{"result": r} for r in calls]} + out = [dict(turn) for _ in range(turns)] + if truncated: + out = out[: max(turns - 1, 0)] + [{"truncated_and_retried": True}] + return {"status": status, "error": error, "turns": out} + + +class AlwaysTrouble: + def trouble(self, start, end): + return "memory pressure" + + +CASES = [ + (record(), "ok"), + (record("incomplete"), "ok"), + (record("turn_limit"), "ok"), + (record("provider_error", "RateLimitError: slow down"), "provider"), + (record("provider_error", "BadRequestError: too long"), "model"), + (record("provider_error", "BadRequestError: bad", turns=0), "harness"), + (record("provider_error", "BadRequestError: x", truncated=True), "harness"), + (record("provider_error", "BadRequestError: x", 2, truncated=True), "model"), + (record("provider_error", "ContentPolicyViolationError: x", turns=0), "harness"), + (record("provider_error", "ContentPolicyViolationError: x"), "model"), + (record("provider_error", "RejectedRequestError: x", turns=0), "harness"), + # The escalated request's timeout is the model's long generation. + (record("provider_error", "Timeout: 300 s", truncated=True), "model"), + (record("provider_error", "Timeout: 300 s", 3, truncated=True), "model"), + (record("provider_error", "Timeout: 300 s"), "provider"), + # A truncated return is a return: no longer `none`. + (record("provider_error", "RateLimitError: x", truncated=True), "provider"), + ( + record("provider_error", "ProviderEmpty: no choices", truncated=True), + "inconclusive", + ), + (record("provider_error", "ProviderEmpty: no choices", turns=0), "none"), + (record("provider_error", "JSONSchemaValidationError: x"), "provider"), + (record("provider_error", "AuthenticationError: key", turns=0), "operator"), + (record("provider_error", "RateLimitError: insufficient_quota"), "operator"), + (record("provider_error", "ProviderEmpty: no choices"), "inconclusive"), + (record("provider_error", "RateLimitError: x", turns=0), "none"), + ( + record("harness_error", "RuntimeError: every tool worker was lost", turns=0), + "none", + ), + (record("harness_error", "the tool pool had no worker"), "inconclusive"), + (record("budget_stop"), "ceiling"), + (record("budget_stop", turns=0), "ceiling"), + (record(calls=[{"error": "timeout", "message": "slow"}]), "inconclusive"), + ( + record(calls=[{"error": "engine", "message": "tool worker failed: EOF"}]), + "inconclusive", + ), + (record(calls=[{"error": "engine", "message": "MemoryError: x"}]), "inconclusive"), + (record(calls=[{"error": "engine", "message": "ValueError: no person"}]), "model"), + (record(calls=[{"results": {"snap": {"error": "OSError: disk"}}}]), "inconclusive"), + (record(calls=[{"results": {"snap": {"error": "KeyError: x"}}}]), "model"), + # The explain feature's trace of a variable, which catches its own errors. + ( + record(calls=[{"explanations": {"snap": {"error": "MemoryError: x"}}}]), + "inconclusive", + ), + ( + record(calls=[{"explanations": {"snap": {"error": "OSError: disk"}}}]), + "inconclusive", + ), + (record(calls=[{"explanations": {"snap": {"error": "KeyError: x"}}}]), "model"), + (record(calls=[{"explanations": {"snap": {"value": 1.0}}}]), "ok"), + (record("running"), "inconclusive"), +] + + +@pytest.mark.parametrize("rec, want", CASES) +def test_each_failure_takes_its_class(rec, want): + assert cr.classify(rec, 0.0, 1.0, None)[0] == want + + +@pytest.mark.parametrize("rec, want", CASES) +def test_host_trouble_outranks_every_class_but_the_early_four(rec, want): + got = cr.classify(rec, 0.0, 1.0, AlwaysTrouble())[0] + assert got == (want if want in cr.EARLY else "host") + + +@pytest.mark.parametrize("rec, want", CASES) +def test_an_early_class_needs_no_host_evidence(rec, want): + early = cr.classify(rec, 0.0, 1.0, None)[0] + assert (early in cr.EARLY) == (want in cr.EARLY) + + +def test_actions_follow_the_class_and_the_attempt(): + for cls in ("ok", "model", "provider", "host", "none", "operator", "harness"): + for n in (1, 2, 3): + got = cr.action(cls, n) + if cls == "harness": + assert got == "stop" + elif cls in ("provider", "host", "none") and n <= 2: + assert got == "rerun" + elif cls == "operator" and n <= 2: + assert got == "pause-rerun" + else: + assert got == "stands" + assert cr.action("ceiling", 1) == "stopped" + assert cr.action("inconclusive", 1) == "stands" + + +# ---------------------------------------------------------------- host evidence + + +def guard_line(t: float, what: str) -> str: + stamp = datetime.fromtimestamp(t, ZoneInfo(TZ)).strftime("%Y-%m-%d %H:%M:%S") + return f"{stamp} [poll] lid closed on battery: {what} pids: 1 2" + + +def power_line(t: float, kind: str) -> str: + stamp = datetime.fromtimestamp(t, ZoneInfo(TZ)).strftime("%Y-%m-%d %H:%M:%S %z") + return f"{stamp} {kind:<20}\t{kind} due to a test" + + +def test_guard_pauses_and_power_sleeps_are_spans(): + t = 1_800_000_000.0 + guard = "\n".join( + [ + guard_line(t, "paused"), + guard_line(t + 10, "resumed"), + "junk", + guard_line(t + 20, "paused"), + ] + ) + assert cr.guard_spans(guard, TZ, end=t + 99) == [ + (t, t + 10, "guard pause"), + (t + 20, t + 99, "guard pause"), + ] + power = "\n".join( + [ + power_line(t, "Sleep"), + power_line(t + 5, "DarkWake"), + power_line(t + 7, "Sleep"), + ] + ) + assert cr.power_spans(power, end=t + 50) == [ + (t, t + 5, "sleep"), + (t + 7, t + 50, "sleep"), + ] + + +def test_an_excerpt_keeps_the_lines_stamped_in_its_window(): + t = 1_800_000_000.0 + text = "\n".join(guard_line(t + k, "paused") for k in (-5, 0, 5, 10)) + "\nx" + zone = ZoneInfo(TZ) + kept = cr.excerpt(text, lambda x: cr.guard_times(x, zone), t, t + 5) + assert kept.splitlines() == [guard_line(t, "paused"), guard_line(t + 5, "paused")] + + +class Readers: + """The host log's inputs, which a test can change while it runs.""" + + def __init__(self): + self.pressure = 1 + self.pressing = 0 # attempts under way that raise the pressure + self.stall = 0.0 + self.guard: list[str] = [] + self.power: list[str] = [] + self.lock = threading.Lock() + + def read_pressure(self): + with self.lock: + stall, self.stall = self.stall, 0.0 + if stall: + time.sleep(stall) + return 2 if self.pressing else self.pressure + + def host(self, folder): + return cr.HostLog( + folder, + timezone=TZ, + interval=0.02, + read_load=lambda: (1.0, 2.0, 3.0), + read_pressure=self.read_pressure, + read_guard=lambda: "\n".join(self.guard), + read_power=lambda: "\n".join(self.power), + ) + + +def wait_for_samples(log: Path, enough, seconds: float = 5.0) -> None: + """Wait until the host log's complete samples satisfy `enough`. A loaded + host can stretch the 0.02 s sampler past any fixed sleep.""" + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + text = log.read_text() if log.exists() else "" + if enough([json.loads(x) for x in text.split("\n")[:-1]]): + return + time.sleep(0.01) + + +@pytest.fixture +def fast(monkeypatch): + """Short evidence windows, with room for the shared host's scheduler. + Intentional gaps scale with this bound; ordinary stalls must not turn + an untouched fake run into a host-failure test.""" + monkeypatch.setattr(cr, "PRESSURE_WINDOW", 0.06) + monkeypatch.setattr(cr, "HOST_LOG_GAP", 5.0) + monkeypatch.setattr(cr, "WAIT_SECONDS", 0.005) + monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) + monkeypatch.setattr(cr, "SAMPLER_SILENCE", 1.0) + monkeypatch.setattr(cr, "READ_WAIT", 0.01) + monkeypatch.setattr(cr, "ANCHOR_SECONDS", 0.3) + handlers = {s: signal.getsignal(s) for s in (signal.SIGINT, signal.SIGTERM)} + yield + for signum, handler in handlers.items(): + signal.signal(signum, handler) + + +def test_the_host_log_samples_and_closes(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + wait_for_samples(tmp_path / "host-log.jsonl", lambda samples: len(samples) >= 4) + host.close() + meta = json.loads((tmp_path / "host.json").read_text()) + samples = [ + json.loads(x) for x in (tmp_path / "host-log.jsonl").read_text().splitlines() + ] + assert meta["timezone"] == TZ and meta["start"] <= samples[0]["time"] + assert samples[-1]["time"] <= meta["end"] + assert len(samples) >= 4 + assert samples[0]["load1"] == 1.0 and samples[0]["pressure"] == 1 + times = [s["time"] for s in samples] + assert times == sorted(times) + + +def test_pressure_near_an_attempt_is_host_trouble_once_settled(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + start = time.time() + readers.pressure = 2 + wait_for_samples( + tmp_path / "host-log.jsonl", + lambda samples: any(x["pressure"] == 2 for x in samples), + ) + readers.pressure = 1 + end = time.time() + host.wait_settled(end) + assert host.settled(end) + host.refresh(after=end) + assert host.trouble(start, end) == "memory pressure" + assert host.trouble(end + 1, end + 2) is None + host.close() + + +def test_a_stalled_host_log_is_a_gap(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + time.sleep(0.03) + start = time.time() + stall = cr.HOST_LOG_GAP + 0.3 + readers.stall = stall + time.sleep(stall + 0.05) + end = time.time() + host.wait_settled(end) + host.refresh(after=end) + assert host.trouble(start, end) == "host log gap" + host.close() + + +def test_guard_pauses_and_sleeps_reach_an_attempt_after_a_refresh(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + start = time.time() + readers.guard += [guard_line(start, "paused"), guard_line(start + 1, "resumed")] + end = time.time() + host.wait_settled(end) + assert host.trouble(start, end) is None # not read yet + host.refresh(after=end) + assert host.trouble(start, end) == "guard pause" + readers.guard.clear() + readers.power += [power_line(start, "Sleep"), power_line(start + 1, "Wake")] + host.refresh(after=time.time()) + assert host.trouble(start, end) == "sleep" + host.close() + assert "Sleep" in (tmp_path / "pmset.log").read_text() + + +def test_a_failed_pressure_reading_is_no_sample(tmp_path, fast): + """A sample with no pressure reading is written, but it is no sample: a + failing reading under pressure would hide the evidence the rule reads, + so the silence it leaves is a gap.""" + readers = Readers() + host = readers.host(tmp_path) + time.sleep(0.03) + start = time.time() + readers.pressure = None + time.sleep(cr.HOST_LOG_GAP + 0.2) + readers.pressure = 1 + end = time.time() + host.wait_settled(end) + host.refresh(after=end) + assert host.trouble(start, end) == "host log gap" + host.close() + samples = [ + json.loads(x) for x in (tmp_path / "host-log.jsonl").read_text().splitlines() + ] + assert any(x["pressure"] is None for x in samples) + folder_host = host_from_files(tmp_path) + assert folder_host.trouble(start, end) == "host log gap" # the report's reading + + +def test_the_host_log_is_a_hash_chain_with_a_monotonic_time(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + time.sleep(0.1) + host.close() + path = tmp_path / "host-log.jsonl" + assert chain_problems(path, cr.HOST_SEED) == [] + lines = path.read_text().splitlines() + assert host.heads() == {"lines": len(lines), "sha256": cr.line_sha(lines[-1])} + ticks = [json.loads(x)["monotonic"] for x in lines] + assert ticks == sorted(ticks) + + +def test_a_failed_log_read_is_tried_again(tmp_path, fast): + calls = [] + + def power(): + calls.append(1) + if len(calls) <= 2: + raise subprocess.CalledProcessError(1, ["pmset"]) + return "" + + host = cr.HostLog( + tmp_path, + timezone=TZ, + interval=0.02, + read_load=lambda: (1.0, 2.0, 3.0), + read_pressure=lambda: 1, + read_guard=lambda: "", + read_power=power, + ) + host.refresh(after=time.time()) + assert len(calls) == 3 # two failures, then a read + host.close() + assert (tmp_path / "pmset.log").exists() + + +def test_logs_that_cannot_be_read_are_a_runner_error(tmp_path, fast): + def power(): + raise subprocess.TimeoutExpired(["pmset"], 120) + + host = cr.HostLog( + tmp_path, + timezone=TZ, + interval=0.02, + read_load=lambda: (1.0, 2.0, 3.0), + read_pressure=lambda: 1, + read_guard=lambda: "", + read_power=power, + ) + with pytest.raises(cr.HostEvidenceError): + host.refresh(after=time.time()) + host.close() + assert "TimeoutExpired" in (tmp_path / "host-error.txt").read_text() + assert not (tmp_path / "pmset.log").exists() + + +def test_a_sampler_whose_samples_fail_is_dead_after_its_silence(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + assert host.dead() is None + readers.pressure = None + host.sample() + time.sleep(cr.SAMPLER_SILENCE + 0.2) + assert "samples failing since" in host.dead() + readers.pressure = 1 + host.sample() + assert host.dead() is None # a good sample: alive again + host.close() + + +# ---------------------------------------------------------------- the ceiling + + +def test_the_clock_never_goes_back(monkeypatch): + monkeypatch.setattr(cr, "_clock_last", cr._clock_last) # restored after + base = time.time() + 1000.0 + readings = iter([base, base - 1e-4, base + 2e-4, base - 5.0, base + 3e-4]) + monkeypatch.setattr(time, "time", lambda: next(readings)) + assert [cr.clock() for _ in range(5)] == [ + base, + base, + base + 2e-4, + base + 2e-4, + base + 3e-4, + ] + + +def test_a_backward_wall_clock_does_not_delay_a_due_anchor(monkeypatch): + """Once the ordered clock reaches the due time, a wall-clock step cannot + make the anchor not due again.""" + run = cr.Run.__new__(cr.Run) + run._pushing = threading.Lock() + run._push_error = None + base = time.time() + 1000.0 + run._next_anchor = base + monkeypatch.setattr(cr, "_clock_last", base - 1.0) + readings = iter([base - 1.0, base + 1.0, base - 5.0]) + monkeypatch.setattr(time, "time", lambda: next(readings)) + assert [run.anchor_due() for _ in range(3)] == [False, True, True] + + +def test_the_anchor_interval_and_retry_wait_are_registered_durations(): + """paper/preregistration-draft.md:218: anchors every 15 minutes, and retries + a minute after the failed push ends.""" + assert cr.ANCHOR_SECONDS == 900.0 + assert cr.ANCHOR_RETRY == 60.0 + + +def test_a_wall_clock_that_steps_back_keeps_the_run_procedure(world, monkeypatch): + """Every reading of the wall clock up to 0.2 ms early (steps back of up to + 0.1 ms seen on this host): the run's times still order its attempts, + stops and decisions.""" + import random + + jitter, real = random.Random(7), time.time + monkeypatch.setattr(time, "time", lambda: real() - jitter.uniform(0.0, 2e-4)) + w = world().execute() + assert run_problems(w) == [] + for model in w.models: + assert len(w.attempts(model)) == N_DRAW * len(labels_of(model)) + + +def test_the_ceiling_crosses_at_its_amount_and_stays_crossed(): + ceiling = cr.Ceiling(1.0, 0.25) + ceiling.add(0.5) + assert not ceiling.exhausted() and ceiling.crossed_at is None + ceiling.add(0.25) + assert ceiling.exhausted() and ceiling.crossed_at is not None + first = ceiling.crossed_at + ceiling.add(None) + assert ceiling.crossed_at == first and ceiling.spent_usd == pytest.approx(1.0) + + +def test_the_ceiling_ignores_a_reserve(): + # No conversation stops for a model's spend: only the ceiling stops one. + assert not cr.Ceiling(1.0, 0.0).exhausted(reserve_usd=5.0) + + +# ---------------------------------------------------------------- fake runs + + +def scenario(sid: str) -> Scenario: + i = POOL.index(sid) + return Scenario( + id=sid, + state="CO", + filing_status="single", + adults=[Person(name="head", age=30 + i, employment_income=10_000.0 * (i + 1))], + children=[], + ) + + +class FakePool: + """An engine pool that answers every call and keeps what it was sent.""" + + def __init__(self, lanes, fail: bool = False): + if fail: + raise RuntimeError("tool worker failed to start: a test") + self.messages: list = [] + self.lost = 0 + self.closed = False + + def call(self, message): + self.messages.append(message) + return {"results": {"snap": {"values": {"spm": 1.0}}}}, 0.0 + + def counters(self): + return {"timeouts": 0, "crashes": 0, "lost": self.lost} + + def close(self): + self.closed = True + + +class FakeProvider: + """Conversations a test plants. specs[(model, label, household, n)] holds a + record's `status` and `error`, `turns` (default 1), `cost` per turn, + `sleep` per turn, `hold` (an event each turn after the first waits for), + `calls` (tool arguments, one call each, per turn), + `pressure` (high pressure during the attempt), `stall` (a host-log stall, + seconds), `guard` (a guard pause over the attempt), `resume_after` + (seconds after which the operator resumes), `raise` (the conversation + raises before its first turn), `raise_after` (it raises after its turns) + and `check` (a function of the lanes, called first).""" + + def __init__(self, world, specs=None): + self.world, self.specs = world, specs or {} + self.count: dict = {} + self.lock = threading.Lock() + + def __call__(self, lanes, sid, label, tool_call, spend): + model = lanes.model + with self.lock: + n = self.count[(model, label, sid)] = ( + self.count.get((model, label, sid), 0) + 1 + ) + spec = self.specs.get((model, label, sid, n), {}) + readers = self.world.readers + if spec.get("check"): + spec["check"](lanes) + if spec.get("raise"): + raise RuntimeError("a test raised") + if spec.get("pressure"): + with readers.lock: + readers.pressing += 1 + lanes.run.host.sample() # the planted pressure is actually observed + if spec.get("stall"): + readers.stall = spec["stall"] + start = time.time() + rec = { + "scenario_id": sid, + "model": model, + "arm": cr.arm_string(label), + "variables": list(VARIABLES), + "status": "running", + "turns": [], + "predictions": {v: 0.0 for v in VARIABLES}, + } + for i in range(spec.get("turns", 1)): + if spend.exhausted(): + rec["status"] = "budget_stop" + break + started = time.time() + if i and spec.get("hold"): + assert spec["hold"].wait(30) + time.sleep(spec.get("sleep", 0.0)) + completed = time.time() + cost = spec.get("cost", COST) + spend.add(cost) + turn = { + "index": i, + "request_started_at": started, + "request_completed_at": completed, + "total_cost_usd": cost, + "calls": [], + } + for arguments in spec.get("calls", []): + result, latency = tool_call(arguments) + turn["calls"].append({"arguments": arguments, "result": result}) + rec["turns"].append(turn) + if spec.get("raise_after"): + raise RuntimeError("a test raised after spending") + if rec["status"] == "running": + rec["status"] = spec.get("status", "complete") + if spec.get("error"): + rec["error"] = spec["error"] + if spec.get("guard"): + readers.guard += [ + guard_line(math.floor(start) - 1, "paused"), + guard_line(math.ceil(time.time()) + 1, "resumed"), + ] + if spec.get("stall"): + time.sleep(spec["stall"] + 0.02) + if spec.get("pressure"): + with readers.lock: + readers.pressing -= 1 + rec["totals"] = {"cost_usd": sum(t["total_cost_usd"] for t in rec["turns"])} + if spec.get("resume_after") is not None: + folder = lanes.folder + + def resume(): + with lanes.cond: + if ( + not lanes.cond.wait_for( + lambda: lanes.paused or lanes.halted(), timeout=30 + ) + or not lanes.paused + ): + return + time.sleep(spec["resume_after"]) + cr.write_json( + folder / "RESUME.json", + {"time": cr.clock(), "note": "fixed in a test"}, + ) + + threading.Thread(target=resume, daemon=True).start() + return rec + + +class World: + """A fake board, design and run.""" + + def __init__( + self, + tmp: Path, + monkeypatch, + specs=None, + caps=None, + reserve=7 * 3 * COST, + overall_cap=100.0, + spent_before=1.0, + no_output=(), + models=None, + pool=FakePool, + ): + self.tmp = tmp + prereg = tmp / "prereg.md" + prereg.write_text(PREREG) + frozen = cr.sha256_file(prereg) + master = cr.master_seed(frozen) + self.draw_order = cr.draw(master, POOL, n=N_DRAW) + board_path = tmp / "board" + board_path.mkdir() + (board_path / "reference_outputs.csv").write_text( + "scenario_id,variable,value\n" + ) + self.no_output = set(no_output) + scored = [ + (sid, v) for sid in POOL if sid not in self.no_output for v in VARIABLES + ] + self.board = { + "scenarios": {sid: scenario(sid) for sid in POOL}, + "variables": {sid: list(VARIABLES) for sid in POOL}, + "scored": pd.DataFrame(scored, columns=["scenario_id", "variable"]), + } + monkeypatch.setattr(pilot, "load_board", lambda path: self.board) + monkeypatch.setattr(pilot, "board_engine", lambda path=None: ENGINE) + installed = {**pilot._versions(), "policyengine-us": ENGINE} + monkeypatch.setattr(pilot, "_versions", lambda: installed) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + monkeypatch.setitem( + pilot._STATED_ENGINE, "version", pilot._STATED_ENGINE["version"] + ) + self.models = tuple(models or cr.CHEAP + cr.FRONTIER) + budget_models = tuple(dict.fromkeys(cr.CHEAP + cr.FRONTIER + self.models)) + self.design = { + "preregistration": str(prereg), + "frozen_sha256": frozen, + "runs": str(tmp / "runs"), + "stage": "confirm", + "board": str(board_path), + "engine": f"policyengine-us {ENGINE}", + "draw_order": self.draw_order, + "arms": {a: a for a in cr.ARMS}, + "frontier_arms": {a: f"f-{a}" for a in cr.FRONTIER_ARMS}, + "caps": {m: (caps or {}).get(m, 10.0) for m in budget_models}, + "reserves": {m: reserve for m in budget_models}, + "overall_cap": overall_cap, + "spent_before": spent_before, + "workers": 2, + "anchor_command": [sys.executable, str(ANCHOR_SCRIPT), str(tmp)], + } + self.readers = Readers() + self.provider = FakeProvider(self, specs) + self.run = cr.Run( + self.design, + models=self.models, + converse=self.provider, + pool_factory=pool, + completion_for=lambda profile: None, + host_factory=self.readers.host, + out=io.StringIO(), + ) + + def execute(self): + self.run.execute() + return self + + # What the runner wrote. + + def runs(self) -> Path: + return Path(self.design["runs"]) + + def lines(self, model, name="run-log.jsonl") -> list[dict]: + path = self.runs() / model / name + if not path.exists(): + return [] + return [json.loads(x) for x in path.read_text().splitlines() if x.strip()] + + def attempts(self, model) -> list[dict]: + return [x for x in self.lines(model) if "event" not in x] + + def events(self, model, kind=None) -> list[dict]: + return [ + x + for x in self.lines(model) + if x.get("event") and kind in (None, x["event"]) + ] + + def folder(self, model, label) -> Path: + arms = ( + self.design["arms"] if model in cr.CHEAP else self.design["frontier_arms"] + ) + return self.runs() / model / arms[label] / "confirm" / "households" + + +def labels_of(model): + return cr.ARMS if model in cr.CHEAP else cr.FRONTIER_ARMS + + +def host_from_files(folder: Path): + """The host evidence as the report builds it, from the archived files.""" + meta = json.loads((folder / "host.json").read_text()) + start, end = float(meta["start"]), float(meta["end"]) + samples = [ + json.loads(x) for x in (folder / "host-log.jsonl").read_text().splitlines() if x + ] + spans = cr.guard_spans( + (folder / "clamshell-guard.log").read_text(), meta["timezone"], end + ) + spans += cr.power_spans((folder / "pmset.log").read_text(), end) + # A sample with no pressure reading is no sample. + good = [s for s in samples if s.get("pressure") is not None] + times = sorted(float(s["time"]) for s in good) + for a, b in zip([start] + times, times + [end]): + if b - a > cr.HOST_LOG_GAP: + spans.append((a, b, "host log gap")) + pressure = [float(s["time"]) for s in good if int(s["pressure"]) > 1] + + class Host: + def trouble(self, s, e): + for a, b, what in spans: + if a <= e and b >= s: + return what + for t in pressure: + if s - cr.PRESSURE_WINDOW <= t <= e + cr.PRESSURE_WINDOW: + return "memory pressure" + return None + + return Host() + + +def chain_problems(path: Path, seed: str) -> list[str]: + """Whether each line's `prev` is the line before's sha256 (the first's, + `seed`).""" + head = seed + for i, text in enumerate(path.read_text().splitlines() if path.exists() else []): + if json.loads(text).get("prev") != head: + return [f"{path.name} line {i + 1}: its prev is not the line before's hash"] + head = cr.line_sha(text) + return [] + + +def chain_heads(path: Path, seed: str) -> list[str]: + """The chain's head after each count of lines, from none.""" + heads = [seed] + for text in path.read_text().splitlines() if path.exists() else []: + heads.append(cr.line_sha(text)) + return heads + + +def anchor_problems(world: World) -> list[str]: + """The anchors: a chain of their own; each holds every log's head at its + count; the last holds every log whole; the last was pushed.""" + problems: list[str] = [] + runs, frozen = world.runs(), world.design["frozen_sha256"] + path = runs / "anchors.jsonl" + problems += chain_problems(path, cr.chain_seed(frozen, "", "anchors.jsonl")) + anchors = [json.loads(x) for x in path.read_text().splitlines()] + heads = { + (model, name): chain_heads( + runs / model / name, cr.chain_seed(frozen, model, name) + ) + for model in world.models + for name in ("run-log.jsonl", "admission-log.jsonl") + } + host = chain_heads(runs / "host" / "host-log.jsonl", cr.HOST_SEED) + for i, anchor in enumerate(anchors): + for (model, name), chain in heads.items(): + got = anchor["models"][model][name] + if got["lines"] >= len(chain) or chain[got["lines"]] != got["sha256"]: + problems.append(f"anchor {i}: {model}/{name} is not the log's head") + got = anchor["host-log.jsonl"] + if got["lines"] >= len(host) or host[got["lines"]] != got["sha256"]: + problems.append(f"anchor {i}: the host log is not the log's head") + last = anchors[-1] + for (model, name), chain in heads.items(): + if last["models"][model][name]["lines"] != len(chain) - 1: + problems.append(f"the last anchor leaves out lines of {model}/{name}") + if last["host-log.jsonl"]["lines"] != len(host) - 1: + problems.append("the last anchor leaves out host-log lines") + for a, b in zip(anchors, anchors[1:]): + if b["time"] - a["time"] > cr.ANCHOR_SECONDS + ANCHOR_SLACK: + problems.append(f"anchors {a['time']!r} and {b['time']!r} too far apart") + pushes = [ + json.loads(x) for x in (runs / "anchor-pushes.jsonl").read_text().splitlines() + ] + final = cr.line_sha(path.read_text().splitlines()[-1]) + if not any(x["anchor"] == final and x["returncode"] == 0 for x in pushes): + problems.append("the last anchor was not pushed") + copy = world.tmp / "anchored.jsonl" + if not copy.exists() or copy.read_text() != path.read_text(): + problems.append("the anchor command's copy is not the anchors") + return problems + + +def run_problems(world: World) -> list[str]: + """Every rule of the run procedure, recomputed from the runner's files.""" + problems: list[str] = [] + host = host_from_files(world.runs() / "host") + frozen = world.design["frozen_sha256"] + problems += chain_problems(world.runs() / "host" / "host-log.jsonl", cr.HOST_SEED) + problems += anchor_problems(world) + late_total = 0.0 + position = {sid: i for i, sid in enumerate(world.draw_order)} + master = cr.master_seed(world.design["frozen_sha256"]) + all_attempts = [] + ceiling_events = {} + for model in world.models: + labels = labels_of(model) + drawn = cr.orders(master, model, world.draw_order, labels) + lines = world.attempts(model) + events = world.events(model) + for name in ("run-log.jsonl", "admission-log.jsonl"): + problems += chain_problems( + world.runs() / model / name, cr.chain_seed(frozen, model, name) + ) + for e in events: + if e["event"] == "stop" and e.get("cause") not in cr.STOP_CAUSES: + problems.append(f"{model}: a stop with no cause ({e})") + if e["event"] == "ceiling" and e.get("cause") != "ceiling": + problems.append(f"{model}: a ceiling event with another cause") + late_total += sum(e["cost_usd"] for e in events if e["event"] == "late-cost") + causes = {e.get("cause") for e in events if e["event"] == "stop"} + stops = [e["time"] for e in events if e["event"] in ("stop", "ceiling")] + stopped_at = min(stops) if stops else math.inf + ceiling_events[model] = [e["time"] for e in events if e["event"] == "ceiling"] + cost: dict[str, float] = {} + span: dict[str, list[float]] = {} + by_run: dict = {} + cut_short: set[str] = set() # an arm or a rerun the stop kept from running + for x in lines: + by_run.setdefault((x["household"], x["arm"]), []).append(x) + # Records and hashes. + for label in labels: + folder = world.folder(model, label) + finals = sorted(folder.glob("*.json")) if folder.is_dir() else [] + for path in finals: + if (path.stem, label) not in by_run: + problems.append( + f"{model}/{label}/{path.stem}: a record with no line" + ) + for (sid, label), xs in sorted(by_run.items()): + at = f"{model}/{label}/{sid}" + xs.sort(key=lambda x: x["attempt"]) + if [x["attempt"] for x in xs] != list(range(1, len(xs) + 1)): + problems.append(f"{at}: attempts {[x['attempt'] for x in xs]}") + continue + final_path = world.folder(model, label) / f"{sid}.json" + final = json.loads(final_path.read_text()) + entries = final.get("attempts") or [] + if len(entries) != len(xs) - 1: + problems.append(f"{at}: {len(entries)} entries for {len(xs)} attempts") + continue + for k, x in enumerate(xs, start=1): + if k < len(xs): + entry = entries[k - 1] + path = world.folder(model, label) / entry["file"] + rec = json.loads(path.read_text()) + if entry["n"] != k or entry["sha256"] != cr.sha256_file(path): + problems.append(f"{at}#{k}: its entry does not hash its file") + if entry["class"] != x["class"] or entry["status"] != rec["status"]: + problems.append(f"{at}#{k}: its entry is not its line's") + if x["sha256"] != entry["sha256"]: + problems.append(f"{at}#{k}: its line's hash is not its file's") + else: + rec = final + if x["sha256"] != cr.sha256_file(final_path): + problems.append( + f"{at}#{k}: its line's hash is not its record's" + ) + cls, _ = cr.classify(rec, x["start"], x["end"], host) + if x["class"] == "unsettled" and "runner-error" in causes: + cls = "unsettled" # classed when the runner failed: it stands + rule = cr.action(cls, k) + if x["class"] != cls or x["action"] != rule: + problems.append( + f"{at}#{k}: logged {x['class']}/{x['action']}, " + f"recomputed {cls}/{rule}" + ) + if x["status"] != rec["status"] or x["lane"] != 1 + position[sid] % 2: + problems.append(f"{at}#{k}: status or lane") + if k < len(xs) and rule not in ("rerun", "pause-rerun"): + problems.append(f"{at}#{k}: rerun after {cls}") + if k == len(xs) and rule in ("rerun", "pause-rerun"): + cut_short.add(sid) + if not stops: + problems.append( + f"{at}#{k}: a rerun due and never run, with no stop" + ) + times = [ + t[key] + for t in rec.get("turns") or [] + for key in ("request_started_at", "request_completed_at") + ] + if times and ( + min(times) < x["start"] - CLOCK_SLACK + or max(times) > x["end"] + CLOCK_SLACK + ): + problems.append( + f"{at}#{k}: requests {min(times)!r}-{max(times)!r} outside " + f"its start and end {x['start']!r}-{x['end']!r}" + ) + if x["start"] > stopped_at: + problems.append(f"{at}#{k}: started after a stop") + cost[sid] = cost.get(sid, 0.0) + cr.attempt_cost(rec) + s, e = span.get(sid, [math.inf, -math.inf]) + span[sid] = [min(s, x["start"]), max(e, x["end"])] + all_attempts.append((model, sid, label, k, x, rec)) + # The schedule. + for lane in (1, 2): + mine = sorted( + (x for x in lines if x["lane"] == lane), key=lambda x: x["start"] + ) + for a, b in zip(mine, mine[1:]): + if b["start"] < a["end"]: + problems.append(f"{model} lane {lane}: attempts overlap") + blocks: list = [] + for x in mine: + if blocks and blocks[-1][0] == x["household"]: + blocks[-1][1].append(x) + else: + blocks.append((x["household"], [x])) + sids = [s for s, _ in blocks] + if len(set(sids)) != len(sids) or sids != sorted(sids, key=position.get): + problems.append( + f"{model} lane {lane}: households split or out of order" + ) + for i, (sid, block) in enumerate(blocks): + got = [(x["attempt"], x["arm"]) for x in block] + firsts = [a for n, a in got if n == 1] + want = [(1, a) for a in firsts] + for n in range(2, cr.MAX_RERUNS + 2): + again = {x["arm"] for x in block if x["attempt"] == n} + want += [(n, a) for a in drawn[sid] if a in again] + cut = firsts != drawn[sid] + if cut: + cut_short.add(sid) + if got != want or firsts != drawn[sid][: len(firsts)]: + problems.append(f"{model}/{sid}: ran {got}, drawn {drawn[sid]}") + if cut and not (stops and i == len(blocks) - 1): + problems.append(f"{model}/{sid}: cut short without a stop") + # Admission. + decisions = sorted( + world.lines(model, "admission-log.jsonl"), key=lambda x: x["time"] + ) + cap = world.design["caps"][model] + reserve = world.design["reserves"][model] + admitted: dict[str, float] = {} + refused = False + # A household a stop cut short is under way until the stop. + ended = { + h: max(e, stopped_at) if h in cut_short else e for h, (_, e) in span.items() + } + for x in decisions: + sid, t = x["household"], x["time"] + if sid in world.no_output: + problems.append( + f"{model}/{sid}: a household with no scored output decided" + ) + finished = [h for h in admitted if h in ended and ended[h] <= t] + spend = sum(cost[h] for h in finished) + under = sum(1 for h in admitted if h not in finished) + 1 + ok = not refused and spend + under * reserve <= cap + cr.ADMISSION_SLACK + if x["decision"] != ("admit" if ok else "refuse"): + problems.append(f"{model}/{sid}: logged {x['decision']}, rule {ok}") + if abs(x["finished_spend"] - spend) > 1e-9 or x["under_way"] != under: + problems.append( + f"{model}/{sid}: spend {x['finished_spend']} vs {spend}" + ) + if x["lane"] != 1 + position[sid] % 2: + problems.append(f"{model}/{sid}: decided on the other lane") + if x["decision"] == "admit": + admitted[sid] = t + else: + refused = True + for sid, (s, _) in span.items(): + if sid not in admitted or s < admitted[sid]: + problems.append(f"{model}/{sid}: ran without, or before, its admission") + for lane in (1, 2): + expect = [ + s + for s in world.draw_order + if 1 + position[s] % 2 == lane and s not in world.no_output + ] + decided = [x["household"] for x in decisions if x["lane"] == lane] + if decided != expect[: len(decided)]: + problems.append(f"{model} lane {lane}: decisions skip a household") + if len(decided) < len(expect) and not (refused or stops): + problems.append(f"{model} lane {lane}: stopped deciding early") + if not stops and set( + a for a in admitted if 1 + position[a] % 2 == lane + ) - set(span): + problems.append(f"{model} lane {lane}: admitted and never run") + # Pauses. + resumes = sorted(e["time"] for e in events if e["event"] == "resume") + for x in lines: + if x["class"] != "operator": + continue + pauses = [ + e for e in events if e["event"] == "pause" and e["time"] >= x["end"] - 1 + ] + if not pauses: + problems.append( + f"{model}/{x['household']}: an operator failure, no pause" + ) + continue + until = next((t for t in resumes if t >= pauses[0]["time"]), math.inf) + if any(x["end"] < y["start"] < until for y in lines): + problems.append(f"{model}: an attempt started during a pause") + # The ceiling, over every model. + total = world.design["spent_before"] + sum( + cr.attempt_cost(r) for *_, r in all_attempts + ) + ceiling = world.design["overall_cap"] + cr.CEILING_MARGIN + if total >= ceiling and not all(ceiling_events.values()): + problems.append(f"spend {total} reached the ceiling and a model has no event") + for times in ceiling_events.values(): + for t in times: + if any(x["start"] > t for *_, x, _ in all_attempts): + problems.append("an attempt started after the ceiling") + # Conservation: the ceiling counted exactly the records' spend, and the + # spend of abandoned attempts after their deadline. + if abs(world.run.ceiling.spent_usd - total - late_total) > 1e-9: + problems.append( + f"the ceiling counted {world.run.ceiling.spent_usd}, records {total} " + f"and late spend {late_total}" + ) + problems += trigger_problems(world, all_attempts) + return problems + + +def trigger_problems(world: World, all_attempts) -> list[str]: + """Each stop rule's trigger has its stop, and each stop of those causes + its trigger: a harness failure and an abandoned attempt stop the model at + their end, before any other attempt starts; one inconclusive error ending + runs in a sixth household stops it once that household's pass is classed.""" + problems: list[str] = [] + for model in world.models: + events = world.events(model) + mine = [(sid, k, x, rec) for m, sid, _, k, x, rec in all_attempts if m == model] + triggers: list[tuple[str, float]] = [] + joined: dict[str, dict[str, float]] = {} # key -> household -> pass end + for sid, k, x, rec in mine: + if x["class"] == "harness": + triggers.append(("harness", x["end"])) + if str(rec.get("error") or "").startswith("AttemptDeadline"): + triggers.append(("deadline", x["end"])) + status = rec.get("status") + if x["class"] == "inconclusive" and status in ( + "provider_error", + "harness_error", + ): + key = f"{status}: {cr.error_class(rec) or rec.get('error')}" + end = max(y["end"] for s, n, y, _ in mine if s == sid and n == k) + households = joined.setdefault(key, {}) + households[sid] = min(households.get(sid, math.inf), end) + for households in joined.values(): + ends = sorted(households.values()) + if len(ends) > cr.INCONCLUSIVE_STOP: + triggers.append(("inconclusive", ends[cr.INCONCLUSIVE_STOP])) + stops = [e for e in events if e["event"] == "stop"] + for cause, t in triggers: + if not any( + e.get("cause") == cause and t <= e["time"] <= t + TRIGGER_BOUND + for e in stops + ): + problems.append(f"{model}: a {cause} trigger at {t!r} with no stop") + if cause != "inconclusive" and any(x["start"] > t for _, _, x, _ in mine): + problems.append(f"{model}: an attempt started after a {cause} stop") + for e in stops: + if e.get("cause") in ("harness", "deadline", "inconclusive") and not any( + cause == e["cause"] and t <= e["time"] <= t + TRIGGER_BOUND + for cause, t in triggers + ): + problems.append(f"{model}: a {e['cause']} stop with no trigger") + return problems + + +@pytest.fixture +def world(tmp_path, monkeypatch, fast): + """A fake run to plant failures in, then execute.""" + + def make(**kwargs): + return World(tmp_path, monkeypatch, **kwargs) + + return make + + +def test_an_untouched_run_runs_every_household_and_arm(world): + w = world().execute() + assert run_problems(w) == [] + for model in w.models: + labels = labels_of(model) + attempts = w.attempts(model) + assert len(attempts) == N_DRAW * len(labels) + assert {x["class"] for x in attempts} == {"ok"} + assert all( + x["decision"] == "admit" for x in w.lines(model, "admission-log.jsonl") + ) + manifests = list((w.runs() / model).glob("manifest-*.json")) + assert len(manifests) == 1 + m = json.loads(manifests[0].read_text()) + assert m["arms"] == { + label: { + "arm": cr.arm_string(label), + "features": sorted(cr.arm_profile(model, label).tool_features), + } + for label in labels + } + assert m["treatment_commit"] == cr.TREATMENT_COMMIT + assert m["cap_usd"] == 10.0 and m["reserve_usd"] == pytest.approx(0.21) + assert m["board_engine"] == f"policyengine-us {ENGINE}" + assert not (w.runs() / model / "in-flight").exists() or not any( + (w.runs() / model / "in-flight").iterdir() + ) + host = json.loads((w.runs() / "host" / "host.json").read_text()) + assert host["timezone"] == TZ and host["end"] > host["start"] + + +def test_households_alternate_lanes_in_draw_order(world): + w = world(models=("gpt-6-luna",)).execute() + lanes = {x["household"]: x["lane"] for x in w.attempts("gpt-6-luna")} + assert [lanes[s] for s in w.draw_order] == [1, 2] * (N_DRAW // 2) + + +def test_each_household_runs_its_arms_in_its_drawn_order(world): + w = world(models=("claude-haiku-5.5",)).execute() + master = cr.master_seed(w.design["frozen_sha256"]) + drawn = cr.orders(master, "claude-haiku-5.5", w.draw_order, cr.ARMS) + for sid in w.draw_order: + mine = sorted( + (x for x in w.attempts("claude-haiku-5.5") if x["household"] == sid), + key=lambda x: x["start"], + ) + assert [x["arm"] for x in mine] == drawn[sid] + + +def test_each_call_carries_its_arm_s_features(world): + calls = {"situation": "{}", "variables": ["snap"]} + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + w.provider.specs = {(model, label, sid, 1): {"calls": [calls]} for label in cr.ARMS} + w.execute() + messages = w.run.lanes[model].pool.messages + assert len(messages) == len(cr.ARMS) + raw = [m for m in messages if not isinstance(m, tuple)] + framed = [m for m in messages if isinstance(m, tuple)] + assert raw == [calls] # the tool v1 arm: as a featureless pool sends it + want = { + tuple(sorted(cr.arm_profile(model, label).tool_features)) + for label in cr.ARMS + if label != "pe" + } + assert {m[1] for m in framed} == want + assert all(m[0] == pilot.CALL_FRAME and m[2] == calls for m in framed) + # The worker reads each frame's features as the arm's. + for m in framed: + assert pilot.frame_features(m[1]) == frozenset(m[1]) + assert run_problems(w) == [] + + +def test_households_with_no_scored_output_are_never_decided(world): + model = "gemini-4-argon" + master = cr.master_seed(hashlib.sha256(PREREG.encode()).hexdigest()) + skipped = cr.draw(master, POOL, n=N_DRAW)[1] + w = world(models=(model,), no_output=(skipped,)).execute() + assert skipped not in { + x["household"] for x in w.lines(model, "admission-log.jsonl") + } + assert skipped not in {x["household"] for x in w.attempts(model)} + assert run_problems(w) == [] + + +def test_a_provider_error_is_rerun_after_the_household_s_first_pass(world): + model = "claude-haiku-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + master = cr.master_seed(w.design["frozen_sha256"]) + first = cr.orders(master, model, w.draw_order, cr.ARMS)[sid][0] + error = {"status": "provider_error", "error": "RateLimitError: slow down"} + w.provider.specs = {(model, first, sid, 1): error, (model, first, sid, 2): error} + w.execute() + mine = sorted( + (x for x in w.attempts(model) if x["household"] == sid), + key=lambda x: x["start"], + ) + assert [(x["attempt"], x["arm"]) for x in mine][-2:] == [(2, first), (3, first)] + assert [x["class"] for x in mine if x["arm"] == first] == ["provider"] * 2 + ["ok"] + final = json.loads((w.folder(model, first) / f"{sid}.json").read_text()) + assert [e["n"] for e in final["attempts"]] == [1, 2] + assert final["attempts"][0]["reason"] == "RateLimitError" + assert (w.folder(model, first) / "attempts" / f"{sid}-1.json").exists() + assert run_problems(w) == [] + + +def test_a_third_attempt_stands_whatever_its_class(world): + model = "gpt-6-sol" + w = world(models=(model,)) + sid = w.draw_order[1] + error = {"status": "provider_error", "error": "APIConnectionError: reset"} + w.provider.specs = {(model, "pe3", sid, n): error for n in (1, 2, 3)} + w.execute() + mine = [x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe3"] + assert sorted((x["attempt"], x["action"]) for x in mine) == [ + (1, "rerun"), + (2, "rerun"), + (3, "stands"), + ] + assert run_problems(w) == [] + + +def test_host_trouble_during_an_attempt_reruns_it(world): + model = "gemini-3.5-flash-lite" + w = world(models=(model,)) + pressured, stalled = w.draw_order[0], w.draw_order[2] # both lane 1 + w.provider.specs = { + (model, "loo-flat", pressured, 1): {"pressure": True, "sleep": 0.05}, + (model, "pe", stalled, 1): {"stall": cr.HOST_LOG_GAP + 0.3}, + } + w.execute() + + def first(sid, label): + line = next( + x + for x in w.attempts(model) + if (x["household"], x["arm"], x["attempt"]) == (sid, label, 1) + ) + final = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + return line, final["attempts"][0] + + line, entry = first(pressured, "loo-flat") + assert line["class"] == entry["class"] == "host" and line["action"] == "rerun" + assert "memory pressure 2 at" in entry["reason"] + line, entry = first(stalled, "pe") + assert line["class"] == "host" and "host log gap" in entry["reason"] + assert run_problems(w) == [] + + +def test_a_guard_pause_over_an_attempt_reruns_it(world, monkeypatch): + # Guard lines have whole seconds and the fake brackets by another + # second: keep the lookahead longer than its possible two-second backdate. + monkeypatch.setattr(cr, "PRESSURE_WINDOW", 2.1) + model = "gpt-6-sol" + w = world(models=(model,)) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe3", sid, 1): {"guard": True}} + w.execute() + line = next( + x + for x in w.attempts(model) + if (x["household"], x["arm"], x["attempt"]) == (sid, "pe3", 1) + ) + assert line["class"] == "host" + final = json.loads((w.folder(model, "pe3") / f"{sid}.json").read_text()) + assert final["attempts"][0]["reason"].startswith("guard pause: guard pause ") + archived = (w.runs() / "host" / "clamshell-guard.log").read_text() + assert "paused pids" in archived + assert run_problems(w) == [] + + +def test_an_operator_failure_pauses_the_model_until_resumed(world): + model = "gpt-6-luna" + w = world(models=(model, "claude-haiku-5.5")) + sid = w.draw_order[0] + w.provider.specs = { + (model, "pe3", sid, 1): { + "status": "provider_error", + "error": "AuthenticationError: bad key", + "turns": 0, + "resume_after": 0.4, + } + } + w.execute() + pauses, resumes = w.events(model, "pause"), w.events(model, "resume") + assert len(pauses) == 1 and len(resumes) == 1 + assert resumes[0]["reason"] == "fixed in a test" + started = [x["start"] for x in w.attempts(model)] + assert not [t for t in started if pauses[0]["time"] < t < resumes[0]["time"]] + assert resumes[0]["time"] - pauses[0]["time"] >= 0.35 + rerun = [ + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe3" + ] + assert [x["action"] for x in sorted(rerun, key=lambda x: x["attempt"])] == [ + "pause-rerun", + "stands", + ] + assert not w.events("claude-haiku-5.5", "pause") # another model runs on + assert list((w.runs() / model).glob("resume-*.json")) + assert run_problems(w) == [] + + +def test_a_request_class_on_the_first_request_stops_the_model(world): + model = "claude-opus-5.5" + w = world(models=(model, "gpt-6-sol")) + sid = w.draw_order[2] + w.provider.specs = { + (model, "pe", sid, 1): { + "status": "provider_error", + "error": "BadRequestError: bad tools", + "turns": 0, + } + } + w.execute() + stops = w.events(model, "stop") + assert len(stops) == 1 and "first request" in stops[0]["reason"] + assert stops[0]["cause"] == "harness" + assert not [x for x in w.attempts(model) if x["start"] > stops[0]["time"]] + assert len(w.attempts("gpt-6-sol")) == N_DRAW * 2 # the other model finishes + assert run_problems(w) == [] + + +def test_one_inconclusive_error_in_six_households_stops_the_model(world): + model = "gpt-6-luna" + w = world(models=(model,)) + error = {"status": "provider_error", "error": "ProviderEmpty: no choices"} + w.provider.specs = {(model, "pe", sid, 1): error for sid in w.draw_order} + w.execute() + stops = w.events(model, "stop") + assert len(stops) == 1 + assert ( + stops[0]["reason"] == "provider_error: ProviderEmpty ended runs in 6 households" + ) + assert stops[0]["cause"] == "inconclusive" + assert run_problems(w) == [] + + +def test_five_households_with_one_inconclusive_error_do_not_stop(world): + model = "gpt-6-luna" + w = world(models=(model,)) + error = {"status": "provider_error", "error": "ProviderEmpty: no choices"} + w.provider.specs = {(model, "pe", sid, 1): error for sid in w.draw_order[:5]} + w.execute() + assert not w.events(model, "stop") + assert len(w.attempts(model)) == N_DRAW * len(cr.ARMS) + assert run_problems(w) == [] + + +def test_a_refusal_ends_both_lanes_decisions(world): + model = "claude-haiku-5.5" + # Each household costs 7 arms x $0.01; the reserve is $0.21 per household. + w = world(models=(model,), caps={model: 0.5}).execute() + decisions = w.lines(model, "admission-log.jsonl") + refused = [x for x in decisions if x["decision"] == "refuse"] + after = decisions[decisions.index(refused[0]) :] + assert refused and all(x["decision"] == "refuse" for x in after) + admitted = {x["household"] for x in decisions if x["decision"] == "admit"} + assert {x["household"] for x in w.attempts(model)} == admitted + assert run_problems(w) == [] + + +def test_spend_never_stops_a_conversation_below_the_ceiling(world): + model = "gemini-4-argon" + w = world(models=(model,), caps={model: 0.05}, reserve=0.01) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe", sid, 1): {"turns": 4, "cost": 0.5}} + w.execute() + # $2 on one arm, far past the cap: it ran to the end; later households + # were refused. + ran = [x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe"] + assert ran[0]["status"] == "complete" + assert any(x["decision"] == "refuse" for x in w.lines(model, "admission-log.jsonl")) + assert run_problems(w) == [] + + +def test_the_hard_ceiling_stops_every_model(world): + w = world( + models=("claude-haiku-5.5", "gpt-6-sol", "gemini-4-argon"), + overall_cap=0.0, + spent_before=24.0, + ) + sid = w.draw_order[1] + # One long conversation crosses $25 while the others run. + w.provider.specs = { + ("gpt-6-sol", "pe3", sid, 1): {"turns": 3, "cost": 0.5, "sleep": 0.02} + } + w.execute() + events = {m: w.events(m, "ceiling") for m in w.models} + assert all(len(e) == 1 for e in events.values()) + crossed = events["gpt-6-sol"][0]["time"] + assert {e[0]["time"] for e in events.values()} == {crossed} + assert w.run.ceiling.spent_usd >= 25.0 + for m in w.models: + assert not [x for x in w.attempts(m) if x["start"] > crossed] + assert run_problems(w) == [] + + +def test_a_model_that_decides_nothing_still_has_both_logs(world): + """haiku's first attempt crosses the ceiling before gpt-6-sol's lanes, + held back, decide: gpt-6-sol's run log holds only the ceiling event and + its admission log is empty.""" + w = world( + models=("claude-haiku-5.5", "gpt-6-sol"), overall_cap=0.0, spent_before=24.5 + ) + sid = w.draw_order[0] + first = w.run.lanes["claude-haiku-5.5"].orders[sid][0] + w.provider.specs = {("claude-haiku-5.5", first, sid, 1): {"cost": 1.0}} + held = w.run.lanes["gpt-6-sol"] + admit = held.admit + + def late(sid, lane): + # A sleep here would race haiku's attempt on a loaded host. + deadline = time.monotonic() + 30 + while not w.run.ceiling.exhausted(): + assert time.monotonic() < deadline, "haiku never crossed the ceiling" + time.sleep(0.01) + return admit(sid, lane) + + held.admit = late + w.execute() + admission = w.runs() / "gpt-6-sol" / "admission-log.jsonl" + assert admission.exists() and admission.read_text() == "" + assert [ + x["event"] + for x in w.lines("gpt-6-sol") + if x["event"] not in ("setup-start", "setup-end") + ] == ["ceiling"] + assert run_problems(w) == [] + + +def test_a_conversation_under_way_at_the_ceiling_ends_budget_stop(world): + model = "gpt-6-luna" + w = world(models=(model,), overall_cap=0.0, spent_before=24.9) + sid = w.draw_order[0] + master = cr.master_seed(w.design["frozen_sha256"]) + first = cr.orders(master, model, w.draw_order, cr.ARMS)[sid][0] + w.provider.specs = {(model, first, sid, 1): {"turns": 5, "cost": 0.06}} + w.execute() + line = next( + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == first + ) + assert line["status"] == "budget_stop" and line["class"] == "ceiling" + assert line["action"] == "stopped" + assert run_problems(w) == [] + + +def test_an_attempt_that_raises_is_saved_as_a_harness_error_record(world): + model = "claude-opus-5.5" + w = world(models=(model,)) + sid = w.draw_order[3] + w.provider.specs = {(model, "pe3", sid, 1): {"raise": True}} + w.execute() + replaced = json.loads( + (w.folder(model, "pe3") / "attempts" / f"{sid}-1.json").read_text() + ) + assert replaced["status"] == "harness_error" and replaced["turns"] == [] + assert replaced["error"] == "RuntimeError: a test raised" + assert replaced["tool_features"] == sorted( + cr.arm_profile(model, "pe3").tool_features + ) + line = next( + x + for x in w.attempts(model) + if x["household"] == sid and x["attempt"] == 1 and x["arm"] == "pe3" + ) + assert line["class"] == "none" and line["action"] == "rerun" + assert run_problems(w) == [] + + +def test_a_pool_that_lost_every_worker_is_started_again(world): + model = "gpt-6-sol" + starts = [] + + class Losing(FakePool): + def __init__(self, lanes): + super().__init__(lanes) + starts.append(self) + + def call(self, message): + self.lost = 2 # every worker gone after this call + return super().call(message) + + w = world(models=(model,), pool=Losing) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe", sid, 1): {"calls": [{"situation": "{}"}]}} + w.execute() + assert len(starts) == 2 and starts[0].closed + setup = [e for e in w.events(model) if e.get("phase") == "restart"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[0]["start"] == setup[0]["time"] == setup[1]["start"] + assert setup[1]["time"] >= setup[0]["time"] + assert setup[1]["outcome"] == "ready" + assert run_problems(w) == [] + + +def test_a_pool_that_cannot_start_again_stops_the_model(world, monkeypatch): + model = "gpt-6-sol" + made = [] + + def pool(lanes): + made.append(1) + return FakePool(lanes, fail=len(made) > 1) + + w = world(models=(model,), pool=pool) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe", sid, 1): {"calls": [{"situation": "{}"}]}} + original = FakePool.call + + def losing(self, message): + self.lost = 2 + return original(self, message) + + monkeypatch.setattr(FakePool, "call", losing) + w.execute() + stops = w.events(model, "stop") + assert stops and "engine pool" in stops[0]["reason"] + assert stops[0]["cause"] == "pool" + setup = [e for e in w.events(model) if e.get("phase") == "restart"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[1]["outcome"] == "failed" + assert run_problems(w) == [] + + +def test_a_household_a_stop_cuts_short_is_under_way_until_the_stop(world): + """Lane 1's first household loses the pool in its first arm, and the + restart waits while lane 2 finishes its household and + decides its next: the household lane 1 cut short has ended its last + attempt, but it is still under way, so its spend is not counted.""" + model = "gpt-6-sol" + made = [] + other_second, restarting, decided_after = ( + threading.Event(), + threading.Event(), + threading.Event(), + ) + + class Losing(FakePool): + def call(self, message): + assert other_second.wait(30) + self.lost = 2 + return super().call(message) + + def pool(lanes): + made.append(1) + if len(made) > 1: + restarting.set() + assert decided_after.wait(30) + raise RuntimeError("tool worker failed to start: a test") + return Losing(lanes) + + w = world(models=(model,), pool=pool) + cut, other, after = w.draw_order[0], w.draw_order[1], w.draw_order[3] + lanes = w.run.lanes[model] + orders, admit = lanes.orders, lanes.admit + + def in_second(lanes): + other_second.set() + assert restarting.wait(30) + + def admitted(sid, lane): + ok = admit(sid, lane) + if sid == after and ok: + decided_after.set() + # Keep this admitted household from racing the failed restart + # for the pool lock before lane 1 records its stop. + with lanes.cond: + assert lanes.cond.wait_for(lambda: lanes.stopped, timeout=30) + return ok + + lanes.admit = admitted + w.provider.specs = { + (model, orders[cut][0], cut, 1): {"calls": [{"situation": "{}"}]}, + (model, orders[other][1], other, 1): {"check": in_second}, + } + w.execute() + stop = w.events(model, "stop")[0]["time"] + decided = {x["household"]: x for x in w.lines(model, "admission-log.jsonl")} + assert decided[after]["time"] < stop # decided while the cut household waited + assert decided[after]["finished_spend"] == pytest.approx(2 * COST) # `other` only + assert decided[after]["under_way"] == 2 # `cut` and `after` + assert [x["arm"] for x in w.attempts(model) if x["household"] == cut] == [ + orders[cut][0] + ] + setup = [e for e in w.events(model) if e.get("phase") == "restart"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[0]["start"] == setup[0]["time"] == setup[1]["start"] + assert setup[1]["time"] >= decided[after]["time"] + assert setup[1]["outcome"] == "failed" + assert run_problems(w) == [] + + +def test_an_interrupt_stops_every_model_and_lets_attempts_end(world): + w = world(models=("gpt-6-luna", "gpt-6-sol")) + sid = w.draw_order[0] + first = w.run.lanes["gpt-6-luna"].orders[sid][0] + + def interrupt_inside(lanes): + # The signal arrives during the household's first attempt, which + # finishes only once the main loop has stopped the lanes, so no + # timing on a loaded host lets the household's later arms start first. + w.run.interrupt(2, None) + until = time.time() + 30 + while not lanes.stopped and time.time() < until: + time.sleep(0.01) + + w.provider.specs = {("gpt-6-luna", first, sid, 1): {"check": interrupt_inside}} + w.execute() + for m in w.models: + stops = w.events(m, "stop") + assert len(stops) == 1 and stops[0]["reason"] == "operator stop (signal 2)" + assert stops[0]["cause"] == "interrupt" + assert not [x for x in w.attempts(m) if x["start"] > stops[0]["time"]] + luna = [x for x in w.attempts("gpt-6-luna") if x["household"] == sid] + assert luna and len(luna) < len(cr.ARMS) # cut by the stop + assert run_problems(w) == [] + + +def test_a_failed_explain_trace_is_inconclusive(world): + """The explain feature's trace catches its own errors and returns them + under `explanations`: a MemoryError there is the host's, not a miss.""" + + class Explaining(FakePool): + def call(self, message): + self.messages.append(message) + return { + "results": {"snap": {"values": {"spm": 1.0}}}, + "explanations": {"snap": {"error": "MemoryError: a test"}}, + }, 0.0 + + model = "gpt-6-luna" + w = world(models=(model,), pool=Explaining) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe3", sid, 1): {"calls": [{"situation": "{}"}]}} + w.execute() + line = next( + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe3" + ) + assert line["class"] == "inconclusive" and line["action"] == "stands" + assert run_problems(w) == [] + + +def test_no_household_is_decided_after_the_crossing(world): + """The crossing lands between the lanes' halt check and the decision: the + decision's time is taken under the ceiling's lock, after a second look.""" + w = world(models=("gpt-6-luna",)) + lanes = w.run.lanes["gpt-6-luna"] + halted = lanes.halted + + def crossing(): + result = halted() + if not result: + w.run.ceiling.add(1e6) # another lane's spend, just after the check + return result + + lanes.halted = crossing + decided = lanes.admit(w.draw_order[0], 1) + lines = w.lines("gpt-6-luna", "admission-log.jsonl") + assert decided is False and lines == [] + + +@pytest.mark.parametrize( + "when", ["before the run", "before the run, misdated", "during the pause"] +) +def test_a_resume_written_before_the_pause_is_ignored(world, when, monkeypatch): + """A resume left from before a pause would end that pause unseen: it is + set aside, and the pause holds until a resume written during it. One + found while no pause holds is set aside at once, whatever time it + states; one found during the pause but stamped before it is set aside + too.""" + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + folder = w.runs() / model + folder.mkdir(parents=True) + stale = {"time": time.time(), "note": "stale"} + if when.startswith("before the run"): + if when.endswith("misdated"): + stale["time"] += 3600.0 # a writer whose clock runs ahead + cr.write_json(folder / "RESUME.json", stale) + + lanes, watch = w.run.lanes[model], w.run.watch + planted, fresh = False, False + + def supervised(*args, **kwargs): + nonlocal planted, fresh + with lanes.cond: + if when == "during the pause" and lanes.paused and not planted: + cr.write_json(folder / "RESUME.json", stale) + planted = True + watch(*args, **kwargs) + with lanes.cond: + if ( + lanes.paused + and list(folder.glob("resume-ignored-*.json")) + and not fresh + ): + # The stale note has been read and archived before its + # replacement exists; the next watch sees only the fresh one. + cr.write_json( + folder / "RESUME.json", + {"time": cr.clock(), "note": "fixed in a test"}, + ) + fresh = True + + monkeypatch.setattr(w.run, "watch", supervised) + + w.provider.specs = { + (model, "pe3", sid, 1): { + "status": "provider_error", + "error": "AuthenticationError: bad key", + "turns": 0, + } + } + w.execute() + pauses, resumes = w.events(model, "pause"), w.events(model, "resume") + assert len(pauses) == 1 and len(resumes) == 1 + assert resumes[0]["reason"] == "fixed in a test" + assert resumes[0]["time"] >= pauses[0]["time"] + ignored = list(folder.glob("resume-ignored-*.json")) + assert len(ignored) == 1 and json.loads(ignored[0].read_text()) == stale + assert run_problems(w) == [] + + +def test_the_signal_handler_only_notes_the_signal(world, monkeypatch): + """The handler runs on the main thread between any two of its bytecodes, + perhaps holding a lock it would need: it takes none. A second signal + exits at once.""" + w = world(models=("gpt-6-luna",)) + lanes = w.run.lanes["gpt-6-luna"] + exits = [] + monkeypatch.setattr(os, "_exit", lambda code: exits.append(code)) + with w.run._say_lock, lanes.cond, lanes._log_lock: + w.run.interrupt(2, None) # would deadlock if it took any of these + assert w.run.signalled == 2 and not lanes.stopped + assert w.lines("gpt-6-luna") == [] + w.run.interrupt(15, None) + assert exits == [130] + + +def test_an_attempt_that_raises_after_spending_keeps_its_cost(world): + model = "claude-haiku-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = { + (model, first, sid, 1): {"turns": 2, "cost": 0.03, "raise_after": True} + } + w.execute() + replaced = json.loads( + (w.folder(model, first) / "attempts" / f"{sid}-1.json").read_text() + ) + assert replaced["status"] == "harness_error" and replaced["turns"] == [] + assert replaced["totals"]["cost_usd"] == pytest.approx(0.06) + assert run_problems(w) == [] # the ceiling counted it, and so do the records + + +def test_an_attempt_past_its_deadline_is_abandoned_and_stops_the_model( + world, monkeypatch +): + """The conversation runs on in its own thread; at the deadline its record + is a harness error with the spend so far, the model stops, and the + conversation ends `budget_stop` at its next turn. Its spend after the + deadline is a late-cost event. The second turn waits for the abandon, so + the split does not depend on when the waiting thread wakes, and the join's + own timeout pins the deadline, because that wait hides when the abandon came.""" + monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 1.0) + abandoned = threading.Event() + abandon = cr.AttemptSpend.abandon + join = threading.Thread.join + waits = {} + + def recorded_join(thread, timeout=None): + waits.setdefault(thread.name, timeout) + return join(thread, timeout) + + monkeypatch.setattr(threading.Thread, "join", recorded_join) + + def abandon_then_release(spend): + spent = abandon(spend) + abandoned.set() + return spent + + monkeypatch.setattr(cr.AttemptSpend, "abandon", abandon_then_release) + model = "gpt-6-luna" + w = world(models=(model, "claude-haiku-5.5")) + sid = w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = { + (model, first, sid, 1): {"turns": 4, "cost": 0.01, "hold": abandoned}, + **{ + ("claude-haiku-5.5", label, s, 1): {"sleep": 0.2} + for label in cr.ARMS + for s in w.draw_order[:2] + }, + } + w.execute() + assert waits[f"{model}-{sid}-{first}-1"] == 1.0 + line = next( + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == first + ) + final = json.loads((w.folder(model, first) / f"{sid}.json").read_text()) + assert final["error"].startswith("AttemptDeadline: past the attempt deadline") + assert final["totals"]["cost_usd"] == pytest.approx(0.01) # turn 1 only + assert line["class"] == "none" # its rerun is cut short by the stop + stops = w.events(model, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "deadline" + late = w.events(model, "late-cost") + assert len(late) == 1 and late[0]["cost_usd"] == pytest.approx(0.01) + assert not [x for x in w.attempts(model) if x["start"] > line["end"]] + assert len(w.attempts("claude-haiku-5.5")) == N_DRAW * len(cr.ARMS) + assert run_problems(w) == [] + + +def test_a_sampler_whose_samples_fail_stops_every_model(world): + models = ("gpt-6-luna", "gpt-6-sol") + w = world(models=models) + w.provider.specs = { + (m, label, sid, 1): {"sleep": 0.1} + for m in models + for label in labels_of(m) + for sid in w.draw_order + } + threading.Timer(0.3, lambda: setattr(w.readers, "pressure", None)).start() + w.execute() + for m in models: + stops = w.events(m, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "sampler" + assert "samples failing since" in stops[0]["reason"] + assert run_problems(w) == [] + + +def failing_settle(lanes, sid, times): + """Lanes.settle, failing its first `times` calls for household `sid`.""" + original, calls = lanes.settle, [] + + def settle(at, done): + if at == sid and len(calls) < times: + calls.append(at) + raise RuntimeError("a test's runner defect") + return original(at, done) + + lanes.settle = settle + return calls + + +@pytest.mark.parametrize("times", [1, 2]) +def test_a_runner_error_stops_every_model_and_writes_what_ended(world, times): + """A runner error in a household's pass: every model stops, and that + household's ended attempts are classed, or `unsettled` if classing fails + again, and written with their lines before the lane ends.""" + models = ("gpt-6-luna", "gpt-6-sol") + w = world(models=models) + sid = w.draw_order[2] + failing_settle(w.run.lanes["gpt-6-luna"], sid, times) + w.execute() + for m in models: + stops = w.events(m, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "runner-error" + mine = [x for x in w.attempts("gpt-6-luna") if x["household"] == sid] + assert len(mine) == len(cr.ARMS) # the pass ended before the error + assert {x["class"] for x in mine} == ({"ok"} if times == 1 else {"unsettled"}) + for label in cr.ARMS: + assert (w.folder("gpt-6-luna", label) / f"{sid}.json").exists() + assert run_problems(w) == [] + + +def test_each_attempt_is_on_disk_before_the_next_starts(world, monkeypatch): + """An ended attempt is written, synced, before its household's next + attempt starts; a replaced one is saved, with its line, before its rerun + starts.""" + model = "claude-haiku-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + order = w.run.lanes[model].orders[sid] + import fcntl + + if not hasattr(fcntl, "F_GETPATH"): + pytest.skip("no F_GETPATH: the paths synced cannot be told") + synced: list[str] = [] + real = os.fsync + + def fsync(fd): + path = fcntl.fcntl(fd, fcntl.F_GETPATH, bytes(1024)) + synced.append(path.rstrip(b"\0").decode()) + return real(fd) + + monkeypatch.setattr(os, "fsync", fsync) + seen = {} + + def before_second(lanes): + seen["in-flight"] = ( + lanes.folder / "in-flight" / sid / f"{order[0]}-1.json" + ).exists() + + def before_rerun(lanes): + seen["replaced"] = ( + w.folder(model, order[1]) / "attempts" / f"{sid}-1.json" + ).exists() + seen["line"] = any( + x["household"] == sid and x["arm"] == order[1] and x["attempt"] == 1 + for x in w.attempts(model) + ) + + error = {"status": "provider_error", "error": "RateLimitError: slow"} + w.provider.specs = { + (model, order[1], sid, 1): {**error, "check": before_second}, + (model, order[1], sid, 2): {"check": before_rerun}, + } + w.execute() + assert seen == {"in-flight": True, "replaced": True, "line": True} + # Each whole-file write is synced before its rename. + assert any(f"/in-flight/{sid}/.{order[0]}-1.json.tmp" in p for p in synced) + assert any(f"/attempts/.{sid}-1.json.tmp" in p for p in synced) + assert run_problems(w) == [] + + +def test_the_worker_count_is_the_design_s(tmp_path, capsys): + design = tmp_path / "design.json" + design.write_text(json.dumps({"runs": str(tmp_path / "runs"), "workers": 2})) + assert cr.main(["run", str(design), "--workers", "3"]) == 1 + assert "the design runs 2 engine workers" in capsys.readouterr().out + + +def test_the_checker_catches_a_runner_that_ignores_a_harness_failure( + world, monkeypatch +): + """trigger_problems is not vacuous: a runner whose harness failure does + not stop the model fails it.""" + model = "claude-opus-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + w.provider.specs = { + (model, "pe", sid, 1): { + "status": "provider_error", + "error": "BadRequestError: bad tools", + "turns": 0, + } + } + lanes = w.run.lanes[model] + real = lanes.event + + def no_stop(kind, reason, at=None, cause=None, **extra): + if cause == "harness": + lanes.stopped = False # the fault: the stop is not kept + return + real(kind, reason, at, cause, **extra) + + lanes.event = no_stop + w.execute() + problems = run_problems(w) + assert any("a harness trigger" in p and "with no stop" in p for p in problems) + assert any("an attempt started after a harness stop" in p for p in problems) + + +def test_the_checker_catches_an_edited_log_line(world): + w = world(models=("gpt-6-sol",)).execute() + path = w.runs() / "gpt-6-sol" / "admission-log.jsonl" + lines = path.read_text().splitlines() + edited = json.loads(lines[1]) + edited["finished_spend"] += 1.0 + path.write_text("\n".join([lines[0], json.dumps(edited)] + lines[2:]) + "\n") + problems = run_problems(w) + assert any("its prev is not the line before's hash" in p for p in problems) + # An interior edit need not change a later anchor's head: its successor + # still has its old bytes. A tail edit separately exercises the anchor. + edited = json.loads(lines[-1]) + edited["finished_spend"] += 1.0 + path.write_text("\n".join(lines[:-1] + [json.dumps(edited)]) + "\n") + problems = run_problems(w) + assert any("is not the log's head" in p for p in problems) + + +KINDS = { + "ok": {}, + "provider": {"status": "provider_error", "error": "RateLimitError: slow"}, + "none": {"status": "harness_error", "error": "RuntimeError: lost", "turns": 0}, + "model": {"status": "provider_error", "error": "BadRequestError: long"}, + "inconclusive": {"status": "provider_error", "error": "ProviderEmpty: none"}, + "pressure": {"pressure": True, "sleep": 0.04}, + "incomplete": {"status": "incomplete"}, + "harness": {"status": "provider_error", "error": "BadRequestError: x", "turns": 0}, + "costly": {"turns": 3, "cost": 0.2}, +} + + +@settings( + max_examples=12, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture, HealthCheck.too_slow], +) +@given( + planted=st.lists( + st.tuples( + st.sampled_from(("claude-haiku-5.5", "gpt-6-sol")), + st.sampled_from(cr.FRONTIER_ARMS), + st.integers(0, N_DRAW - 1), + st.integers(1, 3), + st.sampled_from(sorted(KINDS)), + ), + max_size=10, + ), + cap=st.sampled_from([0.3, 0.6, 10.0]), + spent_before=st.sampled_from([0.0, 24.0]), +) +def test_every_run_keeps_the_run_procedure( + tmp_path_factory, fast, planted, cap, spent_before +): + """Whatever fails, and wherever spend lands, the records follow every rule.""" + tmp = tmp_path_factory.mktemp("run") + mp = pytest.MonkeyPatch() + try: + models = ("claude-haiku-5.5", "gpt-6-sol") + w = World( + tmp, + mp, + models=models, + caps={m: cap for m in models}, + overall_cap=0.0, + spent_before=spent_before, + ) + w.provider.specs = { + (model, label, w.draw_order[i], n): KINDS[kind] + for model, label, i, n, kind in planted + } + w.execute() + assert run_problems(w) == [] + finally: + mp.undo() + + +def test_the_checker_catches_a_planted_fault(world): + """run_problems is not vacuous: an edited log line fails it.""" + w = world(models=("gpt-6-sol",)).execute() + path = w.runs() / "gpt-6-sol" / "run-log.jsonl" + lines = path.read_text().splitlines() + at = next(i for i, line in enumerate(lines) if "event" not in json.loads(line)) + first = json.loads(lines[at]) + first["class"] = "provider" + lines[at] = json.dumps(first) + path.write_text("\n".join(lines) + "\n") + assert any("recomputed" in p for p in run_problems(w)) + + +# ---------------------------------------------------------------- before a run + + +def test_the_preflight_names_a_missing_field(): + assert cr.preflight({}, cr.CHEAP) == [f"the design has no {', '.join(cr.REQUIRED)}"] + + +def complete_preflight_design(w, tmp_path): + """Each calculator preflight starts after standing no-tools finals that pass + the no-tools script's treatment checks: built through the harness from the + fake board, whose files that script's loader checks before it calls the + patched harness loader.""" + import csv + + from policybench.scenarios import scenario_to_dict + from tests.test_confirmatory_no_tools_records import full_record + + board = Path(w.design["board"]) + (board / "reference_outputs.csv.meta.json").write_text( + json.dumps( + { + "programs": ["snap"], + "revisions": [ + {"kind": "engine_upgrade", "engine_version": w.design["engine"]} + ], + } + ) + ) + with (board / "scenarios.csv").open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_json"]) + writer.writerows( + [json.dumps(scenario_to_dict(w.board["scenarios"][sid]))] for sid in POOL + ) + pool = tmp_path / "pool.json" + pool.write_text(json.dumps([{"id": s} for s in POOL])) + baseline = tmp_path / "baseline-exclusions.json" + baseline.write_text('{"exclusions": []}\n') + design = dict( + w.design, + eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, + baseline_exclusions={"path": str(baseline), "sha256": cr.sha256_file(baseline)}, + treatment_harness=str(cr.ROOT), + run_commit="0" * 40, + no_tools={ + "runs": str(tmp_path / "no-tools"), + "folder": "none", + "stage": "confirm", + }, + ) + for model in cr.CHEAP: + for sid in design["draw_order"]: + final = full_record(w.board, model, design["engine"], sid) + final["predictions"] = {v: 1.0 for v in w.board["variables"][sid]} + final["totals"] = {"cost_usd": 0.0} + cr.write_json( + Path(design["no_tools"]["runs"]) + / model + / "none/confirm/households" + / f"{sid}.json", + final, + ) + return design + + +def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( + world, tmp_path, monkeypatch +): + w = world(models=("gpt-6-luna",)) + design = complete_preflight_design(w, tmp_path) + design["draw_order"] = list(reversed(w.draw_order)) + monkeypatch.setattr(cr, "host_power", lambda: (True, False)) + problems = cr.preflight(design, ("gpt-6-luna",)) + text = "\n".join(problems) + assert "the draw, recomputed" in text + assert "not the run commit, clean" in text + assert "lid open False" in text + design["draw_order"] = w.draw_order + assert "the draw" not in "\n".join(cr.preflight(design, ("gpt-6-luna",))) + assert not any( + p.startswith("no-tools") for p in cr.preflight(design, ("gpt-6-luna",)) + ) + + +@pytest.mark.parametrize( + "drawing, clamshell, expected", + [ + ("AC Power", "No", (True, True)), + ("Battery Power", "No", (False, True)), + ("AC Power", "Yes", (True, False)), + ("Battery Power", "Yes", (False, False)), + ], +) +def test_host_power_reads_pmset_and_ioreg(monkeypatch, drawing, clamshell, expected): + output = { + "pmset": f"Now drawing from '{drawing}'\n", + "ioreg": f' "AppleClamshellState" = {clamshell}\n', + } + + def run(command, **kwargs): + return subprocess.CompletedProcess(command, 0, stdout=output[command[0]]) + + monkeypatch.setattr(cr.subprocess, "run", run) + assert cr.host_power() == expected + + +def test_the_preflight_refuses_a_runs_folder_with_records(world, tmp_path): + w = world(models=("gpt-6-luna",)).execute() + design = complete_preflight_design(w, tmp_path) + text = "\n".join(cr.preflight(design, ("gpt-6-luna",))) + assert "already holds records" in text and "already holds a host log" in text + + +@pytest.mark.parametrize( + "fault,expected", + [ + ("missing", "missing"), + ("harness", "stop"), + ("ceiling", "stopped"), + ("history", "attempts list"), + ], +) +def test_preflight_refuses_an_unfinished_no_tools_stage( + world, tmp_path, fault, expected +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + model, sid = cr.CHEAP[-1], w.draw_order[-1] + path = ( + Path(design["no_tools"]["runs"]) + / model + / "none/confirm/households" + / f"{sid}.json" + ) + payload = json.loads(path.read_text()) + if fault == "missing": + path.unlink() + else: + if fault == "history": + payload.pop("attempts") + else: + # A failed call leaves no reply: no answer and no prompt evidence. + from tests.test_confirmatory_no_tools_records import OPTIONAL_EVIDENCE + + for key in OPTIONAL_EVIDENCE: + payload.pop(key) + payload.update( + status="budget_stop" if fault == "ceiling" else "provider_error", + error="BadRequestError" if fault == "harness" else None, + predictions={v: None for v in payload["variables"]}, + turns=[], + ) + path.write_text(json.dumps(payload)) + text = "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert "no-tools" in text and model in text and sid in text and expected in text + + +@pytest.mark.parametrize("relationship", ["equal", "inside", "contains"]) +def test_preflight_refuses_overlapping_no_tools_and_calculator_roots( + world, tmp_path, relationship +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + calculator = Path(design["runs"]) + design["no_tools"]["runs"] = str( + { + "equal": calculator, + "inside": calculator / "no-tools", + "contains": calculator.parent, + }[relationship] + ) + assert any("roots overlap" in p for p in cr.preflight(design, (cr.CHEAP[0],))) + + +def test_resume_writes_the_operator_s_note(tmp_path): + design = tmp_path / "design.json" + design.write_text(json.dumps({"runs": str(tmp_path / "runs")})) + assert cr.main(["resume", str(design), "gpt-6-luna", "--note", "key rotated"]) == 0 + note = json.loads((tmp_path / "runs" / "gpt-6-luna" / "RESUME.json").read_text()) + assert note["note"] == "key rotated" + + +# ---------------------------------------------------------------- the harness + + +def chat_response(calls): + from types import SimpleNamespace + + message = SimpleNamespace( + role="assistant", content=None, tool_calls=calls, function_call=None + ) + usage = SimpleNamespace( + prompt_tokens=1000, + completion_tokens=200, + total_tokens=1200, + reasoning_tokens=0, + prompt_tokens_details=SimpleNamespace(cached_tokens=0), + completion_tokens_details=None, + ) + return SimpleNamespace( + choices=[SimpleNamespace(message=message, finish_reason="tool_calls")], + usage=usage, + model="claude-haiku-5-5", + ) + + +def chat_call(name, arguments, call_id): + from types import SimpleNamespace + + return SimpleNamespace( + id=call_id, + type="function", + function=SimpleNamespace(name=name, arguments=json.dumps(arguments)), + ) + + +def stateless_model(**kwargs): + """A tool-contract model: one calculation, then its answers. It reads the + conversation so far, so the two lanes can share it.""" + if not any(m.get("role") == "tool" for m in kwargs["messages"]): + arguments = {"situation": json.dumps({"people": {}}), "variables": ["snap"]} + return chat_response([chat_call(pilot.TOOL_NAME, arguments, "c1")]) + outputs = {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + return chat_response([chat_call("submit_outputs", {"outputs": outputs}, "s1")]) + + +def stateless_responses_model(**kwargs): + """A Responses tool-contract model: calculate, then submit the answers.""" + from types import SimpleNamespace + + if "previous_response_id" not in kwargs: + name, call_id = pilot.TOOL_NAME, "c1" + arguments = {"situation": json.dumps({"people": {}}), "variables": ["snap"]} + else: + name, call_id = "submit_outputs", "s1" + arguments = { + "outputs": {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + } + return SimpleNamespace( + id=f"resp_{call_id}", + output=[ + SimpleNamespace( + type="function_call", + id=f"fc_{call_id}", + call_id=call_id, + name=name, + arguments=json.dumps(arguments), + status="completed", + ) + ], + output_text=None, + status="completed", + incomplete_details=None, + usage=SimpleNamespace( + input_tokens=1000, + output_tokens=200, + total_tokens=1200, + input_tokens_details=SimpleNamespace(cached_tokens=0), + output_tokens_details=SimpleNamespace(reasoning_tokens=0), + ), + model=kwargs["model"], + ) + + +def stateless_json_model(**kwargs): + """A Gemini JSON-contract model: calculate, then answer in JSON text.""" + if not any(m.get("role") == "tool" for m in kwargs["messages"]): + response = stateless_model(**kwargs) + response.model = kwargs["model"] + return response + response = chat_response([]) + response.model = kwargs["model"] + response.choices[0].finish_reason = "stop" + response.choices[0].message.content = json.dumps( + {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + ) + return response + + +@pytest.mark.parametrize( + "model, completion", + [ + ("claude-haiku-5.5", stateless_model), + ("gpt-6-luna", stateless_responses_model), + ("gemini-3.1-pro-preview", stateless_json_model), + ], +) +def test_the_harness_conversation_runs_under_the_runner(world, model, completion): + """Each API and answer contract completes real harness conversations in + both runner lanes, with the registered feature frame for every arm. + + The confirmatory roster's Geminis use tools. The older board profile + covers the harness's Gemini JSON path in a fake design, without changing + the production roster or any model's answer contract.""" + w = world(models=(model,)) + w.run.converse = cr.converse_with_harness + w.run.completion_for = lambda profile: completion + w.execute() + labels = labels_of(model) + attempts = w.attempts(model) + assert len(attempts) == N_DRAW * len(labels) + assert {x["status"] for x in attempts} == {"complete"} + messages = w.run.lanes[model].pool.messages + assert len(messages) == N_DRAW * len(labels) # one calculation per attempt + frames = [m for m in messages if isinstance(m, tuple)] + assert len(frames) == N_DRAW * (len(labels) - 1) # every arm but pe + sid = w.draw_order[0] + label = "loo-names" if "loo-names" in labels else "pe3" + rec = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + profile = cr.arm_profile(model, label) + assert rec["arm"] == cr.arm_string(label) and rec["attempts"] == [] + assert rec["tool_features"] == sorted(profile.tool_features) + assert rec["predictions"] == {v: 0.0 for v in VARIABLES} + assert run_problems(w) == [] + + +@pytest.mark.parametrize( + "model, completion, api, contract, label", + [ + (model, completion, api, contract, label) + for model, completion, api, contract in [ + ("gpt-6-luna", stateless_responses_model, "responses", "tool"), + ("gemini-3.1-pro-preview", stateless_json_model, "chat", "json"), + ] + for label in labels_of(model) + ], +) +def test_the_runner_sends_the_treatment_conversation( + world, label, model, completion, api, contract +): + """Compare every provider request and calculation against the unchanged + treatment harness at 6b00a6f0, including the tool-result continuation. + + Gemini 3.1's board profile exercises JSON; both confirmatory Geminis use + the tool contract. No contract or model card is replaced for this test.""" + w = world(models=(model,)) + lanes, sid = w.run.lanes[model], w.draw_order[0] + requests, calculations = [], [] + + def provider(**kwargs): + """Keep each request before the conversation can extend its messages.""" + requests.append(copy.deepcopy(kwargs)) + return completion(**kwargs) + + def tool(arguments): + """Keep exactly what the conversation sends to the calculator.""" + calculations.append(arguments) + return {"results": {"snap": {"values": {"spm": 1.0}}}}, 0.0 + + lanes.completion = provider + record = cr.converse_with_harness( + lanes, sid, label, tool, cr.AttemptSpend(w.run.ceiling) + ) + sent, called = requests[:], calculations[:] + requests.clear() + calculations.clear() + profile = pilot.model_profile( + model, arm=cr.arm_string(label), prompt_version="v2-pilot" + ) + treatment = pilot.run_household( + w.board["scenarios"][sid], + w.board["variables"][sid], + tool, + provider, + pilot.Spend(100.0), + profile=profile, + ) + assert record["api"] == treatment["api"] == api + assert record["contract"] == treatment["contract"] == contract + assert record["status"] == treatment["status"] == "complete" + assert ( + record["predictions"] == treatment["predictions"] == {v: 0.0 for v in VARIABLES} + ) + assert len(sent) == len(requests) == 2 + assert sent == requests and called == calculations and len(called) == 1 + assert record["messages"] == treatment["messages"] + first, second = sent + if api == "responses": + assert "messages" not in first and "previous_response_id" not in first + assert first["tool_choice"] == "required" + assert [t["name"] for t in first["tools"]] == [ + pilot.TOOL_NAME, + "submit_outputs", + ] + assert "max_output_tokens" in first + assert second["previous_response_id"] == "resp_c1" + assert second["input"] == [ + { + "type": "function_call_output", + "call_id": "c1", + "output": json.dumps({"results": {"snap": {"values": {"spm": 1.0}}}}), + } + ] + else: + assert "input" not in first and first["tool_choice"] == "auto" + assert [t["function"]["name"] for t in first["tools"]] == [pilot.TOOL_NAME] + assert second["messages"][-1]["role"] == "tool" + assert second["messages"][-1]["tool_call_id"] == "c1" + + +def test_the_ceiling_ends_a_harness_conversation_at_its_next_turn(world): + model = "claude-haiku-5.5" + w = world(models=(model,), overall_cap=0.0, spent_before=25.0 - 1e-9) + w.run.converse = cr.converse_with_harness + w.run.completion_for = lambda profile: stateless_model + w.execute() + # The first turns cross the ceiling; each conversation under way ends + # budget_stop before its next request, and no attempt starts after. + attempts = w.attempts(model) + assert attempts and {x["status"] for x in attempts} == {"budget_stop"} + assert len(attempts) <= 2 # one per lane at most + assert len(w.events(model, "ceiling")) == 1 + assert run_problems(w) == [] + + +def test_a_reader_that_keeps_raising_starts_the_sampler_silence( + tmp_path, fast, monkeypatch +): + readers = Readers() + host = readers.host(tmp_path) + host._stop.set() + host._thread.join() + now, reads = [time.time()], [] + + def pressure(): + reads.append(now[0]) + raise RuntimeError("pressure reader failed") + + class Ticks: + def wait(self, timeout): + now[0] += cr.SAMPLER_SILENCE + 1.0 + return len(reads) == 3 + + # Run the real sampling loop across three failures with a controlled + # clock; no scheduling or sleep decides whether its silence has elapsed. + with monkeypatch.context() as patch: + patch.setattr(cr.time, "time", lambda: now[0]) + patch.setattr(host, "_stop", Ticks()) + patch.setattr(host, "_read_pressure", pressure) + patch.setattr(host._thread, "is_alive", lambda: True) + host._loop() + assert len(reads) == 3 + assert host.failing_since == reads[0] + assert "samples failing since" in host.dead() + assert "RuntimeError: pressure reader failed" in host.dead() + patch.setattr(host, "_read_pressure", lambda: 1) + host.sample() + assert host.failing_since is None and host.failure is None + assert host.dead() is None + host.close() + + +def test_a_second_lane_s_authentication_failure_invalidates_a_resume( + world, monkeypatch +): + model = "gpt-6-luna" + w = world(models=(model,)) + lanes = w.run.lanes[model] + first, second = w.draw_order[:2] + lanes.spent.update({first: 0.0, second: 0.0}) + now = [time.time()] + + def tick(): + now[0] += 1.0 + return now[0] + + monkeypatch.setattr(cr, "clock", tick) + auth = record("provider_error", "AuthenticationError: bad key", turns=0) + lanes.end_attempt(first, "pe3", 1, now[0], auth, False) + note = {"time": lanes.paused_at, "note": "first key fixed"} + cr.write_json(lanes.folder / "RESUME.json", note) + resume = lanes.resume + + def second_failure(*args, **kwargs): + # The other lane finishes after watch reads the note and immediately + # before resume validates it. The second pause requires its own note. + lanes.end_attempt(second, "pe3", 1, now[0], auth, False) + return resume(*args, **kwargs) + + monkeypatch.setattr(lanes, "resume", second_failure) + w.run.watch(anchor=False) + assert lanes.paused and lanes.paused_at > note["time"] + assert len(w.events(model, "pause")) == 2 + assert w.events(model, "resume") == [] + assert list(lanes.folder.glob("resume-ignored-*.json")) + monkeypatch.setattr(lanes, "resume", resume) + cr.write_json( + lanes.folder / "RESUME.json", + {"time": lanes.paused_at, "note": "both keys fixed"}, + ) + w.run.watch(anchor=False) + assert not lanes.paused + assert [event["reason"] for event in w.events(model, "resume")] == [ + "both keys fixed" + ] + + +def test_an_interrupt_during_setup_starts_no_pool_or_lane(world, monkeypatch): + started = [] + w = world( + models=("gpt-6-luna", "gpt-6-sol"), + pool=lambda lanes: started.append((lanes.model, "pool")), + ) + + def completion(profile): + if w.run.signalled is None: + w.run.interrupt(2, None) + + w.run.completion_for = completion + for model, lanes in w.run.lanes.items(): + monkeypatch.setattr( + lanes, "lane", lambda lane, m=model: started.append((m, lane)) + ) + w.execute() + assert started == [] and w.provider.count == {} + for model in w.models: + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + setup = [e for e in w.events(model) if e.get("phase") == "startup"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[0]["start"] == setup[0]["time"] == setup[1]["start"] + stop = w.events(model, "stop")[0]["time"] + assert setup[0]["time"] <= stop <= setup[1]["time"] + assert setup[1]["outcome"] == "stopped" + assert run_problems(w) == [] + + +def test_a_slow_pool_start_is_polled_and_anchored_before_it_finishes( + world, monkeypatch +): + model = "gpt-6-sol" + entered, release, finished = threading.Event(), threading.Event(), threading.Event() + polls, anchors = [], [] + + def pool(lanes): + entered.set() + # Readiness depends on supervision, never on which sleep wins. + assert release.wait(10), "startup was not supervised" + finished.set() + return FakePool(lanes) + + w = world(models=(model,), pool=pool) + watch, anchor = w.run.watch, w.run.anchor + + def supervised(*args, **kwargs): + watch(*args, **kwargs) + if entered.is_set() and not finished.is_set(): + polls.append(cr.clock()) + if len(polls) == 3: + release.set() + + def anchored(*args, **kwargs): + if entered.is_set() and not finished.is_set(): + anchors.append(cr.clock()) + return anchor(*args, **kwargs) + + monkeypatch.setattr(w.run, "watch", supervised) + monkeypatch.setattr(w.run, "anchor", anchored) + w.execute() + assert len(polls) >= 3 and anchors + setup = [e for e in w.events(model) if e.get("phase") == "startup"] + assert setup[0]["time"] <= min(polls + anchors) + assert max(polls + anchors) <= setup[1]["time"] + assert setup[1]["outcome"] == "ready" + assert all(a["start"] >= setup[1]["time"] for a in w.attempts(model)) + assert run_problems(w) == [] + + +def test_an_interrupt_during_pool_start_is_recorded_before_readiness( + world, monkeypatch +): + model = "gpt-6-sol" + stopped = threading.Event() + started = [] + + def pool(lanes): + w.run.interrupt(2, None) + assert stopped.wait(10), "the setup interrupt was not processed" + return FakePool(lanes) + + w = world(models=(model,), pool=pool) + watch = w.run.watch + + def supervised(*args, **kwargs): + watch(*args, **kwargs) + if w.run.lanes[model].stopped: + stopped.set() + + monkeypatch.setattr(w.run, "watch", supervised) + monkeypatch.setattr(w.run.lanes[model], "lane", started.append) + w.execute() + assert stopped.is_set() and started == [] + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + assert w.events(model, "setup-end")[0]["outcome"] == "stopped" + assert w.run.lanes[model].pool.closed + assert run_problems(w) == [] + + +def test_an_interrupt_between_lane_launches_prevents_the_next_lane(world, monkeypatch): + model = "gpt-6-sol" + w = world(models=(model,)) + signalled = threading.Event() + started = [] + start = threading.Thread.start + + def lane(number): + started.append(number) + w.run.interrupt(2, None) + signalled.set() + + def start_and_signal(thread): + start(thread) + if thread.name == f"{model}-1": + assert signalled.wait(10) + + monkeypatch.setattr(w.run.lanes[model], "lane", lane) + monkeypatch.setattr(threading.Thread, "start", start_and_signal) + w.execute() + assert started == [1] + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + assert run_problems(w) == [] + + +def test_stop_all_notifies_every_model_before_a_failed_event_write(world, monkeypatch): + w = world(models=("gpt-6-luna", "gpt-6-sol")) + notified = [] + for model, lanes in w.run.lanes.items(): + notify = lanes.cond.notify_all + + def observed(model=model, notify=notify): + notified.append(model) + notify() + + monkeypatch.setattr(lanes.cond, "notify_all", observed) + first = w.run.lanes[w.models[0]] + + def broken(line, name="run-log.jsonl"): + assert all(lanes.stopped for lanes in w.run.lanes.values()) + raise OSError("the first model's event log is unavailable") + + monkeypatch.setattr(first, "write_line", broken) + w.run.stop_all("disk failure", "runner-error") + assert notified == list(w.models) + assert all(lanes.stopped for lanes in w.run.lanes.values()) + assert w.events(w.models[1], "stop")[0]["cause"] == "runner-error" + + +@pytest.mark.parametrize( + "kind,error", [("pause", "AuthenticationError"), ("stop", "BadRequestError")] +) +def test_end_attempt_retains_the_record_before_a_failed_event_write( + world, monkeypatch, kind, error +): + model = "gpt-6-luna" + w = world(models=(model, "gpt-6-sol")) + sid = w.draw_order[0] + lanes = w.run.lanes[model] + label = lanes.orders[sid][0] + w.provider.specs = { + (model, label, sid, 1): { + "status": "provider_error", + "error": f"{error}: a test", + "turns": 0 if kind == "stop" else 1, + } + } + write = lanes.write_line + failed = [] + + def broken(line, name="run-log.jsonl"): + if line.get("event") == kind and not failed: + failed.append(line) + raise OSError("end_attempt's event write failed") + write(line, name) + + monkeypatch.setattr(lanes, "write_line", broken) + w.execute() + assert len(failed) == 1 + record = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + assert record["error"] == f"{error}: a test" + assert record["totals"]["cost_usd"] == pytest.approx(0 if kind == "stop" else COST) + lines = w.lines(model) + saved = next( + i + for i, x in enumerate(lines) + if x.get("household") == sid and x.get("arm") == label + ) + stopped = next(i for i, x in enumerate(lines) if x.get("cause") == "runner-error") + assert saved < stopped + assert all(lanes.stopped for lanes in w.run.lanes.values()) + assert ( + lanes.folder / "in-flight" / sid + ).exists() is False # event failed before disk write + + +def test_a_final_record_write_failure_still_recovers_the_other_arms(world, monkeypatch): + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + w.run.no_output.update(w.draw_order[1:]) + first = w.run.lanes[model].orders[sid][0] + target = w.folder(model, first) / f"{sid}.json" + write = cr.write_json + + def broken(path, payload, indent=1): + if Path(path) == target: + raise OSError("one final record cannot be written") + return write(path, payload, indent) + + monkeypatch.setattr(cr, "write_json", broken) + w.execute() + assert len(w.attempts(model)) == len(cr.ARMS) - 1 + assert all( + (w.folder(model, a) / f"{sid}.json").exists() for a in cr.ARMS if a != first + ) + assert (w.run.lanes[model].folder / "in-flight" / sid / f"{first}-1.json").exists() + assert w.lines(model)[-1]["cause"] == "runner-error" + + +def blocked_cleanup_request(w, monkeypatch): + """One real request, released by the test after the abandoned join.""" + monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 0.02) + monkeypatch.setattr(cr, "LATE_JOIN", 0.0) + monkeypatch.setattr(cr, "CEILING_MARGIN", 0.0) + w.design.update(overall_cap=0.10, spent_before=0.0) + w.run.no_output.update(w.draw_order[1:]) + w.run.ceiling = cr.Ceiling(0.10, 0.0) + release, returned = threading.Event(), threading.Event() + + def converse(lanes, sid, label, tool_call, spend): + assert release.wait(30) + spend.add(0.20) + returned.set() + return {"status": "budget_stop", "turns": [], "totals": {"cost_usd": 0.20}} + + w.run.converse = converse + return release, returned + + +def test_cleanup_counts_a_request_returning_during_host_close(world, monkeypatch): + model = "gpt-6-luna" + w = world(models=(model,)) + release, returned = blocked_cleanup_request(w, monkeypatch) + host_factory = w.run.host_factory + + def host(folder): + evidence = host_factory(folder) + close = evidence.close + + def finish(): + release.set() + assert returned.wait(10) + for lanes in w.run.lanes.values(): + for item in list(lanes.abandoned): + item["thread"].join(timeout=10) + assert not item["thread"].is_alive() + close() + + evidence.close = finish + return evidence + + w.run.host_factory = host + try: + w.execute() + finally: + release.set() + assert returned.is_set() + assert w.run.ceiling.spent_usd == pytest.approx(0.20) + assert w.run.ceiling.closed + assert [e["cost_usd"] for e in w.events(model, "late-cost")] == [ + pytest.approx(0.20) + ] + assert len(w.events(model, "ceiling")) == 1 + assert w.events(model, "late-unresolved") == [] + assert anchor_problems(w) == [] + assert run_problems(w) == [] + + +@pytest.mark.parametrize("returns_at_close", [False, True]) +def test_cleanup_records_an_unresolved_request_before_closing_the_ledger( + world, monkeypatch, returns_at_close +): + model = "gpt-6-luna" + w = world(models=(model,)) + release, returned = blocked_cleanup_request(w, monkeypatch) + if returns_at_close: + close = w.run.ceiling.close + + def finish(pending=None): + unresolved = close(pending) + release.set() + assert returned.wait(10) + return unresolved + + monkeypatch.setattr(w.run.ceiling, "close", finish) + try: + w.execute() + unresolved = w.events(model, "late-unresolved") + assert len(unresolved) == 1 + assert unresolved[0]["outcome"] == "unresolved-at-ledger-close" + assert unresolved[0]["counted_usd"] == 0.0 + assert w.events(model, "stop")[-1]["cause"] == "runner-error" + assert w.run.ceiling.closed + assert anchor_problems(w) == [] + before = (w.runs() / model / "run-log.jsonl").read_bytes() + finally: + release.set() + assert returned.wait(10) + assert w.run.ceiling.spent_usd == 0.0 + assert w.run.ceiling.unrecorded_usd == pytest.approx(0.20) + assert (w.runs() / model / "run-log.jsonl").read_bytes() == before + + +@pytest.mark.parametrize("during", ["host-close", "final-push"]) +def test_a_first_interrupt_during_cleanup_is_recorded_and_anchored( + world, monkeypatch, during +): + model = "gpt-6-sol" + w = world(models=(model,)) + w.run.no_output.update(w.draw_order) + if during == "host-close": + host_factory = w.run.host_factory + + def host(folder): + evidence = host_factory(folder) + close = evidence.close + + def finish(): + w.run.interrupt(2, None) + close() + + evidence.close = finish + return evidence + + w.run.host_factory = host + else: + anchor = w.run.anchor + signalled = [] + + def push(wait=False): + anchor(wait=wait) + if wait and not signalled: + signalled.append(True) + w.run.interrupt(2, None) + + monkeypatch.setattr(w.run, "anchor", push) + w.execute() + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + assert w.run._interrupted + assert anchor_problems(w) == [] + + +def test_a_replacement_log_failure_keeps_a_valid_final_history(world, monkeypatch): + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + w.run.no_output.update(w.draw_order[1:]) + lanes = w.run.lanes[model] + label = lanes.orders[sid][0] + w.provider.specs = { + (model, label, sid, 1): { + "status": "provider_error", + "error": "RateLimitError: a test", + } + } + write, failed = lanes.write_line, [] + + def broken(line, name="run-log.jsonl"): + if line.get("arm") == label and line.get("attempt") == 1 and not failed: + failed.append(line) + raise OSError("the replaced attempt's log write failed") + write(line, name) + + monkeypatch.setattr(lanes, "write_line", broken) + w.execute() + rec = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + assert rec["attempts"] == [] + assert w.provider.count[(model, label, sid)] == 1 + logged = [x for x in w.attempts(model) if x["arm"] == label] + assert len(logged) == 1 + assert all( + logged[0][key] == value for key, value in failed[0].items() if key != "sha256" + ) + assert logged[0]["sha256"] == cr.sha256_file(w.folder(model, label) / f"{sid}.json") + assert run_problems(w) == [] + + +def test_a_supervision_error_drains_active_lanes_before_final_anchor( + world, monkeypatch +): + model = "gpt-6-sol" + w = world(models=(model,)) + w.run.no_output.update(w.draw_order[1:]) + started, returned = threading.Event(), threading.Event() + converse, watch = w.run.converse, w.run.watch + failed = [] + + def request(*args): + started.set() + assert w.run._failed.wait(10) + rec = converse(*args) + returned.set() + return rec + + def broken_watch(anchor=True): + if started.is_set() and not failed: + failed.append(True) + raise OSError("supervision failed while a request was active") + watch(anchor=anchor) + + w.run.converse = request + monkeypatch.setattr(w.run, "watch", broken_watch) + with pytest.raises(OSError, match="supervision failed"): + w.execute() + assert returned.is_set() + assert len(w.attempts(model)) == 1 + assert w.lines(model)[-1]["cause"] == "runner-error" + assert w.run.ceiling.spent_usd == pytest.approx(1.0 + COST) + assert anchor_problems(w) == [] + assert run_problems(w) == [] + + +def final_push_world(world, monkeypatch, fail_all=False, models=("gpt-6-luna",)): + """A run with no periodic pushes, whose first final push fails.""" + w = world(models=models) + w.run.no_output.update(w.draw_order) + monkeypatch.setattr(w.run, "anchor_due", lambda: False) + monkeypatch.setattr(cr, "ANCHOR_RETRY", 0.013) + script = w.tmp / "retrying-anchor.py" + script.write_text( + "import shutil, sys\n" + "from pathlib import Path\n" + "folder = Path(sys.argv[1])\n" + "counter = folder / 'push-count'\n" + "n = int(counter.read_text()) + 1 if counter.exists() else 1\n" + "counter.write_text(str(n))\n" + "shutil.copyfile(sys.argv[3], folder / f'pushed-{n}.jsonl')\n" + "raise SystemExit(1 if n == 1 or sys.argv[2] == '1' else 0)\n" + ) + w.design["anchor_command"] = [ + sys.executable, + str(script), + str(w.tmp), + str(int(fail_all)), + ] + return w + + +@pytest.mark.parametrize("fail_all", [False, True], ids=["then-success", "all-fail"]) +def test_the_final_push_is_retried_until_success_or_three_tries( + world, monkeypatch, fail_all +): + """Final retries stop at the first success, or after three failures.""" + w = final_push_world(world, monkeypatch, fail_all=fail_all) + w.execute() + anchors = [ + json.loads(x) for x in (w.runs() / "anchors.jsonl").read_text().splitlines() + ] + pushes = [ + json.loads(x) + for x in (w.runs() / "anchor-pushes.jsonl").read_text().splitlines() + ] + expected = 3 if fail_all else 2 + assert cr.FINAL_PUSH_TRIES == 3 + assert len(pushes) == len(anchors) == expected + assert [row["outcome"] for row in pushes] == ( + ["failed"] * 3 if fail_all else ["failed", "success"] + ) + final = (w.runs() / "anchors.jsonl").read_bytes() + assert (w.tmp / f"pushed-{expected}.jsonl").read_bytes() == final + assert pushes[-1]["anchor"] == cr.line_sha(final.decode().splitlines()[-1]) + assert pushes[-1]["anchor_time"] == anchors[-1]["time"] + assert all(a["models"] == anchors[0]["models"] for a in anchors) + assert all(a["host-log.jsonl"] == anchors[0]["host-log.jsonl"] for a in anchors) + assert all(a["time"] < b["time"] for a, b in zip(anchors, anchors[1:])) + + +def test_each_final_push_retry_waits_from_the_previous_try_s_end(world, monkeypatch): + """Each final retry starts at least ANCHOR_RETRY after the prior try ends, + even when the failed command itself takes time.""" + from types import SimpleNamespace + + w = world(models=("gpt-6-luna",)) + now = [1000.0] + durations = iter([7.0, 11.0, 5.0]) + monkeypatch.setattr(cr, "clock", lambda: now[0]) + + def sleep(seconds): + """Advance the controlled clock by the requested wait.""" + now[0] += seconds + + def failed_command(*args, **kwargs): + """A failed anchor command with a controlled, nonzero duration.""" + + def communicate(timeout): + """Finish the command after its planted duration.""" + now[0] += next(durations) + return "", "controlled push failure" + + return SimpleNamespace(returncode=1, communicate=communicate) + + monkeypatch.setattr(time, "sleep", sleep) + monkeypatch.setattr(subprocess, "Popen", failed_command) + w.run.finish() + pushes = [ + json.loads(x) + for x in (w.runs() / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert len(pushes) == cr.FINAL_PUSH_TRIES == 3 + assert [p["outcome"] for p in pushes] == ["failed"] * 3 + assert [p["ended"] - p["started"] for p in pushes] == [7.0, 11.0, 5.0] + assert all( + b["started"] >= a["ended"] + cr.ANCHOR_RETRY for a, b in zip(pushes, pushes[1:]) + ) + + +def test_a_first_signal_during_final_push_retries_is_logged_and_anchored( + world, monkeypatch +): + """A first signal during the retry wait is covered by the last push.""" + w = final_push_world(world, monkeypatch, models=("gpt-6-luna", "gpt-6-sol")) + sleep, delivered = time.sleep, [] + + def during_retry(seconds): + if seconds == cr.ANCHOR_RETRY and not delivered: + delivered.append(True) + w.run.interrupt(signal.SIGTERM, None) + sleep(seconds) + + monkeypatch.setattr(time, "sleep", during_retry) + w.execute() + assert delivered and w.run._interrupted + anchors = [ + json.loads(x) for x in (w.runs() / "anchors.jsonl").read_text().splitlines() + ] + pushes = [ + json.loads(x) + for x in (w.runs() / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert [row["outcome"] for row in pushes] == ["failed", "success", "success"] + for model in w.models: + assert [row["cause"] for row in w.events(model, "stop")] == ["interrupt"] + assert ( + anchors[-1]["models"][model]["run-log.jsonl"] + == (w.run.lanes[model].heads()["run-log.jsonl"]) + ) + assert anchors[-2]["models"][model]["run-log.jsonl"]["lines"] == ( + len(w.lines(model)) - 1 + ) + assert (w.tmp / "pushed-3.jsonl").read_bytes() == ( + w.runs() / "anchors.jsonl" + ).read_bytes() + + +def test_a_second_signal_during_final_push_retries_exits_at_once(world, monkeypatch): + """The retry wait leaves the second signal's immediate exit intact.""" + w = final_push_world(world, monkeypatch) + sleep, delivered = time.sleep, [] + + def exit_now(code): + raise SystemExit(code) + + def during_retry(seconds): + if seconds == cr.ANCHOR_RETRY and not delivered: + delivered.append(True) + w.run.interrupt(signal.SIGINT, None) + w.run.interrupt(signal.SIGTERM, None) + sleep(seconds) + + monkeypatch.setattr(os, "_exit", exit_now) + monkeypatch.setattr(time, "sleep", during_retry) + with pytest.raises(SystemExit) as stopped: + w.execute() + assert stopped.value.code == 130 and delivered + assert len((w.runs() / "anchor-pushes.jsonl").read_text().splitlines()) == 1 + assert all(not w.events(model, "stop") for model in w.models) + + +def test_a_signal_at_the_final_ignore_hook_is_logged_and_anchored(world, monkeypatch): + """A signal between the last ordinary check and SIG_IGN is not lost.""" + w = world(models=("gpt-6-luna", "gpt-6-sol")) + w.run.no_output.update(w.draw_order) + monkeypatch.setattr(w.run, "anchor_due", lambda: False) + ignore = w.run._ignore_signals + ignored, delivered = [], [] + monkeypatch.setattr( + signal, "signal", lambda signum, handler: ignored.append((signum, handler)) + ) + + def at_end(): + if not delivered: + delivered.append(True) + w.run.interrupt(signal.SIGTERM, None) + ignore() + + monkeypatch.setattr(w.run, "_ignore_signals", at_end) + w.execute() + assert delivered and w.run._interrupted + assert (signal.SIGINT, signal.SIG_IGN) in ignored + assert (signal.SIGTERM, signal.SIG_IGN) in ignored + for model in w.models: + stops = w.events(model, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "interrupt" + assert stops[0]["reason"] == f"operator stop (signal {signal.SIGTERM})" + assert anchor_problems(w) == [] + + +def test_a_final_push_log_failure_is_stopped_and_locally_anchored(world, monkeypatch): + model = "gpt-6-sol" + w = world(models=(model,)) + w.run.no_output.update(w.draw_order) + push = w.run._push_anchor + + def broken(head, anchor_time): + if w.run.host.end is not None: + with w.run._pushing: + w.run._push_error = OSError("the final push outcome cannot be written") + else: + push(head, anchor_time) + + monkeypatch.setattr(w.run, "_push_anchor", broken) + with pytest.raises(OSError, match="push outcome could not be recorded"): + w.execute() + assert w.events(model, "stop")[-1]["cause"] == "runner-error" + last = json.loads((w.runs() / "anchors.jsonl").read_text().splitlines()[-1]) + assert last["models"][model]["run-log.jsonl"]["lines"] == len(w.lines(model)) + assert last["host-log.jsonl"] == w.run.host.heads() diff --git a/tests/test_confirmatory_unreadable_evidence.py b/tests/test_confirmatory_unreadable_evidence.py new file mode 100644 index 00000000..cac7af15 --- /dev/null +++ b/tests/test_confirmatory_unreadable_evidence.py @@ -0,0 +1,558 @@ +"""File and folder failures must remain named refusals before providers.""" + +import contextlib +import errno +import io +import json +import os +import re +import threading +import time +from pathlib import Path + +import pytest + +from tests.test_confirmatory_no_tools import ( + Provider, + nt, + stage, + write_board_fixture, +) +from tests.test_confirmatory_no_tools import ( + setup as _setup_fixture, +) +from tests.test_confirmatory_run import ( + complete_preflight_design, + cr, +) +from tests.test_confirmatory_run import ( + fast as _fast_fixture, +) +from tests.test_confirmatory_run import ( + world as _world_fixture, +) + +setup = _setup_fixture +world = _world_fixture +fast = _fast_fixture + + +def deny_listing(monkeypatch, target): + for name in ("scandir", "listdir"): + original = getattr(os, name) + + def denied(path, *args, _original=original, **kwargs): + if not isinstance(path, int) and Path(path) == target: + raise PermissionError("synthetic denied listing") + return _original(path, *args, **kwargs) + + monkeypatch.setattr(os, name, denied) + + +def deny_stat(monkeypatch, target): + original = os.stat + + def denied(path, *args, **kwargs): + if not isinstance(path, int) and Path(path) == target: + # Path predicates suppress ELOOP on the pinned Python 3.12. + raise OSError(errno.ELOOP, "synthetic inaccessible metadata") + return original(path, *args, **kwargs) + + monkeypatch.setattr(os, "stat", denied) + + +@pytest.mark.parametrize("which", ["finals", "attempts"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_unlistable_optional_history_refuses( + setup, world, tmp_path, monkeypatch, which, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + spec = design["no_tools"] + folder = ( + Path(spec["runs"]) / "gpt-6-sol" / spec["folder"] / spec["stage"] / "households" + ) + target = folder if which == "finals" else folder / "attempts" + target.mkdir(parents=True) + deny_listing(monkeypatch, target) + if boundary == "startup": + with pytest.raises(nt.StateError, match=str(target)): + with stage(setup, provider): + pass + else: + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + assert provider.calls == [] + + +@pytest.mark.parametrize("which", ["root-list", "root-stat", "model-stat"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_model_inventory_failure_refuses( + setup, world, tmp_path, monkeypatch, which, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + root = Path(design["no_tools"]["runs"]) + (root / "gpt-6-sol").mkdir(parents=True, exist_ok=True) + target = root / "gpt-6-sol" if which == "model-stat" else root + (deny_listing if which == "root-list" else deny_stat)(monkeypatch, target) + if boundary == "startup": + with state_for_loading(setup) as plan: + with pytest.raises(nt.StateError, match=str(target)): + plan._load_state(setup["board"]) + else: + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + assert provider.calls == [] + + +def vanish(monkeypatch, target): + """A listing returned target, but its metadata read finds it gone.""" + original = os.stat + + def gone(path, *args, **kwargs): + if not isinstance(path, int) and Path(path) == target: + raise FileNotFoundError(errno.ENOENT, "synthetic vanished entry", str(path)) + return original(path, *args, **kwargs) + + monkeypatch.setattr(os, "stat", gone) + + +def plant_listed_entry(monkeypatch, target, fault, tmp_path): + if fault == "dangling": + # A model folder symlinked to an unmounted volume. + target.symlink_to(tmp_path / "unmounted-volume") + else: + target.mkdir() + vanish(monkeypatch, target) + + +@pytest.mark.parametrize("fault", ["dangling", "vanished"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_listed_model_entry_without_metadata_refuses( + setup, world, tmp_path, monkeypatch, fault, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + root = Path(design["no_tools"]["runs"]) + root.mkdir(parents=True, exist_ok=True) + target = root / "ghost-model" + plant_listed_entry(monkeypatch, target, fault, tmp_path) + if boundary == "startup": + with pytest.raises(nt.StateError, match=re.escape(str(target))): + with stage(setup, provider): + pass + else: + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any( + problem.startswith(f"no-tools stage: {target}:") for problem in problems + ), problems + assert not w.provider.count + assert provider.calls == [] + + +@pytest.mark.parametrize("fault", ["dangling", "vanished"]) +@pytest.mark.parametrize("level", ["arm", "stage"]) +def test_preflight_listed_calculator_entry_without_metadata_refuses( + world, tmp_path, monkeypatch, fault, level +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + folder = Path(design["runs"]) / cr.CHEAP[0] + parent = folder if level == "arm" else folder / "pe" + parent.mkdir(parents=True, exist_ok=True) + target = parent / "ghost" + plant_listed_entry(monkeypatch, target, fault, tmp_path) + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(problem.startswith(f"{target}:") for problem in problems), problems + assert not w.provider.count + + +@pytest.mark.parametrize("fault", ["file", "stat"]) +@pytest.mark.parametrize("which", ["finals", "attempts"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_broken_optional_history_folder_refuses( + setup, world, tmp_path, monkeypatch, fault, which, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + spec = design["no_tools"] + folder = ( + Path(spec["runs"]) / "gpt-6-sol" / spec["folder"] / spec["stage"] / "households" + ) + target = folder if which == "finals" else folder / "attempts" + target.parent.mkdir(parents=True, exist_ok=True) + if fault == "file": + target.write_text("{}") + else: + target.mkdir() + deny_stat(monkeypatch, target) + if boundary == "startup": + with pytest.raises(nt.StateError, match=re.escape(str(target))): + with stage(setup, provider): + pass + else: + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + assert provider.calls == [] + + +def state_for_loading(setup): + # Isolate _load_state from the constructor's mkdir and lock operations. + import threading + + plan = object.__new__(nt.Stage) + plan.design = setup["design"] + plan.draw_order = setup["design"]["draw_order"] + plan.root = Path(setup["design"]["no_tools"]["runs"]) + plan.folder, plan.stage = "none", "confirmatory" + plan.models = nt.CHEAP + plan.stopped = threading.Event() + plan._lock_file = None + return plan + + +@pytest.mark.parametrize("which", ["folder-list", "folder-stat", "log-stat"]) +def test_calculator_summary_inventory_failure_refuses(world, monkeypatch, which): + w = world(models=(cr.CHEAP[0],)) + lanes = w.run.lanes[cr.CHEAP[0]] + target = ( + next(iter(lanes.households_dir.values())) + if which != "log-stat" + else lanes.folder / "run-log.jsonl" + ) + target.parent.mkdir(parents=True, exist_ok=True) + if which == "folder-list": + target.mkdir(exist_ok=True) + (deny_listing if which == "folder-list" else deny_stat)(monkeypatch, target) + with pytest.raises(nt.StateError, match=str(target)): + lanes.summary() + assert not w.provider.count + + +@pytest.mark.parametrize( + "which", ["model", "arm", "stage", "households", "run-log", "host-log"] +) +def test_calculator_preflight_inventory_failure_refuses( + world, tmp_path, monkeypatch, which +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + folder = Path(design["runs"]) / cr.CHEAP[0] + targets = { + "model": folder, + "arm": folder / "pe3", + "stage": folder / "pe3/confirm", + "households": folder / "pe3/confirm/households", + "run-log": folder / "run-log.jsonl", + "host-log": Path(design["runs"]) / "host/host-log.jsonl", + } + target = targets[which] + if which.endswith("log"): + target.parent.mkdir(parents=True, exist_ok=True) + deny_stat(monkeypatch, target) + else: + target.mkdir(parents=True, exist_ok=True) + deny_listing(monkeypatch, target) + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + + +@pytest.mark.parametrize("which", ["model", "arm", "stage", "households"]) +def test_calculator_preflight_directory_stat_failure_refuses( + world, tmp_path, monkeypatch, which +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + folder = Path(design["runs"]) / cr.CHEAP[0] + target = ( + folder + / { + "model": "", + "arm": "pe3", + "stage": "pe3/confirm", + "households": "pe3/confirm/households", + }[which] + ) + target.mkdir(parents=True, exist_ok=True) + deny_stat(monkeypatch, target) + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + + +def test_preflight_unreadable_preregistration_refuses(world, tmp_path, monkeypatch): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + target = Path(design["preregistration"]) + original = Path.read_bytes + + def denied(path): + if path == target: + raise PermissionError("synthetic denied read") + return original(path) + + monkeypatch.setattr(Path, "read_bytes", denied) + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + + +@contextlib.contextmanager +def fifo_released_after(path, seconds=5.0): + """Bound a reader that blocks on the FIFO: after `seconds`, open and close a + writer every 50 ms, so each blocked open sees end of file instead of hanging.""" + done = threading.Event() + + def release(): + if done.wait(seconds): + return + while not done.is_set(): + with contextlib.suppress(OSError): # ENXIO while no reader waits + os.close(os.open(path, os.O_WRONLY | os.O_NONBLOCK)) + done.wait(0.05) + + thread = threading.Thread(target=release, daemon=True) + thread.start() + try: + yield + finally: + done.set() + thread.join() + + +@pytest.mark.parametrize("kind", ["fifo", "device"]) +@pytest.mark.parametrize("which", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_saved_record_that_is_not_a_regular_file_refuses_at_once( + setup, boundary, which, kind +): + spec = setup["design"]["no_tools"] + folder = ( + Path(spec["runs"]) / nt.CHEAP[0] / spec["folder"] / spec["stage"] / "households" + ) + target = folder / ("h2.json" if which == "final" else "attempts/h2-1.json") + target.parent.mkdir(parents=True) + if kind == "fifo": + os.mkfifo(target) + else: + target.symlink_to("/dev/null") + provider = Provider() + start = time.monotonic() + with fifo_released_after(target) if kind == "fifo" else contextlib.nullcontext(): + if boundary == "startup": + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("a special file must refuse before execution") + message = str(caught.value) + else: + message = "\n".join(nt.stage_problems(setup["design"])) + found = "FIFO" if kind == "fifo" else "character device" + assert str(target) in message + assert f"expected a regular file, found {found}" in message + assert time.monotonic() - start < 5, "no read may block on the special file" + assert provider.calls == [] + + +def prepared_final(plan, model=nt.CHEAP[0], sid="h2"): + """Save a rerun-eligible final and its history, with no attempts folder.""" + folder = plan.households(model) + folder.mkdir(parents=True) + payload = { + "scenario_id": sid, + "status": "provider_error", + "error": "APIConnectionError", + "turns": [], + "attempts": [], + "totals": {"cost_usd": 0.1}, + } + source = folder / f"{sid}.json" + source.write_text(json.dumps(payload)) + plan.histories[model][sid] = { + "final": payload, + "entries": [], + "final_sha256": nt.sha256_file(source), + } + return folder, source, payload + + +def test_archive_refuses_an_existing_target_and_keeps_both_files(setup): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, payload = prepared_final(plan) + target = folder / "attempts" / "h2-1.json" + target.parent.mkdir() + target.write_text('{"synthetic": "an earlier archive"}') + with pytest.raises( + nt.StateError, match=re.escape(f"{target}: would overwrite") + ): + plan.archive(nt.CHEAP[0], "h2") + assert source.read_text() == json.dumps(payload) + assert target.read_text() == '{"synthetic": "an earlier archive"}' + assert sorted(p.name for p in target.parent.iterdir()) == ["h2-1.json"] + assert plan.histories[nt.CHEAP[0]]["h2"]["entries"] == [] + assert provider.calls == [] + + +def test_archive_digest_refusal_leaves_no_attempts_folder(setup): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, payload = prepared_final(plan) + source.write_text(json.dumps(dict(payload, totals={"cost_usd": 0.2}))) + with pytest.raises(nt.StateError, match=re.escape(f"{source}: final changed")): + plan.archive(nt.CHEAP[0], "h2") + assert json.loads(source.read_text())["totals"] == {"cost_usd": 0.2} + assert not (folder / "attempts").exists() + assert plan.histories[nt.CHEAP[0]]["h2"]["entries"] == [] + assert provider.calls == [] + + +def test_archive_refuses_a_final_replaced_by_a_fifo_at_once(setup): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, _ = prepared_final(plan) + source.unlink() + os.mkfifo(source) + start = time.monotonic() + with fifo_released_after(source): + with pytest.raises( + nt.StateError, match="expected a regular file, found FIFO" + ): + plan.archive(nt.CHEAP[0], "h2") + assert time.monotonic() - start < 5, "no digest read may block on a FIFO" + assert not (folder / "attempts").exists() + assert provider.calls == [] + + +def test_archive_refuses_a_target_replaced_by_a_fifo_after_its_rename( + setup, monkeypatch +): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, _ = prepared_final(plan) + target = folder / "attempts" / "h2-1.json" + original = nt.os.replace + + def rename_then_replace(old, new): + original(old, new) + Path(new).unlink() + os.mkfifo(new) + + monkeypatch.setattr(nt.os, "replace", rename_then_replace) + start = time.monotonic() + with fifo_released_after(target): + with pytest.raises( + nt.StateError, match="expected a regular file, found FIFO" + ): + plan.archive(nt.CHEAP[0], "h2") + assert time.monotonic() - start < 5, "the second hash may not block on a FIFO" + assert not source.exists() + assert plan.histories[nt.CHEAP[0]]["h2"]["entries"] == [] + assert provider.calls == [] + + +@pytest.mark.parametrize("fault", ["second-read", "move-back"]) +def test_a_failed_archive_stops_one_model_and_startup_recovers_it(setup, fault): + """The second hash or the move back fails: that model errors, the others + finish, the invocation releases its lock, and the archive stays with no + final and no link. Startup then rebuilds the link from the bytes present, + changed or not, which is the limit the draft states.""" + model = nt.CHEAP[0] + provider = Provider( + {(model, "h2", 1): {"status": "provider_error", "error": "APIConnectionError"}} + ) + plan = stage(setup, provider) + folder = plan.households(model) + source, target = folder / "h2.json", folder / "attempts" / "h2-1.json" + with pytest.MonkeyPatch.context() as patch: + if fault == "second-read": + read = nt.read_regular + + def unreadable(path): + if Path(path) == target: + raise PermissionError(errno.EACCES, "synthetic denied", str(path)) + return read(path) + + patch.setattr(nt, "read_regular", unreadable) + else: + replace = os.replace + + def failed_move_back(old, new): + if Path(old) == source: + Path(old).write_bytes(Path(old).read_bytes() + b" ") + elif Path(old) == target: + raise OSError(errno.EIO, "synthetic failed move", str(old)) + replace(old, new) + + patch.setattr(os, "replace", failed_move_back) + assert plan.execute() == 1 + assert plan.states[model] == "error" + assert str(target) in plan.out.getvalue() + assert all(plan.states[other] == "complete" for other in nt.CHEAP[1:]) + assert all( + plan.standing(other, sid) for other in nt.CHEAP[1:] for sid in ("h2", "h1") + ) + assert [call for call in provider.calls if call[0] == model] == [(model, "h2", 1)] + assert target.exists() and not source.exists() + digest = nt.sha256_file(target) + again = Provider() + resumed = stage(setup, again, models=[model]) # raises if the lock survived + assert resumed.histories[model]["h2"]["entries"][0]["sha256"] == digest + assert resumed.execute() == 0 + assert again.calls == [(model, "h2", 1), (model, "h1", 1)] + assert json.loads(source.read_text())["attempts"][0]["sha256"] == digest + assert nt.stage_problems(setup["design"]) == [] + + +def test_archive_target_stat_failure_refuses(setup, monkeypatch): + provider = Provider() + with stage(setup, provider) as plan: + model, sid = nt.CHEAP[0], "h2" + folder = plan.households(model) + folder.mkdir(parents=True) + payload = { + "scenario_id": sid, + "status": "provider_error", + "error": "APIConnectionError", + "turns": [], + "attempts": [], + "totals": {"cost_usd": 0.1}, + } + (folder / f"{sid}.json").write_text(json.dumps(payload)) + plan.histories[model][sid] = { + "final": payload, + "entries": [], + "final_sha256": nt.sha256_file(folder / f"{sid}.json"), + } + target = folder / "attempts" / f"{sid}-1.json" + deny_stat(monkeypatch, target) + with pytest.raises(nt.StateError, match=str(target)): + plan.archive(model, sid) + assert (folder / f"{sid}.json").read_text() == json.dumps(payload) + assert not (folder / "attempts").exists() + assert provider.calls == [] + + +def test_board_exclusions_stat_failure_refuses(setup, tmp_path, monkeypatch): + write_board_fixture(setup, tmp_path / "board") + target = Path(setup["design"]["board"]) / "reference_exclusions.json" + deny_stat(monkeypatch, target) + provider = Provider() + with pytest.raises(nt.StateError, match=str(target)): + with nt.Stage(setup["design"], run_household=provider, out=io.StringIO()): + pass + assert provider.calls == [] diff --git a/tests/test_household_set.py b/tests/test_household_set.py new file mode 100644 index 00000000..99d42f9b --- /dev/null +++ b/tests/test_household_set.py @@ -0,0 +1,335 @@ +"""The household-set builder and the tool pilot's --board path. + +The build test runs the real engine on three of the frozen board's own +households, with sampling and the dataset download faked, so the references it +writes must equal the frozen ones on the frozen engine (policyengine-us +2.41.0). It is marked slow. +""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import json +import sys +from pathlib import Path + +import pandas as pd +import pytest +from hypothesis import given +from hypothesis import strategies as st + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import household_set as hs # noqa: E402 +import tool_pilot_argon as pilot # noqa: E402 + +ON_FROZEN_ENGINE = importlib.metadata.version("policyengine-us") == "2.41.0" +SOL = "gpt-6-sol" + + +def test_rename_gives_fresh_ids_in_draw_order(): + scenarios = hs.load_scenarios(hs.FROZEN_BOARD)[:3] + renamed = hs.rename_scenarios(list(reversed(scenarios)), "mc") + assert [s.id for s in renamed] == ["mc_000", "mc_001", "mc_002"] + assert renamed[0].state == scenarios[2].state + assert renamed[0].metadata == scenarios[2].metadata + assert [s.id for s in scenarios] == ["scenario_000", "scenario_001", "scenario_002"] + + +@given(st.lists(st.text(min_size=1), unique=True, max_size=60), st.integers(1, 15)) +def test_batches_partition_the_draw_in_order(ids, size): + batches = hs.split_batches(ids, size) + assert [sid for batch in batches for sid in batch] == ids + assert all(len(batch) == size for batch in batches[:-1]) + assert all(1 <= len(batch) <= size for batch in batches) + + +def test_batches_refuse_a_size_below_one(): + with pytest.raises(ValueError): + hs.split_batches(["a"], 0) + + +def test_differences_report_missing_and_moved_outputs(): + expected = pd.DataFrame( + { + "scenario_id": ["a", "a", "b"], + "variable": ["x", "y", "x"], + "value": [1.0, 2.0, 3.0], + } + ) + computed = pd.DataFrame( + {"scenario_id": ["a", "a"], "variable": ["x", "y"], "value": [1.0, 2.5]} + ) + diff = hs.differences(expected, computed, tolerance=1e-6) + assert sorted(zip(diff.scenario_id, diff.variable)) == [("a", "y"), ("b", "x")] + assert hs.differences(expected, computed, tolerance=1.0).scenario_id.tolist() == [ + "b" + ] + + +def test_fix_module_pins_are_the_frozen_revision(): + pins = hs.fix_module_pins() + assert pins == [ + {"module": p["module"], "path": p["path"], "sha256": p["sha256"]} + for p in hs.frozen_fix_pins() + ] + + +def test_the_frozen_board_reads_as_before(): + assert pilot.board_engine() == pilot.REFERENCE_ENGINE + assert not pilot.is_household_set(pilot.SNAPSHOT) + assert pilot.board_fix_module_sha256() == pilot.FIX_MODULE_SHA256 + assert pilot.board_path(argparse.Namespace()) == pilot.SNAPSHOT + + +@pytest.mark.slow +def test_build_writes_the_frozen_references_for_frozen_households(household_board): + out, sampled = household_board + frozen = pd.read_csv(hs.FROZEN_BOARD / "reference_outputs.csv") + built = pd.read_csv(out / "reference_outputs.csv") + names = {f"mc_{i:03d}": s.id for i, s in enumerate(sampled)} + built["frozen_id"] = built.scenario_id.map(names) + merged = built.merge( + frozen, + left_on=["frozen_id", "variable"], + right_on=["scenario_id", "variable"], + suffixes=("", "_frozen"), + ) + assert ( + len(merged) + == len(built) + == len(frozen[frozen.scenario_id.isin(names.values())]) + ) + from policybench.reference_exclusions import load_reference_exclusions + + excluded = { + (e["scenario_id"], e["variable"]) + for e in load_reference_exclusions(hs.FROZEN_BOARD) + } + scored = merged[ + [(s, v) not in excluded for s, v in zip(merged.frozen_id, merged.variable)] + ] + assert (scored.value - scored.value_frozen).abs().max() < 1e-6 + + +@pytest.mark.slow +def test_build_writes_a_board_the_pilot_reads(household_board): + out, _ = household_board + meta = json.loads((out / "reference_outputs.csv.meta.json").read_text()) + assert meta["board_kind"] == "household_set" + assert meta["seed"] == 7 and meta["num_scenarios"] == 3 + assert meta["reference_csv_sha256"] == hs.sha256_file(out / "reference_outputs.csv") + assert json.loads((out / "batches.json").read_text())["batches"] == [ + ["mc_000", "mc_001"], + ["mc_002"], + ] + board = pilot.load_board(out) + assert sorted(board["scenarios"]) == ["mc_000", "mc_001", "mc_002"] + assert board["excluded"].empty + assert pilot.is_household_set(out) + assert pilot.board_engine(out) == "2.41.0" + pilot.verify_fix_pins(snapshot=out) + pilot._require_engine(out) + + +@pytest.mark.slow +def test_build_refuses_a_written_directory_and_a_dataset_off_the_bundle( + household_board, tmp_path, monkeypatch +): + out, _ = household_board + with pytest.raises(SystemExit, match="not empty"): + hs.main(["build", "--n", "3", "--seed", "7", "--out", str(out)]) + import policybench.policyengine_runtime as runtime + + dataset = tmp_path / "x.h5" + dataset.write_bytes(b"other bytes") + monkeypatch.setattr( + runtime, "materialize_certified_us_dataset", lambda: str(dataset) + ) + monkeypatch.setattr( + runtime, + "policyengine_bundles_for_countries", + lambda countries: {"us": {"certified_data_artifact_sha256": "0" * 64}}, + ) + with pytest.raises(SystemExit, match="not the bundle's"): + hs.main(["build", "--n", "1", "--seed", "1", "--out", str(tmp_path / "new")]) + + +def _record(sid, variables, predictions, model, arm, prompt="v2-pilot"): + record = { + "scenario_id": sid, + "model": model, + "arm": arm, + "prompt_version": prompt, + "benchmark_search_rule": arm in ("web", "web+pe"), + "variables": variables, + "predictions": predictions, + "explanations": {v: "x" for v in variables}, + "status": "complete", + "turns": [{"total_cost_usd": 0.02, "calls": []}], + "cap_hit": False, + "wall_seconds": 1.0, + } + record["totals"] = pilot.household_totals(record) + return record + + +def _write_run(directory, board, model, arm, miss=None, prompt="v2-pilot"): + reference = board["reference"].set_index(["scenario_id", "variable"])["value"] + (directory / "households").mkdir(parents=True) + for sid in sorted(board["scenarios"]): + variables = board["variables"][sid] + predictions = {v: float(reference[(sid, v)]) for v in variables} + if miss is not None and sid == miss[0]: + predictions[miss[1]] = float(reference[(sid, miss[1])]) + 100 + record = _record(sid, variables, predictions, model, arm, prompt) + (directory / "households" / f"{sid}.json").write_text(json.dumps(record)) + + +@pytest.mark.slow +def test_a_household_set_scores_each_arm_against_its_no_tools_run( + household_board, tmp_path +): + out, _ = household_board + board = pilot.load_board(out) + amount = next( + v + for v in board["variables"]["mc_000"] + if not v.endswith("_eligible") and v in set(board["scored"].variable) + ) + none_dir, web_dir = tmp_path / "none", tmp_path / "web" + _write_run(none_dir, board, SOL, "none", miss=("mc_000", amount)) + _write_run(web_dir, board, SOL, "web") + pilot.command_score( + argparse.Namespace( + out_dir=str(web_dir), + complete_only=False, + board=str(out), + baseline_dir=str(none_dir), + ) + ) + score = json.loads((web_dir / "score.json").read_text()) + assert score["unweighted_hits"]["arm2"] == score["scored_outputs"] + assert score["unweighted_hits"]["arm1"] == score["scored_outputs"] - 1 + usage = pd.read_csv(web_dir / "usage_by_household.csv") + assert usage.arm1_cost_usd.round(6).tolist() == [0.02, 0.02, 0.02] + + with pytest.raises(SystemExit, match="--baseline-dir"): + pilot.command_score( + argparse.Namespace( + out_dir=str(web_dir), complete_only=False, board=str(out) + ) + ) + with pytest.raises(SystemExit, match="not gpt-6-sol none"): + pilot.command_score( + argparse.Namespace( + out_dir=str(web_dir), + complete_only=False, + board=str(out), + baseline_dir=str(web_dir), + ) + ) + v1_dir = tmp_path / "none-v1" + _write_run(v1_dir, board, SOL, "none", prompt="v1") + with pytest.raises(SystemExit, match="another prompt version"): + pilot.command_score( + argparse.Namespace( + out_dir=str(web_dir), + complete_only=False, + board=str(out), + baseline_dir=str(v1_dir), + ) + ) + + +def _scenario(sid, filing_status, tax_unit): + from policybench.scenarios import Person, Scenario + + return Scenario( + id=sid, + state="CA", + filing_status=filing_status, + adults=[Person(name="head", age=40, employment_income=0.0)], + metadata={"household_id": 1, "tax_unit_id": tax_unit}, + ) + + +def test_stated_filing_status_is_read_from_the_tax_unit_table(tmp_path): + import h5py + import numpy as np + + path = tmp_path / "set.h5" + rows = np.array( + [(0, 11, b"JOINT"), (1, 12, b"SEPARATE"), (2, 13, b"SURVIVING_SPOUSE")], + dtype=[ + ("index", " AdversaryCase: print("9.9.9 (Claude Code fake)") sys.exit(0) if args[:1] == ["auth"]: + if os.environ.get("ANTHROPIC_API_KEY"): + print(json.dumps({ + "loggedIn": True, "authMethod": "api_key", "apiProvider": "firstParty", + "apiKeySource": "ANTHROPIC_API_KEY", + "configDirectory": os.environ.get("CLAUDE_CONFIG_DIR"), + })) + sys.exit(0) print(json.dumps( {"loggedIn": True, "authMethod": "oauth_token", "apiProvider": "firstParty"} )) @@ -538,6 +545,74 @@ def test_claude_runner_refuses_without_a_lane_login(tmp_path: Path): assert _calls(bin_dir) == [] +def _api_key_env(tmp_path: Path, env: dict, key: str = "sk-ant-judge") -> dict: + """The runner's environment for an API key judge, with a fake secret store.""" + store = tmp_path / "bin" / "secret" + store.write_text( + f"#!{sys.executable}\n" + "import sys\n" + f"print({key!r}) if sys.argv[1:3] == ['get', 'JUDGE_KEY'] else sys.exit(1)\n" + ) + store.chmod(0o755) + env = {**env, "JUDGE_API_KEY_SECRET": "JUDGE_KEY", "AUDIT_SECRET_BIN": str(store)} + for name in ("CLAUDE_CONFIG_DIR", "AUDIT_ACCOUNT", "CLAUDE_CODE_OAUTH_TOKEN"): + env.pop(name, None) + return env + + +def test_claude_runner_judges_on_a_declared_api_key_in_its_own_config(tmp_path: Path): + adversary, bin_dir, env = _setup(tmp_path, "claude") + env = _api_key_env(tmp_path, env) + env["TMPDIR"] = str(tmp_path / "tmp") + (tmp_path / "tmp").mkdir() + result = _run("claude", adversary, env, tmp_path) + assert result.returncode == 0, result.stderr + result.stdout + calls = _calls(bin_dir) + assert calls + configs = {call["env"]["CLAUDE_CONFIG_DIR"] for call in calls} + assert len(configs) == 1 + config = configs.pop() + assert Path(config).parent == (tmp_path / "tmp").resolve() + assert not Path(config).exists() # removed when the run ended + for call in calls: + # The declared key, never the parent's ANTHROPIC_API_KEY. + assert call["env"]["ANTHROPIC_API_KEY"] == "sk-ant-judge" + assert "CLAUDE_CODE_OAUTH_TOKEN" not in call["env"] + for case in CASES: + meta = json.loads( + (adversary / "cases" / case.case_id / "verdict.meta.json").read_text() + ) + assert meta["judge_auth"] == { + "method": "api_key", + "account": None, + "org": None, + "secret": "JUDGE_KEY", + } + assert "JUDGE_KEY" in meta["judge_account_declared"] + assert "sk-ant-judge" not in json.dumps(meta) + + +def test_claude_runner_refuses_an_api_key_mixed_with_a_login(tmp_path: Path): + adversary, bin_dir, env = _setup(tmp_path, "claude") + base = _api_key_env(tmp_path, env) + mixes = { + "CLAUDE_CONFIG_DIR": (str(tmp_path / "lane-config"), "own empty config"), + "AUDIT_ACCOUNT": ("claude:lane@example.org", "unset AUDIT_ACCOUNT"), + "CLAUDE_CODE_OAUTH_TOKEN": ("token", "takes no lane token"), + "JUDGE_ALLOW_DESKTOP_LOGIN": ("1", "are exclusive"), + } + for name, (value, message) in mixes.items(): + result = _run("claude", adversary, {**base, name: value}, tmp_path) + assert result.returncode == 1 + assert message in result.stderr, (name, result.stderr) + result = _run( + "claude", adversary, {**base, "JUDGE_API_KEY_SECRET": "MISSING"}, tmp_path + ) + assert result.returncode == 1 + assert "could not read the API key MISSING" in result.stderr + assert _calls(bin_dir) == [] + + # --- Codex ------------------------------------------------------------------------- diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index a5d531b0..4e586374 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -2293,6 +2293,8 @@ def test_web_notices_name_search_and_keep_the_answer_instruction(): "pe": "gpt-6-sol+pe-tool", "web": "gpt-6-sol+web", "web+pe": "gpt-6-sol+web+pe-tool", + "pe2": "gpt-6-sol+pe-tool-v2", + "pe3": "gpt-6-sol+pe-tool-v3", } with pytest.raises(SystemExit): pilot.model_profile(SOL, "search") diff --git a/tests/test_tool_v2.py b/tests/test_tool_v2.py new file mode 100644 index 00000000..68248090 --- /dev/null +++ b/tests/test_tool_v2.py @@ -0,0 +1,326 @@ +"""The calculator's tool version 2 (the pe2 arm). + +Version 2 reads the names the prompt uses (labels written with spaces, and the +benchmark's output keys), redirects a contribution set as the realized amount +to its ``_desired`` input, flags an input that replaces a formula, places an +input set on another entity where exactly one unit holds it, and explains a +variable on request. Version 1 must behave exactly as before. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import tool_pilot_argon as pilot # noqa: E402 + +V = pilot.VariableInfo +REGISTRY = { + "age": V("person", "int", "age", "year"), + "employment_income": V("person", "float", "employment income", "year"), + "rent": V("person", "float", "rent", "year"), + "traditional_401k_contributions": V( + "person", "float", "401(k)", "year", computed=True + ), + "traditional_401k_contributions_desired": V( + "person", "float", "401(k) asked", "year" + ), + "is_medicaid_eligible": V("person", "bool", "Medicaid", "year", computed=True), + "snap": V("spm_unit", "float", "SNAP", "month", computed=True), + "free_school_meals": V("spm_unit", "float", "free meals", "year", computed=True), + "qualified_passenger_vehicle_loan_interest": V( + "household", "float", "car loan interest", "year" + ), + "state_code": V("household", "Enum", "state", "year", ("CA", "TX")), + "filing_status": V( + "tax_unit", "Enum", "filing status", "year", ("SINGLE", "JOINT"), True + ), +} + + +def _situation(**overrides): + base = { + "people": { + "head": {"age": {"2026": 40}, "employment_income": 30000}, + "kid": {"age": {"2026": 8}}, + }, + "spm_units": {"spm": {"members": ["head", "kid"]}}, + "households": {"h": {"members": ["head", "kid"], "state_code": "CA"}}, + } + base.update(overrides) + return base + + +def _call(situation, variables=("snap",), **extra): + return {"situation": json.dumps(situation), "variables": list(variables), **extra} + + +def _v2(args): + return pilot.normalize_request(args, REGISTRY, 2) + + +def test_version_1_still_refuses_what_version_2_reads(): + situation = _situation() + situation["people"]["head"]["traditional 401k contributions desired"] = 1000 + situation["people"]["head"]["qualified_passenger_vehicle_loan_interest"] = 500 + request, error = pilot.normalize_request( + _call(situation, ("free_school_meals_eligible",), explain=["snap"]), REGISTRY + ) + assert request is None + problems = " ".join(error["problems"]) + assert "unknown arguments: explain" in problems + assert "unknown variable 'traditional 401k contributions desired'" in problems + assert "is a household variable; set it under households" in problems + assert "unknown variable 'free_school_meals_eligible'" in problems + + +def test_version_2_reads_a_label_written_with_spaces(): + situation = _situation() + situation["people"]["head"]["Traditional 401k contributions desired"] = 1000 + request, error = _v2(_call(situation)) + assert error is None + head = request["situation"]["people"]["head"] + assert head["traditional_401k_contributions_desired"] == {"2026": 1000} + assert any( + "as traditional_401k_contributions_desired" in n for n in request["notes"] + ) + + +def test_version_2_sets_the_asked_amount_for_a_computed_contribution(): + situation = _situation() + situation["people"]["head"]["traditional_401k_contributions"] = 18523 + request, error = _v2(_call(situation)) + assert error is None + head = request["situation"]["people"]["head"] + assert "traditional_401k_contributions" not in head + assert head["traditional_401k_contributions_desired"] == {"2026": 18523} + assert any("before limits" in n for n in request["notes"]) + + +def test_version_2_flags_an_input_that_replaces_a_formula(): + situation = _situation() + situation["spm_units"]["spm"]["snap"] = 100 + request, error = _v2(_call(situation)) + assert error is None + assert request["situation"]["spm_units"]["spm"]["snap"] == {"2026": 100} + assert any("snap is computed by the engine" in n for n in request["notes"]) + + +def test_version_2_moves_a_household_input_to_the_persons_household(): + situation = _situation() + situation["people"]["head"]["qualified_passenger_vehicle_loan_interest"] = 500 + request, error = _v2(_call(situation)) + assert error is None + assert request["situation"]["households"]["h"][ + "qualified_passenger_vehicle_loan_interest" + ] == {"2026": 500} + assert ( + "qualified_passenger_vehicle_loan_interest" + not in (request["situation"]["people"]["head"]) + ) + assert any( + "moved qualified_passenger_vehicle_loan_interest" in n for n in request["notes"] + ) + + +def test_version_2_makes_the_one_unit_the_engine_would_when_a_group_is_left_out(): + situation = _situation() + del situation["households"] + situation["people"]["kid"]["state_code"] = "TX" + request, error = _v2(_call(situation)) + assert error is None + households = request["situation"]["households"] + assert households == { + "all": {"members": ["head", "kid"], "state_code": {"2026": "TX"}} + } + + +def test_version_2_moves_a_person_input_off_a_one_person_unit_only(): + one = _situation( + spm_units={"a": {"members": ["head"], "rent": 9000}, "b": {"members": ["kid"]}} + ) + request, error = _v2(_call(one)) + assert error is None + assert request["situation"]["people"]["head"]["rent"] == {"2026": 9000} + both = _situation() + both["spm_units"]["spm"]["rent"] = 9000 + request, error = _v2(_call(both)) + assert request is None + assert any("set it on one person" in p for p in error["problems"]) + + +def test_version_2_refuses_two_different_values_for_one_input(): + situation = _situation() + situation["households"]["h"]["qualified_passenger_vehicle_loan_interest"] = 400 + situation["people"]["head"]["qualified_passenger_vehicle_loan_interest"] = 500 + request, error = _v2(_call(situation)) + assert request is None + assert any("set twice with different values" in p for p in error["problems"]) + + +def test_version_2_refuses_a_group_input_no_single_unit_holds(): + situation = _situation( + households={"h1": {"members": ["head"]}, "h2": {"members": ["kid"]}}, + tax_units={"t": {"members": ["head", "kid"], "state_code": "CA"}}, + ) + request, error = _v2(_call(situation)) + assert request is None + assert any("no single unit there holds" in p for p in error["problems"]) + + +def test_version_2_reads_output_keys_as_their_engine_variables(): + request, error = _v2( + _call(_situation(), ("free_school_meals_eligible", "head_medicaid_eligible")) + ) + assert error is None + assert request["variables"] == ["free_school_meals", "is_medicaid_eligible"] + notes = " ".join(request["notes"]) + assert "1 when free_school_meals is above 0" in notes + assert "is_medicaid_eligible for the person the key names (head)" in notes + + +def test_version_2_explains_at_most_three_known_variables(): + request, error = _v2(_call(_situation(), explain=["snap", "free school meals"])) + assert error is None + assert request["explain"] == ["snap", "free_school_meals"] + request, error = _v2(_call(_situation(), explain=["snap", "a", "b", "c"])) + assert request is None + assert any("explain at most 3" in p for p in error["problems"]) + request, error = _v2(_call(_situation(), explain=["no_such_thing"])) + assert request is None + assert any("explain: unknown variable" in p for p in error["problems"]) + + +@given(st.sampled_from(sorted(REGISTRY)), st.sampled_from([" ", " ", "-", " - "])) +def test_a_label_written_with_any_separator_reads_back_to_its_name(name, separator): + written = name.replace("_", separator).upper() + resolved, note = pilot.resolve_name(written, REGISTRY, output_names=False) + assert resolved == name + assert (note is None) == (written == name) + + +def test_the_stated_engine_follows_the_board_and_defaults_to_the_frozen_one(): + profile = pilot.ModelProfile("m", "gpt-x", "chat", "tool", arm="pe") + frozen_notice = profile.notice + frozen_schema = pilot.calculate_tool_schema(1) + assert "policyengine-us 2.41.0" in frozen_notice + try: + pilot.set_stated_engine("2.60.4") + assert "policyengine-us 2.60.4" in profile.notice + assert "2.41.0" not in profile.notice + assert ( + "policyengine-us 2.60.4" + in pilot.calculate_tool_schema(2)["function"]["description"] + ) + finally: + pilot.set_stated_engine(pilot.REFERENCE_ENGINE) + assert profile.notice == frozen_notice + assert pilot.calculate_tool_schema(1) == frozen_schema + + +def test_the_pe2_arm_offers_tool_version_2(): + profile = pilot.ModelProfile("m", "gpt-x", "chat", "tool", arm="pe2") + assert profile.uses_calculator and not profile.uses_web + assert profile.tool_version == 2 + assert profile.notice == pilot.TOOL_NOTICE_V2 + assert "explain" in profile.notice + assert profile.arm2_label == "m+pe-tool-v2" + json_profile = pilot.ModelProfile("m", "claude-x", "chat", "json", arm="pe2") + assert json_profile.notice.endswith("single JSON object described above.") + for arm in ("pe", "web+pe"): + assert ( + pilot.ModelProfile("m", "gpt-x", "chat", "tool", arm=arm).tool_version == 1 + ) + v2 = pilot.calculate_tool_schema(2)["function"] + assert "explain" in v2["parameters"]["properties"] + assert ( + "explain" + not in pilot.calculate_tool_schema(1)["function"]["parameters"]["properties"] + ) + + +def _echo_worker(conn, fix_path, reform): + conn.send(("ready", None)) + while True: + args = conn.recv() + if args is None: + return + conn.send(("ok", {"received": args})) + + +def test_the_pool_tells_its_workers_the_tool_version(): + pool = pilot.ToolPool( + size=1, timeout=5.0, worker_target=_echo_worker, tool_version=2 + ) + try: + result, _ = pool.call({"situation": "{}"}) + # A frame no model's JSON arguments can be (tool v3 review r1). + assert result == { + "received": ( + "calculate", + ("explain", "names", "overrides", "placement"), + {"situation": "{}"}, + ) + } + finally: + pool.close() + pool = pilot.ToolPool(size=1, timeout=5.0, worker_target=_echo_worker) + try: + result, _ = pool.call({"situation": "{}"}) + assert result == {"received": {"situation": "{}"}} + finally: + pool.close() + + +@pytest.mark.slow +def test_an_explanation_traces_the_engine_and_cites_its_parameters(): + system = pilot.build_tool_system(reform=True) + registry = pilot.build_registry(system) + # Computed: the engine derives it and no stated fact sets it. + assert registry["snap"].computed and registry["is_medicaid_eligible"].computed + assert not registry["employment_income"].computed # summed, but stated + assert not registry["age"].computed + assert not registry["takes_up_snap_if_eligible"].computed + situation = { + "people": { + "head": { + "age": {"2026": 40}, + "employment_income": {"2026": 20000}, + "traditional_401k_contributions": {"2026": 1000}, + } + }, + "households": {"h": {"members": ["head"], "state_code": "TX"}}, + } + request, error = pilot.normalize_request( + { + "situation": json.dumps(situation), + "variables": ["federal_income_tax_before_refundable_credits"], + "explain": ["income_tax_before_refundable_credits"], + }, + registry, + 2, + ) + assert error is None, error + assert request["variables"] == ["income_tax_before_refundable_credits"] + result = pilot.execute_request(system, registry, request) + if "traditional_401k_contributions_desired" in registry: + assert any("before limits" in n for n in result["notes"]) + explanation = result["explanations"]["income_tax_before_refundable_credits"] + assert explanation["steps"][0].startswith( + "income_tax_before_refundable_credits (2026)" + ) + assert len(explanation["steps"]) <= pilot.EXPLAIN_MAX_LINES + assert explanation["parameters"], explanation + assert not any( + p["name"].startswith("gov.abolitions.") for p in explanation["parameters"] + ) + assert any(p["sources"] for p in explanation["parameters"]) + value = result["results"]["income_tax_before_refundable_credits"]["values"] + assert set(value) == {"0"} or len(value) == 1 diff --git a/tests/test_tool_v3.py b/tests/test_tool_v3.py new file mode 100644 index 00000000..71bc73c3 --- /dev/null +++ b/tests/test_tool_v3.py @@ -0,0 +1,1015 @@ +"""The calculator's features one at a time (tool v3 and the ablation arms). + +Each feature (names, placement, overrides, explain, concise, flat) must work +on its own and change nothing else, so a study can turn one on at a time. The +version 2 feature set must reproduce tool v2's notice byte for byte, and the +flat household format must build the units the benchmark's own reference +situations have. +""" + +from __future__ import annotations + +import itertools +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import tool_pilot_argon as pilot # noqa: E402 + +V = pilot.VariableInfo +REGISTRY = { + "age": V("person", "int", "age", "year"), + "employment_income": V("person", "float", "employment income", "year"), + "is_tax_unit_head": V("person", "bool", "head", "year"), + "is_tax_unit_spouse": V("person", "bool", "spouse", "year"), + "traditional_401k_contributions": V( + "person", "float", "401(k)", "year", computed=True + ), + "traditional_401k_contributions_desired": V("person", "float", "asked", "year"), + "snap": V("spm_unit", "float", "SNAP", "month", computed=True), + "pre_subsidy_rent": V("person", "float", "rent", "year"), + "spm_unit_energy_subsidy": V("spm_unit", "float", "energy", "year"), + "qualified_passenger_vehicle_loan_interest": V( + "household", "float", "car loan", "year" + ), + "state_code": V("household", "Enum", "state", "year", ("CA", "LA")), + "tax_unit_childcare_expenses": V("tax_unit", "float", "childcare", "year"), + "marital_unit_flag": V("marital_unit", "bool", "flag", "year"), + "has_esi": V("person", "bool", "ESI", "year"), + "hours_worked_last_week": V("person", "float", "hours", "year"), + "weekly_hours_worked_before_lsr": V("person", "float", "hours", "year"), + "partnership_self_employment_net_earnings": V("person", "float", "p", "year"), +} +F = pilot + + +def _situation(): + return { + "people": {"head": {"age": 40}, "kid": {"age": 8}}, + "spm_units": {"spm": {"members": ["head", "kid"]}}, + "households": {"h": {"members": ["head", "kid"], "state_code": "CA"}}, + } + + +def _norm(args, *features): + return pilot.normalize_request(args, REGISTRY, features=frozenset(features)) + + +def test_the_version_2_set_reproduces_tool_v2s_notice(): + assert F.compose_tool_notice(F.V2_FEATURES, True) == F.TOOL_NOTICE_V2 + assert F.compose_tool_notice(F.V2_FEATURES, False) == F.JSON_TOOL_NOTICE_V2 + assert F.compose_tool_notice(frozenset(), True) == F.TOOL_NOTICE + assert F.compose_tool_notice(frozenset(), False) == F.JSON_TOOL_NOTICE + + +def test_each_feature_adds_its_own_sentence_only(): + base = F.compose_tool_notice(frozenset({F.EXPLAIN}), True) + assert "`explain`" in base and "names it read" not in base + assert "written with spaces" not in base + names = F.compose_tool_notice(frozenset({F.NAMES}), True) + assert "written with spaces" in names and "`explain`" not in names + flat = F.compose_tool_notice(frozenset({F.FLAT}), True) + assert "household you describe" in flat and "`household`" in flat + concise = F.compose_tool_notice(frozenset({F.EXPLAIN, F.CONCISE}), True) + assert '`detail` set to "full"' in concise + + +def test_arms_name_their_features(): + assert F.arm_features("pe") == frozenset() + assert F.arm_features("pe2") == F.V2_FEATURES + assert F.arm_features("pe3") == frozenset(F.TOOL_FEATURES) + assert len(F.TOOL_FEATURES) == 7 and F.LABELS in F.arm_features("pe3") + assert F.arm_features("pe:flat+names") == frozenset({"flat", "names"}) + assert F.arm_features("none") is None and F.arm_features("web") is None + assert F.arm_suffix("pe:flat+names") == "+pe-tool[names+flat]" + with pytest.raises(SystemExit, match="unknown tool features"): + F.arm_features("pe:names+telepathy") + with pytest.raises(SystemExit, match="concise feature needs explain"): + F.arm_features("pe:concise") + profile = F.ModelProfile("m", "gpt-x", "chat", "tool", arm="pe:names") + assert profile.uses_calculator and profile.tool_features == frozenset({"names"}) + assert profile.tool_version == 1 + assert "written with spaces" in profile.notice + assert profile.arm2_label == "m+pe-tool[names]" + + +def test_names_alone_reads_labels_but_places_and_redirects_nothing(): + situation = _situation() + situation["people"]["head"]["Employment income"] = 30000 + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.NAMES) + assert error is None + assert request["situation"]["people"]["head"]["employment_income"] == { + "2026": 30000 + } + misplaced = _situation() + misplaced["people"]["head"]["qualified_passenger_vehicle_loan_interest"] = 5 + request, error = _norm({"situation": misplaced, "variables": ["snap"]}, F.NAMES) + assert request is None and any( + "set it under households" in p for p in error["problems"] + ) + realized = _situation() + realized["people"]["head"]["traditional_401k_contributions"] = 100 + request, error = _norm({"situation": realized, "variables": ["snap"]}, F.NAMES) + assert "traditional_401k_contributions" in request["situation"]["people"]["head"] + request, error = _norm( + {"situation": _situation(), "variables": ["snap"], "explain": ["snap"]}, F.NAMES + ) + assert request is None and any( + "unknown arguments: explain" in p for p in error["problems"] + ) + + +def test_placement_alone_moves_inputs_but_reads_no_labels(): + situation = _situation() + situation["people"]["head"]["qualified_passenger_vehicle_loan_interest"] = 5 + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.PLACEMENT) + assert error is None + assert request["situation"]["households"]["h"][ + "qualified_passenger_vehicle_loan_interest" + ] == {"2026": 5} + labeled = _situation() + labeled["people"]["head"]["employment income"] = 1 + request, error = _norm({"situation": labeled, "variables": ["snap"]}, F.PLACEMENT) + assert request is None + + +def test_overrides_alone_redirects_and_flags(): + situation = _situation() + situation["people"]["head"]["traditional_401k_contributions"] = 100 + situation["spm_units"]["spm"]["snap"] = 5 + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.OVERRIDES) + assert error is None + head = request["situation"]["people"]["head"] + assert head == { + "age": {"2026": 40}, + "traditional_401k_contributions_desired": {"2026": 100}, + } + assert any("snap is computed" in n for n in request["notes"]) + + +def test_concise_takes_a_detail_and_defaults_to_brief(): + request, error = _norm( + {"situation": _situation(), "variables": ["snap"], "explain": ["snap"]}, + F.EXPLAIN, + F.CONCISE, + ) + assert error is None and request["detail"] == "brief" + request, _ = _norm( + { + "situation": _situation(), + "variables": ["snap"], + "explain": ["snap"], + "detail": "full", + }, + F.EXPLAIN, + F.CONCISE, + ) + assert request["detail"] == "full" + request, error = _norm( + {"situation": _situation(), "variables": ["snap"], "detail": "short"}, + F.EXPLAIN, + F.CONCISE, + ) + assert request is None + request, _ = _norm( + {"situation": _situation(), "variables": ["snap"], "explain": ["snap"]}, + F.EXPLAIN, + ) + assert "detail" not in request # the trace, as tool version 2 explains + + +def test_a_brief_explanation_keeps_the_value_its_direct_inputs_and_main_parameters(): + full = { + "steps": [ + "snap (2026) = 288", + " snap_normal_allotment (2026) = 288", + " snap_net_income (2026) = 1700", + " is_snap_eligible (2026) = True", + ], + "parameters": [{"name": f"p{i}", "value": i, "sources": []} for i in range(9)], + } + brief = F.brief_explanation(full) + assert brief["value"] == "snap (2026) = 288" + assert brief["from"] == [ + "snap_normal_allotment (2026) = 288", + "is_snap_eligible (2026) = True", + ] + assert [p["name"] for p in brief["parameters"]] == ["p0", "p1", "p2", "p3", "p4"] + assert F.brief_explanation({"error": "x"}) == {"error": "x"} + + +def test_labels_reads_the_facts_as_the_prompt_labels_them(): + situation = _situation() + situation["people"]["head"].update( + { + "gross wages and salaries": 30000, + "Has Employer-Sponsored Insurance": True, + "usual weekly hours worked": 40, + "self-employment partnership income": 5, + } + ) + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.LABELS) + assert error is None, error + head = request["situation"]["people"]["head"] + assert head["employment_income"] == {"2026": 30000} + assert head["has_esi"] == {"2026": True} + assert head["hours_worked_last_week"] == {"2026": 40} + assert head["partnership_self_employment_net_earnings"] == {"2026": 5} + # The reference also sets stated hours where the SNAP work rules read them. + assert head["weekly_hours_worked_before_lsr"] == {"2026": 40} + assert any("also set weekly_hours_worked_before_lsr" in n for n in request["notes"]) + # A name written with spaces is the names feature's, not this one's. + spaced = _situation() + spaced["people"]["head"]["employment income"] = 1 + request, error = _norm({"situation": spaced, "variables": ["snap"]}, F.LABELS) + assert request is None and "unknown variable 'employment income'" in str(error) + + +def test_labels_leaves_hours_already_set_where_the_rules_read_them(): + situation = _situation() + situation["people"]["head"].update( + {"hours_worked_last_week": 40, "weekly_hours_worked_before_lsr": 20} + ) + request, _ = _norm({"situation": situation, "variables": ["snap"]}, F.LABELS) + head = request["situation"]["people"]["head"] + assert head["weekly_hours_worked_before_lsr"] == {"2026": 20} + + +def test_pe2_reads_no_labels_and_sets_no_alias_as_it_ran(): + situation = _situation() + situation["people"]["head"]["gross wages and salaries"] = 30000 + request, error = _norm( + {"situation": situation, "variables": ["snap"]}, *F.V2_FEATURES + ) + assert request is None and "gross wages and salaries" in str(error) + situation = _situation() + situation["people"]["head"]["hours_worked_last_week"] = 40 + request, _ = _norm({"situation": situation, "variables": ["snap"]}, *F.V2_FEATURES) + assert ( + "weekly_hours_worked_before_lsr" not in request["situation"]["people"]["head"] + ) + + +def test_only_pe2_says_the_facts_carry_variable_names(): + claim = "labeled with its variable names" + assert claim in F.TOOL_NOTICE_V2 + for size in range(1, len(F.TOOL_FEATURES) + 1): + for features in itertools.combinations(F.TOOL_FEATURES, size): + features = frozenset(features) + if F.CONCISE in features and F.EXPLAIN not in features: + continue + notice = F.compose_tool_notice(features, True) + assert (claim in notice) == (features == F.V2_FEATURES), features + assert notice.endswith("as instructed above.") + assert ("names it read" in notice) == bool(features & {F.NAMES, F.LABELS}) + + +def test_every_label_a_prompt_shows_reads_back_to_its_engine_name(): + from policybench.prompts import _format_input_line + from policybench.scenarios import PE_DROPPED_INPUTS, PE_INPUT_RENAMES + + labels = F.prompt_labels() + fields = {spec.output_name: spec.value_type for spec in F._promptable_specs()} + fields |= {"age": "int", "employment_income": "float"} + dropped = {f for names in PE_DROPPED_INPUTS.values() for f in names} + for field, value_type in {**fields, **dict.fromkeys(dropped, "float")}.items(): + for label in F.rendered_labels(field, value_type): + want = (PE_INPUT_RENAMES.get(field, field), field in dropped) + assert want in labels[F._label_key(label)], (field, label) + # Fallback labels (the name with spaces) are the prompt's too. + assert ("first_home_mortgage_balance", False) in labels[ + F._label_key("first home mortgage balance") + ] + # Special renderers: the Marketplace plan line is labeled on its own. + line = _format_input_line("selected_marketplace_plan_benchmark_ratio", 0.8) + assert line.startswith("- selected Marketplace plan: ") + assert ("selected_marketplace_plan_benchmark_ratio", False) in labels[ + F._label_key("selected Marketplace plan") + ] + # The v2-pilot prompt's relabelings, as that prompt writes them. + for field, label in ( + ("spm_unit_energy_subsidy", "home energy assistance received"), + ( + "employer_sponsored_insurance_premiums", + "employer-paid health insurance premiums (paid by the employer, " + "not by the household)", + ), + ): + assert label in F.rendered_labels(field) + assert (field, False) in labels[F._label_key(label)] + assert ("state_code", False) in labels[F._label_key("state")] + assert ("first_home_mortgage_interest", True) in labels[ + F._label_key("first home mortgage interest") + ] + assert ("roth_401k_contributions_desired", False) in labels[ + F._label_key("Roth 401k contributions desired") + ] + assert ("partnership_s_corp_income", False) in labels[ + F._label_key("partnership or S-corp income") + ] + + +def test_labels_reads_no_label_that_names_two_of_the_engines_inputs(monkeypatch): + two = frozenset({("has_esi", False), ("age", False)}) + monkeypatch.setattr(F, "prompt_labels", lambda: {F._label_key("covered"): two}) + situation = _situation() + situation["people"]["head"]["covered"] = True + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.LABELS) + assert request is None and "unknown variable 'covered'" in str(error) + one = frozenset({("has_esi", False), ("uk_only_input", False)}) + monkeypatch.setattr(F, "prompt_labels", lambda: {F._label_key("covered"): one}) + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.LABELS) + assert error is None and request["situation"]["people"]["head"]["has_esi"] + + +def test_labels_ignores_a_fact_the_reference_leaves_out(): + situation = _situation() + situation["tax_units"] = { + "t": {"members": ["head", "kid"], "first home mortgage interest": 900} + } + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.LABELS) + assert error is None, error + assert request["situation"]["tax_units"]["t"] == {"members": ["head", "kid"]} + assert any("leaves it out too" in n for n in request["notes"]) + request, error = _norm({"situation": situation, "variables": ["snap"]}, F.NAMES) + assert request is None and "first home mortgage interest" in str(error) + + +def _flat(**overrides): + household = { + "state": "LA", + "people": [ + { + "name": "Head", + "role": "head", + "facts": {"age": 40, "employment_income": 30000}, + }, + {"name": "Spouse", "role": "spouse", "facts": {"age": 38}}, + {"name": "Kid", "role": "dependent", "facts": {"age": 6}}, + ], + "household_facts": { + "spm_unit_energy_subsidy": 900, + "tax_unit_childcare_expenses": 1200, + "qualified_passenger_vehicle_loan_interest": 300, + }, + } + household.update(overrides) + return {"household": json.dumps(household), "variables": ["snap"]} + + +def test_flat_builds_the_benchmarks_units(): + request, error = _norm(_flat(), F.FLAT) + assert error is None, error + s = request["situation"] + assert s["marital_units"] == { + "marital_unit_1": {"members": ["Head", "Spouse"]}, + "marital_unit_2": {"members": ["Kid"]}, + } + for group, unit in ( + ("tax_units", "tax_unit"), + ("spm_units", "spm_unit"), + ("families", "family"), + ): + assert s[group][unit]["members"] == ["Head", "Spouse", "Kid"] + assert s["households"]["household"]["state_code"] == {"2026": "LA"} + assert s["spm_units"]["spm_unit"]["spm_unit_energy_subsidy"] == {"2026": 900} + assert s["tax_units"]["tax_unit"]["tax_unit_childcare_expenses"] == {"2026": 1200} + assert s["households"]["household"][ + "qualified_passenger_vehicle_loan_interest" + ] == {"2026": 300} + assert s["people"]["Head"]["is_tax_unit_head"] == {"2026": True} + assert s["people"]["Spouse"]["is_tax_unit_spouse"] == {"2026": True} + assert s["people"]["Kid"]["is_tax_unit_head"] == {"2026": False} + + +def test_flat_names_a_person_by_role_when_no_name_is_given(): + people = [ + {"role": "head", "facts": {"age": 40}}, + {"role": "dependent", "facts": {"age": 6}}, + {"name": "Kid", "role": "dependent", "facts": {"age": 4}}, + {"role": "dependent", "facts": {"age": 2}}, + ] + request, error = _norm(_flat(people=people), F.FLAT) + assert error is None, error + assert list(request["situation"]["people"]) == [ + "head", + "dependent_1", + "Kid", + "dependent_3", + ] + assert "people[0]: named head" in request["notes"] + people = [ + {"role": "head", "facts": {}}, + {"name": "head", "role": "dependent", "facts": {}}, + ] + request, error = _norm(_flat(people=people), F.FLAT) + assert any("listed twice" in p for p in error["problems"]) + people = [{"name": "", "role": "head", "facts": {}}] + request, error = _norm(_flat(people=people), F.FLAT) + assert any("name must be a short string" in p for p in error["problems"]) + + +def test_flat_placement_gives_a_household_of_ones_person_fact_to_that_person(): + one = [{"name": "Solo", "role": "head", "facts": {"age": 70}}] + args = _flat(people=one, household_facts={"pre_subsidy_rent": 9000}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert error is None, error + solo = request["situation"]["people"]["Solo"] + assert solo["pre_subsidy_rent"] == {"2026": 9000} + assert "pre_subsidy_rent" not in request["situation"]["spm_units"]["spm_unit"] + assert any("moved pre_subsidy_rent" in n for n in request["notes"]) + one[0]["facts"]["pre_subsidy_rent"] = 100 + conflict = _flat(people=one, household_facts={"pre_subsidy_rent": 9000}) + request, error = _norm(conflict, F.FLAT, F.PLACEMENT) + assert any("different value" in p for p in error["problems"]) + # Two people: whose rent it is is not said, so it is refused. + request, error = _norm( + _flat(household_facts={"pre_subsidy_rent": 9000}), F.FLAT, F.PLACEMENT + ) + assert any("whose facts list it" in p for p in error["problems"]) + # Without placement, even a household of one is refused. + request, error = _norm(args, F.FLAT) + assert any("whose facts list it" in p for p in error["problems"]) + + +def test_flat_refuses_facts_given_beside_the_person(): + people = [{"role": "head", "age": 40, "employment_income": 30000}] + request, error = _norm(_flat(people=people), F.FLAT) + assert request is None + assert any( + "takes only name, role and facts; give age, employment_income under facts" in p + for p in error["problems"] + ) + + +def test_flat_reads_a_fact_written_two_ways_once_whatever_the_order(): + for facts in ( + {"spm_unit_energy_subsidy": 900, "spm unit energy subsidy": 800}, + {"spm unit energy subsidy": 800, "spm_unit_energy_subsidy": 900}, + ): + request, error = _norm(_flat(household_facts=facts), F.FLAT, F.NAMES) + assert request is None + assert any("set twice with different values" in p for p in error["problems"]) + same = {"spm_unit_energy_subsidy": 900, "spm unit energy subsidy": {"2026": 900}} + request, error = _norm(_flat(household_facts=same), F.FLAT, F.NAMES) + assert error is None, error + + +def test_flat_roles_set_the_relationship_inputs_and_refuse_a_contradiction(): + for written in ("is_tax_unit_head", "is tax unit head"): + people = [{"name": "A", "role": "head", "facts": {written: False}}] + request, error = _norm(_flat(people=people), F.FLAT, F.NAMES) + assert request is None, written + assert any("contradicts the role head" in p for p in error["problems"]) + people = [{"name": "A", "role": "head", "facts": {written: True}}] + request, error = _norm(_flat(people=people), F.FLAT, F.NAMES) + assert error is None, (written, error) + head = request["situation"]["people"]["A"] + assert head["is_tax_unit_head"] == {"2026": True} + assert head["is_tax_unit_spouse"] == {"2026": False} + + +def test_flat_placement_compares_values_once_normalized(): + one = [ + {"name": "Solo", "role": "head", "facts": {"pre_subsidy_rent": {"2026": 9000}}} + ] + args = _flat(people=one, household_facts={"pre_subsidy_rent": 9000}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert error is None, error + assert request["situation"]["people"]["Solo"]["pre_subsidy_rent"] == {"2026": 9000} + + +def test_the_descriptions_promise_only_what_the_features_do(): + flat = F.calculate_tool_schema(features=frozenset({F.FLAT}))["function"] + assert '"employment_income": 30000' in flat["description"] + example = [ + {"role": "head", "facts": {"age": 40, "employment_income": 30000}}, + ] + request, error = _norm(_flat(people=example), F.FLAT) + assert error is None, error + entity_error = "a variable set on the wrong entity" + placed = F.calculate_tool_schema(features=frozenset({F.PLACEMENT}))["function"] + assert entity_error not in placed["description"] + assert "moved to the one unit holding its people" in placed["description"] + named = F.calculate_tool_schema(features=frozenset({F.NAMES}))["function"] + assert entity_error in named["description"] + concise = F.calculate_tool_schema(features=frozenset({F.EXPLAIN, F.CONCISE})) + detail = concise["function"]["parameters"]["properties"]["detail"]["description"] + assert f"up to {F.EXPLAIN_MAX_LINES} steps" in detail + for features in (frozenset({F.EXPLAIN, F.CONCISE}), frozenset(F.TOOL_FEATURES)): + assert "every step" not in F.compose_tool_notice(features, True) + + +def test_flat_refuses_what_it_cannot_place(): + people = [ + {"name": "A", "role": "head", "facts": {}}, + {"name": "B", "role": "head", "facts": {}}, + ] + request, error = _norm(_flat(people=people), F.FLAT) + assert any("exactly one head" in p for p in error["problems"]) + request, error = _norm(_flat(household_facts={"age": 3}), F.FLAT) + assert any("person's fact" in p for p in error["problems"]) + people = [{"name": "A", "role": "boss", "facts": {}}] + request, error = _norm(_flat(people=people), F.FLAT) + assert any("role must be one of" in p for p in error["problems"]) + request, error = _norm({"situation": _situation(), "variables": ["snap"]}, F.FLAT) + assert any("unknown arguments: situation" in p for p in error["problems"]) + request, error = _norm(_flat(), F.NAMES) + assert any("unknown arguments: household" in p for p in error["problems"]) + + +def test_flat_with_names_reads_labeled_household_facts(): + request, error = _norm( + _flat(household_facts={"spm unit energy subsidy": 900}), F.FLAT, F.NAMES + ) + assert error is None + spm_unit = request["situation"]["spm_units"]["spm_unit"] + assert spm_unit["spm_unit_energy_subsidy"] == {"2026": 900} + assert any("as spm_unit_energy_subsidy" in n for n in request["notes"]) + + +def test_the_schema_offers_each_features_arguments(): + flat = F.calculate_tool_schema(features=frozenset({F.FLAT}))["function"] + props = flat["parameters"]["properties"] + assert "household" in props and "situation" not in props + assert flat["parameters"]["required"] == ["household", "variables"] + concise = F.calculate_tool_schema(features=frozenset({F.EXPLAIN, F.CONCISE})) + props = concise["function"]["parameters"]["properties"] + assert "explain" in props and props["detail"]["enum"] == ["brief", "full"] + assert F.calculate_tool_schema(features=F.V2_FEATURES) == F.calculate_tool_schema(2) + assert F.calculate_tool_schema(features=frozenset()) == F.calculate_tool_schema(1) + + +def _echo_worker(conn, fix_path, reform): + conn.send(("ready", None)) + while True: + args = conn.recv() + if args is None: + return + conn.send(("ok", {"received": args})) + + +def test_the_pool_tells_its_workers_the_features_in_a_frame(): + pool = F.ToolPool( + size=1, + timeout=5.0, + worker_target=_echo_worker, + tool_features=frozenset({F.FLAT, F.NAMES}), + ) + try: + result, _ = pool.call({"x": 1}) + assert result == {"received": ("calculate", ("flat", "names"), {"x": 1})} + finally: + pool.close() + + +def test_a_model_cannot_turn_on_a_feature_its_arm_does_not_offer(): + forged = { + "__tool_features__": ["names"], + "arguments": { + "situation": {"people": {"head": {"Employment income": 12345}}}, + "variables": ["age"], + }, + } + for message in (forged, {"__tool_version__": 2, "arguments": forged["arguments"]}): + request, error = F.worker_request(message, REGISTRY) + assert request is None + assert "unknown arguments: " in error["problems"][0] + request, error = F.worker_request( + F.tool_call_frame(frozenset({F.NAMES}), forged["arguments"]), REGISTRY + ) + assert error is None + assert request["situation"]["people"]["head"]["employment_income"] == { + "2026": 12345 + } + for names in (("telepathy",), ("concise",), ({},), ["names"]): + request, error = F.worker_request(("calculate", names, {}), REGISTRY) + assert request is None and error["error"] == "configuration", names + + +def _profile(arm): + return F.ModelProfile("m", "gpt-x", "chat", "tool", arm=arm) + + +def _record(profile, **changes): + record = { + "scenario_id": "mt_000", + "tool_notice_sha256": __import__("hashlib") + .sha256(profile.notice.encode()) + .hexdigest(), + **F.profile_treatment(profile), + } + record.update(changes) + return record + + +def test_a_resume_refuses_households_run_under_another_treatment(): + pe3 = _profile("pe3") + assert F.record_treatment_problem(_record(pe3), pe3) is None + six = sorted(set(F.TOOL_FEATURES) - {F.LABELS}) + assert "tool features" in F.record_treatment_problem( + _record(pe3, tool_features=six), pe3 + ) + assert "another tool notice" in F.record_treatment_problem( + _record(pe3, tool_notice_sha256="0" * 64), pe3 + ) + assert "calculator schema" in F.record_treatment_problem( + _record(pe3, calculate_tool_sha256="0" * 64), pe3 + ) + old = _record(pe3) + del old["tool_features"] + assert "no recorded tool features" in F.record_treatment_problem(old, pe3) + # Arms whose treatment never changed accept records from before the fields. + for arm in ("pe", "pe2", "web+pe", "none", "web"): + profile = _profile(arm) + legacy = _record(profile) + for key in ("tool_features", "calculate_tool_sha256", "tool_notice_sha256"): + legacy.pop(key) + assert F.record_treatment_problem(legacy, profile) is None, arm + pe2 = _profile("pe2") + assert "tool features" in F.record_treatment_problem(_record(_profile("pe")), pe2) + + +def test_diagnostics_read_flat_people_by_their_default_names(): + household = { + "people": [ + {"role": "head", "facts": {}}, + {"role": "spouse", "facts": {}}, + {"name": "Kid", "role": "dependent", "facts": {}}, + {"role": "dependent", "facts": {}}, + ] + } + names = F._person_names({"household": json.dumps(household)}) + assert names == ["head", "spouse", "Kid", "dependent_2"] + + +@pytest.mark.slow +def test_a_household_described_flat_computes_what_the_benchmarks_own_does(): + """Differential: on the installed engine, each frozen-board household + described in the flat format computes every board output exactly as the + benchmark's own situation (Scenario.to_pe_household) does.""" + import household_set as hs + import pandas as pd + from policyengine_us import Simulation + + from policybench.ground_truth import _extract_person_value, _pe_variable_for_output + + system = F.build_tool_system(reform=True) + registry = F.build_registry(system) + reference = pd.read_csv(hs.FROZEN_BOARD / "reference_outputs.csv") + variables = reference.groupby("scenario_id").variable.agg(list) + relations = {"is_tax_unit_head", "is_tax_unit_spouse"} + checked = 0 + for scenario in hs.load_scenarios(hs.FROZEN_BOARD)[:8]: + situation = hs.situation_for(scenario) + couple = scenario.marital_couple() or () + head = couple[0] if couple else scenario.adults[0].name + people = [ + { + "name": name, + "role": "head" + if name == head + else ("spouse" if name in couple else "dependent"), + "facts": {k: v for k, v in facts.items() if k not in relations}, + } + for name, facts in situation["people"].items() + ] + household_facts = {} + for group in ("tax_units", "spm_units", "families", "households"): + for unit in situation[group].values(): + household_facts.update( + { + k: v + for k, v in unit.items() + if k not in ("members", "state_code") + } + ) + household = { + "state": scenario.state, + "people": people, + "household_facts": household_facts, + } + request, error = F.normalize_request( + {"household": json.dumps(household), "variables": ["snap"]}, + registry, + features=frozenset({F.FLAT}), + ) + assert error is None, (scenario.id, error) + own = Simulation(tax_benefit_system=system, situation=situation) + flat = Simulation(tax_benefit_system=system, situation=request["situation"]) + for variable in variables[scenario.id]: + name = _pe_variable_for_output(variable, "us") + values = [ + float( + _extract_person_value( + simulation.calculate(name, 2026), scenario, variable + ) + ) + for simulation in (own, flat) + ] + assert values[0] == values[1], (scenario.id, variable, values) + checked += 1 + assert checked > 80 + + +@pytest.mark.slow +def test_a_household_described_from_its_prompt_computes_the_same(): + """Differential, with the labels feature alone: each of 30 frozen-board + households described flat from the facts its v2-pilot prompt shows, each + under the label the prompt shows it under (dropped facts included), with + the state and roles the prompt gives, computes every board output as the + benchmark's own situation does. The take-up conventions, which the prompt + states in words, are set under their engine names.""" + import household_set as hs + import pandas as pd + from policyengine_us import Simulation + + from policybench.ground_truth import _extract_person_value, _pe_variable_for_output + from policybench.scenarios import ( + DEFAULT_TAKEUP_INPUTS, + is_excluded_prompt_input_name, + ) + + system = F.build_tool_system(reform=True) + registry = F.build_registry(system) + reference = pd.read_csv(hs.FROZEN_BOARD / "reference_outputs.csv") + variables = reference.groupby("scenario_id").variable.agg(list) + shown = set() + + def label(field, value): + value_type = ( + "bool" + if isinstance(value, bool) + else ("str" if isinstance(value, str) else "float") + ) + written = F.rendered_labels(field, value_type)[-1] + shown.add((field, written)) + return written + + def prompt_facts(inputs): + return { + label(k, v): v + for k, v in inputs.items() + if not is_excluded_prompt_input_name(k) + } + + checked = 0 + for scenario in hs.load_scenarios(hs.FROZEN_BOARD)[:30]: + couple = scenario.marital_couple() or () + head = couple[0] if couple else scenario.adults[0].name + people = [] + for person in scenario.all_people: + facts = {"age": person.age} + if person.name.startswith("adult") or abs(person.employment_income) > 1e-6: + facts[label("employment_income", 1.0)] = person.employment_income + facts |= prompt_facts(person.inputs) + facts |= DEFAULT_TAKEUP_INPUTS["person"] + role = ( + "head" + if person.name == head + else ("spouse" if person.name in couple else "dependent") + ) + people.append({"name": person.name, "role": role, "facts": facts}) + household_facts = {} + for inputs in ( + scenario.tax_unit_inputs, + scenario.spm_unit_inputs, + scenario.household_inputs, + ): + household_facts |= prompt_facts(inputs) + for entity in ("tax_unit", "spm_unit", "household"): + household_facts |= DEFAULT_TAKEUP_INPUTS[entity] + household = { + "state": scenario.state, + "people": people, + "household_facts": household_facts, + } + request, error = F.normalize_request( + {"household": json.dumps(household), "variables": ["snap"]}, + registry, + features=frozenset({F.FLAT, F.LABELS}), + ) + assert error is None, (scenario.id, error) + own = Simulation( + tax_benefit_system=system, situation=hs.situation_for(scenario) + ) + flat = Simulation(tax_benefit_system=system, situation=request["situation"]) + for variable in variables[scenario.id]: + name = _pe_variable_for_output(variable, "us") + values = [ + float( + _extract_person_value( + simulation.calculate(name, 2026), scenario, variable + ) + ) + for simulation in (own, flat) + ] + assert values[0] == values[1], (scenario.id, variable, values) + checked += 1 + worded = {w for f, w in shown if w != f.replace("_", " ")} + assert checked > 300 and len(worded) > 10, (checked, sorted(worded)) + # No label a US prompt can show names two of this engine's inputs. + ambiguous = { + key: targets + for key, targets in F.prompt_labels().items() + if len({e for e, dropped in targets if not dropped and e in registry}) > 1 + } + assert not ambiguous, ambiguous + + +def test_flat_reports_a_malformed_repeat_of_a_fact_it_drops(): + # Review r2: a repeat that fails validation was dropped without a word. + one = [{"name": "Solo", "role": "head", "facts": {"employment_income": 30000}}] + for bad in ("40000", [40000], {"26": 40000}, {}): + args = _flat(people=one, household_facts={"employment_income": bad}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert request is None, bad + assert any("employment_income" in m for m in error["problems"]), (bad, error) + # An equal repeat, scalar or by period, still merges quietly; a different + # one is still refused. + for same in (30000, {"2026": 30000}): + args = _flat(people=one, household_facts={"employment_income": same}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert error is None, (same, error) + args = _flat(people=one, household_facts={"employment_income": 40000}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert request is None + assert any("set twice with different values" in m for m in error["problems"]) + + +def test_a_harness_error_record_resumes_under_its_own_arm(): + # Review r2: the fallback record carried no treatment, so one harness + # failure blocked every custom-arm resume. + for arm in ("pe3", "pe:labels+flat", "pe", "pe2", "web+pe", "none", "web"): + profile = _profile(arm) + record = F.harness_error_record("mt_000", ["snap"], profile, RuntimeError("x")) + assert record["status"] in F.RERUNNABLE_STATUSES + assert F.record_treatment_problem(record, profile) is None, arm + assert record["predictions"] == {"snap": None} + assert record["totals"]["cost_usd"] == 0 + + +def test_a_rerunnable_record_from_another_treatment_is_refused_too(): + # Review r3: a rerun is not sure to replace a rerunnable record (a budget + # stop, lost workers, a narrower --scenarios), so every record is checked. + pe3 = _profile("pe3") + stale = _record(pe3, tool_features=["names"]) + records = { + "mt_000": dict(stale, status="complete"), + "mt_001": dict(stale, scenario_id="mt_001", status="provider_error"), + "mt_002": F.harness_error_record("mt_002", ["snap"], pe3, RuntimeError("x")), + } + problems = F.treatment_problems(records, pe3) + assert [p.split()[0] for p in problems] == ["mt_000", "mt_001"] + + +SIX = ("concise", "explain", "flat", "names", "overrides", "placement") + + +def _scored_record(sid, profile, treatment_of=None, status="complete", **changes): + """A household record as command_run writes it, with the treatment of + `treatment_of` (another arm's profile) when given.""" + import hashlib + + board = pilot.load_board() + reference = board["reference"].set_index(["scenario_id", "variable"])["value"] + variables = board["variables"][sid] + source = treatment_of or profile + record = { + "scenario_id": sid, + "model": profile.name, + "arm": profile.arm, + "prompt_version": profile.prompt_version, + "benchmark_search_rule": profile.benchmark_search_rule, + "tool_notice_sha256": hashlib.sha256(source.notice.encode()).hexdigest(), + **F.profile_treatment(source), + "variables": variables, + "predictions": {v: float(reference[(sid, v)]) for v in variables}, + "explanations": {v: "x" for v in variables}, + "status": status, + "turns": [{"total_cost_usd": 1.25, "calls": []}], + "cap_hit": False, + "wall_seconds": 1.0, + } + record.update(changes) + record["totals"] = F.household_totals(record) + return record + + +def _write(directory, *records): + (directory / "households").mkdir(parents=True, exist_ok=True) + for record in records: + path = directory / "households" / f"{record['scenario_id']}.json" + path.write_text(json.dumps(record)) + + +def test_a_resume_refuses_an_old_treatments_provider_error_before_running( + tmp_path, monkeypatch +): + # The review's case: a pe3 folder holding a provider_error written under + # six features, with the budget already spent. Before the fix the resume + # passed, `one` stopped on the budget without replacing it, and scoring + # read its predictions as pe3's. + import argparse + + pe3 = F.model_profile(F.MODEL_NAME, "pe3") + six = F.model_profile(F.MODEL_NAME, "pe:" + "+".join(SIX)) + _write(tmp_path, _scored_record("scenario_000", pe3, six, status="provider_error")) + monkeypatch.setattr(F, "_require_engine", lambda board: None) + monkeypatch.setenv(F.provider_key(pe3.model_id), "test-key") + + def no_pool(**kwargs): + raise AssertionError("the run got as far as starting the calculator") + + monkeypatch.setattr(F, "ToolPool", no_pool) + args = argparse.Namespace( + out_dir=str(tmp_path), + board=None, + model=F.MODEL_NAME, + arm="pe3", + prompt="v1", + forbid_benchmark_search=False, + scenarios=None, + limit=None, + workers=1, + budget_usd=1.0, + reserve_usd=0.0, + ) + with pytest.raises(SystemExit, match="another treatment"): + F.command_run(args) + assert not list(tmp_path.glob("manifest-*.json")) + + +def test_score_refuses_a_folder_mixing_treatments(tmp_path): + import argparse + + pe3 = F.model_profile(F.MODEL_NAME, "pe3") + six = F.model_profile(F.MODEL_NAME, "pe:" + "+".join(SIX)) + _write( + tmp_path, + _scored_record("scenario_000", pe3), + _scored_record("scenario_001", pe3, six, status="provider_error"), + ) + args = argparse.Namespace(out_dir=str(tmp_path), complete_only=False) + with pytest.raises(SystemExit, match="another treatment"): + F.command_score(args) + assert not (tmp_path / "score.json").exists() + # The same folder without the stray record scores. + (tmp_path / "households" / "scenario_001.json").unlink() + F.command_score(args) + assert json.loads((tmp_path / "score.json").read_text())["households"] == 1 + + +def test_score_rebuilds_notices_on_the_boards_engine_and_restores_it( + tmp_path, monkeypatch +): + # A household set's records state its engine (2.60.4) in their notices and + # schema, so a check on the reference engine's text would refuse them all. + import argparse + + monkeypatch.setattr(F, "board_engine", lambda snapshot=None: "2.60.4") + pe3 = F.model_profile(F.MODEL_NAME, "pe3") + with F.stating_engine("2.60.4"): + current = _scored_record("scenario_000", pe3) + _write(tmp_path, current) + args = argparse.Namespace(out_dir=str(tmp_path), complete_only=False) + F.command_score(args) + assert F._STATED_ENGINE["version"] == F.REFERENCE_ENGINE + # Beside it, a record sent under the reference engine's text mixes two. + _write(tmp_path, _scored_record("scenario_001", pe3)) + with pytest.raises(SystemExit, match="another tool notice"): + F.command_score(args) + assert F._STATED_ENGINE["version"] == F.REFERENCE_ENGINE + + +def test_score_takes_a_folder_written_wholly_before_the_texts_stated_the_engine( + tmp_path, monkeypatch, capsys +): + # Test batch 0's frontier calculator arms (b6d835a8) were told 2.41.0 while + # the board and calculator ran 2.60.4: one treatment, scored as it ran. A + # resume into it is still refused, since new records would say 2.60.4. + import argparse + + monkeypatch.setattr(F, "board_engine", lambda snapshot=None: "2.60.4") + pe = F.model_profile(F.MODEL_NAME, "pe") + _write( + tmp_path, _scored_record("scenario_000", pe), _scored_record("scenario_001", pe) + ) + F.command_score(argparse.Namespace(out_dir=str(tmp_path), complete_only=False)) + assert "name policyengine-us 2.41.0" in capsys.readouterr().out + assert json.loads((tmp_path / "score.json").read_text())["households"] == 2 + with F.stating_engine("2.60.4"): + assert len(F.treatment_problems(F.load_records(tmp_path), pe)) == 2 + + +def test_flat_reports_a_malformed_first_value_once_and_not_as_a_conflict(): + one = [{"name": "Solo", "role": "head", "facts": {"employment_income": "30000"}}] + args = _flat(people=one, household_facts={"employment_income": 40000}) + request, error = _norm(args, F.FLAT, F.PLACEMENT) + assert request is None + assert not any("set twice" in m for m in error["problems"]), error + assert sum("employment_income" in m for m in error["problems"]) == 1, error