From 6e15d713b0a9a095f5a00d63a331abbaadc5966c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 02:48:28 -0400 Subject: [PATCH 01/48] Confirmatory runner: lanes, one engine pool per model, admission, the ceiling scripts/confirmatory_run.py runs the frozen pre-registration's run procedure on the treatment harness (6b00a6f0's tool_pilot_argon.py, unchanged) and writes the run records the confirmatory report checks: - two lanes per model, households alternating in draw order, each household's arms in its drawn order, then reruns in passes; - one ToolPool per model with no features of its own, each call framed with its arm's features; - admission against each model's cap and reserve, waiting while a pass's reruns are undecided; - the hard ceiling over every model's spend, through the harness's Spend; - the failure classes and actions, with operator pauses, harness and inconclusive stops, and replaced attempts saved before their reruns; - the 30-second host log and the guard's and power log's lines. Tests follow. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 1388 +++++++++++++++++++++++++++++++++++ 1 file changed, 1388 insertions(+) create mode 100644 scripts/confirmatory_run.py diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py new file mode 100644 index 00000000..cc61b52b --- /dev/null +++ b/scripts/confirmatory_run.py @@ -0,0 +1,1388 @@ +"""The confirmatory calculator-interface run. + +Usage: + python scripts/confirmatory_run.py run DESIGN [--workers N] [--dry-run] + python scripts/confirmatory_run.py resume DESIGN MODEL [--note TEXT] + +Runs the frozen pre-registration's "Run procedure" and writes its "Run +records", which the confirmatory report checks before it scores anything. +DESIGN is the JSON file written at the freeze, the file the report reads. + +- **Lanes.** Each model has two lanes. Households alternate between them in + draw order, the first drawn to lane 1. A lane runs one household's arms one + after another, in the order drawn for that household and model (the stream + ``order/``), then its reruns in passes (second attempts before + third, each pass in the household's order), and only then decides its next + household. The six models run at once. +- **One engine pool per model**, started once, with no features of its own. + Each call carries its arm's features in a call frame (``tool_call_frame``), + the message a pool started with those features would send. +- **Admission.** A lane admits its next household when the spend of the + model's finished households (every attempt, replaced ones included) plus + the reserve for each household under way, this one included, is at most + the model's cap plus $1e-9. A household is finished when its last attempt + has ended. After one refusal every later decision is a refusal. A decision + waits while an operator pause holds, and while any of the model's + households has ended a pass whose reruns are undecided, so that whether it + has finished is known. +- **Spend.** No conversation is stopped for spend, except at the hard + ceiling: the overall cap plus $25, over the spend before the calculator + stage and every model's spend since. At the crossing, no attempt starts, + the conversations under way end ``budget_stop`` at their next turn, and + every model's run log gets a ``ceiling`` event at the crossing's time. +- **Failures.** Each attempt's class follows the pre-registration's table, in + the precedence the report's ``classify`` uses, from its record and the host + evidence for its start and end; its action follows from its class and its + attempt number. An operator failure pauses the model's lanes until + ``resume``. A harness failure stops them, and so does one inconclusive + error that ends runs in more than five of the model's households. A + replaced attempt is saved, before its rerun, as + ``households/attempts/-.json``. +- **The host log.** Every 30 seconds: the time, the load average and the + kernel's memory-pressure level. At the end, the guard's and the power log's + lines from a day before the booking to its end. An attempt's class waits + for its host evidence to be complete: a sample more than 60 seconds after + the pass's last end, or no sample for more than 90 seconds. + +The records' format is the confirmatory report's (its module docstring). The +report recomputes every class, action, order and admission decision from the +records, with its own copy of these rules; the study's differential test runs +this runner on fakes and checks that its records pass the report. + +The runner runs the treatment harness's code. It refuses to start unless its +``tool_pilot_argon.py``, ``policybench`` package and fix module are the +treatment commit's (6b00a6f0), its tree is clean at the design's run commit, +the machine is on AC power with the lid open, and the runs folder holds no +records: a run is never resumed. An interrupt (SIGINT or SIGTERM) stops every +model's lanes with a stop event and lets the conversations under way end; a +second interrupt exits at once. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import shutil +import signal +import subprocess +import sys +import threading +import time +import traceback +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Callable +from zoneinfo import ZoneInfo + +import numpy as np + +# The harness and policybench come from this checkout, whatever else is on the +# path: the preflight refuses any other. +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT), str(ROOT / "scripts")] + +import tool_pilot_argon as pilot # noqa: E402 + +TREATMENT_COMMIT = "6b00a6f0c4997e73b9968b412b57646957cad631" +# The treatment's code, which must be the treatment commit's. +TREATMENT_PATHS = ( + "scripts/tool_pilot_argon.py", + "policybench", + "reference_audit/2026-09-28/fixes", +) +PROMPT_VERSION = "v2-pilot" +CHEAP = ("claude-haiku-5.5", "gpt-6-luna", "gemini-3.5-flash-lite") +FRONTIER = ("gpt-6-sol", "claude-opus-5.5", "gemini-4-argon") +ARMS = ( + "pe", + "pe3", + "pe3-b", + "loo-labels", + "loo-names", + "loo-flat", + "loo-labels-flat", +) +FRONTIER_ARMS = ("pe", "pe3") +# The features each label drops from pe3; None for the tool v1 arm. +DROPPED = { + "pe": None, + "pe3": (), + "pe3-b": (), + "loo-labels": ("labels",), + "loo-names": ("names",), + "loo-flat": ("flat",), + "loo-labels-flat": ("labels", "flat"), +} +HOUSEHOLDS = 120 +MAX_RERUNS = 2 +INCONCLUSIVE_STOP = 5 # households of a model one inconclusive error may end +PRESSURE_WINDOW = 60.0 # seconds +HOST_LOG_GAP = 90.0 # seconds +SAMPLE_SECONDS = 30.0 +CEILING_MARGIN = 25.0 # dollars above the overall cap +ADMISSION_SLACK = 1e-9 # dollars +EXCERPT_LEAD = 86_400.0 # host logs are kept from a day before the booking +WAIT_SECONDS = 1.0 # how often a waiting lane looks again + +# The pre-registration's error classes (its "Failures" table). +TRANSPORT = frozenset( + { + "APIConnectionError", + "APIError", + "APIResponseValidationError", + "BadGatewayError", + "InternalServerError", + "RateLimitError", + "ServiceUnavailableError", + "Timeout", + "RequestWallTimeoutError", + } +) +REQUEST = frozenset( + { + "BadRequestError", + "ContextWindowExceededError", + "InvalidRequestError", + "UnprocessableEntityError", + "UnsupportedParamsError", + } +) +ACCOUNT = frozenset({"AuthenticationError", "PermissionDeniedError", "NotFoundError"}) +HOST_ENGINE = frozenset({"MemoryError", "OSError"}) +RERUN = frozenset({"provider", "host", "none", "operator"}) +FINISHED = frozenset({"complete", "incomplete", "turn_limit"}) +# Classes decided without the host evidence: they come before host trouble. +EARLY = frozenset({"operator", "harness", "ceiling", "none"}) + +REQUIRED = ( + "preregistration", + "frozen_sha256", + "runs", + "stage", + "board", + "engine", + "eligible_pool", + "draw_order", + "arms", + "frontier_arms", + "treatment_harness", + "run_commit", + "caps", + "reserves", + "overall_cap", + "spent_before", +) + + +# ---------------------------------------------------------------- streams + + +def master_seed(frozen_sha256: str) -> int: + if not re.fullmatch(r"[0-9a-f]{64}", frozen_sha256 or ""): + raise ValueError(f"not a sha256: {frozen_sha256!r}") + return int(frozen_sha256[:16], 16) + + +def stream(master: int, name: str) -> np.random.Generator: + key = int.from_bytes(hashlib.sha256(name.encode()).digest()[:8], "big") + return np.random.default_rng(np.random.SeedSequence(master, spawn_key=(key,))) + + +def draw(master: int, pool_ids: list[str], n: int | None = None) -> list[str]: + ids = np.array(sorted(pool_ids)) + size = HOUSEHOLDS if n is None else n + return [ + str(s) for s in stream(master, "draw").choice(ids, size=size, replace=False) + ] + + +def orders( + master: int, model: str, draw_order: list[str], arms: tuple[str, ...] +) -> dict[str, list[str]]: + """Each drawn household's arm order for a model, drawn for every household + in draw order, so no order depends on which households run.""" + generator = stream(master, f"order/{model}") + return { + sid: [str(a) for a in generator.permutation(list(arms))] for sid in draw_order + } + + +# ---------------------------------------------------------------- arms + + +def arm_string(label: str) -> str: + """The harness's arm string for a design label (`TOOL_FEATURES` order).""" + dropped = DROPPED[label] + if dropped is None: + return "pe" + if not dropped: + return "pe3" + return "pe:" + "+".join(f for f in pilot.TOOL_FEATURES if f not in dropped) + + +def arm_profile(model: str, label: str) -> pilot.ModelProfile: + """The treatment harness's profile for a model and design label; refused + unless its calculator features are the ones the label registers.""" + profile = pilot.model_profile( + model, arm=arm_string(label), prompt_version=PROMPT_VERSION + ) + dropped = DROPPED[label] + want = set() if dropped is None else set(pilot.TOOL_FEATURES) - set(dropped) + if not profile.uses_calculator or set(profile.tool_features) != want: + raise ValueError( + f"{model}/{label}: the harness gives features " + f"{sorted(profile.tool_features)}, not {sorted(want)}" + ) + return profile + + +# ---------------------------------------------------------------- classes + + +def error_class(record: dict) -> str: + return str(record.get("error") or "").split(":", 1)[0].strip() + + +def completed_turns(record: dict) -> list: + return [t for t in record.get("turns") or [] if not t.get("truncated_and_retried")] + + +def call_failures(record: dict) -> tuple[list[str], list[str]]: + """(the inconclusive tool failures, the model's engine exceptions) in its + calls.""" + host, model = [], [] + for turn in record.get("turns") or []: + for call in turn.get("calls") or []: + result = call.get("result") + if not isinstance(result, dict): + continue + error, message = result.get("error"), str(result.get("message") or "") + name = message.split(":", 1)[0] + if error == "timeout": + host.append("tool timeout") + elif error == "engine" and message.startswith("tool worker failed:"): + host.append("tool worker failure") + elif error == "engine": + (host if name in HOST_ENGINE else model).append(name) + for value in (result.get("results") or {}).values(): + if isinstance(value, dict) and value.get("error"): + name = str(value["error"]).split(":", 1)[0] + (host if name in HOST_ENGINE else model).append(name) + return host, model + + +def classify(record: dict, start: float, end: float, host) -> tuple[str, str]: + """(class, detail) of one attempt. In order of precedence: operator (an + account class or insufficient_quota); harness (a request class with no + model request yet returned); ceiling (budget_stop); none (no model request + returned); host (host trouble during the attempt); provider (a transport + class); model (a request class); inconclusive (any other provider error, + any harness error, a tool timeout, a worker failure, MemoryError or + OSError); otherwise ok, or model when the engine raised on a household the + model built. `host` is None to decide only the classes before host.""" + status, error = record.get("status"), str(record.get("error") or "") + name = error_class(record) + reached = bool(completed_turns(record)) + if status == "provider_error" and ( + name in ACCOUNT or "insufficient_quota" in error + ): + return "operator", name + if status == "provider_error" and name in REQUEST and not reached: + return "harness", name + if status == "budget_stop": + return "ceiling", status + if not reached: + return "none", f"{status}: {name}" if name else str(status) + trouble = host.trouble(start, end) if host is not None else None + if trouble: + return "host", trouble + if status == "provider_error": + if name in TRANSPORT: + return "provider", name + if name in REQUEST: + return "model", name + return "inconclusive", name or "provider_error" + if status == "harness_error": + return "inconclusive", error or "harness_error" + tool, engine = call_failures(record) + if tool: + return "inconclusive", tool[0] + if status not in FINISHED: + return "inconclusive", f"status {status}" + return ("model", engine[0]) if engine else ("ok", str(status)) + + +def action(cls: str, attempt: int) -> str: + if cls == "harness": + return "stop" + if cls == "ceiling": + return "stopped" + if cls in RERUN and attempt <= MAX_RERUNS: + return "pause-rerun" if cls == "operator" else "rerun" + return "stands" + + +def attempt_cost(record: dict) -> float: + return float((record.get("totals") or {}).get("cost_usd") or 0.0) + + +# ---------------------------------------------------------------- host evidence + + +GUARD_LINE = re.compile(r"^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d) \[[^\]]*\] (.*)$") +POWER_LINE = re.compile(r"^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d [+-]\d{4}) ([^\t]*?)\s*\t") +GUARD_LOG = Path.home() / "Library/Logs/clamshell-guard.log" + + +def guard_time(line: str, zone: ZoneInfo) -> float | None: + match = GUARD_LINE.match(line) + if not match: + return None + stamp = datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S") + return stamp.replace(tzinfo=zone).timestamp() + + +def power_time(line: str) -> float | None: + match = POWER_LINE.match(line) + if not match: + return None + return datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S %z").timestamp() + + +def guard_spans(text: str, timezone: str, end: float) -> list: + """Pauses: from a 'paused pids:' line to the next 'resumed pids:' line (or + `end`).""" + zone, spans, opened = ZoneInfo(timezone), [], None + for line in text.splitlines(): + t = guard_time(line, zone) + if t is None: + continue + what = GUARD_LINE.match(line).group(2) + if "paused pids:" in what and opened is None: + opened = t + elif "resumed pids:" in what and opened is not None: + spans.append((opened, t, "guard pause")) + opened = None + if opened is not None: + spans.append((opened, end, "guard pause")) + return spans + + +def power_spans(text: str, end: float) -> list: + """Sleeps: from a Sleep line to the next Wake or DarkWake line (or `end`).""" + spans, opened = [], None + for line in text.splitlines(): + t = power_time(line) + if t is None: + continue + kind = POWER_LINE.match(line).group(2).strip() + if kind == "Sleep" and opened is None: + opened = t + elif kind in ("Wake", "DarkWake") and opened is not None: + spans.append((opened, t, "sleep")) + opened = None + if opened is not None: + spans.append((opened, end, "sleep")) + return spans + + +def excerpt(text: str, when: Callable[[str], float | None], since: float, until: float): + """The lines of a log stamped within [since, until].""" + keep = [] + for line in text.splitlines(): + t = when(line) + if t is not None and since <= t <= until: + keep.append(line + "\n") + return "".join(keep) + + +def memory_pressure() -> int | None: + """`sysctl kern.memorystatus_vm_pressure_level`: 1 when normal on this host.""" + try: + out = subprocess.run( + ["sysctl", "-n", "kern.memorystatus_vm_pressure_level"], + capture_output=True, + text=True, + timeout=10, + check=True, + ) + return int(out.stdout.strip()) + except (OSError, subprocess.SubprocessError, ValueError): + return None + + +def read_guard_log() -> str: + try: + return GUARD_LOG.read_text(errors="replace") + except OSError: + return "" + + +def read_power_log() -> str: + out = subprocess.run( + ["pmset", "-g", "log"], capture_output=True, text=True, timeout=120, check=True + ) + return out.stdout + + +def local_timezone() -> str: + """The IANA name of the host's zone, which the guard log's times are in.""" + target = os.path.realpath("/etc/localtime") + if "zoneinfo/" not in target: + raise SystemExit(f"cannot tell the host's time zone from {target}") + return target.split("zoneinfo/", 1)[1] + + +class HostLog: + """The host log, written every `interval` seconds, and the host evidence an + attempt's class reads: the guard's pauses, the power log's sleeps, the + host log's gaps and its memory-pressure samples.""" + + def __init__( + self, + folder: Path, + timezone: str | None = None, + interval: float = SAMPLE_SECONDS, + read_load: Callable[[], tuple] = os.getloadavg, + read_pressure: Callable[[], int | None] = memory_pressure, + read_guard: Callable[[], str] = read_guard_log, + read_power: Callable[[], str] = read_power_log, + ): + self.folder = Path(folder) + self.folder.mkdir(parents=True, exist_ok=True) + self.timezone = timezone or local_timezone() + ZoneInfo(self.timezone) # refused here, not at the first pause + self.interval = interval + self._read_load, self._read_pressure = read_load, read_pressure + self._read_guard, self._read_power = read_guard, read_power + self._cond = threading.Condition() + self._samples: list[tuple[float, int | None]] = [] + self._spans: list = [] # guard pauses and sleeps at the last refresh + self._read_at = -math.inf + self._refresh_lock = threading.Lock() + self._stop = threading.Event() + self.start = time.time() + self.end: float | None = None + self._write_meta() + self.sample() + self._thread = threading.Thread(target=self._loop, name="host-log", daemon=True) + self._thread.start() + + def _write_meta(self) -> None: + meta = {"timezone": self.timezone, "start": self.start} + if self.end is not None: + meta["end"] = self.end + write_json(self.folder / "host.json", meta) + + def sample(self) -> None: + load1, load5, load15 = self._read_load() + pressure = self._read_pressure() + with self._cond: + t = time.time() + line = { + "time": t, + "load1": load1, + "load5": load5, + "load15": load15, + "pressure": pressure, + } + with (self.folder / "host-log.jsonl").open("a") as handle: + handle.write(json.dumps(line) + "\n") + self._samples.append((t, pressure)) + self._cond.notify_all() + + def _loop(self) -> None: + k = 1 + while not self._stop.wait( + max(0.0, self.start + k * self.interval - time.time()) + ): + self.sample() + # After a stall, the next sample is the next tick, with no burst. + k = max(k + 1, math.ceil((time.time() - self.start) / self.interval)) + + def settled(self, end: float) -> bool: + """Whether every sample that could bear on an attempt ending at `end` is + in: one later than the pressure window, or a silence longer than a + gap (which is itself host trouble when it covers the attempt).""" + with self._cond: + latest = self._samples[-1][0] + return latest > end + PRESSURE_WINDOW or time.time() - latest > HOST_LOG_GAP + + def wait_settled(self, end: float) -> None: + with self._cond: + while not self.settled(end): + self._cond.wait(timeout=min(self.interval, 5.0)) + + def refresh(self, after: float) -> None: + """Read the guard's and the power log again unless read since `after`.""" + with self._refresh_lock: + if self._read_at > after: + return + now = time.time() + since = self.start - EXCERPT_LEAD + zone = ZoneInfo(self.timezone) + guard = excerpt( + self._read_guard(), lambda x: guard_time(x, zone), since, now + ) + power = excerpt(self._read_power(), power_time, since, now) + spans = guard_spans(guard, self.timezone, now) + power_spans(power, now) + with self._cond: + self._spans, self._read_at = spans, now + + def _evidence(self) -> tuple[list, list[float]]: + """(spans, the times of samples above 1), as the report builds them + from the archived logs, with a silence so far as an open gap.""" + with self._cond: + spans = list(self._spans) + times = [t for t, _ in self._samples] + pressure = [t for t, p in self._samples if int(p or 0) > 1] + for a, b in zip([self.start] + times, times): + if b - a > HOST_LOG_GAP: + spans.append((a, b, "host log gap")) + now = time.time() if self.end is None else self.end + if now - times[-1] > HOST_LOG_GAP: + spans.append((times[-1], now, "host log gap")) + return spans, pressure + + def trouble(self, start: float, end: float) -> str | None: + spans, pressure = self._evidence() + for a, b, what in spans: + if a <= end and b >= start: + return what + for t in pressure: + if start - PRESSURE_WINDOW <= t <= end + PRESSURE_WINDOW: + return "memory pressure" + return None + + def describe(self, start: float, end: float) -> str: + """The host evidence behind an attempt's host class, for its rerun's + reason.""" + spans, _ = self._evidence() + parts = [ + f"{what} {a:.1f}-{b:.1f}" for a, b, what in spans if a <= end and b >= start + ] + with self._cond: + parts += [ + f"memory pressure {p} at {t:.1f}" + for t, p in self._samples + if int(p or 0) > 1 + and start - PRESSURE_WINDOW <= t <= end + PRESSURE_WINDOW + ] + return "; ".join(parts) + + def close(self) -> None: + """A last sample, the booking's end, and the guard's and power log's + lines for the booking.""" + self._stop.set() + self._thread.join(timeout=self.interval + 10) + self.sample() + with self._cond: + self.end = time.time() + self._write_meta() + since, zone = self.start - EXCERPT_LEAD, ZoneInfo(self.timezone) + guard = excerpt( + self._read_guard(), lambda x: guard_time(x, zone), since, self.end + ) + (self.folder / "clamshell-guard.log").write_text(guard) + power = excerpt(self._read_power(), power_time, since, self.end) + (self.folder / "pmset.log").write_text(power) + + +# ---------------------------------------------------------------- spend + + +class Ceiling(pilot.Spend): + """The hard ceiling over every model's spend: once the total reaches it, + `exhausted` is true for good, which ends each conversation under way at + its next turn and lets no attempt start.""" + + def __init__(self, ceiling_usd: float, spent_before: float): + super().__init__(ceiling_usd, spent_before) + self.crossed_at: float | None = None + + @property + def lock(self) -> threading.Lock: + return self._lock + + def add(self, usd: float | None) -> None: + with self._lock: + self.spent_usd += float(usd or 0.0) + if self.crossed_at is None and self.spent_usd >= self.budget_usd: + self.crossed_at = time.time() + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + with self._lock: + return self.crossed_at is not None + + +# ---------------------------------------------------------------- files + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + return sha256_bytes(Path(path).read_bytes()) + + +def write_json(path: Path, payload, indent: int | None = 1) -> str: + """Write a file whole (to a temporary name, then renamed); its sha256.""" + data = json.dumps(payload, indent=indent, default=str).encode() + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.tmp") + tmp.write_bytes(data) + os.replace(tmp, path) + return sha256_bytes(data) + + +# ---------------------------------------------------------------- one model + + +@dataclass +class Attempt: + label: str + n: int + start: float + end: float + record: dict + cls: str = "" + detail: str = "" + reason: str = "" + + +def converse_with_harness(lanes: "Lanes", sid: str, label: str, tool_call) -> dict: + """One attempt: the treatment harness's conversation for the arm, under the + hard ceiling.""" + board = lanes.run.board + return pilot.run_household( + board["scenarios"][sid], + board["variables"][sid], + tool_call, + lanes.completion, + lanes.run.ceiling, + profile=lanes.profiles[label], + ) + + +class Lanes: + """One model's two lanes: its admission, its attempts and its records.""" + + def __init__(self, run: "Run", model: str, labels: tuple[str, ...], folders: dict): + self.run, self.model, self.labels = run, model, labels + design = run.design + self.folder = Path(design["runs"]) / model + self.households_dir = { + label: self.folder / folders[label] / design["stage"] / "households" + for label in labels + } + self.profiles = {label: arm_profile(model, label) for label in labels} + self.cap = float(design["caps"][model]) + self.reserve = float(design["reserves"][model]) + self.orders = orders(run.master, model, run.draw_order, labels) + self.cond = threading.Condition() + self._log_lock = threading.Lock() + self.state: dict[str, str] = {} # household -> running, settling, finished + self.spent: dict[str, float] = {} # household -> its attempts' cost so far + self.refused = False + self.paused = False + self.stopped = False + self.inconclusive: dict[str, list[str]] = {} + self.pool = None + self._pool_lock = threading.Lock() + self.completion = None + + # The records. + + def households(self, lane: int) -> list[str]: + return [ + sid + for i, sid in enumerate(self.run.draw_order) + if i % 2 == lane - 1 and sid not in self.run.no_output + ] + + def write_line(self, line: dict, name: str = "run-log.jsonl") -> None: + with self._log_lock: + self.folder.mkdir(parents=True, exist_ok=True) + with (self.folder / name).open("a") as handle: + handle.write(json.dumps(line) + "\n") + handle.flush() + os.fsync(handle.fileno()) + + def event(self, kind: str, reason: str, at: float | None = None) -> None: + self.write_line( + {"event": kind, "time": time.time() if at is None else at, "reason": reason} + ) + self.run.say(f"{self.model}: {kind} ({reason})") + + def attempt_line(self, sid: str, lane: int, a: Attempt, sha: str) -> None: + self.write_line( + { + "household": sid, + "arm": a.label, + "lane": lane, + "attempt": a.n, + "start": a.start, + "end": a.end, + "status": a.record.get("status"), + "class": a.cls, + "action": action(a.cls, a.n), + "sha256": sha, + } + ) + + # Stops and pauses. + + def halted(self) -> bool: + return self.stopped or self.run.ceiling.exhausted() + + def stop(self, reason: str, kind: str = "stop", at: float | None = None) -> None: + """Stop the lanes. Every rule that stops them writes its own event.""" + with self.cond: + self.stopped = True + self.event(kind, reason, at) + self.cond.notify_all() + + def resume(self, note: str) -> None: + with self.cond: + if self.paused: + self.paused = False + self.event("resume", note) + self.cond.notify_all() + + # One household. + + def admit(self, sid: str, lane: int) -> bool: + with self.cond: + while not self.halted() and ( + self.paused or "settling" in self.state.values() + ): + self.cond.wait(timeout=WAIT_SECONDS) + if self.halted(): + return False + now = time.time() + finished = [h for h, s in self.state.items() if s == "finished"] + spend = sum(self.spent[h] for h in finished) + under_way = len(self.state) - len(finished) + 1 + ok = ( + not self.refused + and spend + under_way * self.reserve <= self.cap + ADMISSION_SLACK + ) + self.write_line( + { + "household": sid, + "lane": lane, + "time": now, + "decision": "admit" if ok else "refuse", + "finished_spend": spend, + "under_way": under_way, + "reserve": self.reserve, + "cap": self.cap, + }, + "admission-log.jsonl", + ) + if ok: + self.state[sid], self.spent[sid] = "running", 0.0 + else: + self.refused = True + self.cond.notify_all() + return ok + + def start_attempt(self) -> float | None: + """The attempt's start, or None when the lanes have stopped. Waits out a + pause. The start is taken under the ceiling's lock, so no attempt + starts after the crossing.""" + with self.cond: + while self.paused and not self.halted(): + self.cond.wait(timeout=WAIT_SECONDS) + with self.run.ceiling.lock: + if self.stopped or self.run.ceiling.crossed_at is not None: + return None + return time.time() + + def end_attempt( + self, sid: str, label: str, n: int, start: float, record: dict, last: bool + ) -> Attempt: + """The attempt's end. A class decided without the host evidence acts at + once: an operator failure pauses the lanes, a harness failure stops + them, before any other attempt can start.""" + with self.cond: + end = time.time() + self.spent[sid] += attempt_cost(record) + attempt = Attempt(label, n, start, end, record) + cls, detail = classify(record, start, end, None) + if cls in EARLY: + attempt.cls, attempt.detail = cls, detail + if cls == "operator": + self.paused = True + self.event("pause", f"{label}/{sid} attempt {n}: {detail}") + if cls == "harness": + self.stopped = True + self.event( + "stop", + f"{label}/{sid}: a request class on the first request ({detail})", + ) + if last: + self.state[sid] = "settling" + self.cond.notify_all() + return attempt + + def settle(self, sid: str, done: list[Attempt]) -> None: + """Each attempt's class once the host evidence for the pass is in.""" + last = max(a.end for a in done) + host = self.run.host + host.wait_settled(last) + host.refresh(after=last) + stop = None + for a in done: + cls, detail = classify(a.record, a.start, a.end, host) + if a.cls and a.cls != cls: + raise RuntimeError(f"{a.label}/{sid}: class {a.cls} became {cls}") + a.cls, a.detail = cls, detail + a.reason = ( + f"{detail}: {host.describe(a.start, a.end)}" + if cls == "host" + else detail + ) + status = a.record.get("status") + if cls == "inconclusive" and status in ("provider_error", "harness_error"): + key = f"{status}: {error_class(a.record) or a.record.get('error')}" + with self.cond: + seen = self.inconclusive.setdefault(key, []) + if sid not in seen: + seen.append(sid) + if len(seen) > INCONCLUSIVE_STOP and stop is None: + stop = f"{key} ended runs in {len(seen)} households" + if stop: + self.stop(stop) + + def converse(self, sid: str, label: str) -> dict: + profile = self.profiles[label] + board = self.run.board + features = frozenset(profile.tool_features) + + def tool_call(arguments): + message = ( + pilot.tool_call_frame(features, arguments) if features else arguments + ) + return self.pool.call(message) + + try: + return self.run.converse(self, sid, label, tool_call) + except Exception as error: # an attempt that left no record of its own + return pilot.harness_error_record( + sid, board["variables"][sid], profile, error + ) + + def ensure_pool(self) -> bool: + """The model's pool, started again if it has lost every worker; False + if it cannot be.""" + with self._pool_lock: + if ( + self.pool is not None + and self.pool.counters()["lost"] < self.run.workers + ): + return True + old = self.pool + try: + self.pool = self.run.pool_factory(self) + except RuntimeError as error: + self.run.say(f"{self.model}: the engine pool did not start: {error}") + return False + if old is not None: + self.run.say( + f"{self.model}: the engine pool lost every worker; started again" + ) + old.close() + return True + + def replace(self, sid: str, lane: int, previous: Attempt, entries: list) -> None: + """Save the attempt a rerun replaces, before the rerun.""" + folder = self.households_dir[previous.label] / "attempts" + name = f"{sid}-{previous.n}.json" + sha = write_json(folder / name, previous.record) + entries.append( + { + "n": previous.n, + "file": f"attempts/{name}", + "sha256": sha, + "status": previous.record.get("status"), + "class": previous.cls, + "reason": previous.reason, + } + ) + self.attempt_line(sid, lane, previous, sha) + + def run_household(self, sid: str, lane: int) -> None: + order = self.orders[sid] + attempts: dict[str, list[Attempt]] = {label: [] for label in order} + entries: dict[str, list[dict]] = {label: [] for label in order} + in_flight = self.folder / "in-flight" / sid + todo = [(1, label) for label in order] + while todo: + done: list[Attempt] = [] + for i, (n, label) in enumerate(todo): + if not self.ensure_pool(): + self.stop("the engine pool could not be started again") + break + start = self.start_attempt() + if start is None: + break + if n > 1: + self.replace(sid, lane, attempts[label][-1], entries[label]) + record = self.converse(sid, label) + a = self.end_attempt( + sid, label, n, start, record, last=i == len(todo) - 1 + ) + write_json(in_flight / f"{label}-{n}.json", record) + attempts[label].append(a) + done.append(a) + self.run.say( + f"{self.model} lane {lane} {sid} {label} #{n}: " + f"{record.get('status')} ${attempt_cost(record):.4f} " + f"(total ${self.run.ceiling.spent_usd:.2f})" + ) + if not done: + break + self.settle(sid, done) + cut = len(done) < len(todo) + todo = ( + [] + if cut or self.halted() + else [ + (a.n + 1, a.label) + for a in done + if a.n <= MAX_RERUNS and a.cls in RERUN + ] + ) + with self.cond: + self.state[sid] = "running" if todo else "finished" + self.cond.notify_all() + for label, mine in attempts.items(): + if mine: + final = mine[-1] + sha = write_json( + self.households_dir[label] / f"{sid}.json", + {**final.record, "attempts": entries[label]}, + ) + self.attempt_line(sid, lane, final, sha) + with self.cond: + self.state[sid] = "finished" + self.cond.notify_all() + shutil.rmtree(in_flight, ignore_errors=True) + + def lane(self, lane: int) -> None: + try: + for sid in self.households(lane): + if not self.admit(sid, lane): + return + self.run_household(sid, lane) + except Exception as error: # a runner defect: this model stops, the rest run on + self.run.say(traceback.format_exc()) + self.stop(f"runner error on lane {lane}: {type(error).__name__}: {error}") + + def manifest(self) -> dict: + board, design = self.run.board, self.run.design + pe3 = arm_profile(self.model, "pe3") + first = sorted(board["scenarios"])[0] + + def timeout(profile): + built = pilot.build_initial_request( + board["scenarios"][first], board["variables"][first], profile + ) + return built[1].get("timeout") + + board_path = Path(design["board"]) + return { + "started_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "git_head": pilot._git("rev-parse", "HEAD"), + "git_dirty": bool(pilot._git("status", "--porcelain")), + "script_sha256": sha256_file(Path(__file__)), + "harness_sha256": sha256_file(Path(pilot.__file__)), + "treatment_commit": TREATMENT_COMMIT, + "versions": pilot._versions(), + "fix_module": str(pilot.FIX_MODULE.relative_to(pilot.ROOT)), + "fix_module_sha256": sha256_file(pilot.FIX_MODULE), + "board": str(board_path), + "board_engine": f"policyengine-us {pilot.board_engine(board_path)}", + "board_reference_sha256": sha256_file(board_path / "reference_outputs.csv"), + "model": self.model, + "model_id": pe3.model_id, + "api": pe3.api, + "answer_contract": pe3.contract, + "prompt_version": PROMPT_VERSION, + "prompt_wording_sha256": pilot.prompt_wording_sha256(pe3), + "request_timeout_seconds": timeout(pe3), + "turn_cap": pilot.TOOL_TURN_CAP, + "max_calculations_per_turn": pilot.MAX_CALCULATIONS_PER_TURN, + "tool_timeout_seconds": pilot.TOOL_TIMEOUT_SECONDS, + "arms": { + label: { + "arm": arm_string(label), + "features": sorted(self.profiles[label].tool_features), + } + for label in self.labels + }, + "arm_settings": { + label: { + "tool_notice_sha256": hashlib.sha256( + profile.notice.encode() + ).hexdigest(), + **pilot.profile_treatment(profile), + "tool_version": profile.tool_version, + "request_timeout_seconds": timeout(profile), + } + for label, profile in self.profiles.items() + }, + "folders": { + label: str(path) for label, path in self.households_dir.items() + }, + "cap_usd": self.cap, + "reserve_usd": self.reserve, + "overall_cap_usd": float(design["overall_cap"]), + "ceiling_usd": self.run.ceiling.budget_usd, + "spent_before_usd": float(design["spent_before"]), + "lanes": {lane: self.households(lane) for lane in (1, 2)}, + "no_scored_output": sorted(self.run.no_output), + "workers": self.run.workers, + } + + def summary(self) -> dict: + records = [] + for label, folder in self.households_dir.items(): + for path in sorted(folder.glob("*.json")) if folder.is_dir() else []: + records.append(json.loads(path.read_text())) + lines = [] + path = self.folder / "run-log.jsonl" + if path.exists(): + lines = [json.loads(x) for x in path.read_text().splitlines() if x.strip()] + attempts = [x for x in lines if "event" not in x] + count: dict[str, int] = {} + for x in attempts: + count[x["class"]] = count.get(x["class"], 0) + 1 + return { + "finished_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "model": self.model, + "households_finished": sorted( + h for h, s in self.state.items() if s == "finished" + ), + "refused": self.refused, + "stopped": self.stopped, + "spent_usd": sum(self.spent.values()), + "final_records": len(records), + "attempts": len(attempts), + "classes": dict(sorted(count.items())), + "events": [x for x in lines if "event" in x], + "tool_pool": self.pool.counters() if self.pool is not None else None, + } + + +# ---------------------------------------------------------------- the run + + +def default_pool(lanes: Lanes): + return pilot.ToolPool( + size=lanes.run.workers, + board=Path(lanes.run.design["board"]), + tool_features=frozenset(), + ) + + +def default_completion(profile: pilot.ModelProfile): + from litellm import completion, responses + + return responses if profile.api == "responses" else completion + + +class Run: + """The six models' lanes, the host log and the hard ceiling.""" + + def __init__( + self, + design: dict, + workers: int = 2, + models: tuple[str, ...] | None = None, + converse: Callable = converse_with_harness, + pool_factory: Callable = default_pool, + completion_for: Callable = default_completion, + host_factory: Callable | None = None, + out=sys.stdout, + ): + self.design, self.workers, self.out = design, workers, out + self.converse, self.pool_factory = converse, pool_factory + self.completion_for = completion_for + self.host_factory = host_factory or (lambda folder: HostLog(folder)) + self._say_lock = threading.Lock() + self.master = master_seed(design["frozen_sha256"]) + self.draw_order = list(design["draw_order"]) + board_path = Path(design["board"]) + pilot.set_stated_engine(pilot.board_engine(board_path)) + self.board = pilot.load_board(board_path) + scored = {str(s) for s in self.board["scored"]["scenario_id"]} + self.no_output = {sid for sid in self.draw_order if sid not in scored} + self.ceiling = Ceiling( + float(design["overall_cap"]) + CEILING_MARGIN, float(design["spent_before"]) + ) + self.models = tuple(models or CHEAP + FRONTIER) + self.lanes = { + model: Lanes( + self, + model, + ARMS if model in CHEAP else FRONTIER_ARMS, + design["arms"] if model in CHEAP else design["frontier_arms"], + ) + for model in self.models + } + self.host: HostLog | None = None + self._ceiling_written = False + self.interrupts = 0 + + def say(self, text: str) -> None: + with self._say_lock: + print(text, file=self.out, flush=True) + + def stop_all(self, reason: str) -> None: + for lanes in self.lanes.values(): + lanes.stop(reason) + + def watch(self) -> None: + """The ceiling's events, and resumes after operator pauses.""" + crossed = self.ceiling.crossed_at + if crossed is not None and not self._ceiling_written: + self._ceiling_written = True + reason = ( + f"total spend ${self.ceiling.spent_usd:.2f} reached the hard ceiling " + f"${self.ceiling.budget_usd:.2f}" + ) + for lanes in self.lanes.values(): + lanes.stop(reason, kind="ceiling", at=crossed) + for lanes in self.lanes.values(): + path = lanes.folder / "RESUME.json" + if lanes.paused and path.exists(): + note = json.loads(path.read_text()).get("note") or "resumed" + path.rename(lanes.folder / f"resume-{time.time():.0f}.json") + lanes.resume(note) + + def execute(self) -> None: + runs = Path(self.design["runs"]) + self.host = self.host_factory(runs / "host") + try: + starting = {} + for model, lanes in self.lanes.items(): + lanes.completion = self.completion_for(lanes.profiles["pe3"]) + stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) + write_json( + lanes.folder / f"manifest-{stamp}.json", lanes.manifest(), indent=2 + ) + starting[model] = threading.Thread( + target=lanes.ensure_pool, daemon=True + ) + starting[model].start() + for thread in starting.values(): + thread.join() + threads = [ + threading.Thread( + target=lanes.lane, args=(lane,), name=f"{model}-{lane}", daemon=True + ) + for model, lanes in self.lanes.items() + for lane in (1, 2) + ] + for thread in threads: + thread.start() + while any(t.is_alive() for t in threads): + self.watch() + for t in threads: + t.join(timeout=0.5 / len(threads)) + self.watch() + finally: + for model, lanes in self.lanes.items(): + stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) + write_json( + lanes.folder / f"run-summary-{stamp}.json", + lanes.summary(), + indent=2, + ) + if lanes.pool is not None: + lanes.pool.close() + self.host.close() + + def interrupt(self, signum, frame) -> None: + self.interrupts += 1 + if self.interrupts > 1: + raise KeyboardInterrupt + self.say( + f"signal {signum}: stopping every model's lanes; " + "the conversations under way end" + ) + self.stop_all(f"operator stop (signal {signum})") + + +# ---------------------------------------------------------------- before a run + + +def host_power() -> tuple[bool, bool]: + """(on AC power, lid open).""" + batt = subprocess.run( + ["pmset", "-g", "batt"], capture_output=True, text=True + ).stdout + lid = subprocess.run( + ["ioreg", "-r", "-k", "AppleClamshellState", "-d", "1"], + capture_output=True, + text=True, + ).stdout + return "'AC Power'" in batt, '"AppleClamshellState" = No' in lid + + +def preflight(design: dict, models: tuple[str, ...]) -> list[str]: + """What stops the run from starting.""" + missing = [k for k in REQUIRED if k not in design] + if missing: + return [f"the design has no {', '.join(missing)}"] + problems = [] + if design.get("stated_engine_overrides"): + problems.append("the confirmatory run states the board's engine: no overrides") + if sha256_file(Path(design["preregistration"])) != design["frozen_sha256"]: + problems.append("the frozen copy does not hash to frozen_sha256") + pool = design["eligible_pool"] + if sha256_file(Path(pool["path"])) != pool["sha256"]: + problems.append("the eligible pool does not hash as frozen") + else: + ids = [e["id"] for e in json.loads(Path(pool["path"]).read_text())] + if draw(master_seed(design["frozen_sha256"]), ids) != list( + design["draw_order"] + ): + problems.append( + "the draw, recomputed from the stream 'draw', is not the design's" + ) + head, dirty = pilot._git("rev-parse", "HEAD"), pilot._git("status", "--porcelain") + if head != design["run_commit"] or dirty: + problems.append( + f"the runner is at {head[:12]} (dirty {bool(dirty)}), " + "not the run commit, clean" + ) + changed = subprocess.run( + [ + "git", + "-C", + str(pilot.ROOT), + "diff", + "--quiet", + TREATMENT_COMMIT, + "HEAD", + "--", + *TREATMENT_PATHS, + ] + ).returncode + if changed: + problems.append(f"the treatment's code differs from {TREATMENT_COMMIT[:12]}'s") + import policybench + + places = [Path(p).resolve() for p in policybench.__path__] + if pilot.ROOT != ROOT or not all(p.is_relative_to(ROOT) for p in places): + problems.append( + f"the harness ({pilot.__file__}) or policybench ({places}) is not " + "this checkout's" + ) + harness = design["treatment_harness"] + harness_head = subprocess.run( + ["git", "-C", harness, "rev-parse", "HEAD"], capture_output=True, text=True + ).stdout.strip() + if harness_head != TREATMENT_COMMIT: + problems.append( + f"the treatment harness is at {harness_head[:12]}, " + f"not {TREATMENT_COMMIT[:12]}" + ) + board = Path(design["board"]) + engine = pilot.board_engine(board) + if f"policyengine-us {engine}" != design["engine"]: + problems.append( + f"the board's engine {engine} is not the design's {design['engine']}" + ) + installed = pilot._versions()["policyengine-us"] + if installed != engine: + problems.append(f"policyengine-us {installed} is installed, not {engine}") + if set(design["arms"]) != set(ARMS) or set(design["frontier_arms"]) != set( + FRONTIER_ARMS + ): + problems.append("the design's arms are not the registered ones") + for name in ("arms", "frontier_arms"): + if len(set(design[name].values())) != len(design[name]): + problems.append(f"two {name} share a folder") + for model in models: + if model not in design["caps"] or model not in design["reserves"]: + problems.append(f"{model}: no cap or reserve") + for label in ARMS if model in CHEAP else FRONTIER_ARMS: + try: + arm_profile(model, label) + except (ValueError, SystemExit) as error: + problems.append(str(error)) + key = pilot.provider_key(pilot.model_profile(model).model_id) + if not os.environ.get(key): + problems.append(f"{model}: {key} is not set") + folder = Path(design["runs"]) / model + if (folder / "run-log.jsonl").exists() or any( + folder.glob("*/*/households/*.json") + ): + problems.append(f"{folder} already holds records; a run is never resumed") + if (Path(design["runs"]) / "host" / "host-log.jsonl").exists(): + problems.append("the runs folder already holds a host log") + ac, lid = host_power() + if not (ac and lid): + problems.append( + f"the host is not on AC power with the lid open (AC {ac}, lid open {lid})" + ) + return problems + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run", help="run the confirmatory stage") + run.add_argument("design", type=Path) + run.add_argument("--workers", type=int, default=2, help="engine workers per model") + run.add_argument("--dry-run", action="store_true", help="check and print the plan") + resume = sub.add_parser("resume", help="end an operator pause") + resume.add_argument("design", type=Path) + resume.add_argument("model") + resume.add_argument("--note", default="the account was fixed") + args = parser.parse_args(argv) + design = json.loads(args.design.read_text()) + if args.command == "resume": + folder = Path(design["runs"]) / args.model + write_json(folder / "RESUME.json", {"time": time.time(), "note": args.note}) + print(f"{args.model}: resume requested; the runner resumes within a second") + return 0 + models = CHEAP + FRONTIER + problems = preflight(design, models) + pilot._require_engine(Path(design["board"])) + if problems: + print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) + return 1 + plan = Run(design, workers=args.workers) + if args.dry_run: + for model, lanes in plan.lanes.items(): + sizes = [len(lanes.households(lane)) for lane in (1, 2)] + print( + f"{model}: arms {', '.join(lanes.labels)}; lanes {sizes[0]} and " + f"{sizes[1]} households; cap ${lanes.cap:.2f}, " + f"reserve ${lanes.reserve:.2f}" + ) + print( + f"no scored output: {sorted(plan.no_output)}; hard ceiling " + f"${plan.ceiling.budget_usd:.2f} " + f"(${plan.ceiling.spent_usd:.2f} spent before)" + ) + return 0 + signal.signal(signal.SIGINT, plan.interrupt) + signal.signal(signal.SIGTERM, plan.interrupt) + plan.execute() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From c04ffe5b4178b8bde3f2d6d3e7af8ce39d313287 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 02:56:22 -0400 Subject: [PATCH 02/48] Confirmatory runner: tests Unit tests for the streams, arm strings and features, the classes and actions, the host evidence and the ceiling. Fake runs (a fake provider, pool and host log, on a fake board, with the windows scaled down) plant reruns, host trouble, an operator pause, a harness stop, the inconclusive stop, a refusal, the hard ceiling, a raising attempt, pool restarts and an interrupt. run_problems recomputes every rule of the run procedure from the files the runner wrote, and a property test checks it over random failures and spend. Two tests run the harness's own conversation under the runner. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 1435 ++++++++++++++++++++++++++++++++ 1 file changed, 1435 insertions(+) create mode 100644 tests/test_confirmatory_run.py diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py new file mode 100644 index 00000000..f42a1c8e --- /dev/null +++ b/tests/test_confirmatory_run.py @@ -0,0 +1,1435 @@ +"""The confirmatory runner (scripts/confirmatory_run.py). + +The runs here are fakes: a fake provider whose conversations a test plants, +a fake engine pool and a host log sampled every 20 ms, on a fake board. Each +run is checked by `run_problems`, which recomputes, from the files the runner +wrote and nothing else, every rule the run procedure states: the records and +their hashes, each attempt's class and action, the lanes and the schedule, +admission, pauses, stops and the ceiling. The study's differential test runs +the runner and checks its records with the confirmatory report itself. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import math +import sys +import threading +import time +from datetime import datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +import numpy as np +import pandas as pd +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import confirmatory_run as cr # noqa: E402 +import tool_pilot_argon as pilot # noqa: E402 + +from policybench.scenarios import Person, Scenario # noqa: E402 + +VARIABLES = ["snap", "federal_income_tax_before_refundable_credits"] +POOL = [f"h{i:02d}" for i in range(12)] +N_DRAW = 6 +COST = 0.01 +TZ = "America/New_York" +PREREG = "A frozen pre-registration for a test.\n" +ENGINE = "2.60.4" + + +# ---------------------------------------------------------------- streams + + +def test_the_master_seed_is_the_first_sixteen_hex_digits(): + sha = hashlib.sha256(b"x").hexdigest() + assert cr.master_seed(sha) == int(sha[:16], 16) + with pytest.raises(ValueError): + cr.master_seed("not a sha") + + +def test_a_stream_is_keyed_by_its_name_alone(): + a = cr.stream(7, "order/m").integers(0, 1 << 30, 5) + b = cr.stream(7, "order/m").integers(0, 1 << 30, 5) + c = cr.stream(7, "order/n").integers(0, 1 << 30, 5) + assert list(a) == list(b) and list(a) != list(c) + key = int.from_bytes(hashlib.sha256(b"draw").digest()[:8], "big") + want = np.random.default_rng(np.random.SeedSequence(7, spawn_key=(key,))) + assert list(cr.stream(7, "draw").integers(0, 100, 8)) == list( + want.integers(0, 100, 8) + ) + + +@given(st.integers(0, (1 << 64) - 1), st.integers(1, 12)) +def test_the_draw_is_seeded_and_without_replacement(master, n): + got = cr.draw(master, list(reversed(POOL)), n=n) + assert got == cr.draw(master, POOL, n=n) # the pool is sorted first + assert len(set(got)) == n and set(got) <= set(POOL) + + +def test_orders_are_permutations_drawn_for_every_household_in_draw_order(): + got = cr.orders(9, "m", POOL, cr.ARMS) + assert list(got) == POOL + assert all(sorted(o) == sorted(cr.ARMS) for o in got.values()) + generator = cr.stream(9, "order/m") + assert got[POOL[0]] == [str(a) for a in generator.permutation(list(cr.ARMS))] + # The first households' orders do not depend on the later ones. + assert cr.orders(9, "m", POOL[:3], cr.ARMS) == {s: got[s] for s in POOL[:3]} + + +# ---------------------------------------------------------------- arms + + +def test_arm_strings_follow_the_harness_feature_order(): + assert cr.arm_string("pe") == "pe" + assert cr.arm_string("pe3") == cr.arm_string("pe3-b") == "pe3" + for label, dropped in cr.DROPPED.items(): + if dropped: + kept = [f for f in pilot.TOOL_FEATURES if f not in dropped] + assert cr.arm_string(label) == "pe:" + "+".join(kept) + + +@pytest.mark.parametrize("model", cr.CHEAP + cr.FRONTIER) +def test_each_arm_has_its_registered_features(model): + for label in cr.ARMS if model in cr.CHEAP else cr.FRONTIER_ARMS: + profile = cr.arm_profile(model, label) + dropped = cr.DROPPED[label] + want = set() if dropped is None else set(pilot.TOOL_FEATURES) - set(dropped) + assert profile.uses_calculator and set(profile.tool_features) == want + assert profile.prompt_version == cr.PROMPT_VERSION + + +def test_an_arm_whose_features_are_not_registered_is_refused(monkeypatch): + # The harness builds an arm's features from its arm string: a string that + # does not give the label's registered features is refused. + monkeypatch.setattr(cr, "arm_string", lambda label: "pe:names+labels") + with pytest.raises(ValueError, match="features"): + cr.arm_profile("gpt-6-luna", "loo-flat") + + +def test_the_error_classes_are_policybench_s(): + from policybench import eval_no_tools as e + + assert cr.TRANSPORT == {c.__name__ for c in e.RETRYABLE_PROVIDER_ERRORS} + assert cr.REQUEST | cr.ACCOUNT == {c.__name__ for c in e.MODEL_FATAL_ERRORS} + assert not cr.REQUEST & cr.ACCOUNT + assert not cr.TRANSPORT & (cr.REQUEST | cr.ACCOUNT) + + +# ---------------------------------------------------------------- classes + + +def record(status="complete", error=None, turns=1, calls=(), truncated=False): + turn = {"calls": [{"result": r} for r in calls]} + out = [dict(turn) for _ in range(turns)] + if truncated: + out = [{"truncated_and_retried": True}] + return {"status": status, "error": error, "turns": out} + + +class AlwaysTrouble: + def trouble(self, start, end): + return "memory pressure" + + +CASES = [ + (record(), "ok"), + (record("incomplete"), "ok"), + (record("turn_limit"), "ok"), + (record("provider_error", "RateLimitError: slow down"), "provider"), + (record("provider_error", "BadRequestError: too long"), "model"), + (record("provider_error", "BadRequestError: bad", turns=0), "harness"), + (record("provider_error", "BadRequestError: x", truncated=True), "harness"), + (record("provider_error", "AuthenticationError: key", turns=0), "operator"), + (record("provider_error", "RateLimitError: insufficient_quota"), "operator"), + (record("provider_error", "ProviderEmpty: no choices"), "inconclusive"), + (record("provider_error", "RateLimitError: x", turns=0), "none"), + ( + record("harness_error", "RuntimeError: every tool worker was lost", turns=0), + "none", + ), + (record("harness_error", "the tool pool had no worker"), "inconclusive"), + (record("budget_stop"), "ceiling"), + (record("budget_stop", turns=0), "ceiling"), + (record(calls=[{"error": "timeout", "message": "slow"}]), "inconclusive"), + ( + record(calls=[{"error": "engine", "message": "tool worker failed: EOF"}]), + "inconclusive", + ), + (record(calls=[{"error": "engine", "message": "MemoryError: x"}]), "inconclusive"), + (record(calls=[{"error": "engine", "message": "ValueError: no person"}]), "model"), + (record(calls=[{"results": {"snap": {"error": "OSError: disk"}}}]), "inconclusive"), + (record(calls=[{"results": {"snap": {"error": "KeyError: x"}}}]), "model"), + (record("running"), "inconclusive"), +] + + +@pytest.mark.parametrize("rec, want", CASES) +def test_each_failure_takes_its_class(rec, want): + assert cr.classify(rec, 0.0, 1.0, None)[0] == want + + +@pytest.mark.parametrize("rec, want", CASES) +def test_host_trouble_outranks_every_class_but_the_early_four(rec, want): + got = cr.classify(rec, 0.0, 1.0, AlwaysTrouble())[0] + assert got == (want if want in cr.EARLY else "host") + + +@pytest.mark.parametrize("rec, want", CASES) +def test_an_early_class_needs_no_host_evidence(rec, want): + early = cr.classify(rec, 0.0, 1.0, None)[0] + assert (early in cr.EARLY) == (want in cr.EARLY) + + +def test_actions_follow_the_class_and_the_attempt(): + for cls in ("ok", "model", "provider", "host", "none", "operator", "harness"): + for n in (1, 2, 3): + got = cr.action(cls, n) + if cls == "harness": + assert got == "stop" + elif cls in ("provider", "host", "none") and n <= 2: + assert got == "rerun" + elif cls == "operator" and n <= 2: + assert got == "pause-rerun" + else: + assert got == "stands" + assert cr.action("ceiling", 1) == "stopped" + assert cr.action("inconclusive", 1) == "stands" + + +# ---------------------------------------------------------------- host evidence + + +def guard_line(t: float, what: str) -> str: + stamp = datetime.fromtimestamp(t, ZoneInfo(TZ)).strftime("%Y-%m-%d %H:%M:%S") + return f"{stamp} [poll] lid closed on battery: {what} pids: 1 2" + + +def power_line(t: float, kind: str) -> str: + stamp = datetime.fromtimestamp(t, ZoneInfo(TZ)).strftime("%Y-%m-%d %H:%M:%S %z") + return f"{stamp} {kind:<20}\t{kind} due to a test" + + +def test_guard_pauses_and_power_sleeps_are_spans(): + t = 1_800_000_000.0 + guard = "\n".join( + [ + guard_line(t, "paused"), + guard_line(t + 10, "resumed"), + "junk", + guard_line(t + 20, "paused"), + ] + ) + assert cr.guard_spans(guard, TZ, end=t + 99) == [ + (t, t + 10, "guard pause"), + (t + 20, t + 99, "guard pause"), + ] + power = "\n".join( + [ + power_line(t, "Sleep"), + power_line(t + 5, "DarkWake"), + power_line(t + 7, "Sleep"), + ] + ) + assert cr.power_spans(power, end=t + 50) == [ + (t, t + 5, "sleep"), + (t + 7, t + 50, "sleep"), + ] + + +def test_an_excerpt_keeps_the_lines_stamped_in_its_window(): + t = 1_800_000_000.0 + text = "\n".join(guard_line(t + k, "paused") for k in (-5, 0, 5, 10)) + "\nx" + zone = ZoneInfo(TZ) + kept = cr.excerpt(text, lambda x: cr.guard_time(x, zone), t, t + 5) + assert kept.splitlines() == [guard_line(t, "paused"), guard_line(t + 5, "paused")] + + +class Readers: + """The host log's inputs, which a test can change while it runs.""" + + def __init__(self): + self.pressure = 1 + self.pressing = 0 # attempts under way that raise the pressure + self.stall = 0.0 + self.guard: list[str] = [] + self.power: list[str] = [] + self.lock = threading.Lock() + + def read_pressure(self): + with self.lock: + stall, self.stall = self.stall, 0.0 + if stall: + time.sleep(stall) + return 2 if self.pressing else self.pressure + + def host(self, folder): + return cr.HostLog( + folder, + timezone=TZ, + interval=0.02, + read_load=lambda: (1.0, 2.0, 3.0), + read_pressure=self.read_pressure, + read_guard=lambda: "\n".join(self.guard), + read_power=lambda: "\n".join(self.power), + ) + + +@pytest.fixture +def fast(monkeypatch): + """Windows scaled down a thousandfold, so a fake run takes a second.""" + monkeypatch.setattr(cr, "PRESSURE_WINDOW", 0.06) + monkeypatch.setattr(cr, "HOST_LOG_GAP", 0.09) + monkeypatch.setattr(cr, "WAIT_SECONDS", 0.005) + monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) + + +def test_the_host_log_samples_and_closes(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + time.sleep(0.1) + host.close() + meta = json.loads((tmp_path / "host.json").read_text()) + samples = [ + json.loads(x) for x in (tmp_path / "host-log.jsonl").read_text().splitlines() + ] + assert meta["timezone"] == TZ and meta["start"] <= samples[0]["time"] + assert samples[-1]["time"] <= meta["end"] + assert len(samples) >= 4 + assert samples[0]["load1"] == 1.0 and samples[0]["pressure"] == 1 + times = [s["time"] for s in samples] + assert times == sorted(times) + + +def test_pressure_near_an_attempt_is_host_trouble_once_settled(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + start = time.time() + readers.pressure = 2 + time.sleep(0.05) + readers.pressure = 1 + end = time.time() + host.wait_settled(end) + assert host.settled(end) + host.refresh(after=end) + assert host.trouble(start, end) == "memory pressure" + assert host.trouble(end + 1, end + 2) is None + host.close() + + +def test_a_stalled_host_log_is_a_gap(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + time.sleep(0.03) + start = time.time() + readers.stall = 0.2 + time.sleep(0.25) + end = time.time() + host.wait_settled(end) + host.refresh(after=end) + assert host.trouble(start, end) == "host log gap" + host.close() + + +def test_guard_pauses_and_sleeps_reach_an_attempt_after_a_refresh(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + start = time.time() + readers.guard += [guard_line(start, "paused"), guard_line(start + 1, "resumed")] + end = time.time() + host.wait_settled(end) + assert host.trouble(start, end) is None # not read yet + host.refresh(after=end) + assert host.trouble(start, end) == "guard pause" + readers.guard.clear() + readers.power += [power_line(start, "Sleep"), power_line(start + 1, "Wake")] + host.refresh(after=time.time()) + assert host.trouble(start, end) == "sleep" + host.close() + assert "Sleep" in (tmp_path / "pmset.log").read_text() + + +# ---------------------------------------------------------------- the ceiling + + +def test_the_ceiling_crosses_at_its_amount_and_stays_crossed(): + ceiling = cr.Ceiling(1.0, 0.25) + ceiling.add(0.5) + assert not ceiling.exhausted() and ceiling.crossed_at is None + ceiling.add(0.25) + assert ceiling.exhausted() and ceiling.crossed_at is not None + first = ceiling.crossed_at + ceiling.add(None) + assert ceiling.crossed_at == first and ceiling.spent_usd == pytest.approx(1.0) + + +def test_the_ceiling_ignores_a_reserve(): + # No conversation stops for a model's spend: only the ceiling stops one. + assert not cr.Ceiling(1.0, 0.0).exhausted(reserve_usd=5.0) + + +# ---------------------------------------------------------------- fake runs + + +def scenario(sid: str) -> Scenario: + i = POOL.index(sid) + return Scenario( + id=sid, + state="CO", + filing_status="single", + adults=[Person(name="head", age=30 + i, employment_income=10_000.0 * (i + 1))], + children=[], + ) + + +class FakePool: + """An engine pool that answers every call and keeps what it was sent.""" + + def __init__(self, lanes, fail: bool = False): + if fail: + raise RuntimeError("tool worker failed to start: a test") + self.messages: list = [] + self.lost = 0 + self.closed = False + + def call(self, message): + self.messages.append(message) + return {"results": {"snap": {"values": {"spm": 1.0}}}}, 0.0 + + def counters(self): + return {"timeouts": 0, "crashes": 0, "lost": self.lost} + + def close(self): + self.closed = True + + +class FakeProvider: + """Conversations a test plants. specs[(model, label, household, n)] holds a + record's `status` and `error`, `turns` (default 1), `cost` per turn, + `sleep` per turn, `calls` (tool arguments, one call each, per turn), + `pressure` (high pressure during the attempt), `stall` (a host-log stall, + seconds), `guard` (a guard pause over the attempt), `resume_after` + (seconds after which the operator resumes) and `raise` (the conversation + raises).""" + + def __init__(self, world, specs=None): + self.world, self.specs = world, specs or {} + self.count: dict = {} + self.lock = threading.Lock() + + def __call__(self, lanes, sid, label, tool_call): + model = lanes.model + with self.lock: + n = self.count[(model, label, sid)] = ( + self.count.get((model, label, sid), 0) + 1 + ) + spec = self.specs.get((model, label, sid, n), {}) + readers = self.world.readers + if spec.get("raise"): + raise RuntimeError("a test raised") + if spec.get("pressure"): + with readers.lock: + readers.pressing += 1 + if spec.get("stall"): + readers.stall = spec["stall"] + start = time.time() + rec = { + "scenario_id": sid, + "model": model, + "arm": cr.arm_string(label), + "variables": list(VARIABLES), + "status": "running", + "turns": [], + "predictions": {v: 0.0 for v in VARIABLES}, + } + spend = lanes.run.ceiling + for i in range(spec.get("turns", 1)): + if spend.exhausted(): + rec["status"] = "budget_stop" + break + started = time.time() + time.sleep(spec.get("sleep", 0.0)) + completed = time.time() + cost = spec.get("cost", COST) + spend.add(cost) + turn = { + "index": i, + "request_started_at": started, + "request_completed_at": completed, + "total_cost_usd": cost, + "calls": [], + } + for arguments in spec.get("calls", []): + result, latency = tool_call(arguments) + turn["calls"].append({"arguments": arguments, "result": result}) + rec["turns"].append(turn) + if rec["status"] == "running": + rec["status"] = spec.get("status", "complete") + if spec.get("error"): + rec["error"] = spec["error"] + if spec.get("guard"): + readers.guard += [ + guard_line(math.floor(start) - 1, "paused"), + guard_line(math.ceil(time.time()) + 1, "resumed"), + ] + if spec.get("stall"): + time.sleep(spec["stall"] + 0.02) + if spec.get("pressure"): + with readers.lock: + readers.pressing -= 1 + rec["totals"] = {"cost_usd": sum(t["total_cost_usd"] for t in rec["turns"])} + if spec.get("resume_after") is not None: + folder = lanes.folder + + def resume(): + cr.write_json(folder / "RESUME.json", {"note": "fixed in a test"}) + + threading.Timer(spec["resume_after"], resume).start() + return rec + + +class World: + """A fake board, design and run.""" + + def __init__( + self, + tmp: Path, + monkeypatch, + specs=None, + caps=None, + reserve=7 * 3 * COST, + overall_cap=100.0, + spent_before=1.0, + no_output=(), + models=None, + pool=FakePool, + ): + self.tmp = tmp + prereg = tmp / "prereg.md" + prereg.write_text(PREREG) + frozen = cr.sha256_file(prereg) + master = cr.master_seed(frozen) + self.draw_order = cr.draw(master, POOL, n=N_DRAW) + board_path = tmp / "board" + board_path.mkdir() + (board_path / "reference_outputs.csv").write_text( + "scenario_id,variable,value\n" + ) + self.no_output = set(no_output) + scored = [ + (sid, v) for sid in POOL if sid not in self.no_output for v in VARIABLES + ] + self.board = { + "scenarios": {sid: scenario(sid) for sid in POOL}, + "variables": {sid: list(VARIABLES) for sid in POOL}, + "scored": pd.DataFrame(scored, columns=["scenario_id", "variable"]), + } + monkeypatch.setattr(pilot, "load_board", lambda path: self.board) + monkeypatch.setattr(pilot, "board_engine", lambda path=None: ENGINE) + monkeypatch.setitem( + pilot._STATED_ENGINE, "version", pilot._STATED_ENGINE["version"] + ) + self.models = tuple(models or cr.CHEAP + cr.FRONTIER) + self.design = { + "preregistration": str(prereg), + "frozen_sha256": frozen, + "runs": str(tmp / "runs"), + "stage": "confirm", + "board": str(board_path), + "engine": f"policyengine-us {ENGINE}", + "draw_order": self.draw_order, + "arms": {a: a for a in cr.ARMS}, + "frontier_arms": {a: f"f-{a}" for a in cr.FRONTIER_ARMS}, + "caps": {m: (caps or {}).get(m, 10.0) for m in cr.CHEAP + cr.FRONTIER}, + "reserves": {m: reserve for m in cr.CHEAP + cr.FRONTIER}, + "overall_cap": overall_cap, + "spent_before": spent_before, + } + self.readers = Readers() + self.provider = FakeProvider(self, specs) + self.run = cr.Run( + self.design, + models=self.models, + converse=self.provider, + pool_factory=pool, + completion_for=lambda profile: None, + host_factory=self.readers.host, + out=io.StringIO(), + ) + + def execute(self): + self.run.execute() + return self + + # What the runner wrote. + + def runs(self) -> Path: + return Path(self.design["runs"]) + + def lines(self, model, name="run-log.jsonl") -> list[dict]: + path = self.runs() / model / name + if not path.exists(): + return [] + return [json.loads(x) for x in path.read_text().splitlines() if x.strip()] + + def attempts(self, model) -> list[dict]: + return [x for x in self.lines(model) if "event" not in x] + + def events(self, model, kind=None) -> list[dict]: + return [ + x + for x in self.lines(model) + if x.get("event") and kind in (None, x["event"]) + ] + + def folder(self, model, label) -> Path: + arms = ( + self.design["arms"] if model in cr.CHEAP else self.design["frontier_arms"] + ) + return self.runs() / model / arms[label] / "confirm" / "households" + + +def labels_of(model): + return cr.ARMS if model in cr.CHEAP else cr.FRONTIER_ARMS + + +def host_from_files(folder: Path): + """The host evidence as the report builds it, from the archived files.""" + meta = json.loads((folder / "host.json").read_text()) + start, end = float(meta["start"]), float(meta["end"]) + samples = [ + json.loads(x) for x in (folder / "host-log.jsonl").read_text().splitlines() if x + ] + spans = cr.guard_spans( + (folder / "clamshell-guard.log").read_text(), meta["timezone"], end + ) + spans += cr.power_spans((folder / "pmset.log").read_text(), end) + times = sorted(float(s["time"]) for s in samples) + for a, b in zip([start] + times, times + [end]): + if b - a > cr.HOST_LOG_GAP: + spans.append((a, b, "host log gap")) + pressure = [float(s["time"]) for s in samples if int(s.get("pressure") or 0) > 1] + + class Host: + def trouble(self, s, e): + for a, b, what in spans: + if a <= e and b >= s: + return what + for t in pressure: + if s - cr.PRESSURE_WINDOW <= t <= e + cr.PRESSURE_WINDOW: + return "memory pressure" + return None + + return Host() + + +def run_problems(world: World) -> list[str]: + """Every rule of the run procedure, recomputed from the runner's files.""" + problems: list[str] = [] + host = host_from_files(world.runs() / "host") + position = {sid: i for i, sid in enumerate(world.draw_order)} + master = cr.master_seed(world.design["frozen_sha256"]) + all_attempts = [] + ceiling_events = {} + for model in world.models: + labels = labels_of(model) + drawn = cr.orders(master, model, world.draw_order, labels) + lines = world.attempts(model) + events = world.events(model) + stops = [e["time"] for e in events if e["event"] in ("stop", "ceiling")] + stopped_at = min(stops) if stops else math.inf + ceiling_events[model] = [e["time"] for e in events if e["event"] == "ceiling"] + cost: dict[str, float] = {} + span: dict[str, list[float]] = {} + by_run: dict = {} + for x in lines: + by_run.setdefault((x["household"], x["arm"]), []).append(x) + # Records and hashes. + for label in labels: + folder = world.folder(model, label) + finals = sorted(folder.glob("*.json")) if folder.is_dir() else [] + for path in finals: + if (path.stem, label) not in by_run: + problems.append( + f"{model}/{label}/{path.stem}: a record with no line" + ) + for (sid, label), xs in sorted(by_run.items()): + at = f"{model}/{label}/{sid}" + xs.sort(key=lambda x: x["attempt"]) + if [x["attempt"] for x in xs] != list(range(1, len(xs) + 1)): + problems.append(f"{at}: attempts {[x['attempt'] for x in xs]}") + continue + final_path = world.folder(model, label) / f"{sid}.json" + final = json.loads(final_path.read_text()) + entries = final.get("attempts") or [] + if len(entries) != len(xs) - 1: + problems.append(f"{at}: {len(entries)} entries for {len(xs)} attempts") + continue + for k, x in enumerate(xs, start=1): + if k < len(xs): + entry = entries[k - 1] + path = world.folder(model, label) / entry["file"] + rec = json.loads(path.read_text()) + if entry["n"] != k or entry["sha256"] != cr.sha256_file(path): + problems.append(f"{at}#{k}: its entry does not hash its file") + if entry["class"] != x["class"] or entry["status"] != rec["status"]: + problems.append(f"{at}#{k}: its entry is not its line's") + if x["sha256"] != entry["sha256"]: + problems.append(f"{at}#{k}: its line's hash is not its file's") + else: + rec = final + if x["sha256"] != cr.sha256_file(final_path): + problems.append( + f"{at}#{k}: its line's hash is not its record's" + ) + cls, _ = cr.classify(rec, x["start"], x["end"], host) + rule = cr.action(cls, k) + if x["class"] != cls or x["action"] != rule: + problems.append( + f"{at}#{k}: logged {x['class']}/{x['action']}, " + f"recomputed {cls}/{rule}" + ) + if x["status"] != rec["status"] or x["lane"] != 1 + position[sid] % 2: + problems.append(f"{at}#{k}: status or lane") + if k < len(xs) and rule not in ("rerun", "pause-rerun"): + problems.append(f"{at}#{k}: rerun after {cls}") + if k == len(xs) and rule in ("rerun", "pause-rerun") and not stops: + problems.append( + f"{at}#{k}: a rerun due and never run, with no stop" + ) + times = [ + t[key] + for t in rec.get("turns") or [] + for key in ("request_started_at", "request_completed_at") + ] + if times and (min(times) < x["start"] or max(times) > x["end"]): + problems.append(f"{at}#{k}: requests outside its start and end") + if x["start"] > stopped_at: + problems.append(f"{at}#{k}: started after a stop") + cost[sid] = cost.get(sid, 0.0) + cr.attempt_cost(rec) + s, e = span.get(sid, [math.inf, -math.inf]) + span[sid] = [min(s, x["start"]), max(e, x["end"])] + all_attempts.append((model, sid, label, k, x, rec)) + # The schedule. + for lane in (1, 2): + mine = sorted( + (x for x in lines if x["lane"] == lane), key=lambda x: x["start"] + ) + for a, b in zip(mine, mine[1:]): + if b["start"] < a["end"]: + problems.append(f"{model} lane {lane}: attempts overlap") + blocks: list = [] + for x in mine: + if blocks and blocks[-1][0] == x["household"]: + blocks[-1][1].append(x) + else: + blocks.append((x["household"], [x])) + sids = [s for s, _ in blocks] + if len(set(sids)) != len(sids) or sids != sorted(sids, key=position.get): + problems.append( + f"{model} lane {lane}: households split or out of order" + ) + for i, (sid, block) in enumerate(blocks): + got = [(x["attempt"], x["arm"]) for x in block] + firsts = [a for n, a in got if n == 1] + want = [(1, a) for a in firsts] + for n in range(2, cr.MAX_RERUNS + 2): + again = {x["arm"] for x in block if x["attempt"] == n} + want += [(n, a) for a in drawn[sid] if a in again] + cut = firsts != drawn[sid] + if got != want or firsts != drawn[sid][: len(firsts)]: + problems.append(f"{model}/{sid}: ran {got}, drawn {drawn[sid]}") + if cut and not (stops and i == len(blocks) - 1): + problems.append(f"{model}/{sid}: cut short without a stop") + # Admission. + decisions = sorted( + world.lines(model, "admission-log.jsonl"), key=lambda x: x["time"] + ) + cap = world.design["caps"][model] + reserve = world.design["reserves"][model] + admitted: dict[str, float] = {} + refused = False + for x in decisions: + sid, t = x["household"], x["time"] + if sid in world.no_output: + problems.append( + f"{model}/{sid}: a household with no scored output decided" + ) + finished = [h for h in admitted if h in span and span[h][1] <= t] + spend = sum(cost[h] for h in finished) + under = sum(1 for h in admitted if h not in finished) + 1 + ok = not refused and spend + under * reserve <= cap + cr.ADMISSION_SLACK + if x["decision"] != ("admit" if ok else "refuse"): + problems.append(f"{model}/{sid}: logged {x['decision']}, rule {ok}") + if abs(x["finished_spend"] - spend) > 1e-9 or x["under_way"] != under: + problems.append( + f"{model}/{sid}: spend {x['finished_spend']} vs {spend}" + ) + if x["lane"] != 1 + position[sid] % 2: + problems.append(f"{model}/{sid}: decided on the other lane") + if x["decision"] == "admit": + admitted[sid] = t + else: + refused = True + for sid, (s, _) in span.items(): + if sid not in admitted or s < admitted[sid]: + problems.append(f"{model}/{sid}: ran without, or before, its admission") + for lane in (1, 2): + expect = [ + s + for s in world.draw_order + if 1 + position[s] % 2 == lane and s not in world.no_output + ] + decided = [x["household"] for x in decisions if x["lane"] == lane] + if decided != expect[: len(decided)]: + problems.append(f"{model} lane {lane}: decisions skip a household") + if len(decided) < len(expect) and not (refused or stops): + problems.append(f"{model} lane {lane}: stopped deciding early") + if not stops and set( + a for a in admitted if 1 + position[a] % 2 == lane + ) - set(span): + problems.append(f"{model} lane {lane}: admitted and never run") + # Pauses. + resumes = sorted(e["time"] for e in events if e["event"] == "resume") + for x in lines: + if x["class"] != "operator": + continue + pauses = [ + e for e in events if e["event"] == "pause" and e["time"] >= x["end"] - 1 + ] + if not pauses: + problems.append( + f"{model}/{x['household']}: an operator failure, no pause" + ) + continue + until = next((t for t in resumes if t >= pauses[0]["time"]), math.inf) + if any(x["end"] < y["start"] < until for y in lines): + problems.append(f"{model}: an attempt started during a pause") + # The ceiling, over every model. + total = world.design["spent_before"] + sum( + cr.attempt_cost(r) for *_, r in all_attempts + ) + ceiling = world.design["overall_cap"] + cr.CEILING_MARGIN + if total >= ceiling and not all(ceiling_events.values()): + problems.append(f"spend {total} reached the ceiling and a model has no event") + for times in ceiling_events.values(): + for t in times: + if any(x["start"] > t for *_, x, _ in all_attempts): + problems.append("an attempt started after the ceiling") + # Conservation: the ceiling counted exactly the records' spend. + if abs(world.run.ceiling.spent_usd - total) > 1e-9: + problems.append( + f"the ceiling counted {world.run.ceiling.spent_usd}, records {total}" + ) + return problems + + +@pytest.fixture +def world(tmp_path, monkeypatch, fast): + """A fake run to plant failures in, then execute.""" + + def make(**kwargs): + return World(tmp_path, monkeypatch, **kwargs) + + return make + + +def test_an_untouched_run_runs_every_household_and_arm(world): + w = world().execute() + assert run_problems(w) == [] + for model in w.models: + labels = labels_of(model) + attempts = w.attempts(model) + assert len(attempts) == N_DRAW * len(labels) + assert {x["class"] for x in attempts} == {"ok"} + assert all( + x["decision"] == "admit" for x in w.lines(model, "admission-log.jsonl") + ) + manifests = list((w.runs() / model).glob("manifest-*.json")) + assert len(manifests) == 1 + m = json.loads(manifests[0].read_text()) + assert m["arms"] == { + label: { + "arm": cr.arm_string(label), + "features": sorted(cr.arm_profile(model, label).tool_features), + } + for label in labels + } + assert m["treatment_commit"] == cr.TREATMENT_COMMIT + assert m["cap_usd"] == 10.0 and m["reserve_usd"] == pytest.approx(0.21) + assert m["board_engine"] == f"policyengine-us {ENGINE}" + assert not (w.runs() / model / "in-flight").exists() or not any( + (w.runs() / model / "in-flight").iterdir() + ) + host = json.loads((w.runs() / "host" / "host.json").read_text()) + assert host["timezone"] == TZ and host["end"] > host["start"] + + +def test_households_alternate_lanes_in_draw_order(world): + w = world(models=("gpt-6-luna",)).execute() + lanes = {x["household"]: x["lane"] for x in w.attempts("gpt-6-luna")} + assert [lanes[s] for s in w.draw_order] == [1, 2] * (N_DRAW // 2) + + +def test_each_household_runs_its_arms_in_its_drawn_order(world): + w = world(models=("claude-haiku-5.5",)).execute() + master = cr.master_seed(w.design["frozen_sha256"]) + drawn = cr.orders(master, "claude-haiku-5.5", w.draw_order, cr.ARMS) + for sid in w.draw_order: + mine = sorted( + (x for x in w.attempts("claude-haiku-5.5") if x["household"] == sid), + key=lambda x: x["start"], + ) + assert [x["arm"] for x in mine] == drawn[sid] + + +def test_each_call_carries_its_arm_s_features(world): + calls = {"situation": "{}", "variables": ["snap"]} + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + w.provider.specs = {(model, label, sid, 1): {"calls": [calls]} for label in cr.ARMS} + w.execute() + messages = w.run.lanes[model].pool.messages + assert len(messages) == len(cr.ARMS) + raw = [m for m in messages if not isinstance(m, tuple)] + framed = [m for m in messages if isinstance(m, tuple)] + assert raw == [calls] # the tool v1 arm: as a featureless pool sends it + want = { + tuple(sorted(cr.arm_profile(model, label).tool_features)) + for label in cr.ARMS + if label != "pe" + } + assert {m[1] for m in framed} == want + assert all(m[0] == pilot.CALL_FRAME and m[2] == calls for m in framed) + # The worker reads each frame's features as the arm's. + for m in framed: + assert pilot.frame_features(m[1]) == frozenset(m[1]) + assert run_problems(w) == [] + + +def test_households_with_no_scored_output_are_never_decided(world): + model = "gemini-4-argon" + master = cr.master_seed(hashlib.sha256(PREREG.encode()).hexdigest()) + skipped = cr.draw(master, POOL, n=N_DRAW)[1] + w = world(models=(model,), no_output=(skipped,)).execute() + assert skipped not in { + x["household"] for x in w.lines(model, "admission-log.jsonl") + } + assert skipped not in {x["household"] for x in w.attempts(model)} + assert run_problems(w) == [] + + +def test_a_provider_error_is_rerun_after_the_household_s_first_pass(world): + model = "claude-haiku-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + master = cr.master_seed(w.design["frozen_sha256"]) + first = cr.orders(master, model, w.draw_order, cr.ARMS)[sid][0] + error = {"status": "provider_error", "error": "RateLimitError: slow down"} + w.provider.specs = {(model, first, sid, 1): error, (model, first, sid, 2): error} + w.execute() + mine = sorted( + (x for x in w.attempts(model) if x["household"] == sid), + key=lambda x: x["start"], + ) + assert [(x["attempt"], x["arm"]) for x in mine][-2:] == [(2, first), (3, first)] + assert [x["class"] for x in mine if x["arm"] == first] == ["provider"] * 2 + ["ok"] + final = json.loads((w.folder(model, first) / f"{sid}.json").read_text()) + assert [e["n"] for e in final["attempts"]] == [1, 2] + assert final["attempts"][0]["reason"] == "RateLimitError" + assert (w.folder(model, first) / "attempts" / f"{sid}-1.json").exists() + assert run_problems(w) == [] + + +def test_a_third_attempt_stands_whatever_its_class(world): + model = "gpt-6-sol" + w = world(models=(model,)) + sid = w.draw_order[1] + error = {"status": "provider_error", "error": "APIConnectionError: reset"} + w.provider.specs = {(model, "pe3", sid, n): error for n in (1, 2, 3)} + w.execute() + mine = [x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe3"] + assert sorted((x["attempt"], x["action"]) for x in mine) == [ + (1, "rerun"), + (2, "rerun"), + (3, "stands"), + ] + assert run_problems(w) == [] + + +def test_host_trouble_during_an_attempt_reruns_it(world): + model = "gemini-3.5-flash-lite" + w = world(models=(model,)) + pressured, stalled = w.draw_order[0], w.draw_order[2] # both lane 1 + w.provider.specs = { + (model, "loo-flat", pressured, 1): {"pressure": True, "sleep": 0.05}, + (model, "pe", stalled, 1): {"stall": 0.15}, + } + w.execute() + + def first(sid, label): + line = next( + x + for x in w.attempts(model) + if (x["household"], x["arm"], x["attempt"]) == (sid, label, 1) + ) + final = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + return line, final["attempts"][0] + + line, entry = first(pressured, "loo-flat") + assert line["class"] == entry["class"] == "host" and line["action"] == "rerun" + assert "memory pressure 2 at" in entry["reason"] + line, entry = first(stalled, "pe") + assert line["class"] == "host" and "host log gap" in entry["reason"] + assert run_problems(w) == [] + + +def test_a_guard_pause_over_an_attempt_reruns_it(world): + model = "gpt-6-sol" + w = world(models=(model,)) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe3", sid, 1): {"guard": True}} + w.execute() + line = next( + x + for x in w.attempts(model) + if (x["household"], x["arm"], x["attempt"]) == (sid, "pe3", 1) + ) + assert line["class"] == "host" + final = json.loads((w.folder(model, "pe3") / f"{sid}.json").read_text()) + assert final["attempts"][0]["reason"].startswith("guard pause: guard pause ") + archived = (w.runs() / "host" / "clamshell-guard.log").read_text() + assert "paused pids" in archived + assert run_problems(w) == [] + + +def test_an_operator_failure_pauses_the_model_until_resumed(world): + model = "gpt-6-luna" + w = world(models=(model, "claude-haiku-5.5")) + sid = w.draw_order[0] + w.provider.specs = { + (model, "pe3", sid, 1): { + "status": "provider_error", + "error": "AuthenticationError: bad key", + "turns": 0, + "resume_after": 0.4, + } + } + w.execute() + pauses, resumes = w.events(model, "pause"), w.events(model, "resume") + assert len(pauses) == 1 and len(resumes) == 1 + assert resumes[0]["reason"] == "fixed in a test" + started = [x["start"] for x in w.attempts(model)] + assert not [t for t in started if pauses[0]["time"] < t < resumes[0]["time"]] + assert resumes[0]["time"] - pauses[0]["time"] >= 0.35 + rerun = [ + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe3" + ] + assert [x["action"] for x in sorted(rerun, key=lambda x: x["attempt"])] == [ + "pause-rerun", + "stands", + ] + assert not w.events("claude-haiku-5.5", "pause") # another model runs on + assert list((w.runs() / model).glob("resume-*.json")) + assert run_problems(w) == [] + + +def test_a_request_class_on_the_first_request_stops_the_model(world): + model = "claude-opus-5.5" + w = world(models=(model, "gpt-6-sol")) + sid = w.draw_order[2] + w.provider.specs = { + (model, "pe", sid, 1): { + "status": "provider_error", + "error": "BadRequestError: bad tools", + "turns": 0, + } + } + w.execute() + stops = w.events(model, "stop") + assert len(stops) == 1 and "first request" in stops[0]["reason"] + assert not [x for x in w.attempts(model) if x["start"] > stops[0]["time"]] + assert len(w.attempts("gpt-6-sol")) == N_DRAW * 2 # the other model finishes + assert run_problems(w) == [] + + +def test_one_inconclusive_error_in_six_households_stops_the_model(world): + model = "gpt-6-luna" + w = world(models=(model,)) + error = {"status": "provider_error", "error": "ProviderEmpty: no choices"} + w.provider.specs = {(model, "pe", sid, 1): error for sid in w.draw_order} + w.execute() + stops = w.events(model, "stop") + assert len(stops) == 1 + assert ( + stops[0]["reason"] == "provider_error: ProviderEmpty ended runs in 6 households" + ) + assert run_problems(w) == [] + + +def test_five_households_with_one_inconclusive_error_do_not_stop(world): + model = "gpt-6-luna" + w = world(models=(model,)) + error = {"status": "provider_error", "error": "ProviderEmpty: no choices"} + w.provider.specs = {(model, "pe", sid, 1): error for sid in w.draw_order[:5]} + w.execute() + assert not w.events(model, "stop") + assert len(w.attempts(model)) == N_DRAW * len(cr.ARMS) + assert run_problems(w) == [] + + +def test_a_refusal_ends_both_lanes_decisions(world): + model = "claude-haiku-5.5" + # Each household costs 7 arms x $0.01; the reserve is $0.21 per household. + w = world(models=(model,), caps={model: 0.5}).execute() + decisions = w.lines(model, "admission-log.jsonl") + refused = [x for x in decisions if x["decision"] == "refuse"] + after = decisions[decisions.index(refused[0]) :] + assert refused and all(x["decision"] == "refuse" for x in after) + admitted = {x["household"] for x in decisions if x["decision"] == "admit"} + assert {x["household"] for x in w.attempts(model)} == admitted + assert run_problems(w) == [] + + +def test_spend_never_stops_a_conversation_below_the_ceiling(world): + model = "gemini-4-argon" + w = world(models=(model,), caps={model: 0.05}, reserve=0.01) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe", sid, 1): {"turns": 4, "cost": 0.5}} + w.execute() + # $2 on one arm, far past the cap: it ran to the end; later households + # were refused. + ran = [x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe"] + assert ran[0]["status"] == "complete" + assert any(x["decision"] == "refuse" for x in w.lines(model, "admission-log.jsonl")) + assert run_problems(w) == [] + + +def test_the_hard_ceiling_stops_every_model(world): + w = world( + models=("claude-haiku-5.5", "gpt-6-sol", "gemini-4-argon"), + overall_cap=0.0, + spent_before=24.0, + ) + sid = w.draw_order[1] + # One long conversation crosses $25 while the others run. + w.provider.specs = { + ("gpt-6-sol", "pe3", sid, 1): {"turns": 3, "cost": 0.5, "sleep": 0.02} + } + w.execute() + events = {m: w.events(m, "ceiling") for m in w.models} + assert all(len(e) == 1 for e in events.values()) + crossed = events["gpt-6-sol"][0]["time"] + assert {e[0]["time"] for e in events.values()} == {crossed} + assert w.run.ceiling.spent_usd >= 25.0 + for m in w.models: + assert not [x for x in w.attempts(m) if x["start"] > crossed] + assert run_problems(w) == [] + + +def test_a_conversation_under_way_at_the_ceiling_ends_budget_stop(world): + model = "gpt-6-luna" + w = world(models=(model,), overall_cap=0.0, spent_before=24.9) + sid = w.draw_order[0] + master = cr.master_seed(w.design["frozen_sha256"]) + first = cr.orders(master, model, w.draw_order, cr.ARMS)[sid][0] + w.provider.specs = {(model, first, sid, 1): {"turns": 5, "cost": 0.06}} + w.execute() + line = next( + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == first + ) + assert line["status"] == "budget_stop" and line["class"] == "ceiling" + assert line["action"] == "stopped" + assert run_problems(w) == [] + + +def test_an_attempt_that_raises_is_saved_as_a_harness_error_record(world): + model = "claude-opus-5.5" + w = world(models=(model,)) + sid = w.draw_order[3] + w.provider.specs = {(model, "pe3", sid, 1): {"raise": True}} + w.execute() + replaced = json.loads( + (w.folder(model, "pe3") / "attempts" / f"{sid}-1.json").read_text() + ) + assert replaced["status"] == "harness_error" and replaced["turns"] == [] + assert replaced["error"] == "RuntimeError: a test raised" + assert replaced["tool_features"] == sorted( + cr.arm_profile(model, "pe3").tool_features + ) + line = next( + x + for x in w.attempts(model) + if x["household"] == sid and x["attempt"] == 1 and x["arm"] == "pe3" + ) + assert line["class"] == "none" and line["action"] == "rerun" + assert run_problems(w) == [] + + +def test_a_pool_that_lost_every_worker_is_started_again(world): + model = "gpt-6-sol" + starts = [] + + class Losing(FakePool): + def __init__(self, lanes): + super().__init__(lanes) + starts.append(self) + + def call(self, message): + self.lost = 2 # every worker gone after this call + return super().call(message) + + w = world(models=(model,), pool=Losing) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe", sid, 1): {"calls": [{"situation": "{}"}]}} + w.execute() + assert len(starts) == 2 and starts[0].closed + assert run_problems(w) == [] + + +def test_a_pool_that_cannot_start_again_stops_the_model(world, monkeypatch): + model = "gpt-6-sol" + made = [] + + def pool(lanes): + made.append(1) + return FakePool(lanes, fail=len(made) > 1) + + w = world(models=(model,), pool=pool) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe", sid, 1): {"calls": [{"situation": "{}"}]}} + original = FakePool.call + + def losing(self, message): + self.lost = 2 + return original(self, message) + + monkeypatch.setattr(FakePool, "call", losing) + w.execute() + stops = w.events(model, "stop") + assert stops and "engine pool" in stops[0]["reason"] + assert run_problems(w) == [] + + +def test_an_interrupt_stops_every_model_and_lets_attempts_end(world): + w = world(models=("gpt-6-luna", "gpt-6-sol")) + sid = w.draw_order[0] + w.provider.specs = {("gpt-6-luna", lbl, sid, 1): {"sleep": 0.1} for lbl in cr.ARMS} + threading.Timer(0.15, lambda: w.run.interrupt(2, None)).start() + w.execute() + for m in w.models: + stops = w.events(m, "stop") + assert len(stops) == 1 and stops[0]["reason"] == "operator stop (signal 2)" + assert not [x for x in w.attempts(m) if x["start"] > stops[0]["time"]] + luna = [x for x in w.attempts("gpt-6-luna") if x["household"] == sid] + assert luna and len(luna) < len(cr.ARMS) # cut by the stop + assert run_problems(w) == [] + + +KINDS = { + "ok": {}, + "provider": {"status": "provider_error", "error": "RateLimitError: slow"}, + "none": {"status": "harness_error", "error": "RuntimeError: lost", "turns": 0}, + "model": {"status": "provider_error", "error": "BadRequestError: long"}, + "inconclusive": {"status": "provider_error", "error": "ProviderEmpty: none"}, + "pressure": {"pressure": True, "sleep": 0.04}, + "incomplete": {"status": "incomplete"}, + "harness": {"status": "provider_error", "error": "BadRequestError: x", "turns": 0}, + "costly": {"turns": 3, "cost": 0.2}, +} + + +@settings( + max_examples=12, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture, HealthCheck.too_slow], +) +@given( + planted=st.lists( + st.tuples( + st.sampled_from(("claude-haiku-5.5", "gpt-6-sol")), + st.sampled_from(cr.FRONTIER_ARMS), + st.integers(0, N_DRAW - 1), + st.integers(1, 3), + st.sampled_from(sorted(KINDS)), + ), + max_size=10, + ), + cap=st.sampled_from([0.3, 0.6, 10.0]), + spent_before=st.sampled_from([0.0, 24.0]), +) +def test_every_run_keeps_the_run_procedure( + tmp_path_factory, fast, planted, cap, spent_before +): + """Whatever fails, and wherever spend lands, the records follow every rule.""" + tmp = tmp_path_factory.mktemp("run") + mp = pytest.MonkeyPatch() + try: + models = ("claude-haiku-5.5", "gpt-6-sol") + w = World( + tmp, + mp, + models=models, + caps={m: cap for m in models}, + overall_cap=0.0, + spent_before=spent_before, + ) + w.provider.specs = { + (model, label, w.draw_order[i], n): KINDS[kind] + for model, label, i, n, kind in planted + } + w.execute() + assert run_problems(w) == [] + finally: + mp.undo() + + +def test_the_checker_catches_a_planted_fault(world): + """run_problems is not vacuous: an edited log line fails it.""" + w = world(models=("gpt-6-sol",)).execute() + path = w.runs() / "gpt-6-sol" / "run-log.jsonl" + lines = path.read_text().splitlines() + first = json.loads(lines[0]) + first["class"] = "provider" + path.write_text("\n".join([json.dumps(first)] + lines[1:]) + "\n") + assert any("recomputed" in p for p in run_problems(w)) + + +# ---------------------------------------------------------------- before a run + + +def test_the_preflight_names_a_missing_field(): + assert cr.preflight({}, cr.CHEAP) == [f"the design has no {', '.join(cr.REQUIRED)}"] + + +def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( + world, tmp_path, monkeypatch +): + w = world(models=("gpt-6-luna",)) + pool = tmp_path / "pool.json" + pool.write_text(json.dumps([{"id": s} for s in POOL])) + design = dict( + w.design, + eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, + treatment_harness=str(cr.ROOT), + run_commit="0" * 40, + draw_order=list(reversed(w.draw_order)), + ) + monkeypatch.setattr(cr, "host_power", lambda: (True, False)) + problems = cr.preflight(design, ("gpt-6-luna",)) + text = "\n".join(problems) + assert "the draw, recomputed" in text + assert "not the run commit, clean" in text + assert "lid open False" in text + design["draw_order"] = w.draw_order + assert "the draw" not in "\n".join(cr.preflight(design, ("gpt-6-luna",))) + + +def test_the_preflight_refuses_a_runs_folder_with_records(world, tmp_path): + w = world(models=("gpt-6-luna",)).execute() + pool = tmp_path / "pool.json" + pool.write_text(json.dumps([{"id": s} for s in POOL])) + design = dict( + w.design, + eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, + treatment_harness=str(cr.ROOT), + run_commit="0" * 40, + ) + text = "\n".join(cr.preflight(design, ("gpt-6-luna",))) + assert "already holds records" in text and "already holds a host log" in text + + +def test_resume_writes_the_operator_s_note(tmp_path): + design = tmp_path / "design.json" + design.write_text(json.dumps({"runs": str(tmp_path / "runs")})) + assert cr.main(["resume", str(design), "gpt-6-luna", "--note", "key rotated"]) == 0 + note = json.loads((tmp_path / "runs" / "gpt-6-luna" / "RESUME.json").read_text()) + assert note["note"] == "key rotated" + + +# ---------------------------------------------------------------- the harness + + +def chat_response(calls): + from types import SimpleNamespace + + message = SimpleNamespace( + role="assistant", content=None, tool_calls=calls, function_call=None + ) + usage = SimpleNamespace( + prompt_tokens=1000, + completion_tokens=200, + total_tokens=1200, + reasoning_tokens=0, + prompt_tokens_details=SimpleNamespace(cached_tokens=0), + completion_tokens_details=None, + ) + return SimpleNamespace( + choices=[SimpleNamespace(message=message, finish_reason="tool_calls")], + usage=usage, + model="claude-haiku-5-5", + ) + + +def chat_call(name, arguments, call_id): + from types import SimpleNamespace + + return SimpleNamespace( + id=call_id, + type="function", + function=SimpleNamespace(name=name, arguments=json.dumps(arguments)), + ) + + +def stateless_model(**kwargs): + """A tool-contract model: one calculation, then its answers. It reads the + conversation so far, so the two lanes can share it.""" + if not any(m.get("role") == "tool" for m in kwargs["messages"]): + arguments = {"situation": json.dumps({"people": {}}), "variables": ["snap"]} + return chat_response([chat_call(pilot.TOOL_NAME, arguments, "c1")]) + outputs = {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + return chat_response([chat_call("submit_outputs", {"outputs": outputs}, "s1")]) + + +def test_the_harness_conversation_runs_under_the_runner(world): + model = "claude-haiku-5.5" + w = world(models=(model,)) + w.run.converse = cr.converse_with_harness + w.run.completion_for = lambda profile: stateless_model + w.execute() + attempts = w.attempts(model) + assert len(attempts) == N_DRAW * len(cr.ARMS) + assert {x["status"] for x in attempts} == {"complete"} + messages = w.run.lanes[model].pool.messages + assert len(messages) == N_DRAW * len(cr.ARMS) # one calculation per attempt + frames = [m for m in messages if isinstance(m, tuple)] + assert len(frames) == N_DRAW * (len(cr.ARMS) - 1) # every arm but pe + sid = w.draw_order[0] + rec = json.loads((w.folder(model, "loo-names") / f"{sid}.json").read_text()) + profile = cr.arm_profile(model, "loo-names") + assert rec["arm"] == cr.arm_string("loo-names") and rec["attempts"] == [] + assert rec["tool_features"] == sorted(profile.tool_features) + assert rec["predictions"] == {v: 0.0 for v in VARIABLES} + assert run_problems(w) == [] + + +def test_the_ceiling_ends_a_harness_conversation_at_its_next_turn(world): + model = "claude-haiku-5.5" + w = world(models=(model,), overall_cap=0.0, spent_before=25.0 - 1e-9) + w.run.converse = cr.converse_with_harness + w.run.completion_for = lambda profile: stateless_model + w.execute() + # The first turns cross the ceiling; each conversation under way ends + # budget_stop before its next request, and no attempt starts after. + attempts = w.attempts(model) + assert attempts and {x["status"] for x in attempts} == {"budget_stop"} + assert len(attempts) <= 2 # one per lane at most + assert len(w.events(model, "ceiling")) == 1 + assert run_problems(w) == [] From ec31e835ece3adc83dd6cbed9a5972542a9d7a6e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 03:22:37 -0400 Subject: [PATCH 03/48] Confirmatory runner: create both logs at the start A model whose lanes never decide, because the ceiling was crossed before its first admission, wrote no admission log, and the confirmatory report reads a missing log as a lost one. The runner now creates each model's run and admission logs when it writes the model's manifest. Found by the differential test in the study's scripts repo, which runs this runner and checks its records with the report. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 3 +++ tests/test_confirmatory_run.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index cc61b52b..1b7b0566 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -1179,6 +1179,9 @@ def execute(self) -> None: write_json( lanes.folder / f"manifest-{stamp}.json", lanes.manifest(), indent=2 ) + # A model whose lanes never decide still has both logs. + for name in ("run-log.jsonl", "admission-log.jsonl"): + (lanes.folder / name).touch() starting[model] = threading.Thread( target=lanes.ensure_pool, daemon=True ) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index f42a1c8e..c7651586 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -1134,6 +1134,31 @@ def test_the_hard_ceiling_stops_every_model(world): assert run_problems(w) == [] +def test_a_model_that_decides_nothing_still_has_both_logs(world): + """haiku's first attempt crosses the ceiling before gpt-6-sol's lanes, + held back, decide: gpt-6-sol's run log holds only the ceiling event and + its admission log is empty.""" + w = world( + models=("claude-haiku-5.5", "gpt-6-sol"), overall_cap=0.0, spent_before=24.5 + ) + sid = w.draw_order[0] + first = w.run.lanes["claude-haiku-5.5"].orders[sid][0] + w.provider.specs = {("claude-haiku-5.5", first, sid, 1): {"cost": 1.0}} + held = w.run.lanes["gpt-6-sol"] + admit = held.admit + + def late(sid, lane): + time.sleep(0.3) + return admit(sid, lane) + + held.admit = late + w.execute() + admission = w.runs() / "gpt-6-sol" / "admission-log.jsonl" + assert admission.exists() and admission.read_text() == "" + assert [x["event"] for x in w.lines("gpt-6-sol")] == ["ceiling"] + assert run_problems(w) == [] + + def test_a_conversation_under_way_at_the_ceiling_ends_budget_stop(world): model = "gpt-6-luna" w = world(models=(model,), overall_cap=0.0, spent_before=24.9) From 808ccc8abc3962e534d52633f2ff5909d02a64da Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 04:11:22 -0400 Subject: [PATCH 04/48] Confirmatory runner: a clock that never goes back, and three test-oracle fixes The wall clock steps back by up to about 0.1 ms between threads on the run's host (40 backward steps in 190 million readings over 12 threads, the largest 91 us). The run records order attempts, stops, admission decisions and the ceiling's crossing by their times, and the confirmatory report compares them exactly: an attempt's end and the next start in its lane can be microseconds apart, so a step back could make a valid run fail the report. `clock()` gives time.time(), never earlier than a time it gave before, and each of those times is taken under the lock that orders it. A test reads the wall clock up to 0.2 ms early everywhere; without `clock()` it fails every time (attempts overlap, a household runs before its admission). The test oracle had three faults, two of them exposed by the full suite: - A household a stop cut short was taken to finish at its last attempt. The runner counts it under way until the stop, as the report now does. A new test makes the case deterministic (it failed the old oracle with the flake's message, spend 0.02 vs 0.03). - The scaled host-log gap was 90 ms, and under a load of 12 the sampler stalled past it, so an untouched run got host reruns. The gap is 0.5 s in the tests, and the stall tests stall 0.8 s. - The oracle allows the harness's raw request times a second outside their attempt, as the report does. 121 tests pass, 8 runs of the full suite in a row at a load of 12 to 146. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 25 ++++++-- tests/test_confirmatory_run.py | 113 +++++++++++++++++++++++++++++---- 2 files changed, 121 insertions(+), 17 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 1b7b0566..0b7f0f9b 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -331,6 +331,21 @@ def attempt_cost(record: dict) -> float: return float((record.get("totals") or {}).get("cost_usd") or 0.0) +_clock_lock = threading.Lock() +_clock_last = 0.0 + + +def clock() -> float: + """`time.time()`, never earlier than a time it gave before. The wall clock + can step back by about 0.1 ms, even within a thread, and the run records order + attempts, stops, admission decisions and the ceiling's crossing by the + times this gives; each is taken under the lock that orders it.""" + global _clock_last + with _clock_lock: + _clock_last = max(time.time(), _clock_last) + return _clock_last + + # ---------------------------------------------------------------- host evidence @@ -613,7 +628,7 @@ def add(self, usd: float | None) -> None: with self._lock: self.spent_usd += float(usd or 0.0) if self.crossed_at is None and self.spent_usd >= self.budget_usd: - self.crossed_at = time.time() + self.crossed_at = clock() def exhausted(self, reserve_usd: float = 0.0) -> bool: with self._lock: @@ -716,7 +731,7 @@ def write_line(self, line: dict, name: str = "run-log.jsonl") -> None: def event(self, kind: str, reason: str, at: float | None = None) -> None: self.write_line( - {"event": kind, "time": time.time() if at is None else at, "reason": reason} + {"event": kind, "time": clock() if at is None else at, "reason": reason} ) self.run.say(f"{self.model}: {kind} ({reason})") @@ -765,7 +780,7 @@ def admit(self, sid: str, lane: int) -> bool: self.cond.wait(timeout=WAIT_SECONDS) if self.halted(): return False - now = time.time() + now = clock() finished = [h for h, s in self.state.items() if s == "finished"] spend = sum(self.spent[h] for h in finished) under_way = len(self.state) - len(finished) + 1 @@ -803,7 +818,7 @@ def start_attempt(self) -> float | None: with self.run.ceiling.lock: if self.stopped or self.run.ceiling.crossed_at is not None: return None - return time.time() + return clock() def end_attempt( self, sid: str, label: str, n: int, start: float, record: dict, last: bool @@ -812,7 +827,7 @@ def end_attempt( once: an operator failure pauses the lanes, a harness failure stops them, before any other attempt can start.""" with self.cond: - end = time.time() + end = clock() self.spent[sid] += attempt_cost(record) attempt = Attempt(label, n, start, end, record) cls, detail = classify(record, start, end, None) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index c7651586..63acc702 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -42,6 +42,10 @@ TZ = "America/New_York" PREREG = "A frozen pre-registration for a test.\n" ENGINE = "2.60.4" +# The harness times requests with the raw wall clock, which steps back by +# about 0.1 ms on this host, even within a thread; the report allows a second, as +# here, which also covers a small correction of the clock. +CLOCK_SLACK = 1.0 # ---------------------------------------------------------------- streams @@ -283,9 +287,11 @@ def host(self, folder): @pytest.fixture def fast(monkeypatch): - """Windows scaled down a thousandfold, so a fake run takes a second.""" + """Windows scaled down a thousandfold, so a fake run takes a second; the + host-log gap less, so a sampler a busy host stalls past 90 ms (seen at a + load of 12) is not a gap.""" monkeypatch.setattr(cr, "PRESSURE_WINDOW", 0.06) - monkeypatch.setattr(cr, "HOST_LOG_GAP", 0.09) + monkeypatch.setattr(cr, "HOST_LOG_GAP", 0.5) monkeypatch.setattr(cr, "WAIT_SECONDS", 0.005) monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) @@ -328,8 +334,8 @@ def test_a_stalled_host_log_is_a_gap(tmp_path, fast): host = readers.host(tmp_path) time.sleep(0.03) start = time.time() - readers.stall = 0.2 - time.sleep(0.25) + readers.stall = 0.8 + time.sleep(0.85) end = time.time() host.wait_settled(end) host.refresh(after=end) @@ -358,6 +364,34 @@ def test_guard_pauses_and_sleeps_reach_an_attempt_after_a_refresh(tmp_path, fast # ---------------------------------------------------------------- the ceiling +def test_the_clock_never_goes_back(monkeypatch): + monkeypatch.setattr(cr, "_clock_last", cr._clock_last) # restored after + base = time.time() + 1000.0 + readings = iter([base, base - 1e-4, base + 2e-4, base - 5.0, base + 3e-4]) + monkeypatch.setattr(time, "time", lambda: next(readings)) + assert [cr.clock() for _ in range(5)] == [ + base, + base, + base + 2e-4, + base + 2e-4, + base + 3e-4, + ] + + +def test_a_wall_clock_that_steps_back_keeps_the_run_procedure(world, monkeypatch): + """Every reading of the wall clock up to 0.2 ms early (steps back of up to + 0.1 ms seen on this host): the run's times still order its attempts, + stops and decisions.""" + import random + + jitter, real = random.Random(7), time.time + monkeypatch.setattr(time, "time", lambda: real() - jitter.uniform(0.0, 2e-4)) + w = world().execute() + assert run_problems(w) == [] + for model in w.models: + assert len(w.attempts(model)) == N_DRAW * len(labels_of(model)) + + def test_the_ceiling_crosses_at_its_amount_and_stays_crossed(): ceiling = cr.Ceiling(1.0, 0.25) ceiling.add(0.5) @@ -648,6 +682,7 @@ def run_problems(world: World) -> list[str]: cost: dict[str, float] = {} span: dict[str, list[float]] = {} by_run: dict = {} + cut_short: set[str] = set() # an arm or a rerun the stop kept from running for x in lines: by_run.setdefault((x["household"], x["arm"]), []).append(x) # Records and hashes. @@ -699,17 +734,25 @@ def run_problems(world: World) -> list[str]: problems.append(f"{at}#{k}: status or lane") if k < len(xs) and rule not in ("rerun", "pause-rerun"): problems.append(f"{at}#{k}: rerun after {cls}") - if k == len(xs) and rule in ("rerun", "pause-rerun") and not stops: - problems.append( - f"{at}#{k}: a rerun due and never run, with no stop" - ) + if k == len(xs) and rule in ("rerun", "pause-rerun"): + cut_short.add(sid) + if not stops: + problems.append( + f"{at}#{k}: a rerun due and never run, with no stop" + ) times = [ t[key] for t in rec.get("turns") or [] for key in ("request_started_at", "request_completed_at") ] - if times and (min(times) < x["start"] or max(times) > x["end"]): - problems.append(f"{at}#{k}: requests outside its start and end") + if times and ( + min(times) < x["start"] - CLOCK_SLACK + or max(times) > x["end"] + CLOCK_SLACK + ): + problems.append( + f"{at}#{k}: requests {min(times)!r}-{max(times)!r} outside " + f"its start and end {x['start']!r}-{x['end']!r}" + ) if x["start"] > stopped_at: problems.append(f"{at}#{k}: started after a stop") cost[sid] = cost.get(sid, 0.0) + cr.attempt_cost(rec) @@ -743,6 +786,8 @@ def run_problems(world: World) -> list[str]: again = {x["arm"] for x in block if x["attempt"] == n} want += [(n, a) for a in drawn[sid] if a in again] cut = firsts != drawn[sid] + if cut: + cut_short.add(sid) if got != want or firsts != drawn[sid][: len(firsts)]: problems.append(f"{model}/{sid}: ran {got}, drawn {drawn[sid]}") if cut and not (stops and i == len(blocks) - 1): @@ -755,13 +800,17 @@ def run_problems(world: World) -> list[str]: reserve = world.design["reserves"][model] admitted: dict[str, float] = {} refused = False + # A household a stop cut short is under way until the stop. + ended = { + h: max(e, stopped_at) if h in cut_short else e for h, (_, e) in span.items() + } for x in decisions: sid, t = x["household"], x["time"] if sid in world.no_output: problems.append( f"{model}/{sid}: a household with no scored output decided" ) - finished = [h for h in admitted if h in span and span[h][1] <= t] + finished = [h for h in admitted if h in ended and ended[h] <= t] spend = sum(cost[h] for h in finished) under = sum(1 for h in admitted if h not in finished) + 1 ok = not refused and spend + under * reserve <= cap + cr.ADMISSION_SLACK @@ -970,7 +1019,7 @@ def test_host_trouble_during_an_attempt_reruns_it(world): pressured, stalled = w.draw_order[0], w.draw_order[2] # both lane 1 w.provider.specs = { (model, "loo-flat", pressured, 1): {"pressure": True, "sleep": 0.05}, - (model, "pe", stalled, 1): {"stall": 0.15}, + (model, "pe", stalled, 1): {"stall": 0.8}, } w.execute() @@ -1243,6 +1292,46 @@ def losing(self, message): assert run_problems(w) == [] +def test_a_household_a_stop_cuts_short_is_under_way_until_the_stop(world): + """Lane 1's first household loses the pool in its first arm, and the + restart fails 0.6 s later. In between, lane 2 finishes its household and + decides its next: the household lane 1 cut short has ended its last + attempt, but it is still under way, so its spend is not counted.""" + model = "gpt-6-sol" + made = [] + + class Losing(FakePool): + def call(self, message): + self.lost = 2 + return super().call(message) + + def pool(lanes): + made.append(1) + if len(made) > 1: + time.sleep(0.6) + raise RuntimeError("tool worker failed to start: a test") + return Losing(lanes) + + w = world(models=(model,), pool=pool) + cut, other, after = w.draw_order[0], w.draw_order[1], w.draw_order[3] + orders = w.run.lanes[model].orders + w.provider.specs = { + (model, orders[cut][0], cut, 1): {"sleep": 0.1, "calls": [{"situation": "{}"}]}, + (model, orders[other][0], other, 1): {"sleep": 0.02}, + (model, orders[other][1], other, 1): {"sleep": 0.3}, + } + w.execute() + stop = w.events(model, "stop")[0]["time"] + decided = {x["household"]: x for x in w.lines(model, "admission-log.jsonl")} + assert decided[after]["time"] < stop # decided while the cut household waited + assert decided[after]["finished_spend"] == pytest.approx(2 * COST) # `other` only + assert decided[after]["under_way"] == 2 # `cut` and `after` + assert [x["arm"] for x in w.attempts(model) if x["household"] == cut] == [ + orders[cut][0] + ] + assert run_problems(w) == [] + + def test_an_interrupt_stops_every_model_and_lets_attempts_end(world): w = world(models=("gpt-6-luna", "gpt-6-sol")) sid = w.draw_order[0] From fdbd411111d83556be2b8848a57052c3073529d0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 08:11:31 -0400 Subject: [PATCH 05/48] Confirmatory runner: the review's eight runner findings, stop causes, chains and anchors Answers runner-r1 (R1-R8) and the runner side of the pre-registration and report reviews: - A failed read of the guard's or the power log is tried five times; one that still fails is a runner error, which stops every model's lanes after settling and writing what had ended. A failed memory-pressure reading is no sample. - An attempt still running at an hour is abandoned: a harness-error record (AttemptDeadline) with its spend so far, a stop of cause deadline, and its later spend in late-cost events. An attempt that raises keeps the spend it counted. - Admission takes its time and checks the ceiling under the ceiling's lock. - A resume is accepted only if written after its pause began. - The signal handler only notes the signal; the main loop stops the lanes. - A sampler whose samples fail for five minutes, or whose thread ends, stops every model. - --workers must equal the design's. - Every stop names its cause. The error classes add the litellm subclasses the harness groups by isinstance, explain traces count as calls, and a Timeout on the escalated request after a truncated turn is class model. - Run, admission and host logs are hash chains; anchors.jsonl records every log's head every 15 minutes and at the end, and the design's anchor command (confirmatory_anchor.py) commits and pushes it. - In-flight records are fsynced before their rename. 189 tests pass (tests/test_confirmatory_run.py, tests/test_confirmatory_anchor.py). 19 mutants of the new rules each fail a named test. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 49 ++ scripts/confirmatory_run.py | 711 +++++++++++++++++++++++++----- tests/fake_anchor.py | 13 + tests/test_confirmatory_anchor.py | 91 ++++ tests/test_confirmatory_run.py | 641 ++++++++++++++++++++++++++- 5 files changed, 1378 insertions(+), 127 deletions(-) create mode 100644 scripts/confirmatory_anchor.py create mode 100644 tests/fake_anchor.py create mode 100644 tests/test_confirmatory_anchor.py diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py new file mode 100644 index 00000000..a4103d8a --- /dev/null +++ b/scripts/confirmatory_anchor.py @@ -0,0 +1,49 @@ +"""Push the confirmatory run's anchors outside the run. + +Usage: + python scripts/confirmatory_anchor.py --repo DIR ANCHORS + +The runner calls this every 15 minutes, and once more at its end, with the +path of its `anchors.jsonl`: each line holds every log's line count and chain +head at a time. This copies the file into DIR, a clone checked out on a branch +of its own, commits it and pushes the branch to its remote. The remote's record +of each push is the anchor's time from outside the run: a log edited after an +anchor was pushed no longer matches that anchor's head. +""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +import subprocess +import sys +from pathlib import Path + + +def git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), *args], check=True, capture_output=True, text=True + ).stdout + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--repo", type=Path, required=True, help="the anchors clone") + parser.add_argument("anchors", type=Path, help="the run's anchors.jsonl") + args = parser.parse_args(argv) + repo = args.repo.expanduser() + target = repo / "anchors.jsonl" + shutil.copyfile(args.anchors, target) + lines = target.read_text().splitlines() + git(repo, "add", "anchors.jsonl") + if git(repo, "status", "--porcelain", "--", "anchors.jsonl"): + head = hashlib.sha256(lines[-1].encode()).hexdigest() if lines else "none" + git(repo, "commit", "-q", "-m", f"anchor {len(lines)}: {head}") + git(repo, "push", "-q", "origin", "HEAD") + print(f"pushed {len(lines)} anchors") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 0b7f0f9b..de5f4089 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -30,19 +30,48 @@ stage and every model's spend since. At the crossing, no attempt starts, the conversations under way end ``budget_stop`` at their next turn, and every model's run log gets a ``ceiling`` event at the crossing's time. + The decision to admit a household and the crossing share one lock, so no + household is decided after the crossing. +- **The attempt deadline.** A conversation still running an hour after it + started is abandoned: its attempt gets a harness-error record + (``AttemptDeadline``) holding the spend so far, the model's lanes stop, + and the conversation ends ``budget_stop`` at its next turn. Its spend after + the deadline still counts toward the ceiling, and goes to the run log as + ``late-cost`` events. - **Failures.** Each attempt's class follows the pre-registration's table, in the precedence the report's ``classify`` uses, from its record and the host evidence for its start and end; its action follows from its class and its attempt number. An operator failure pauses the model's lanes until - ``resume``. A harness failure stops them, and so does one inconclusive - error that ends runs in more than five of the model's households. A - replaced attempt is saved, before its rerun, as - ``households/attempts/-.json``. -- **The host log.** Every 30 seconds: the time, the load average and the - kernel's memory-pressure level. At the end, the guard's and the power log's - lines from a day before the booking to its end. An attempt's class waits - for its host evidence to be complete: a sample more than 60 seconds after - the pass's last end, or no sample for more than 90 seconds. + ``resume``, which is accepted only if written after the pause began. A + harness failure stops them, and so does one inconclusive error that ends + runs in more than five of the model's households. A replaced attempt is + saved, before its rerun, as ``households/attempts/-.json``. +- **Stops.** Every ``stop`` event names its cause: ``harness``, + ``inconclusive``, ``deadline``, ``pool`` (the engine pool could not be + started again), ``runner-error`` (an exception in the runner itself, which + stops every model's lanes after writing what had ended), ``interrupt`` or + ``sampler``. The ceiling's events are ``ceiling`` events of cause + ``ceiling``. +- **The host log.** Every 30 seconds: the time, the monotonic clock, the + load average and the kernel's memory-pressure level (None when it could not + be read, which is no sample). At the end, the guard's and the power log's + lines from a day before the booking to its end. A read of those logs is + tried five times; one that still fails during the run is a runner error, + and at the end leaves ``host-error.txt`` in place of the excerpts. An + attempt's class waits for its host evidence to + be complete: a good sample more than 60 seconds after the pass's last end, + or no good sample for more than 90 seconds. A sampler whose samples have + failed for five minutes, or whose thread has ended, stops every model. +- **Chains and anchors.** Every line of the run logs, the admission logs and + the host log carries ``prev``, the sha256 of the line before it. The first + line's is a seed: for a model's logs, the sha256 of the frozen hash, the + model and the file's name; for the host log, of its name. Every 15 + minutes, and at the end, ``anchors.jsonl`` gets a line with every log's + line count and head and every manifest's sha256, itself chained, and the + design's ``anchor_command`` is run on it in the background; each run's + result goes to ``anchor-pushes.jsonl``. ``confirmatory_anchor.py`` is that + command: it commits the file to a clone and pushes it, so the remote holds + each anchor's time from outside the run. The records' format is the confirmatory report's (its module docstring). The report recomputes every class, action, order and admission decision from the @@ -53,9 +82,11 @@ ``tool_pilot_argon.py``, ``policybench`` package and fix module are the treatment commit's (6b00a6f0), its tree is clean at the design's run commit, the machine is on AC power with the lid open, and the runs folder holds no -records: a run is never resumed. An interrupt (SIGINT or SIGTERM) stops every -model's lanes with a stop event and lets the conversations under way end; a -second interrupt exits at once. +records: a run is never resumed. ``--workers`` must equal the design's +engine workers per model. An interrupt (SIGINT or SIGTERM) only notes the +signal; the main loop then stops every model's lanes with cause +``interrupt`` and lets the conversations under way end. A second interrupt +exits at once. """ from __future__ import annotations @@ -128,8 +159,29 @@ ADMISSION_SLACK = 1e-9 # dollars EXCERPT_LEAD = 86_400.0 # host logs are kept from a day before the booking WAIT_SECONDS = 1.0 # how often a waiting lane looks again +ATTEMPT_DEADLINE = 3600.0 # seconds; the longest development attempt took 714 +SAMPLER_SILENCE = 300.0 # seconds without a host-log sample that stop the run +READ_TRIES = 5 # reads of the guard's and the power log before a runner error +READ_WAIT = 10.0 # seconds between them +ANCHOR_SECONDS = 900.0 # how often the logs' chain heads are anchored +LATE_JOIN = 60.0 # seconds the run's end waits for an abandoned attempt +# What a stop event names as its cause. +STOP_CAUSES = frozenset( + { + "harness", + "inconclusive", + "pool", + "deadline", + "runner-error", + "interrupt", + "sampler", + } +) -# The pre-registration's error classes (its "Failures" table). +# The pre-registration's error classes (its "Failures" table): the names of +# litellm's exception classes under the harness's retryable classes (transport) +# and under its model-fatal ones (request and account). A record keeps only +# the raised class's name. TRANSPORT = frozenset( { "APIConnectionError", @@ -137,6 +189,8 @@ "APIResponseValidationError", "BadGatewayError", "InternalServerError", + "JSONSchemaValidationError", + "MidStreamFallbackError", "RateLimitError", "ServiceUnavailableError", "Timeout", @@ -146,13 +200,20 @@ REQUEST = frozenset( { "BadRequestError", + "ContentPolicyViolationError", "ContextWindowExceededError", + "ImageFetchError", "InvalidRequestError", + "LiteLLMUnknownProvider", + "RejectedRequestError", "UnprocessableEntityError", "UnsupportedParamsError", } ) ACCOUNT = frozenset({"AuthenticationError", "PermissionDeniedError", "NotFoundError"}) +# A timeout on the escalated request, the one asking for the escalation +# ceiling after a truncated turn, is the model's long generation timing out. +TIMEOUTS = frozenset({"Timeout", "RequestWallTimeoutError"}) HOST_ENGINE = frozenset({"MemoryError", "OSError"}) RERUN = frozenset({"provider", "host", "none", "operator"}) FINISHED = frozenset({"complete", "incomplete", "turn_limit"}) @@ -176,6 +237,8 @@ "reserves", "overall_cap", "spent_before", + "workers", + "anchor_command", ) @@ -254,7 +317,8 @@ def completed_turns(record: dict) -> list: def call_failures(record: dict) -> tuple[list[str], list[str]]: """(the inconclusive tool failures, the model's engine exceptions) in its - calls.""" + calls: a call's own error, and the errors in its results and in its + explanations (the explain feature's trace of a variable).""" host, model = [], [] for turn in record.get("turns") or []: for call in turn.get("calls") or []: @@ -269,39 +333,59 @@ def call_failures(record: dict) -> tuple[list[str], list[str]]: host.append("tool worker failure") elif error == "engine": (host if name in HOST_ENGINE else model).append(name) - for value in (result.get("results") or {}).values(): - if isinstance(value, dict) and value.get("error"): - name = str(value["error"]).split(":", 1)[0] - (host if name in HOST_ENGINE else model).append(name) + for group in ("results", "explanations"): + values = result.get(group) + for value in values.values() if isinstance(values, dict) else (): + if isinstance(value, dict) and value.get("error"): + name = str(value["error"]).split(":", 1)[0] + (host if name in HOST_ENGINE else model).append(name) return host, model +def escalation_timed_out(record: dict) -> bool: + """Whether the attempt ended on a timeout of its escalated request: the + harness records a truncated turn, then asks again at the escalation + ceiling, and an error there leaves the truncated turn last.""" + turns = record.get("turns") or [] + return ( + record.get("status") == "provider_error" + and error_class(record) in TIMEOUTS + and bool(turns) + and bool(turns[-1].get("truncated_and_retried")) + ) + + def classify(record: dict, start: float, end: float, host) -> tuple[str, str]: """(class, detail) of one attempt. In order of precedence: operator (an account class or insufficient_quota); harness (a request class with no - model request yet returned); ceiling (budget_stop); none (no model request - returned); host (host trouble during the attempt); provider (a transport - class); model (a request class); inconclusive (any other provider error, - any harness error, a tool timeout, a worker failure, MemoryError or - OSError); otherwise ok, or model when the engine raised on a household the - model built. `host` is None to decide only the classes before host.""" + completed model turn yet); ceiling (budget_stop); none (no turn recorded: + no model request returned, truncated or not); host (host trouble during + the attempt); model (a timeout of the escalated request); provider (a + transport class); model (a request class); inconclusive (any other + provider error, any harness error, a tool timeout, a worker failure, + MemoryError or OSError); otherwise ok, or model when the engine raised on + a household the model built. `host` is None to decide only the classes + before host.""" status, error = record.get("status"), str(record.get("error") or "") name = error_class(record) - reached = bool(completed_turns(record)) + returned = bool(record.get("turns")) + completed = bool(completed_turns(record)) if status == "provider_error" and ( name in ACCOUNT or "insufficient_quota" in error ): return "operator", name - if status == "provider_error" and name in REQUEST and not reached: + if status == "provider_error" and name in REQUEST and not completed: return "harness", name if status == "budget_stop": return "ceiling", status - if not reached: + if not returned: return "none", f"{status}: {name}" if name else str(status) trouble = host.trouble(start, end) if host is not None else None if trouble: return "host", trouble if status == "provider_error": + if escalation_timed_out(record): + return "model", f"{name} on the escalated request" if name in TRANSPORT: return "provider", name if name in REQUEST: @@ -432,9 +516,11 @@ def memory_pressure() -> int | None: def read_guard_log() -> str: + """The guard's log; empty when there is none (the guard never paused + anything). Any other error is raised, for the reader to try again.""" try: return GUARD_LOG.read_text(errors="replace") - except OSError: + except FileNotFoundError: return "" @@ -453,10 +539,19 @@ def local_timezone() -> str: return target.split("zoneinfo/", 1)[1] +# The `prev` of the host log's first line. +HOST_SEED = hashlib.sha256(b"host-log.jsonl").hexdigest() + + +class HostEvidenceError(RuntimeError): + """The guard's or the power log could not be read: a runner error.""" + + class HostLog: """The host log, written every `interval` seconds, and the host evidence an attempt's class reads: the guard's pauses, the power log's sleeps, the - host log's gaps and its memory-pressure samples.""" + host log's gaps and its memory-pressure samples. A sample whose pressure + could not be read is no sample: the silence around it is a gap.""" def __init__( self, @@ -481,7 +576,11 @@ def __init__( self._read_at = -math.inf self._refresh_lock = threading.Lock() self._stop = threading.Event() + self.failure: str | None = None # the sampler's last error + self.failing_since: float | None = None # failed samples since then + self._head, self._lines = HOST_SEED, 0 # the log's hash chain self.start = time.time() + self.written = self.start # the last sample written self.end: float | None = None self._write_meta() self.sample() @@ -495,37 +594,82 @@ def _write_meta(self) -> None: write_json(self.folder / "host.json", meta) def sample(self) -> None: - load1, load5, load15 = self._read_load() + """One sample. A load or pressure the host would not give is written + as null; a sample that cannot be written is the sampler's failure, + and the silence it leaves is a gap.""" + try: + load1, load5, load15 = self._read_load() + except OSError: + load1 = load5 = load15 = None pressure = self._read_pressure() with self._cond: t = time.time() line = { "time": t, + "monotonic": time.monotonic(), "load1": load1, "load5": load5, "load15": load15, "pressure": pressure, } - with (self.folder / "host-log.jsonl").open("a") as handle: - handle.write(json.dumps(line) + "\n") + text = json.dumps({**line, "prev": self._head}) + try: + with (self.folder / "host-log.jsonl").open("a") as handle: + handle.write(text + "\n") + except OSError as error: + self.failure = f"{type(error).__name__}: {error}" + self.failing_since = self.failing_since or t + return + self._head, self._lines = line_sha(text), self._lines + 1 + self.written = t + if pressure is None: + self.failure = "no memory-pressure reading" + self.failing_since = self.failing_since or t + else: + self.failing_since = None self._samples.append((t, pressure)) self._cond.notify_all() + def heads(self) -> dict: + """The host log's line count and last line's sha256, for an anchor.""" + with self._cond: + return {"lines": self._lines, "sha256": self._head} + + def dead(self) -> str | None: + """Why the sampler is taken for dead: no good sample for + SAMPLER_SILENCE seconds while its samples fail, or its thread gone. + A host that slept or was paused is not this: its sampler failed + nothing.""" + if self.end is None and not self._thread.is_alive(): + return "the sampler's thread ended" + since = self.failing_since + if since is not None and time.time() - since > SAMPLER_SILENCE: + return f"samples failing since {since:.1f} ({self.failure})" + return None + def _loop(self) -> None: k = 1 while not self._stop.wait( max(0.0, self.start + k * self.interval - time.time()) ): - self.sample() + try: + self.sample() + except Exception as error: # the sampler runs on; watch() sees the silence + self.failure = f"{type(error).__name__}: {error}" # After a stall, the next sample is the next tick, with no burst. k = max(k + 1, math.ceil((time.time() - self.start) / self.interval)) + def _good(self) -> list[float]: + """The times of the samples with a pressure reading.""" + return [t for t, p in self._samples if p is not None] + def settled(self, end: float) -> bool: """Whether every sample that could bear on an attempt ending at `end` is in: one later than the pressure window, or a silence longer than a gap (which is itself host trouble when it covers the attempt).""" with self._cond: - latest = self._samples[-1][0] + good = self._good() + latest = good[-1] if good else self.start return latest > end + PRESSURE_WINDOW or time.time() - latest > HOST_LOG_GAP def wait_settled(self, end: float) -> None: @@ -533,18 +677,30 @@ def wait_settled(self, end: float) -> None: while not self.settled(end): self._cond.wait(timeout=min(self.interval, 5.0)) + def _read_logs(self) -> tuple[str, str]: + """The guard's and the power log, read up to READ_TRIES times.""" + for attempt in range(READ_TRIES): + try: + return self._read_guard(), self._read_power() + except (OSError, subprocess.SubprocessError) as error: + failure = f"{type(error).__name__}: {error}" + if attempt < READ_TRIES - 1: + time.sleep(READ_WAIT) + raise HostEvidenceError(f"the guard's or the power log: {failure}") + def refresh(self, after: float) -> None: - """Read the guard's and the power log again unless read since `after`.""" + """Read the guard's and the power log again unless read since `after`. + A failed read keeps the last spans and tries again; HostEvidenceError + after READ_TRIES.""" with self._refresh_lock: if self._read_at > after: return now = time.time() since = self.start - EXCERPT_LEAD zone = ZoneInfo(self.timezone) - guard = excerpt( - self._read_guard(), lambda x: guard_time(x, zone), since, now - ) - power = excerpt(self._read_power(), power_time, since, now) + guard_text, power_text = self._read_logs() + guard = excerpt(guard_text, lambda x: guard_time(x, zone), since, now) + power = excerpt(power_text, power_time, since, now) spans = guard_spans(guard, self.timezone, now) + power_spans(power, now) with self._cond: self._spans, self._read_at = spans, now @@ -554,14 +710,15 @@ def _evidence(self) -> tuple[list, list[float]]: from the archived logs, with a silence so far as an open gap.""" with self._cond: spans = list(self._spans) - times = [t for t, _ in self._samples] - pressure = [t for t, p in self._samples if int(p or 0) > 1] + times = self._good() + pressure = [t for t, p in self._samples if p is not None and int(p) > 1] for a, b in zip([self.start] + times, times): if b - a > HOST_LOG_GAP: spans.append((a, b, "host log gap")) now = time.time() if self.end is None else self.end - if now - times[-1] > HOST_LOG_GAP: - spans.append((times[-1], now, "host log gap")) + last = times[-1] if times else self.start + if now - last > HOST_LOG_GAP: + spans.append((last, now, "host log gap")) return spans, pressure def trouble(self, start: float, end: float) -> str | None: @@ -591,20 +748,28 @@ def describe(self, start: float, end: float) -> str: return "; ".join(parts) def close(self) -> None: - """A last sample, the booking's end, and the guard's and power log's - lines for the booking.""" + """A last sample, the run's end, and the guard's and power log's lines + from a day before the run's start to its end. If the logs cannot be + read, `host-error.txt` says why and the excerpts are not written: the + report then has no host evidence to check against.""" self._stop.set() self._thread.join(timeout=self.interval + 10) - self.sample() + try: + self.sample() + except Exception as error: + self.failure = f"{type(error).__name__}: {error}" with self._cond: self.end = time.time() self._write_meta() since, zone = self.start - EXCERPT_LEAD, ZoneInfo(self.timezone) - guard = excerpt( - self._read_guard(), lambda x: guard_time(x, zone), since, self.end - ) + try: + guard_text, power_text = self._read_logs() + except HostEvidenceError as error: + (self.folder / "host-error.txt").write_text(f"{error}\n") + return + guard = excerpt(guard_text, lambda x: guard_time(x, zone), since, self.end) (self.folder / "clamshell-guard.log").write_text(guard) - power = excerpt(self._read_power(), power_time, since, self.end) + power = excerpt(power_text, power_time, since, self.end) (self.folder / "pmset.log").write_text(power) @@ -635,6 +800,38 @@ def exhausted(self, reserve_usd: float = 0.0) -> bool: return self.crossed_at is not None +class AttemptSpend: + """The hard ceiling as one attempt's conversation sees it. Every cost goes + to the ceiling, and the attempt keeps its own total, so an attempt that + raises still records what it spent. Once the attempt is abandoned at its + deadline, the conversation ends `budget_stop` at its next turn, and what + it spends after is kept apart as late spend.""" + + def __init__(self, ceiling: Ceiling): + self.ceiling = ceiling + self._lock = threading.Lock() + self.spent_usd = 0.0 + self.late_usd = 0.0 + self.abandoned = False + + def add(self, usd: float | None) -> None: + with self._lock: + if self.abandoned: + self.late_usd += float(usd or 0.0) + else: + self.spent_usd += float(usd or 0.0) + self.ceiling.add(usd) + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + return self.abandoned or self.ceiling.exhausted() + + def abandon(self) -> float: + """Abandon the attempt; what it had spent.""" + with self._lock: + self.abandoned = True + return self.spent_usd + + # ---------------------------------------------------------------- files @@ -647,15 +844,29 @@ def sha256_file(path: Path) -> str: def write_json(path: Path, payload, indent: int | None = 1) -> str: - """Write a file whole (to a temporary name, then renamed); its sha256.""" + """Write a file whole (to a temporary name, synced, then renamed); its + sha256.""" data = json.dumps(payload, indent=indent, default=str).encode() path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f".{path.name}.tmp") - tmp.write_bytes(data) + with tmp.open("wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) os.replace(tmp, path) return sha256_bytes(data) +def chain_seed(frozen_sha256: str, model: str, name: str) -> str: + """The `prev` of a log's first line: the log's place in the design.""" + return sha256_bytes(f"{frozen_sha256}/{model}/{name}".encode()) + + +def line_sha(text: str) -> str: + """A log line's hash, the next line's `prev`: its JSON, no newline.""" + return sha256_bytes(text.encode()) + + # ---------------------------------------------------------------- one model @@ -669,9 +880,17 @@ class Attempt: cls: str = "" detail: str = "" reason: str = "" + settled: bool = False # classed with its pass's host evidence + logged: bool = False # its run-log line written + + +class AttemptDeadline(RuntimeError): + """An attempt ran past ATTEMPT_DEADLINE and was abandoned.""" -def converse_with_harness(lanes: "Lanes", sid: str, label: str, tool_call) -> dict: +def converse_with_harness( + lanes: "Lanes", sid: str, label: str, tool_call, spend: AttemptSpend +) -> dict: """One attempt: the treatment harness's conversation for the arm, under the hard ceiling.""" board = lanes.run.board @@ -680,7 +899,7 @@ def converse_with_harness(lanes: "Lanes", sid: str, label: str, tool_call) -> di board["variables"][sid], tool_call, lanes.completion, - lanes.run.ceiling, + spend, profile=lanes.profiles[label], ) @@ -706,11 +925,21 @@ def __init__(self, run: "Run", model: str, labels: tuple[str, ...], folders: dic self.spent: dict[str, float] = {} # household -> its attempts' cost so far self.refused = False self.paused = False + self.paused_at: float | None = None self.stopped = False self.inconclusive: dict[str, list[str]] = {} self.pool = None self._pool_lock = threading.Lock() self.completion = None + self.manifest_sha256: str | None = None + # Each log is a hash chain: a line's `prev` is the line before's sha256. + self._heads = { + name: chain_seed(design["frozen_sha256"], model, name) + for name in ("run-log.jsonl", "admission-log.jsonl") + } + self._counts = {name: 0 for name in self._heads} + # Attempts abandoned at their deadline, whose spend after it is logged. + self.abandoned: list[dict] = [] # The records. @@ -723,16 +952,30 @@ def households(self, lane: int) -> list[str]: def write_line(self, line: dict, name: str = "run-log.jsonl") -> None: with self._log_lock: + text = json.dumps({**line, "prev": self._heads[name]}) self.folder.mkdir(parents=True, exist_ok=True) with (self.folder / name).open("a") as handle: - handle.write(json.dumps(line) + "\n") + handle.write(text + "\n") handle.flush() os.fsync(handle.fileno()) + self._heads[name] = line_sha(text) + self._counts[name] += 1 - def event(self, kind: str, reason: str, at: float | None = None) -> None: - self.write_line( - {"event": kind, "time": clock() if at is None else at, "reason": reason} - ) + def heads(self) -> dict: + """Each log's line count and last line's sha256, for an anchor.""" + with self._log_lock: + return { + name: {"lines": self._counts[name], "sha256": self._heads[name]} + for name in self._heads + } + + def event( + self, kind: str, reason: str, at: float | None = None, cause=None, **extra + ) -> None: + line = {"event": kind, "time": clock() if at is None else at, "reason": reason} + if cause is not None: + line["cause"] = cause + self.write_line({**line, **extra}) self.run.say(f"{self.model}: {kind} ({reason})") def attempt_line(self, sid: str, lane: int, a: Attempt, sha: str) -> None: @@ -750,17 +993,23 @@ def attempt_line(self, sid: str, lane: int, a: Attempt, sha: str) -> None: "sha256": sha, } ) + a.logged = True # Stops and pauses. def halted(self) -> bool: return self.stopped or self.run.ceiling.exhausted() - def stop(self, reason: str, kind: str = "stop", at: float | None = None) -> None: - """Stop the lanes. Every rule that stops them writes its own event.""" + def stop( + self, reason: str, cause: str, kind: str = "stop", at: float | None = None + ) -> None: + """Stop the lanes. Every rule that stops them writes its own event, + naming its cause.""" + if (kind, cause) != ("ceiling", "ceiling") and cause not in STOP_CAUSES: + raise ValueError(f"no stop has the cause {cause!r}") with self.cond: self.stopped = True - self.event(kind, reason, at) + self.event(kind, reason, at, cause=cause) self.cond.notify_all() def resume(self, note: str) -> None: @@ -778,9 +1027,11 @@ def admit(self, sid: str, lane: int) -> bool: self.paused or "settling" in self.state.values() ): self.cond.wait(timeout=WAIT_SECONDS) - if self.halted(): - return False - now = clock() + # Taken under the ceiling's lock, so no decision follows the crossing. + with self.run.ceiling.lock: + if self.stopped or self.run.ceiling.crossed_at is not None: + return False + now = clock() finished = [h for h, s in self.state.items() if s == "finished"] spend = sum(self.spent[h] for h in finished) under_way = len(self.state) - len(finished) + 1 @@ -821,11 +1072,19 @@ def start_attempt(self) -> float | None: return clock() def end_attempt( - self, sid: str, label: str, n: int, start: float, record: dict, last: bool + self, + sid: str, + label: str, + n: int, + start: float, + record: dict, + last: bool, + deadline: bool = False, ) -> Attempt: """The attempt's end. A class decided without the host evidence acts at once: an operator failure pauses the lanes, a harness failure stops - them, before any other attempt can start.""" + them, before any other attempt can start; so does an attempt + abandoned at its deadline.""" with self.cond: end = clock() self.spent[sid] += attempt_cost(record) @@ -834,13 +1093,24 @@ def end_attempt( if cls in EARLY: attempt.cls, attempt.detail = cls, detail if cls == "operator": - self.paused = True - self.event("pause", f"{label}/{sid} attempt {n}: {detail}") + self.paused, self.paused_at = True, clock() + self.event( + "pause", f"{label}/{sid} attempt {n}: {detail}", at=self.paused_at + ) if cls == "harness": self.stopped = True self.event( "stop", f"{label}/{sid}: a request class on the first request ({detail})", + cause="harness", + ) + if deadline: + self.stopped = True + self.event( + "stop", + f"{label}/{sid} attempt {n} ran past the attempt deadline " + f"of {ATTEMPT_DEADLINE:.0f} s", + cause="deadline", ) if last: self.state[sid] = "settling" @@ -858,7 +1128,7 @@ def settle(self, sid: str, done: list[Attempt]) -> None: cls, detail = classify(a.record, a.start, a.end, host) if a.cls and a.cls != cls: raise RuntimeError(f"{a.label}/{sid}: class {a.cls} became {cls}") - a.cls, a.detail = cls, detail + a.cls, a.detail, a.settled = cls, detail, True a.reason = ( f"{detail}: {host.describe(a.start, a.end)}" if cls == "host" @@ -874,12 +1144,19 @@ def settle(self, sid: str, done: list[Attempt]) -> None: if len(seen) > INCONCLUSIVE_STOP and stop is None: stop = f"{key} ended runs in {len(seen)} households" if stop: - self.stop(stop) - - def converse(self, sid: str, label: str) -> dict: + self.stop(stop, cause="inconclusive") + + def converse(self, sid: str, label: str, n: int) -> tuple[dict, bool]: + """(the attempt's record, whether it was abandoned). The conversation + runs in a thread of its own. One that raises leaves a harness-error + record with the spend it counted; one still running at + ATTEMPT_DEADLINE is abandoned with a harness-error record holding + the spend so far, ends `budget_stop` at its next turn, and writes + nothing.""" profile = self.profiles[label] - board = self.run.board + variables = self.run.board["variables"][sid] features = frozenset(profile.tool_features) + spend = AttemptSpend(self.run.ceiling) def tool_call(arguments): message = ( @@ -887,12 +1164,67 @@ def tool_call(arguments): ) return self.pool.call(message) - try: - return self.run.converse(self, sid, label, tool_call) - except Exception as error: # an attempt that left no record of its own - return pilot.harness_error_record( - sid, board["variables"][sid], profile, error + box: dict = {} + + def work(): + try: + box["record"] = self.run.converse(self, sid, label, tool_call, spend) + except Exception as error: # an attempt that left no record of its own + box["error"] = error + + thread = threading.Thread( + target=work, name=f"{self.model}-{sid}-{label}-{n}", daemon=True + ) + thread.start() + thread.join(timeout=ATTEMPT_DEADLINE) + if thread.is_alive(): + spent = spend.abandon() + with self.cond: + self.abandoned.append( + { + "household": sid, + "arm": label, + "attempt": n, + "spend": spend, + "thread": thread, + "logged": 0.0, + } + ) + error = AttemptDeadline( + f"past the attempt deadline of {ATTEMPT_DEADLINE:.0f} s" ) + record = pilot.harness_error_record(sid, variables, profile, error) + record["totals"]["cost_usd"] = spent + return record, True + if "error" in box: + record = pilot.harness_error_record(sid, variables, profile, box["error"]) + record["totals"]["cost_usd"] = spend.spent_usd + return record, False + return box["record"], False + + def log_late(self, final: bool = False) -> None: + """A `late-cost` event for each abandoned attempt that has ended (or, + at the end of the run, for each one still running) with spend after + its deadline not yet logged.""" + with self.cond: + pending = list(self.abandoned) + for item in pending: + ended = not item["thread"].is_alive() + late = item["spend"].late_usd + if (ended or final) and late > item["logged"]: + self.event( + "late-cost", + f"{item['arm']}/{item['household']} attempt {item['attempt']}: " + "spend after its deadline", + household=item["household"], + arm=item["arm"], + attempt=item["attempt"], + cost_usd=late - item["logged"], + ) + item["logged"] = late + if ended: + with self.cond: + self.abandoned.remove(item) def ensure_pool(self) -> bool: """The model's pool, started again if it has lost every worker; False @@ -933,30 +1265,30 @@ def replace(self, sid: str, lane: int, previous: Attempt, entries: list) -> None ) self.attempt_line(sid, lane, previous, sha) - def run_household(self, sid: str, lane: int) -> None: - order = self.orders[sid] - attempts: dict[str, list[Attempt]] = {label: [] for label in order} - entries: dict[str, list[dict]] = {label: [] for label in order} - in_flight = self.folder / "in-flight" / sid - todo = [(1, label) for label in order] + def passes( + self, sid: str, lane: int, attempts: dict, entries: dict, in_flight: Path + ) -> None: + """The household's arms in its drawn order, then its reruns in passes, + each pass classed once its host evidence is in.""" + todo = [(1, label) for label in self.orders[sid]] while todo: done: list[Attempt] = [] for i, (n, label) in enumerate(todo): if not self.ensure_pool(): - self.stop("the engine pool could not be started again") + self.stop("the engine pool could not be started again", "pool") break start = self.start_attempt() if start is None: break if n > 1: self.replace(sid, lane, attempts[label][-1], entries[label]) - record = self.converse(sid, label) + record, abandoned = self.converse(sid, label, n) a = self.end_attempt( - sid, label, n, start, record, last=i == len(todo) - 1 + sid, label, n, start, record, i == len(todo) - 1, abandoned ) - write_json(in_flight / f"{label}-{n}.json", record) attempts[label].append(a) done.append(a) + write_json(in_flight / f"{label}-{n}.json", record) self.run.say( f"{self.model} lane {lane} {sid} {label} #{n}: " f"{record.get('status')} ${attempt_cost(record):.4f} " @@ -978,8 +1310,39 @@ def run_household(self, sid: str, lane: int) -> None: with self.cond: self.state[sid] = "running" if todo else "finished" self.cond.notify_all() + + def run_household(self, sid: str, lane: int) -> bool: + """One household; False when the runner failed part way. Then every + model stops, and the attempts that ended are classed (as `unsettled` + when their host evidence cannot be read) and written before the lane + ends.""" + attempts: dict[str, list[Attempt]] = {label: [] for label in self.orders[sid]} + entries: dict[str, list[dict]] = {label: [] for label in self.orders[sid]} + in_flight = self.folder / "in-flight" / sid + failed = False + try: + self.passes(sid, lane, attempts, entries, in_flight) + except Exception as error: + failed = True + self.run.say(traceback.format_exc()) + self.run.stop_all( + f"runner error on {self.model} lane {lane} ({sid}): " + f"{type(error).__name__}: {error}", + cause="runner-error", + ) + pending = [a for mine in attempts.values() for a in mine if not a.settled] + if pending: + try: + self.settle(sid, pending) + except Exception as again: + self.run.say(traceback.format_exc()) + for a in pending: + a.cls, a.settled = "unsettled", True + a.detail = a.reason = ( + f"unsettled: {type(again).__name__}: {again}" + ) for label, mine in attempts.items(): - if mine: + if mine and not mine[-1].logged: final = mine[-1] sha = write_json( self.households_dir[label] / f"{sid}.json", @@ -989,17 +1352,22 @@ def run_household(self, sid: str, lane: int) -> None: with self.cond: self.state[sid] = "finished" self.cond.notify_all() - shutil.rmtree(in_flight, ignore_errors=True) + if not failed: + shutil.rmtree(in_flight, ignore_errors=True) + return not failed def lane(self, lane: int) -> None: try: for sid in self.households(lane): - if not self.admit(sid, lane): + if not self.admit(sid, lane) or not self.run_household(sid, lane): return - self.run_household(sid, lane) - except Exception as error: # a runner defect: this model stops, the rest run on + except Exception as error: # a runner defect: every model stops self.run.say(traceback.format_exc()) - self.stop(f"runner error on lane {lane}: {type(error).__name__}: {error}") + self.run.stop_all( + f"runner error on {self.model} lane {lane}: " + f"{type(error).__name__}: {error}", + cause="runner-error", + ) def manifest(self) -> dict: board, design = self.run.board, self.run.design @@ -1065,6 +1433,9 @@ def timeout(profile): "lanes": {lane: self.households(lane) for lane in (1, 2)}, "no_scored_output": sorted(self.run.no_output), "workers": self.run.workers, + "attempt_deadline_seconds": ATTEMPT_DEADLINE, + "anchor_seconds": ANCHOR_SECONDS, + "anchor_command": list(design["anchor_command"]), } def summary(self) -> dict: @@ -1094,6 +1465,7 @@ def summary(self) -> dict: "classes": dict(sorted(count.items())), "events": [x for x in lines if "event" in x], "tool_pool": self.pool.counters() if self.pool is not None else None, + "abandoned_running": len(self.abandoned), } @@ -1120,7 +1492,7 @@ class Run: def __init__( self, design: dict, - workers: int = 2, + workers: int | None = None, models: tuple[str, ...] | None = None, converse: Callable = converse_with_harness, pool_factory: Callable = default_pool, @@ -1128,7 +1500,8 @@ def __init__( host_factory: Callable | None = None, out=sys.stdout, ): - self.design, self.workers, self.out = design, workers, out + self.design, self.out = design, out + self.workers = int(design["workers"]) if workers is None else workers self.converse, self.pool_factory = converse, pool_factory self.completion_for = completion_for self.host_factory = host_factory or (lambda folder: HostLog(folder)) @@ -1156,17 +1529,83 @@ def __init__( self.host: HostLog | None = None self._ceiling_written = False self.interrupts = 0 + self.signalled: int | None = None # set by the signal handler alone + self._interrupted = False + self._sampler_stopped = False + self._anchor_head = chain_seed(design["frozen_sha256"], "", "anchors.jsonl") + self._anchored = -math.inf + self._pushing = threading.Lock() + self._push: threading.Thread | None = None def say(self, text: str) -> None: with self._say_lock: print(text, file=self.out, flush=True) - def stop_all(self, reason: str) -> None: + def stop_all(self, reason: str, cause: str) -> None: for lanes in self.lanes.values(): - lanes.stop(reason) + lanes.stop(reason, cause) + + # Anchors: the logs' chain heads, written and pushed outside the run. + + def anchor(self, wait: bool = False) -> None: + """Every log's chain head to `anchors.jsonl`, itself a chain, then the + design's anchor command on that file, in the background (one at a + time; an anchor whose push would wait is written but not pushed until + the next). Each push's result goes to `anchor-pushes.jsonl`.""" + runs = Path(self.design["runs"]) + line = { + "time": clock(), + "models": { + model: {**lanes.heads(), "manifest": lanes.manifest_sha256} + for model, lanes in self.lanes.items() + }, + "host-log.jsonl": self.host.heads() if self.host is not None else None, + "prev": self._anchor_head, + } + text = json.dumps(line) + with (runs / "anchors.jsonl").open("a") as handle: + handle.write(text + "\n") + handle.flush() + os.fsync(handle.fileno()) + self._anchor_head, self._anchored = line_sha(text), time.time() + if self._push is not None and self._push.is_alive(): + if not wait: + return + self._push.join() + self._push = threading.Thread( + target=self._push_anchor, args=(self._anchor_head,), daemon=True + ) + self._push.start() + if wait: + self._push.join() + + def _push_anchor(self, head: str) -> None: + runs = Path(self.design["runs"]) + command = [str(x) for x in self.design["anchor_command"]] + command.append(str(runs / "anchors.jsonl")) + started = time.time() + try: + out = subprocess.run(command, capture_output=True, text=True, timeout=300) + code, text = out.returncode, (out.stdout + out.stderr)[-2000:] + except (OSError, subprocess.SubprocessError) as error: + code, text = None, f"{type(error).__name__}: {error}" + line = { + "anchor": head, + "started": started, + "ended": time.time(), + "returncode": code, + "output": text, + } + with (runs / "anchor-pushes.jsonl").open("a") as handle: + handle.write(json.dumps(line) + "\n") + if code != 0: + self.say(f"the anchor push failed ({code}): {text}") + + # The main loop's work. def watch(self) -> None: - """The ceiling's events, and resumes after operator pauses.""" + """The ceiling's events, an interrupt, a dead sampler, resumes after + operator pauses, late spend and the anchors.""" crossed = self.ceiling.crossed_at if crossed is not None and not self._ceiling_written: self._ceiling_written = True @@ -1175,13 +1614,40 @@ def watch(self) -> None: f"${self.ceiling.budget_usd:.2f}" ) for lanes in self.lanes.values(): - lanes.stop(reason, kind="ceiling", at=crossed) + lanes.stop(reason, "ceiling", kind="ceiling", at=crossed) + if self.signalled is not None and not self._interrupted: + self._interrupted = True + self.say( + f"signal {self.signalled}: stopping every model's lanes; " + "the conversations under way finish" + ) + self.stop_all(f"operator stop (signal {self.signalled})", "interrupt") + dead = self.host.dead() if self.host is not None else None + if dead and not self._sampler_stopped: + self._sampler_stopped = True + self.stop_all(f"the host log stopped: {dead}", "sampler") for lanes in self.lanes.values(): path = lanes.folder / "RESUME.json" - if lanes.paused and path.exists(): - note = json.loads(path.read_text()).get("note") or "resumed" - path.rename(lanes.folder / f"resume-{time.time():.0f}.json") - lanes.resume(note) + if not path.exists(): + continue + try: + note = json.loads(path.read_text()) + except (OSError, ValueError): + continue # still being written + with lanes.cond: + paused, since = lanes.paused, lanes.paused_at + stamp = f"{time.time():.3f}" + if paused and float(note.get("time") or 0.0) >= since: + path.rename(lanes.folder / f"resume-{stamp}.json") + lanes.resume(note.get("note") or "resumed") + else: + # Written before the pause it would end, or with no pause. + path.rename(lanes.folder / f"resume-ignored-{stamp}.json") + self.say(f"{lanes.model}: a resume written before any pause, ignored") + for lanes in self.lanes.values(): + lanes.log_late() + if time.time() - self._anchored >= ANCHOR_SECONDS: + self.anchor() def execute(self) -> None: runs = Path(self.design["runs"]) @@ -1191,7 +1657,7 @@ def execute(self) -> None: for model, lanes in self.lanes.items(): lanes.completion = self.completion_for(lanes.profiles["pe3"]) stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) - write_json( + lanes.manifest_sha256 = write_json( lanes.folder / f"manifest-{stamp}.json", lanes.manifest(), indent=2 ) # A model whose lanes never decide still has both logs. @@ -1219,6 +1685,9 @@ def execute(self) -> None: self.watch() finally: for model, lanes in self.lanes.items(): + for item in list(lanes.abandoned): + item["thread"].join(timeout=LATE_JOIN) + lanes.log_late(final=True) stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) write_json( lanes.folder / f"run-summary-{stamp}.json", @@ -1228,16 +1697,15 @@ def execute(self) -> None: if lanes.pool is not None: lanes.pool.close() self.host.close() + self.anchor(wait=True) def interrupt(self, signum, frame) -> None: + """The signal handler. It only notes the signal; the main loop stops + every model's lanes. A second signal exits at once.""" self.interrupts += 1 if self.interrupts > 1: - raise KeyboardInterrupt - self.say( - f"signal {signum}: stopping every model's lanes; " - "the conversations under way end" - ) - self.stop_all(f"operator stop (signal {signum})") + os._exit(130) + self.signalled = signum # ---------------------------------------------------------------- before a run @@ -1262,6 +1730,13 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: if missing: return [f"the design has no {', '.join(missing)}"] problems = [] + command = design["anchor_command"] + if ( + not isinstance(command, list) + or not command + or shutil.which(str(command[0])) is None + ): + problems.append(f"the anchor command {command!r} is not a command here") if design.get("stated_engine_overrides"): problems.append("the confirmatory run states the board's engine: no overrides") if sha256_file(Path(design["preregistration"])) != design["frozen_sha256"]: @@ -1362,7 +1837,9 @@ def main(argv=None) -> int: sub = parser.add_subparsers(dest="command", required=True) run = sub.add_parser("run", help="run the confirmatory stage") run.add_argument("design", type=Path) - run.add_argument("--workers", type=int, default=2, help="engine workers per model") + run.add_argument( + "--workers", type=int, help="engine workers per model: the design's alone" + ) run.add_argument("--dry-run", action="store_true", help="check and print the plan") resume = sub.add_parser("resume", help="end an operator pause") resume.add_argument("design", type=Path) @@ -1375,13 +1852,19 @@ def main(argv=None) -> int: write_json(folder / "RESUME.json", {"time": time.time(), "note": args.note}) print(f"{args.model}: resume requested; the runner resumes within a second") return 0 + if args.workers is not None and args.workers != design.get("workers"): + print( + f"The run cannot start: --workers {args.workers}, and the design " + f"runs {design.get('workers')} engine workers per model." + ) + return 1 models = CHEAP + FRONTIER problems = preflight(design, models) pilot._require_engine(Path(design["board"])) if problems: print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) return 1 - plan = Run(design, workers=args.workers) + plan = Run(design) if args.dry_run: for model, lanes in plan.lanes.items(): sizes = [len(lanes.households(lane)) for lane in (1, 2)] diff --git a/tests/fake_anchor.py b/tests/fake_anchor.py new file mode 100644 index 00000000..73b5c32f --- /dev/null +++ b/tests/fake_anchor.py @@ -0,0 +1,13 @@ +"""The confirmatory runner's anchor command in its tests: a copy of the run's +anchors.jsonl to FOLDER/anchored.jsonl. + +Usage: python fake_anchor.py FOLDER ANCHORS +""" + +import shutil +import sys +from pathlib import Path + +folder, anchors = Path(sys.argv[1]), Path(sys.argv[2]) +shutil.copyfile(anchors, folder / "anchored.jsonl") +print("anchored") diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py new file mode 100644 index 00000000..626f3ba2 --- /dev/null +++ b/tests/test_confirmatory_anchor.py @@ -0,0 +1,91 @@ +"""The anchor command: each call commits the run's anchors and pushes them.""" + +import hashlib +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import confirmatory_anchor # noqa: E402 + +SCRIPT = Path(confirmatory_anchor.__file__) + + +@pytest.fixture +def clone(tmp_path, monkeypatch): + """A clone of an empty bare remote, with git's own settings only.""" + empty = tmp_path / "gitconfig" + empty.write_text("") + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(empty)) + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + for role in ("AUTHOR", "COMMITTER"): + monkeypatch.setenv(f"GIT_{role}_NAME", "anchor test") + monkeypatch.setenv(f"GIT_{role}_EMAIL", "anchor@example.invalid") + remote = tmp_path / "remote.git" + subprocess.run(["git", "init", "-q", "--bare", str(remote)], check=True) + repo = tmp_path / "anchors" + subprocess.run(["git", "clone", "-q", str(remote), str(repo)], check=True) + return remote, repo + + +def remote_log(remote: Path) -> list[str]: + out = subprocess.run( + ["git", "-C", str(remote), "log", "--format=%s", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + return out.stdout.splitlines() + + +def remote_file(remote: Path) -> str: + out = subprocess.run( + ["git", "-C", str(remote), "show", "HEAD:anchors.jsonl"], + check=True, + capture_output=True, + text=True, + ) + return out.stdout + + +def test_each_anchor_is_committed_and_pushed(clone, tmp_path): + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + lines = ['{"time": 1.0, "prev": "a"}', '{"time": 2.0, "prev": "b"}'] + shas = [hashlib.sha256(line.encode()).hexdigest() for line in lines] + anchors.write_text(lines[0] + "\n") + assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 + anchors.write_text("\n".join(lines) + "\n") + assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 + assert remote_log(remote) == [f"anchor 2: {shas[1]}", f"anchor 1: {shas[0]}"] + assert remote_file(remote) == anchors.read_text() + + +def test_an_unchanged_file_makes_no_commit(clone, tmp_path): + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + anchors.write_text('{"time": 1.0}\n') + for _ in range(2): + assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 + assert len(remote_log(remote)) == 1 + + +def test_a_failed_push_exits_nonzero(clone, tmp_path): + """The runner records the command's return code; a push that fails must + not look like one that landed.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + anchors.write_text('{"time": 1.0}\n') + subprocess.run( + ["git", "-C", str(repo), "remote", "set-url", "origin", str(tmp_path / "gone")], + check=True, + ) + out = subprocess.run( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + capture_output=True, + text=True, + ) + assert out.returncode != 0 diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 63acc702..e6034e12 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -15,6 +15,8 @@ import io import json import math +import os +import subprocess import sys import threading import time @@ -46,6 +48,12 @@ # about 0.1 ms on this host, even within a thread; the report allows a second, as # here, which also covers a small correction of the clock. CLOCK_SLACK = 1.0 +# The anchor command in a test: a copy of the anchors to the test's folder. +ANCHOR_SCRIPT = Path(__file__).resolve().parent / "fake_anchor.py" +# How far apart anchors and their triggers' stops may be in a test, on a +# loaded host. +ANCHOR_SLACK = 5.0 +TRIGGER_BOUND = 5.0 # ---------------------------------------------------------------- streams @@ -117,23 +125,50 @@ def test_an_arm_whose_features_are_not_registered_is_refused(monkeypatch): cr.arm_profile("gpt-6-luna", "loo-flat") +def litellm_names(groups) -> set[str]: + """The names of litellm's exception classes under any of `groups`.""" + import inspect + + import litellm + + found = set() + for module in (litellm, litellm.exceptions): + for obj in vars(module).values(): + if inspect.isclass(obj) and issubclass(obj, BaseException): + if issubclass(obj, groups): + found.add(obj.__name__) + return found + + def test_the_error_classes_are_policybench_s(): + """A record keeps the raised class's name, and the harness groups classes + by inheritance: so each list holds every litellm class under the + harness's group.""" from policybench import eval_no_tools as e - assert cr.TRANSPORT == {c.__name__ for c in e.RETRYABLE_PROVIDER_ERRORS} - assert cr.REQUEST | cr.ACCOUNT == {c.__name__ for c in e.MODEL_FATAL_ERRORS} + fatal = litellm_names(e.MODEL_FATAL_ERRORS) + retryable = litellm_names(e.RETRYABLE_PROVIDER_ERRORS) - fatal + assert cr.REQUEST | cr.ACCOUNT == fatal + assert cr.TRANSPORT - {"RequestWallTimeoutError"} == retryable + assert "RequestWallTimeoutError" in { + c.__name__ for c in e.RETRYABLE_PROVIDER_ERRORS + } + assert {"ContentPolicyViolationError", "RejectedRequestError"} <= cr.REQUEST assert not cr.REQUEST & cr.ACCOUNT assert not cr.TRANSPORT & (cr.REQUEST | cr.ACCOUNT) + assert cr.TIMEOUTS <= cr.TRANSPORT # ---------------------------------------------------------------- classes def record(status="complete", error=None, turns=1, calls=(), truncated=False): + """`truncated`: the record's last turn is a truncated one, its escalated + request then failing (with `turns` - 1 completed turns before it).""" turn = {"calls": [{"result": r} for r in calls]} out = [dict(turn) for _ in range(turns)] if truncated: - out = [{"truncated_and_retried": True}] + out = out[: max(turns - 1, 0)] + [{"truncated_and_retried": True}] return {"status": status, "error": error, "turns": out} @@ -150,6 +185,22 @@ def trouble(self, start, end): (record("provider_error", "BadRequestError: too long"), "model"), (record("provider_error", "BadRequestError: bad", turns=0), "harness"), (record("provider_error", "BadRequestError: x", truncated=True), "harness"), + (record("provider_error", "BadRequestError: x", 2, truncated=True), "model"), + (record("provider_error", "ContentPolicyViolationError: x", turns=0), "harness"), + (record("provider_error", "ContentPolicyViolationError: x"), "model"), + (record("provider_error", "RejectedRequestError: x", turns=0), "harness"), + # The escalated request's timeout is the model's long generation. + (record("provider_error", "Timeout: 300 s", truncated=True), "model"), + (record("provider_error", "Timeout: 300 s", 3, truncated=True), "model"), + (record("provider_error", "Timeout: 300 s"), "provider"), + # A truncated return is a return: no longer `none`. + (record("provider_error", "RateLimitError: x", truncated=True), "provider"), + ( + record("provider_error", "ProviderEmpty: no choices", truncated=True), + "inconclusive", + ), + (record("provider_error", "ProviderEmpty: no choices", turns=0), "none"), + (record("provider_error", "JSONSchemaValidationError: x"), "provider"), (record("provider_error", "AuthenticationError: key", turns=0), "operator"), (record("provider_error", "RateLimitError: insufficient_quota"), "operator"), (record("provider_error", "ProviderEmpty: no choices"), "inconclusive"), @@ -170,6 +221,17 @@ def trouble(self, start, end): (record(calls=[{"error": "engine", "message": "ValueError: no person"}]), "model"), (record(calls=[{"results": {"snap": {"error": "OSError: disk"}}}]), "inconclusive"), (record(calls=[{"results": {"snap": {"error": "KeyError: x"}}}]), "model"), + # The explain feature's trace of a variable, which catches its own errors. + ( + record(calls=[{"explanations": {"snap": {"error": "MemoryError: x"}}}]), + "inconclusive", + ), + ( + record(calls=[{"explanations": {"snap": {"error": "OSError: disk"}}}]), + "inconclusive", + ), + (record(calls=[{"explanations": {"snap": {"error": "KeyError: x"}}}]), "model"), + (record(calls=[{"explanations": {"snap": {"value": 1.0}}}]), "ok"), (record("running"), "inconclusive"), ] @@ -294,6 +356,9 @@ def fast(monkeypatch): monkeypatch.setattr(cr, "HOST_LOG_GAP", 0.5) monkeypatch.setattr(cr, "WAIT_SECONDS", 0.005) monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) + monkeypatch.setattr(cr, "SAMPLER_SILENCE", 1.0) + monkeypatch.setattr(cr, "READ_WAIT", 0.01) + monkeypatch.setattr(cr, "ANCHOR_SECONDS", 0.3) def test_the_host_log_samples_and_closes(tmp_path, fast): @@ -361,6 +426,100 @@ def test_guard_pauses_and_sleeps_reach_an_attempt_after_a_refresh(tmp_path, fast assert "Sleep" in (tmp_path / "pmset.log").read_text() +def test_a_failed_pressure_reading_is_no_sample(tmp_path, fast): + """A sample with no pressure reading is written, but it is no sample: a + failing reading under pressure would hide the evidence the rule reads, + so the silence it leaves is a gap.""" + readers = Readers() + host = readers.host(tmp_path) + time.sleep(0.03) + start = time.time() + readers.pressure = None + time.sleep(0.7) + readers.pressure = 1 + end = time.time() + host.wait_settled(end) + host.refresh(after=end) + assert host.trouble(start, end) == "host log gap" + host.close() + samples = [ + json.loads(x) for x in (tmp_path / "host-log.jsonl").read_text().splitlines() + ] + assert any(x["pressure"] is None for x in samples) + folder_host = host_from_files(tmp_path) + assert folder_host.trouble(start, end) == "host log gap" # the report's reading + + +def test_the_host_log_is_a_hash_chain_with_a_monotonic_time(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + time.sleep(0.1) + host.close() + path = tmp_path / "host-log.jsonl" + assert chain_problems(path, cr.HOST_SEED) == [] + lines = path.read_text().splitlines() + assert host.heads() == {"lines": len(lines), "sha256": cr.line_sha(lines[-1])} + ticks = [json.loads(x)["monotonic"] for x in lines] + assert ticks == sorted(ticks) + + +def test_a_failed_log_read_is_tried_again(tmp_path, fast): + calls = [] + + def power(): + calls.append(1) + if len(calls) <= 2: + raise subprocess.CalledProcessError(1, ["pmset"]) + return "" + + host = cr.HostLog( + tmp_path, + timezone=TZ, + interval=0.02, + read_load=lambda: (1.0, 2.0, 3.0), + read_pressure=lambda: 1, + read_guard=lambda: "", + read_power=power, + ) + host.refresh(after=time.time()) + assert len(calls) == 3 # two failures, then a read + host.close() + assert (tmp_path / "pmset.log").exists() + + +def test_logs_that_cannot_be_read_are_a_runner_error(tmp_path, fast): + def power(): + raise subprocess.TimeoutExpired(["pmset"], 120) + + host = cr.HostLog( + tmp_path, + timezone=TZ, + interval=0.02, + read_load=lambda: (1.0, 2.0, 3.0), + read_pressure=lambda: 1, + read_guard=lambda: "", + read_power=power, + ) + with pytest.raises(cr.HostEvidenceError): + host.refresh(after=time.time()) + host.close() + assert "TimeoutExpired" in (tmp_path / "host-error.txt").read_text() + assert not (tmp_path / "pmset.log").exists() + + +def test_a_sampler_whose_samples_fail_is_dead_after_its_silence(tmp_path, fast): + readers = Readers() + host = readers.host(tmp_path) + assert host.dead() is None + readers.pressure = None + time.sleep(cr.SAMPLER_SILENCE + 0.2) + assert "samples failing since" in host.dead() + readers.pressure = 1 + time.sleep(0.1) + assert host.dead() is None # a good sample: alive again + host.close() + + # ---------------------------------------------------------------- the ceiling @@ -449,15 +608,16 @@ class FakeProvider: `sleep` per turn, `calls` (tool arguments, one call each, per turn), `pressure` (high pressure during the attempt), `stall` (a host-log stall, seconds), `guard` (a guard pause over the attempt), `resume_after` - (seconds after which the operator resumes) and `raise` (the conversation - raises).""" + (seconds after which the operator resumes), `raise` (the conversation + raises before its first turn), `raise_after` (it raises after its turns) + and `check` (a function of the lanes, called first).""" def __init__(self, world, specs=None): self.world, self.specs = world, specs or {} self.count: dict = {} self.lock = threading.Lock() - def __call__(self, lanes, sid, label, tool_call): + def __call__(self, lanes, sid, label, tool_call, spend): model = lanes.model with self.lock: n = self.count[(model, label, sid)] = ( @@ -465,6 +625,8 @@ def __call__(self, lanes, sid, label, tool_call): ) spec = self.specs.get((model, label, sid, n), {}) readers = self.world.readers + if spec.get("check"): + spec["check"](lanes) if spec.get("raise"): raise RuntimeError("a test raised") if spec.get("pressure"): @@ -482,7 +644,6 @@ def __call__(self, lanes, sid, label, tool_call): "turns": [], "predictions": {v: 0.0 for v in VARIABLES}, } - spend = lanes.run.ceiling for i in range(spec.get("turns", 1)): if spend.exhausted(): rec["status"] = "budget_stop" @@ -503,6 +664,8 @@ def __call__(self, lanes, sid, label, tool_call): result, latency = tool_call(arguments) turn["calls"].append({"arguments": arguments, "result": result}) rec["turns"].append(turn) + if spec.get("raise_after"): + raise RuntimeError("a test raised after spending") if rec["status"] == "running": rec["status"] = spec.get("status", "complete") if spec.get("error"): @@ -522,7 +685,10 @@ def __call__(self, lanes, sid, label, tool_call): folder = lanes.folder def resume(): - cr.write_json(folder / "RESUME.json", {"note": "fixed in a test"}) + cr.write_json( + folder / "RESUME.json", + {"time": time.time(), "note": "fixed in a test"}, + ) threading.Timer(spec["resume_after"], resume).start() return rec @@ -584,6 +750,8 @@ def __init__( "reserves": {m: reserve for m in cr.CHEAP + cr.FRONTIER}, "overall_cap": overall_cap, "spent_before": spent_before, + "workers": 2, + "anchor_command": [sys.executable, str(ANCHOR_SCRIPT), str(tmp)], } self.readers = Readers() self.provider = FakeProvider(self, specs) @@ -644,11 +812,13 @@ def host_from_files(folder: Path): (folder / "clamshell-guard.log").read_text(), meta["timezone"], end ) spans += cr.power_spans((folder / "pmset.log").read_text(), end) - times = sorted(float(s["time"]) for s in samples) + # A sample with no pressure reading is no sample. + good = [s for s in samples if s.get("pressure") is not None] + times = sorted(float(s["time"]) for s in good) for a, b in zip([start] + times, times + [end]): if b - a > cr.HOST_LOG_GAP: spans.append((a, b, "host log gap")) - pressure = [float(s["time"]) for s in samples if int(s.get("pressure") or 0) > 1] + pressure = [float(s["time"]) for s in good if int(s["pressure"]) > 1] class Host: def trouble(self, s, e): @@ -663,10 +833,78 @@ def trouble(self, s, e): return Host() +def chain_problems(path: Path, seed: str) -> list[str]: + """Whether each line's `prev` is the line before's sha256 (the first's, + `seed`).""" + head = seed + for i, text in enumerate(path.read_text().splitlines() if path.exists() else []): + if json.loads(text).get("prev") != head: + return [f"{path.name} line {i + 1}: its prev is not the line before's hash"] + head = cr.line_sha(text) + return [] + + +def chain_heads(path: Path, seed: str) -> list[str]: + """The chain's head after each count of lines, from none.""" + heads = [seed] + for text in path.read_text().splitlines() if path.exists() else []: + heads.append(cr.line_sha(text)) + return heads + + +def anchor_problems(world: World) -> list[str]: + """The anchors: a chain of their own; each holds every log's head at its + count; the last holds every log whole; the last was pushed.""" + problems: list[str] = [] + runs, frozen = world.runs(), world.design["frozen_sha256"] + path = runs / "anchors.jsonl" + problems += chain_problems(path, cr.chain_seed(frozen, "", "anchors.jsonl")) + anchors = [json.loads(x) for x in path.read_text().splitlines()] + heads = { + (model, name): chain_heads( + runs / model / name, cr.chain_seed(frozen, model, name) + ) + for model in world.models + for name in ("run-log.jsonl", "admission-log.jsonl") + } + host = chain_heads(runs / "host" / "host-log.jsonl", cr.HOST_SEED) + for i, anchor in enumerate(anchors): + for (model, name), chain in heads.items(): + got = anchor["models"][model][name] + if got["lines"] >= len(chain) or chain[got["lines"]] != got["sha256"]: + problems.append(f"anchor {i}: {model}/{name} is not the log's head") + got = anchor["host-log.jsonl"] + if got["lines"] >= len(host) or host[got["lines"]] != got["sha256"]: + problems.append(f"anchor {i}: the host log is not the log's head") + last = anchors[-1] + for (model, name), chain in heads.items(): + if last["models"][model][name]["lines"] != len(chain) - 1: + problems.append(f"the last anchor leaves out lines of {model}/{name}") + if last["host-log.jsonl"]["lines"] != len(host) - 1: + problems.append("the last anchor leaves out host-log lines") + for a, b in zip(anchors, anchors[1:]): + if b["time"] - a["time"] > cr.ANCHOR_SECONDS + ANCHOR_SLACK: + problems.append(f"anchors {a['time']!r} and {b['time']!r} too far apart") + pushes = [ + json.loads(x) for x in (runs / "anchor-pushes.jsonl").read_text().splitlines() + ] + final = cr.line_sha(path.read_text().splitlines()[-1]) + if not any(x["anchor"] == final and x["returncode"] == 0 for x in pushes): + problems.append("the last anchor was not pushed") + copy = world.tmp / "anchored.jsonl" + if not copy.exists() or copy.read_text() != path.read_text(): + problems.append("the anchor command's copy is not the anchors") + return problems + + def run_problems(world: World) -> list[str]: """Every rule of the run procedure, recomputed from the runner's files.""" problems: list[str] = [] host = host_from_files(world.runs() / "host") + frozen = world.design["frozen_sha256"] + problems += chain_problems(world.runs() / "host" / "host-log.jsonl", cr.HOST_SEED) + problems += anchor_problems(world) + late_total = 0.0 position = {sid: i for i, sid in enumerate(world.draw_order)} master = cr.master_seed(world.design["frozen_sha256"]) all_attempts = [] @@ -676,6 +914,17 @@ def run_problems(world: World) -> list[str]: drawn = cr.orders(master, model, world.draw_order, labels) lines = world.attempts(model) events = world.events(model) + for name in ("run-log.jsonl", "admission-log.jsonl"): + problems += chain_problems( + world.runs() / model / name, cr.chain_seed(frozen, model, name) + ) + for e in events: + if e["event"] == "stop" and e.get("cause") not in cr.STOP_CAUSES: + problems.append(f"{model}: a stop with no cause ({e})") + if e["event"] == "ceiling" and e.get("cause") != "ceiling": + problems.append(f"{model}: a ceiling event with another cause") + late_total += sum(e["cost_usd"] for e in events if e["event"] == "late-cost") + causes = {e.get("cause") for e in events if e["event"] == "stop"} stops = [e["time"] for e in events if e["event"] in ("stop", "ceiling")] stopped_at = min(stops) if stops else math.inf ceiling_events[model] = [e["time"] for e in events if e["event"] == "ceiling"] @@ -724,6 +973,8 @@ def run_problems(world: World) -> list[str]: f"{at}#{k}: its line's hash is not its record's" ) cls, _ = cr.classify(rec, x["start"], x["end"], host) + if x["class"] == "unsettled" and "runner-error" in causes: + cls = "unsettled" # classed when the runner failed: it stands rule = cr.action(cls, k) if x["class"] != cls or x["action"] != rule: problems.append( @@ -871,11 +1122,61 @@ def run_problems(world: World) -> list[str]: for t in times: if any(x["start"] > t for *_, x, _ in all_attempts): problems.append("an attempt started after the ceiling") - # Conservation: the ceiling counted exactly the records' spend. - if abs(world.run.ceiling.spent_usd - total) > 1e-9: + # Conservation: the ceiling counted exactly the records' spend, and the + # spend of abandoned attempts after their deadline. + if abs(world.run.ceiling.spent_usd - total - late_total) > 1e-9: problems.append( - f"the ceiling counted {world.run.ceiling.spent_usd}, records {total}" + f"the ceiling counted {world.run.ceiling.spent_usd}, records {total} " + f"and late spend {late_total}" ) + problems += trigger_problems(world, all_attempts) + return problems + + +def trigger_problems(world: World, all_attempts) -> list[str]: + """Each stop rule's trigger has its stop, and each stop of those causes + its trigger: a harness failure and an abandoned attempt stop the model at + their end, before any other attempt starts; one inconclusive error ending + runs in a sixth household stops it once that household's pass is classed.""" + problems: list[str] = [] + for model in world.models: + events = world.events(model) + mine = [(sid, k, x, rec) for m, sid, _, k, x, rec in all_attempts if m == model] + triggers: list[tuple[str, float]] = [] + joined: dict[str, dict[str, float]] = {} # key -> household -> pass end + for sid, k, x, rec in mine: + if x["class"] == "harness": + triggers.append(("harness", x["end"])) + if str(rec.get("error") or "").startswith("AttemptDeadline"): + triggers.append(("deadline", x["end"])) + status = rec.get("status") + if x["class"] == "inconclusive" and status in ( + "provider_error", + "harness_error", + ): + key = f"{status}: {cr.error_class(rec) or rec.get('error')}" + end = max(y["end"] for s, n, y, _ in mine if s == sid and n == k) + households = joined.setdefault(key, {}) + households[sid] = min(households.get(sid, math.inf), end) + for households in joined.values(): + ends = sorted(households.values()) + if len(ends) > cr.INCONCLUSIVE_STOP: + triggers.append(("inconclusive", ends[cr.INCONCLUSIVE_STOP])) + stops = [e for e in events if e["event"] == "stop"] + for cause, t in triggers: + if not any( + e.get("cause") == cause and t <= e["time"] <= t + TRIGGER_BOUND + for e in stops + ): + problems.append(f"{model}: a {cause} trigger at {t!r} with no stop") + if cause != "inconclusive" and any(x["start"] > t for _, _, x, _ in mine): + problems.append(f"{model}: an attempt started after a {cause} stop") + for e in stops: + if e.get("cause") in ("harness", "deadline", "inconclusive") and not any( + cause == e["cause"] and t <= e["time"] <= t + TRIGGER_BOUND + for cause, t in triggers + ): + problems.append(f"{model}: a {e['cause']} stop with no trigger") return problems @@ -1104,6 +1405,7 @@ def test_a_request_class_on_the_first_request_stops_the_model(world): w.execute() stops = w.events(model, "stop") assert len(stops) == 1 and "first request" in stops[0]["reason"] + assert stops[0]["cause"] == "harness" assert not [x for x in w.attempts(model) if x["start"] > stops[0]["time"]] assert len(w.attempts("gpt-6-sol")) == N_DRAW * 2 # the other model finishes assert run_problems(w) == [] @@ -1120,6 +1422,7 @@ def test_one_inconclusive_error_in_six_households_stops_the_model(world): assert ( stops[0]["reason"] == "provider_error: ProviderEmpty ended runs in 6 households" ) + assert stops[0]["cause"] == "inconclusive" assert run_problems(w) == [] @@ -1289,6 +1592,7 @@ def losing(self, message): w.execute() stops = w.events(model, "stop") assert stops and "engine pool" in stops[0]["reason"] + assert stops[0]["cause"] == "pool" assert run_problems(w) == [] @@ -1341,12 +1645,323 @@ def test_an_interrupt_stops_every_model_and_lets_attempts_end(world): for m in w.models: stops = w.events(m, "stop") assert len(stops) == 1 and stops[0]["reason"] == "operator stop (signal 2)" + assert stops[0]["cause"] == "interrupt" assert not [x for x in w.attempts(m) if x["start"] > stops[0]["time"]] luna = [x for x in w.attempts("gpt-6-luna") if x["household"] == sid] assert luna and len(luna) < len(cr.ARMS) # cut by the stop assert run_problems(w) == [] +def test_a_failed_explain_trace_is_inconclusive(world): + """The explain feature's trace catches its own errors and returns them + under `explanations`: a MemoryError there is the host's, not a miss.""" + + class Explaining(FakePool): + def call(self, message): + self.messages.append(message) + return { + "results": {"snap": {"values": {"spm": 1.0}}}, + "explanations": {"snap": {"error": "MemoryError: a test"}}, + }, 0.0 + + model = "gpt-6-luna" + w = world(models=(model,), pool=Explaining) + sid = w.draw_order[0] + w.provider.specs = {(model, "pe3", sid, 1): {"calls": [{"situation": "{}"}]}} + w.execute() + line = next( + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == "pe3" + ) + assert line["class"] == "inconclusive" and line["action"] == "stands" + assert run_problems(w) == [] + + +def test_no_household_is_decided_after_the_crossing(world): + """The crossing lands between the lanes' halt check and the decision: the + decision's time is taken under the ceiling's lock, after a second look.""" + w = world(models=("gpt-6-luna",)) + lanes = w.run.lanes["gpt-6-luna"] + halted = lanes.halted + + def crossing(): + result = halted() + if not result: + w.run.ceiling.add(1e6) # another lane's spend, just after the check + return result + + lanes.halted = crossing + decided = lanes.admit(w.draw_order[0], 1) + lines = w.lines("gpt-6-luna", "admission-log.jsonl") + assert decided is False and lines == [] + + +@pytest.mark.parametrize( + "when", ["before the run", "before the run, misdated", "during the pause"] +) +def test_a_resume_written_before_the_pause_is_ignored(world, when): + """A resume left from before a pause would end that pause unseen: it is + set aside, and the pause holds until a resume written during it. One + found while no pause holds is set aside at once, whatever time it + states; one found during the pause but stamped before it is set aside + too.""" + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + folder = w.runs() / model + folder.mkdir(parents=True) + stale = {"time": time.time(), "note": "stale"} + if when.startswith("before the run"): + if when.endswith("misdated"): + stale["time"] += 3600.0 # a writer whose clock runs ahead + cr.write_json(folder / "RESUME.json", stale) + + def plant(lanes): + # Lands once the pause holds: the conversation returns at once. + threading.Timer( + 0.1, lambda: cr.write_json(folder / "RESUME.json", stale) + ).start() + + w.provider.specs = { + (model, "pe3", sid, 1): { + "status": "provider_error", + "error": "AuthenticationError: bad key", + "turns": 0, + "resume_after": 1.5, # the main loop looks every 0.5 s + **({"check": plant} if when == "during the pause" else {}), + } + } + w.execute() + pauses, resumes = w.events(model, "pause"), w.events(model, "resume") + assert len(pauses) == 1 and len(resumes) == 1 + assert resumes[0]["reason"] == "fixed in a test" + assert resumes[0]["time"] - pauses[0]["time"] >= 1.4 + assert list(folder.glob("resume-ignored-*.json")) + assert run_problems(w) == [] + + +def test_the_signal_handler_only_notes_the_signal(world, monkeypatch): + """The handler runs on the main thread between any two of its bytecodes, + perhaps holding a lock it would need: it takes none. A second signal + exits at once.""" + w = world(models=("gpt-6-luna",)) + lanes = w.run.lanes["gpt-6-luna"] + exits = [] + monkeypatch.setattr(os, "_exit", lambda code: exits.append(code)) + with w.run._say_lock, lanes.cond, lanes._log_lock: + w.run.interrupt(2, None) # would deadlock if it took any of these + assert w.run.signalled == 2 and not lanes.stopped + assert w.lines("gpt-6-luna") == [] + w.run.interrupt(15, None) + assert exits == [130] + + +def test_an_attempt_that_raises_after_spending_keeps_its_cost(world): + model = "claude-haiku-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = { + (model, first, sid, 1): {"turns": 2, "cost": 0.03, "raise_after": True} + } + w.execute() + replaced = json.loads( + (w.folder(model, first) / "attempts" / f"{sid}-1.json").read_text() + ) + assert replaced["status"] == "harness_error" and replaced["turns"] == [] + assert replaced["totals"]["cost_usd"] == pytest.approx(0.06) + assert run_problems(w) == [] # the ceiling counted it, and so do the records + + +def test_an_attempt_past_its_deadline_is_abandoned_and_stops_the_model( + world, monkeypatch +): + """The conversation runs on in its own thread; at the deadline its record + is a harness error with the spend so far, the model stops, and the + conversation ends `budget_stop` at its next turn. Its spend after the + deadline is a late-cost event.""" + monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 0.3) + model = "gpt-6-luna" + w = world(models=(model, "claude-haiku-5.5")) + sid = w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = { + (model, first, sid, 1): {"turns": 4, "cost": 0.01, "sleep": 0.2}, + **{ + ("claude-haiku-5.5", label, s, 1): {"sleep": 0.05} + for label in cr.ARMS + for s in w.draw_order[:2] + }, + } + w.execute() + line = next( + x for x in w.attempts(model) if x["household"] == sid and x["arm"] == first + ) + final = json.loads((w.folder(model, first) / f"{sid}.json").read_text()) + assert final["error"].startswith("AttemptDeadline: past the attempt deadline") + assert final["totals"]["cost_usd"] == pytest.approx(0.01) # turn 1 only + assert line["class"] == "none" # its rerun is cut short by the stop + stops = w.events(model, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "deadline" + late = w.events(model, "late-cost") + assert len(late) == 1 and late[0]["cost_usd"] == pytest.approx(0.01) + assert not [x for x in w.attempts(model) if x["start"] > line["end"]] + assert len(w.attempts("claude-haiku-5.5")) == N_DRAW * len(cr.ARMS) + assert run_problems(w) == [] + + +def test_a_sampler_whose_samples_fail_stops_every_model(world): + models = ("gpt-6-luna", "gpt-6-sol") + w = world(models=models) + w.provider.specs = { + (m, label, sid, 1): {"sleep": 0.1} + for m in models + for label in labels_of(m) + for sid in w.draw_order + } + threading.Timer(0.3, lambda: setattr(w.readers, "pressure", None)).start() + w.execute() + for m in models: + stops = w.events(m, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "sampler" + assert "samples failing since" in stops[0]["reason"] + assert run_problems(w) == [] + + +def failing_settle(lanes, sid, times): + """Lanes.settle, failing its first `times` calls for household `sid`.""" + original, calls = lanes.settle, [] + + def settle(at, done): + if at == sid and len(calls) < times: + calls.append(at) + raise RuntimeError("a test's runner defect") + return original(at, done) + + lanes.settle = settle + return calls + + +@pytest.mark.parametrize("times", [1, 2]) +def test_a_runner_error_stops_every_model_and_writes_what_ended(world, times): + """A runner error in a household's pass: every model stops, and that + household's ended attempts are classed, or `unsettled` if classing fails + again, and written with their lines before the lane ends.""" + models = ("gpt-6-luna", "gpt-6-sol") + w = world(models=models) + sid = w.draw_order[2] + failing_settle(w.run.lanes["gpt-6-luna"], sid, times) + w.execute() + for m in models: + stops = w.events(m, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "runner-error" + mine = [x for x in w.attempts("gpt-6-luna") if x["household"] == sid] + assert len(mine) == len(cr.ARMS) # the pass ended before the error + assert {x["class"] for x in mine} == ({"ok"} if times == 1 else {"unsettled"}) + for label in cr.ARMS: + assert (w.folder("gpt-6-luna", label) / f"{sid}.json").exists() + assert run_problems(w) == [] + + +def test_each_attempt_is_on_disk_before_the_next_starts(world, monkeypatch): + """An ended attempt is written, synced, before its household's next + attempt starts; a replaced one is saved, with its line, before its rerun + starts.""" + model = "claude-haiku-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + order = w.run.lanes[model].orders[sid] + import fcntl + + if not hasattr(fcntl, "F_GETPATH"): + pytest.skip("no F_GETPATH: the paths synced cannot be told") + synced: list[str] = [] + real = os.fsync + + def fsync(fd): + path = fcntl.fcntl(fd, fcntl.F_GETPATH, bytes(1024)) + synced.append(path.rstrip(b"\0").decode()) + return real(fd) + + monkeypatch.setattr(os, "fsync", fsync) + seen = {} + + def before_second(lanes): + seen["in-flight"] = ( + lanes.folder / "in-flight" / sid / f"{order[0]}-1.json" + ).exists() + + def before_rerun(lanes): + seen["replaced"] = ( + w.folder(model, order[1]) / "attempts" / f"{sid}-1.json" + ).exists() + seen["line"] = any( + x["household"] == sid and x["arm"] == order[1] and x["attempt"] == 1 + for x in w.attempts(model) + ) + + error = {"status": "provider_error", "error": "RateLimitError: slow"} + w.provider.specs = { + (model, order[1], sid, 1): {**error, "check": before_second}, + (model, order[1], sid, 2): {"check": before_rerun}, + } + w.execute() + assert seen == {"in-flight": True, "replaced": True, "line": True} + # Each whole-file write is synced before its rename. + assert any(f"/in-flight/{sid}/.{order[0]}-1.json.tmp" in p for p in synced) + assert any(f"/attempts/.{sid}-1.json.tmp" in p for p in synced) + assert run_problems(w) == [] + + +def test_the_worker_count_is_the_design_s(tmp_path, capsys): + design = tmp_path / "design.json" + design.write_text(json.dumps({"runs": str(tmp_path / "runs"), "workers": 2})) + assert cr.main(["run", str(design), "--workers", "3"]) == 1 + assert "the design runs 2 engine workers" in capsys.readouterr().out + + +def test_the_checker_catches_a_runner_that_ignores_a_harness_failure( + world, monkeypatch +): + """trigger_problems is not vacuous: a runner whose harness failure does + not stop the model fails it.""" + model = "claude-opus-5.5" + w = world(models=(model,)) + sid = w.draw_order[0] + w.provider.specs = { + (model, "pe", sid, 1): { + "status": "provider_error", + "error": "BadRequestError: bad tools", + "turns": 0, + } + } + lanes = w.run.lanes[model] + real = lanes.event + + def no_stop(kind, reason, at=None, cause=None, **extra): + if cause == "harness": + lanes.stopped = False # the fault: the stop is not kept + return + real(kind, reason, at, cause, **extra) + + lanes.event = no_stop + w.execute() + problems = run_problems(w) + assert any("a harness trigger" in p and "with no stop" in p for p in problems) + assert any("an attempt started after a harness stop" in p for p in problems) + + +def test_the_checker_catches_an_edited_log_line(world): + w = world(models=("gpt-6-sol",)).execute() + path = w.runs() / "gpt-6-sol" / "admission-log.jsonl" + lines = path.read_text().splitlines() + edited = json.loads(lines[1]) + edited["finished_spend"] = 0.0 + path.write_text("\n".join([lines[0], json.dumps(edited)] + lines[2:]) + "\n") + problems = run_problems(w) + assert any("its prev is not the line before's hash" in p for p in problems) + assert any("is not the log's head" in p for p in problems) + + KINDS = { "ok": {}, "provider": {"status": "provider_error", "error": "RateLimitError: slow"}, From 06abd1fdf92c3047096161ec71b01b5b187b12e0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 11:11:03 -0400 Subject: [PATCH 06/48] Confirmatory runner: log excerpts from a day before the run's start The docstring and EXCERPT_LEAD's comment said "the booking"; the code keeps the guard's and power log's lines from a day before the host log's start. No behaviour changes. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index de5f4089..02048665 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -55,7 +55,7 @@ - **The host log.** Every 30 seconds: the time, the monotonic clock, the load average and the kernel's memory-pressure level (None when it could not be read, which is no sample). At the end, the guard's and the power log's - lines from a day before the booking to its end. A read of those logs is + lines from a day before the run's start to its end. A read of those logs is tried five times; one that still fails during the run is a runner error, and at the end leaves ``host-error.txt`` in place of the excerpts. An attempt's class waits for its host evidence to @@ -157,7 +157,7 @@ SAMPLE_SECONDS = 30.0 CEILING_MARGIN = 25.0 # dollars above the overall cap ADMISSION_SLACK = 1e-9 # dollars -EXCERPT_LEAD = 86_400.0 # host logs are kept from a day before the booking +EXCERPT_LEAD = 86_400.0 # host logs are kept from a day before the run's start WAIT_SECONDS = 1.0 # how often a waiting lane looks again ATTEMPT_DEADLINE = 3600.0 # seconds; the longest development attempt took 714 SAMPLER_SILENCE = 300.0 # seconds without a host-log sample that stop the run From f5099dbcc26fc6747e147eb8d413c15a40a56e54 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 11:23:22 -0400 Subject: [PATCH 07/48] Confirmatory runner tests: the interrupt test without a timer The test sent its interrupt on a 0.15 s timer, and at load 49 the run had not started the household's first attempt by then, so no attempt was cut short and the test failed. The interrupt now arrives from inside that first attempt, which finishes only once the main loop has stopped the lanes. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index e6034e12..4616ff93 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -1639,8 +1639,18 @@ def pool(lanes): def test_an_interrupt_stops_every_model_and_lets_attempts_end(world): w = world(models=("gpt-6-luna", "gpt-6-sol")) sid = w.draw_order[0] - w.provider.specs = {("gpt-6-luna", lbl, sid, 1): {"sleep": 0.1} for lbl in cr.ARMS} - threading.Timer(0.15, lambda: w.run.interrupt(2, None)).start() + first = w.run.lanes["gpt-6-luna"].orders[sid][0] + + def interrupt_inside(lanes): + # The signal arrives during the household's first attempt, which + # finishes only once the main loop has stopped the lanes, so no + # timing on a loaded host lets the household's later arms start first. + w.run.interrupt(2, None) + until = time.time() + 30 + while not lanes.stopped and time.time() < until: + time.sleep(0.01) + + w.provider.specs = {("gpt-6-luna", first, sid, 1): {"check": interrupt_inside}} w.execute() for m in w.models: stops = w.events(m, "stop") From f210b6e556fe6341052d1c9b71335210da9dda3d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 13:14:01 -0400 Subject: [PATCH 08/48] Confirmatory runner: round-2 fixes, part 1 (not yet tested) Answers the first part of review runner-r2: - The ledger can close. Ceiling.charge counts spend with the lock held and, once close() has run, counts nothing more and keeps the amount apart as unrecorded_usd. AttemptSpend now changes its totals under the ceiling's lock, so the attempt's totals and the ceiling's always agree. - The sampler's failure timer starts on every caught sampling exception (finding 3), so a reader that keeps raising stops the run after the silence limit instead of never. - Guard-log lines carry no offset, so guard_times reads each as both readings of a repeated or skipped local hour (New York repeats 01:00-02:00 on 2026-11-01). A pause opens at its earliest reading and closes at its latest; excerpts keep a line if either reading falls in the window. - ANCHOR_RETRY and ANCHOR_PUSH_TIMEOUT are defined for the anchor changes that follow; LATE_JOIN becomes one overall deadline. The suite has not run on this commit: the next commit runs it and finishes the round (stop_all, end_attempt, runner errors, startup, finish(), the resume race and the anchor push). Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 98 +++++++++++++++++++++++----------- tests/test_confirmatory_run.py | 2 +- 2 files changed, 69 insertions(+), 31 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 02048665..3dfe346f 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -164,7 +164,9 @@ READ_TRIES = 5 # reads of the guard's and the power log before a runner error READ_WAIT = 10.0 # seconds between them ANCHOR_SECONDS = 900.0 # how often the logs' chain heads are anchored -LATE_JOIN = 60.0 # seconds the run's end waits for an abandoned attempt +ANCHOR_RETRY = 60.0 # seconds to the next anchor after a failed push +ANCHOR_PUSH_TIMEOUT = 300.0 # seconds; then the command's process group is killed +LATE_JOIN = 60.0 # seconds the run's end waits, in all, for abandoned attempts # What a stop event names as its cause. STOP_CAUSES = frozenset( { @@ -438,34 +440,42 @@ def clock() -> float: GUARD_LOG = Path.home() / "Library/Logs/clamshell-guard.log" -def guard_time(line: str, zone: ZoneInfo) -> float | None: +def guard_times(line: str, zone: ZoneInfo) -> tuple[float, float] | None: + """The earliest and latest instants a guard line's local time can be. They + differ only within an hour that a change of offset repeats or skips: on + 2026-11-01, New York's 01:00-02:00 comes twice, and the guard's lines + carry no offset.""" match = GUARD_LINE.match(line) if not match: return None stamp = datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S") - return stamp.replace(tzinfo=zone).timestamp() + both = [stamp.replace(tzinfo=zone, fold=fold).timestamp() for fold in (0, 1)] + return min(both), max(both) -def power_time(line: str) -> float | None: +def power_times(line: str) -> tuple[float, float] | None: + """A power-log line's instant, twice (its time carries its offset).""" match = POWER_LINE.match(line) if not match: return None - return datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S %z").timestamp() + t = datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S %z").timestamp() + return t, t def guard_spans(text: str, timezone: str, end: float) -> list: """Pauses: from a 'paused pids:' line to the next 'resumed pids:' line (or - `end`).""" + `end`), each from the earliest reading of its pause to the latest of its + resume.""" zone, spans, opened = ZoneInfo(timezone), [], None for line in text.splitlines(): - t = guard_time(line, zone) - if t is None: + times = guard_times(line, zone) + if times is None: continue what = GUARD_LINE.match(line).group(2) if "paused pids:" in what and opened is None: - opened = t + opened = times[0] elif "resumed pids:" in what and opened is not None: - spans.append((opened, t, "guard pause")) + spans.append((opened, times[1], "guard pause")) opened = None if opened is not None: spans.append((opened, end, "guard pause")) @@ -476,9 +486,10 @@ def power_spans(text: str, end: float) -> list: """Sleeps: from a Sleep line to the next Wake or DarkWake line (or `end`).""" spans, opened = [], None for line in text.splitlines(): - t = power_time(line) - if t is None: + times = power_times(line) + if times is None: continue + t = times[0] kind = POWER_LINE.match(line).group(2).strip() if kind == "Sleep" and opened is None: opened = t @@ -490,12 +501,18 @@ def power_spans(text: str, end: float) -> list: return spans -def excerpt(text: str, when: Callable[[str], float | None], since: float, until: float): - """The lines of a log stamped within [since, until].""" +def excerpt( + text: str, + when: Callable[[str], tuple[float, float] | None], + since: float, + until: float, +): + """The lines of a log that one reading of its time puts within [since, + until].""" keep = [] for line in text.splitlines(): - t = when(line) - if t is not None and since <= t <= until: + times = when(line) + if times is not None and times[0] <= until and times[1] >= since: keep.append(line + "\n") return "".join(keep) @@ -656,6 +673,7 @@ def _loop(self) -> None: self.sample() except Exception as error: # the sampler runs on; watch() sees the silence self.failure = f"{type(error).__name__}: {error}" + self.failing_since = self.failing_since or time.time() # After a stall, the next sample is the next tick, with no burst. k = max(k + 1, math.ceil((time.time() - self.start) / self.interval)) @@ -699,8 +717,8 @@ def refresh(self, after: float) -> None: since = self.start - EXCERPT_LEAD zone = ZoneInfo(self.timezone) guard_text, power_text = self._read_logs() - guard = excerpt(guard_text, lambda x: guard_time(x, zone), since, now) - power = excerpt(power_text, power_time, since, now) + guard = excerpt(guard_text, lambda x: guard_times(x, zone), since, now) + power = excerpt(power_text, power_times, since, now) spans = guard_spans(guard, self.timezone, now) + power_spans(power, now) with self._cond: self._spans, self._read_at = spans, now @@ -767,9 +785,9 @@ def close(self) -> None: except HostEvidenceError as error: (self.folder / "host-error.txt").write_text(f"{error}\n") return - guard = excerpt(guard_text, lambda x: guard_time(x, zone), since, self.end) + guard = excerpt(guard_text, lambda x: guard_times(x, zone), since, self.end) (self.folder / "clamshell-guard.log").write_text(guard) - power = excerpt(power_text, power_time, since, self.end) + power = excerpt(power_text, power_times, since, self.end) (self.folder / "pmset.log").write_text(power) @@ -784,16 +802,33 @@ class Ceiling(pilot.Spend): def __init__(self, ceiling_usd: float, spent_before: float): super().__init__(ceiling_usd, spent_before) self.crossed_at: float | None = None + self.closed = False # the records are final: spend is no longer counted + self.unrecorded_usd = 0.0 # spend after that, in no record @property def lock(self) -> threading.Lock: return self._lock + def charge(self, usd: float) -> bool: + """Count `usd`, with the lock held; False, and the spend unrecorded, + once the ledger has closed.""" + if self.closed: + self.unrecorded_usd += usd + return False + self.spent_usd += usd + if self.crossed_at is None and self.spent_usd >= self.budget_usd: + self.crossed_at = clock() + return True + def add(self, usd: float | None) -> None: with self._lock: - self.spent_usd += float(usd or 0.0) - if self.crossed_at is None and self.spent_usd >= self.budget_usd: - self.crossed_at = clock() + self.charge(float(usd or 0.0)) + + def close(self) -> None: + """Close the ledger at the run's end: what the ceiling has counted is + then what the records hold.""" + with self._lock: + self.closed = True def exhausted(self, reserve_usd: float = 0.0) -> bool: with self._lock: @@ -805,29 +840,32 @@ class AttemptSpend: to the ceiling, and the attempt keeps its own total, so an attempt that raises still records what it spent. Once the attempt is abandoned at its deadline, the conversation ends `budget_stop` at its next turn, and what - it spends after is kept apart as late spend.""" + it spends after is kept apart as late spend. The attempt's totals and the + ceiling's change under the ceiling's lock, so they always agree; once + the ledger has closed, neither counts more.""" def __init__(self, ceiling: Ceiling): self.ceiling = ceiling - self._lock = threading.Lock() self.spent_usd = 0.0 self.late_usd = 0.0 self.abandoned = False def add(self, usd: float | None) -> None: - with self._lock: + usd = float(usd or 0.0) + with self.ceiling.lock: + if not self.ceiling.charge(usd): + return if self.abandoned: - self.late_usd += float(usd or 0.0) + self.late_usd += usd else: - self.spent_usd += float(usd or 0.0) - self.ceiling.add(usd) + self.spent_usd += usd def exhausted(self, reserve_usd: float = 0.0) -> bool: return self.abandoned or self.ceiling.exhausted() def abandon(self) -> float: """Abandon the attempt; what it had spent.""" - with self._lock: + with self.ceiling.lock: self.abandoned = True return self.spent_usd diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 4616ff93..1e962383 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -313,7 +313,7 @@ def test_an_excerpt_keeps_the_lines_stamped_in_its_window(): t = 1_800_000_000.0 text = "\n".join(guard_line(t + k, "paused") for k in (-5, 0, 5, 10)) + "\nx" zone = ZoneInfo(TZ) - kept = cr.excerpt(text, lambda x: cr.guard_time(x, zone), t, t + 5) + kept = cr.excerpt(text, lambda x: cr.guard_times(x, zone), t, t + 5) assert kept.splitlines() == [guard_line(t, "paused"), guard_line(t + 5, "paused")] From 36d553a10a4b193eb47720b0c8ad3622eac3e2dd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 07:26:56 -0700 Subject: [PATCH 09/48] Confirmatory runner: round-2 fixes, part 2 Answers review runner-r2 and finishes f210b6e5: - Set and notify every model's stop state before event I/O. Retain ended attempts before pause/stop writes; recover their classes and final records best-effort before runner-error stops, and block new starts during recovery. - Finish against one abandoned-request join deadline. Close pools and host before the ledger, then record late costs, crossings and interrupts before the final anchor. Snapshot unresolved requests under the ledger lock and record their unknown future spend explicitly as a runner error. - Confirm persistent sampler exceptions stop the run and a good sample clears the failure timer. Validate a resume against the current pause under the condition that clears it. - Poll pool startup with supervision and process pending stops before every lane launch. Record startup/restart spans for the report's idle calculation; drain supervisors on runner errors before closing their host and ledger. - Kill the whole anchor command process group at ANCHOR_PUSH_TIMEOUT. Retry failed pushes using ANCHOR_RETRY and sync each outcome, anchor timestamp and push interval. Surface push-log I/O failures and retain final local coverage. - Exercise disk-error recovery, late cleanup spend, unresolved requests, persistent sampler errors, the resume race, startup signals and slow pools, failed pushes and real child/grandchild timeout termination. The report must subtract validated setup spans from lane idle time and compute remote exposure from successful pushed snapshots. Local cadence alone cannot support the draft's claimed fifteen-minute maximum; neither report nor draft is changed here. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 475 +++++++++++++++++---- tests/test_confirmatory_anchor.py | 184 ++++++++ tests/test_confirmatory_run.py | 676 ++++++++++++++++++++++++++++-- 3 files changed, 1212 insertions(+), 123 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 3dfe346f..fd3d01da 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -16,7 +16,9 @@ household. The six models run at once. - **One engine pool per model**, started once, with no features of its own. Each call carries its arm's features in a call frame (``tool_call_frame``), - the message a pool started with those features would send. + the message a pool started with those features would send. ``setup-start`` + and ``setup-end`` events bracket startup and pool restarts, so setup time + can be distinguished from an idle lane. - **Admission.** A lane admits its next household when the spend of the model's finished households (every attempt, replaced ones included) plus the reserve for each household under way, this one included, is at most @@ -643,6 +645,7 @@ def sample(self) -> None: self.failure = "no memory-pressure reading" self.failing_since = self.failing_since or t else: + self.failure = None self.failing_since = None self._samples.append((t, pressure)) self._cond.notify_all() @@ -824,11 +827,13 @@ def add(self, usd: float | None) -> None: with self._lock: self.charge(float(usd or 0.0)) - def close(self) -> None: + def close(self, pending: list[dict] | None = None) -> list[dict]: """Close the ledger at the run's end: what the ceiling has counted is - then what the records hold.""" + then what the records hold. Snapshot unresolved requests under the + same lock: one finishing afterward cannot disappear from recovery.""" with self._lock: self.closed = True + return [item for item in pending or [] if item["thread"].is_alive()] def exhausted(self, reserve_usd: float = 0.0) -> bool: with self._lock: @@ -1036,7 +1041,7 @@ def attempt_line(self, sid: str, lane: int, a: Attempt, sha: str) -> None: # Stops and pauses. def halted(self) -> bool: - return self.stopped or self.run.ceiling.exhausted() + return self.stopped or self.run._failed.is_set() or self.run.ceiling.exhausted() def stop( self, reason: str, cause: str, kind: str = "stop", at: float | None = None @@ -1047,15 +1052,19 @@ def stop( raise ValueError(f"no stop has the cause {cause!r}") with self.cond: self.stopped = True - self.event(kind, reason, at, cause=cause) self.cond.notify_all() + self.event(kind, reason, at, cause=cause) - def resume(self, note: str) -> None: + def resume(self, note: str, noted_at: float) -> bool: + """A note may end only a pause it was written during; another lane's + later operator failure cannot be cleared by the older note.""" with self.cond: - if self.paused: - self.paused = False - self.event("resume", note) - self.cond.notify_all() + if not (self.paused and noted_at >= self.paused_at): + return False + self.paused = False + self.cond.notify_all() + self.event("resume", note) + return True # One household. @@ -1067,7 +1076,11 @@ def admit(self, sid: str, lane: int) -> bool: self.cond.wait(timeout=WAIT_SECONDS) # Taken under the ceiling's lock, so no decision follows the crossing. with self.run.ceiling.lock: - if self.stopped or self.run.ceiling.crossed_at is not None: + if ( + self.stopped + or self.run._failed.is_set() + or self.run.ceiling.crossed_at is not None + ): return False now = clock() finished = [h for h, s in self.state.items() if s == "finished"] @@ -1105,7 +1118,11 @@ def start_attempt(self) -> float | None: while self.paused and not self.halted(): self.cond.wait(timeout=WAIT_SECONDS) with self.run.ceiling.lock: - if self.stopped or self.run.ceiling.crossed_at is not None: + if ( + self.stopped + or self.run._failed.is_set() + or self.run.ceiling.crossed_at is not None + ): return None return clock() @@ -1118,6 +1135,7 @@ def end_attempt( record: dict, last: bool, deadline: bool = False, + retained: list[Attempt] | None = None, ) -> Attempt: """The attempt's end. A class decided without the host evidence acts at once: an operator failure pauses the lanes, a harness failure stops @@ -1127,32 +1145,35 @@ def end_attempt( end = clock() self.spent[sid] += attempt_cost(record) attempt = Attempt(label, n, start, end, record) + if retained is not None: + retained.append(attempt) # recovery owns it before any event I/O cls, detail = classify(record, start, end, None) if cls in EARLY: attempt.cls, attempt.detail = cls, detail if cls == "operator": self.paused, self.paused_at = True, clock() + if cls == "harness" or deadline: + self.stopped = True + if last: + self.state[sid] = "settling" + self.cond.notify_all() + if cls == "operator": self.event( "pause", f"{label}/{sid} attempt {n}: {detail}", at=self.paused_at ) if cls == "harness": - self.stopped = True self.event( "stop", f"{label}/{sid}: a request class on the first request ({detail})", cause="harness", ) if deadline: - self.stopped = True self.event( "stop", f"{label}/{sid} attempt {n} ran past the attempt deadline " f"of {ATTEMPT_DEADLINE:.0f} s", cause="deadline", ) - if last: - self.state[sid] = "settling" - self.cond.notify_all() return attempt def settle(self, sid: str, done: list[Attempt]) -> None: @@ -1264,27 +1285,52 @@ def log_late(self, final: bool = False) -> None: with self.cond: self.abandoned.remove(item) - def ensure_pool(self) -> bool: + def ensure_pool(self, startup: bool = False) -> bool: """The model's pool, started again if it has lost every worker; False if it cannot be.""" with self._pool_lock: + if self.halted(): + return False if ( self.pool is not None and self.pool.counters()["lost"] < self.run.workers ): return True old = self.pool - try: - self.pool = self.run.pool_factory(self) - except RuntimeError as error: - self.run.say(f"{self.model}: the engine pool did not start: {error}") - return False - if old is not None: - self.run.say( - f"{self.model}: the engine pool lost every worker; started again" + start, outcome = clock(), "failed" + if not startup: + self.event( + "setup-start", + "engine pool restart", + at=start, + phase="restart", + start=start, ) - old.close() - return True + try: + try: + self.pool = self.run.pool_factory(self) + except RuntimeError as error: + self.run.say( + f"{self.model}: the engine pool did not start: {error}" + ) + return False + if old is not None: + self.run.say( + f"{self.model}: the engine pool lost every worker; " + "started again" + ) + old.close() + outcome = "stopped" if self.halted() else "ready" + return outcome == "ready" + finally: + if not startup: + self.event( + "setup-end", + "engine pool restart", + phase="restart", + start=start, + outcome=outcome, + ) def replace(self, sid: str, lane: int, previous: Attempt, entries: list) -> None: """Save the attempt a rerun replaces, before the rerun.""" @@ -1313,7 +1359,8 @@ def passes( done: list[Attempt] = [] for i, (n, label) in enumerate(todo): if not self.ensure_pool(): - self.stop("the engine pool could not be started again", "pool") + if not self.halted(): + self.stop("the engine pool could not be started again", "pool") break start = self.start_attempt() if start is None: @@ -1322,9 +1369,15 @@ def passes( self.replace(sid, lane, attempts[label][-1], entries[label]) record, abandoned = self.converse(sid, label, n) a = self.end_attempt( - sid, label, n, start, record, i == len(todo) - 1, abandoned + sid, + label, + n, + start, + record, + i == len(todo) - 1, + abandoned, + retained=attempts[label], ) - attempts[label].append(a) done.append(a) write_json(in_flight / f"{label}-{n}.json", record) self.run.say( @@ -1357,17 +1410,16 @@ def run_household(self, sid: str, lane: int) -> bool: attempts: dict[str, list[Attempt]] = {label: [] for label in self.orders[sid]} entries: dict[str, list[dict]] = {label: [] for label in self.orders[sid]} in_flight = self.folder / "in-flight" / sid - failed = False + failure = None try: self.passes(sid, lane, attempts, entries, in_flight) except Exception as error: - failed = True - self.run.say(traceback.format_exc()) - self.run.stop_all( + self.run._failed.set() # block starts while records are recovered + failure = ( f"runner error on {self.model} lane {lane} ({sid}): " - f"{type(error).__name__}: {error}", - cause="runner-error", + f"{type(error).__name__}: {error}" ) + self.run.say(traceback.format_exc()) pending = [a for mine in attempts.values() for a in mine if not a.settled] if pending: try: @@ -1382,17 +1434,34 @@ def run_household(self, sid: str, lane: int) -> bool: for label, mine in attempts.items(): if mine and not mine[-1].logged: final = mine[-1] - sha = write_json( - self.households_dir[label] / f"{sid}.json", - {**final.record, "attempts": entries[label]}, - ) - self.attempt_line(sid, lane, final, sha) + try: + sha = write_json( + self.households_dir[label] / f"{sid}.json", + { + **final.record, + "attempts": [ + entry + for entry in entries[label] + if entry["n"] < final.n + ], + }, + ) + self.attempt_line(sid, lane, final, sha) + except Exception as error: # the other arms still get written + self.run._failed.set() + failure = failure or ( + f"runner error writing {self.model} lane {lane} ({sid}): " + f"{type(error).__name__}: {error}" + ) + self.run.say(traceback.format_exc()) + if failure is not None: + self.run.stop_all(failure, cause="runner-error") with self.cond: self.state[sid] = "finished" self.cond.notify_all() - if not failed: + if failure is None: shutil.rmtree(in_flight, ignore_errors=True) - return not failed + return failure is None def lane(self, lane: int) -> None: try: @@ -1570,22 +1639,49 @@ def __init__( self.signalled: int | None = None # set by the signal handler alone self._interrupted = False self._sampler_stopped = False + self._failed = threading.Event() # recovery blocks starts before stop I/O self._anchor_head = chain_seed(design["frozen_sha256"], "", "anchors.jsonl") self._anchored = -math.inf self._pushing = threading.Lock() + self._next_anchor = -math.inf self._push: threading.Thread | None = None + self._push_error: OSError | None = None def say(self, text: str) -> None: with self._say_lock: print(text, file=self.out, flush=True) - def stop_all(self, reason: str, cause: str) -> None: + def stop_all( + self, reason: str, cause: str, kind: str = "stop", at: float | None = None + ) -> None: + # Set every state before the first fallible write; notify even if a + # disk error prevents a model's event from being recorded. for lanes in self.lanes.values(): - lanes.stop(reason, cause) + with lanes.cond: + lanes.stopped = True + lanes.cond.notify_all() + for lanes in self.lanes.values(): + try: + lanes.event(kind, reason, at, cause=cause) + except Exception: + self.say(traceback.format_exc()) # Anchors: the logs' chain heads, written and pushed outside the run. - def anchor(self, wait: bool = False) -> None: + def _check_push_error(self) -> None: + with self._pushing: + error = self._push_error + if error is not None: + raise OSError( + f"anchor push outcome could not be recorded: {error}" + ) from error + + def anchor_due(self) -> bool: + self._check_push_error() + with self._pushing: + return time.time() >= self._next_anchor + + def anchor(self, wait: bool = False, push: bool = True) -> None: """Every log's chain head to `anchors.jsonl`, itself a chain, then the design's anchor command on that file, in the background (one at a time; an anchor whose push would wait is written but not pushed until @@ -1606,42 +1702,83 @@ def anchor(self, wait: bool = False) -> None: handle.flush() os.fsync(handle.fileno()) self._anchor_head, self._anchored = line_sha(text), time.time() + with self._pushing: + self._next_anchor = self._anchored + ANCHOR_SECONDS + if not push: + return # a push-log failure still leaves the final local heads if self._push is not None and self._push.is_alive(): if not wait: return self._push.join() + self._check_push_error() self._push = threading.Thread( - target=self._push_anchor, args=(self._anchor_head,), daemon=True + target=self._push_anchor, + args=(self._anchor_head, line["time"]), + daemon=True, ) self._push.start() if wait: self._push.join() + self._check_push_error() - def _push_anchor(self, head: str) -> None: + def _push_anchor(self, head: str, anchor_time: float) -> None: runs = Path(self.design["runs"]) command = [str(x) for x in self.design["anchor_command"]] command.append(str(runs / "anchors.jsonl")) started = time.time() try: - out = subprocess.run(command, capture_output=True, text=True, timeout=300) - code, text = out.returncode, (out.stdout + out.stderr)[-2000:] + child = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + stdout, stderr = child.communicate(timeout=ANCHOR_PUSH_TIMEOUT) + outcome = "success" if child.returncode == 0 else "failed" + except subprocess.TimeoutExpired: + # The command is a session leader: git and ssh share its group. + # Killing just the command leaves its push running after timeout. + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, stderr = child.communicate() + outcome = "timeout" + code, text = child.returncode, (stdout + stderr)[-2000:] except (OSError, subprocess.SubprocessError) as error: - code, text = None, f"{type(error).__name__}: {error}" + code, outcome, text = None, "error", f"{type(error).__name__}: {error}" + ended = time.time() + if outcome != "success": + with self._pushing: + self._next_anchor = min(self._next_anchor, ended + ANCHOR_RETRY) line = { "anchor": head, + "anchor_time": anchor_time, "started": started, - "ended": time.time(), + "ended": ended, + "outcome": outcome, "returncode": code, "output": text, } - with (runs / "anchor-pushes.jsonl").open("a") as handle: - handle.write(json.dumps(line) + "\n") - if code != 0: - self.say(f"the anchor push failed ({code}): {text}") + try: + with (runs / "anchor-pushes.jsonl").open("a") as handle: + handle.write(json.dumps(line) + "\n") + handle.flush() + os.fsync(handle.fileno()) + except OSError as error: + # A daemon's exception cannot stop the run: hand this one back + # to the supervisor, or to the final caller after it joins us. + with self._pushing: + self._push_error = self._push_error or error + return + if outcome != "success": + self.say(f"the anchor push failed ({outcome}, {code}): {text}") # The main loop's work. - def watch(self) -> None: + def watch(self, anchor: bool = True) -> None: """The ceiling's events, an interrupt, a dead sampler, resumes after operator pauses, late spend and the anchors.""" crossed = self.ceiling.crossed_at @@ -1651,8 +1788,7 @@ def watch(self) -> None: f"total spend ${self.ceiling.spent_usd:.2f} reached the hard ceiling " f"${self.ceiling.budget_usd:.2f}" ) - for lanes in self.lanes.values(): - lanes.stop(reason, "ceiling", kind="ceiling", at=crossed) + self.stop_all(reason, "ceiling", kind="ceiling", at=crossed) if self.signalled is not None and not self._interrupted: self._interrupted = True self.say( @@ -1672,27 +1808,39 @@ def watch(self) -> None: note = json.loads(path.read_text()) except (OSError, ValueError): continue # still being written - with lanes.cond: - paused, since = lanes.paused, lanes.paused_at stamp = f"{time.time():.3f}" - if paused and float(note.get("time") or 0.0) >= since: + if lanes.resume( + note.get("note") or "resumed", float(note.get("time") or 0.0) + ): path.rename(lanes.folder / f"resume-{stamp}.json") - lanes.resume(note.get("note") or "resumed") else: # Written before the pause it would end, or with no pause. path.rename(lanes.folder / f"resume-ignored-{stamp}.json") self.say(f"{lanes.model}: a resume written before any pause, ignored") for lanes in self.lanes.values(): lanes.log_late() - if time.time() - self._anchored >= ANCHOR_SECONDS: + if anchor and self.anchor_due(): self.anchor() def execute(self) -> None: runs = Path(self.design["runs"]) - self.host = self.host_factory(runs / "host") + starting, threads = {}, [] + setup_open = set() try: - starting = {} + self.host = self.host_factory(runs / "host") + ready, failures = {}, {} + startup = clock() + for lanes in self.lanes.values(): + lanes.event( + "setup-start", + "run startup", + at=startup, + phase="startup", + start=startup, + ) + setup_open.add(lanes.model) for model, lanes in self.lanes.items(): + self.watch(anchor=False) lanes.completion = self.completion_for(lanes.profiles["pe3"]) stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) lanes.manifest_sha256 = write_json( @@ -1701,41 +1849,188 @@ def execute(self) -> None: # A model whose lanes never decide still has both logs. for name in ("run-log.jsonl", "admission-log.jsonl"): (lanes.folder / name).touch() + + def start_pool(model, lanes): + try: + ready[model] = lanes.ensure_pool(startup=True) + except Exception as error: + failures[model] = error + self._failed.set() + + for model, lanes in self.lanes.items(): + self.watch() + if lanes.halted(): + continue starting[model] = threading.Thread( - target=lanes.ensure_pool, daemon=True + target=start_pool, args=(model, lanes), daemon=True ) starting[model].start() - for thread in starting.values(): - thread.join() - threads = [ - threading.Thread( - target=lanes.lane, args=(lane,), name=f"{model}-{lane}", daemon=True + while any(t.is_alive() for t in starting.values()): + self.watch() + for thread in starting.values(): + thread.join(timeout=0.5 / len(starting)) + self.watch() + for model, lanes in self.lanes.items(): + outcome = ( + "failed" + if model in failures + else "stopped" + if lanes.halted() + else "ready" + if ready.get(model) + else "failed" ) - for model, lanes in self.lanes.items() - for lane in (1, 2) - ] - for thread in threads: - thread.start() + lanes.event( + "setup-end", + "run startup", + phase="startup", + start=startup, + outcome=outcome, + ) + setup_open.discard(model) + if outcome == "failed" and model not in failures: + lanes.stop("the engine pool could not be started", "pool") + if failures: + raise next(iter(failures.values())) + threads = [] + for model, lanes in self.lanes.items(): + for lane in (1, 2): + self.watch() + with lanes.cond: + if lanes.halted() or self.signalled is not None: + continue + thread = threading.Thread( + target=lanes.lane, + args=(lane,), + name=f"{model}-{lane}", + daemon=True, + ) + thread.start() + threads.append(thread) while any(t.is_alive() for t in threads): self.watch() for t in threads: t.join(timeout=0.5 / len(threads)) self.watch() + except Exception as error: + self._failed.set() + # Supervisors finish their bounded attempts and recovery before + # the ledger, host and final anchor can close underneath them. + for lanes in self.lanes.values(): + with lanes.cond: + lanes.cond.notify_all() + for thread in [*starting.values(), *threads]: + while thread.is_alive(): + thread.join(timeout=WAIT_SECONDS) + try: + self.watch(anchor=False) + except Exception: + self.say(traceback.format_exc()) + for model in setup_open: + try: + self.lanes[model].event( + "setup-end", + "run startup", + phase="startup", + start=startup, + outcome="failed", + ) + except Exception: + self.say(traceback.format_exc()) + self.stop_all( + f"runner error: {type(error).__name__}: {error}", "runner-error" + ) + raise finally: - for model, lanes in self.lanes.items(): - for item in list(lanes.abandoned): - item["thread"].join(timeout=LATE_JOIN) - lanes.log_late(final=True) + self.finish() + + def finish(self) -> None: + """One join deadline, then close pools and host before closing the + ledger. Its final late costs, crossings and signals precede the last + anchor. An unresolved request makes future spend unknowable: record + it explicitly, with a runner-error stop, rather than claim it settled.""" + errors = [] + + def attempt(work): + try: + work() + return True + except Exception as error: + errors.append(error) + self._failed.set() + self.say(traceback.format_exc()) + self.stop_all( + f"runner error during cleanup: {type(error).__name__}: {error}", + "runner-error", + ) + return False + + until = time.monotonic() + LATE_JOIN + for lanes in self.lanes.values(): + for item in list(lanes.abandoned): + while item["thread"].is_alive() and time.monotonic() < until: + item["thread"].join( + timeout=min(WAIT_SECONDS, max(0.0, until - time.monotonic())) + ) + attempt(lambda: self.watch(anchor=False)) + for lanes in self.lanes.values(): + if lanes.pool is not None: + attempt(lanes.pool.close) + if self.host is not None: + attempt(self.host.close) + pending = [] + for lanes in self.lanes.values(): + with lanes.cond: + pending.extend({**item, "lanes": lanes} for item in lanes.abandoned) + unresolved = self.ceiling.close(pending) + for lanes in self.lanes.values(): + attempt(lambda: lanes.log_late(final=True)) + for item in unresolved: + attempt( + lambda: item["lanes"].event( + "late-unresolved", + "request unresolved when the ledger closed; " + "future spend is unrecorded", + household=item["household"], + arm=item["arm"], + attempt=item["attempt"], + outcome="unresolved-at-ledger-close", + counted_usd=item["spend"].spent_usd + item["spend"].late_usd, + ) + ) + if unresolved: + self._failed.set() + self.stop_all( + f"{len(unresolved)} abandoned requests unresolved at ledger close; " + "future spend is unrecorded", + "runner-error", + ) + # A first signal can also arrive while the final push is running. + while True: + attempt(lambda: self.watch(anchor=False)) + for lanes in self.lanes.values(): stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) - write_json( - lanes.folder / f"run-summary-{stamp}.json", - lanes.summary(), - indent=2, + attempt( + lambda: write_json( + lanes.folder / f"run-summary-{stamp}.json", + lanes.summary(), + indent=2, + ) ) - if lanes.pool is not None: - lanes.pool.close() - self.host.close() - self.anchor(wait=True) + if not attempt(lambda: self.anchor(wait=True)): + attempt(lambda: self.watch(anchor=False)) + # The error handler appended stops after that snapshot. Keep + # them covered locally even if pushing cannot be recorded. + try: + self.anchor(push=False) + except Exception as error: + errors.append(error) + self.say(traceback.format_exc()) + break + if self.signalled is None or self._interrupted: + break + if errors: + raise errors[0] def interrupt(self, signum, frame) -> None: """The signal handler. It only notes the signal; the main loop stops diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index 626f3ba2..8bc4c175 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -1,8 +1,15 @@ """The anchor command: each call commits the run's anchors and pushes them.""" import hashlib +import io +import json +import math +import os +import signal import subprocess import sys +import threading +import time from pathlib import Path import pytest @@ -10,6 +17,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) import confirmatory_anchor # noqa: E402 +import confirmatory_run as cr # noqa: E402 SCRIPT = Path(confirmatory_anchor.__file__) @@ -89,3 +97,179 @@ def test_a_failed_push_exits_nonzero(clone, tmp_path): text=True, ) assert out.returncode != 0 + + +def bare_run(folder, command): + """Just the anchor writer, without the board, engines or providers.""" + run = object.__new__(cr.Run) + run.design = {"runs": str(folder), "anchor_command": command} + run.out, run._say_lock = io.StringIO(), threading.Lock() + run.lanes, run.host = {}, None + run._anchor_head = cr.chain_seed("a" * 64, "", "anchors.jsonl") + run._anchored = run._next_anchor = -math.inf + run._pushing, run._push = threading.Lock(), None + run._push_error = None + return run + + +def test_failed_pushes_retry_and_keep_every_outcome(tmp_path, monkeypatch): + """A failed remote never looks anchored; retry follows the failure's end, + then a successful push restores the ordinary cadence.""" + now, calls = [1000.0], [] + monkeypatch.setattr(cr.time, "time", lambda: now[0]) + monkeypatch.setattr(cr, "clock", lambda: now[0]) + monkeypatch.setattr(cr, "ANCHOR_SECONDS", 900.0) + monkeypatch.setattr(cr, "ANCHOR_RETRY", 60.0) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 123.0) + + class Push: + def __init__(self, command, **kwargs): + calls.append((command, kwargs)) + self.returncode = 1 if len(calls) < 4 else 0 + + def communicate(self, timeout): + assert timeout == cr.ANCHOR_PUSH_TIMEOUT + now[0] += 7.0 + return "", "remote unavailable" if self.returncode else "pushed" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + run = bare_run(tmp_path, ["push-anchors"]) + assert run.anchor_due() + for i in range(4): + started = now[0] + run.anchor(wait=True) + assert not run.anchor_due() + expected = now[0] + cr.ANCHOR_RETRY if i < 3 else started + cr.ANCHOR_SECONDS + assert run._next_anchor == expected + now[0] = expected + assert run.anchor_due() + anchors = [ + json.loads(x) for x in (tmp_path / "anchors.jsonl").read_text().splitlines() + ] + pushes = [ + json.loads(x) + for x in (tmp_path / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert [x["outcome"] for x in pushes] == ["failed"] * 3 + ["success"] + assert [x["returncode"] for x in pushes] == [1, 1, 1, 0] + assert [x["anchor_time"] for x in pushes] == [x["time"] for x in anchors] + assert all(x["started"] == x["anchor_time"] < x["ended"] for x in pushes) + assert all(x[1]["start_new_session"] for x in calls) + assert len({x["anchor"] for x in pushes}) == 4 + + +def test_an_anchor_command_that_cannot_start_is_recorded(tmp_path): + run = bare_run(tmp_path, [str(tmp_path / "missing-command")]) + run.anchor(wait=True) + push = json.loads((tmp_path / "anchor-pushes.jsonl").read_text()) + assert push["outcome"] == "error" and push["returncode"] is None + assert "FileNotFoundError" in push["output"] + assert run._next_anchor == push["ended"] + cr.ANCHOR_RETRY + + +@pytest.mark.parametrize("wait", [False, True]) +def test_push_log_io_failure_reaches_the_supervisor(tmp_path, monkeypatch, wait): + """A successful remote push whose outcome could not be synced is still a + runner error, both in supervision and in the final blocking anchor.""" + calls, fsync = [], os.fsync + + class Push: + returncode = 0 + + def __init__(self, *args, **kwargs): + pass + + def communicate(self, timeout): + return "pushed", "" + + def fail_push_sync(fd): + calls.append(fd) + if len(calls) == 2: # The anchor itself was synced; its push outcome was not. + raise OSError("push-log fsync failed") + return fsync(fd) + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + monkeypatch.setattr(cr.os, "fsync", fail_push_sync) + run = bare_run(tmp_path, ["push-anchors"]) + if wait: + with pytest.raises(OSError, match="anchor push outcome could not be recorded"): + run.anchor(wait=True) + else: + run.anchor() + run._push.join() + with pytest.raises(OSError, match="push-log fsync failed"): + run.anchor_due() + assert not run._push.is_alive() + assert len(calls) == 2 + + +def test_a_push_timeout_kills_its_children_and_grandchildren(tmp_path, monkeypatch): + """All descendants inherit the captured pipe: reaping the command cannot + finish until they close it. The timeout starts after both have launched.""" + script = tmp_path / "push.py" + script.write_text( + "import os, subprocess, sys, threading\n" + "from pathlib import Path\n" + "folder, level = Path(sys.argv[1]), int(sys.argv[2])\n" + "(folder / f'pid-{level}').write_text(str(os.getpid()))\n" + "if level < 2:\n" + " child = subprocess.Popen(\n" + " [sys.executable, __file__, str(folder), str(level + 1)])\n" + "threading.Event().wait()\n" + ) + run = bare_run(tmp_path, [sys.executable, str(script), str(tmp_path), "0"]) + real_popen, real_killpg = subprocess.Popen, os.killpg + children, kills, errors = [], [], [] + done = threading.Event() + + def popen(*args, **kwargs): + child = real_popen(*args, **kwargs) + children.append(child) + communicate = child.communicate + + def ready_communicate(timeout=None): + if timeout is not None: + deadline = time.monotonic() + 30.0 + ready = tmp_path / "pid-2" + while not ready.exists() or not ready.read_text().strip(): + assert time.monotonic() < deadline, "the descendants did not start" + done.wait(0.01) + return communicate(timeout=timeout) + + child.communicate = ready_communicate + return child + + def killpg(pid, signum): + kills.append((pid, signum)) + return real_killpg(pid, signum) + + def push(): + try: + run.anchor(wait=True) + except BaseException as error: + errors.append(error) + finally: + done.set() + + with monkeypatch.context() as patch: + patch.setattr(cr.subprocess, "Popen", popen) + patch.setattr(cr.os, "killpg", killpg) + patch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 0.05) + thread = threading.Thread(target=push, daemon=True) + thread.start() + finished = done.wait(45.0) + if not finished: + for child in children: + try: + real_killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + thread.join(timeout=5.0) + assert finished, "the timed-out command left descendants holding its pipes" + assert not errors + assert len(children) == 1 and kills == [(children[0].pid, signal.SIGKILL)] + push = json.loads((tmp_path / "anchor-pushes.jsonl").read_text()) + assert push["outcome"] == "timeout" and push["returncode"] == -signal.SIGKILL + assert push["started"] <= push["ended"] + for level in (1, 2): + assert int((tmp_path / f"pid-{level}").read_text()) > 0 diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 1e962383..1be358f9 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -52,7 +52,7 @@ ANCHOR_SCRIPT = Path(__file__).resolve().parent / "fake_anchor.py" # How far apart anchors and their triggers' stops may be in a test, on a # loaded host. -ANCHOR_SLACK = 5.0 +ANCHOR_SLACK = 30.0 TRIGGER_BOUND = 5.0 @@ -78,6 +78,7 @@ def test_a_stream_is_keyed_by_its_name_alone(): ) +@settings(deadline=None) @given(st.integers(0, (1 << 64) - 1), st.integers(1, 12)) def test_the_draw_is_seeded_and_without_replacement(master, n): got = cr.draw(master, list(reversed(POOL)), n=n) @@ -349,11 +350,11 @@ def host(self, folder): @pytest.fixture def fast(monkeypatch): - """Windows scaled down a thousandfold, so a fake run takes a second; the - host-log gap less, so a sampler a busy host stalls past 90 ms (seen at a - load of 12) is not a gap.""" + """Short evidence windows, with room for the shared host's scheduler. + Intentional gaps scale with this bound; ordinary stalls must not turn + an untouched fake run into a host-failure test.""" monkeypatch.setattr(cr, "PRESSURE_WINDOW", 0.06) - monkeypatch.setattr(cr, "HOST_LOG_GAP", 0.5) + monkeypatch.setattr(cr, "HOST_LOG_GAP", 5.0) monkeypatch.setattr(cr, "WAIT_SECONDS", 0.005) monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) monkeypatch.setattr(cr, "SAMPLER_SILENCE", 1.0) @@ -399,8 +400,9 @@ def test_a_stalled_host_log_is_a_gap(tmp_path, fast): host = readers.host(tmp_path) time.sleep(0.03) start = time.time() - readers.stall = 0.8 - time.sleep(0.85) + stall = cr.HOST_LOG_GAP + 0.3 + readers.stall = stall + time.sleep(stall + 0.05) end = time.time() host.wait_settled(end) host.refresh(after=end) @@ -435,7 +437,7 @@ def test_a_failed_pressure_reading_is_no_sample(tmp_path, fast): time.sleep(0.03) start = time.time() readers.pressure = None - time.sleep(0.7) + time.sleep(cr.HOST_LOG_GAP + 0.2) readers.pressure = 1 end = time.time() host.wait_settled(end) @@ -512,10 +514,11 @@ def test_a_sampler_whose_samples_fail_is_dead_after_its_silence(tmp_path, fast): host = readers.host(tmp_path) assert host.dead() is None readers.pressure = None + host.sample() time.sleep(cr.SAMPLER_SILENCE + 0.2) assert "samples failing since" in host.dead() readers.pressure = 1 - time.sleep(0.1) + host.sample() assert host.dead() is None # a good sample: alive again host.close() @@ -632,6 +635,7 @@ def __call__(self, lanes, sid, label, tool_call, spend): if spec.get("pressure"): with readers.lock: readers.pressing += 1 + lanes.run.host.sample() # the planted pressure is actually observed if spec.get("stall"): readers.stall = spec["stall"] start = time.time() @@ -685,12 +689,21 @@ def __call__(self, lanes, sid, label, tool_call, spend): folder = lanes.folder def resume(): + with lanes.cond: + if ( + not lanes.cond.wait_for( + lambda: lanes.paused or lanes.halted(), timeout=30 + ) + or not lanes.paused + ): + return + time.sleep(spec["resume_after"]) cr.write_json( folder / "RESUME.json", - {"time": time.time(), "note": "fixed in a test"}, + {"time": cr.clock(), "note": "fixed in a test"}, ) - threading.Timer(spec["resume_after"], resume).start() + threading.Thread(target=resume, daemon=True).start() return rec @@ -1320,7 +1333,7 @@ def test_host_trouble_during_an_attempt_reruns_it(world): pressured, stalled = w.draw_order[0], w.draw_order[2] # both lane 1 w.provider.specs = { (model, "loo-flat", pressured, 1): {"pressure": True, "sleep": 0.05}, - (model, "pe", stalled, 1): {"stall": 0.8}, + (model, "pe", stalled, 1): {"stall": cr.HOST_LOG_GAP + 0.3}, } w.execute() @@ -1341,7 +1354,10 @@ def first(sid, label): assert run_problems(w) == [] -def test_a_guard_pause_over_an_attempt_reruns_it(world): +def test_a_guard_pause_over_an_attempt_reruns_it(world, monkeypatch): + # Guard lines have whole seconds and the fake brackets by another + # second: keep the lookahead longer than its possible two-second backdate. + monkeypatch.setattr(cr, "PRESSURE_WINDOW", 2.1) model = "gpt-6-sol" w = world(models=(model,)) sid = w.draw_order[0] @@ -1507,7 +1523,11 @@ def late(sid, lane): w.execute() admission = w.runs() / "gpt-6-sol" / "admission-log.jsonl" assert admission.exists() and admission.read_text() == "" - assert [x["event"] for x in w.lines("gpt-6-sol")] == ["ceiling"] + assert [ + x["event"] + for x in w.lines("gpt-6-sol") + if x["event"] not in ("setup-start", "setup-end") + ] == ["ceiling"] assert run_problems(w) == [] @@ -1568,6 +1588,11 @@ def call(self, message): w.provider.specs = {(model, "pe", sid, 1): {"calls": [{"situation": "{}"}]}} w.execute() assert len(starts) == 2 and starts[0].closed + setup = [e for e in w.events(model) if e.get("phase") == "restart"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[0]["start"] == setup[0]["time"] == setup[1]["start"] + assert setup[1]["time"] >= setup[0]["time"] + assert setup[1]["outcome"] == "ready" assert run_problems(w) == [] @@ -1593,36 +1618,62 @@ def losing(self, message): stops = w.events(model, "stop") assert stops and "engine pool" in stops[0]["reason"] assert stops[0]["cause"] == "pool" + setup = [e for e in w.events(model) if e.get("phase") == "restart"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[1]["outcome"] == "failed" assert run_problems(w) == [] def test_a_household_a_stop_cuts_short_is_under_way_until_the_stop(world): """Lane 1's first household loses the pool in its first arm, and the - restart fails 0.6 s later. In between, lane 2 finishes its household and + restart waits while lane 2 finishes its household and decides its next: the household lane 1 cut short has ended its last attempt, but it is still under way, so its spend is not counted.""" model = "gpt-6-sol" made = [] + other_second, restarting, decided_after = ( + threading.Event(), + threading.Event(), + threading.Event(), + ) class Losing(FakePool): def call(self, message): + assert other_second.wait(30) self.lost = 2 return super().call(message) def pool(lanes): made.append(1) if len(made) > 1: - time.sleep(0.6) + restarting.set() + assert decided_after.wait(30) raise RuntimeError("tool worker failed to start: a test") return Losing(lanes) w = world(models=(model,), pool=pool) cut, other, after = w.draw_order[0], w.draw_order[1], w.draw_order[3] - orders = w.run.lanes[model].orders + lanes = w.run.lanes[model] + orders, admit = lanes.orders, lanes.admit + + def in_second(lanes): + other_second.set() + assert restarting.wait(30) + + def admitted(sid, lane): + ok = admit(sid, lane) + if sid == after and ok: + decided_after.set() + # Keep this admitted household from racing the failed restart + # for the pool lock before lane 1 records its stop. + with lanes.cond: + assert lanes.cond.wait_for(lambda: lanes.stopped, timeout=30) + return ok + + lanes.admit = admitted w.provider.specs = { - (model, orders[cut][0], cut, 1): {"sleep": 0.1, "calls": [{"situation": "{}"}]}, - (model, orders[other][0], other, 1): {"sleep": 0.02}, - (model, orders[other][1], other, 1): {"sleep": 0.3}, + (model, orders[cut][0], cut, 1): {"calls": [{"situation": "{}"}]}, + (model, orders[other][1], other, 1): {"check": in_second}, } w.execute() stop = w.events(model, "stop")[0]["time"] @@ -1633,6 +1684,11 @@ def pool(lanes): assert [x["arm"] for x in w.attempts(model) if x["household"] == cut] == [ orders[cut][0] ] + setup = [e for e in w.events(model) if e.get("phase") == "restart"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[0]["start"] == setup[0]["time"] == setup[1]["start"] + assert setup[1]["time"] >= decided[after]["time"] + assert setup[1]["outcome"] == "failed" assert run_problems(w) == [] @@ -1708,7 +1764,7 @@ def crossing(): @pytest.mark.parametrize( "when", ["before the run", "before the run, misdated", "during the pause"] ) -def test_a_resume_written_before_the_pause_is_ignored(world, when): +def test_a_resume_written_before_the_pause_is_ignored(world, when, monkeypatch): """A resume left from before a pause would end that pause unseen: it is set aside, and the pause holds until a resume written during it. One found while no pause holds is set aside at once, whatever time it @@ -1725,27 +1781,46 @@ def test_a_resume_written_before_the_pause_is_ignored(world, when): stale["time"] += 3600.0 # a writer whose clock runs ahead cr.write_json(folder / "RESUME.json", stale) - def plant(lanes): - # Lands once the pause holds: the conversation returns at once. - threading.Timer( - 0.1, lambda: cr.write_json(folder / "RESUME.json", stale) - ).start() + lanes, watch = w.run.lanes[model], w.run.watch + planted, fresh = False, False + + def supervised(*args, **kwargs): + nonlocal planted, fresh + with lanes.cond: + if when == "during the pause" and lanes.paused and not planted: + cr.write_json(folder / "RESUME.json", stale) + planted = True + watch(*args, **kwargs) + with lanes.cond: + if ( + lanes.paused + and list(folder.glob("resume-ignored-*.json")) + and not fresh + ): + # The stale note has been read and archived before its + # replacement exists; the next watch sees only the fresh one. + cr.write_json( + folder / "RESUME.json", + {"time": cr.clock(), "note": "fixed in a test"}, + ) + fresh = True + + monkeypatch.setattr(w.run, "watch", supervised) w.provider.specs = { (model, "pe3", sid, 1): { "status": "provider_error", "error": "AuthenticationError: bad key", "turns": 0, - "resume_after": 1.5, # the main loop looks every 0.5 s - **({"check": plant} if when == "during the pause" else {}), } } w.execute() pauses, resumes = w.events(model, "pause"), w.events(model, "resume") assert len(pauses) == 1 and len(resumes) == 1 assert resumes[0]["reason"] == "fixed in a test" - assert resumes[0]["time"] - pauses[0]["time"] >= 1.4 - assert list(folder.glob("resume-ignored-*.json")) + assert resumes[0]["time"] >= pauses[0]["time"] + ignored = list(folder.glob("resume-ignored-*.json")) + assert len(ignored) == 1 and json.loads(ignored[0].read_text()) == stale assert run_problems(w) == [] @@ -1965,10 +2040,16 @@ def test_the_checker_catches_an_edited_log_line(world): path = w.runs() / "gpt-6-sol" / "admission-log.jsonl" lines = path.read_text().splitlines() edited = json.loads(lines[1]) - edited["finished_spend"] = 0.0 + edited["finished_spend"] += 1.0 path.write_text("\n".join([lines[0], json.dumps(edited)] + lines[2:]) + "\n") problems = run_problems(w) assert any("its prev is not the line before's hash" in p for p in problems) + # An interior edit need not change a later anchor's head: its successor + # still has its old bytes. A tail edit separately exercises the anchor. + edited = json.loads(lines[-1]) + edited["finished_spend"] += 1.0 + path.write_text("\n".join(lines[:-1] + [json.dumps(edited)]) + "\n") + problems = run_problems(w) assert any("is not the log's head" in p for p in problems) @@ -2035,9 +2116,11 @@ def test_the_checker_catches_a_planted_fault(world): w = world(models=("gpt-6-sol",)).execute() path = w.runs() / "gpt-6-sol" / "run-log.jsonl" lines = path.read_text().splitlines() - first = json.loads(lines[0]) + at = next(i for i, line in enumerate(lines) if "event" not in json.loads(line)) + first = json.loads(lines[at]) first["class"] = "provider" - path.write_text("\n".join([json.dumps(first)] + lines[1:]) + "\n") + lines[at] = json.dumps(first) + path.write_text("\n".join(lines) + "\n") assert any("recomputed" in p for p in run_problems(w)) @@ -2172,3 +2255,530 @@ def test_the_ceiling_ends_a_harness_conversation_at_its_next_turn(world): assert len(attempts) <= 2 # one per lane at most assert len(w.events(model, "ceiling")) == 1 assert run_problems(w) == [] + + +def test_a_reader_that_keeps_raising_starts_the_sampler_silence( + tmp_path, fast, monkeypatch +): + readers = Readers() + host = readers.host(tmp_path) + host._stop.set() + host._thread.join() + now, reads = [time.time()], [] + + def pressure(): + reads.append(now[0]) + raise RuntimeError("pressure reader failed") + + class Ticks: + def wait(self, timeout): + now[0] += cr.SAMPLER_SILENCE + 1.0 + return len(reads) == 3 + + # Run the real sampling loop across three failures with a controlled + # clock; no scheduling or sleep decides whether its silence has elapsed. + with monkeypatch.context() as patch: + patch.setattr(cr.time, "time", lambda: now[0]) + patch.setattr(host, "_stop", Ticks()) + patch.setattr(host, "_read_pressure", pressure) + patch.setattr(host._thread, "is_alive", lambda: True) + host._loop() + assert len(reads) == 3 + assert host.failing_since == reads[0] + assert "samples failing since" in host.dead() + assert "RuntimeError: pressure reader failed" in host.dead() + patch.setattr(host, "_read_pressure", lambda: 1) + host.sample() + assert host.failing_since is None and host.failure is None + assert host.dead() is None + host.close() + + +def test_a_second_lane_s_authentication_failure_invalidates_a_resume( + world, monkeypatch +): + model = "gpt-6-luna" + w = world(models=(model,)) + lanes = w.run.lanes[model] + first, second = w.draw_order[:2] + lanes.spent.update({first: 0.0, second: 0.0}) + now = [time.time()] + + def tick(): + now[0] += 1.0 + return now[0] + + monkeypatch.setattr(cr, "clock", tick) + auth = record("provider_error", "AuthenticationError: bad key", turns=0) + lanes.end_attempt(first, "pe3", 1, now[0], auth, False) + note = {"time": lanes.paused_at, "note": "first key fixed"} + cr.write_json(lanes.folder / "RESUME.json", note) + resume = lanes.resume + + def second_failure(*args, **kwargs): + # The other lane finishes after watch reads the note and immediately + # before resume validates it. The second pause requires its own note. + lanes.end_attempt(second, "pe3", 1, now[0], auth, False) + return resume(*args, **kwargs) + + monkeypatch.setattr(lanes, "resume", second_failure) + w.run.watch(anchor=False) + assert lanes.paused and lanes.paused_at > note["time"] + assert len(w.events(model, "pause")) == 2 + assert w.events(model, "resume") == [] + assert list(lanes.folder.glob("resume-ignored-*.json")) + monkeypatch.setattr(lanes, "resume", resume) + cr.write_json( + lanes.folder / "RESUME.json", + {"time": lanes.paused_at, "note": "both keys fixed"}, + ) + w.run.watch(anchor=False) + assert not lanes.paused + assert [event["reason"] for event in w.events(model, "resume")] == [ + "both keys fixed" + ] + + +def test_an_interrupt_during_setup_starts_no_pool_or_lane(world, monkeypatch): + started = [] + w = world( + models=("gpt-6-luna", "gpt-6-sol"), + pool=lambda lanes: started.append((lanes.model, "pool")), + ) + + def completion(profile): + if w.run.signalled is None: + w.run.interrupt(2, None) + + w.run.completion_for = completion + for model, lanes in w.run.lanes.items(): + monkeypatch.setattr( + lanes, "lane", lambda lane, m=model: started.append((m, lane)) + ) + w.execute() + assert started == [] and w.provider.count == {} + for model in w.models: + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + setup = [e for e in w.events(model) if e.get("phase") == "startup"] + assert [e["event"] for e in setup] == ["setup-start", "setup-end"] + assert setup[0]["start"] == setup[0]["time"] == setup[1]["start"] + stop = w.events(model, "stop")[0]["time"] + assert setup[0]["time"] <= stop <= setup[1]["time"] + assert setup[1]["outcome"] == "stopped" + assert run_problems(w) == [] + + +def test_a_slow_pool_start_is_polled_and_anchored_before_it_finishes( + world, monkeypatch +): + model = "gpt-6-sol" + entered, release, finished = threading.Event(), threading.Event(), threading.Event() + polls, anchors = [], [] + + def pool(lanes): + entered.set() + # Readiness depends on supervision, never on which sleep wins. + assert release.wait(10), "startup was not supervised" + finished.set() + return FakePool(lanes) + + w = world(models=(model,), pool=pool) + watch, anchor = w.run.watch, w.run.anchor + + def supervised(*args, **kwargs): + watch(*args, **kwargs) + if entered.is_set() and not finished.is_set(): + polls.append(cr.clock()) + if len(polls) == 3: + release.set() + + def anchored(*args, **kwargs): + if entered.is_set() and not finished.is_set(): + anchors.append(cr.clock()) + return anchor(*args, **kwargs) + + monkeypatch.setattr(w.run, "watch", supervised) + monkeypatch.setattr(w.run, "anchor", anchored) + w.execute() + assert len(polls) >= 3 and anchors + setup = [e for e in w.events(model) if e.get("phase") == "startup"] + assert setup[0]["time"] <= min(polls + anchors) + assert max(polls + anchors) <= setup[1]["time"] + assert setup[1]["outcome"] == "ready" + assert all(a["start"] >= setup[1]["time"] for a in w.attempts(model)) + assert run_problems(w) == [] + + +def test_an_interrupt_during_pool_start_is_recorded_before_readiness( + world, monkeypatch +): + model = "gpt-6-sol" + stopped = threading.Event() + started = [] + + def pool(lanes): + w.run.interrupt(2, None) + assert stopped.wait(10), "the setup interrupt was not processed" + return FakePool(lanes) + + w = world(models=(model,), pool=pool) + watch = w.run.watch + + def supervised(*args, **kwargs): + watch(*args, **kwargs) + if w.run.lanes[model].stopped: + stopped.set() + + monkeypatch.setattr(w.run, "watch", supervised) + monkeypatch.setattr(w.run.lanes[model], "lane", started.append) + w.execute() + assert stopped.is_set() and started == [] + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + assert w.events(model, "setup-end")[0]["outcome"] == "stopped" + assert w.run.lanes[model].pool.closed + assert run_problems(w) == [] + + +def test_an_interrupt_between_lane_launches_prevents_the_next_lane(world, monkeypatch): + model = "gpt-6-sol" + w = world(models=(model,)) + signalled = threading.Event() + started = [] + start = threading.Thread.start + + def lane(number): + started.append(number) + w.run.interrupt(2, None) + signalled.set() + + def start_and_signal(thread): + start(thread) + if thread.name == f"{model}-1": + assert signalled.wait(10) + + monkeypatch.setattr(w.run.lanes[model], "lane", lane) + monkeypatch.setattr(threading.Thread, "start", start_and_signal) + w.execute() + assert started == [1] + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + assert run_problems(w) == [] + + +def test_stop_all_notifies_every_model_before_a_failed_event_write(world, monkeypatch): + w = world(models=("gpt-6-luna", "gpt-6-sol")) + notified = [] + for model, lanes in w.run.lanes.items(): + notify = lanes.cond.notify_all + + def observed(model=model, notify=notify): + notified.append(model) + notify() + + monkeypatch.setattr(lanes.cond, "notify_all", observed) + first = w.run.lanes[w.models[0]] + + def broken(line, name="run-log.jsonl"): + assert all(lanes.stopped for lanes in w.run.lanes.values()) + raise OSError("the first model's event log is unavailable") + + monkeypatch.setattr(first, "write_line", broken) + w.run.stop_all("disk failure", "runner-error") + assert notified == list(w.models) + assert all(lanes.stopped for lanes in w.run.lanes.values()) + assert w.events(w.models[1], "stop")[0]["cause"] == "runner-error" + + +@pytest.mark.parametrize( + "kind,error", [("pause", "AuthenticationError"), ("stop", "BadRequestError")] +) +def test_end_attempt_retains_the_record_before_a_failed_event_write( + world, monkeypatch, kind, error +): + model = "gpt-6-luna" + w = world(models=(model, "gpt-6-sol")) + sid = w.draw_order[0] + lanes = w.run.lanes[model] + label = lanes.orders[sid][0] + w.provider.specs = { + (model, label, sid, 1): { + "status": "provider_error", + "error": f"{error}: a test", + "turns": 0 if kind == "stop" else 1, + } + } + write = lanes.write_line + failed = [] + + def broken(line, name="run-log.jsonl"): + if line.get("event") == kind and not failed: + failed.append(line) + raise OSError("end_attempt's event write failed") + write(line, name) + + monkeypatch.setattr(lanes, "write_line", broken) + w.execute() + assert len(failed) == 1 + record = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + assert record["error"] == f"{error}: a test" + assert record["totals"]["cost_usd"] == pytest.approx(0 if kind == "stop" else COST) + lines = w.lines(model) + saved = next( + i + for i, x in enumerate(lines) + if x.get("household") == sid and x.get("arm") == label + ) + stopped = next(i for i, x in enumerate(lines) if x.get("cause") == "runner-error") + assert saved < stopped + assert all(lanes.stopped for lanes in w.run.lanes.values()) + assert ( + lanes.folder / "in-flight" / sid + ).exists() is False # event failed before disk write + + +def test_a_final_record_write_failure_still_recovers_the_other_arms(world, monkeypatch): + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + w.run.no_output.update(w.draw_order[1:]) + first = w.run.lanes[model].orders[sid][0] + target = w.folder(model, first) / f"{sid}.json" + write = cr.write_json + + def broken(path, payload, indent=1): + if Path(path) == target: + raise OSError("one final record cannot be written") + return write(path, payload, indent) + + monkeypatch.setattr(cr, "write_json", broken) + w.execute() + assert len(w.attempts(model)) == len(cr.ARMS) - 1 + assert all( + (w.folder(model, a) / f"{sid}.json").exists() for a in cr.ARMS if a != first + ) + assert (w.run.lanes[model].folder / "in-flight" / sid / f"{first}-1.json").exists() + assert w.lines(model)[-1]["cause"] == "runner-error" + + +def blocked_cleanup_request(w, monkeypatch): + """One real request, released by the test after the abandoned join.""" + monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 0.02) + monkeypatch.setattr(cr, "LATE_JOIN", 0.0) + monkeypatch.setattr(cr, "CEILING_MARGIN", 0.0) + w.design.update(overall_cap=0.10, spent_before=0.0) + w.run.no_output.update(w.draw_order[1:]) + w.run.ceiling = cr.Ceiling(0.10, 0.0) + release, returned = threading.Event(), threading.Event() + + def converse(lanes, sid, label, tool_call, spend): + assert release.wait(30) + spend.add(0.20) + returned.set() + return {"status": "budget_stop", "turns": [], "totals": {"cost_usd": 0.20}} + + w.run.converse = converse + return release, returned + + +def test_cleanup_counts_a_request_returning_during_host_close(world, monkeypatch): + model = "gpt-6-luna" + w = world(models=(model,)) + release, returned = blocked_cleanup_request(w, monkeypatch) + host_factory = w.run.host_factory + + def host(folder): + evidence = host_factory(folder) + close = evidence.close + + def finish(): + release.set() + assert returned.wait(10) + for lanes in w.run.lanes.values(): + for item in list(lanes.abandoned): + item["thread"].join(timeout=10) + assert not item["thread"].is_alive() + close() + + evidence.close = finish + return evidence + + w.run.host_factory = host + try: + w.execute() + finally: + release.set() + assert returned.is_set() + assert w.run.ceiling.spent_usd == pytest.approx(0.20) + assert w.run.ceiling.closed + assert [e["cost_usd"] for e in w.events(model, "late-cost")] == [ + pytest.approx(0.20) + ] + assert len(w.events(model, "ceiling")) == 1 + assert w.events(model, "late-unresolved") == [] + assert anchor_problems(w) == [] + assert run_problems(w) == [] + + +@pytest.mark.parametrize("returns_at_close", [False, True]) +def test_cleanup_records_an_unresolved_request_before_closing_the_ledger( + world, monkeypatch, returns_at_close +): + model = "gpt-6-luna" + w = world(models=(model,)) + release, returned = blocked_cleanup_request(w, monkeypatch) + if returns_at_close: + close = w.run.ceiling.close + + def finish(pending=None): + unresolved = close(pending) + release.set() + assert returned.wait(10) + return unresolved + + monkeypatch.setattr(w.run.ceiling, "close", finish) + try: + w.execute() + unresolved = w.events(model, "late-unresolved") + assert len(unresolved) == 1 + assert unresolved[0]["outcome"] == "unresolved-at-ledger-close" + assert unresolved[0]["counted_usd"] == 0.0 + assert w.events(model, "stop")[-1]["cause"] == "runner-error" + assert w.run.ceiling.closed + assert anchor_problems(w) == [] + before = (w.runs() / model / "run-log.jsonl").read_bytes() + finally: + release.set() + assert returned.wait(10) + assert w.run.ceiling.spent_usd == 0.0 + assert w.run.ceiling.unrecorded_usd == pytest.approx(0.20) + assert (w.runs() / model / "run-log.jsonl").read_bytes() == before + + +@pytest.mark.parametrize("during", ["host-close", "final-push"]) +def test_a_first_interrupt_during_cleanup_is_recorded_and_anchored( + world, monkeypatch, during +): + model = "gpt-6-sol" + w = world(models=(model,)) + w.run.no_output.update(w.draw_order) + if during == "host-close": + host_factory = w.run.host_factory + + def host(folder): + evidence = host_factory(folder) + close = evidence.close + + def finish(): + w.run.interrupt(2, None) + close() + + evidence.close = finish + return evidence + + w.run.host_factory = host + else: + anchor = w.run.anchor + signalled = [] + + def push(wait=False): + anchor(wait=wait) + if wait and not signalled: + signalled.append(True) + w.run.interrupt(2, None) + + monkeypatch.setattr(w.run, "anchor", push) + w.execute() + assert [e["cause"] for e in w.events(model, "stop")] == ["interrupt"] + assert w.run._interrupted + assert anchor_problems(w) == [] + + +def test_a_replacement_log_failure_keeps_a_valid_final_history(world, monkeypatch): + model = "gpt-6-luna" + w = world(models=(model,)) + sid = w.draw_order[0] + w.run.no_output.update(w.draw_order[1:]) + lanes = w.run.lanes[model] + label = lanes.orders[sid][0] + w.provider.specs = { + (model, label, sid, 1): { + "status": "provider_error", + "error": "RateLimitError: a test", + } + } + write, failed = lanes.write_line, [] + + def broken(line, name="run-log.jsonl"): + if line.get("arm") == label and line.get("attempt") == 1 and not failed: + failed.append(line) + raise OSError("the replaced attempt's log write failed") + write(line, name) + + monkeypatch.setattr(lanes, "write_line", broken) + w.execute() + rec = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + assert rec["attempts"] == [] + assert w.provider.count[(model, label, sid)] == 1 + logged = [x for x in w.attempts(model) if x["arm"] == label] + assert len(logged) == 1 + assert all( + logged[0][key] == value for key, value in failed[0].items() if key != "sha256" + ) + assert logged[0]["sha256"] == cr.sha256_file(w.folder(model, label) / f"{sid}.json") + assert run_problems(w) == [] + + +def test_a_supervision_error_drains_active_lanes_before_final_anchor( + world, monkeypatch +): + model = "gpt-6-sol" + w = world(models=(model,)) + w.run.no_output.update(w.draw_order[1:]) + started, returned = threading.Event(), threading.Event() + converse, watch = w.run.converse, w.run.watch + failed = [] + + def request(*args): + started.set() + assert w.run._failed.wait(10) + rec = converse(*args) + returned.set() + return rec + + def broken_watch(anchor=True): + if started.is_set() and not failed: + failed.append(True) + raise OSError("supervision failed while a request was active") + watch(anchor=anchor) + + w.run.converse = request + monkeypatch.setattr(w.run, "watch", broken_watch) + with pytest.raises(OSError, match="supervision failed"): + w.execute() + assert returned.is_set() + assert len(w.attempts(model)) == 1 + assert w.lines(model)[-1]["cause"] == "runner-error" + assert w.run.ceiling.spent_usd == pytest.approx(1.0 + COST) + assert anchor_problems(w) == [] + assert run_problems(w) == [] + + +def test_a_final_push_log_failure_is_stopped_and_locally_anchored(world, monkeypatch): + model = "gpt-6-sol" + w = world(models=(model,)) + w.run.no_output.update(w.draw_order) + push = w.run._push_anchor + + def broken(head, anchor_time): + if w.run.host.end is not None: + with w.run._pushing: + w.run._push_error = OSError("the final push outcome cannot be written") + else: + push(head, anchor_time) + + monkeypatch.setattr(w.run, "_push_anchor", broken) + with pytest.raises(OSError, match="push outcome could not be recorded"): + w.execute() + assert w.events(model, "stop")[-1]["cause"] == "runner-error" + last = json.loads((w.runs() / "anchors.jsonl").read_text().splitlines()[-1]) + assert last["models"][model]["run-log.jsonl"]["lines"] == len(w.lines(model)) + assert last["host-log.jsonl"] == w.run.host.heads() From 00d267ae17e124f4c62d4f8169d6ce1402ced638 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 09:55:38 -0700 Subject: [PATCH 10/48] Confirmatory runner: round-3 fixes Answers review runner-r3 findings 2, 4, 5 and 6, and orders push times: - Push a synced, atomically installed snapshot of the anchors file. Give each push its own path, including after a thread is abandoned, and keep the remote record named anchors.jsonl. - Try the final push up to three times per set of heads, a minute apart, writing a new anchor for each try without changing the covered logs. - Bound pipe draining after process-group termination and every push join. Record a stuck thread once as an error and reject its late outcome. - Ignore signals before the last cleanup check, then log and anchor a first signal that arrived in the gap. Keep first and second signal handling active throughout retries. - Take anchor cadence and push intervals from the runner's ordered clock. Exercise overlapping pushes, retry exhaustion, escaped descendants, abandoned threads, cleanup signals and a backward wall-clock step. The report's record fields and outcome names are unchanged. Push snapshots are copies, not records; the draft should distinguish final retry exhaustion from the ongoing run's retry cadence, and retries after newly logged signals. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 9 +- scripts/confirmatory_run.py | 167 ++++++++++++++----- tests/test_confirmatory_anchor.py | 260 +++++++++++++++++++++++++++++- tests/test_confirmatory_run.py | 150 +++++++++++++++++ 4 files changed, 536 insertions(+), 50 deletions(-) diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py index a4103d8a..68b4f37f 100644 --- a/scripts/confirmatory_anchor.py +++ b/scripts/confirmatory_anchor.py @@ -3,9 +3,10 @@ Usage: python scripts/confirmatory_anchor.py --repo DIR ANCHORS -The runner calls this every 15 minutes, and once more at its end, with the -path of its `anchors.jsonl`: each line holds every log's line count and chain -head at a time. This copies the file into DIR, a clone checked out on a branch +The runner calls this every 15 minutes and at its end (with up to three final +tries), with a snapshot of its `anchors.jsonl`: each line holds every log's +line count and chain head at a time. The snapshot is a copy, not a record. +This copies it to `anchors.jsonl` in DIR, a clone checked out on a branch of its own, commits it and pushes the branch to its remote. The remote's record of each push is the anchor's time from outside the run: a log edited after an anchor was pushed no longer matches that anchor's head. @@ -30,7 +31,7 @@ def git(repo: Path, *args: str) -> str: def main(argv=None) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--repo", type=Path, required=True, help="the anchors clone") - parser.add_argument("anchors", type=Path, help="the run's anchors.jsonl") + parser.add_argument("anchors", type=Path, help="a snapshot of the run's anchors.jsonl") args = parser.parse_args(argv) repo = args.repo.expanduser() target = repo / "anchors.jsonl" diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index fd3d01da..eff73e76 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -70,7 +70,9 @@ model and the file's name; for the host log, of its name. Every 15 minutes, and at the end, ``anchors.jsonl`` gets a line with every log's line count and head and every manifest's sha256, itself chained, and the - design's ``anchor_command`` is run on it in the background; each run's + design's ``anchor_command`` is run on a synced, atomic copy in the + background. Each ``anchors..push-snapshot.jsonl`` is a copy, not a + record, and is never replaced by a later push's snapshot. Each push's result goes to ``anchor-pushes.jsonl``. ``confirmatory_anchor.py`` is that command: it commits the file to a clone and pushes it, so the remote holds each anchor's time from outside the run. @@ -168,6 +170,8 @@ ANCHOR_SECONDS = 900.0 # how often the logs' chain heads are anchored ANCHOR_RETRY = 60.0 # seconds to the next anchor after a failed push ANCHOR_PUSH_TIMEOUT = 300.0 # seconds; then the command's process group is killed +KILL_GRACE = 10.0 # seconds to drain pipes after killing the process group +FINAL_PUSH_TRIES = 3 # includes the first final push LATE_JOIN = 60.0 # seconds the run's end waits, in all, for abandoned attempts # What a stop event names as its cause. STOP_CAUSES = frozenset( @@ -1646,6 +1650,8 @@ def __init__( self._next_anchor = -math.inf self._push: threading.Thread | None = None self._push_error: OSError | None = None + self._push_pending: dict | None = None + self._last_push_outcome: str | None = None def say(self, text: str) -> None: with self._say_lock: @@ -1679,13 +1685,15 @@ def _check_push_error(self) -> None: def anchor_due(self) -> bool: self._check_push_error() with self._pushing: - return time.time() >= self._next_anchor + return clock() >= self._next_anchor def anchor(self, wait: bool = False, push: bool = True) -> None: """Every log's chain head to `anchors.jsonl`, itself a chain, then the - design's anchor command on that file, in the background (one at a + design's anchor command on a synced snapshot, in the background (one at a time; an anchor whose push would wait is written but not pushed until - the next). Each push's result goes to `anchor-pushes.jsonl`.""" + the next). A snapshot is a copy, not a record, and its path is unique + even if a push thread has to be abandoned. Each push's result goes to + `anchor-pushes.jsonl`.""" runs = Path(self.design["runs"]) line = { "time": clock(), @@ -1701,7 +1709,7 @@ def anchor(self, wait: bool = False, push: bool = True) -> None: handle.write(text + "\n") handle.flush() os.fsync(handle.fileno()) - self._anchor_head, self._anchored = line_sha(text), time.time() + self._anchor_head, self._anchored = line_sha(text), clock() with self._pushing: self._next_anchor = self._anchored + ANCHOR_SECONDS if not push: @@ -1709,8 +1717,21 @@ def anchor(self, wait: bool = False, push: bool = True) -> None: if self._push is not None and self._push.is_alive(): if not wait: return - self._push.join() - self._check_push_error() + self._join_push() + snapshot = runs / f"anchors.{self._anchor_head}.push-snapshot.jsonl" + tmp = snapshot.with_suffix(".tmp") + with tmp.open("wb") as handle: + handle.write((runs / "anchors.jsonl").read_bytes()) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, snapshot) + with self._pushing: + self._push_pending = { + "anchor": self._anchor_head, + "anchor_time": line["time"], + "started": clock(), + } + self._last_push_outcome = None self._push = threading.Thread( target=self._push_anchor, args=(self._anchor_head, line["time"]), @@ -1718,14 +1739,64 @@ def anchor(self, wait: bool = False, push: bool = True) -> None: ) self._push.start() if wait: - self._push.join() - self._check_push_error() + self._join_push() + + def _join_push(self) -> None: + """Bound the push thread's join and claim an error if it cannot end.""" + self._push.join(timeout=ANCHOR_PUSH_TIMEOUT + KILL_GRACE + 30) + if self._push.is_alive(): + with self._pushing: + pending = self._push_pending + if pending is not None: + self._record_push( + { + **pending, + "ended": clock(), + "outcome": "error", + "returncode": None, + "output": "the anchor push thread did not end after its time limit", + } + ) + self._check_push_error() + + def _record_push(self, line: dict) -> None: + """Claim one outcome under the push lock; a late worker cannot repeat it.""" + with self._pushing: + pending = self._push_pending + if pending is None or pending["anchor"] != line["anchor"]: + return + self._push_pending = None + self._last_push_outcome = line["outcome"] + if line["outcome"] != "success": + self._next_anchor = min( + self._next_anchor, line["ended"] + ANCHOR_RETRY + ) + try: + path = Path(self.design["runs"]) / "anchor-pushes.jsonl" + with path.open("a") as handle: + handle.write(json.dumps(line) + "\n") + handle.flush() + os.fsync(handle.fileno()) + except OSError as error: + # A daemon's exception must reach the supervisor or final join. + self._push_error = self._push_error or error + return + if line["outcome"] != "success": + self.say( + f"the anchor push failed ({line['outcome']}, " + f"{line['returncode']}): {line['output']}" + ) def _push_anchor(self, head: str, anchor_time: float) -> None: + """Push one immutable snapshot and record its bounded command's outcome.""" runs = Path(self.design["runs"]) command = [str(x) for x in self.design["anchor_command"]] - command.append(str(runs / "anchors.jsonl")) - started = time.time() + command.append(str(runs / f"anchors.{head}.push-snapshot.jsonl")) + with self._pushing: + pending = self._push_pending + if pending is None or pending["anchor"] != head: + return + started = pending["started"] = clock() try: child = subprocess.Popen( command, @@ -1744,15 +1815,23 @@ def _push_anchor(self, head: str, anchor_time: float) -> None: os.killpg(child.pid, signal.SIGKILL) except ProcessLookupError: pass - stdout, stderr = child.communicate() + try: + stdout, stderr = child.communicate(timeout=KILL_GRACE) + except subprocess.TimeoutExpired as error: + for pipe in (child.stdout, child.stderr): + if pipe is not None: + pipe.close() + stdout, stderr = error.output or b"", error.stderr or b"" + if isinstance(stdout, bytes): + stdout = stdout.decode(errors="replace") + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + stderr += "\npipes were held open after the kill" outcome = "timeout" code, text = child.returncode, (stdout + stderr)[-2000:] except (OSError, subprocess.SubprocessError) as error: code, outcome, text = None, "error", f"{type(error).__name__}: {error}" - ended = time.time() - if outcome != "success": - with self._pushing: - self._next_anchor = min(self._next_anchor, ended + ANCHOR_RETRY) + ended = clock() line = { "anchor": head, "anchor_time": anchor_time, @@ -1762,19 +1841,7 @@ def _push_anchor(self, head: str, anchor_time: float) -> None: "returncode": code, "output": text, } - try: - with (runs / "anchor-pushes.jsonl").open("a") as handle: - handle.write(json.dumps(line) + "\n") - handle.flush() - os.fsync(handle.fileno()) - except OSError as error: - # A daemon's exception cannot stop the run: hand this one back - # to the supervisor, or to the final caller after it joins us. - with self._pushing: - self._push_error = self._push_error or error - return - if outcome != "success": - self.say(f"the anchor push failed ({outcome}, {code}): {text}") + self._record_push(line) # The main loop's work. @@ -1948,7 +2015,9 @@ def finish(self) -> None: """One join deadline, then close pools and host before closing the ledger. Its final late costs, crossings and signals precede the last anchor. An unresolved request makes future spend unknowable: record - it explicitly, with a runner-error stop, rather than claim it settled.""" + it explicitly, with a runner-error stop, rather than claim it settled. + Retry the final push up to three times per set of heads, then ignore + signals and check once more for a first signal needing coverage.""" errors = [] def attempt(work): @@ -2017,21 +2086,35 @@ def attempt(work): indent=2, ) ) - if not attempt(lambda: self.anchor(wait=True)): - attempt(lambda: self.watch(anchor=False)) - # The error handler appended stops after that snapshot. Keep - # them covered locally even if pushing cannot be recorded. - try: - self.anchor(push=False) - except Exception as error: - errors.append(error) - self.say(traceback.format_exc()) - break - if self.signalled is None or self._interrupted: - break + for push_try in range(FINAL_PUSH_TRIES): + if push_try: + time.sleep(ANCHOR_RETRY) + if not attempt(lambda: self.anchor(wait=True)): + attempt(lambda: self.watch(anchor=False)) + # The error handler appended stops after that snapshot. Keep + # them covered locally even if pushing cannot be recorded. + try: + self.anchor(push=False) + except Exception as error: + errors.append(error) + self.say(traceback.format_exc()) + break + if self._last_push_outcome == "success": + break + if self.signalled is not None and not self._interrupted: + continue + self._ignore_signals() + if self.signalled is not None and not self._interrupted: + continue + break if errors: raise errors[0] + def _ignore_signals(self) -> None: + """Close signal delivery before the last check; the process is ending.""" + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + def interrupt(self, signum, frame) -> None: """The signal handler. It only notes the signal; the main loop stops every model's lanes. A second signal exits at once.""" diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index 8bc4c175..f6944c42 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -59,9 +59,11 @@ def remote_file(remote: Path) -> str: return out.stdout -def test_each_anchor_is_committed_and_pushed(clone, tmp_path): +@pytest.mark.parametrize("name", ["anchors.jsonl", "anchors.push-snapshot.jsonl"]) +def test_each_anchor_is_committed_and_pushed(clone, tmp_path, name): + """A snapshot's local name never changes the remote record's name.""" remote, repo = clone - anchors = tmp_path / "anchors.jsonl" + anchors = tmp_path / name lines = ['{"time": 1.0, "prev": "a"}', '{"time": 2.0, "prev": "b"}'] shas = [hashlib.sha256(line.encode()).hexdigest() for line in lines] anchors.write_text(lines[0] + "\n") @@ -70,6 +72,9 @@ def test_each_anchor_is_committed_and_pushed(clone, tmp_path): assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 assert remote_log(remote) == [f"anchor 2: {shas[1]}", f"anchor 1: {shas[0]}"] assert remote_file(remote) == anchors.read_text() + assert sorted(path.name for path in repo.iterdir() if path.name != ".git") == [ + "anchors.jsonl" + ] def test_an_unchanged_file_makes_no_commit(clone, tmp_path): @@ -112,6 +117,20 @@ def bare_run(folder, command): return run +def bare_finish_run(folder, command): + """An anchor writer with an empty, already closed run to finish.""" + + class Ledger: + def close(self, pending): + return [] + + run = bare_run(folder, command) + run.ceiling, run._failed = Ledger(), threading.Event() + run.signalled, run._interrupted = None, False + run.watch = lambda anchor=True: None + return run + + def test_failed_pushes_retry_and_keep_every_outcome(tmp_path, monkeypatch): """A failed remote never looks anchored; retry follows the failure's end, then a successful push restores the ordinary cadence.""" @@ -184,7 +203,8 @@ def communicate(self, timeout): def fail_push_sync(fd): calls.append(fd) - if len(calls) == 2: # The anchor itself was synced; its push outcome was not. + if (tmp_path / "anchor-pushes.jsonl").exists(): + # The anchor and its copy synced; only the outcome's sync fails. raise OSError("push-log fsync failed") return fsync(fd) @@ -200,7 +220,92 @@ def fail_push_sync(fd): with pytest.raises(OSError, match="push-log fsync failed"): run.anchor_due() assert not run._push.is_alive() - assert len(calls) == 2 + assert len(calls) == 3 + + +def test_a_running_push_reads_a_snapshot_while_the_final_anchor_is_appended( + tmp_path, monkeypatch +): + """A push released after the final append still sees its own whole prefix.""" + reading, release, joining = threading.Event(), threading.Event(), threading.Event() + calls, copies, errors = [], [], [] + + class Push: + returncode = 0 + + def __init__(self, command, **kwargs): + self.source = Path(command[-1]) + calls.append(self.source) + self.first = len(calls) == 1 + + def communicate(self, timeout): + if self.first: + reading.set() + assert release.wait(15.0), "the first push was never released" + copies.append(self.source.read_bytes()) + return "pushed", "" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + run = bare_run(tmp_path, ["push-anchors"]) + run.anchor() + assert reading.wait(15.0), "the first push did not start" + first, join = run._push, run._push.join + + def joining_push(timeout=None): + joining.set() + return join(timeout=timeout) + + def final_anchor(): + try: + run.anchor(wait=True) + except BaseException as error: + errors.append(error) + + monkeypatch.setattr(first, "join", joining_push) + final = threading.Thread(target=final_anchor, daemon=True) + final.start() + try: + assert joining.wait(15.0), "the final anchor did not join the blocked push" + live = (tmp_path / "anchors.jsonl").read_bytes() + assert len(live.splitlines()) == 2 + assert calls[0] != tmp_path / "anchors.jsonl" + assert calls[0].name.endswith("push-snapshot.jsonl") + assert len(calls[0].read_bytes().splitlines()) == 1 + finally: + release.set() + final.join(timeout=15.0) + assert not final.is_alive() and not errors + live = (tmp_path / "anchors.jsonl").read_bytes() + assert len(copies) == 2 and copies[-1] == live + assert all(copy.endswith(b"\n") and live.startswith(copy) for copy in copies) + assert len(set(calls)) == 2 + + +def test_push_times_use_the_ordered_clock_after_a_wall_clock_step_back( + tmp_path, monkeypatch +): + """The anchor, its cadence and its push never precede that anchor's time.""" + base = time.time() + 1000.0 + readings = iter([base, base - 0.0001]) + monkeypatch.setattr(cr, "_clock_last", 0.0) + monkeypatch.setattr(cr.time, "time", lambda: next(readings, base - 0.0001)) + + class Push: + returncode = 0 + + def __init__(self, *args, **kwargs): + pass + + def communicate(self, timeout): + return "pushed", "" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + run = bare_run(tmp_path, ["push-anchors"]) + run.anchor(wait=True) + anchor = json.loads((tmp_path / "anchors.jsonl").read_text()) + push = json.loads((tmp_path / "anchor-pushes.jsonl").read_text()) + assert anchor["time"] <= run._anchored <= push["started"] <= push["ended"] + assert push["anchor_time"] == anchor["time"] def test_a_push_timeout_kills_its_children_and_grandchildren(tmp_path, monkeypatch): @@ -273,3 +378,150 @@ def push(): assert push["started"] <= push["ended"] for level in (1, 2): assert int((tmp_path / f"pid-{level}").read_text()) > 0 + + +def test_finish_abandons_pipes_held_by_a_descendant_outside_the_push_group( + tmp_path, monkeypatch +): + """An escaped grandchild cannot hold finish open after the kill's grace.""" + script = tmp_path / "escaped-push.py" + script.write_text( + "import os, subprocess, sys, threading\n" + "from pathlib import Path\n" + "folder = Path(sys.argv[1])\n" + "if len(sys.argv) > 2 and sys.argv[2] == 'escaped':\n" + " (folder / 'escaped-pid').write_text(str(os.getpid()))\n" + "else:\n" + " subprocess.Popen(\n" + " [sys.executable, __file__, str(folder), 'escaped'],\n" + " start_new_session=True)\n" + "threading.Event().wait()\n" + ) + run = bare_finish_run(tmp_path, [sys.executable, str(script), str(tmp_path)]) + real_popen = subprocess.Popen + ready, done = threading.Event(), threading.Event() + children, errors = [], [] + + def popen(*args, **kwargs): + child = real_popen(*args, **kwargs) + children.append(child) + communicate = child.communicate + + def ready_communicate(timeout=None): + if not ready.is_set(): + deadline = time.monotonic() + 15.0 + pid = tmp_path / "escaped-pid" + while not pid.exists() or not pid.read_text().strip(): + assert time.monotonic() < deadline, "the grandchild did not start" + done.wait(0.01) + ready.set() + return communicate(timeout=timeout) + + child.communicate = ready_communicate + return child + + def finish(): + try: + run.finish() + except BaseException as error: + errors.append(error) + finally: + done.set() + + monkeypatch.setattr(cr.subprocess, "Popen", popen) + monkeypatch.setattr(cr.signal, "signal", lambda *args: None) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 0.05) + monkeypatch.setattr(cr, "KILL_GRACE", 0.05) + monkeypatch.setattr(cr, "FINAL_PUSH_TRIES", 1) + thread = threading.Thread(target=finish, daemon=True) + thread.start() + try: + assert ready.wait(15.0), "the push never launched its grandchild" + started = time.monotonic() + finished = done.wait(10.0) + elapsed = time.monotonic() - started + finally: + pid = tmp_path / "escaped-pid" + if pid.exists() and pid.read_text().strip(): + try: + os.kill(int(pid.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass + for child in children: + if child.poll() is None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + thread.join(timeout=15.0) + assert finished and elapsed < 10.0, "finish waited for the escaped child's pipe" + assert not errors and not thread.is_alive() + rows = [ + json.loads(line) + for line in (tmp_path / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert len(rows) == 1 and rows[0]["outcome"] == "timeout" + assert "pipes" in rows[0]["output"] and "held open after the kill" in rows[0]["output"] + + +def test_finish_records_a_stuck_push_once_even_if_its_thread_later_ends( + tmp_path, monkeypatch +): + """A bounded join retires the worker's right to append a second outcome.""" + started, release = threading.Event(), threading.Event() + calls, waits, copies = [], [], {} + + class Push: + returncode = 0 + + def __init__(self, command, **kwargs): + calls.append(command) + self.first = len(calls) == 1 + self.source = Path(command[-1]) + self.returncode = 1 if self.first else 0 + + def communicate(self, timeout): + if self.first: + started.set() + assert release.wait(15.0), "the abandoned worker was never released" + copies[self.source] = self.source.read_bytes() + return "pushed", "" + + monkeypatch.setattr(cr.subprocess, "Popen", Push) + monkeypatch.setattr(cr.signal, "signal", lambda *args: None) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 0.05) + monkeypatch.setattr(cr, "KILL_GRACE", 0.05) + run = bare_finish_run(tmp_path, ["push-anchors"]) + run.anchor() + assert started.wait(15.0), "the periodic push did not start" + worker, join = run._push, run._push.join + + def short_join(timeout=None): + waits.append(timeout) + return join(timeout=0.02) + + monkeypatch.setattr(worker, "join", short_join) + try: + began = time.monotonic() + run.finish() + assert time.monotonic() - began < 10.0 + path = tmp_path / "anchor-pushes.jsonl" + before = path.read_bytes() + rows = [json.loads(line) for line in before.splitlines()] + assert waits == [pytest.approx(cr.ANCHOR_PUSH_TIMEOUT + cr.KILL_GRACE + 30)] + assert len(rows) == 2 + assert rows[0]["outcome"] == "error" and rows[0]["returncode"] is None + assert "push thread did not end" in rows[0]["output"] + assert rows[1]["outcome"] == "success" + next_anchor = run._next_anchor + finally: + release.set() + join(timeout=15.0) + assert not worker.is_alive() + assert path.read_bytes() == before + assert len({row["anchor"] for row in rows}) == len(rows) + assert run._last_push_outcome == "success" and run._next_anchor == next_anchor + live = (tmp_path / "anchors.jsonl").read_bytes() + assert len(copies[Path(calls[0][-1])].splitlines()) == 1 + assert len(copies[Path(calls[1][-1])].splitlines()) == 2 + assert all(copy.endswith(b"\n") and live.startswith(copy) for copy in copies.values()) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 1be358f9..3f3b2b71 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -16,6 +16,7 @@ import json import math import os +import signal import subprocess import sys import threading @@ -360,6 +361,10 @@ def fast(monkeypatch): monkeypatch.setattr(cr, "SAMPLER_SILENCE", 1.0) monkeypatch.setattr(cr, "READ_WAIT", 0.01) monkeypatch.setattr(cr, "ANCHOR_SECONDS", 0.3) + handlers = {s: signal.getsignal(s) for s in (signal.SIGINT, signal.SIGTERM)} + yield + for signum, handler in handlers.items(): + signal.signal(signum, handler) def test_the_host_log_samples_and_closes(tmp_path, fast): @@ -2762,6 +2767,151 @@ def broken_watch(anchor=True): assert run_problems(w) == [] +def final_push_world(world, monkeypatch, fail_all=False, models=("gpt-6-luna",)): + """A run with no periodic pushes, whose first final push fails.""" + w = world(models=models) + w.run.no_output.update(w.draw_order) + monkeypatch.setattr(w.run, "anchor_due", lambda: False) + monkeypatch.setattr(cr, "ANCHOR_RETRY", 0.013) + script = w.tmp / "retrying-anchor.py" + script.write_text( + "import shutil, sys\n" + "from pathlib import Path\n" + "folder = Path(sys.argv[1])\n" + "counter = folder / 'push-count'\n" + "n = int(counter.read_text()) + 1 if counter.exists() else 1\n" + "counter.write_text(str(n))\n" + "shutil.copyfile(sys.argv[3], folder / f'pushed-{n}.jsonl')\n" + "raise SystemExit(1 if n == 1 or sys.argv[2] == '1' else 0)\n" + ) + w.design["anchor_command"] = [ + sys.executable, + str(script), + str(w.tmp), + str(int(fail_all)), + ] + return w + + +@pytest.mark.parametrize("fail_all", [False, True], ids=["then-success", "all-fail"]) +def test_the_final_push_is_retried_until_success_or_three_tries( + world, monkeypatch, fail_all +): + """Final retries stop at the first success, or after three failures.""" + w = final_push_world(world, monkeypatch, fail_all=fail_all) + w.execute() + anchors = [ + json.loads(x) for x in (w.runs() / "anchors.jsonl").read_text().splitlines() + ] + pushes = [ + json.loads(x) + for x in (w.runs() / "anchor-pushes.jsonl").read_text().splitlines() + ] + expected = 3 if fail_all else 2 + assert cr.FINAL_PUSH_TRIES == 3 + assert len(pushes) == len(anchors) == expected + assert [row["outcome"] for row in pushes] == ( + ["failed"] * 3 if fail_all else ["failed", "success"] + ) + final = (w.runs() / "anchors.jsonl").read_bytes() + assert (w.tmp / f"pushed-{expected}.jsonl").read_bytes() == final + assert pushes[-1]["anchor"] == cr.line_sha(final.decode().splitlines()[-1]) + assert pushes[-1]["anchor_time"] == anchors[-1]["time"] + assert all(a["models"] == anchors[0]["models"] for a in anchors) + assert all(a["host-log.jsonl"] == anchors[0]["host-log.jsonl"] for a in anchors) + assert all(a["time"] < b["time"] for a, b in zip(anchors, anchors[1:])) + + +def test_a_first_signal_during_final_push_retries_is_logged_and_anchored( + world, monkeypatch +): + """A first signal during the retry wait is covered by the last push.""" + w = final_push_world(world, monkeypatch, models=("gpt-6-luna", "gpt-6-sol")) + sleep, delivered = time.sleep, [] + + def during_retry(seconds): + if seconds == cr.ANCHOR_RETRY and not delivered: + delivered.append(True) + w.run.interrupt(signal.SIGTERM, None) + sleep(seconds) + + monkeypatch.setattr(time, "sleep", during_retry) + w.execute() + assert delivered and w.run._interrupted + anchors = [ + json.loads(x) for x in (w.runs() / "anchors.jsonl").read_text().splitlines() + ] + pushes = [ + json.loads(x) + for x in (w.runs() / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert [row["outcome"] for row in pushes] == ["failed", "success", "success"] + for model in w.models: + assert [row["cause"] for row in w.events(model, "stop")] == ["interrupt"] + assert anchors[-1]["models"][model]["run-log.jsonl"] == ( + w.run.lanes[model].heads()["run-log.jsonl"] + ) + assert anchors[-2]["models"][model]["run-log.jsonl"]["lines"] == ( + len(w.lines(model)) - 1 + ) + assert (w.tmp / "pushed-3.jsonl").read_bytes() == ( + w.runs() / "anchors.jsonl" + ).read_bytes() + + +def test_a_second_signal_during_final_push_retries_exits_at_once(world, monkeypatch): + """The retry wait leaves the second signal's immediate exit intact.""" + w = final_push_world(world, monkeypatch) + sleep, delivered = time.sleep, [] + + def exit_now(code): + raise SystemExit(code) + + def during_retry(seconds): + if seconds == cr.ANCHOR_RETRY and not delivered: + delivered.append(True) + w.run.interrupt(signal.SIGINT, None) + w.run.interrupt(signal.SIGTERM, None) + sleep(seconds) + + monkeypatch.setattr(os, "_exit", exit_now) + monkeypatch.setattr(time, "sleep", during_retry) + with pytest.raises(SystemExit) as stopped: + w.execute() + assert stopped.value.code == 130 and delivered + assert len((w.runs() / "anchor-pushes.jsonl").read_text().splitlines()) == 1 + assert all(not w.events(model, "stop") for model in w.models) + + +def test_a_signal_at_the_final_ignore_hook_is_logged_and_anchored(world, monkeypatch): + """A signal between the last ordinary check and SIG_IGN is not lost.""" + w = world(models=("gpt-6-luna", "gpt-6-sol")) + w.run.no_output.update(w.draw_order) + monkeypatch.setattr(w.run, "anchor_due", lambda: False) + ignore = w.run._ignore_signals + ignored, delivered = [], [] + monkeypatch.setattr( + signal, "signal", lambda signum, handler: ignored.append((signum, handler)) + ) + + def at_end(): + if not delivered: + delivered.append(True) + w.run.interrupt(signal.SIGTERM, None) + ignore() + + monkeypatch.setattr(w.run, "_ignore_signals", at_end) + w.execute() + assert delivered and w.run._interrupted + assert (signal.SIGINT, signal.SIG_IGN) in ignored + assert (signal.SIGTERM, signal.SIG_IGN) in ignored + for model in w.models: + stops = w.events(model, "stop") + assert len(stops) == 1 and stops[0]["cause"] == "interrupt" + assert stops[0]["reason"] == f"operator stop (signal {signal.SIGTERM})" + assert anchor_problems(w) == [] + + def test_a_final_push_log_failure_is_stopped_and_locally_anchored(world, monkeypatch): model = "gpt-6-sol" w = world(models=(model,)) From 019f8424449e6385567b5a753e7726fe323be7e1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 13:31:42 -0700 Subject: [PATCH 11/48] Confirmatory runner: round-4 fixes Answers review runner-r4 finding 1 and tests the remaining clock and retry spacing requirements: - Hold an exclusive, non-blocking lock in the anchor clone's .git directory through the push. Refuse busy helpers and snapshots that do not extend the clone's anchors byte for byte; allow equality and never force-push. Leave a stale index.lock to fail normally. - Keep worker retirement and the next push, but say the retired command may still be running. A helper holding the lock makes later pushes fail as busy; final anchors left local are a deviation. - Exercise older helpers after newer pushes, shared-clone retirement, lock contention, interior edits, equality and stale git locks against a temporary clone and local bare remote. - Test anchor_due across a backward wall-clock step and measure each final retry from the preceding try's end, with a controlled clock. - Clarify that final retries wait a minute after a failed try ends. First and second signal handling stays active only until cleanup ignores signals before its last check; signals during re-anchoring after that check are ignored. State that write/fsync under _pushing can delay supervision and retirement beyond the thread join bound. The report's record fields and outcome names are unchanged. Retired-worker output now identifies the surviving-command risk; helper refusals are failed pushes. The clone's persistent lock file is auxiliary, outside run records. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 42 +++++-- scripts/confirmatory_run.py | 43 +++++--- tests/test_confirmatory_anchor.py | 177 ++++++++++++++++++++++++++++++ tests/test_confirmatory_run.py | 54 +++++++++ 4 files changed, 293 insertions(+), 23 deletions(-) diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py index 68b4f37f..838a4bba 100644 --- a/scripts/confirmatory_anchor.py +++ b/scripts/confirmatory_anchor.py @@ -10,39 +10,63 @@ of its own, commits it and pushes the branch to its remote. The remote's record of each push is the anchor's time from outside the run: a log edited after an anchor was pushed no longer matches that anchor's head. + +An exclusive non-blocking lock in the clone's `.git` directory covers every +read, copy, commit and push. A busy clone fails without changing its records. +Under the lock, the clone's current file must be a byte prefix of the snapshot: +a delayed older command cannot replace newer evidence. Git's own index.lock +is left alone; a git failure is a failed push. """ from __future__ import annotations import argparse +import fcntl import hashlib -import shutil import subprocess import sys from pathlib import Path def git(repo: Path, *args: str) -> str: + """Run git in the anchor clone, retaining failures for the caller.""" return subprocess.run( ["git", "-C", str(repo), *args], check=True, capture_output=True, text=True ).stdout def main(argv=None) -> int: + """Push a whole snapshot only while exclusively extending the clone's file.""" parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--repo", type=Path, required=True, help="the anchors clone") parser.add_argument("anchors", type=Path, help="a snapshot of the run's anchors.jsonl") args = parser.parse_args(argv) repo = args.repo.expanduser() target = repo / "anchors.jsonl" - shutil.copyfile(args.anchors, target) - lines = target.read_text().splitlines() - git(repo, "add", "anchors.jsonl") - if git(repo, "status", "--porcelain", "--", "anchors.jsonl"): - head = hashlib.sha256(lines[-1].encode()).hexdigest() if lines else "none" - git(repo, "commit", "-q", "-m", f"anchor {len(lines)}: {head}") - git(repo, "push", "-q", "origin", "HEAD") - print(f"pushed {len(lines)} anchors") + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print("anchor clone busy: another helper holds its lock", file=sys.stderr) + return 1 + snapshot = args.anchors.read_bytes() + current = target.read_bytes() if target.exists() else b"" + if not snapshot.startswith(current): + print( + "refusing anchor snapshot: the clone's anchors.jsonl is not a byte " + "prefix of the snapshot", + file=sys.stderr, + ) + return 1 + if snapshot != current or not target.exists(): + target.write_bytes(snapshot) + lines = snapshot.decode().splitlines() + git(repo, "add", "anchors.jsonl") + if git(repo, "status", "--porcelain", "--", "anchors.jsonl"): + head = hashlib.sha256(lines[-1].encode()).hexdigest() if lines else "none" + git(repo, "commit", "-q", "-m", f"anchor {len(lines)}: {head}") + git(repo, "push", "-q", "origin", "HEAD") + print(f"pushed {len(lines)} anchors") return 0 diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index eff73e76..1ee029c4 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -74,8 +74,13 @@ background. Each ``anchors..push-snapshot.jsonl`` is a copy, not a record, and is never replaced by a later push's snapshot. Each push's result goes to ``anchor-pushes.jsonl``. ``confirmatory_anchor.py`` is that - command: it commits the file to a clone and pushes it, so the remote holds - each anchor's time from outside the run. + command: under an exclusive, non-blocking clone lock, it accepts only a + snapshot extending the clone's anchors, commits it and pushes it, so the + remote holds each anchor's time from outside the run. Retiring a worker + does not ensure its command has ended. If an old helper still holds the + lock, later pushes fail as busy; a final segment left local is a deviation. + Final pushes get up to three tries per set of heads, waiting a minute + after each failed try ends before starting the next. The records' format is the confirmatory report's (its module docstring). The report recomputes every class, action, order and admission decision from the @@ -90,7 +95,9 @@ engine workers per model. An interrupt (SIGINT or SIGTERM) only notes the signal; the main loop then stops every model's lanes with cause ``interrupt`` and lets the conversations under way end. A second interrupt -exits at once. +exits at once until cleanup ignores signals before its final check. Signals +during any re-anchoring after that check are ignored, including a second +interrupt. """ from __future__ import annotations @@ -1689,11 +1696,11 @@ def anchor_due(self) -> bool: def anchor(self, wait: bool = False, push: bool = True) -> None: """Every log's chain head to `anchors.jsonl`, itself a chain, then the - design's anchor command on a synced snapshot, in the background (one at a - time; an anchor whose push would wait is written but not pushed until - the next). A snapshot is a copy, not a record, and its path is unique - even if a push thread has to be abandoned. Each push's result goes to - `anchor-pushes.jsonl`.""" + design's anchor command on a synced snapshot, in the background (one + current worker at a time; an anchor whose push would wait is written + but not pushed until the next). A snapshot is a copy, not a record, + and its path is unique even if a push thread has to be abandoned. + Each push's result goes to `anchor-pushes.jsonl`.""" runs = Path(self.design["runs"]) line = { "time": clock(), @@ -1742,7 +1749,9 @@ def anchor(self, wait: bool = False, push: bool = True) -> None: self._join_push() def _join_push(self) -> None: - """Bound the push thread's join and claim an error if it cannot end.""" + """Bound the thread join and retire its outcome if it cannot end. + Its command may survive; the helper's lock and prefix check protect + the clone. Waiting for _pushing still assumes a responsive filesystem.""" self._push.join(timeout=ANCHOR_PUSH_TIMEOUT + KILL_GRACE + 30) if self._push.is_alive(): with self._pushing: @@ -1754,13 +1763,16 @@ def _join_push(self) -> None: "ended": clock(), "outcome": "error", "returncode": None, - "output": "the anchor push thread did not end after its time limit", + "output": "the anchor push thread did not end after its time limit; " + "its command may still be running", } ) self._check_push_error() def _record_push(self, line: dict) -> None: - """Claim one outcome under the push lock; a late worker cannot repeat it.""" + """Claim one outcome under the push lock; a late worker cannot repeat it. + The lock covers writing and fsync, so a wedged filesystem can delay + supervision and retirement beyond the thread join's bound.""" with self._pushing: pending = self._push_pending if pending is None or pending["anchor"] != line["anchor"]: @@ -2016,8 +2028,10 @@ def finish(self) -> None: ledger. Its final late costs, crossings and signals precede the last anchor. An unresolved request makes future spend unknowable: record it explicitly, with a runner-error stop, rather than claim it settled. - Retry the final push up to three times per set of heads, then ignore - signals and check once more for a first signal needing coverage.""" + Retry the final push up to three times per set of heads, waiting + ANCHOR_RETRY after each failed try ends. Then ignore signals and check + once more for a first signal needing coverage. Signals during the + resulting re-anchoring are ignored, including a second interrupt.""" errors = [] def attempt(work): @@ -2117,7 +2131,8 @@ def _ignore_signals(self) -> None: def interrupt(self, signum, frame) -> None: """The signal handler. It only notes the signal; the main loop stops - every model's lanes. A second signal exits at once.""" + every model's lanes. A second signal exits at once while this handler + is installed; cleanup ignores signals before its final check.""" self.interrupts += 1 if self.interrupts > 1: os._exit(130) diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index f6944c42..5fa338ff 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -1,10 +1,12 @@ """The anchor command: each call commits the run's anchors and pushes them.""" +import fcntl import hashlib import io import json import math import os +import select import signal import subprocess import sys @@ -59,6 +61,15 @@ def remote_file(remote: Path) -> str: return out.stdout +def anchor_command(repo: Path, anchors: Path): + """Run the real helper and retain its failure message and exit status.""" + return subprocess.run( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + capture_output=True, + text=True, + ) + + @pytest.mark.parametrize("name", ["anchors.jsonl", "anchors.push-snapshot.jsonl"]) def test_each_anchor_is_committed_and_pushed(clone, tmp_path, name): """A snapshot's local name never changes the remote record's name.""" @@ -78,12 +89,84 @@ def test_each_anchor_is_committed_and_pushed(clone, tmp_path, name): def test_an_unchanged_file_makes_no_commit(clone, tmp_path): + """An equal snapshot succeeds without changing the file or remote history.""" remote, repo = clone anchors = tmp_path / "anchors.jsonl" anchors.write_text('{"time": 1.0}\n') for _ in range(2): assert confirmatory_anchor.main(["--repo", str(repo), str(anchors)]) == 0 assert len(remote_log(remote)) == 1 + assert remote_file(remote) == anchors.read_text() + + +def test_an_older_snapshot_cannot_replace_a_newer_remote_file(clone, tmp_path): + """A delayed helper cannot commit or push a prefix after the newer snapshot.""" + remote, repo = clone + older, newer = tmp_path / "older.jsonl", tmp_path / "newer.jsonl" + older.write_bytes(b'{"time": 1.0}\n') + newer.write_bytes(older.read_bytes() + b'{"time": 2.0}\n') + assert anchor_command(repo, newer).returncode == 0 + history, index = remote_log(remote), (repo / ".git" / "index").read_bytes() + out = anchor_command(repo, older) + assert out.returncode != 0 and "not a byte prefix" in out.stderr + assert (repo / "anchors.jsonl").read_bytes() == newer.read_bytes() + assert (repo / ".git" / "index").read_bytes() == index + assert remote_file(remote).encode() == newer.read_bytes() + assert remote_log(remote) == history + + +def test_a_busy_clone_is_refused_without_changing_its_records(clone, tmp_path): + """A held flock prevents target, index, commit and remote writes.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + history, index = remote_log(remote), (repo / ".git" / "index").read_bytes() + anchors.write_bytes(first + b'{"time": 2.0}\n') + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + out = anchor_command(repo, anchors) + assert out.returncode != 0 and "busy" in out.stderr + assert (repo / "anchors.jsonl").read_bytes() == first + assert (repo / ".git" / "index").read_bytes() == index + assert remote_file(remote).encode() == first + assert remote_log(remote) == history + + +def test_a_changed_middle_line_is_refused_without_writing(clone, tmp_path): + """A longer snapshot still fails when it rewrites an existing line.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n{"time": 2.0}\n{"time": 3.0}\n' + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + history, index = remote_log(remote), (repo / ".git" / "index").read_bytes() + anchors.write_bytes(first.replace(b"2.0", b"9.0") + b'{"time": 4.0}\n') + out = anchor_command(repo, anchors) + assert out.returncode != 0 and "not a byte prefix" in out.stderr + assert (repo / "anchors.jsonl").read_bytes() == first + assert (repo / ".git" / "index").read_bytes() == index + assert remote_file(remote).encode() == first + assert remote_log(remote) == history + + +def test_a_stale_git_index_lock_is_left_alone(clone, tmp_path): + """A failed git command never removes another command's index lock.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + history = remote_log(remote) + index_lock = repo / ".git" / "index.lock" + index_lock.write_bytes(b"stale lock kept for its owner\n") + anchors.write_bytes(first + b'{"time": 2.0}\n') + out = anchor_command(repo, anchors) + assert out.returncode != 0 and "CalledProcessError" in out.stderr + assert index_lock.read_bytes() == b"stale lock kept for its owner\n" + assert remote_file(remote).encode() == first + assert remote_log(remote) == history def test_a_failed_push_exits_nonzero(clone, tmp_path): @@ -512,6 +595,7 @@ def short_join(timeout=None): assert len(rows) == 2 assert rows[0]["outcome"] == "error" and rows[0]["returncode"] is None assert "push thread did not end" in rows[0]["output"] + assert "command may still be running" in rows[0]["output"] assert rows[1]["outcome"] == "success" next_anchor = run._next_anchor finally: @@ -525,3 +609,96 @@ def short_join(timeout=None): assert len(copies[Path(calls[0][-1])].splitlines()) == 1 assert len(copies[Path(calls[1][-1])].splitlines()) == 2 assert all(copy.endswith(b"\n") and live.startswith(copy) for copy in copies.values()) + + +def test_a_retired_command_cannot_shrink_the_remote_after_the_final_push( + clone, tmp_path, monkeypatch +): + """A live old command released after finish cannot replace newer evidence.""" + remote, repo = clone + folder = tmp_path / "run" + folder.mkdir() + delayed = tmp_path / "delayed-helper.py" + delayed.write_text( + "import os, sys\n" + "ready, release = map(int, sys.argv[1:3])\n" + "os.write(ready, b'1')\n" + "os.close(ready)\n" + "os.read(release, 1)\n" + "os.close(release)\n" + "os.execv(sys.executable, [sys.executable, *sys.argv[3:]])\n" + ) + ready_read, ready_write = os.pipe() + release_read, release_write = os.pipe() + real_popen, children, waits = subprocess.Popen, [], [] + run = bare_finish_run( + folder, [sys.executable, str(SCRIPT), "--repo", str(repo)] + ) + + def popen(command, **kwargs): + """Delay only the first real helper, before it can acquire the clone lock.""" + if len(command) > 1 and command[1] == str(SCRIPT): + if not children: + command = [ + sys.executable, + str(delayed), + str(ready_write), + str(release_read), + *command[1:], + ] + kwargs["pass_fds"] = (ready_write, release_read) + child = real_popen(command, **kwargs) + children.append(child) + return child + return real_popen(command, **kwargs) + + def short_join(timeout=None): + """Exercise retirement without waiting for the production join bound.""" + waits.append(timeout) + return join(timeout=0.02) + + monkeypatch.setattr(cr.subprocess, "Popen", popen) + monkeypatch.setattr(cr.signal, "signal", lambda *args: None) + monkeypatch.setattr(cr, "ANCHOR_PUSH_TIMEOUT", 30.0) + worker = join = None + try: + run.anchor() + worker, join = run._push, run._push.join + assert select.select([ready_read], [], [], 15.0)[0], ( + "the old command did not start" + ) + assert os.read(ready_read, 1) == b"1" + monkeypatch.setattr(worker, "join", short_join) + run.finish() + assert children[0].poll() is None + newer = (folder / "anchors.jsonl").read_bytes() + assert len(newer.splitlines()) == 2 + assert remote_file(remote).encode() == newer + history = remote_log(remote) + path = folder / "anchor-pushes.jsonl" + before = path.read_bytes() + rows = [json.loads(line) for line in before.splitlines()] + assert waits == [pytest.approx(cr.ANCHOR_PUSH_TIMEOUT + cr.KILL_GRACE + 30)] + assert [row["outcome"] for row in rows] == ["error", "success"] + assert rows[0]["returncode"] is None + assert "command may still be running" in rows[0]["output"] + finally: + try: + os.write(release_write, b"1") + except BrokenPipeError: + pass + if join is not None: + join(timeout=45.0) + for child in children: + if child.poll() is None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + for fd in (ready_read, ready_write, release_read, release_write): + os.close(fd) + assert not worker.is_alive() and len(children) == 2 + assert children[0].returncode == 1 + assert path.read_bytes() == before + assert remote_file(remote).encode() == newer + assert remote_log(remote) == history diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 3f3b2b71..ed103b04 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -545,6 +545,20 @@ def test_the_clock_never_goes_back(monkeypatch): ] +def test_a_backward_wall_clock_does_not_delay_a_due_anchor(monkeypatch): + """Once the ordered clock reaches the due time, a wall-clock step cannot + make the anchor not due again.""" + run = cr.Run.__new__(cr.Run) + run._pushing = threading.Lock() + run._push_error = None + base = time.time() + 1000.0 + run._next_anchor = base + monkeypatch.setattr(cr, "_clock_last", base - 1.0) + readings = iter([base - 1.0, base + 1.0, base - 5.0]) + monkeypatch.setattr(time, "time", lambda: next(readings)) + assert [run.anchor_due() for _ in range(3)] == [False, True, True] + + def test_a_wall_clock_that_steps_back_keeps_the_run_procedure(world, monkeypatch): """Every reading of the wall clock up to 0.2 ms early (steps back of up to 0.1 ms seen on this host): the run's times still order its attempts, @@ -2822,6 +2836,46 @@ def test_the_final_push_is_retried_until_success_or_three_tries( assert all(a["time"] < b["time"] for a, b in zip(anchors, anchors[1:])) +def test_each_final_push_retry_waits_from_the_previous_try_s_end(world, monkeypatch): + """Each final retry starts at least ANCHOR_RETRY after the prior try ends, + even when the failed command itself takes time.""" + from types import SimpleNamespace + + w = world(models=("gpt-6-luna",)) + now = [1000.0] + durations = iter([7.0, 11.0, 5.0]) + monkeypatch.setattr(cr, "clock", lambda: now[0]) + + def sleep(seconds): + """Advance the controlled clock by the requested wait.""" + now[0] += seconds + + def failed_command(*args, **kwargs): + """A failed anchor command with a controlled, nonzero duration.""" + + def communicate(timeout): + """Finish the command after its planted duration.""" + now[0] += next(durations) + return "", "controlled push failure" + + return SimpleNamespace(returncode=1, communicate=communicate) + + monkeypatch.setattr(time, "sleep", sleep) + monkeypatch.setattr(subprocess, "Popen", failed_command) + w.run.finish() + pushes = [ + json.loads(x) + for x in (w.runs() / "anchor-pushes.jsonl").read_text().splitlines() + ] + assert len(pushes) == cr.FINAL_PUSH_TRIES == 3 + assert [p["outcome"] for p in pushes] == ["failed"] * 3 + assert [p["ended"] - p["started"] for p in pushes] == [7.0, 11.0, 5.0] + assert all( + b["started"] >= a["ended"] + cr.ANCHOR_RETRY + for a, b in zip(pushes, pushes[1:]) + ) + + def test_a_first_signal_during_final_push_retries_is_logged_and_anchored( world, monkeypatch ): From 9b30a778a2e85d5b6bce983ef2b07342b97df840 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 23:21:24 -0700 Subject: [PATCH 12/48] Confirmatory runner: round-5 fixes Answers review runner-r5 finding 1 and fills the retry and conversation coverage gaps: - Pass the clone's flock descriptor to every Git command. A surviving Git writer, hook or remote helper retains the lock after helper-only death; later helpers still refuse busy clones and non-extending snapshots. Disable fsmonitor startup, whose daemon closes inherited descriptors. Keep ordinary, non-force pushes and leave Git's own index.lock alone. - Kill the real helper after Git has entered pre-commit, pre-push and local pre-receive hooks. Check busy refusals, inherited hook descriptors and unchanged clone file, index, HEAD and remote, then newer pushes and every remote commit's byte-prefix extension. Test that configured fsmonitor is never started. - Pin the registered 900-second anchor interval and 60-second retry wait independently of production constants, citing draft line 218. - Compare fake OpenAI Responses and Gemini JSON conversations against the unchanged treatment harness, including complete requests and calculator inputs. Exercise both through the runner's lanes. Use the existing Gemini 3.1 JSON profile; both confirmatory Geminis retain their tool contracts. - Describe surviving Git lock holders and advisory/custom-command limits. The report's record fields and outcome names are unchanged. Busy refusal output now identifies a helper or Git process as the holder. The existing clone lock remains auxiliary, outside run records. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 45 ++++++-- scripts/confirmatory_run.py | 5 +- tests/test_confirmatory_anchor.py | 152 +++++++++++++++++++++++++ tests/test_confirmatory_run.py | 183 ++++++++++++++++++++++++++++-- 4 files changed, 360 insertions(+), 25 deletions(-) diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py index 838a4bba..2dd7f2b3 100644 --- a/scripts/confirmatory_anchor.py +++ b/scripts/confirmatory_anchor.py @@ -12,10 +12,17 @@ anchor was pushed no longer matches that anchor's head. An exclusive non-blocking lock in the clone's `.git` directory covers every -read, copy, commit and push. A busy clone fails without changing its records. -Under the lock, the clone's current file must be a byte prefix of the snapshot: -a delayed older command cannot replace newer evidence. Git's own index.lock -is left alone; a git failure is a failed push. +read, copy, commit and push. Every Git command inherits its descriptor, so a +surviving command keeps the clone busy even if this helper alone is killed. +Git's hooks and remote helpers inherit it too; automatic fsmonitor startup, +which closes inherited descriptors, is disabled. Configured commands must +preserve the descriptor in any background clone writer. This advisory lock +does not cover independent writers or replacement of the lock file. +A busy clone fails without changing its records. Under the lock, the clone's +current file must be a byte prefix of the snapshot (equality is allowed): a +delayed older command cannot replace newer evidence. This checks bytes, not +JSON or line boundaries; the runner supplies complete, synced lines. Git's +own index.lock is left alone; a git failure is a failed push. Never force-push. """ from __future__ import annotations @@ -28,10 +35,14 @@ from pathlib import Path -def git(repo: Path, *args: str) -> str: - """Run git in the anchor clone, retaining failures for the caller.""" +def git(repo: Path, lock_fd: int, *args: str) -> str: + """Run Git with the clone lock inherited, retaining failures for the caller.""" return subprocess.run( - ["git", "-C", str(repo), *args], check=True, capture_output=True, text=True + ["git", "-c", "core.fsmonitor=false", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + pass_fds=(lock_fd,), ).stdout @@ -47,7 +58,10 @@ def main(argv=None) -> int: try: fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: - print("anchor clone busy: another helper holds its lock", file=sys.stderr) + print( + "anchor clone busy: another helper or Git process holds its lock", + file=sys.stderr, + ) return 1 snapshot = args.anchors.read_bytes() current = target.read_bytes() if target.exists() else b"" @@ -61,11 +75,18 @@ def main(argv=None) -> int: if snapshot != current or not target.exists(): target.write_bytes(snapshot) lines = snapshot.decode().splitlines() - git(repo, "add", "anchors.jsonl") - if git(repo, "status", "--porcelain", "--", "anchors.jsonl"): + git(repo, lock.fileno(), "add", "anchors.jsonl") + if git(repo, lock.fileno(), "status", "--porcelain", "--", "anchors.jsonl"): head = hashlib.sha256(lines[-1].encode()).hexdigest() if lines else "none" - git(repo, "commit", "-q", "-m", f"anchor {len(lines)}: {head}") - git(repo, "push", "-q", "origin", "HEAD") + git( + repo, + lock.fileno(), + "commit", + "-q", + "-m", + f"anchor {len(lines)}: {head}", + ) + git(repo, lock.fileno(), "push", "-q", "origin", "HEAD") print(f"pushed {len(lines)} anchors") return 0 diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 1ee029c4..d4dd254b 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -77,8 +77,9 @@ command: under an exclusive, non-blocking clone lock, it accepts only a snapshot extending the clone's anchors, commits it and pushes it, so the remote holds each anchor's time from outside the run. Retiring a worker - does not ensure its command has ended. If an old helper still holds the - lock, later pushes fail as busy; a final segment left local is a deviation. + does not ensure its command has ended. If an old helper or its Git child + still holds the lock, later pushes fail as busy; a final segment left local + is a deviation. Final pushes get up to three tries per set of heads, waiting a minute after each failed try ends before starting the next. diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index 5fa338ff..153a2f1d 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -134,6 +134,135 @@ def test_a_busy_clone_is_refused_without_changing_its_records(clone, tmp_path): assert remote_log(remote) == history +@pytest.mark.parametrize("hook_name", ["pre-commit", "pre-push", "pre-receive"]) +def test_killing_only_the_helper_keeps_its_git_writer_locked( + clone, tmp_path, hook_name +): + """A Git child already using the clone retains the lock after helper death.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + older = first + b'{"time": 2.0}\n' + snapshots = [older + b'{"time": 3.0}\n', older + b'{"time": 3.0}\n{"time": 4.0}\n'] + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + ready, release = tmp_path / "ready", tmp_path / "release" + for path in (ready, release): + os.mkfifo(path) + hook_lock = tmp_path / "hook-lock.json" + hooks = remote / "hooks" if hook_name == "pre-receive" else repo / ".git" / "hooks" + hook = hooks / hook_name + hook.write_text( + f"#!{sys.executable}\n" + "import json, os\n" + "from pathlib import Path\n" + "if os.environ.get('ANCHOR_TEST_BLOCK') == '1':\n" + f" lock = Path({str(repo / '.git' / 'confirmatory-anchor.lock')!r}).stat()\n" + " inherited = []\n" + " for fd in range(3, 256):\n" + " try:\n" + " info = os.fstat(fd)\n" + " except OSError:\n" + " continue\n" + " if (info.st_dev, info.st_ino) == (lock.st_dev, lock.st_ino):\n" + " inherited.append(os.get_inheritable(fd))\n" + f" Path({str(hook_lock)!r}).write_text(json.dumps(inherited))\n" + f" with open({str(ready)!r}, 'wb', buffering=0) as pipe:\n" + " pipe.write(b'1')\n" + f" with open({str(release)!r}, 'rb', buffering=0) as pipe:\n" + " pipe.read(1)\n" + ) + hook.chmod(0o755) + + def records(): + """Read the clone's file, index, HEAD and the remote without writing.""" + objects = [ + subprocess.run( + ["git", "-C", str(repo), "show", obj], check=True, capture_output=True + ).stdout + for obj in (":anchors.jsonl", "HEAD:anchors.jsonl") + ] + return ( + (repo / "anchors.jsonl").read_bytes(), + (repo / ".git" / "index").read_bytes(), + *objects, + remote_file(remote).encode(), + remote_log(remote), + ) + + ready_fd, release_fd = (os.open(path, os.O_RDWR) for path in (ready, release)) + child = None + try: + anchors.write_bytes(older) + child = subprocess.Popen( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + env={**os.environ, "ANCHOR_TEST_BLOCK": "1"}, + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert select.select([ready_fd], [], [], 15.0)[0], "Git did not reach its hook" + assert os.read(ready_fd, 1) == b"1" + before = records() + assert before[0] == before[2] == older + child.kill() + assert child.wait(timeout=15.0) == -signal.SIGKILL + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 1 and "busy" in out.stderr, ( + "a surviving Git writer lost the helper's clone lock: " + out.stderr + ) + assert records() == before + assert json.loads(hook_lock.read_text()) == [True] + os.write(release_fd, b"1") + child.communicate(timeout=15.0) + # Pipe closure need not mean every inherited lock holder has ended. + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + deadline = time.monotonic() + 15.0 + while True: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, "a Git child did not finish" + threading.Event().wait(0.01) + after = records() + assert after[0] == after[2] == after[3] == older + assert after[4] in (first, older) + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + state = records() + assert state[0] == state[2] == state[3] == state[4] == snapshot + history = subprocess.run( + ["git", "-C", str(remote), "rev-list", "--reverse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + previous = b"" + for commit in history: + content = subprocess.run( + ["git", "-C", str(remote), "show", f"{commit}:anchors.jsonl"], + check=True, + capture_output=True, + ).stdout + assert content.startswith(previous) + previous = content + assert previous == snapshots[-1] + finally: + if child is not None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.communicate(timeout=15.0) + for fd in (ready_fd, release_fd): + os.close(fd) + + def test_a_changed_middle_line_is_refused_without_writing(clone, tmp_path): """A longer snapshot still fails when it rewrites an existing line.""" remote, repo = clone @@ -169,6 +298,29 @@ def test_a_stale_git_index_lock_is_left_alone(clone, tmp_path): assert remote_log(remote) == history +def test_git_fsmonitor_is_disabled_even_when_the_clone_enables_it(clone, tmp_path): + """Git must not launch a monitor that can discard the inherited clone lock.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + anchors.write_bytes(b'{"time": 1.0}\n') + assert anchor_command(repo, anchors).returncode == 0 + invoked, monitor = tmp_path / "monitor-invoked", tmp_path / "fsmonitor" + monitor.write_text( + f"#!{sys.executable}\n" + "from pathlib import Path\n" + f"Path({str(invoked)!r}).touch()\n" + ) + monitor.chmod(0o755) + subprocess.run( + ["git", "-C", str(repo), "config", "core.fsmonitor", str(monitor)], check=True + ) + anchors.write_bytes(anchors.read_bytes() + b'{"time": 2.0}\n') + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + assert not invoked.exists() + assert remote_file(remote).encode() == anchors.read_bytes() + + def test_a_failed_push_exits_nonzero(clone, tmp_path): """The runner records the command's return code; a push that fails must not look like one that landed.""" diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index ed103b04..d4e3622f 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -11,6 +11,7 @@ from __future__ import annotations +import copy import hashlib import io import json @@ -559,6 +560,13 @@ def test_a_backward_wall_clock_does_not_delay_a_due_anchor(monkeypatch): assert [run.anchor_due() for _ in range(3)] == [False, True, True] +def test_the_anchor_interval_and_retry_wait_are_registered_durations(): + """paper/preregistration-draft.md:218: anchors every 15 minutes, and retries + a minute after the failed push ends.""" + assert cr.ANCHOR_SECONDS == 900.0 + assert cr.ANCHOR_RETRY == 60.0 + + def test_a_wall_clock_that_steps_back_keeps_the_run_procedure(world, monkeypatch): """Every reading of the wall clock up to 0.2 ms early (steps back of up to 0.1 ms seen on this host): the run's times still order its attempts, @@ -768,6 +776,7 @@ def __init__( pilot._STATED_ENGINE, "version", pilot._STATED_ENGINE["version"] ) self.models = tuple(models or cr.CHEAP + cr.FRONTIER) + budget_models = tuple(dict.fromkeys(cr.CHEAP + cr.FRONTIER + self.models)) self.design = { "preregistration": str(prereg), "frozen_sha256": frozen, @@ -778,8 +787,8 @@ def __init__( "draw_order": self.draw_order, "arms": {a: a for a in cr.ARMS}, "frontier_arms": {a: f"f-{a}" for a in cr.FRONTIER_ARMS}, - "caps": {m: (caps or {}).get(m, 10.0) for m in cr.CHEAP + cr.FRONTIER}, - "reserves": {m: reserve for m in cr.CHEAP + cr.FRONTIER}, + "caps": {m: (caps or {}).get(m, 10.0) for m in budget_models}, + "reserves": {m: reserve for m in budget_models}, "overall_cap": overall_cap, "spent_before": spent_before, "workers": 2, @@ -2239,28 +2248,180 @@ def stateless_model(**kwargs): return chat_response([chat_call("submit_outputs", {"outputs": outputs}, "s1")]) -def test_the_harness_conversation_runs_under_the_runner(world): - model = "claude-haiku-5.5" +def stateless_responses_model(**kwargs): + """A Responses tool-contract model: calculate, then submit the answers.""" + from types import SimpleNamespace + + if "previous_response_id" not in kwargs: + name, call_id = pilot.TOOL_NAME, "c1" + arguments = {"situation": json.dumps({"people": {}}), "variables": ["snap"]} + else: + name, call_id = "submit_outputs", "s1" + arguments = { + "outputs": {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + } + return SimpleNamespace( + id=f"resp_{call_id}", + output=[ + SimpleNamespace( + type="function_call", + id=f"fc_{call_id}", + call_id=call_id, + name=name, + arguments=json.dumps(arguments), + status="completed", + ) + ], + output_text=None, + status="completed", + incomplete_details=None, + usage=SimpleNamespace( + input_tokens=1000, + output_tokens=200, + total_tokens=1200, + input_tokens_details=SimpleNamespace(cached_tokens=0), + output_tokens_details=SimpleNamespace(reasoning_tokens=0), + ), + model=kwargs["model"], + ) + + +def stateless_json_model(**kwargs): + """A Gemini JSON-contract model: calculate, then answer in JSON text.""" + if not any(m.get("role") == "tool" for m in kwargs["messages"]): + response = stateless_model(**kwargs) + response.model = kwargs["model"] + return response + response = chat_response([]) + response.model = kwargs["model"] + response.choices[0].finish_reason = "stop" + response.choices[0].message.content = json.dumps( + {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + ) + return response + + +@pytest.mark.parametrize( + "model, completion", + [ + ("claude-haiku-5.5", stateless_model), + ("gpt-6-luna", stateless_responses_model), + ("gemini-3.1-pro-preview", stateless_json_model), + ], +) +def test_the_harness_conversation_runs_under_the_runner(world, model, completion): + """Each API and answer contract completes real harness conversations in + both runner lanes, with the registered feature frame for every arm. + + The confirmatory roster's Geminis use tools. The older board profile + covers the harness's Gemini JSON path in a fake design, without changing + the production roster or any model's answer contract.""" w = world(models=(model,)) w.run.converse = cr.converse_with_harness - w.run.completion_for = lambda profile: stateless_model + w.run.completion_for = lambda profile: completion w.execute() + labels = labels_of(model) attempts = w.attempts(model) - assert len(attempts) == N_DRAW * len(cr.ARMS) + assert len(attempts) == N_DRAW * len(labels) assert {x["status"] for x in attempts} == {"complete"} messages = w.run.lanes[model].pool.messages - assert len(messages) == N_DRAW * len(cr.ARMS) # one calculation per attempt + assert len(messages) == N_DRAW * len(labels) # one calculation per attempt frames = [m for m in messages if isinstance(m, tuple)] - assert len(frames) == N_DRAW * (len(cr.ARMS) - 1) # every arm but pe + assert len(frames) == N_DRAW * (len(labels) - 1) # every arm but pe sid = w.draw_order[0] - rec = json.loads((w.folder(model, "loo-names") / f"{sid}.json").read_text()) - profile = cr.arm_profile(model, "loo-names") - assert rec["arm"] == cr.arm_string("loo-names") and rec["attempts"] == [] + label = "loo-names" if "loo-names" in labels else "pe3" + rec = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + profile = cr.arm_profile(model, label) + assert rec["arm"] == cr.arm_string(label) and rec["attempts"] == [] assert rec["tool_features"] == sorted(profile.tool_features) assert rec["predictions"] == {v: 0.0 for v in VARIABLES} assert run_problems(w) == [] +@pytest.mark.parametrize( + "model, completion, api, contract, label", + [ + (model, completion, api, contract, label) + for model, completion, api, contract in [ + ("gpt-6-luna", stateless_responses_model, "responses", "tool"), + ("gemini-3.1-pro-preview", stateless_json_model, "chat", "json"), + ] + for label in labels_of(model) + ], +) +def test_the_runner_sends_the_treatment_conversation( + world, label, model, completion, api, contract +): + """Compare every provider request and calculation against the unchanged + treatment harness at 6b00a6f0, including the tool-result continuation. + + Gemini 3.1's board profile exercises JSON; both confirmatory Geminis use + the tool contract. No contract or model card is replaced for this test.""" + w = world(models=(model,)) + lanes, sid = w.run.lanes[model], w.draw_order[0] + requests, calculations = [], [] + + def provider(**kwargs): + """Keep each request before the conversation can extend its messages.""" + requests.append(copy.deepcopy(kwargs)) + return completion(**kwargs) + + def tool(arguments): + """Keep exactly what the conversation sends to the calculator.""" + calculations.append(arguments) + return {"results": {"snap": {"values": {"spm": 1.0}}}}, 0.0 + + lanes.completion = provider + record = cr.converse_with_harness( + lanes, sid, label, tool, cr.AttemptSpend(w.run.ceiling) + ) + sent, called = requests[:], calculations[:] + requests.clear() + calculations.clear() + profile = pilot.model_profile( + model, arm=cr.arm_string(label), prompt_version="v2-pilot" + ) + treatment = pilot.run_household( + w.board["scenarios"][sid], + w.board["variables"][sid], + tool, + provider, + pilot.Spend(100.0), + profile=profile, + ) + assert record["api"] == treatment["api"] == api + assert record["contract"] == treatment["contract"] == contract + assert record["status"] == treatment["status"] == "complete" + assert record["predictions"] == treatment["predictions"] == { + v: 0.0 for v in VARIABLES + } + assert len(sent) == len(requests) == 2 + assert sent == requests and called == calculations and len(called) == 1 + assert record["messages"] == treatment["messages"] + first, second = sent + if api == "responses": + assert "messages" not in first and "previous_response_id" not in first + assert first["tool_choice"] == "required" + assert [t["name"] for t in first["tools"]] == [ + pilot.TOOL_NAME, + "submit_outputs", + ] + assert "max_output_tokens" in first + assert second["previous_response_id"] == "resp_c1" + assert second["input"] == [ + { + "type": "function_call_output", + "call_id": "c1", + "output": json.dumps({"results": {"snap": {"values": {"spm": 1.0}}}}), + } + ] + else: + assert "input" not in first and first["tool_choice"] == "auto" + assert [t["function"]["name"] for t in first["tools"]] == [pilot.TOOL_NAME] + assert second["messages"][-1]["role"] == "tool" + assert second["messages"][-1]["tool_call_id"] == "c1" + + def test_the_ceiling_ends_a_harness_conversation_at_its_next_turn(world): model = "claude-haiku-5.5" w = world(models=(model,), overall_cap=0.0, spent_before=25.0 - 1e-9) From 4002f8ee4454f06c2373d9742aa2d365a629e920 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 23:48:23 -0700 Subject: [PATCH 13/48] Confirmatory runner: round-6 wording and coverage Answers review runner-r6's two P3 items: - Narrow the fsmonitor claim to disabling normal index-triggered startup. A configured hook can explicitly start the descriptor-closing daemon; configured background clone writers must retain the locked descriptor without unlocking it. The flock is local, while an external remote's safety rests on the byte-prefix check and ordinary non-force pushes. - Describe credential-cache and other descriptor holders keeping later helpers busy for their lifetimes, including the default 900-second cache. Every try while the lock is retained, including all three final tries, fails as busy and is reported as a failed push. - Block real Git clean filters during add and status, kill only the helper, and check later busy refusals leave file, raw index, staged contents, HEAD and remote history unchanged. Release the writer and verify newer snapshots push with prefix-preserving history. - Double-fork a hook descendant into a new session. Let its hook and helper finish, check the detached holder still prevents later pushes, and verify pushes succeed after it exits. Check each new regression against a temporary helper with inheritance removed only from its target command. Production behavior, report fields and outcomes are unchanged. This adds no dependencies and leaves the report implementation and draft untouched. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 29 ++- tests/test_confirmatory_anchor.py | 295 +++++++++++++++++++++++++++++- 2 files changed, 313 insertions(+), 11 deletions(-) diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py index 2dd7f2b3..8cd02865 100644 --- a/scripts/confirmatory_anchor.py +++ b/scripts/confirmatory_anchor.py @@ -14,15 +14,26 @@ An exclusive non-blocking lock in the clone's `.git` directory covers every read, copy, commit and push. Every Git command inherits its descriptor, so a surviving command keeps the clone busy even if this helper alone is killed. -Git's hooks and remote helpers inherit it too; automatic fsmonitor startup, -which closes inherited descriptors, is disabled. Configured commands must -preserve the descriptor in any background clone writer. This advisory lock -does not cover independent writers or replacement of the lock file. -A busy clone fails without changing its records. Under the lock, the clone's -current file must be a byte prefix of the snapshot (equality is allowed): a -delayed older command cannot replace newer evidence. This checks bytes, not -JSON or line boundaries; the runner supplies complete, synced lines. Git's -own index.lock is left alone; a git failure is a failed push. Never force-push. +Ordinary local hooks and remote helpers inherit it too. The per-command +`core.fsmonitor=false` disables normal index-triggered fsmonitor startup. +A configured hook can still run `git fsmonitor--daemon start`, whose child +closes inherited descriptors; this falls under the configured-command limit: +configured commands must retain the descriptor without unlocking it in any +background clone writer. This advisory lock does not cover independent +writers or replacement of the lock file. The lock is local: processes on an +external remote server never hold it. The remote's safety rests on the byte-prefix +check and ordinary non-force pushes. + +A credential-cache daemon or any other descriptor holder can keep the clone +busy up to the holder's lifetime (the credential cache defaults to 900 +seconds), indefinitely if it never exits or closes the descriptor. Every +later try, including all three final tries, then fails as busy and is reported +as a failed push. A busy clone fails without changing its records. Under the +lock, the clone's current file must be a byte prefix of the snapshot (equality +is allowed): a delayed older command cannot replace newer evidence. This +checks bytes, not JSON or line boundaries; the runner supplies complete, +synced lines. Git's own index.lock is left alone; a git failure is a failed +push. Never force-push. """ from __future__ import annotations diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index 153a2f1d..fbc5a367 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -263,6 +263,297 @@ def records(): os.close(fd) +@pytest.mark.parametrize("command", ["add", "status"]) +def test_killing_only_the_helper_during_add_or_status_keeps_the_clone_locked( + clone, tmp_path, command +): + """A clean filter pins each real Git command across helper-only death.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + older = first + b'{"time": 2.0}\n' + snapshots = [older + b'{"time": 3.0}\n', older + b'{"time": 3.0}\n{"time": 4.0}\n'] + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + ready, release = tmp_path / "ready", tmp_path / "release" + for path in (ready, release): + os.mkfifo(path) + invocation = tmp_path / "filter-invocation" + filter_lock = tmp_path / "filter-lock.json" + clean = tmp_path / "clean-filter" + block_at = 1 if command == "add" else 2 + clean.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "content = sys.stdin.buffer.read()\n" + "if os.environ.get('ANCHOR_TEST_BLOCK') == '1':\n" + f" invocation = Path({str(invocation)!r})\n" + " count = int(invocation.read_text()) + 1 if invocation.exists() else 1\n" + " invocation.write_text(str(count))\n" + f" if count == {block_at}:\n" + f" lock = Path({str(repo / '.git' / 'confirmatory-anchor.lock')!r})" + ".stat()\n" + " inherited = []\n" + " for fd in range(3, 256):\n" + " try:\n" + " info = os.fstat(fd)\n" + " except OSError:\n" + " continue\n" + " if (info.st_dev, info.st_ino) == (lock.st_dev, lock.st_ino):\n" + " inherited.append(os.get_inheritable(fd))\n" + f" Path({str(filter_lock)!r}).write_text(json.dumps(inherited))\n" + f" with open({str(ready)!r}, 'wb', buffering=0) as pipe:\n" + " pipe.write(b'1')\n" + f" with open({str(release)!r}, 'rb', buffering=0) as pipe:\n" + " pipe.read(1)\n" + f" elif count == 1 and {command == 'status'!r}:\n" + f" target = Path({str(repo / 'anchors.jsonl')!r})\n" + " info = target.stat()\n" + " os.utime(target, ns=(info.st_atime_ns, " + "info.st_mtime_ns - 2000000000))\n" + "sys.stdout.buffer.write(content)\n" + ) + clean.chmod(0o755) + (repo / ".git" / "info" / "attributes").write_text("anchors.jsonl filter=anchor\n") + for name, value in (("clean", str(clean)), ("required", "true")): + subprocess.run( + ["git", "-C", str(repo), "config", f"filter.anchor.{name}", value], + check=True, + ) + + def records(): + """Read file, raw index, staged and committed bytes and remote history.""" + objects = [ + subprocess.run( + ["git", "-C", str(repo), "show", obj], check=True, capture_output=True + ).stdout + for obj in (":anchors.jsonl", "HEAD:anchors.jsonl") + ] + return ( + (repo / "anchors.jsonl").read_bytes(), + (repo / ".git" / "index").read_bytes(), + *objects, + remote_file(remote).encode(), + remote_log(remote), + ) + + ready_fd, release_fd = (os.open(path, os.O_RDWR) for path in (ready, release)) + child = None + try: + anchors.write_bytes(older) + child = subprocess.Popen( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + env={**os.environ, "ANCHOR_TEST_BLOCK": "1"}, + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert select.select([ready_fd], [], [], 15.0)[0], ( + f"Git {command} did not reach its clean filter" + ) + assert os.read(ready_fd, 1) == b"1" + # The first conversion runs in add; changing its stat forces status's second. + assert invocation.read_text() == str(block_at) + before = records() + assert before[0] == older + assert before[2] == (first if command == "add" else older) + assert before[3] == before[4] == first + child.kill() + assert child.wait(timeout=15.0) == -signal.SIGKILL + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 1 and "busy" in out.stderr, ( + f"a surviving Git {command} lost the helper's clone lock: " + out.stderr + ) + assert records() == before + assert json.loads(filter_lock.read_text()) == [True] + os.write(release_fd, b"1") + child.communicate(timeout=15.0) + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + deadline = time.monotonic() + 15.0 + while True: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, f"Git {command} did not finish" + threading.Event().wait(0.01) + after = records() + assert after[0] == after[2] == older + assert after[3] == after[4] == first + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + state = records() + assert state[0] == state[2] == state[3] == state[4] == snapshot + history = subprocess.run( + ["git", "-C", str(remote), "rev-list", "--reverse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + previous = b"" + for commit in history: + content = subprocess.run( + ["git", "-C", str(remote), "show", f"{commit}:anchors.jsonl"], + check=True, + capture_output=True, + ).stdout + assert content.startswith(previous) + previous = content + assert previous == snapshots[-1] + finally: + if child is not None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.communicate(timeout=15.0) + for fd in (ready_fd, release_fd): + os.close(fd) + + +def test_a_detached_hook_descendant_keeps_the_clone_locked(clone, tmp_path): + """A double-forked holder outlives its hook and helper until explicitly released.""" + remote, repo = clone + anchors = tmp_path / "anchors.jsonl" + first = b'{"time": 1.0}\n' + older = first + b'{"time": 2.0}\n' + snapshots = [older + b'{"time": 3.0}\n', older + b'{"time": 3.0}\n{"time": 4.0}\n'] + anchors.write_bytes(first) + assert anchor_command(repo, anchors).returncode == 0 + ready, release = tmp_path / "ready", tmp_path / "release" + for path in (ready, release): + os.mkfifo(path) + holder = tmp_path / "detached-holder.json" + hook = repo / ".git" / "hooks" / "pre-push" + hook.write_text( + f"#!{sys.executable}\n" + "import json, os\n" + "from pathlib import Path\n" + "if os.environ.get('ANCHOR_TEST_DETACH') == '1':\n" + f" ready = os.open({str(ready)!r}, os.O_WRONLY)\n" + f" release = os.open({str(release)!r}, os.O_RDONLY)\n" + " hook_pid = os.getpid()\n" + " child = os.fork()\n" + " if child:\n" + " os.waitpid(child, 0)\n" + " raise SystemExit(0)\n" + " os.setsid()\n" + " if os.fork():\n" + " os._exit(0)\n" + " with open(os.devnull, 'r+b', buffering=0) as null:\n" + " for fd in range(3):\n" + " os.dup2(null.fileno(), fd)\n" + f" lock = Path({str(repo / '.git' / 'confirmatory-anchor.lock')!r}).stat()\n" + " inherited = []\n" + " for fd in range(3, 256):\n" + " try:\n" + " info = os.fstat(fd)\n" + " except OSError:\n" + " continue\n" + " if (info.st_dev, info.st_ino) == (lock.st_dev, lock.st_ino):\n" + " inherited.append(os.get_inheritable(fd))\n" + " state = dict(pid=os.getpid(), sid=os.getsid(0), pgid=os.getpgrp(),\n" + " hook_pid=hook_pid, inherited=inherited)\n" + f" Path({str(holder)!r}).write_text(json.dumps(state))\n" + " os.write(ready, b'1')\n" + " os.read(release, 1)\n" + " os._exit(0)\n" + ) + hook.chmod(0o755) + + def records(): + """Read the clone's file, index, HEAD and remote without writing.""" + objects = [ + subprocess.run( + ["git", "-C", str(repo), "show", obj], check=True, capture_output=True + ).stdout + for obj in (":anchors.jsonl", "HEAD:anchors.jsonl") + ] + return ( + (repo / "anchors.jsonl").read_bytes(), + (repo / ".git" / "index").read_bytes(), + *objects, + remote_file(remote).encode(), + remote_log(remote), + ) + + ready_fd, release_fd = (os.open(path, os.O_RDWR) for path in (ready, release)) + child = None + descendant = None + try: + anchors.write_bytes(older) + child = subprocess.Popen( + [sys.executable, str(SCRIPT), "--repo", str(repo), str(anchors)], + env={**os.environ, "ANCHOR_TEST_DETACH": "1"}, + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert select.select([ready_fd], [], [], 15.0)[0], ( + "hook did not detach its holder" + ) + assert os.read(ready_fd, 1) == b"1" + state = json.loads(holder.read_text()) + descendant = state["pid"] + stdout, stderr = child.communicate(timeout=15.0) + assert child.returncode == 0, stderr.decode() + assert stdout == b"pushed 2 anchors\n" + assert state["sid"] != child.pid and state["pgid"] != child.pid + assert descendant != state["sid"] + os.kill(descendant, 0) + with pytest.raises(ProcessLookupError): + os.kill(state["hook_pid"], 0) + before = records() + assert before[0] == before[2] == before[3] == before[4] == older + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 1 and "busy" in out.stderr, ( + "a detached hook descendant lost the helper's clone lock: " + out.stderr + ) + assert records() == before + assert state["inherited"] == [True] + os.write(release_fd, b"1") + with (repo / ".git" / "confirmatory-anchor.lock").open("a") as lock: + deadline = time.monotonic() + 15.0 + while True: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, ( + "the detached holder did not exit" + ) + threading.Event().wait(0.01) + descendant = None + for snapshot in snapshots: + anchors.write_bytes(snapshot) + out = anchor_command(repo, anchors) + assert out.returncode == 0, out.stderr + state = records() + assert state[0] == state[2] == state[3] == state[4] == snapshot + finally: + os.write(release_fd, b"1") + if descendant is not None: + try: + os.kill(descendant, signal.SIGKILL) + except ProcessLookupError: + pass + if child is not None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.communicate(timeout=15.0) + for fd in (ready_fd, release_fd): + os.close(fd) + + def test_a_changed_middle_line_is_refused_without_writing(clone, tmp_path): """A longer snapshot still fails when it rewrites an existing line.""" remote, repo = clone @@ -298,8 +589,8 @@ def test_a_stale_git_index_lock_is_left_alone(clone, tmp_path): assert remote_log(remote) == history -def test_git_fsmonitor_is_disabled_even_when_the_clone_enables_it(clone, tmp_path): - """Git must not launch a monitor that can discard the inherited clone lock.""" +def test_normal_index_fsmonitor_startup_is_disabled(clone, tmp_path): + """The per-command override suppresses a configured index-triggered monitor.""" remote, repo = clone anchors = tmp_path / "anchors.jsonl" anchors.write_bytes(b'{"time": 1.0}\n') From 047c8682f426381687ae93321cddc8732b9cd635 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 00:52:57 -0700 Subject: [PATCH 14/48] Run the no-tools stage with recoverable attempts and cumulative spend. Run each selected model in its own thread, one drawn household at a time, using the pinned harness's none profile and a tool callback that raises. Reuse the calculator runner's classification and action rules without host evidence. Rerun eligible provider and unanswered attempts immediately. Pause account failures until re-invocation, stop a model on a harness class, and stop every model on a budget stop. Limit reruns to two while retaining the action helper's harness and ceiling stop precedence. Move each replaced final's exact bytes into consecutively numbered attempt files before rerunning. Atomically write finals with complete ordered number, path, hash, status, class and reason links. Recover an interrupted archive from the files alone. Refuse native prior_attempts, missing attempts lists, numbering gaps, altered history links and ineligible archives. Carry every model's final and archived spend across invocations, including optional model folders. Share the five-dollar cap across all model threads, check it before each attempt, and retain an exception's spend in the harness fallback record. Show each attempt, model state and cumulative spend; dry runs show standing and owed households. Return success only when every selected model has a standing final for every drawn household. Cover synthetic providers, retry exhaustion, operator resume, harness and budget stops, interrupted archives, corrupted histories, exception spend, optional models and agreement with the separate report's registered cap. The existing calculator runner remains unchanged. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 411 +++++++++++++++++++++++++++ tests/test_confirmatory_no_tools.py | 426 ++++++++++++++++++++++++++++ 2 files changed, 837 insertions(+) create mode 100644 scripts/confirmatory_no_tools.py create mode 100644 tests/test_confirmatory_no_tools.py diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py new file mode 100644 index 00000000..c48f06ac --- /dev/null +++ b/scripts/confirmatory_no_tools.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +"""Run the confirmatory no-tools stage with complete, recoverable attempts. + +Every model runs in its own thread, in draw order, under a shared $5 ceiling. +A replaced final is moved byte for byte into ``households/attempts`` before +its rerun; the final links every archive by number, hash, status and reason. +Re-invocation resumes solely from those files and is an operator's approval +to rerun a paused account failure. Native harness resume is never used. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import re +import sys +import threading +from pathlib import Path + +# confirmatory_run pins the harness and policybench to this checkout. +from confirmatory_run import ( + CHEAP, + MAX_RERUNS, + PROMPT_VERSION, + action, + attempt_cost, + classify, + default_completion, + pilot, + sha256_file, + write_json, +) + +NO_TOOLS_CAP = 5.0 + + +class StateError(ValueError): + """The files cannot describe an intact no-tools attempt history.""" + + +def component(value: str) -> str: + """A record/model name must name one ordinary path component.""" + if ( + not isinstance(value, str) + or value in ("", ".", "..") + or Path(value).name != value + ): + raise StateError(f"invalid path component {value!r}") + return value + + +def read_record(path: Path) -> dict: + try: + record = json.loads(path.read_text()) + if not isinstance(record, dict): + raise ValueError("record is not a JSON object") + if "prior_attempts" in record: + raise ValueError("native prior_attempts cannot preserve full attempts") + if not isinstance(record.get("attempts"), list): + raise ValueError("record lacks an attempts list") + cost = attempt_cost(record) + if not math.isfinite(cost) or cost < 0: + raise ValueError("attempt cost must be finite and nonnegative") + return record + except (OSError, ValueError, TypeError, AttributeError) as error: + raise StateError(f"{path}: {error}") from error + + +def entry(path: Path, record: dict, n: int) -> dict: + cls, reason = classify(record, 0.0, 0.0, None) + return { + "n": n, + "file": f"attempts/{path.name}", + "sha256": sha256_file(path), + "status": record.get("status"), + "class": cls, + "reason": reason, + } + + +def history(folder: Path) -> tuple[dict[str, dict], float]: + """Validate a model's histories, including an interrupted archive, and sum spend.""" + archives: dict[str, list[tuple[int, Path]]] = {} + for path in sorted((folder / "attempts").glob("*.json")): + match = re.fullmatch(r"(.+)-([1-9][0-9]*)\.json", path.name) + if not match: + raise StateError(f"{path}: invalid attempt number") + archives.setdefault(match[1], []).append((int(match[2]), path)) + finals = {path.stem: path for path in sorted(folder.glob("*.json"))} + states, spent = {}, 0.0 + for sid in sorted(set(finals) | set(archives)): + paths = sorted(archives.get(sid, [])) + numbers = [n for n, _ in paths] + if numbers != list(range(1, len(paths) + 1)): + raise StateError( + f"{folder}: attempt-number gap for {sid}: " + + ", ".join(path.name for _, path in paths) + ) + if len(paths) > MAX_RERUNS: + raise StateError(f"{folder}: {sid} exceeds {MAX_RERUNS} reruns") + entries = [] + for n, path in paths: + record = read_record(path) + if record.get("scenario_id") != sid: + raise StateError(f"{path}: household differs from archive name {sid}") + if record["attempts"] != entries: + raise StateError(f"{path}: attempts disagree with earlier archives") + cls, _ = classify(record, 0.0, 0.0, None) + if action(cls, n) not in ("rerun", "pause-rerun"): + raise StateError(f"{path}: archived attempt was ineligible for rerun") + entries.append(entry(path, record, n)) + spent += attempt_cost(record) + final = None + if sid in finals: + path = finals[sid] + final = read_record(path) + if final.get("scenario_id") != sid: + raise StateError(f"{path}: household differs from final name {sid}") + if final["attempts"] != entries: + raise StateError(f"{path}: attempts disagree with archived files") + spent += attempt_cost(final) + states[sid] = {"entries": entries, "final": final} + return states, spent + + +class Spend: + """The stage's recorded spend, shared by every model and every invocation.""" + + def __init__(self, spent: float, stopped: threading.Event): + self.spent_usd, self.budget_usd = spent, NO_TOOLS_CAP + self.lock, self.stopped = threading.Lock(), stopped + if spent >= self.budget_usd: + stopped.set() + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + with self.lock: + return ( + self.stopped.is_set() or self.spent_usd + reserve_usd >= self.budget_usd + ) + + +class AttemptSpend: + """Count a request in both the stage and its own exception fallback record.""" + + def __init__(self, stage: Spend): + self.stage, self.spent_usd = stage, 0.0 + + def add(self, usd: float | None) -> None: + cost = float(usd or 0.0) + with self.stage.lock: + self.spent_usd += cost + self.stage.spent_usd += cost + if self.stage.spent_usd >= self.stage.budget_usd: + self.stage.stopped.set() + + def exhausted(self, reserve_usd: float = 0.0) -> bool: + return self.stage.exhausted(reserve_usd) + + +class Stage: + """One stage, with injectable providers/board for a provider-free rehearsal.""" + + def __init__( + self, + design: dict, + models=None, + run_household=None, + completion_for=default_completion, + board=None, + out=None, + ): + self.design = design + self.models = tuple(dict.fromkeys(models if models is not None else CHEAP)) + if not self.models: + raise StateError("at least one model is required") + for model in self.models: + component(model) + self.draw_order = [component(sid) for sid in design["draw_order"]] + if len(self.draw_order) != len(set(self.draw_order)): + raise StateError("draw_order repeats a household") + spec = design["no_tools"] + self.root = Path(spec["runs"]) + self.folder, self.stage = component(spec["folder"]), component(spec["stage"]) + self.out = sys.stdout if out is None else out + self._say_lock = threading.Lock() + self.stopped = threading.Event() + self.completion_for = completion_for + self.run_household = ( + run_household if run_household is not None else pilot.run_household + ) + pilot.set_stated_engine(design["engine"].split()[-1]) + if board is None: + board_path = Path(design["board"]) + if f"policyengine-us {pilot.board_engine(board_path)}" != design["engine"]: + raise StateError("the board's engine differs from the design") + pilot._require_engine(board_path) + board = pilot.load_board(board_path) + self.board = board + missing = set(self.draw_order) - set(board["scenarios"]) + missing |= set(self.draw_order) - set(board["variables"]) + if missing: + raise StateError( + f"drawn households missing from the board: {sorted(missing)}" + ) + self.histories = {} + spent = 0.0 + folders = ( + {p.name for p in self.root.iterdir() if p.is_dir()} + if self.root.is_dir() + else set() + ) + for model in sorted(folders | set(self.models)): + states, cost = history(self.households(model)) + self.histories[model] = states + spent += cost + # A saved ceiling failure is a stage-wide stop on every invocation. + if any( + state["final"] is not None + and action( + classify(state["final"], 0.0, 0.0, None)[0], + len(state["entries"]) + 1, + ) + == "stopped" + for state in states.values() + ): + self.stopped.set() + self.spend = Spend(spent, self.stopped) + self.states = {model: "pending" for model in self.models} + + def households(self, model: str) -> Path: + return self.root / model / self.folder / self.stage / "households" + + def say(self, text: str) -> None: + with self._say_lock: + print(text, file=self.out, flush=True) + + def standing(self, model: str, sid: str) -> bool: + state = self.histories[model].get(sid) + return bool( + state + and state["final"] is not None + and action( + classify(state["final"], 0.0, 0.0, None)[0], len(state["entries"]) + 1 + ) + == "stands" + ) + + def dry_run(self) -> int: + for model in self.models: + done = [sid for sid in self.draw_order if self.standing(model, sid)] + owed = [] + for sid in self.draw_order: + if sid in done: + continue + state = self.histories[model].get(sid, {"entries": [], "final": None}) + n = len(state["entries"]) + 1 + if state["final"] is not None: + outcome = action(classify(state["final"], 0.0, 0.0, None)[0], n) + if outcome in ("stop", "stopped"): + owed.append(f"{sid}:blocked {outcome} at attempt {n}") + continue + n += 1 + owed.append(f"{sid}:{n}") + self.say(f"{model}: done {done}; owed (household:next attempt) {owed}") + self.say(f"stage spend ${self.spend.spent_usd:.6f} of ${NO_TOOLS_CAP:.2f}") + return ( + 0 + if all( + self.standing(model, sid) + for model in self.models + for sid in self.draw_order + ) + else 1 + ) + + def archive(self, model: str, sid: str) -> None: + """Move an eligible final unchanged, then extend its reconstructed links.""" + state = self.histories[model][sid] + record, entries = state["final"], state["entries"] + n = len(entries) + 1 + cls, _ = classify(record, 0.0, 0.0, None) + if action(cls, n) not in ("rerun", "pause-rerun"): + raise StateError(f"{model}/{sid}: attempt {n} is ineligible for rerun") + folder = self.households(model) + target = folder / "attempts" / f"{sid}-{n}.json" + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + raise StateError(f"{target}: would overwrite an archived attempt") + os.replace(folder / f"{sid}.json", target) + entries.append(entry(target, record, n)) + state["final"] = None + + def run_model(self, model: str) -> None: + try: + profile = pilot.model_profile( + model, arm="none", prompt_version=PROMPT_VERSION + ) + if profile.uses_calculator or profile.uses_web: + raise StateError(f"{model}: the none profile offers a tool") + completion = None + completion_loaded = False + for sid in self.draw_order: + state = self.histories[model].setdefault( + sid, {"entries": [], "final": None} + ) + while True: + if state["final"] is not None: + cls, _ = classify(state["final"], 0.0, 0.0, None) + outcome = action(cls, len(state["entries"]) + 1) + if outcome == "stands": + break + if outcome == "stop": + self.states[model] = "harness-stopped" + return + if outcome == "stopped": + self.stopped.set() + if self.spend.exhausted(): + self.states[model] = "budget-stopped" + return + if state["final"] is not None: + self.archive(model, sid) + if not completion_loaded: + completion = self.completion_for(profile) + completion_loaded = True + if self.spend.exhausted(): + self.states[model] = "budget-stopped" + return + spend = AttemptSpend(self.spend) + + def no_tool(arguments): + raise RuntimeError("the no-tools arm cannot call a tool") + + try: + record = self.run_household( + self.board["scenarios"][sid], + self.board["variables"][sid], + no_tool, + completion, + spend, + profile=profile, + ) + except Exception as error: + record = pilot.harness_error_record( + sid, self.board["variables"][sid], profile, error + ) + record["totals"]["cost_usd"] = spend.spent_usd + record["attempts"] = list(state["entries"]) + write_json(self.households(model) / f"{sid}.json", record) + state["final"] = record + n = len(state["entries"]) + 1 + cls, _ = classify(record, 0.0, 0.0, None) + outcome = action(cls, n) + self.say( + f"{model} {sid} {n} {cls} {outcome} ${attempt_cost(record):.6f}" + ) + if outcome == "pause-rerun": + self.states[model] = "operator-paused" + return + if outcome == "stop": + self.states[model] = "harness-stopped" + return + if outcome == "stopped": + self.stopped.set() + self.states[model] = "budget-stopped" + return + if outcome == "stands": + break + self.states[model] = "complete" + except Exception as error: + self.states[model] = "error" + self.say(f"{model}: {type(error).__name__}: {error}") + + def execute(self) -> int: + threads = [ + threading.Thread( + target=self.run_model, args=(model,), name=f"no-tools-{model}" + ) + for model in self.models + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + for model in self.models: + self.say(f"{model}: {self.states[model]}") + self.say(f"stage spend ${self.spend.spent_usd:.6f} of ${NO_TOOLS_CAP:.2f}") + return ( + 0 if all(self.states[model] == "complete" for model in self.models) else 1 + ) + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run") + run.add_argument("design", type=Path) + run.add_argument("--model", action="extend", nargs="+") + run.add_argument("--dry-run", action="store_true") + args = parser.parse_args(argv) + try: + stage = Stage(json.loads(args.design.read_text()), models=args.model) + return stage.dry_run() if args.dry_run else stage.execute() + except (StateError, OSError, ValueError, KeyError, TypeError) as error: + print(f"The no-tools stage cannot start: {error}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py new file mode 100644 index 00000000..1a2b388e --- /dev/null +++ b/tests/test_confirmatory_no_tools.py @@ -0,0 +1,426 @@ +"""Provider-free tests of the no-tools stage's files, retries and shared spend.""" + +from __future__ import annotations + +import ast +import io +import json +import os +import sys +import threading +from pathlib import Path +from types import SimpleNamespace + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import confirmatory_no_tools as nt # noqa: E402 + + +@pytest.fixture +def setup(tmp_path): + draw = ["h2", "h1"] + return { + "design": { + "board": "synthetic.csv", + "engine": "policyengine-us 2.60.4", + "draw_order": draw, + "no_tools": { + "runs": str(tmp_path / "no-tools"), + "folder": "none", + "stage": "confirmatory", + }, + }, + "board": { + "scenarios": {sid: SimpleNamespace(id=sid) for sid in draw}, + "variables": {sid: ["snap"] for sid in draw}, + }, + } + + +def record(sid, profile, status="complete", error=None, cost=0.1): + return { + "scenario_id": sid, + "model": profile.name, + "arm": profile.arm, + "status": status, + "error": error, + "turns": [] if status == "harness_error" else [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": cost}, + } + + +class Provider: + def __init__(self, planted=None): + self.planted, self.calls = planted or {}, [] + self.lock = threading.Lock() + + def __call__( + self, scenario, variables, tool_call, completion_fn, spend, *, profile + ): + key = profile.name, scenario.id + with self.lock: + n = 1 + sum(call[:2] == key for call in self.calls) + self.calls.append((*key, n)) + spec = self.planted.get((*key, n), {}) + cost = spec.get("cost", 0.1) + spend.add(cost) + if spec.get("raise"): + raise RuntimeError("synthetic harness failure") + return record( + scenario.id, + profile, + cost=cost, + status=spec.get("status", "complete"), + error=spec.get("error"), + ) + + +def stage(setup, provider=None, models=None, **kwargs): + return nt.Stage( + setup["design"], + models=models, + board=setup["board"], + run_household=provider or Provider(), + completion_for=lambda profile: None, + out=io.StringIO(), + **kwargs, + ) + + +def final(plan, model=nt.CHEAP[0], sid="h2"): + return plan.households(model) / f"{sid}.json" + + +def load(path): + return json.loads(path.read_text()) + + +def plant(setup, status, error, n=1, **extra): + return {(nt.CHEAP[0], "h2", n): {"status": status, "error": error, **extra}} + + +def test_every_household_ok_in_draw_order_and_each_model_has_its_thread(setup): + provider, threads = Provider(), {} + + def fake(*args, **kwargs): + threads.setdefault(kwargs["profile"].name, threading.current_thread().name) + return provider(*args, **kwargs) + + plan = stage(setup, fake) + assert plan.execute() == 0 + assert all(plan.states[m] == "complete" for m in nt.CHEAP) + assert len(set(threads.values())) == len(nt.CHEAP) + for model in nt.CHEAP: + assert [call[1] for call in provider.calls if call[0] == model] == ["h2", "h1"] + for sid in setup["design"]["draw_order"]: + assert load(final(plan, model, sid))["attempts"] == [] + assert plan.spend.spent_usd == pytest.approx(0.6) + + +def test_provider_rerun_links_the_unchanged_archived_bytes(setup): + provider = Provider(plant(setup, "provider_error", "APIConnectionError: synthetic")) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + written = {} + original = nt.write_json + + def capture(path, payload, *args, **kwargs): + result = original(path, payload, *args, **kwargs) + written.setdefault(path.name, path.read_bytes()) + return result + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(nt, "write_json", capture) + assert plan.execute() == 0 + path = final(plan) + archive = path.parent / "attempts/h2-1.json" + assert archive.read_bytes() == written["h2.json"] + assert load(path)["attempts"] == [ + { + "n": 1, + "file": "attempts/h2-1.json", + "sha256": nt.sha256_file(archive), + "status": "provider_error", + "class": "provider", + "reason": "APIConnectionError", + } + ] + assert stage(setup, models=[nt.CHEAP[0]]).spend.spent_usd == pytest.approx(0.3) + + +def test_third_provider_failure_stands_and_has_only_two_archives(setup): + planted = { + **plant(setup, "provider_error", "APIConnectionError", 1), + **plant(setup, "provider_error", "APIConnectionError", 2), + **plant(setup, "provider_error", "APIConnectionError", 3), + } + provider = Provider(planted) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.execute() == 0 + assert [n for m, sid, n in provider.calls if sid == "h2"] == [1, 2, 3] + assert len(load(final(plan))["attempts"]) == nt.MAX_RERUNS + assert load(final(plan))["status"] == "provider_error" + + +def test_operator_pause_stops_its_model_and_reinvocation_resumes(setup): + provider = Provider( + plant(setup, "provider_error", "AuthenticationError: synthetic") + ) + first = stage(setup, provider) + assert first.execute() == 1 + assert first.states[nt.CHEAP[0]] == "operator-paused" + assert not final(first, sid="h1").exists() + assert first.states[nt.CHEAP[1]] == "complete" + before = final(first).read_bytes() + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.execute() == 0 + assert (final(resumed).parent / "attempts/h2-1.json").read_bytes() == before + assert load(final(resumed))["attempts"][0]["class"] == "operator" + + +def test_harness_stop_prevents_later_households_and_persists(setup): + provider = Provider(plant(setup, "provider_error", "BadRequestError: synthetic")) + + # A request error is a harness class only before a completed turn. + def early_failure(*args, **kwargs): + result = provider(*args, **kwargs) + if result["status"] == "provider_error": + result["turns"] = [] + return result + + plan = stage(setup, early_failure, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert plan.states[nt.CHEAP[0]] == "harness-stopped" + assert not final(plan, sid="h1").exists() + resumed_provider = Provider() + resumed = stage(setup, resumed_provider, models=[nt.CHEAP[0]]) + assert resumed.execute() == 1 + assert resumed_provider.calls == [] + + +def paused(setup): + plan = stage( + setup, + Provider(plant(setup, "provider_error", "AuthenticationError")), + models=[nt.CHEAP[0]], + ) + assert plan.execute() == 1 + return plan + + +def test_crash_after_archive_resumes_at_the_next_attempt_number(setup, monkeypatch): + first = paused(setup) + original = nt.os.replace + + def interrupted(source, target): + original(source, target) + if Path(target).parent.name == "attempts": + raise RuntimeError("synthetic crash after archive") + + monkeypatch.setattr(nt.os, "replace", interrupted) + with pytest.raises(RuntimeError, match="synthetic crash"): + first.archive(nt.CHEAP[0], "h2") + monkeypatch.setattr(nt.os, "replace", original) + assert not final(first).exists() + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.execute() == 0 + assert load(final(resumed))["attempts"][0]["n"] == 1 + assert " h2 2 ok stands " in resumed.out.getvalue() + + +def test_spend_in_archives_and_optional_models_carries_across_invocations(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + optional = stage(setup, models=["gpt-6-sol"]) + assert optional.execute() == 0 + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.spend.spent_usd == pytest.approx(0.3) + assert resumed.execute() == 0 + assert resumed.spend.spent_usd == pytest.approx(0.5) + + +def test_reaching_the_cap_prevents_every_new_attempt_across_invocations( + setup, monkeypatch +): + monkeypatch.setattr(nt, "NO_TOOLS_CAP", 0.1) + plan = stage(setup, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert len(list(plan.households(nt.CHEAP[0]).glob("*.json"))) == 1 + provider = Provider() + resumed = stage(setup, provider, models=[nt.CHEAP[1]]) + assert resumed.execute() == 1 + assert provider.calls == [] + assert resumed.spend.spent_usd == pytest.approx(0.1) + + +def test_budget_stop_is_shared_by_all_models(setup): + provider = Provider(plant(setup, "budget_stop", None)) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert plan.stopped.is_set() + resumed = stage(setup, Provider(), models=[nt.CHEAP[1]]) + assert resumed.execute() == 1 + assert resumed.states[nt.CHEAP[1]] == "budget-stopped" + + +@pytest.mark.parametrize( + "mutation,match", + [ + (lambda r: r.update(prior_attempts=[]), "prior_attempts"), + (lambda r: r.pop("attempts"), "attempts list"), + ], +) +def test_refuse_a_final_not_written_by_this_script(setup, mutation, match): + first = paused(setup) + path = final(first) + payload = load(path) + mutation(payload) + path.write_text(json.dumps(payload)) + with pytest.raises(nt.StateError, match=match): + stage(setup) + + +def test_refuse_an_attempt_number_gap_and_name_its_files(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + archive = final(first).parent / "attempts/h2-1.json" + archive.rename(archive.with_name("h2-2.json")) + with pytest.raises(nt.StateError, match="attempt-number gap.*h2-2.json"): + stage(setup) + + +@pytest.mark.parametrize( + "field,value", [("sha256", "wrong"), ("n", 2), ("reason", "edited")] +) +def test_refuse_a_final_with_corrupted_attempt_links(setup, field, value): + paused(setup) + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.execute() == 0 + path = final(resumed) + payload = load(path) + payload["attempts"][0][field] = value + path.write_text(json.dumps(payload)) + with pytest.raises(nt.StateError, match="attempts disagree"): + stage(setup) + + +def test_an_attempt_that_raises_keeps_its_spend(setup): + provider = Provider(plant(setup, "complete", None, **{"raise": True, "cost": 0.27})) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.execute() == 0 + archive = final(plan).parent / "attempts/h2-1.json" + assert load(archive)["status"] == "harness_error" + assert load(archive)["totals"]["cost_usd"] == 0.27 + assert plan.spend.spent_usd == pytest.approx(0.47) + assert stage(setup, models=[nt.CHEAP[0]]).spend.spent_usd == pytest.approx(0.47) + + +def test_dry_run_lists_done_households_and_next_owed_attempt_without_providers(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + provider = Provider() + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.dry_run() == 1 + text = plan.out.getvalue() + assert "done []" in text and "h2:2" in text and "h1:1" in text + assert "$0.100000" in text + assert provider.calls == [] + + +def test_each_attempt_uses_none_profile_and_refuses_a_tool_call(setup): + def fake(scenario, variables, tool_call, completion, spend, *, profile): + assert profile.arm == "none" and profile.prompt_version == nt.PROMPT_VERSION + assert variables == ["snap"] + with pytest.raises(RuntimeError, match="cannot call a tool"): + tool_call({}) + return record(scenario.id, profile, cost=0) + + assert stage(setup, fake, models=[nt.CHEAP[0]]).execute() == 0 + + +def test_registered_cap_matches_the_report_when_available(): + """The local report is separate from this public repository's dependency tree.""" + configured = os.environ.get("CONFIRMATORY_REPORT") + candidates = ( + [Path(configured)] + if configured + else [Path(__file__).resolve().parents[2] / "report" / "confirmatory_report.py"] + ) + path = next((p for p in candidates if p.is_file()), None) + if path is None: + pytest.skip("set CONFIRMATORY_REPORT to compare the separate report checkout") + tree = ast.parse(path.read_text()) + cap = next( + ast.literal_eval(node.value) + for node in tree.body + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "NO_TOOLS_CAP" + for target in node.targets + ) + ) + assert nt.NO_TOOLS_CAP == cap == 5.0 + + +def test_main_supports_multiple_model_options_and_dry_run( + setup, tmp_path, monkeypatch, capsys +): + path = tmp_path / "design.json" + path.write_text(json.dumps(setup["design"])) + monkeypatch.setattr(nt.pilot, "board_engine", lambda path: "2.60.4") + monkeypatch.setattr(nt.pilot, "_require_engine", lambda path: None) + monkeypatch.setattr(nt.pilot, "load_board", lambda path: setup["board"]) + assert ( + nt.main( + [ + "run", + str(path), + "--model", + nt.CHEAP[0], + nt.CHEAP[1], + "--model", + nt.CHEAP[2], + "--dry-run", + ] + ) + == 1 + ) + output = capsys.readouterr().out + assert all(model in output for model in nt.CHEAP) + + +def test_in_flight_models_observe_the_shared_cap_before_their_next_turn( + setup, monkeypatch +): + monkeypatch.setattr(nt, "NO_TOOLS_CAP", 0.1) + ready, charged = threading.Barrier(3), threading.Barrier(3) + calls = [] + + def fake(scenario, variables, tool_call, completion, spend, *, profile): + calls.append((profile.name, scenario.id)) + ready.wait(timeout=30) + cost = 0.1 if profile.name == nt.CHEAP[0] else 0.0 + spend.add(cost) + charged.wait(timeout=30) + assert spend.exhausted() + return record(scenario.id, profile, cost=cost) + + plan = stage(setup, fake) + assert plan.execute() == 1 + assert len(calls) == 3 and all(sid == "h2" for _, sid in calls) + assert plan.spend.spent_usd == pytest.approx(0.1) + + +def test_dry_run_marks_harness_and_budget_finals_as_blocked(setup): + plan = stage(setup, models=[nt.CHEAP[0]]) + profile = nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + payload = record("h2", profile, status="budget_stop", cost=0) + payload["attempts"] = [] + nt.write_json(final(plan), payload) + resumed = stage(setup, models=[nt.CHEAP[0]]) + assert resumed.dry_run() == 1 + assert "h2:blocked stopped at attempt 1" in resumed.out.getvalue() From 060b28793eae03e768090ed05a9746c8bc342527 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 02:30:20 -0700 Subject: [PATCH 15/48] Confirmatory runner: wait for host-log samples in two timing tests Two host-log tests slept a fixed time and expected the 0.02 s sampler to have run in it. On a loaded host it did not: test_the_host_log_samples_and_closes got three samples where it expects four (09:26Z run at 047c8682), and test_pressure_near_an_attempt_is_host_trouble_once_settled found no sample during its 0.05 s of pressure (09:39Z run). Neither failure was a fault in the runner. A helper, wait_for_samples, now polls the log's complete lines for up to 5 seconds until they hold what each test needs: four samples, or one sample under pressure. The assertions are unchanged. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index d4e3622f..246b2533 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -350,6 +350,17 @@ def host(self, folder): ) +def wait_for_samples(log: Path, enough, seconds: float = 5.0) -> None: + """Wait until the host log's complete samples satisfy `enough`. A loaded + host can stretch the 0.02 s sampler past any fixed sleep.""" + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + text = log.read_text() if log.exists() else "" + if enough([json.loads(x) for x in text.split("\n")[:-1]]): + return + time.sleep(0.01) + + @pytest.fixture def fast(monkeypatch): """Short evidence windows, with room for the shared host's scheduler. @@ -371,7 +382,7 @@ def fast(monkeypatch): def test_the_host_log_samples_and_closes(tmp_path, fast): readers = Readers() host = readers.host(tmp_path) - time.sleep(0.1) + wait_for_samples(tmp_path / "host-log.jsonl", lambda samples: len(samples) >= 4) host.close() meta = json.loads((tmp_path / "host.json").read_text()) samples = [ @@ -390,7 +401,10 @@ def test_pressure_near_an_attempt_is_host_trouble_once_settled(tmp_path, fast): host = readers.host(tmp_path) start = time.time() readers.pressure = 2 - time.sleep(0.05) + wait_for_samples( + tmp_path / "host-log.jsonl", + lambda samples: any(x["pressure"] == 2 for x in samples), + ) readers.pressure = 1 end = time.time() host.wait_settled(end) From 08d7b186448af15aa3bc42dd03708b4034ef6305 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 03:33:43 -0700 Subject: [PATCH 16/48] Confirmatory runner: coordinate no-tools shutdown and freeze inputs Install and restore run-scoped SIGINT and SIGTERM handlers. The first signal lets active attempts save, prevents further archives and attempts, and returns a nonzero interrupted state. A second signal warns that the active attempt and its spend are lost before immediate exit. Hold an exclusive nonblocking stage-root lock through initialization, history scanning and execution, including dry runs. Require the baseline exclusions file to exist and match its declared sha256 before startup. Provide a board-free stage validator that requires every cheap-model drawn final to stand and rejects invalid histories in every model folder. Exercise archive household identity, rerun eligibility, third-attempt harness stops, both blocked dry-run classes, and the real harness turn check after another model crosses the shared cap. Regression evidence records seven expected failures before these fixes and eleven passing focused cases afterwards; the expanded no-tools file passes all 33 tests. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 134 ++++++++++++++++- tests/test_confirmatory_no_tools.py | 221 +++++++++++++++++++++++++++- 2 files changed, 348 insertions(+), 7 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index c48f06ac..d165a950 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -11,10 +11,12 @@ from __future__ import annotations import argparse +import fcntl import json import math import os import re +import signal import sys import threading from pathlib import Path @@ -125,6 +127,42 @@ def history(folder: Path) -> tuple[dict[str, dict], float]: return states, spent +def stage_problems(design: dict) -> list[str]: + """Read stage histories and standing finals without a board or provider.""" + problems = [] + try: + spec = design["no_tools"] + root = Path(spec["runs"]) + folder, stage = component(spec["folder"]), component(spec["stage"]) + draw_order = [component(sid) for sid in design["draw_order"]] + models = set(CHEAP) + if root.is_dir(): + models.update(path.name for path in root.iterdir() if path.is_dir()) + except (OSError, KeyError, TypeError, ValueError) as error: + return [f"no-tools stage: {error}"] + for model in sorted(models): + try: + states, _ = history(root / model / folder / stage / "households") + if model not in CHEAP: + continue + for sid in draw_order: + state = states.get(sid) + if state is None or state["final"] is None: + problems.append(f"no-tools {model}/{sid}: missing standing final") + continue + outcome = action( + classify(state["final"], 0.0, 0.0, None)[0], + len(state["entries"]) + 1, + ) + if outcome != "stands": + problems.append( + f"no-tools {model}/{sid}: final action is {outcome}" + ) + except (OSError, TypeError, ValueError, AttributeError, KeyError) as error: + problems.append(f"no-tools {model}: {error}") + return problems + + class Spend: """The stage's recorded spend, shared by every model and every invocation.""" @@ -184,12 +222,37 @@ def __init__( self.root = Path(spec["runs"]) self.folder, self.stage = component(spec["folder"]), component(spec["stage"]) self.out = sys.stdout if out is None else out - self._say_lock = threading.Lock() + self._say_lock = threading.RLock() self.stopped = threading.Event() + self.interrupted = threading.Event() self.completion_for = completion_for self.run_household = ( run_household if run_household is not None else pilot.run_household ) + self._lock_file = None + try: + baseline = design["baseline_exclusions"] + if sha256_file(Path(baseline["path"])) != baseline["sha256"]: + raise StateError("baseline exclusions do not hash as declared") + except (OSError, KeyError, TypeError) as error: + raise StateError(f"baseline exclusions: {error}") from error + self.root.mkdir(parents=True, exist_ok=True) + self._lock_file = (self.root / ".confirmatory-no-tools.lock").open("a") + try: + fcntl.flock(self._lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + self.close() + raise StateError( + f"another no-tools invocation holds the lock in {self.root}" + ) from error + try: + self._load_state(board) + except BaseException: + self.close() + raise + + def _load_state(self, board) -> None: + design = self.design pilot.set_stated_engine(design["engine"].split()[-1]) if board is None: board_path = Path(design["board"]) @@ -229,6 +292,22 @@ def __init__( self.spend = Spend(spent, self.stopped) self.states = {model: "pending" for model in self.models} + def close(self) -> None: + """Release the invocation lock, including after initialization failure.""" + lock = getattr(self, "_lock_file", None) + if lock is not None: + lock.close() + self._lock_file = None + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __del__(self): + self.close() + def households(self, model: str) -> Path: return self.root / model / self.folder / self.stage / "households" @@ -248,6 +327,12 @@ def standing(self, model: str, sid: str) -> bool: ) def dry_run(self) -> int: + try: + return self._dry_run() + finally: + self.close() + + def _dry_run(self) -> int: for model in self.models: done = [sid for sid in self.draw_order if self.standing(model, sid)] owed = [] @@ -302,10 +387,16 @@ def run_model(self, model: str) -> None: completion = None completion_loaded = False for sid in self.draw_order: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return state = self.histories[model].setdefault( sid, {"entries": [], "final": None} ) while True: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return if state["final"] is not None: cls, _ = classify(state["final"], 0.0, 0.0, None) outcome = action(cls, len(state["entries"]) + 1) @@ -320,6 +411,9 @@ def run_model(self, model: str) -> None: self.states[model] = "budget-stopped" return if state["final"] is not None: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return self.archive(model, sid) if not completion_loaded: completion = self.completion_for(profile) @@ -327,6 +421,9 @@ def run_model(self, model: str) -> None: if self.spend.exhausted(): self.states[model] = "budget-stopped" return + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return spend = AttemptSpend(self.spend) def no_tool(arguments): @@ -355,6 +452,9 @@ def no_tool(arguments): self.say( f"{model} {sid} {n} {cls} {outcome} ${attempt_cost(record):.6f}" ) + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return if outcome == "pause-rerun": self.states[model] = "operator-paused" return @@ -373,6 +473,29 @@ def no_tool(arguments): self.say(f"{model}: {type(error).__name__}: {error}") def execute(self) -> int: + previous = {} + + def interrupt(signum, frame): + if self.interrupted.is_set(): + self.say( + "Forced interruption: an attempt under way, and its spend, " + "are then lost." + ) + os._exit(128 + signum) + self.interrupted.set() + self.say("Interrupted: attempts under way will finish and save.") + + try: + if threading.current_thread() is threading.main_thread(): + for sig in (signal.SIGINT, signal.SIGTERM): + previous[sig] = signal.signal(sig, interrupt) + return self._execute() + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + self.close() + + def _execute(self) -> int: threads = [ threading.Thread( target=self.run_model, args=(model,), name=f"no-tools-{model}" @@ -387,7 +510,10 @@ def execute(self) -> int: self.say(f"{model}: {self.states[model]}") self.say(f"stage spend ${self.spend.spent_usd:.6f} of ${NO_TOOLS_CAP:.2f}") return ( - 0 if all(self.states[model] == "complete" for model in self.models) else 1 + 0 + if not self.interrupted.is_set() + and all(self.states[model] == "complete" for model in self.models) + else 1 ) @@ -400,8 +526,8 @@ def main(argv=None) -> int: run.add_argument("--dry-run", action="store_true") args = parser.parse_args(argv) try: - stage = Stage(json.loads(args.design.read_text()), models=args.model) - return stage.dry_run() if args.dry_run else stage.execute() + with Stage(json.loads(args.design.read_text()), models=args.model) as stage: + return stage.dry_run() if args.dry_run else stage.execute() except (StateError, OSError, ValueError, KeyError, TypeError) as error: print(f"The no-tools stage cannot start: {error}") return 1 diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 1a2b388e..7c77f876 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -6,6 +6,7 @@ import io import json import os +import signal import sys import threading from pathlib import Path @@ -21,11 +22,17 @@ @pytest.fixture def setup(tmp_path): draw = ["h2", "h1"] + baseline = tmp_path / "baseline-exclusions.json" + baseline.write_text("[]\n") return { "design": { "board": "synthetic.csv", "engine": "policyengine-us 2.60.4", "draw_order": draw, + "baseline_exclusions": { + "path": str(baseline), + "sha256": nt.sha256_file(baseline), + }, "no_tools": { "runs": str(tmp_path / "no-tools"), "folder": "none", @@ -413,14 +420,222 @@ def fake(scenario, variables, tool_call, completion, spend, *, profile): assert plan.spend.spent_usd == pytest.approx(0.1) -def test_dry_run_marks_harness_and_budget_finals_as_blocked(setup): +@pytest.mark.parametrize( + "status,error,expected", + [ + ("budget_stop", None, "stopped"), + ("provider_error", "BadRequestError: synthetic", "stop"), + ], +) +def test_dry_run_marks_harness_and_budget_finals_as_blocked( + setup, status, error, expected +): plan = stage(setup, models=[nt.CHEAP[0]]) profile = nt.pilot.model_profile( nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION ) - payload = record("h2", profile, status="budget_stop", cost=0) + payload = record("h2", profile, status=status, error=error, cost=0) + payload["turns"] = [] payload["attempts"] = [] nt.write_json(final(plan), payload) + if hasattr(plan, "close"): + plan.close() resumed = stage(setup, models=[nt.CHEAP[0]]) assert resumed.dry_run() == 1 - assert "h2:blocked stopped at attempt 1" in resumed.out.getvalue() + assert f"h2:blocked {expected} at attempt 1" in resumed.out.getvalue() + + +def test_sigint_finishes_saves_and_stops_before_the_next_household(setup): + calls = [] + saved_handlers = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + continued = threading.Event() + + def interrupted(scenario, variables, tool_call, completion, spend, *, profile): + calls.append(scenario.id) + spend.add(0.17) + if scenario.id == "h2": + os.kill(os.getpid(), signal.SIGINT) + continued.wait(0.05) + return record(scenario.id, profile, cost=0.17) + + plan = stage(setup, interrupted, models=[nt.CHEAP[0]]) + try: + result = plan.execute() + except KeyboardInterrupt: + # Let the old implementation's uncoordinated worker finish before the + # regression assertion, so a red run leaves no worker behind. + continued.set() + for worker in threading.enumerate(): + if worker.name.startswith("no-tools-"): + worker.join(timeout=30) + result = 0 + assert result != 0, "SIGINT must give a coordinated nonzero exit" + assert calls == ["h2"] + assert load(final(plan))["totals"]["cost_usd"] == 0.17 + assert plan.states[nt.CHEAP[0]] == "interrupted" + assert not plan.stopped.is_set(), "interrupts must preserve the budget event" + assert not final(plan, sid="h1").exists() + assert all( + signal.getsignal(sig) == handler for sig, handler in saved_handlers.items() + ) + assert "finish and save" in plan.out.getvalue() + + +def test_stage_lock_refuses_a_second_invocation_and_releases_after_dry_run(setup): + first = stage(setup, models=[nt.CHEAP[0]]) + with pytest.raises(nt.StateError, match="another no-tools invocation"): + stage(setup, models=[nt.CHEAP[1]]) + assert first.dry_run() == 1 + second = stage(setup, models=[nt.CHEAP[1]]) + assert second.dry_run() == 1 + + +@pytest.mark.parametrize("fault", ["missing", "hash"]) +def test_baseline_exclusions_must_exist_and_hash_before_startup(setup, fault): + spec = setup["design"]["baseline_exclusions"] + if fault == "missing": + Path(spec["path"]).unlink() + else: + spec["sha256"] = "0" * 64 + with pytest.raises(nt.StateError, match="baseline exclusions"): + stage(setup) + + +def test_archive_identity_and_rerun_eligibility_are_checked_directly(setup): + first = paused(setup) + first.archive(nt.CHEAP[0], "h2") + path = final(first).parent / "attempts/h2-1.json" + payload = load(path) + payload["scenario_id"] = "h1" + path.write_text(json.dumps(payload)) + with pytest.raises(nt.StateError, match="household differs from archive name h2"): + stage(setup) + payload.update(scenario_id="h2", status="complete", error=None) + path.write_text(json.dumps(payload)) + with pytest.raises(nt.StateError, match="archived attempt was ineligible"): + stage(setup) + + +def test_two_provider_failures_then_harness_failure_has_no_standing_final(setup): + provider = Provider( + { + **plant(setup, "provider_error", "APIConnectionError", 1), + **plant(setup, "provider_error", "APIConnectionError", 2), + **plant(setup, "provider_error", "BadRequestError", 3), + } + ) + + def failure(*args, **kwargs): + result = provider(*args, **kwargs) + if result["error"] == "BadRequestError": + result["turns"] = [] + return result + + plan = stage(setup, failure, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert plan.states[nt.CHEAP[0]] == "harness-stopped" + assert [n for _, _, n in provider.calls] == [1, 2, 3] + assert len(load(final(plan))["attempts"]) == 2 + assert not plan.standing(nt.CHEAP[0], "h2") + assert not final(plan, sid="h1").exists() + + +@pytest.mark.parametrize( + "status,error,expected", + [ + ("provider_error", "BadRequestError", "stop"), + ("budget_stop", None, "stopped"), + ], +) +def test_stage_problems_requires_every_cheap_model_final_to_stand( + setup, status, error, expected, monkeypatch +): + plan = stage(setup) + assert plan.execute() == 0 + path = final(plan, model=nt.CHEAP[-1], sid="h1") + payload = load(path) + payload.update(status=status, error=error, turns=[]) + path.write_text(json.dumps(payload)) + monkeypatch.setattr( + nt.pilot, "load_board", lambda _: pytest.fail("helper loads board") + ) + problems = nt.stage_problems(setup["design"]) + assert any(nt.CHEAP[-1] in p and "h1" in p and expected in p for p in problems) + + +def test_stage_problems_reports_missing_finals_and_invalid_optional_history(setup): + plan = stage(setup) + assert plan.execute() == 0 + assert nt.stage_problems(setup["design"]) == [] + final(plan, sid="h1").unlink() + optional = plan.households("gpt-6-sol") / "attempts/h2-2.json" + optional.parent.mkdir(parents=True) + optional.write_text("{}") + problems = nt.stage_problems(setup["design"]) + assert any(nt.CHEAP[0] in p and "h1" in p and "missing" in p for p in problems) + assert any("attempt-number gap" in p and "gpt-6-sol" in p for p in problems) + + +def test_real_harness_checks_other_models_spend_before_the_next_turn( + setup, monkeypatch +): + from policybench import eval_no_tools + + monkeypatch.setattr(nt, "NO_TOOLS_CAP", 0.1) + for scenario in setup["board"]["scenarios"].values(): + scenario.country = "us" + + def prompt(scenario, variables, profile): + return "Synthetic household", { + "_board_prompt_sha256": "synthetic", + "_sent_board_prompt_sha256": "synthetic", + "_budget_ceiling": 100, + "_answer_tool": nt.pilot.answer_tool_for( + profile, variables, scenario.country + ), + "max_tokens": 100, + } + + monkeypatch.setattr(nt.pilot, "build_initial_request", prompt) + monkeypatch.setattr( + eval_no_tools, + "_extract_usage_metadata", + lambda response, *args: {"total_cost_usd": response.synthetic_cost}, + ) + entered = threading.Barrier(2) + requests = [] + models = [nt.CHEAP[0], nt.CHEAP[2]] + first_id = nt.pilot.model_profile(models[0], arm="none").model_id + + def completion(**kwargs): + requests.append(kwargs["model"]) + entered.wait(timeout=30) + if kwargs["model"] != first_id: + assert plan.stopped.wait(timeout=30), "other model must cross shared cap" + message = SimpleNamespace( + role="assistant", content="", tool_calls=None, function_call=None + ) + return SimpleNamespace( + choices=[SimpleNamespace(message=message, finish_reason="stop")], + synthetic_cost=0.11 if kwargs["model"] == first_id else 0.0, + id="synthetic-response", + ) + + plan = nt.Stage( + setup["design"], + models=models, + board=setup["board"], + run_household=nt.pilot.run_household, + completion_for=lambda profile: completion, + out=io.StringIO(), + ) + assert plan.execute() == 1 + assert len(requests) == 2 + for model in models: + payload = load(final(plan, model=model)) + assert payload["status"] == "budget_stop", payload.get("error") + assert len(payload["turns"]) == 1 + assert not final(plan, model=model, sid="h1").exists() + assert plan.spend.spent_usd == pytest.approx(0.11) From 5a9735b0e4984ca84013dffa49b305ad273f8f32 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 03:33:52 -0700 Subject: [PATCH 17/48] Confirmatory runner: require standing no-tools finals before calculation Calculator preflight calls the no-tools history validator and refuses an unfinished stage. Every drawn household must have a standing final for every required cheap model, and every saved history must pass the writer rules. Synthetic completed no-tools histories keep existing preflight tests meaningful. Require the calculator and no-tools runs roots to be disjoint after resolving their paths. Equal roots and either direction of containment are refused. Changes to the calculator runner are confined to preflight. All seven new preflight regressions failed before the change. The complete calculator test file passes 230 tests after the change, including those regressions. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 12 ++++ tests/test_confirmatory_run.py | 103 ++++++++++++++++++++++++++++----- 2 files changed, 102 insertions(+), 13 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index d4dd254b..3949f239 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2162,6 +2162,18 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: if missing: return [f"the design has no {', '.join(missing)}"] problems = [] + from confirmatory_no_tools import stage_problems + + problems.extend(stage_problems(design)) + try: + no_tools_root = Path(design["no_tools"]["runs"]).resolve() + calculator_root = Path(design["runs"]).resolve() + if no_tools_root.is_relative_to( + calculator_root + ) or calculator_root.is_relative_to(no_tools_root): + problems.append("the no-tools and calculator runs roots overlap") + except (KeyError, TypeError, OSError, ValueError) as error: + problems.append(f"the no-tools runs root is invalid: {error}") command = design["anchor_command"] if ( not isinstance(command, list) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 246b2533..0b39c5a6 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2173,10 +2173,8 @@ def test_the_preflight_names_a_missing_field(): assert cr.preflight({}, cr.CHEAP) == [f"the design has no {', '.join(cr.REQUIRED)}"] -def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( - world, tmp_path, monkeypatch -): - w = world(models=("gpt-6-luna",)) +def complete_preflight_design(w, tmp_path): + """Each calculator preflight starts after synthetic standing no-tools finals.""" pool = tmp_path / "pool.json" pool.write_text(json.dumps([{"id": s} for s in POOL])) design = dict( @@ -2184,8 +2182,36 @@ def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, treatment_harness=str(cr.ROOT), run_commit="0" * 40, - draw_order=list(reversed(w.draw_order)), + no_tools={ + "runs": str(tmp_path / "no-tools"), + "folder": "none", + "stage": "confirm", + }, ) + for model in cr.CHEAP: + for sid in design["draw_order"]: + cr.write_json( + Path(design["no_tools"]["runs"]) + / model + / "none/confirm/households" + / f"{sid}.json", + { + "scenario_id": sid, + "status": "complete", + "turns": [{}], + "totals": {"cost_usd": 0.0}, + "attempts": [], + }, + ) + return design + + +def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( + world, tmp_path, monkeypatch +): + w = world(models=("gpt-6-luna",)) + design = complete_preflight_design(w, tmp_path) + design["draw_order"] = list(reversed(w.draw_order)) monkeypatch.setattr(cr, "host_power", lambda: (True, False)) problems = cr.preflight(design, ("gpt-6-luna",)) text = "\n".join(problems) @@ -2194,22 +2220,73 @@ def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( assert "lid open False" in text design["draw_order"] = w.draw_order assert "the draw" not in "\n".join(cr.preflight(design, ("gpt-6-luna",))) + assert not any( + p.startswith("no-tools") for p in cr.preflight(design, ("gpt-6-luna",)) + ) def test_the_preflight_refuses_a_runs_folder_with_records(world, tmp_path): w = world(models=("gpt-6-luna",)).execute() - pool = tmp_path / "pool.json" - pool.write_text(json.dumps([{"id": s} for s in POOL])) - design = dict( - w.design, - eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, - treatment_harness=str(cr.ROOT), - run_commit="0" * 40, - ) + design = complete_preflight_design(w, tmp_path) text = "\n".join(cr.preflight(design, ("gpt-6-luna",))) assert "already holds records" in text and "already holds a host log" in text +@pytest.mark.parametrize( + "fault,expected", + [ + ("missing", "missing"), + ("harness", "stop"), + ("ceiling", "stopped"), + ("history", "attempts list"), + ], +) +def test_preflight_refuses_an_unfinished_no_tools_stage( + world, tmp_path, fault, expected +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + model, sid = cr.CHEAP[-1], w.draw_order[-1] + path = ( + Path(design["no_tools"]["runs"]) + / model + / "none/confirm/households" + / f"{sid}.json" + ) + payload = json.loads(path.read_text()) + if fault == "missing": + path.unlink() + else: + if fault == "history": + payload.pop("attempts") + else: + payload.update( + status="budget_stop" if fault == "ceiling" else "provider_error", + error="BadRequestError" if fault == "harness" else None, + turns=[], + ) + path.write_text(json.dumps(payload)) + text = "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert "no-tools" in text and model in text and sid in text and expected in text + + +@pytest.mark.parametrize("relationship", ["equal", "inside", "contains"]) +def test_preflight_refuses_overlapping_no_tools_and_calculator_roots( + world, tmp_path, relationship +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + calculator = Path(design["runs"]) + design["no_tools"]["runs"] = str( + { + "equal": calculator, + "inside": calculator / "no-tools", + "contains": calculator.parent, + }[relationship] + ) + assert any("roots overlap" in p for p in cr.preflight(design, (cr.CHEAP[0],))) + + def test_resume_writes_the_operator_s_note(tmp_path): design = tmp_path / "design.json" design.write_text(json.dumps({"runs": str(tmp_path / "runs")})) From c60086b05f77a8aaa49fa732033e9884e1d34988 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 05:21:39 -0700 Subject: [PATCH 18/48] Confirmatory runner: coordinate interruption and worker cleanup Archive moves and attempt admissions share a short lock with the first signal handler, so every admission after that handler returns is refused. Providers run outside the admission lock, and admitted attempts finish and save. Exceptional shutdown interrupts and joins every started worker before restoring signal handlers or releasing the invocation lock. A second signal writes a nonblocking stderr message and exits immediately without taking shared locks; every active unsaved attempt and its spend are lost, while saved finals and archives survive. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 74 ++++++++--- tests/test_confirmatory_no_tools.py | 197 ++++++++++++++++++++++++++++ 2 files changed, 253 insertions(+), 18 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index d165a950..ffaa42e5 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -6,6 +6,14 @@ its rerun; the final links every archive by number, hash, status and reason. Re-invocation resumes solely from those files and is an operator's approval to rerun a paused account failure. Native harness resume is never used. +An archive move or attempt is admitted when it takes the admission lock and +commits before the first signal handler finishes; everything else is refused. +The lock covers only an interruption check and a rename or an admission state +change. Admitted attempts finish and save outside the admission lock. +Exceptional shutdown interrupts and joins all started workers before restoring +signal handlers or releasing the invocation lock. A second signal immediately +uses a nonblocking stderr write and exits, losing every active unsaved attempt +and its spend; saved finals and archives survive. """ from __future__ import annotations @@ -225,6 +233,9 @@ def __init__( self._say_lock = threading.RLock() self.stopped = threading.Event() self.interrupted = threading.Event() + self._admission_lock = threading.Lock() + self._admitted_attempts = {} + self._threads = [] self.completion_for = completion_for self.run_household = ( run_household if run_household is not None else pilot.run_household @@ -360,8 +371,8 @@ def _dry_run(self) -> int: else 1 ) - def archive(self, model: str, sid: str) -> None: - """Move an eligible final unchanged, then extend its reconstructed links.""" + def archive(self, model: str, sid: str) -> bool: + """Atomically admit a prepared archive move, then extend its links.""" state = self.histories[model][sid] record, entries = state["final"], state["entries"] n = len(entries) + 1 @@ -369,13 +380,20 @@ def archive(self, model: str, sid: str) -> None: if action(cls, n) not in ("rerun", "pause-rerun"): raise StateError(f"{model}/{sid}: attempt {n} is ineligible for rerun") folder = self.households(model) + source = folder / f"{sid}.json" target = folder / "attempts" / f"{sid}-{n}.json" target.parent.mkdir(parents=True, exist_ok=True) if target.exists(): raise StateError(f"{target}: would overwrite an archived attempt") - os.replace(folder / f"{sid}.json", target) - entries.append(entry(target, record, n)) - state["final"] = None + link = entry(source, record, n) + link["file"] = f"attempts/{target.name}" + with self._admission_lock: + if self.interrupted.is_set(): + return False + os.replace(source, target) + state["final"] = None + entries.append(link) + return True def run_model(self, model: str) -> None: try: @@ -414,17 +432,21 @@ def run_model(self, model: str) -> None: if self.interrupted.is_set(): self.states[model] = "interrupted" return - self.archive(model, sid) + if not self.archive(model, sid): + self.states[model] = "interrupted" + return if not completion_loaded: completion = self.completion_for(profile) completion_loaded = True if self.spend.exhausted(): self.states[model] = "budget-stopped" return - if self.interrupted.is_set(): - self.states[model] = "interrupted" - return spend = AttemptSpend(self.spend) + with self._admission_lock: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return + self._admitted_attempts[model, sid] = len(state["entries"]) + 1 def no_tool(arguments): raise RuntimeError("the no-tools arm cannot call a tool") @@ -474,15 +496,24 @@ def no_tool(arguments): def execute(self) -> int: previous = {} + first_signal_seen = False def interrupt(signum, frame): - if self.interrupted.is_set(): - self.say( - "Forced interruption: an attempt under way, and its spend, " - "are then lost." - ) + nonlocal first_signal_seen + if first_signal_seen: + try: + os.set_blocking(2, False) + os.write( + 2, + b"Forced interruption: every attempt under way, and its spend, " + b"are lost; saved finals and archives survive.\n", + ) + except OSError: + pass os._exit(128 + signum) - self.interrupted.set() + first_signal_seen = True + with self._admission_lock: + self.interrupted.set() self.say("Interrupted: attempts under way will finish and save.") try: @@ -490,21 +521,28 @@ def interrupt(signum, frame): for sig in (signal.SIGINT, signal.SIGTERM): previous[sig] = signal.signal(sig, interrupt) return self._execute() + except BaseException: + with self._admission_lock: + self.interrupted.set() + for thread in self._threads: + if thread.ident is not None: + thread.join() + raise finally: for sig, handler in previous.items(): signal.signal(sig, handler) self.close() def _execute(self) -> int: - threads = [ + self._threads = [ threading.Thread( target=self.run_model, args=(model,), name=f"no-tools-{model}" ) for model in self.models ] - for thread in threads: + for thread in self._threads: thread.start() - for thread in threads: + for thread in self._threads: thread.join() for model in self.models: self.say(f"{model}: {self.states[model]}") diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 7c77f876..55e3d0c5 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -639,3 +639,200 @@ def completion(**kwargs): assert len(payload["turns"]) == 1 assert not final(plan, model=model, sid="h1").exists() assert plan.spend.spent_usd == pytest.approx(0.11) + + +def track_interrupt_return(monkeypatch): + """Observe the actual installed first-signal handler returning.""" + returned = threading.Event() + register = signal.signal + + def install(signum, handler): + if callable(handler) and handler.__name__ == "interrupt": + + def tracked(sig, frame): + handler(sig, frame) + returned.set() + + return register(signum, tracked) + return register(signum, handler) + + monkeypatch.setattr(signal, "signal", install) + return returned + + +@pytest.mark.parametrize("boundary", ["archive", "attempt"]) +def test_signal_between_check_and_action_refuses_new_admission( + setup, monkeypatch, boundary +): + """R1: a real SIGINT returns before the proposed action reaches admission.""" + if boundary == "archive": + paused(setup) + provider = Provider() + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + returned = track_interrupt_return(monkeypatch) + + def signal_and_wait(): + os.kill(os.getpid(), signal.SIGINT) + assert returned.wait(timeout=10), "the signal handler must finish" + + if boundary == "archive": + original = plan.archive + + def archive(*args): + signal_and_wait() + return original(*args) + + monkeypatch.setattr(plan, "archive", archive) + else: + original = nt.AttemptSpend + + def attempt_spend(*args): + signal_and_wait() + return original(*args) + + monkeypatch.setattr(nt, "AttemptSpend", attempt_spend) + assert plan.execute() == 1 + assert returned.is_set() + assert provider.calls == [], "R1: no provider attempt begins after interruption" + archives = plan.households(nt.CHEAP[0]) / "attempts" + assert not list(archives.glob("*.json")), "R1: no archive moves after interruption" + + +def test_failed_second_thread_start_keeps_invocation_lock_until_worker_finishes( + setup, monkeypatch +): + """R2: exceptional shutdown waits for every worker before closing its lock.""" + entered, release, failed_start, closed = (threading.Event() for _ in range(4)) + original_start = threading.Thread.start + original_close = nt.Stage.close + observed = {} + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + entered.set() + assert release.wait(timeout=10) + return record(scenario.id, profile) + + plan = stage(setup, provider) + + def start(thread): + if thread.name == f"no-tools-{nt.CHEAP[1]}": + assert entered.wait(timeout=10) + failed_start.set() + raise RuntimeError("synthetic second thread start failure") + return original_start(thread) + + def close(current): + original_close(current) + if current is plan: + closed.set() + + def observe(): + assert failed_start.wait(timeout=10) + observed["closed_while_live"] = closed.wait(timeout=0.2) + lock = (plan.root / ".confirmatory-no-tools.lock").open("a") + try: + import fcntl + + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + observed["held"] = True + else: + observed["held"] = False + finally: + lock.close() + release.set() + + watcher = threading.Thread(target=observe) + watcher.start() + monkeypatch.setattr(threading.Thread, "start", start) + monkeypatch.setattr(nt.Stage, "close", close) + try: + with pytest.raises(RuntimeError, match="second thread start"): + plan.execute() + finally: + release.set() + watcher.join(timeout=10) + for worker in threading.enumerate(): + if worker.name.startswith("no-tools-"): + worker.join(timeout=10) + assert observed["held"], "R2: a live worker must retain the invocation lock" + assert not observed["closed_while_live"], "R2: cleanup joins before unlocking" + assert plan.interrupted.is_set(), "R2: exceptional shutdown refuses further work" + assert plan._lock_file is None + + +def test_second_signal_exits_while_output_lock_is_held(setup, tmp_path): + """R4: forced termination is independent of the worker-shared output lock.""" + import select + import subprocess + import textwrap + + probe = tmp_path / "forced_exit.py" + probe.write_text( + textwrap.dedent("""\ + import io + import json + import os + import signal + import sys + import threading + import time + from types import SimpleNamespace + import confirmatory_no_tools as nt + + design = json.loads(sys.argv[1]) + draw = design['draw_order'] + board = { + 'scenarios': {sid: SimpleNamespace(id=sid) for sid in draw}, + 'variables': {sid: ['snap'] for sid in draw}, + } + held = threading.Event() + plan = nt.Stage(design, models=[nt.CHEAP[0]], board=board, + completion_for=lambda profile: None, out=io.StringIO()) + + def worker(model): + with plan._say_lock: + held.set() + time.sleep(30) + + plan.run_model = worker + + def signals(): + assert held.wait(10) + os.write(1, b'ready\\n') + os.kill(os.getpid(), signal.SIGINT) + time.sleep(0.1) + os.kill(os.getpid(), signal.SIGTERM) + + threading.Thread(target=signals, daemon=True).start() + plan.execute() + raise AssertionError('forced exit returned') + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + process = subprocess.Popen( + [sys.executable, str(probe), json.dumps(setup["design"])], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + try: + ready, _, _ = select.select([process.stdout], [], [], 90) + assert ready, "R4: subprocess must reach output contention before signals" + assert process.stdout.readline() == b"ready\n" + stdout, stderr = process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + pytest.fail("R4: the second signal blocked on the held output lock") + finally: + if process.poll() is None: + process.kill() + process.communicate() + assert process.returncode == 128 + signal.SIGTERM, (stdout, stderr) + assert b"every attempt under way" in stderr + assert b"saved finals and archives survive" in stderr From 248e777abddd83ff62637b8efc79eeb50a76d65d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 05:21:44 -0700 Subject: [PATCH 19/48] Confirmatory runner: validate every saved final and baseline inventory The shared history reader classifies every saved final, including optional models and households outside the draw, before initialization and calculator preflight accept its history. Preflight then requires every drawn cheap-model final to stand. Startup validates the hash-bound baseline as an object containing an exclusions list, with unique household and output cells identified by nonempty strings, before any provider call. The empty baseline fixture uses the report schema. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 33 ++++++++++++++++++++-- tests/test_confirmatory_no_tools.py | 43 ++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index ffaa42e5..ac189818 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -130,6 +130,12 @@ def history(folder: Path) -> tuple[dict[str, dict], float]: raise StateError(f"{path}: household differs from final name {sid}") if final["attempts"] != entries: raise StateError(f"{path}: attempts disagree with archived files") + try: + classify(final, 0.0, 0.0, None) + except (TypeError, ValueError, AttributeError, KeyError) as error: + raise StateError( + f"{path}: cannot classify saved final: {error}" + ) from error spent += attempt_cost(final) states[sid] = {"entries": entries, "final": final} return states, spent @@ -243,9 +249,32 @@ def __init__( self._lock_file = None try: baseline = design["baseline_exclusions"] - if sha256_file(Path(baseline["path"])) != baseline["sha256"]: + if not isinstance(baseline, dict) or not isinstance( + baseline.get("path"), str + ): + raise StateError("baseline exclusions need path and sha256") + baseline_path = Path(baseline["path"]) + if sha256_file(baseline_path) != baseline.get("sha256"): raise StateError("baseline exclusions do not hash as declared") - except (OSError, KeyError, TypeError) as error: + payload = json.loads(baseline_path.read_text()) + exclusions = ( + payload.get("exclusions") if isinstance(payload, dict) else None + ) + if not isinstance(exclusions, list): + raise StateError("baseline exclusions must contain an exclusions list") + cells = [] + for exclusion in exclusions: + if not isinstance(exclusion, dict) or any( + not isinstance(exclusion.get(key), str) or not exclusion[key] + for key in ("scenario_id", "variable") + ): + raise StateError( + "baseline exclusion needs household and output strings" + ) + cells.append((exclusion["scenario_id"], exclusion["variable"])) + if len(cells) != len(set(cells)): + raise StateError("baseline exclusions repeat a cell") + except (OSError, KeyError, TypeError, ValueError) as error: raise StateError(f"baseline exclusions: {error}") from error self.root.mkdir(parents=True, exist_ok=True) self._lock_file = (self.root / ".confirmatory-no-tools.lock").open("a") diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 55e3d0c5..e034d000 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -23,7 +23,7 @@ def setup(tmp_path): draw = ["h2", "h1"] baseline = tmp_path / "baseline-exclusions.json" - baseline.write_text("[]\n") + baseline.write_text('{"exclusions": []}\n') return { "design": { "board": "synthetic.csv", @@ -836,3 +836,44 @@ def signals(): assert process.returncode == 128 + signal.SIGTERM, (stdout, stderr) assert b"every attempt under way" in stderr assert b"saved finals and archives survive" in stderr + + +@pytest.mark.parametrize("category", ["optional", "outside-draw"]) +def test_stage_problems_classifies_every_saved_final(setup, category): + """R3: optional and off-draw finals use the initialization classifier.""" + plan = stage(setup) + assert plan.execute() == 0 + model = "gpt-6-sol" if category == "optional" else nt.CHEAP[0] + sid = "h2" if category == "optional" else "outside-draw" + profile = nt.pilot.model_profile( + model, arm="none", prompt_version=nt.PROMPT_VERSION + ) + payload = record(sid, profile) + payload.update(attempts=[], turns=[None]) + path = plan.households(model) / f"{sid}.json" + nt.write_json(path, payload) + with pytest.raises((nt.StateError, AttributeError), match="get"): + stage(setup) + problems = nt.stage_problems(setup["design"]) + assert any(model in problem and "get" in problem for problem in problems), ( + "R3: every final refused by initialization must fail stage preflight", + problems, + ) + + +@pytest.mark.parametrize( + "payload", ["[]", "{invalid json", '{"exclusions": [{"scenario_id": "h2"}]}'] +) +def test_baseline_schema_is_usable_before_any_provider_call(setup, payload): + """R5: correctly hashed malformed inventories refuse without provider calls.""" + spec = setup["design"]["baseline_exclusions"] + path = Path(spec["path"]) + path.write_text(payload) + spec["sha256"] = nt.sha256_file(path) + provider = Provider() + try: + with pytest.raises(nt.StateError, match="baseline exclusion"): + plan = stage(setup, provider) + plan.close() + finally: + assert provider.calls == [], "R5: schema checks precede provider calls" From 4ce7f8ef93209bb23c9c8f4eba8f8a2bbcf3ab9f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 05:23:44 -0700 Subject: [PATCH 20/48] Confirmatory runner: keep exceptional cleanup signal-safe Exceptional cleanup sets interruption directly before joining every started worker. Keeping the main thread outside the admission lock lets a first signal safely acquire that lock if it interrupts cleanup. The invocation lock remains held until every worker finishes and saves. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index ac189818..dcd52029 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -551,8 +551,9 @@ def interrupt(signum, frame): previous[sig] = signal.signal(sig, interrupt) return self._execute() except BaseException: - with self._admission_lock: - self.interrupted.set() + # The main thread must remain outside admission here: a first signal + # can interrupt cleanup and its handler must take that same lock. + self.interrupted.set() for thread in self._threads: if thread.ident is not None: thread.join() From 2f795eae44a9dd65514f00ccf5b60fddbeb052c3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 23:54:32 -0400 Subject: [PATCH 21/48] Confirmatory runner: defer signals and track worker completion Record signals and wake a pipe watcher with a handler that takes no lock and raises nothing. The watcher sets interruption under admission and announces admitted work after releasing the lock. Archive moves and attempts commit under the same lock before interruption; admitted attempts finish and save outside it. Track worker began and done events independently of Thread identity and join bookkeeping. Escaping exceptions retain handlers and the invocation lock through completion, allow ten seconds for uncertain startup, tolerate repeated wait exceptions and preserve the original exception. Late targets return before file access. Apply completion tracking and safe late entry to the watcher as well. A caught model exception marks that model error and leaves global interruption unset. A second signal exits with 128 plus its number and preserves saved finals, archives and their recorded spend. Every unsaved attempt then under way is lost with its spend. A signal before handler restoration requires graceful exit 1 even when all workers completed. Run signal and cleanup regressions in bounded subprocesses, synchronize through events and use a thirty-second forced-exit deadline. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 199 +++- tests/test_confirmatory_no_tools.py | 1484 ++++++++++++++++++++++++--- 2 files changed, 1519 insertions(+), 164 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index dcd52029..0f19a730 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -6,14 +6,22 @@ its rerun; the final links every archive by number, hash, status and reason. Re-invocation resumes solely from those files and is an operator's approval to rerun a paused account failure. Native harness resume is never used. -An archive move or attempt is admitted when it takes the admission lock and -commits before the first signal handler finishes; everything else is refused. -The lock covers only an interruption check and a rename or an admission state -change. Admitted attempts finish and save outside the admission lock. -Exceptional shutdown interrupts and joins all started workers before restoring -signal handlers or releasing the invocation lock. A second signal immediately -uses a nonblocking stderr write and exits, losing every active unsaved attempt -and its spend; saved finals and archives survive. +An archive move or attempt is admitted when its interruption check and commit +under the admission lock precede the watcher setting the interrupt under that +same lock. After that interrupt is set, no archive move or attempt is admitted. +The lock covers only the check and rename or admission state change; admitted +attempts finish and save outside it. The first SIGINT or SIGTERM only records +its number and wakes the watcher through a nonblocking pipe. The handler takes +no lock and raises nothing. The watcher sets the interrupt, releases admission, +and then announces that attempts already admitted will finish and save. +An exception escaping execute interrupts and waits for script-owned worker +completion events before restoring handlers or releasing the invocation lock. +An exception caught in a model thread instead marks that model error without +setting the interrupt. A second signal immediately uses a nonblocking stderr +write and exits, losing every unsaved attempt then under way with its spend; +saved finals, archives and their recorded spend survive. Any signal before +handler restoration requires exit 1 on graceful completion, even if every +worker completed. """ from __future__ import annotations @@ -27,6 +35,7 @@ import signal import sys import threading +import time from pathlib import Path # confirmatory_run pins the harness and policybench to this checkout. @@ -44,6 +53,7 @@ ) NO_TOOLS_CAP = 5.0 +START_GRACE_SECONDS = 10 class StateError(ValueError): @@ -525,54 +535,189 @@ def no_tool(arguments): def execute(self) -> int: previous = {} - first_signal_seen = False + signals = [] + closing = False + watcher = None + watcher_done = threading.Event() + watcher_began = threading.Event() + watcher_started = False + failure = None + read_fd = write_fd = None + self._started_workers = [] + self._starting_worker = None + self._worker_began = {} + self._worker_done = {} + + def wake_watcher(): + try: + os.write(write_fd, b"!") + except OSError: + pass def interrupt(signum, frame): - nonlocal first_signal_seen - if first_signal_seen: + signals.append(signum) + if len(signals) >= 2: try: os.set_blocking(2, False) os.write( 2, - b"Forced interruption: every attempt under way, and its spend, " - b"are lost; saved finals and archives survive.\n", + b"Forced interruption: every unsaved attempt then under way " + b"is lost, with its spend; saved finals, archives and the " + b"spend they record survive.\n", ) except OSError: pass os._exit(128 + signum) - first_signal_seen = True - with self._admission_lock: - self.interrupted.set() - self.say("Interrupted: attempts under way will finish and save.") + wake_watcher() + + def stop_admission(): + while True: + try: + with self._admission_lock: + self.interrupted.set() + return + except BaseException: + continue + + def watch_signals(): + watcher_began.set() + notified = False + try: + if closing: + return + while True: + os.read(read_fd, 1) + if closing: + return + if signals and not notified: + with self._admission_lock: + self.interrupted.set() + notified = True + self.say( + "Interrupted: attempts already admitted " + "will finish and save." + ) + finally: + watcher_done.set() try: + read_fd, write_fd = os.pipe() + os.set_blocking(write_fd, False) if threading.current_thread() is threading.main_thread(): for sig in (signal.SIGINT, signal.SIGTERM): previous[sig] = signal.signal(sig, interrupt) - return self._execute() - except BaseException: - # The main thread must remain outside admission here: a first signal - # can interrupt cleanup and its handler must take that same lock. - self.interrupted.set() - for thread in self._threads: - if thread.ident is not None: - thread.join() + watcher = threading.Thread( + target=watch_signals, name="no-tools-signal-watcher" + ) + watcher.start() + watcher_started = True + result = self._execute() + except BaseException as error: + failure = error + # SIGINT/SIGTERM handlers never raise, so production waits are not + # interrupted by them. Tolerant waits defend against exceptions from + # elsewhere (for example a test SIGUSR1 handler raising KeyboardInterrupt). + stop_admission() + self._wait_for_workers() raise finally: + closing = True + if write_fd is not None: + wake_watcher() + shutdown_error = None + if watcher is not None: + # start() can raise after native startup, even before ident is set. + # A watcher beginning after grace sees closing before reading a pipe. + deadline = time.monotonic() + START_GRACE_SECONDS + while not watcher_started and not watcher_began.is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + watcher_began.wait(remaining) + except BaseException as error: + if shutdown_error is None: + shutdown_error = error + if watcher_started or watcher_began.is_set(): + while True: + try: + watcher_done.wait() + watcher.join() + break + except BaseException as error: + if shutdown_error is None: + shutdown_error = error + if failure is None and shutdown_error is not None: + stop_admission() + for fd in (read_fd, write_fd): + if fd is not None: + os.close(fd) for sig, handler in previous.items(): signal.signal(sig, handler) self.close() + if failure is None and shutdown_error is not None: + raise shutdown_error + return 1 if signals else result + + def _wait_for_workers(self) -> None: + """Retain protection through completion despite exceptional wait interruptions. + + The installed SIGINT and SIGTERM handlers never raise. These tolerant + waits defend against an exception from elsewhere, such as a test-only + SIGUSR1 handler that raises KeyboardInterrupt, and preserve the original + exception escaping execute. + """ + workers = list(self._started_workers) + starting = self._starting_worker + if starting is not None and starting not in workers: + model = self.models[self._threads.index(starting)] + deadline = time.monotonic() + START_GRACE_SECONDS + while not self._worker_began[model].is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + self._worker_began[model].wait(remaining) + except BaseException: + continue + if self._worker_began[model].is_set(): + workers.append(starting) + for thread in workers: + model = self.models[self._threads.index(thread)] + while True: + try: + self._worker_done[model].wait() + thread.join() + break + except BaseException: + continue + + def _run_worker(self, model: str) -> None: + self._worker_began[model].set() + try: + if self.interrupted.is_set(): + self.states[model] = "interrupted" + return + self.run_model(model) + finally: + self._worker_done[model].set() def _execute(self) -> int: + self._worker_began = {model: threading.Event() for model in self.models} + self._worker_done = {model: threading.Event() for model in self.models} self._threads = [ threading.Thread( - target=self.run_model, args=(model,), name=f"no-tools-{model}" + target=self._run_worker, args=(model,), name=f"no-tools-{model}" ) for model in self.models ] for thread in self._threads: + self._starting_worker = thread thread.start() - for thread in self._threads: + self._started_workers.append(thread) + self._starting_worker = None + for model, thread in zip(self.models, self._threads): + self._worker_done[model].wait() thread.join() for model in self.models: self.say(f"{model}: {self.states[model]}") diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index e034d000..08620921 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -445,42 +445,48 @@ def test_dry_run_marks_harness_and_budget_finals_as_blocked( assert f"h2:blocked {expected} at attempt 1" in resumed.out.getvalue() -def test_sigint_finishes_saves_and_stops_before_the_next_household(setup): - calls = [] - saved_handlers = { - sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) - } - continued = threading.Event() +def test_sigint_finishes_saves_and_stops_before_the_next_household(setup, tmp_path): + run_no_tools_probe( + setup, + tmp_path, + "finish_admitted_attempt", + r""" + calls = [] + saved_handlers = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } - def interrupted(scenario, variables, tool_call, completion, spend, *, profile): - calls.append(scenario.id) - spend.add(0.17) - if scenario.id == "h2": - os.kill(os.getpid(), signal.SIGINT) - continued.wait(0.05) - return record(scenario.id, profile, cost=0.17) - plan = stage(setup, interrupted, models=[nt.CHEAP[0]]) - try: + def interrupted( + scenario, variables, tool_call, completion, spend, *, profile + ): + calls.append(scenario.id) + spend.add(0.17) + if scenario.id == "h2": + os.kill(os.getpid(), signal.SIGINT) + assert plan.interrupted.wait(30), ( + "SIGINT must set the interruption event" + ) + return record(scenario.id, profile, cost=0.17) + + + plan = stage(setup, interrupted, models=[nt.CHEAP[0]]) result = plan.execute() - except KeyboardInterrupt: - # Let the old implementation's uncoordinated worker finish before the - # regression assertion, so a red run leaves no worker behind. - continued.set() - for worker in threading.enumerate(): - if worker.name.startswith("no-tools-"): - worker.join(timeout=30) - result = 0 - assert result != 0, "SIGINT must give a coordinated nonzero exit" - assert calls == ["h2"] - assert load(final(plan))["totals"]["cost_usd"] == 0.17 - assert plan.states[nt.CHEAP[0]] == "interrupted" - assert not plan.stopped.is_set(), "interrupts must preserve the budget event" - assert not final(plan, sid="h1").exists() - assert all( - signal.getsignal(sig) == handler for sig, handler in saved_handlers.items() + assert result != 0, "SIGINT must give a coordinated nonzero exit" + assert calls == ["h2"] + assert load(final(plan))["totals"]["cost_usd"] == 0.17 + assert plan.states[nt.CHEAP[0]] == "interrupted" + assert not plan.stopped.is_set(), ( + "interrupts must preserve the budget event" + ) + assert not final(plan, sid="h1").exists() + assert all( + signal.getsignal(sig) == handler + for sig, handler in saved_handlers.items() + ) + assert "finish and save" in plan.out.getvalue() + """, ) - assert "finish and save" in plan.out.getvalue() def test_stage_lock_refuses_a_second_invocation_and_releases_after_dry_run(setup): @@ -662,110 +668,146 @@ def tracked(sig, frame): @pytest.mark.parametrize("boundary", ["archive", "attempt"]) def test_signal_between_check_and_action_refuses_new_admission( - setup, monkeypatch, boundary + setup, tmp_path, boundary ): - """R1: a real SIGINT returns before the proposed action reaches admission.""" - if boundary == "archive": - paused(setup) - provider = Provider() - plan = stage(setup, provider, models=[nt.CHEAP[0]]) - returned = track_interrupt_return(monkeypatch) + """R1: the watcher interrupts before the proposed action reaches admission.""" + run_no_tools_probe( + setup, + tmp_path, + f"refuse_{boundary}_admission", + f" boundary = {boundary!r}\n" + + r""" + if boundary == "archive": + paused(setup) + provider = Provider() + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + returned = track_interrupt_return(monkeypatch) + + + def signal_and_wait(): + os.kill(os.getpid(), signal.SIGINT) + assert returned.wait(30), "the signal handler must finish" + assert plan.interrupted.wait(30), ( + "the watcher must set the interrupt before admission" + ) - def signal_and_wait(): - os.kill(os.getpid(), signal.SIGINT) - assert returned.wait(timeout=10), "the signal handler must finish" - if boundary == "archive": - original = plan.archive + if boundary == "archive": + original = plan.archive - def archive(*args): - signal_and_wait() - return original(*args) + def archive(*args): + signal_and_wait() + return original(*args) - monkeypatch.setattr(plan, "archive", archive) - else: - original = nt.AttemptSpend + monkeypatch.setattr(plan, "archive", archive) + else: + original = nt.AttemptSpend - def attempt_spend(*args): - signal_and_wait() - return original(*args) + def attempt_spend(*args): + signal_and_wait() + return original(*args) - monkeypatch.setattr(nt, "AttemptSpend", attempt_spend) - assert plan.execute() == 1 - assert returned.is_set() - assert provider.calls == [], "R1: no provider attempt begins after interruption" - archives = plan.households(nt.CHEAP[0]) / "attempts" - assert not list(archives.glob("*.json")), "R1: no archive moves after interruption" + monkeypatch.setattr(nt, "AttemptSpend", attempt_spend) + assert plan.execute() == 1 + assert returned.is_set() + assert provider.calls == [], ( + "R1: no provider attempt begins after interruption" + ) + archives = plan.households(nt.CHEAP[0]) / "attempts" + assert not list(archives.glob("*.json")), ( + "R1: no archive moves after interruption" + ) + """, + ) def test_failed_second_thread_start_keeps_invocation_lock_until_worker_finishes( - setup, monkeypatch + setup, tmp_path ): """R2: exceptional shutdown waits for every worker before closing its lock.""" - entered, release, failed_start, closed = (threading.Event() for _ in range(4)) - original_start = threading.Thread.start - original_close = nt.Stage.close - observed = {} - - def provider(scenario, variables, tool_call, completion, spend, *, profile): - entered.set() - assert release.wait(timeout=10) - return record(scenario.id, profile) - - plan = stage(setup, provider) - - def start(thread): - if thread.name == f"no-tools-{nt.CHEAP[1]}": - assert entered.wait(timeout=10) - failed_start.set() - raise RuntimeError("synthetic second thread start failure") - return original_start(thread) - - def close(current): - original_close(current) - if current is plan: - closed.set() - - def observe(): - assert failed_start.wait(timeout=10) - observed["closed_while_live"] = closed.wait(timeout=0.2) - lock = (plan.root / ".confirmatory-no-tools.lock").open("a") - try: - import fcntl + run_no_tools_probe( + setup, + tmp_path, + "failed_second_start", + r""" + import fcntl + + entered, release, failed_start, closed = ( + threading.Event() for _ in range(4) + ) + original_start = threading.Thread.start + original_close = nt.Stage.close + observed = {} + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + entered.set() + assert release.wait(timeout=30) + return record(scenario.id, profile) + + + plan = stage(setup, provider) + monkeypatch.setattr(nt, "START_GRACE_SECONDS", 0.05, raising=False) + + + def start(thread): + if thread.name == f"no-tools-{nt.CHEAP[1]}": + assert entered.wait(timeout=30) + failed_start.set() + raise RuntimeError("synthetic second thread start failure") + return original_start(thread) + + def close(current): + original_close(current) + if current is plan: + closed.set() + + + def observe(): + assert failed_start.wait(timeout=30) + observed["closed_while_live"] = closed.wait(timeout=0.2) + lock = (plan.root / ".confirmatory-no-tools.lock").open("a") try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError: - observed["held"] = True - else: - observed["held"] = False + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + observed["held"] = True + else: + observed["held"] = False + finally: + lock.close() + release.set() + + + watcher = threading.Thread(target=observe) + watcher.start() + monkeypatch.setattr(threading.Thread, "start", start) + monkeypatch.setattr(nt.Stage, "close", close) + try: + with pytest.raises(RuntimeError, match="second thread start"): + plan.execute() finally: - lock.close() release.set() - - watcher = threading.Thread(target=observe) - watcher.start() - monkeypatch.setattr(threading.Thread, "start", start) - monkeypatch.setattr(nt.Stage, "close", close) - try: - with pytest.raises(RuntimeError, match="second thread start"): - plan.execute() - finally: - release.set() - watcher.join(timeout=10) - for worker in threading.enumerate(): - if worker.name.startswith("no-tools-"): - worker.join(timeout=10) - assert observed["held"], "R2: a live worker must retain the invocation lock" - assert not observed["closed_while_live"], "R2: cleanup joins before unlocking" - assert plan.interrupted.is_set(), "R2: exceptional shutdown refuses further work" - assert plan._lock_file is None + watcher.join(timeout=30) + assert not watcher.is_alive(), "R2: independent lock probe finishes" + assert observed["held"], "R2: a live worker must retain the invocation lock" + assert not observed["closed_while_live"], ( + "R2: cleanup joins before unlocking" + ) + assert plan.interrupted.is_set(), ( + "R2: exceptional shutdown refuses further work" + ) + assert plan._lock_file is None + assert load(final(plan))["status"] == "complete", ( + "R2: admitted worker saves before cleanup returns" + ) + """, + ) def test_second_signal_exits_while_output_lock_is_held(setup, tmp_path): """R4: forced termination is independent of the worker-shared output lock.""" - import select - import subprocess import textwrap probe = tmp_path / "forced_exit.py" @@ -802,7 +844,7 @@ def signals(): assert held.wait(10) os.write(1, b'ready\\n') os.kill(os.getpid(), signal.SIGINT) - time.sleep(0.1) + assert plan.interrupted.wait(30) os.kill(os.getpid(), signal.SIGTERM) threading.Thread(target=signals, daemon=True).start() @@ -814,28 +856,17 @@ def signals(): env["PYTHONPATH"] = os.pathsep.join( [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] ) - process = subprocess.Popen( - [sys.executable, str(probe), json.dumps(setup["design"])], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"])], env=env, + ready_marker=b"ready\n", ) - try: - ready, _, _ = select.select([process.stdout], [], [], 90) - assert ready, "R4: subprocess must reach output contention before signals" - assert process.stdout.readline() == b"ready\n" - stdout, stderr = process.communicate(timeout=5) - except subprocess.TimeoutExpired: - process.kill() - stdout, stderr = process.communicate() - pytest.fail("R4: the second signal blocked on the held output lock") - finally: - if process.poll() is None: - process.kill() - process.communicate() - assert process.returncode == 128 + signal.SIGTERM, (stdout, stderr) - assert b"every attempt under way" in stderr - assert b"saved finals and archives survive" in stderr + stdout, stderr = result.stdout, result.stderr + assert b"ready\n" in stdout, (stdout, stderr) + assert result.returncode == 128 + signal.SIGTERM, (stdout, stderr) + assert b"every unsaved attempt then under way is lost, with its spend" in stderr + assert b"saved finals, archives and the spend they record survive" in stderr @pytest.mark.parametrize("category", ["optional", "outside-draw"]) @@ -877,3 +908,1182 @@ def test_baseline_schema_is_usable_before_any_provider_call(setup, payload): plan.close() finally: assert provider.calls == [], "R5: schema checks precede provider calls" + + +def run_no_tools_probe(setup, tmp_path, name, body): + """Bound a signal or worker-shutdown regression to its own process.""" + import textwrap + + prelude = r""" + import io + import json + import os + import runpy + import signal + import sys + import threading + from types import SimpleNamespace + import pytest + + helpers = runpy.run_path(sys.argv[2]) + for helper in ( + "nt", + "record", + "Provider", + "stage", + "final", + "load", + "paused", + "track_interrupt_return", + ): + globals()[helper] = helpers[helper] + design = json.loads(sys.argv[1]) + setup = { + "design": design, + "board": { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + }, + } + monkeypatch = pytest.MonkeyPatch() + """ + probe = tmp_path / f"{name}.py" + probe.write_text(textwrap.dedent(prelude) + "\n" + textwrap.dedent(body)) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), str(Path(__file__))], + env=env, + ) + assert result.returncode == 0, (result.stdout, result.stderr) + + +def _run_cleanup_regression(setup, tmp_path, case): + """Run every exceptional-cleanup signal probe behind a process deadline.""" + import textwrap + + probe = tmp_path / "cleanup_regression.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import subprocess + import sys + import threading + import time + from types import SimpleNamespace + import confirmatory_no_tools as nt + + nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + print("cleanup-regression-ready", flush=True) + design, case = json.loads(sys.argv[1]), sys.argv[2] + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + model = nt.CHEAP[0] + entered, release, saved, worker_returned = ( + threading.Event() for _ in range(4) + ) + cleanup_entered, closed = threading.Event(), threading.Event() + handler_returned = threading.Event() + order, observed, calls = [], {}, [] + original_handlers = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_start = threading.Thread.start + original_signal = signal.signal + original_close = nt.Stage.close + original_write = nt.write_json + main_ident = threading.get_ident() + nt.START_GRACE_SECONDS = 0.05 + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + calls.append((profile.name, scenario.id)) + entered.set() + assert release.wait(20), "cleanup rule: release the admitted worker" + spend.add(0.1) + return { + "scenario_id": scenario.id, + "model": profile.name, + "arm": profile.arm, + "status": "complete", + "error": None, + "turns": [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": 0.1}, + } + + + models = list(nt.CHEAP[:2]) if case == "condition-signal" else [model] + plan = nt.Stage( + design, + models=models, + board=board, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) + original_model = plan.run_model + + + def run_model(name): + try: + return original_model(name) + finally: + worker_returned.set() + + + plan.run_model = run_model + + + def write(path, payload, *args, **kwargs): + result = original_write(path, payload, *args, **kwargs) + if path == plan.households(model) / "h2.json": + order.append("saved") + saved.set() + return result + + + nt.write_json = write + + + def close(current): + if current is plan and current._lock_file is not None: + order.append("close") + closed.set() + return original_close(current) + + + nt.Stage.close = close + + + def register(sig, handler): + if sig in original_handlers and handler == original_handlers[sig]: + order.append("restore:" + signal.Signals(sig).name) + elif sig in original_handlers and callable(handler): + original = handler + + def tracked(signum, frame): + original(signum, frame) + handler_returned.set() + + handler = tracked + return original_signal(sig, handler) + + + signal.signal = register + + + def lock_held(): + # A separate process opens the lock independently of the worker. + probe = ( + "import fcntl,sys; f=open(sys.argv[1],'a'); " + "\ntry: fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB)" + "\nexcept BlockingIOError: print('held')" + "\nelse: print('released')" + ) + result = subprocess.run( + [ + sys.executable, + "-c", + probe, + str(plan.root / ".confirmatory-no-tools.lock"), + ], + capture_output=True, + text=True, + timeout=10, + check=True, + ) + return result.stdout.strip() == "held" + + + if case == "condition-signal": + + class InterruptInsideSet(threading.Event): + def set(self): + if threading.get_ident() == main_ident and not hasattr( + self, "injected" + ): + self.injected = True + with self._cond: + cleanup_entered.set() + os.kill(os.getpid(), signal.SIGINT) + assert handler_returned.wait(10), ( + "cleanup rule: handler returns inside Event.set" + ) + self._flag = True + self._cond.notify_all() + return + super().set() + + plan.interrupted = InterruptInsideSet() + + def start(thread): + if thread.name == "no-tools-" + nt.CHEAP[1]: + assert entered.wait(10) + raise RuntimeError("synthetic second start failure") + return original_start(thread) + + def observe(): + assert cleanup_entered.wait(10) + assert handler_returned.wait(10) + observed["held"] = lock_held() + observed["closed_while_live"] = closed.is_set() + release.set() + + actor = threading.Thread(target=observe, daemon=True) + actor.start() + threading.Thread.start = start + try: + plan.execute() + except RuntimeError as error: + assert str(error) == "synthetic second start failure" + observed["propagated"] = True + propagated_error = error + else: + raise AssertionError( + "cleanup rule: original start exception propagates" + ) + actor.join(10) + assert handler_returned.is_set(), ( + "cleanup rule: SIGINT handler returns inside Event.set" + ) + assert observed.get("held"), ( + "cleanup rule: invocation lock survives Event.set signal" + ) + assert not observed.get("closed_while_live"), ( + "cleanup rule: admitted worker saves before close" + ) + assert plan.interrupted.is_set() + + elif case in ("join-once", "join-twice"): + raised = [] + original_set = plan.interrupted.set + + def interrupt_cleanup(): + cleanup_entered.set() + return original_set() + + plan.interrupted.set = interrupt_cleanup + + def injected(sig, frame): + raised.append(len(raised) + 1) + raise KeyboardInterrupt( + "synthetic wait interruption " + str(len(raised)) + ) + + signal.signal(signal.SIGUSR1, injected) + + def main_in_worker_wait(): + frame = sys._current_frames().get(main_ident) + while frame is not None: + if frame.f_code.co_name == "_wait_for_tstate_lock": + return True + if frame.f_code.co_name == "wait": + event = frame.f_locals.get("self") + if event in getattr(plan, "_worker_done", {}).values(): + return True + frame = frame.f_back + return False + + def await_wait(cleanup=False): + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if closed.is_set(): + return False + if ( + not cleanup or cleanup_entered.is_set() + ) and main_in_worker_wait(): + return True + time.sleep(0.001) + return False + + def observe(): + assert entered.wait(10) + assert await_wait(), ( + "cleanup rule: main thread reaches a worker wait" + ) + os.kill(os.getpid(), signal.SIGUSR1) + if case == "join-twice": + if await_wait(cleanup=True): + os.kill(os.getpid(), signal.SIGUSR1) + # The original implementation closes early after poisoning Thread's + # tstate lock. Give it that opportunity while the provider is held. + observed["closed_while_live"] = closed.wait(0.1) + observed["held"] = lock_held() + release.set() + + actor = threading.Thread(target=observe, daemon=True) + actor.start() + try: + plan.execute() + except KeyboardInterrupt as error: + observed["exception"] = str(error) + order.append("propagated") + else: + raise AssertionError("cleanup rule: wait exception must propagate") + finally: + release.set() + actor.join(15) + assert worker_returned.wait(10), ( + "cleanup rule: worker finishes after injected wait exception" + ) + assert observed.get("exception") == "synthetic wait interruption 1", ( + "cleanup rule: preserve original exception" + ) + expected = 2 if case == "join-twice" else 1 + assert len(raised) == expected, ( + "cleanup rule: repeat interruption exercises cleanup wait" + ) + assert not observed.get("closed_while_live"), ( + "cleanup rule: close waits for worker done" + ) + assert observed.get("held"), ( + "cleanup rule: invocation lock stays held through worker done" + ) + + elif case in ("start-after", "start-before"): + + def start(thread): + if thread.name == "no-tools-" + model: + if case == "start-after": + original_start(thread) + assert entered.wait(10) + raise RuntimeError("synthetic start interruption") + return original_start(thread) + + def observe(): + assert entered.wait(10) + observed["closed_while_live"] = closed.wait(0.1) + observed["held"] = lock_held() + release.set() + + actor = None + if case == "start-after": + actor = threading.Thread(target=observe, daemon=True) + actor.start() + threading.Thread.start = start + began = time.monotonic() + try: + plan.execute() + except RuntimeError as error: + observed["exception"] = str(error) + order.append("propagated") + else: + raise AssertionError( + "cleanup rule: failed start exception must propagate" + ) + if actor is not None: + actor.join(15) + assert not observed.get("closed_while_live"), ( + "cleanup rule: partially started worker saves before close" + ) + assert observed.get("held"), ( + "cleanup rule: partially started worker retains invocation lock" + ) + else: + elapsed = time.monotonic() - began + assert elapsed >= nt.START_GRACE_SECONDS * 0.9, ( + "cleanup rule: unstarted worker receives start grace" + ) + assert not calls and not entered.is_set(), ( + "cleanup rule: failed native start touches no worker files" + ) + assert not list( + plan.root.glob("*/none/confirmatory/households/*.json") + ) + assert observed.get("exception") == "synthetic start interruption" + + else: + raise AssertionError(case) + + if case != "start-before": + assert saved.is_set(), "cleanup rule: admitted worker saves its final" + assert order.index("saved") < order.index("close"), ( + "cleanup rule: save precedes close" + ) + for sig in (signal.SIGINT, signal.SIGTERM): + assert order.index("saved") < order.index( + "restore:" + signal.Signals(sig).name + ), "cleanup rule: save precedes handler restoration" + assert plan._lock_file is None, ( + "cleanup rule: invocation lock eventually closes" + ) + assert all( + signal.getsignal(sig) == old for sig, old in original_handlers.items() + ), "cleanup rule: handlers eventually restore" + print(json.dumps({"case": case, "order": order, "observed": observed})) + if case == "condition-signal": + raise propagated_error + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), case], + env=env, + ready_marker=b"cleanup-regression-ready\n", + text=True, + ) + assert "cleanup-regression-ready\n" in result.stdout, ( + "cleanup rule: subprocess reaches readiness", + result.stdout, + result.stderr, + ) + assert result.returncode == (1 if case == "condition-signal" else 0), ( + result.stdout, + result.stderr, + ) + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def test_signal_inside_cleanup_event_condition_keeps_worker_protected(setup, tmp_path): + """R10.1: a real signal inside Event.set cannot reenter its condition lock.""" + result = _run_cleanup_regression(setup, tmp_path, "condition-signal") + assert result["observed"]["propagated"] + + +@pytest.mark.parametrize("case", ["join-once", "join-twice"]) +def test_wait_exceptions_preserve_worker_completion_and_original_exception( + setup, tmp_path, case +): + """R10.2: real SIGUSR1 exceptions cannot release protection before done.""" + _run_cleanup_regression(setup, tmp_path, case) + + +@pytest.mark.parametrize("case", ["start-after", "start-before"]) +def test_raised_start_tracks_native_worker_or_waits_start_grace(setup, tmp_path, case): + """R10.2: a raised start has a completion event and bounded began grace.""" + _run_cleanup_regression(setup, tmp_path, case) + + +def admission_probe(setup, tmp_path, boundary, force): + """Exercise real signals behind a paused admission in a bounded subprocess.""" + import textwrap + + if boundary == "archive": + paused(setup) + probe = tmp_path / "admission_probe.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import sys + import threading + import traceback + from pathlib import Path + from types import SimpleNamespace + import confirmatory_no_tools as nt + + design = json.loads(sys.argv[1]) + boundary, force = sys.argv[2], sys.argv[3] == "force" + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + entered, release, returned, watcher_ready, committed = ( + threading.Event() for _ in range(5) + ) + errors, calls, order = [], [], [] + plan = None + + + class AdmissionLock: + def __init__(self): + self.lock = threading.Lock() + + def __enter__(self): + if threading.current_thread().name == "no-tools-signal-watcher": + watcher_ready.set() + self.lock.acquire() + return self + + def __exit__(self, *exc): + self.lock.release() + + def locked(self): + return self.lock.locked() + + + class InterruptEvent(threading.Event): + def set(self): + order.append("interrupt") + super().set() + watcher_ready.set() + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + calls.append(scenario.id) + assert plan.interrupted.wait(20), "R3: watcher eventually interrupts" + spend.add(0.1) + return { + "scenario_id": scenario.id, + "model": profile.name, + "arm": profile.arm, + "status": "complete", + "error": None, + "turns": [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": 0.1}, + } + + + def completion(profile): + if boundary == "archive": + assert plan.interrupted.wait(20), ( + "R3: archive finishes before another attempt" + ) + return None + + + plan = nt.Stage( + design, + models=[nt.CHEAP[0]], + board=board, + run_household=provider, + completion_for=completion, + out=io.StringIO(), + ) + plan.interrupted = InterruptEvent() + plan._admission_lock = AdmissionLock() + original_say = plan.say + + + def say(text): + if text.startswith("Interrupted:"): + assert plan.interrupted.is_set(), "R3: message follows interruption" + assert committed.is_set(), ( + "R3: admitted commit precedes interruption message" + ) + order.append("message") + original_say(text) + + + plan.say = say + original_signal = signal.signal + + + def install(signum, handler): + if callable(handler) and handler.__name__ == "interrupt": + + def tracked(sig, frame): + handler(sig, frame) + returned.set() + + return original_signal(signum, tracked) + return original_signal(signum, handler) + + + signal.signal = install + + + def pause_commit(): + entered.set() + os.write(1, b"ready\n") + assert release.wait(20), "R3: paused admitted commit is released" + + + if boundary == "archive": + original_replace = nt.os.replace + + def replace(source, target): + if Path(target).parent.name == "attempts": + pause_commit() + original_replace(source, target) + order.append("commit") + committed.set() + return + return original_replace(source, target) + + nt.os.replace = replace + else: + + class Admitted(dict): + def __setitem__(self, key, value): + pause_commit() + super().__setitem__(key, value) + order.append("commit") + committed.set() + + plan._admitted_attempts = Admitted() + + + def control(): + try: + assert entered.wait(20), "R3: worker reaches the admission commit" + held_at_commit = plan._admission_lock.locked() + os.kill(os.getpid(), signal.SIGINT) + assert returned.wait(5), ( + "R3: first handler returns while worker holds admission lock" + ) + assert held_at_commit, ( + "R3: worker holds the admission lock at its commit" + ) + assert watcher_ready.wait(5), ( + "R3: watcher reaches the admission boundary" + ) + assert not plan.interrupted.is_set(), ( + "R3: interrupt cannot precede admitted commit" + ) + assert not committed.is_set(), ( + "R3: commit remains paused during first handler" + ) + if force: + os.write(1, b"handler-returned-watcher-waiting\n") + os.kill(os.getpid(), signal.SIGINT) + raise AssertionError("R3: forced exit returned") + except BaseException: + errors.append(traceback.format_exc()) + if force: + os.write(2, errors[-1].encode()) + os._exit(99) + finally: + if not force: + release.set() + + + controller = threading.Thread(target=control, daemon=True) + controller.start() + result = plan.execute() + controller.join(20) + assert not errors, "\n".join(errors) + assert not controller.is_alive(), "R3: controller finishes" + assert result == 1, "R3: signalled run exits nonzero" + assert ( + order.index("commit") + < order.index("interrupt") + < order.index("message") + ), order + assert plan.interrupted.is_set(), ( + "R3: watcher sets interrupt after admitted commit" + ) + households = plan.households(nt.CHEAP[0]) + assert not (households / "h1.json").exists(), ( + "R3: no later household is admitted" + ) + if boundary == "archive": + assert calls == [], ( + "R3: no attempt admitted after the watcher interrupts" + ) + assert (households / "attempts/h2-1.json").exists(), ( + "R3: admitted archive is saved" + ) + assert not (households / "h2.json").exists(), ( + "R3: refused next attempt has no final" + ) + assert plan._admitted_attempts == {}, ( + "R3: no subsequent attempt is admitted" + ) + else: + assert calls == ["h2"], "R3: exactly the admitted attempt runs" + saved = json.loads((households / "h2.json").read_text()) + assert saved["status"] == "complete", ( + "R3: admitted attempt saves its final" + ) + assert saved["totals"]["cost_usd"] == 0.1, ( + "R3: admitted attempt saves spend" + ) + assert len(plan._admitted_attempts) == 1, "R3: nothing else is admitted" + print(json.dumps({"order": order, "calls": calls})) + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), boundary, "force" if force else "finish"], + env=env, + ready_marker=b"ready\n", + ) + assert b"ready\n" in result.stdout, (result.stdout, result.stderr) + return result.returncode, result.stdout, result.stderr + + +@pytest.mark.parametrize("boundary", ["archive", "attempt"]) +def test_signal_waits_for_an_admission_commit_without_blocking_handler( + setup, tmp_path, boundary +): + """R3: watcher interruption waits for commits admitted under the worker lock.""" + code, stdout, stderr = admission_probe(setup, tmp_path, boundary, force=False) + assert code == 0, (stdout, stderr) + + +@pytest.mark.parametrize("boundary", ["archive", "attempt"]) +def test_second_signal_forces_exit_while_watcher_waits_for_admission( + setup, tmp_path, boundary +): + """R3: the forced path remains available while the watcher waits for admission.""" + code, stdout, stderr = admission_probe(setup, tmp_path, boundary, force=True) + assert code == 128 + signal.SIGINT, (stdout, stderr) + assert b"handler-returned-watcher-waiting\n" in stdout + assert ( + b"Forced interruption: every unsaved attempt then under way is lost, " + b"with its spend; saved finals, archives and the spend they record survive." + ) in stderr + + +def test_signal_after_all_workers_complete_still_exits_one(setup, tmp_path): + """R4: a signal before handler restoration makes even a complete run fail.""" + import textwrap + + probe = tmp_path / "signal_after_workers.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import sys + from types import SimpleNamespace + import confirmatory_no_tools as nt + + design = json.loads(sys.argv[1]) + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + plan = nt.Stage( + design, + models=[nt.CHEAP[0]], + board=board, + completion_for=lambda profile: None, + out=io.StringIO(), + ) + + + def complete(model): + plan.states[model] = "complete" + + + plan.run_model = complete + original_signal = signal.signal + previous = signal.getsignal(signal.SIGINT) + sent = False + + + def restore(sig, handler): + global sent + if not sent and sig == signal.SIGINT and handler == previous: + sent = True + os.kill(os.getpid(), signal.SIGINT) + return original_signal(sig, handler) + + + signal.signal = restore + assert plan.execute() == 1, ( + "R4: signal arrival before restoration requires exit 1" + ) + assert all(state == "complete" for state in plan.states.values()) + assert sent, "R4: signal arrives after every worker completed" + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"])], + env=env, + ) + assert result.returncode == 0, (result.stdout, result.stderr) + + +@pytest.mark.parametrize("during_grace", [True, False]) +def test_native_worker_without_ident_obeys_start_grace_and_entry_interrupt( + setup, tmp_path, during_grace +): + """R10.2: native startup can precede ident and must never outlive file safety.""" + import textwrap + + probe = tmp_path / "native_start_regression.py" + probe.write_text( + textwrap.dedent(r""" + import io + import json + import os + import signal + import subprocess + import sys + import threading + import time + from types import SimpleNamespace + import confirmatory_no_tools as nt + + nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + print("native-start-regression-ready", flush=True) + design = json.loads(sys.argv[1]) + during_grace = sys.argv[2] == "True" + model = nt.CHEAP[0] + board = { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + } + native_entered, bootstrap_release = threading.Event(), threading.Event() + cleanup_entered, closed, native_finished = ( + threading.Event() for _ in range(3) + ) + done_held, release_done = threading.Event(), threading.Event() + original_bootstrap = threading.Thread._bootstrap_inner + original_run = threading.Thread.run + original_close = nt.Stage.close + main_ident = threading.get_ident() + worker, calls, order, observed = [], [], [], {} + nt.START_GRACE_SECONDS = 10 if during_grace else 0.05 + + + def provider(scenario, variables, tool_call, completion, spend, *, profile): + calls.append(scenario.id) + return { + "scenario_id": scenario.id, + "model": profile.name, + "arm": profile.arm, + "status": "complete", + "error": None, + "turns": [{"model": profile.model_id}], + "predictions": {"snap": 1}, + "totals": {"cost_usd": 0.0}, + } + + + plan = nt.Stage( + design, + models=[model], + board=board, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) + original_set = plan.interrupted.set + + + def interrupt_cleanup(): + result = original_set() + cleanup_entered.set() + return result + + + plan.interrupted.set = interrupt_cleanup + + + def bootstrap(thread): + if thread.name == "no-tools-" + model: + worker.append(thread) + native_entered.set() + assert bootstrap_release.wait(20), ( + "start grace rule: release native startup" + ) + return original_bootstrap(thread) + + + def run(thread): + try: + return original_run(thread) + finally: + if thread.name == "no-tools-" + model: + order.append("native-finished") + native_finished.set() + + + def close(current): + if current is plan and current._lock_file is not None: + order.append("close") + closed.set() + return original_close(current) + + + threading.Thread._bootstrap_inner = bootstrap + threading.Thread.run = run + nt.Stage.close = close + + + def injected(sig, frame): + raise KeyboardInterrupt("synthetic startup wait interruption") + + + signal.signal(signal.SIGUSR1, injected) + + + def main_in_start_wait(): + frame = sys._current_frames().get(main_ident) + while frame is not None: + if ( + frame.f_code.co_name == "wait" + and frame.f_locals.get("self") is worker[0]._started + ): + return True + frame = frame.f_back + return False + + + def lock_held(): + probe = ( + "import fcntl,sys; f=open(sys.argv[1],'a'); " + "\ntry: fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB)" + "\nexcept BlockingIOError: print('held')" + "\nelse: print('released')" + ) + result = subprocess.run( + [ + sys.executable, + "-c", + probe, + str(plan.root / ".confirmatory-no-tools.lock"), + ], + capture_output=True, + text=True, + timeout=10, + check=True, + ) + return result.stdout.strip() == "held" + + + def observe(): + assert native_entered.wait(10) + assert worker[0].ident is None, ( + "start grace rule: native worker has no ident" + ) + deadline = time.monotonic() + 10 + while not main_in_start_wait(): + assert time.monotonic() < deadline, ( + "start grace rule: main reaches _started.wait" + ) + time.sleep(0.001) + os.kill(os.getpid(), signal.SIGUSR1) + assert cleanup_entered.wait(10) + if during_grace: + # Let the native target begin immediately within grace, then + # hold its completion independently of process-startup load. + done = getattr(plan, "_worker_done", {}).get(model) + if done is not None: + original_done_set = done.set + + def set_done(): + done_held.set() + assert release_done.wait(20), ( + "start grace rule: release begun worker completion" + ) + return original_done_set() + + done.set = set_done + bootstrap_release.set() + if done is not None: + assert done_held.wait(10), ( + "start grace rule: late target reaches completion" + ) + try: + observed["held_at_cleanup"] = lock_held() + finally: + release_done.set() + else: + assert closed.wait(10), ( + "start grace rule: close after bounded grace" + ) + bootstrap_release.set() + + + actor = threading.Thread(target=observe, daemon=True) + actor.start() + try: + plan.execute() + except KeyboardInterrupt as error: + assert str(error) == "synthetic startup wait interruption" + else: + raise AssertionError("start grace rule: original exception propagates") + finally: + if closed.is_set() and not during_grace: + bootstrap_release.set() + actor.join(15) + assert native_finished.wait(10), ( + "start grace rule: native worker eventually completes" + ) + assert not calls, ( + "start grace rule: interrupted late worker makes no provider call" + ) + assert not list(plan.root.glob("*/none/confirmatory/households/*.json")), ( + "start grace rule: interrupted late worker writes no files" + ) + if during_grace: + assert observed.get("held_at_cleanup"), ( + "start grace rule: invocation lock stays held during grace" + ) + assert order.index("native-finished") < order.index("close"), ( + "start grace rule: worker begun during grace completes before close" + ) + else: + assert order.index("close") < order.index("native-finished"), ( + "start grace rule: worker beyond grace exits safely after close" + ) + assert plan._lock_file is None + print(json.dumps({"during_grace": during_grace, "order": order})) + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), str(during_grace)], + env=env, + ready_marker=b"native-start-regression-ready\n", + ) + assert b"native-start-regression-ready\n" in result.stdout, ( + "start grace rule: subprocess reaches readiness", + result.stdout, + result.stderr, + ) + assert result.returncode == 0, (result.stdout, result.stderr) + + +_NO_TOOLS_FORK_WARMED = False + + +def run_no_tools_fork_probe( + probe, + args=(), + *, + env=None, + timeout=30, + ready_marker=None, + ready_timeout=120, + text=False, +): + """Run actual probe code in a forked subprocess with bounded cleanup. + + The parent warms the provider-free profile imports once. A fresh child + executes the probe with its own signals, stdout, stderr and process exit. + Readiness has a separate deadline; the execution deadline begins when the + child reaches its actual readiness marker, or after subprocess setup. + """ + import multiprocessing + import runpy + import subprocess + import time + import traceback + + global _NO_TOOLS_FORK_WARMED + if not _NO_TOOLS_FORK_WARMED: + nt.pilot.model_profile( + nt.CHEAP[0], arm="none", prompt_version=nt.PROMPT_VERSION + ) + _NO_TOOLS_FORK_WARMED = True + probe = Path(probe).resolve() + stdout_path = probe.with_suffix(".stdout") + stderr_path = probe.with_suffix(".stderr") + command = [sys.executable, str(probe), *map(str, args)] + context = multiprocessing.get_context("fork") + startup_ready = context.Event() + + # Flush existing parent streams before fork so the child cannot duplicate + # any buffered pytest output when its streams are replaced. + sys.stdout.flush() + sys.stderr.flush() + with stdout_path.open("wb") as stdout_file, stderr_path.open("wb") as stderr_file: + + def child(): + code = 0 + try: + os.dup2(stdout_file.fileno(), 1) + os.dup2(stderr_file.fileno(), 2) + sys.stdout = os.fdopen(os.dup(1), "w", buffering=1) + sys.stderr = os.fdopen(os.dup(2), "w", buffering=1) + if env is not None: + os.environ.clear() + os.environ.update(env) + sys.argv = command[1:] + signal.signal(signal.SIGINT, signal.default_int_handler) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + startup_ready.set() + runpy.run_path(str(probe), run_name="__main__") + except SystemExit as error: + if error.code is None: + code = 0 + elif isinstance(error.code, int): + code = error.code + else: + print(error.code, file=sys.stderr) + code = 1 + except BaseException: + traceback.print_exc() + code = 1 + finally: + sys.stdout.flush() + sys.stderr.flush() + os._exit(code) + + process = context.Process(target=child, name=f"no-tools-probe-{probe.stem}") + process.start() + initial_deadline = time.monotonic() + ready_timeout + + def output(): + stdout = stdout_path.read_bytes() + stderr = stderr_path.read_bytes() + if text: + return stdout.decode(errors="replace"), stderr.decode(errors="replace") + return stdout, stderr + + def expired(limit): + process.kill() + process.join(timeout=30) + stdout, stderr = output() + raise subprocess.TimeoutExpired( + command, limit, output=stdout, stderr=stderr + ) + + try: + while not startup_ready.is_set() and process.is_alive(): + remaining = initial_deadline - time.monotonic() + if remaining <= 0: + expired(ready_timeout) + startup_ready.wait(timeout=min(remaining, 0.1)) + if ready_marker is not None: + marker = ( + ready_marker.encode() + if isinstance(ready_marker, str) + else ready_marker + ) + while marker not in stdout_path.read_bytes() and process.is_alive(): + if time.monotonic() >= initial_deadline: + expired(ready_timeout) + time.sleep(0.01) + process.join(timeout=timeout) + if process.is_alive(): + expired(timeout) + stdout, stderr = output() + return subprocess.CompletedProcess( + command, process.exitcode, stdout, stderr + ) + finally: + if process.is_alive(): + process.kill() + process.join(timeout=30) + process.close() From 6597adb467090bccb29f8e67e6691662cef3dc47 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 23:55:09 -0400 Subject: [PATCH 22/48] Confirmatory runner: restore signal handlers before closing the wake pipe Until the handlers are restored, a signal still writes to the watcher pipe; closing it first could let that write land in a file that reuses the descriptor. Note that an exception another handler raises before the exceptional wait begins cannot be shielded. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 0f19a730..b4b195cf 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -617,6 +617,8 @@ def watch_signals(): # SIGINT/SIGTERM handlers never raise, so production waits are not # interrupted by them. Tolerant waits defend against exceptions from # elsewhere (for example a test SIGUSR1 handler raising KeyboardInterrupt). + # Python cannot shield the instructions before these calls: another + # handler that raises there escapes without the wait. stop_admission() self._wait_for_workers() raise @@ -649,11 +651,13 @@ def watch_signals(): shutdown_error = error if failure is None and shutdown_error is not None: stop_admission() + # Restore the handlers first: until then a signal still writes to + # write_fd, whose number a closed pipe could hand to another file. + for sig, handler in previous.items(): + signal.signal(sig, handler) for fd in (read_fd, write_fd): if fd is not None: os.close(fd) - for sig, handler in previous.items(): - signal.signal(sig, handler) self.close() if failure is None and shutdown_error is not None: raise shutdown_error From 9acc9205cc717cf6176d4c72aad673b108f38cf5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 00:10:32 -0400 Subject: [PATCH 23/48] Confirmatory runner: let the forced-exit probe wait for the handler The admission probe's control thread sent the second SIGINT and then at once raised "forced exit returned". The handler runs on the main thread, so on a loaded host the control thread could reach its own os._exit(99) first; one of three suite runs at 6597adb4 failed that way, with the forced message printed after the control thread's traceback. The thread now sleeps 20 s, inside the probe's 30 s execution deadline, before it reports a forced exit that never came. Test only. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_no_tools.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 08620921..37c2ad60 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -1388,6 +1388,7 @@ def admission_probe(setup, tmp_path, boundary, force): import signal import sys import threading + import time import traceback from pathlib import Path from types import SimpleNamespace @@ -1551,6 +1552,11 @@ def control(): if force: os.write(1, b"handler-returned-watcher-waiting\n") os.kill(os.getpid(), signal.SIGINT) + # The handler runs on the main thread. Sleep (releasing + # the GIL) so it can exit the process before this thread + # reports that it did not; 20 s stays inside the probe's + # 30 s execution deadline. + time.sleep(20) raise AssertionError("R3: forced exit returned") except BaseException: errors.append(traceback.format_exc()) From a1114fe95d3170afbd71425cfcf89ae7fc6799b3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 02:31:13 -0400 Subject: [PATCH 24/48] Confirmatory runner: cover watcher startup and pipe teardown Exercise real signals after descriptor reuse and watcher shutdown. Require both handlers to restore before either descriptor closes. Pin late first SIGINT to exit 1 with completed finals, an unset interrupt and no announcement. Exercise watcher start failures before and after native startup and target entry beyond startup grace. Require the original start error to propagate, both handlers and descriptors to be released, and delayed targets to avoid closed descriptors and finish. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_no_tools.py | 443 ++++++++++++++++++++++++++++ 1 file changed, 443 insertions(+) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 37c2ad60..961d37c8 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -2093,3 +2093,446 @@ def expired(limit): process.kill() process.join(timeout=30) process.close() + + +def test_signal_handlers_restore_before_either_watcher_descriptor_closes( + setup, tmp_path +): + """R11: every handler that can write is removed before either pipe end closes.""" + run_no_tools_probe( + setup, + tmp_path, + "watcher_descriptor_order", + r""" + plan = stage(setup, models=[nt.CHEAP[0]]) + previous = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_pipe, original_close = os.pipe, os.close + original_signal = signal.signal + watcher_pipe, events = [], [] + + + def pipe(): + descriptors = original_pipe() + assert not watcher_pipe, "R11: execute creates one watcher pipe" + watcher_pipe.extend(descriptors) + return descriptors + + + def register(sig, handler): + old = original_signal(sig, handler) + if sig in previous and handler == previous[sig]: + events.append(("restore", sig)) + return old + + + def close(fd): + original_close(fd) + if fd in watcher_pipe: + events.append(("close", fd)) + + + monkeypatch.setattr(os, "pipe", pipe) + monkeypatch.setattr(os, "close", close) + monkeypatch.setattr(signal, "signal", register) + assert plan.execute() == 0 + restored, closed = set(), [] + for kind, value in events: + if kind == "restore": + restored.add(value) + else: + assert restored == set(previous), ( + "R11: both handlers restore before either pipe descriptor " + f"closes: {events}" + ) + closed.append(value) + assert len(watcher_pipe) == 2 + assert closed == watcher_pipe, "R11: both watcher descriptors close" + assert all( + signal.getsignal(sig) == handler for sig, handler in previous.items() + ), "R11: both prior handlers remain installed" + """, + ) + + +def test_signal_after_watcher_descriptor_reuse_cannot_write_to_replacement( + setup, tmp_path +): + """R11: a real signal after close cannot write into a reused descriptor.""" + run_no_tools_probe( + setup, + tmp_path, + "watcher_descriptor_reuse", + r""" + plan = stage(setup, models=[nt.CHEAP[0]]) + original_pipe, original_close = os.pipe, os.close + original_signal = signal.signal + previous_interrupts = [] + + + def previous_interrupt(sig, frame): + previous_interrupts.append(sig) + + + original_signal(signal.SIGINT, previous_interrupt) + watcher_pipe, replacement = [], [] + sent = False + + + def pipe(): + descriptors = original_pipe() + assert not watcher_pipe, "R11: execute creates one watcher pipe" + watcher_pipe.extend(descriptors) + return descriptors + + + def close(fd): + global sent + original_close(fd) + if fd == watcher_pipe[1] and not replacement: + read_fd, write_fd = original_pipe() + # Preserve the read end if it acquired the descriptor that must + # become the replacement write end. + if read_fd == fd: + copied_read_fd = os.dup(read_fd) + original_close(read_fd) + read_fd = copied_read_fd + if write_fd != fd: + os.dup2(write_fd, fd) + original_close(write_fd) + write_fd = fd + replacement.extend((read_fd, write_fd)) + os.set_blocking(read_fd, False) + sent = True + # This executes on the main thread immediately after the old + # write descriptor closes and its number is safely reused. + os.kill(os.getpid(), signal.SIGINT) + + + monkeypatch.setattr(os, "pipe", pipe) + monkeypatch.setattr(os, "close", close) + try: + result = plan.execute() + assert sent and len(replacement) == 2 + try: + received = os.read(replacement[0], 1) + except BlockingIOError: + received = b"" + assert received == b"", ( + "R11: a restored handler cannot write into the replacement " + f"descriptor; received {received!r}" + ) + assert result == 0 + assert previous_interrupts == [signal.SIGINT], ( + "R11: the real signal reaches the restored prior handler" + ) + finally: + for fd in replacement: + original_close(fd) + """, + ) + + +@pytest.mark.parametrize("case", ["before-native", "after-native", "after-grace"]) +def test_watcher_start_failure_restores_resources_and_late_entry_is_safe( + setup, tmp_path, case +): + """R11: failed watcher startup preserves cleanup and the late-entry guard.""" + run_no_tools_probe( + setup, + tmp_path, + "watcher_start_" + case, + " case = " + + repr(case) + + r""" + import errno + import time + + provider = Provider() + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + original_handlers = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_start = threading.Thread.start + original_run = threading.Thread.run + original_pipe = os.pipe + original_read = os.read + original_fd_close = os.close + original_close = plan.close + native_entered, reading, release_late, target_finished = ( + threading.Event() for _ in range(4) + ) + watchers, pipe_fds, closed_fds, reads, target_errors, order = ( + [] for _ in range(6) + ) + start_error = RuntimeError("synthetic watcher start " + case) + nt.START_GRACE_SECONDS = 0.05 + + + def pipe(): + descriptors = original_pipe() + assert not pipe_fds, "watcher lifecycle: execute creates one pipe" + pipe_fds.extend(descriptors) + return descriptors + + + def read(fd, count): + if fd in pipe_fds: + assert fd not in closed_fds, ( + "watcher lifecycle: a late watcher cannot read a closed pipe" + ) + reads.append(fd) + reading.set() + return original_read(fd, count) + + + def fd_close(fd): + result = original_fd_close(fd) + if fd in pipe_fds: + closed_fds.append(fd) + order.append("descriptor-closed") + return result + + + def close(): + result = original_close() + order.append("stage-closed") + return result + + + def run(thread): + if thread.name != "no-tools-signal-watcher": + return original_run(thread) + native_entered.set() + try: + if case == "after-grace": + assert release_late.wait(20), ( + "watcher lifecycle: release the delayed native target" + ) + assert set(closed_fds) == set(pipe_fds), ( + "watcher lifecycle: delayed target begins after pipe close" + ) + order.append("late-target-entered") + return original_run(thread) + except BaseException as error: + target_errors.append(error) + raise + finally: + target_finished.set() + + + def start(thread): + if thread.name != "no-tools-signal-watcher": + return original_start(thread) + watchers.append(thread) + if case != "before-native": + original_start(thread) + assert native_entered.wait(10), ( + "watcher lifecycle: the native thread starts before failure" + ) + if case == "after-native": + assert reading.wait(10), ( + "watcher lifecycle: the begun watcher reaches its pipe read" + ) + raise start_error + + + monkeypatch.setattr(os, "pipe", pipe) + monkeypatch.setattr(os, "read", read) + monkeypatch.setattr(os, "close", fd_close) + monkeypatch.setattr(threading.Thread, "run", run) + monkeypatch.setattr(threading.Thread, "start", start) + monkeypatch.setattr(plan, "close", close) + started_at = time.monotonic() + try: + plan.execute() + except RuntimeError as error: + assert error is start_error, ( + "watcher lifecycle: propagate the original start exception" + ) + else: + raise AssertionError("watcher lifecycle: startup failure must propagate") + elapsed = time.monotonic() - started_at + + assert len(watchers) == 1 and len(pipe_fds) == 2 + assert sorted(closed_fds) == sorted(pipe_fds), ( + "watcher lifecycle: close both descriptors exactly once" + ) + assert order.count("stage-closed") == 1 + assert all( + signal.getsignal(sig) == handler + for sig, handler in original_handlers.items() + ), "watcher lifecycle: restore both installed handlers" + for fd in pipe_fds: + try: + os.fstat(fd) + except OSError as error: + assert error.errno == errno.EBADF + else: + raise AssertionError("watcher lifecycle: descriptor remains open") + assert plan.interrupted.is_set(), ( + "watcher lifecycle: failed startup stops admission" + ) + assert provider.calls == [] and plan._started_workers == [], ( + "watcher lifecycle: no worker is admitted after startup failure" + ) + assert plan._lock_file is None, ( + "watcher lifecycle: release the stage invocation lock" + ) + if case == "before-native": + assert elapsed >= nt.START_GRACE_SECONDS * 0.9, ( + "watcher lifecycle: allow bounded grace for a possible native start" + ) + assert not native_entered.is_set() + assert not target_finished.is_set() + assert not reads + elif case == "after-native": + assert reads == [pipe_fds[0]] + assert target_finished.is_set(), ( + "watcher lifecycle: join the watcher before cleanup returns" + ) + else: + assert elapsed >= nt.START_GRACE_SECONDS * 0.9, ( + "watcher lifecycle: delayed target outlasts the bounded start grace" + ) + assert not target_finished.is_set() + release_late.set() + watchers[0].join(timeout=10) + assert target_finished.is_set() + assert not reads, ( + "watcher lifecycle: late entry checks closing before reading" + ) + assert order.index("stage-closed") < order.index("late-target-entered") + assert not target_errors, target_errors + assert not watchers[0].is_alive(), ( + "watcher lifecycle: no watcher remains after probe cleanup" + ) + assert not any( + thread.name == "no-tools-signal-watcher" + for thread in threading.enumerate() + ) + monkeypatch.undo() + """, + ) + + +def test_first_sigint_after_watcher_shutdown_preserves_completed_finals( + setup, tmp_path +): + """R11.2: late recording forces exit 1 without watcher interruption or output.""" + import textwrap + + probe = tmp_path / "late_first_sigint.py" + probe.write_text( + textwrap.dedent(r""" + import json + import os + import runpy + import signal + import sys + import threading + from types import SimpleNamespace + + helpers = runpy.run_path(sys.argv[2]) + nt, stage, final, load = ( + helpers[name] for name in ("nt", "stage", "final", "load") + ) + design = json.loads(sys.argv[1]) + setup = { + "design": design, + "board": { + "scenarios": { + sid: SimpleNamespace(id=sid) for sid in design["draw_order"] + }, + "variables": {sid: ["snap"] for sid in design["draw_order"]}, + }, + } + plan = stage(setup) + previous = { + sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM) + } + original_signal = signal.signal + original_start = threading.Thread.start + watcher = [] + restoring, recorded = threading.Event(), threading.Event() + handler_returned = threading.Event() + errors = [] + + + def start(thread): + if thread.name == "no-tools-signal-watcher": + watcher.append(thread) + return original_start(thread) + + + def register(sig, handler): + if sig in previous and handler == previous[sig]: + if sig == signal.SIGINT: + assert len(watcher) == 1 and not watcher[0].is_alive(), ( + "late signal: watcher has finished before restoration" + ) + restoring.set() + assert recorded.wait(20), "late signal: controller finishes" + elif sig in previous: + original_handler = handler + + def tracked(signum, frame): + original_handler(signum, frame) + handler_returned.set() + + handler = tracked + return original_signal(sig, handler) + + + def send(): + try: + assert restoring.wait(20), "late signal: restoration pauses" + os.kill(os.getpid(), signal.SIGINT) + assert handler_returned.wait(20), "late signal: handler returns" + except BaseException as error: + errors.append(error) + finally: + recorded.set() + + + actor = threading.Thread(target=send, daemon=True) + actor.start() + threading.Thread.start = start + signal.signal = register + result = plan.execute() + actor.join(20) + assert not actor.is_alive() and not errors, errors + assert result == 1, "late signal: recorded SIGINT requires exit 1" + assert not plan.interrupted.is_set(), ( + "late signal: a finished watcher cannot set interruption" + ) + assert all(state == "complete" for state in plan.states.values()) + for model in plan.models: + for sid in design["draw_order"]: + assert load(final(plan, model, sid))["status"] == "complete", ( + "late signal: every worker's final is saved" + ) + assert "Interrupted" not in plan.out.getvalue(), ( + "late signal: a finished watcher prints no announcement" + ) + assert all(signal.getsignal(sig) == old for sig, old in previous.items()) + assert not watcher[0].is_alive() and plan._lock_file is None + print("late-first-signal-verified", flush=True) + sys.exit(result) + """) + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(nt.__file__).parent), env.get("PYTHONPATH", "")] + ) + result = run_no_tools_fork_probe( + probe, + [json.dumps(setup["design"]), str(Path(__file__))], + env=env, + ) + assert result.returncode == 1, (result.stdout, result.stderr) + assert b"late-first-signal-verified\n" in result.stdout, ( + result.stdout, + result.stderr, + ) From 89dd9464ecd0a443dbb60ed61f2340cc3d33c510 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 02:34:30 -0400 Subject: [PATCH 25/48] Confirmatory runner: qualify signal and cleanup guarantees Describe the handler recording a signal and waking the watcher, with allocation failure as its exception limit. Describe the watcher setting interruption while running and a first signal recorded after shutdown forcing exit 1 without setting interruption or announcing it. State that guarded waits retry exceptions raised within them. Explain that another handler raising in an unguarded instruction before a guarded call, in cleanup bookkeeping or in finally can replace the original exception and skip waits. Keep executable behavior unchanged. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 39 +++++++++++++++++++------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index b4b195cf..458179e2 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -10,12 +10,17 @@ under the admission lock precede the watcher setting the interrupt under that same lock. After that interrupt is set, no archive move or attempt is admitted. The lock covers only the check and rename or admission state change; admitted -attempts finish and save outside it. The first SIGINT or SIGTERM only records -its number and wakes the watcher through a nonblocking pipe. The handler takes -no lock and raises nothing. The watcher sets the interrupt, releases admission, -and then announces that attempts already admitted will finish and save. -An exception escaping execute interrupts and waits for script-owned worker -completion events before restoring handlers or releasing the invocation lock. +attempts finish and save outside it. The first SIGINT or SIGTERM records its +number and wakes the watcher through a nonblocking pipe. The handler takes no +lock and raises nothing beyond allocation failure. While running, the watcher +sets the interrupt, releases admission, and then announces that attempts already +admitted will finish and save. A first signal recorded after watcher shutdown +still forces exit 1 without setting the interrupt or printing that announcement. +The guarded cleanup path sets the interrupt and waits for script-owned worker +completion before restoring handlers or releasing the invocation lock. +Guarded waits retry exceptions raised within them and preserve the original +exception. Another handler raising in an unguarded instruction, before a guarded +call, in cleanup bookkeeping or in finally, can replace it and skip waits. An exception caught in a model thread instead marks that model error without setting the interrupt. A second signal immediately uses a nonblocking stderr write and exits, losing every unsaved attempt then under way with its spend; @@ -614,11 +619,12 @@ def watch_signals(): result = self._execute() except BaseException as error: failure = error - # SIGINT/SIGTERM handlers never raise, so production waits are not - # interrupted by them. Tolerant waits defend against exceptions from - # elsewhere (for example a test SIGUSR1 handler raising KeyboardInterrupt). - # Python cannot shield the instructions before these calls: another - # handler that raises there escapes without the wait. + # SIGINT/SIGTERM handlers raise nothing beyond allocation failure. + # Guarded waits retry exceptions raised within them, including those + # from another handler (for example SIGUSR1 raising KeyboardInterrupt). + # Another handler raising in an unguarded instruction, before a + # guarded call, in cleanup bookkeeping or in finally, can replace + # the original exception and skip waits. stop_admission() self._wait_for_workers() raise @@ -664,12 +670,13 @@ def watch_signals(): return 1 if signals else result def _wait_for_workers(self) -> None: - """Retain protection through completion despite exceptional wait interruptions. + """Retry exceptions raised within guarded worker-completion waits. - The installed SIGINT and SIGTERM handlers never raise. These tolerant - waits defend against an exception from elsewhere, such as a test-only - SIGUSR1 handler that raises KeyboardInterrupt, and preserve the original - exception escaping execute. + The installed SIGINT and SIGTERM handlers raise nothing beyond allocation + failure. Guarded waits retry exceptions from elsewhere, such as a test-only + SIGUSR1 handler raising KeyboardInterrupt, preserving the original exception. + Another handler raising in unguarded instructions before a guarded call, + in cleanup bookkeeping or in finally can replace it and skip waits. """ workers = list(self._started_workers) starting = self._starting_worker From 4dafb833ea3e34c679ca24725234126003095356 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 09:03:50 -0400 Subject: [PATCH 26/48] Confirmatory runner: refuse undecodable files with their paths Catch recursion failures while reading saved finals, archived attempts, baseline exclusions and the CLI design so deeply nested JSON becomes a refusal naming the file. Include the baseline path in its validation errors, and require the CLI design to be a JSON object before stage initialization. Valid stage decisions stay the same. Saved records and baseline exclusions must refuse decoder recursion failures with a path-bearing StateError. The CLI must refuse unreadable, undecodable and non-object designs with their path and exit 1 before initializing a stage. Invalid UTF-8 must remain a decoding refusal. Synthetic regressions exercise every reader in a subprocess with a timeout and require the path before provider calls; final and archive cases also check preflight. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 18 ++++++-- tests/test_confirmatory_no_tools.py | 68 +++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 458179e2..85a8038c 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -89,7 +89,7 @@ def read_record(path: Path) -> dict: if not math.isfinite(cost) or cost < 0: raise ValueError("attempt cost must be finite and nonnegative") return record - except (OSError, ValueError, TypeError, AttributeError) as error: + except (OSError, ValueError, TypeError, AttributeError, RecursionError) as error: raise StateError(f"{path}: {error}") from error @@ -262,6 +262,7 @@ def __init__( run_household if run_household is not None else pilot.run_household ) self._lock_file = None + baseline_path = None try: baseline = design["baseline_exclusions"] if not isinstance(baseline, dict) or not isinstance( @@ -289,8 +290,11 @@ def __init__( cells.append((exclusion["scenario_id"], exclusion["variable"])) if len(cells) != len(set(cells)): raise StateError("baseline exclusions repeat a cell") - except (OSError, KeyError, TypeError, ValueError) as error: - raise StateError(f"baseline exclusions: {error}") from error + except (OSError, KeyError, TypeError, ValueError, RecursionError) as error: + at = "baseline exclusions" + if baseline_path is not None: + at += f" ({baseline_path})" + raise StateError(f"{at}: {error}") from error self.root.mkdir(parents=True, exist_ok=True) self._lock_file = (self.root / ".confirmatory-no-tools.lock").open("a") try: @@ -750,7 +754,13 @@ def main(argv=None) -> int: run.add_argument("--dry-run", action="store_true") args = parser.parse_args(argv) try: - with Stage(json.loads(args.design.read_text()), models=args.model) as stage: + try: + design = json.loads(args.design.read_text()) + if not isinstance(design, dict): + raise ValueError("design is not a JSON object") + except (OSError, ValueError, RecursionError) as error: + raise StateError(f"{args.design}: {error}") from error + with Stage(design, models=args.model) as stage: return stage.dry_run() if args.dry_run else stage.execute() except (StateError, OSError, ValueError, KeyError, TypeError) as error: print(f"The no-tools stage cannot start: {error}") diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 961d37c8..81fed2e3 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -910,6 +910,74 @@ def test_baseline_schema_is_usable_before_any_provider_call(setup, payload): assert provider.calls == [], "R5: schema checks precede provider calls" +@pytest.mark.parametrize("reader", ["final", "archive", "baseline", "design"]) +@pytest.mark.parametrize("fault", ["nested", "non-object", "invalid-utf8"]) +def test_undecodable_stage_file_refuses_with_path(setup, tmp_path, reader, fault): + """Every file reader names its refusal; deep decoding stays in a child.""" + run_no_tools_probe( + setup, + tmp_path, + f"decode_{reader}_{fault}", + f" reader, fault = {reader!r}, {fault!r}\n" + + r""" + from contextlib import redirect_stdout + from pathlib import Path + + if fault == "nested": + payload = ( + b'{"attempts": [], "exclusions": [], "nested": ' + + b"[" * 20000 + b"0" + b"]" * 20000 + b"}" + ) + elif fault == "non-object": + payload = b"[]" + else: + payload = ( + b'{"scenario_id": "h2", "status": "provider_error", ' + b'"error": "APIConnectionError", "attempts": [], ' + b'"exclusions": [], "note": "\xff"}' + ) + + provider = Provider() + root = Path(setup["design"]["no_tools"]["runs"]) + if reader in ("final", "archive"): + path = root / nt.CHEAP[0] / "none/confirmatory/households" + path /= "h2.json" if reader == "final" else "attempts/h2-1.json" + elif reader == "baseline": + path = Path(setup["design"]["baseline_exclusions"]["path"]) + else: + path = root.parent / "design.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + + if reader == "design": + def unexpected_stage(*args, **kwargs): + pytest.fail("an undecodable design must refuse before Stage") + + monkeypatch.setattr(nt, "Stage", unexpected_stage) + output = io.StringIO() + with redirect_stdout(output): + assert nt.main(["run", str(path), "--dry-run"]) == 1 + assert "The no-tools stage cannot start:" in output.getvalue() + assert str(path) in output.getvalue() + else: + if reader == "baseline": + setup["design"]["baseline_exclusions"]["sha256"] = ( + nt.sha256_file(path) + ) + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("an undecodable file must refuse before execution") + assert str(path) in str(caught.value) + if fault == "invalid-utf8": + assert isinstance(caught.value.__cause__, UnicodeDecodeError) + if reader in ("final", "archive"): + problems = nt.stage_problems(setup["design"]) + assert any(str(path) in problem for problem in problems), problems + assert provider.calls == [] + """, + ) + + def run_no_tools_probe(setup, tmp_path, name, body): """Bound a signal or worker-shutdown regression to its own process.""" import textwrap From 040d206437d1cbf3dbcb43b8e39a2923bc136931 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 12:47:16 -0400 Subject: [PATCH 27/48] Confirmatory runner: name board and cost refusals Check board inputs as strict UTF-8 and required JSON objects before delegating to the pinned harness. Attribute metadata engine checks, CSV parsing, exclusion validation and scenario reconstruction failures to their source files. Preserve absent optional exclusions and return the harness's loaded board for usable inputs. Saved finals and archived attempts must refuse numeric cost overflow with a path-bearing StateError at startup and preflight. Nested fixtures must control the child's recursion limit, derive a deeper payload and require the decoder's RecursionError as the refusal's cause. Allocation exhaustion continues to propagate. Provider-free regressions cover each board decoder, both cost boundaries and valid board results. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 87 +++++++++++++- tests/test_confirmatory_no_tools.py | 179 +++++++++++++++++++++++++++- 2 files changed, 257 insertions(+), 9 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 85a8038c..b92d360d 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -89,10 +89,89 @@ def read_record(path: Path) -> dict: if not math.isfinite(cost) or cost < 0: raise ValueError("attempt cost must be finite and nonnegative") return record - except (OSError, ValueError, TypeError, AttributeError, RecursionError) as error: + except ( + OSError, + ValueError, + TypeError, + AttributeError, + RecursionError, + OverflowError, + ) as error: raise StateError(f"{path}: {error}") from error +def _board_input(path: Path, read): + """Keep a delegated board reader's refusal attached to its input file.""" + try: + return read() + except ( + OSError, + ValueError, + TypeError, + AttributeError, + KeyError, + RecursionError, + OverflowError, + SystemExit, + ) as error: + raise StateError(f"{path}: {error}") from error + + +def load_board(path: Path, engine: str) -> dict: + """Check board inputs locally, then use the harness's unchanged loader.""" + from policybench.reference_exclusions import ( + load_reference_exclusions, + split_reference, + ) + from policybench.scenarios import scenario_from_dict + from policybench.spec import expand_programs_for_scenario + + def object_json(text): + payload = json.loads(text) + if not isinstance(payload, dict): + raise ValueError("board input is not a JSON object") + return payload + + def csv_frame(source): + # pandas also reads the original path in load_board. Check UTF-8 here + # so every decode/parser failure has its concrete source filename. + with source.open(encoding="utf-8") as stream: + return pilot.pd.read_csv(stream) + + metadata = path / "reference_outputs.csv.meta.json" + meta = _board_input( + metadata, lambda: object_json(metadata.read_text(encoding="utf-8")) + ) + + def require_engine(): + if f"policyengine-us {pilot.board_engine(path)}" != engine: + raise StateError("the board's engine differs from the design") + pilot._require_engine(path) + + _board_input(metadata, require_engine) + reference_path = path / "reference_outputs.csv" + reference = _board_input(reference_path, lambda: csv_frame(reference_path)) + exclusions_path = path / "reference_exclusions.json" + if exclusions_path.exists(): + _board_input( + exclusions_path, + lambda: object_json(exclusions_path.read_text(encoding="utf-8")), + ) + exclusions = _board_input(exclusions_path, lambda: load_reference_exclusions(path)) + _board_input(exclusions_path, lambda: split_reference(reference, exclusions)) + scenarios_path = path / "scenarios.csv" + scenarios = _board_input(scenarios_path, lambda: csv_frame(scenarios_path)) + for _, row in scenarios.iterrows(): + scenario = _board_input( + scenarios_path, + lambda: scenario_from_dict(object_json(row["scenario_json"])), + ) + _board_input( + metadata, lambda: expand_programs_for_scenario(meta["programs"], scenario) + ) + return pilot.load_board(path) + + def entry(path: Path, record: dict, n: int) -> dict: cls, reason = classify(record, 0.0, 0.0, None) return { @@ -314,11 +393,7 @@ def _load_state(self, board) -> None: design = self.design pilot.set_stated_engine(design["engine"].split()[-1]) if board is None: - board_path = Path(design["board"]) - if f"policyengine-us {pilot.board_engine(board_path)}" != design["engine"]: - raise StateError("the board's engine differs from the design") - pilot._require_engine(board_path) - board = pilot.load_board(board_path) + board = load_board(Path(design["board"]), design["engine"]) self.board = board missing = set(self.draw_order) - set(board["scenarios"]) missing |= set(self.draw_order) - set(board["variables"]) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 81fed2e3..10b53b1c 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -924,9 +924,11 @@ def test_undecodable_stage_file_refuses_with_path(setup, tmp_path, reader, fault from pathlib import Path if fault == "nested": + sys.setrecursionlimit(1000) + depth = sys.getrecursionlimit() * 20 payload = ( b'{"attempts": [], "exclusions": [], "nested": ' - + b"[" * 20000 + b"0" + b"]" * 20000 + b"}" + + b"[" * depth + b"0" + b"]" * depth + b"}" ) elif fault == "non-object": payload = b"[]" @@ -950,15 +952,25 @@ def test_undecodable_stage_file_refuses_with_path(setup, tmp_path, reader, fault path.write_bytes(payload) if reader == "design": + refusals = [] + + class CapturedStateError(nt.StateError): + def __init__(self, *args): + super().__init__(*args) + refusals.append(self) + def unexpected_stage(*args, **kwargs): pytest.fail("an undecodable design must refuse before Stage") + monkeypatch.setattr(nt, "StateError", CapturedStateError) monkeypatch.setattr(nt, "Stage", unexpected_stage) output = io.StringIO() with redirect_stdout(output): assert nt.main(["run", str(path), "--dry-run"]) == 1 assert "The no-tools stage cannot start:" in output.getvalue() assert str(path) in output.getvalue() + assert len(refusals) == 1 + caught = SimpleNamespace(value=refusals[0]) else: if reader == "baseline": setup["design"]["baseline_exclusions"]["sha256"] = ( @@ -968,16 +980,176 @@ def unexpected_stage(*args, **kwargs): with stage(setup, provider): pytest.fail("an undecodable file must refuse before execution") assert str(path) in str(caught.value) - if fault == "invalid-utf8": - assert isinstance(caught.value.__cause__, UnicodeDecodeError) if reader in ("final", "archive"): problems = nt.stage_problems(setup["design"]) assert any(str(path) in problem for problem in problems), problems + if fault == "nested": + assert isinstance(caught.value.__cause__, RecursionError) + trace = caught.value.__cause__.__traceback__ + frames = [] + while trace is not None: + frames.append(trace.tb_frame.f_code.co_filename) + trace = trace.tb_next + assert any(name.endswith("json/decoder.py") for name in frames), frames + elif fault == "invalid-utf8": + assert isinstance(caught.value.__cause__, UnicodeDecodeError) assert provider.calls == [] """, ) +def write_board_fixture(setup, directory): + """Write a small board consumed by the unmodified harness readers.""" + import csv + + directory.mkdir() + engine = nt.pilot._versions()["policyengine-us"] + setup["design"]["engine"] = f"policyengine-us {engine}" + setup["design"]["board"] = str(directory) + meta = { + "programs": ["snap"], + "revisions": [ + {"kind": "engine_upgrade", "engine_version": f"policyengine-us {engine}"} + ], + } + (directory / "reference_outputs.csv.meta.json").write_text(json.dumps(meta)) + (directory / "reference_exclusions.json").write_text('{"exclusions": []}') + with (directory / "reference_outputs.csv").open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_id", "variable", "value"]) + writer.writerows((sid, "snap", 1) for sid in setup["design"]["draw_order"]) + with (directory / "scenarios.csv").open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_json"]) + writer.writerows( + [json.dumps({"id": sid, "state": "CA", "adults": []})] + for sid in setup["design"]["draw_order"] + ) + + +@pytest.mark.parametrize( + "reader,fault", + [ + (reader, fault) + for reader in ("metadata", "exclusions", "scenarios") + for fault in ("nested", "non-object", "invalid-utf8") + ] + + [("reference", "invalid-utf8")], +) +def test_undecodable_board_file_refuses_with_path(setup, tmp_path, reader, fault): + run_no_tools_probe( + setup, + tmp_path, + f"board_{reader}_{fault}", + f" reader, fault = {reader!r}, {fault!r}\n" + + r""" + import csv + from pathlib import Path + + directory = Path(setup["design"]["no_tools"]["runs"]).parent / "board" + write_board_fixture(setup, directory) + names = { + "metadata": "reference_outputs.csv.meta.json", + "exclusions": "reference_exclusions.json", + "scenarios": "scenarios.csv", + "reference": "reference_outputs.csv", + } + path = directory / names[reader] + if fault == "nested": + sys.setrecursionlimit(1000) + depth = sys.getrecursionlimit() * 20 + payload = b'{"nested":' + b"[" * depth + b"0" + b"]" * depth + b"}" + elif fault == "non-object": + payload = b"[]" + else: + payload = b'{"note":"\xff"}' + if reader == "scenarios": + # Quote the JSON as a CSV field, retaining the corrupt UTF-8 byte. + text = io.StringIO(newline="") + writer = csv.writer(text) + writer.writerow(["scenario_json"]) + writer.writerow([payload.decode("latin1")]) + payload = text.getvalue().encode("latin1") + elif reader == "reference": + payload = b"scenario_id,variable,value\nh2,\xff,1\n" + path.write_bytes(payload) + provider = Provider() + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], board=None, run_household=provider, + completion_for=lambda profile: None, out=io.StringIO(), + ): + pytest.fail("an unusable board must refuse before execution") + assert str(path) in str(caught.value) + if fault == "nested": + assert isinstance(caught.value.__cause__, RecursionError) + trace = caught.value.__cause__.__traceback__ + frames = [] + while trace is not None: + frames.append(trace.tb_frame.f_code.co_filename) + trace = trace.tb_next + assert any(name.endswith("json/decoder.py") for name in frames), frames + elif fault == "invalid-utf8": + assert isinstance(caught.value.__cause__, UnicodeDecodeError) + assert provider.calls == [] + """, + ) + + +@pytest.mark.parametrize("with_exclusions", [True, False]) +def test_valid_board_loads_through_harness(setup, tmp_path, with_exclusions): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + if not with_exclusions: + (directory / "reference_exclusions.json").unlink() + expected = nt.pilot.load_board(directory) + provider = Provider() + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.board.keys() == expected.keys() + for key in ("reference", "scored", "excluded", "scenarios_frame"): + assert plan.board[key].equals(expected[key]) + for key in ("meta", "scenarios", "variables"): + assert plan.board[key] == expected[key] + assert plan.spend.spent_usd == 0 + assert plan.histories == {model: {} for model in nt.CHEAP} + assert provider.calls == [] + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_oversized_record_cost_refuses_with_path(setup, reader, boundary): + root = Path(setup["design"]["no_tools"]["runs"]) + path = root / nt.CHEAP[0] / "none/confirmatory/households" + path /= "h2.json" if reader == "final" else "attempts/h2-1.json" + payload = record( + "h2", + SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic"), + status="provider_error", + error="APIConnectionError", + cost=10**400, + ) + payload["attempts"] = [] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload)) + provider = Provider() + if boundary == "startup": + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("an oversized cost must refuse before execution") + assert str(path) in str(caught.value) + assert isinstance(caught.value.__cause__, OverflowError) + else: + problems = nt.stage_problems(setup["design"]) + assert any(str(path) in problem for problem in problems), problems + assert provider.calls == [] + + def run_no_tools_probe(setup, tmp_path, name, body): """Bound a signal or worker-shutdown regression to its own process.""" import textwrap @@ -1003,6 +1175,7 @@ def run_no_tools_probe(setup, tmp_path, name, body): "load", "paused", "track_interrupt_return", + "write_board_fixture", ): globals()[helper] = helpers[helper] design = json.loads(sys.argv[1]) From 6928b06997bbada1ffad0860d74832d36f1b1d3f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 13:00:02 -0400 Subject: [PATCH 28/48] Confirmatory runner: check the CLI with a real board Give the CLI option test a real small board so it exercises the runner's input boundary and unchanged harness loader. Reference agreement failures must name both the reference CSV and exclusions JSON because either input can make that validation unusable. Preserve the checkpoint branch and its existing commits. The delivery branch retains two commits while including this fixture update and the paired reference paths. The CLI dry run must parse all model options and name its owed work before provider calls. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 7 +++++-- tests/test_confirmatory_no_tools.py | 8 ++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index b92d360d..9f4a06cb 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -100,7 +100,7 @@ def read_record(path: Path) -> dict: raise StateError(f"{path}: {error}") from error -def _board_input(path: Path, read): +def _board_input(path: Path | str, read): """Keep a delegated board reader's refusal attached to its input file.""" try: return read() @@ -158,7 +158,10 @@ def require_engine(): lambda: object_json(exclusions_path.read_text(encoding="utf-8")), ) exclusions = _board_input(exclusions_path, lambda: load_reference_exclusions(path)) - _board_input(exclusions_path, lambda: split_reference(reference, exclusions)) + _board_input( + f"{reference_path} and {exclusions_path}", + lambda: split_reference(reference, exclusions), + ) scenarios_path = path / "scenarios.csv" scenarios = _board_input(scenarios_path, lambda: csv_frame(scenarios_path)) for _, row in scenarios.iterrows(): diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 10b53b1c..9ea28ac0 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -371,14 +371,10 @@ def test_registered_cap_matches_the_report_when_available(): assert nt.NO_TOOLS_CAP == cap == 5.0 -def test_main_supports_multiple_model_options_and_dry_run( - setup, tmp_path, monkeypatch, capsys -): +def test_main_supports_multiple_model_options_and_dry_run(setup, tmp_path, capsys): + write_board_fixture(setup, tmp_path / "board") path = tmp_path / "design.json" path.write_text(json.dumps(setup["design"])) - monkeypatch.setattr(nt.pilot, "board_engine", lambda path: "2.60.4") - monkeypatch.setattr(nt.pilot, "_require_engine", lambda path: None) - monkeypatch.setattr(nt.pilot, "load_board", lambda path: setup["board"]) assert ( nt.main( [ From 8b6c8f70b080c31daae4ad28576bc5f89711f7f4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 17:17:25 -0400 Subject: [PATCH 29/48] Confirmatory runner: guard board rereads and strict saved costs Hash the bytes used by every board input check and guard the unchanged harness loader. Exclusions existence failures must name the exclusions file. Delegated I/O failures must name their filename or the board folder. Other delegated failures must name files whose checked bytes changed, or the board folder when no change is established, while preserving the original cause. Saved finals and archives must contain an object totals with a present numeric cost_usd. Booleans and other nonnumbers are refused. Costs must convert to finite nonnegative floats. Startup and calculator preflight use this reader; calculator attempt_cost and valid spend arithmetic stay unchanged. Provider-free regressions record 65 failures and 49 passes against 6928b069 before these fixes; final verification follows. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 74 ++++-- tests/test_confirmatory_no_tools.py | 335 ++++++++++++++++++++++++++++ 2 files changed, 393 insertions(+), 16 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 9f4a06cb..412b61a4 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -33,6 +33,8 @@ import argparse import fcntl +import hashlib +import io import json import math import os @@ -85,7 +87,13 @@ def read_record(path: Path) -> dict: raise ValueError("native prior_attempts cannot preserve full attempts") if not isinstance(record.get("attempts"), list): raise ValueError("record lacks an attempts list") - cost = attempt_cost(record) + totals = record.get("totals") + if not isinstance(totals, dict): + raise ValueError("record totals must be an object with cost_usd") + raw_cost = totals.get("cost_usd") + if isinstance(raw_cost, bool) or not isinstance(raw_cost, (int, float)): + raise ValueError("record cost_usd must be a present number") + cost = float(raw_cost) if not math.isfinite(cost) or cost < 0: raise ValueError("attempt cost must be finite and nonnegative") return record @@ -100,20 +108,23 @@ def read_record(path: Path) -> dict: raise StateError(f"{path}: {error}") from error +BOARD_INPUT_ERRORS = ( + OSError, + ValueError, + TypeError, + AttributeError, + KeyError, + RecursionError, + OverflowError, + SystemExit, +) + + def _board_input(path: Path | str, read): """Keep a delegated board reader's refusal attached to its input file.""" try: return read() - except ( - OSError, - ValueError, - TypeError, - AttributeError, - KeyError, - RecursionError, - OverflowError, - SystemExit, - ) as error: + except BOARD_INPUT_ERRORS as error: raise StateError(f"{path}: {error}") from error @@ -126,6 +137,13 @@ def load_board(path: Path, engine: str) -> dict: from policybench.scenarios import scenario_from_dict from policybench.spec import expand_programs_for_scenario + checked = {} + + def checked_bytes(source): + data = source.read_bytes() + checked[source] = hashlib.sha256(data).hexdigest() + return data + def object_json(text): payload = json.loads(text) if not isinstance(payload, dict): @@ -135,12 +153,14 @@ def object_json(text): def csv_frame(source): # pandas also reads the original path in load_board. Check UTF-8 here # so every decode/parser failure has its concrete source filename. - with source.open(encoding="utf-8") as stream: + with io.TextIOWrapper( + io.BytesIO(checked_bytes(source)), encoding="utf-8" + ) as stream: return pilot.pd.read_csv(stream) metadata = path / "reference_outputs.csv.meta.json" meta = _board_input( - metadata, lambda: object_json(metadata.read_text(encoding="utf-8")) + metadata, lambda: object_json(checked_bytes(metadata).decode("utf-8")) ) def require_engine(): @@ -152,11 +172,13 @@ def require_engine(): reference_path = path / "reference_outputs.csv" reference = _board_input(reference_path, lambda: csv_frame(reference_path)) exclusions_path = path / "reference_exclusions.json" - if exclusions_path.exists(): + if _board_input(exclusions_path, exclusions_path.exists): _board_input( exclusions_path, - lambda: object_json(exclusions_path.read_text(encoding="utf-8")), + lambda: object_json(checked_bytes(exclusions_path).decode("utf-8")), ) + else: + checked[exclusions_path] = None exclusions = _board_input(exclusions_path, lambda: load_reference_exclusions(path)) _board_input( f"{reference_path} and {exclusions_path}", @@ -172,7 +194,27 @@ def require_engine(): _board_input( metadata, lambda: expand_programs_for_scenario(meta["programs"], scenario) ) - return pilot.load_board(path) + try: + return pilot.load_board(path) + except BOARD_INPUT_ERRORS as error: + if isinstance(error, OSError): + raise StateError(f"{error.filename or path}: {error}") from error + changed, unreadable = [], [] + for source, digest in checked.items(): + try: + current = sha256_file(source) + except FileNotFoundError: + current = None + except BOARD_INPUT_ERRORS as reread_error: + unreadable.append(f"{source}: cannot recheck: {reread_error}") + continue + if current != digest: + changed.append(str(source)) + at = " and ".join(changed) if changed else str(path) + detail = "; changed after it was checked" if changed else "" + if unreadable: + detail += "; " + "; ".join(unreadable) + raise StateError(f"{at}{detail}: {error}") from error def entry(path: Path, record: dict, n: int) -> dict: diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 9ea28ac0..d5b253e1 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -1117,6 +1117,224 @@ def test_valid_board_loads_through_harness(setup, tmp_path, with_exclusions): assert provider.calls == [] +@pytest.mark.parametrize("with_filename", [True, False]) +def test_delegated_board_oserror_refuses_with_path( + setup, tmp_path, monkeypatch, with_filename +): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + reference = directory / "reference_outputs.csv" + error = ( + OSError(5, "synthetic reread failure", str(reference)) + if with_filename + else OSError("synthetic reread failure") + ) + read_csv = nt.pilot.pd.read_csv + rereads = [] + + def fail_reference_reread(source, *args, **kwargs): + # Prechecks pass an opened stream; the unchanged harness passes a path. + if isinstance(source, Path) and source == reference: + rereads.append(source) + raise error + return read_csv(source, *args, **kwargs) + + monkeypatch.setattr(nt.pilot.pd, "read_csv", fail_reference_reread) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("a failed board reread must refuse before execution") + assert rereads == [reference] + assert caught.value.__cause__ is error + assert str(reference if with_filename else directory) in str(caught.value) + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize("reader", ["metadata", "scenarios"]) +def test_board_changed_to_nested_json_after_check_refuses_with_path( + setup, tmp_path, reader +): + run_no_tools_probe( + setup, + tmp_path, + f"board_changed_{reader}", + f" reader = {reader!r}\n" + + r""" + import csv + from pathlib import Path + + sys.setrecursionlimit(1000) + depth = sys.getrecursionlimit() * 20 + payload = '{"nested":' + "[" * depth + "0" + "]" * depth + "}" + directory = Path(setup["design"]["no_tools"]["runs"]).parent / "board" + write_board_fixture(setup, directory) + filename = ( + "reference_outputs.csv.meta.json" if reader == "metadata" + else "scenarios.csv" + ) + path = directory / filename + load_board = nt.pilot.load_board + delegates = [] + + def change_then_load(source): + delegates.append(source) + if reader == "metadata": + path.write_text(payload) + else: + with path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_json"]) + writer.writerow([payload]) + return load_board(source) + + monkeypatch.setattr(nt.pilot, "load_board", change_then_load) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], board=None, run_household=provider, + completion_for=lambda profile: None, out=io.StringIO(), + ): + pytest.fail("a changed board must refuse before execution") + assert delegates == [directory], "all prechecks must finish first" + assert str(path) in str(caught.value) + assert "changed after" in str(caught.value) + assert isinstance(caught.value.__cause__, RecursionError) + trace = caught.value.__cause__.__traceback__ + frames = [] + while trace is not None: + frames.append(trace.tb_frame.f_code.co_filename) + trace = trace.tb_next + assert any(name.endswith("json/decoder.py") for name in frames), frames + finally: + assert provider.calls == [] + """, + ) + + +def test_board_exclusions_exists_failure_refuses_with_path( + setup, tmp_path, monkeypatch +): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + exclusions = directory / "reference_exclusions.json" + error = PermissionError(13, "synthetic unreadable board parent", str(exclusions)) + exists = Path.exists + checks = [] + + def fail_exclusions_exists(path): + if path == exclusions: + checks.append(path) + raise error + return exists(path) + + monkeypatch.setattr(Path, "exists", fail_exclusions_exists) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("an unreadable exclusions path must refuse") + assert checks == [exclusions] + assert caught.value.__cause__ is error + assert str(exclusions) in str(caught.value) + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize( + "error_type", + [ + ValueError, + TypeError, + AttributeError, + KeyError, + RecursionError, + OverflowError, + SystemExit, + ], +) +def test_unchanged_board_delegate_failure_names_folder( + setup, tmp_path, monkeypatch, error_type +): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + error = error_type("synthetic delegated failure") + delegates = [] + + def fail_load(source): + delegates.append(source) + raise error + + monkeypatch.setattr(nt.pilot, "load_board", fail_load) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("a failed board delegate must refuse before execution") + assert delegates == [directory] + assert caught.value.__cause__ is error + assert str(directory) in str(caught.value) + assert "changed after" not in str(caught.value) + finally: + assert provider.calls == [] + + +def test_board_delegate_failure_names_each_changed_file(setup, tmp_path, monkeypatch): + directory = tmp_path / "board" + write_board_fixture(setup, directory) + paths = [ + directory / name + for name in ("reference_outputs.csv.meta.json", "scenarios.csv") + ] + error = ValueError("synthetic delegated failure") + delegates = [] + + def change_then_fail(source): + delegates.append(source) + for path in paths: + path.write_bytes(path.read_bytes() + b"\n") + raise error + + monkeypatch.setattr(nt.pilot, "load_board", change_then_fail) + provider = Provider() + try: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("a changed board must refuse before execution") + assert delegates == [directory] + assert caught.value.__cause__ is error + assert all(str(path) in str(caught.value) for path in paths) + assert "changed after" in str(caught.value) + finally: + assert provider.calls == [] + + @pytest.mark.parametrize("reader", ["final", "archive"]) @pytest.mark.parametrize("boundary", ["startup", "preflight"]) def test_oversized_record_cost_refuses_with_path(setup, reader, boundary): @@ -1146,6 +1364,123 @@ def test_oversized_record_cost_refuses_with_path(setup, reader, boundary): assert provider.calls == [] +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize( + "fault", + [ + "missing-totals", + "null-totals", + "false-totals", + "empty-list-totals", + "empty-string-totals", + "true-totals", + "number-totals", + "list-totals", + "missing-cost", + "null", + "true", + "false", + "numeric-string", + "empty-string", + "invalid-string", + "empty-list", + "list", + "empty-object", + "object", + "nan", + "infinity", + "negative-infinity", + "negative", + "overflow", + ], +) +def test_strict_saved_cost_refuses_with_path(setup, reader, boundary, fault): + root = Path(setup["design"]["no_tools"]["runs"]) + path = root / nt.CHEAP[0] / "none/confirmatory/households" + path /= "h2.json" if reader == "final" else "attempts/h2-1.json" + payload = record( + "h2", + SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic"), + status="provider_error", + error="APIConnectionError", + ) + payload["attempts"] = [] + totals = { + "null-totals": None, + "false-totals": False, + "empty-list-totals": [], + "empty-string-totals": "", + "true-totals": True, + "number-totals": 1, + "list-totals": [1], + } + costs = { + "null": None, + "true": True, + "false": False, + "numeric-string": "0.25", + "empty-string": "", + "invalid-string": "invalid", + "empty-list": [], + "list": [0.25], + "empty-object": {}, + "object": {"value": 0.25}, + "nan": float("nan"), + "infinity": float("inf"), + "negative-infinity": float("-inf"), + "negative": -0.25, + "overflow": 10**400, + } + if fault == "missing-totals": + del payload["totals"] + elif fault in totals: + payload["totals"] = totals[fault] + elif fault == "missing-cost": + payload["totals"] = {} + else: + payload["totals"]["cost_usd"] = costs[fault] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload)) + provider = Provider() + try: + if boundary == "startup": + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("an invalid saved cost must refuse before execution") + assert str(path) in str(caught.value) + else: + problems = nt.stage_problems(setup["design"]) + assert any(str(path) in problem for problem in problems), problems + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize("cost", [0, 0.0, -0.0, 0.25, 2]) +def test_valid_saved_numeric_cost_preserves_spend(setup, cost): + root = Path(setup["design"]["no_tools"]["runs"]) + folder = root / nt.CHEAP[0] / "none/confirmatory/households" + folder.mkdir(parents=True) + profile = SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic") + archive = folder / "attempts/h2-1.json" + archive.parent.mkdir() + prior = record( + "h2", profile, status="provider_error", error="APIConnectionError", cost=cost + ) + prior["attempts"] = [] + archive.write_text(json.dumps(prior)) + saved = record("h2", profile, cost=cost) + saved["attempts"] = [nt.entry(archive, prior, 1)] + (folder / "h2.json").write_text(json.dumps(saved)) + provider = Provider() + with stage(setup, provider) as plan: + assert plan.spend.spent_usd == 2 * float(cost) + assert not any( + str(folder) in problem for problem in nt.stage_problems(setup["design"]) + ) + assert provider.calls == [] + + def run_no_tools_probe(setup, tmp_path, name, body): """Bound a signal or worker-shutdown regression to its own process.""" import textwrap From 0300257c18b032032f525802963e32dc26a29580 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 18:10:47 -0400 Subject: [PATCH 30/48] Confirmatory runner: align saved treatment and link checks Check every saved final and archive against the pinned none-arm treatment before providers or calculator admission. Model, arm, prompt version, wording, notice, explicit null calculator fields, board outputs and provider turns must match. A completed reply or nonnull answer requires its contract, sent prompt and rebuilt hashes. Empty failed attempts may omit that evidence; any evidence present must match. Cheap finals must be in the draw and retain a predictions object. Preflight loads the checked board lazily to apply these same rules. Final link numbers must be consecutive integers excluding booleans. Archived prefixes retain the report equality rule. Known link fields must match while extra keys and an absent status matching null remain permitted, and earlier prefixes are preserved. Archive classification failures must name the archive. Valid numeric spend, rerun eligibility, interrupted-archive recovery, locking and worker execution keep their existing rules. Keep the minimal state-machine fixtures explicit about their injected treatment validator. Add real-board fixtures with harness-built evidence, conditional failure controls and exact archive/spend resume checks. Record 164 agreement failures and 32 controls against the original reader before these fixes. The final selection passes 315 cases, including legacy assertions updated for the new refusal reasons. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 220 +++++++++- tests/test_confirmatory_no_tools.py | 13 +- tests/test_confirmatory_no_tools_records.py | 454 ++++++++++++++++++++ 3 files changed, 674 insertions(+), 13 deletions(-) create mode 100644 tests/test_confirmatory_no_tools_records.py diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 412b61a4..acdf0f88 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -53,6 +53,7 @@ action, attempt_cost, classify, + completed_turns, default_completion, pilot, sha256_file, @@ -61,6 +62,9 @@ NO_TOOLS_CAP = 5.0 START_GRACE_SECONDS = 10 +NO_TOOLS_WORDING_SHA256 = ( + "43b13e4e64463f4ee1654e9546672899f80ca5e1ce18e93cf2eb04091b3a2699" +) class StateError(ValueError): @@ -217,6 +221,150 @@ def require_engine(): raise StateError(f"{at}{detail}: {error}") from error +def _no_tools_answered(record: dict) -> bool: + predictions = record.get("predictions") + return isinstance(predictions, dict) and any( + value is not None for value in predictions.values() + ) + + +def _saved_treatment_problems(record: dict, model: str, board: dict) -> list[str]: + """Report treatment_problems specialized to the fixed none arm. + + Keep the report's conditional evidence requirements and conversions. + In particular, empty failed attempts may omit prompt evidence, variables + use the report's list conversion, and unexamined fields remain unexamined. + """ + problems = [] + if record.get("model") != model: + return [f"record names model {record.get('model')!r}"] + if record.get("arm") != "none": + return [f"arm {record.get('arm')!r} differs from 'none'"] + + profile = pilot.model_profile(model, arm="none", prompt_version=PROMPT_VERSION) + treatment = pilot.profile_treatment(profile) + if treatment["tool_features"] is not None: + raise ValueError( + f"none: the harness gives features {treatment['tool_features']}" + ) + + answered = _no_tools_answered(record) + owed = ( + isinstance(record.get("turns"), list) and bool(completed_turns(record)) + ) or answered + keys = ( + "prompt_version", + "prompt_wording_sha256", + "tool_notice_sha256", + "tool_features", + "calculate_tool_sha256", + "variables", + "turns", + ) + if owed: + keys += ( + "contract", + "messages", + "board_prompt_sha256", + "sent_board_prompt_sha256", + ) + for key in keys: + if key not in record: + problems.append(f"missing no-tools field {key!r}") + if "predictions" in record and not isinstance(record["predictions"], dict): + problems.append("no-tools predictions must be an object from run_household") + + contract = record.get("contract") + if (owed or "contract" in record) and contract != profile.contract: + problems.append( + f"answer contract {contract!r} differs from {profile.contract!r}" + ) + if record.get("prompt_version") != PROMPT_VERSION: + problems.append(f"prompt version {record.get('prompt_version')!r}") + wording = pilot.prompt_wording_sha256(profile) + if ( + wording != NO_TOOLS_WORDING_SHA256 + or record.get("prompt_wording_sha256") != NO_TOOLS_WORDING_SHA256 + ): + problems.append( + f"prompt wording {str(record.get('prompt_wording_sha256'))[:12]}, " + f"harness {str(wording)[:12]}" + ) + notice = hashlib.sha256(profile.notice.encode()).hexdigest() + if record.get("tool_notice_sha256") != notice: + problems.append("the notice hash differs from the arm's") + for key in ("tool_features", "calculate_tool_sha256"): + if record.get(key) != treatment[key]: + problems.append(f"{key} differs from the none arm's") + + sid = record.get("scenario_id") + if sid not in board["scenarios"]: + return problems + ["no such household on the board"] + if list(record.get("variables") or []) != list(board["variables"][sid]): + problems.append("its outputs differ from the board's") + if not isinstance(record.get("turns"), list): + return problems + ["no-tools turns must be an array from run_household"] + if answered and not completed_turns(record): + problems.append("a no-tools answer has no completed turn") + for number, turn in enumerate(record.get("turns") or [], start=1): + served = turn.get("provider_resolved_model") + if not served: + problems.append(f"turn {number} reports no provider's model") + elif served != profile.model_id.split("/")[-1]: + problems.append( + f"turn {number}: providers reported {served!r}, " + f"expected {profile.model_id}" + ) + + messages = record.get("messages") + if (owed or "messages" in record) and not isinstance(messages, list): + return problems + [ + "no-tools messages must be an array from conversation.transcript" + ] + if not messages: + if owed: + problems.append("no no-tools sent prompt in messages") + if completed_turns(record): + problems.append("no sent prompt in messages") + if owed: + return problems + + built, settings = pilot.build_initial_request( + board["scenarios"][sid], board["variables"][sid], profile + ) + if messages: + first = messages[0] if isinstance(messages[0], dict) else {} + text = ( + first.get("input") + if isinstance(first.get("input"), str) + else first.get("content") + ) + if text != built: + problems.append("the sent prompt differs from the harness's rebuilt prompt") + for key, description in ( + ("sent_board_prompt_sha256", "sent board prompt"), + ("board_prompt_sha256", "board prompt"), + ): + if (messages or key in record) and record.get(key) != settings[f"_{key}"]: + problems.append(f"the {description} hash differs from the rebuilt one") + return problems + + +def validate_treatment(path, record, model, board, engine): + """Check each saved record against the report's pinned none-arm treatment.""" + try: + with pilot.stating_engine(engine.split()[-1]): + problems = _saved_treatment_problems( + record, model, board() if callable(board) else board + ) + except BOARD_INPUT_ERRORS as error: + raise StateError( + f"{path}: cannot check saved no-tools treatment: {error}" + ) from error + if problems: + raise StateError(f"{path}: " + "; ".join(problems)) + + def entry(path: Path, record: dict, n: int) -> dict: cls, reason = classify(record, 0.0, 0.0, None) return { @@ -229,7 +377,17 @@ def entry(path: Path, record: dict, n: int) -> dict: } -def history(folder: Path) -> tuple[dict[str, dict], float]: +def _attempts_agree(listed: list, entries: list) -> bool: + """Match known link fields and retain each archive's exact earlier prefix.""" + if len(listed) != len(entries) or listed[:-1] != entries[:-1]: + return False + return not entries or ( + isinstance(listed[-1], dict) + and all(listed[-1].get(key) == value for key, value in entries[-1].items()) + ) + + +def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], float]: """Validate a model's histories, including an interrupted archive, and sum spend.""" archives: dict[str, list[tuple[int, Path]]] = {} for path in sorted((folder / "attempts").glob("*.json")): @@ -254,34 +412,58 @@ def history(folder: Path) -> tuple[dict[str, dict], float]: record = read_record(path) if record.get("scenario_id") != sid: raise StateError(f"{path}: household differs from archive name {sid}") - if record["attempts"] != entries: + if validate is not None: + validate(path, record) + if not _attempts_agree(record["attempts"], entries): raise StateError(f"{path}: attempts disagree with earlier archives") - cls, _ = classify(record, 0.0, 0.0, None) + try: + cls, _ = classify(record, 0.0, 0.0, None) + except BOARD_INPUT_ERRORS as error: + raise StateError( + f"{path}: cannot classify saved archive: {error}" + ) from error if action(cls, n) not in ("rerun", "pause-rerun"): raise StateError(f"{path}: archived attempt was ineligible for rerun") - entries.append(entry(path, record, n)) + entries = record["attempts"] + [entry(path, record, n)] spent += attempt_cost(record) final = None if sid in finals: path = finals[sid] final = read_record(path) + if drawn is not None: + if sid not in drawn: + raise StateError(f"{path}: cheap-model final is outside the draw") + if not isinstance(final.get("predictions"), dict): + raise StateError( + f"{path}: final no-tools predictions must be an object" + ) if final.get("scenario_id") != sid: raise StateError(f"{path}: household differs from final name {sid}") - if final["attempts"] != entries: + for n, link in enumerate(final["attempts"], 1): + if ( + not isinstance(link, dict) + or type(link.get("n")) is not int + or link["n"] != n + ): + raise StateError(f"{path}: attempt link n must be integer {n}") + if validate is not None: + validate(path, final) + if not _attempts_agree(final["attempts"], entries): raise StateError(f"{path}: attempts disagree with archived files") try: classify(final, 0.0, 0.0, None) - except (TypeError, ValueError, AttributeError, KeyError) as error: + except BOARD_INPUT_ERRORS as error: raise StateError( f"{path}: cannot classify saved final: {error}" ) from error + entries = final["attempts"] spent += attempt_cost(final) states[sid] = {"entries": entries, "final": final} return states, spent def stage_problems(design: dict) -> list[str]: - """Read stage histories and standing finals without a board or provider.""" + """Check saved histories, treatment and standing without a provider.""" problems = [] try: spec = design["no_tools"] @@ -293,9 +475,23 @@ def stage_problems(design: dict) -> list[str]: models.update(path.name for path in root.iterdir() if path.is_dir()) except (OSError, KeyError, TypeError, ValueError) as error: return [f"no-tools stage: {error}"] + board = None + + def checked_board(): + nonlocal board + if board is None: + board = load_board(Path(design["board"]), design["engine"]) + return board + for model in sorted(models): try: - states, _ = history(root / model / folder / stage / "households") + states, _ = history( + root / model / folder / stage / "households", + validate=lambda path, record: validate_treatment( + path, record, model, checked_board, design["engine"] + ), + drawn=set(draw_order) if model in CHEAP else None, + ) if model not in CHEAP: continue for sid in draw_order: @@ -454,7 +650,13 @@ def _load_state(self, board) -> None: else set() ) for model in sorted(folders | set(self.models)): - states, cost = history(self.households(model)) + states, cost = history( + self.households(model), + validate=lambda path, record: validate_treatment( + path, record, model, self.board, design["engine"] + ), + drawn=set(self.draw_order) if model in CHEAP else None, + ) self.histories[model] = states spent += cost # A saved ceiling failure is a stage-wide stop on every invocation. diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index d5b253e1..7bead2a2 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -20,7 +20,10 @@ @pytest.fixture -def setup(tmp_path): +def setup(tmp_path, monkeypatch): + # These state-machine fixtures inject a minimal board and fake records. + # Real treatment pins and prompts are covered in the saved-record suite. + monkeypatch.setattr(nt, "validate_treatment", lambda *args: None, raising=False) draw = ["h2", "h1"] baseline = tmp_path / "baseline-exclusions.json" baseline.write_text('{"exclusions": []}\n') @@ -309,7 +312,8 @@ def test_refuse_a_final_with_corrupted_attempt_links(setup, field, value): payload = load(path) payload["attempts"][0][field] = value path.write_text(json.dumps(payload)) - with pytest.raises(nt.StateError, match="attempts disagree"): + match = "attempt link n" if field == "n" else "attempts disagree" + with pytest.raises(nt.StateError, match=match): stage(setup) @@ -879,10 +883,11 @@ def test_stage_problems_classifies_every_saved_final(setup, category): payload.update(attempts=[], turns=[None]) path = plan.households(model) / f"{sid}.json" nt.write_json(path, payload) - with pytest.raises((nt.StateError, AttributeError), match="get"): + match = "outside the draw" if category == "outside-draw" else "get" + with pytest.raises(nt.StateError, match=match): stage(setup) problems = nt.stage_problems(setup["design"]) - assert any(model in problem and "get" in problem for problem in problems), ( + assert any(model in problem and match in problem for problem in problems), ( "R3: every final refused by initialization must fail stage preflight", problems, ) diff --git a/tests/test_confirmatory_no_tools_records.py b/tests/test_confirmatory_no_tools_records.py new file mode 100644 index 00000000..c145012c --- /dev/null +++ b/tests/test_confirmatory_no_tools_records.py @@ -0,0 +1,454 @@ +"""Saved no-tools evidence must agree with the report before any provider call.""" + +from __future__ import annotations + +import hashlib +import io +import json +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests.test_confirmatory_no_tools import Provider, nt, write_board_fixture + +OPTIONAL_EVIDENCE = ( + "contract", + "messages", + "board_prompt_sha256", + "sent_board_prompt_sha256", +) +REQUIRED_FIELDS = ( + "model", + "arm", + "prompt_version", + "prompt_wording_sha256", + "tool_notice_sha256", + "tool_features", + "calculate_tool_sha256", + "variables", + "turns", + *OPTIONAL_EVIDENCE, +) + + +def full_record(board, model, engine, sid="h2"): + """Rebuild the exact treatment pins and prompt through the real harness.""" + profile = nt.pilot.model_profile( + model, arm="none", prompt_version=nt.PROMPT_VERSION + ) + with nt.pilot.stating_engine(engine.split()[-1]): + prompt, settings = nt.pilot.build_initial_request( + board["scenarios"][sid], board["variables"][sid], profile + ) + return { + "scenario_id": sid, + "model": model, + "arm": "none", + "status": "complete", + "error": None, + "prompt_version": nt.PROMPT_VERSION, + "prompt_wording_sha256": nt.pilot.prompt_wording_sha256(profile), + "tool_notice_sha256": hashlib.sha256(profile.notice.encode()).hexdigest(), + **nt.pilot.profile_treatment(profile), + "variables": list(board["variables"][sid]), + "turns": [{"provider_resolved_model": profile.model_id.split("/")[-1]}], + "predictions": {"snap": 1}, + "contract": profile.contract, + "messages": [{"role": "user", "content": prompt}], + "board_prompt_sha256": settings["_board_prompt_sha256"], + "sent_board_prompt_sha256": settings["_sent_board_prompt_sha256"], + "totals": {"cost_usd": 0.1}, + "attempts": [], + } + + +def write_fixture_json(path, payload): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload)) + + +def save_records(saved, change_link=None): + """Keep downstream byte hashes and link prefixes valid after one mutation.""" + + def save(path, payload): + text = json.dumps(payload) + if saved.serialized.get(path) != text: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + saved.serialized[path] = text + + links = [] + for n, payload in enumerate(saved.archives, 1): + payload["attempts"] = deepcopy(links) + path = saved.folder / "attempts" / f"h2-{n}.json" + save(path, payload) + try: + link = nt.entry(path, payload, n) + except (AttributeError, TypeError, ValueError, KeyError): + # Malformed turn evidence must reach the startup reader, rather + # than fail while this fixture refreshes otherwise valid links. + link = deepcopy(saved.original_links[n - 1]) + link["sha256"] = nt.sha256_file(path) + if change_link is not None: + change_link(link) + links.append(link) + for model, payload in saved.finals.items(): + payload["attempts"] = deepcopy(links) if model == nt.CHEAP[0] else [] + path = saved.root / model / "none/confirmatory/households/h2.json" + save(path, payload) + saved.original_links = links + + +@pytest.fixture(scope="module") +def treatment_templates(tmp_path_factory): + """Build actual treatment templates once and retain the installed versions.""" + directory = tmp_path_factory.mktemp("saved-record-board") + versions = nt.pilot._versions() + baseline = directory / "baseline-exclusions.json" + baseline.write_text('{"exclusions": []}\n') + design = { + "draw_order": ["h2", "outside"], + "baseline_exclusions": { + "path": str(baseline), + "sha256": nt.sha256_file(baseline), + }, + "no_tools": { + "runs": "", + "folder": "none", + "stage": "confirmatory", + }, + } + with pytest.MonkeyPatch.context() as patch: + patch.setattr(nt.pilot, "_versions", lambda: versions) + write_board_fixture({"design": design}, directory / "board") + design["draw_order"] = ["h2"] + board = nt.pilot.load_board(Path(design["board"])) + finals = { + model: full_record(board, model, design["engine"]) for model in nt.CHEAP + } + yield design, finals + + +@pytest.fixture +def saved_records(tmp_path, treatment_templates): + template, template_finals = treatment_templates + design, finals = deepcopy(template), deepcopy(template_finals) + design["no_tools"]["runs"] = str(tmp_path / "no-tools") + archives = [deepcopy(finals[nt.CHEAP[0]]) for _ in range(2)] + for payload in archives: + payload.update(status="provider_error", error="APIConnectionError") + root = Path(design["no_tools"]["runs"]) + folder = root / nt.CHEAP[0] / "none/confirmatory/households" + saved = SimpleNamespace( + design=design, + root=root, + folder=folder, + finals=finals, + archives=archives, + original_links=[], + serialized={}, + paths={"final": folder / "h2.json", "archive": folder / "attempts/h2-1.json"}, + ) + save_records(saved) + return saved + + +def target_record(saved, reader): + return saved.finals[nt.CHEAP[0]] if reader == "final" else saved.archives[0] + + +def no_reply(payload, truncated=False): + payload["predictions"] = {"snap": None} + payload["turns"] = ( + [{**payload["turns"][0], "truncated_and_retried": True}] if truncated else [] + ) + for key in OPTIONAL_EVIDENCE: + payload.pop(key, None) + + +BAD_VALUES = { + "model": "synthetic-other-model", + "arm": "pe", + "prompt_version": "v1", + "prompt_wording_sha256": "wrong", + "tool_notice_sha256": "wrong", + "tool_features": [], + "calculate_tool_sha256": "wrong", + "variables": ["other"], + "contract": "synthetic-other-contract", + "board_prompt_sha256": "wrong", + "sent_board_prompt_sha256": "wrong", + "messages": {"content": "wrong"}, + "predictions": [], + "turns": None, +} +MUTATIONS = ( + [f"missing:{field}" for field in REQUIRED_FIELDS] + + [f"wrong:{field}" for field in BAD_VALUES] + + [ + "malformed-turn", + "missing-provider-model", + "wrong-provider-model", + "answer-without-completed-turn", + "empty-messages", + "malformed-message", + "wrong-sent-prompt", + ] + + [f"fallback:{field}" for field in OPTIONAL_EVIDENCE] +) + + +def mutate_record(payload, mutation): + kind, _, field = mutation.partition(":") + if kind == "missing": + payload.pop(field) + elif kind == "wrong": + payload[field] = deepcopy(BAD_VALUES[field]) + elif kind == "fallback": + no_reply(payload) + payload[field] = [] if field == "messages" else "wrong" + if field == "messages": + payload[field] = {"content": "wrong"} + elif mutation == "malformed-turn": + payload["turns"] = [None] + elif mutation == "missing-provider-model": + payload["turns"] = [{}] + elif mutation == "wrong-provider-model": + payload["turns"][0]["provider_resolved_model"] = "synthetic-other-model" + elif mutation == "answer-without-completed-turn": + payload["turns"] = [] + elif mutation == "empty-messages": + payload["messages"] = [] + elif mutation == "malformed-message": + payload["messages"] = [None] + else: + assert mutation == "wrong-sent-prompt" + payload["messages"][0]["content"] = "synthetic other household" + + +def assert_boundary(saved, boundary, refuses, expected_spend=0.5): + provider = Provider() + try: + if boundary == "startup": + if refuses is not None: + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + saved.design, + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail( + "unusable saved evidence must refuse before execution" + ) + assert str(refuses) in str(caught.value) + else: + with nt.Stage( + saved.design, + board=None, + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.spend.spent_usd == pytest.approx(expected_spend) + assert all(plan.standing(model, "h2") for model in nt.CHEAP) + else: + problems = nt.stage_problems(saved.design) + if refuses is not None: + assert any(str(refuses) in problem for problem in problems), problems + else: + assert problems == [] + finally: + assert provider.calls == [] + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("mutation", MUTATIONS) +def test_saved_treatment_refuses_with_path(saved_records, reader, boundary, mutation): + saved = saved_records + mutate_record(target_record(saved, reader), mutation) + save_records(saved) + assert_boundary(saved, boundary, saved.paths[reader]) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("number", [True, 1.0], ids=["boolean", "float"]) +def test_saved_attempt_link_number_requires_integer(saved_records, boundary, number): + saved = saved_records + + def replace_number(link): + if link["n"] == 1: + link["n"] = number + + save_records(saved, replace_number) + assert_boundary(saved, boundary, saved.paths["final"]) + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize( + "form", ["full", "fallback", "truncated-only", "input-message"] +) +def test_valid_saved_treatment_preserves_spend(saved_records, reader, boundary, form): + saved = saved_records + payload = target_record(saved, reader) + if form in ("fallback", "truncated-only"): + no_reply(payload, truncated=form == "truncated-only") + elif form == "input-message": + first = payload["messages"][0] + first["input"] = first.pop("content") + save_records(saved) + assert_boundary(saved, boundary, None) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("form", ["extra-key", "absent-null-status"]) +def test_report_compatible_attempt_links_are_accepted(saved_records, boundary, form): + saved = saved_records + if form == "absent-null-status": + no_reply(saved.archives[0]) + saved.archives[0].pop("status") + + def change_link(link): + if form == "extra-key": + link["synthetic_extra"] = True + elif link["n"] == 1: + link.pop("status") + + save_records(saved, change_link) + assert_boundary(saved, boundary, None) + + +def add_board_household(saved, sid): + board = nt.pilot.load_board(Path(saved.design["board"])) + assert sid in board["scenarios"] and sid not in saved.design["draw_order"] + return board + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_cheap_final_outside_draw_refuses_with_path(saved_records, boundary): + saved = saved_records + board = add_board_household(saved, "outside") + payload = full_record(board, nt.CHEAP[0], saved.design["engine"], "outside") + path = saved.folder / "outside.json" + write_fixture_json(path, payload) + assert_boundary(saved, boundary, path) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_cheap_final_missing_predictions_refuses_with_path(saved_records, boundary): + saved = saved_records + payload = target_record(saved, "final") + no_reply(payload) + payload.pop("predictions") + save_records(saved) + assert_boundary(saved, boundary, saved.paths["final"]) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("reader", ["archive", "optional"]) +def test_absent_predictions_without_reply_remain_accepted( + saved_records, boundary, reader +): + saved = saved_records + if reader == "optional": + board = nt.pilot.load_board(Path(saved.design["board"])) + payload = full_record(board, "gpt-6-sol", saved.design["engine"]) + saved.finals["gpt-6-sol"] = payload + else: + payload = target_record(saved, "archive") + no_reply(payload) + payload.pop("predictions") + save_records(saved) + assert_boundary( + saved, boundary, None, expected_spend=0.6 if reader == "optional" else 0.5 + ) + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_saved_call_classification_failure_refuses_with_path( + saved_records, reader, boundary +): + saved = saved_records + payload = target_record(saved, reader) + payload["status"] = "complete" + payload["turns"][0]["calls"] = [None] + save_records(saved) + assert_boundary(saved, boundary, saved.paths[reader]) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +@pytest.mark.parametrize("number", [True, 1.0], ids=["boolean", "float"]) +def test_archive_prefix_numeric_equality_remains_accepted( + saved_records, boundary, number +): + saved = saved_records + second = saved.archives[1] + second["attempts"][0]["n"] = number + path = saved.folder / "attempts/h2-2.json" + write_fixture_json(path, second) + final = target_record(saved, "final") + final["attempts"][1]["sha256"] = nt.sha256_file(path) + write_fixture_json(saved.paths["final"], final) + assert_boundary(saved, boundary, None) + + +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_nonobject_archive_prefix_refuses_with_path(saved_records, boundary): + saved = saved_records + second = saved.archives[1] + second["attempts"][0] = None + path = saved.folder / "attempts/h2-2.json" + write_fixture_json(path, second) + final = target_record(saved, "final") + final["attempts"][1]["sha256"] = nt.sha256_file(path) + write_fixture_json(saved.paths["final"], final) + assert_boundary(saved, boundary, path) + + +@pytest.mark.parametrize("history", ["interrupted-archive", "rerunnable-final"]) +def test_real_treatment_resume_preserves_attempts_and_spend(saved_records, history): + saved = saved_records + first = saved.paths["archive"] + second = saved.folder / "attempts/h2-2.json" + final = saved.paths["final"] + prior_bytes = first.read_bytes() + second.unlink() + final.unlink() + if history == "rerunnable-final": + first.unlink() + final.write_bytes(prior_bytes) + board = nt.pilot.load_board(Path(saved.design["board"])) + calls = [] + + def fake_household( + scenario, variables, tool_call, completion_fn, spend, *, profile + ): + calls.append((profile.name, scenario.id)) + assert variables == board["variables"][scenario.id] + spend.add(0.1) + return full_record(board, profile.name, saved.design["engine"], scenario.id) + + with nt.Stage( + saved.design, + board=None, + models=[nt.CHEAP[0]], + run_household=fake_household, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.spend.spent_usd == pytest.approx(0.3) + assert plan.execute() == 0 + assert plan.spend.spent_usd == pytest.approx(0.4) + assert calls == [(nt.CHEAP[0], "h2")] + assert first.read_bytes() == prior_bytes + assert not second.exists() + payload = json.loads(final.read_text()) + assert payload["status"] == "complete" + assert payload["attempts"] == [nt.entry(first, saved.archives[0], 1)] + assert nt.stage_problems(saved.design) == [] From 4fb80d9b74194739c7e3f9d2127f86daa32bcd08 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 02:59:55 -0400 Subject: [PATCH 31/48] Confirmatory runner tests: build preflight finals the treatment checks accept 0300257c checks each saved no-tools record against the pinned none arm, and a cheap model's final must hold a predictions object. complete_preflight_design still wrote bare finals, so every preflight refused each model's first household and five tests never reached their planted fault. The fixture now builds each final through the harness with the records tests' full_record and writes the board files the no-tools loader checks. The harness and ceiling faults use the form a failed call saves: no answer and no prompt evidence. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 44 +++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 0b39c5a6..10847f52 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2174,7 +2174,32 @@ def test_the_preflight_names_a_missing_field(): def complete_preflight_design(w, tmp_path): - """Each calculator preflight starts after synthetic standing no-tools finals.""" + """Each calculator preflight starts after standing no-tools finals that pass + the no-tools script's treatment checks: built through the harness from the + fake board, whose files that script's loader checks before it calls the + patched harness loader.""" + import csv + + from policybench.scenarios import scenario_to_dict + from tests.test_confirmatory_no_tools_records import full_record + + board = Path(w.design["board"]) + (board / "reference_outputs.csv.meta.json").write_text( + json.dumps( + { + "programs": ["snap"], + "revisions": [ + {"kind": "engine_upgrade", "engine_version": w.design["engine"]} + ], + } + ) + ) + with (board / "scenarios.csv").open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["scenario_json"]) + writer.writerows( + [json.dumps(scenario_to_dict(w.board["scenarios"][sid]))] for sid in POOL + ) pool = tmp_path / "pool.json" pool.write_text(json.dumps([{"id": s} for s in POOL])) design = dict( @@ -2190,18 +2215,15 @@ def complete_preflight_design(w, tmp_path): ) for model in cr.CHEAP: for sid in design["draw_order"]: + final = full_record(w.board, model, design["engine"], sid) + final["predictions"] = {v: 1.0 for v in w.board["variables"][sid]} + final["totals"] = {"cost_usd": 0.0} cr.write_json( Path(design["no_tools"]["runs"]) / model / "none/confirm/households" / f"{sid}.json", - { - "scenario_id": sid, - "status": "complete", - "turns": [{}], - "totals": {"cost_usd": 0.0}, - "attempts": [], - }, + final, ) return design @@ -2260,9 +2282,15 @@ def test_preflight_refuses_an_unfinished_no_tools_stage( if fault == "history": payload.pop("attempts") else: + # A failed call leaves no reply: no answer and no prompt evidence. + from tests.test_confirmatory_no_tools_records import OPTIONAL_EVIDENCE + + for key in OPTIONAL_EVIDENCE: + payload.pop(key) payload.update( status="budget_stop" if fault == "ceiling" else "provider_error", error="BadRequestError" if fault == "harness" else None, + predictions={v: None for v in payload["variables"]}, turns=[], ) path.write_text(json.dumps(payload)) From 5e334d9a516b228c964e480845dcfb961aeaf295 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 06:52:38 -0400 Subject: [PATCH 32/48] Confirmatory runner: reproduce unreadable evidence refusals Record provider-free regressions for denied history enumeration, saved-evidence metadata failures, hash and decoder buffer disagreement, and board engine failures through calculator preflight and main. Each unusable file or folder must produce a refusal naming its path before providers. Baseline and eligible-pool hashes must bind the bytes decoded, and main must print all collected problems and return 1. The original production code fails all 38 regression cases. Two controls establish that runner startup, its stage check, and the separate report accept object-form variables with the same saved spend. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_r15_baseline.py | 162 +++++++++++++++ tests/test_confirmatory_r15_engine.py | 90 +++++++++ tests/test_confirmatory_r15_files.py | 249 ++++++++++++++++++++++++ 3 files changed, 501 insertions(+) create mode 100644 tests/test_confirmatory_r15_baseline.py create mode 100644 tests/test_confirmatory_r15_engine.py create mode 100644 tests/test_confirmatory_r15_files.py diff --git a/tests/test_confirmatory_r15_baseline.py b/tests/test_confirmatory_r15_baseline.py new file mode 100644 index 00000000..8d4aceb9 --- /dev/null +++ b/tests/test_confirmatory_r15_baseline.py @@ -0,0 +1,162 @@ +"""Hash-bound inputs and report-compatible variables stay provider-free.""" + +from __future__ import annotations + +import importlib.util +import io +import os +import sys +from pathlib import Path + +import pytest + +from tests.test_confirmatory_no_tools import Provider, nt +from tests.test_confirmatory_no_tools import setup as _setup_fixture +from tests.test_confirmatory_no_tools_records import ( + assert_boundary, + save_records, +) +from tests.test_confirmatory_no_tools_records import ( + saved_records as _saved_records_fixture, +) +from tests.test_confirmatory_no_tools_records import ( + treatment_templates as _treatment_templates_fixture, +) +from tests.test_confirmatory_run import N_DRAW, World, complete_preflight_design, cr + +setup = _setup_fixture +saved_records = _saved_records_fixture +treatment_templates = _treatment_templates_fixture + + +def replace_text_read(monkeypatch, path, replacement): + """Simulate replacement after the first byte read, without changing disk.""" + original_bytes, original_text = Path.read_bytes, Path.read_text + reads = [] + + def read_bytes(source): + data = original_bytes(source) + if source == path: + reads.append(("bytes", data)) + return data + + def read_text(source, *args, **kwargs): + if source == path: + reads.append(("text", replacement)) + return replacement + return original_text(source, *args, **kwargs) + + monkeypatch.setattr(Path, "read_bytes", read_bytes) + monkeypatch.setattr(Path, "read_text", read_text) + return reads + + +@pytest.mark.parametrize("data", [b"not JSON", b"\xff"], ids=["json", "utf8"]) +def test_baseline_replacement_cannot_supply_unhashed_decoded_bytes( + setup, monkeypatch, data +): + path = Path(setup["design"]["baseline_exclusions"]["path"]) + path.write_bytes(data) + setup["design"]["baseline_exclusions"]["sha256"] = nt.sha256_file(path) + reads = replace_text_read(monkeypatch, path, '{"exclusions": []}') + provider = Provider() + with pytest.raises(nt.StateError) as caught: + with nt.Stage( + setup["design"], + board=setup["board"], + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ): + pytest.fail("the declared baseline bytes must be decoded") + assert str(path) in str(caught.value) + assert reads == [("bytes", data)] + assert provider.calls == [] + + +def test_baseline_hash_and_decoder_share_one_valid_buffer(setup, monkeypatch): + path = Path(setup["design"]["baseline_exclusions"]["path"]) + data = path.read_bytes() + reads = replace_text_read(monkeypatch, path, "not JSON") + provider = Provider() + with nt.Stage( + setup["design"], + board=setup["board"], + run_household=provider, + completion_for=lambda profile: None, + out=io.StringIO(), + ) as plan: + assert plan.spend.spent_usd == 0 + assert reads == [("bytes", data)] + assert provider.calls == [] + + +def pool_design(tmp_path, monkeypatch): + monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + world = World(tmp_path, monkeypatch, models=(cr.CHEAP[0],)) + return world, complete_preflight_design(world, tmp_path) + + +@pytest.mark.parametrize("data", [b"not JSON", b"\xff"], ids=["json", "utf8"]) +def test_preflight_pool_replacement_cannot_supply_unhashed_decoded_bytes( + tmp_path, monkeypatch, data +): + world, design = pool_design(tmp_path, monkeypatch) + path = Path(design["eligible_pool"]["path"]) + replacement = path.read_text() + path.write_bytes(data) + design["eligible_pool"]["sha256"] = cr.sha256_file(path) + reads = replace_text_read(monkeypatch, path, replacement) + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(str(path) in problem for problem in problems), problems + assert reads == [("bytes", data)] + assert world.provider.count == {} + + +def test_preflight_pool_hash_and_decoder_share_one_valid_buffer(tmp_path, monkeypatch): + world, design = pool_design(tmp_path, monkeypatch) + path = Path(design["eligible_pool"]["path"]) + data = path.read_bytes() + reads = replace_text_read(monkeypatch, path, "[]") + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert not any("the draw, recomputed" in problem for problem in problems), problems + assert not any(str(path) in problem for problem in problems), problems + assert reads == [("bytes", data)] + assert world.provider.count == {} + + +@pytest.fixture(scope="module") +def report_module(): + configured = os.environ.get("CONFIRMATORY_REPORT") + if not configured: + pytest.skip( + "set CONFIRMATORY_REPORT for the separate report's acceptance control" + ) + path = Path(configured) + name = "confirmatory_report_r15_variables_control" + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + with pytest.MonkeyPatch.context() as patch: + patch.syspath_prepend(str(path.parent)) + patch.setitem(sys.modules, name, module) + spec.loader.exec_module(module) + yield module + + +@pytest.mark.parametrize("reader", ["final", "archive"]) +def test_variables_object_is_accepted_by_runner_and_report( + saved_records, report_module, monkeypatch, reader +): + saved = saved_records + payload = saved.finals[nt.CHEAP[0]] if reader == "final" else saved.archives[0] + payload["variables"] = {"snap": {"synthetic_metadata": True}} + save_records(saved) + assert_boundary(saved, "startup", None) + assert_boundary(saved, "preflight", None) + saved.design["treatment_harness"] = "synthetic-harness" + monkeypatch.setattr(report_module, "harness_module", lambda path: nt.pilot) + problems = [] + costs = report_module.no_tools_costs(saved.design, problems) + assert problems == [] + assert costs["registered_usd"] + costs["other_usd"] == pytest.approx(0.5) diff --git a/tests/test_confirmatory_r15_engine.py b/tests/test_confirmatory_r15_engine.py new file mode 100644 index 00000000..11910e52 --- /dev/null +++ b/tests/test_confirmatory_r15_engine.py @@ -0,0 +1,90 @@ +"""Engine sidecars must refuse a calculator start before any provider call.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tests.test_confirmatory_run import ( + ENGINE, + complete_preflight_design, + cr, + pilot, +) +from tests.test_confirmatory_run import ( + fast as _fast_fixture, +) +from tests.test_confirmatory_run import ( + world as _world_fixture, +) + +world = _world_fixture +fast = _fast_fixture + + +REAL_BOARD_ENGINE = pilot.board_engine + + +def engine_fault(world, tmp_path, monkeypatch, fault): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + metadata = Path(design["board"]) / "reference_outputs.csv.meta.json" + monkeypatch.setattr(pilot, "board_engine", REAL_BOARD_ENGINE) + monkeypatch.setattr(pilot, "_versions", lambda: {"policyengine-us": ENGINE}) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + + def no_run(*args, **kwargs): + pytest.fail("a refused preflight must not construct or call a provider") + + monkeypatch.setattr(cr, "Run", no_run) + monkeypatch.setattr(pilot, "run_household", no_run) + if fault == "malformed": + metadata.write_text("{") + else: + read_text = Path.read_text + + def denied(path, *args, **kwargs): + if path == metadata: + raise PermissionError("engine metadata read denied") + return read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", denied) + return w, design, metadata + + +@pytest.mark.parametrize("fault", ["malformed", "unreadable"]) +@pytest.mark.parametrize("entrypoint", ["preflight", "main"]) +def test_engine_metadata_refuses_full_calculator_start( + world, tmp_path, monkeypatch, capsys, fault, entrypoint +): + w, design, metadata = engine_fault(world, tmp_path, monkeypatch, fault) + if entrypoint == "preflight": + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert str(metadata) in "\n".join(problems) + else: + design_path = tmp_path / "design.json" + design_path.write_text(json.dumps(design)) + assert cr.main(["run", str(design_path)]) == 1 + output = capsys.readouterr().out + assert "The run cannot start" in output + assert str(metadata) in output + assert not w.provider.count + + +@pytest.mark.parametrize("fault", ["malformed", "unreadable"]) +def test_main_names_engine_reread_failure_and_preserves_all_problems( + world, tmp_path, monkeypatch, capsys, fault +): + w, design, metadata = engine_fault(world, tmp_path, monkeypatch, fault) + # Isolate main's final engine reread, which must retain preflight's problems. + monkeypatch.setattr(cr, "preflight", lambda *_: ["earlier preflight problem"]) + design_path = tmp_path / "design.json" + design_path.write_text(json.dumps(design)) + assert cr.main(["run", str(design_path)]) == 1 + output = capsys.readouterr().out + assert "The run cannot start" in output + assert "earlier preflight problem" in output + assert str(metadata) in output + assert not w.provider.count diff --git a/tests/test_confirmatory_r15_files.py b/tests/test_confirmatory_r15_files.py new file mode 100644 index 00000000..18a9d75a --- /dev/null +++ b/tests/test_confirmatory_r15_files.py @@ -0,0 +1,249 @@ +"""File and folder failures must remain named refusals before providers.""" + +import errno +import io +import json +import os +from pathlib import Path + +import pytest + +from tests.test_confirmatory_no_tools import ( + Provider, + nt, + stage, + write_board_fixture, +) +from tests.test_confirmatory_no_tools import ( + setup as _setup_fixture, +) +from tests.test_confirmatory_run import ( + complete_preflight_design, + cr, +) +from tests.test_confirmatory_run import ( + fast as _fast_fixture, +) +from tests.test_confirmatory_run import ( + world as _world_fixture, +) + +setup = _setup_fixture +world = _world_fixture +fast = _fast_fixture + + +def deny_listing(monkeypatch, target): + for name in ("scandir", "listdir"): + original = getattr(os, name) + + def denied(path, *args, _original=original, **kwargs): + if not isinstance(path, int) and Path(path) == target: + raise PermissionError("synthetic denied listing") + return _original(path, *args, **kwargs) + + monkeypatch.setattr(os, name, denied) + + +def deny_stat(monkeypatch, target): + original = os.stat + + def denied(path, *args, **kwargs): + if not isinstance(path, int) and Path(path) == target: + # Path predicates suppress ELOOP on the pinned Python 3.12. + raise OSError(errno.ELOOP, "synthetic inaccessible metadata") + return original(path, *args, **kwargs) + + monkeypatch.setattr(os, "stat", denied) + + +@pytest.mark.parametrize("which", ["finals", "attempts"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_unlistable_optional_history_refuses( + setup, world, tmp_path, monkeypatch, which, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + spec = design["no_tools"] + folder = ( + Path(spec["runs"]) / "gpt-6-sol" / spec["folder"] / spec["stage"] / "households" + ) + target = folder if which == "finals" else folder / "attempts" + target.mkdir(parents=True) + deny_listing(monkeypatch, target) + if boundary == "startup": + with pytest.raises(nt.StateError, match=str(target)): + with stage(setup, provider): + pass + else: + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + assert provider.calls == [] + + +@pytest.mark.parametrize("which", ["root-list", "root-stat", "model-stat"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_model_inventory_failure_refuses( + setup, world, tmp_path, monkeypatch, which, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + root = Path(design["no_tools"]["runs"]) + (root / "gpt-6-sol").mkdir(parents=True, exist_ok=True) + target = root / "gpt-6-sol" if which == "model-stat" else root + (deny_listing if which == "root-list" else deny_stat)(monkeypatch, target) + if boundary == "startup": + with state_for_loading(setup) as plan: + with pytest.raises(nt.StateError, match=str(target)): + plan._load_state(setup["board"]) + else: + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + assert provider.calls == [] + + +def state_for_loading(setup): + # Isolate _load_state from the constructor's mkdir and lock operations. + import threading + + plan = object.__new__(nt.Stage) + plan.design = setup["design"] + plan.draw_order = setup["design"]["draw_order"] + plan.root = Path(setup["design"]["no_tools"]["runs"]) + plan.folder, plan.stage = "none", "confirmatory" + plan.models = nt.CHEAP + plan.stopped = threading.Event() + plan._lock_file = None + return plan + + +@pytest.mark.parametrize("which", ["folder-list", "folder-stat", "log-stat"]) +def test_calculator_summary_inventory_failure_refuses(world, monkeypatch, which): + w = world(models=(cr.CHEAP[0],)) + lanes = w.run.lanes[cr.CHEAP[0]] + target = ( + next(iter(lanes.households_dir.values())) + if which != "log-stat" + else lanes.folder / "run-log.jsonl" + ) + target.parent.mkdir(parents=True, exist_ok=True) + if which == "folder-list": + target.mkdir(exist_ok=True) + (deny_listing if which == "folder-list" else deny_stat)(monkeypatch, target) + with pytest.raises(nt.StateError, match=str(target)): + lanes.summary() + assert not w.provider.count + + +@pytest.mark.parametrize( + "which", ["model", "arm", "stage", "households", "run-log", "host-log"] +) +def test_calculator_preflight_inventory_failure_refuses( + world, tmp_path, monkeypatch, which +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + folder = Path(design["runs"]) / cr.CHEAP[0] + targets = { + "model": folder, + "arm": folder / "pe3", + "stage": folder / "pe3/confirm", + "households": folder / "pe3/confirm/households", + "run-log": folder / "run-log.jsonl", + "host-log": Path(design["runs"]) / "host/host-log.jsonl", + } + target = targets[which] + if which.endswith("log"): + target.parent.mkdir(parents=True, exist_ok=True) + deny_stat(monkeypatch, target) + else: + target.mkdir(parents=True, exist_ok=True) + deny_listing(monkeypatch, target) + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + + +@pytest.mark.parametrize("which", ["model", "arm", "stage", "households"]) +def test_calculator_preflight_directory_stat_failure_refuses( + world, tmp_path, monkeypatch, which +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + folder = Path(design["runs"]) / cr.CHEAP[0] + target = ( + folder + / { + "model": "", + "arm": "pe3", + "stage": "pe3/confirm", + "households": "pe3/confirm/households", + }[which] + ) + target.mkdir(parents=True, exist_ok=True) + deny_stat(monkeypatch, target) + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + + +def test_preflight_unreadable_preregistration_refuses(world, tmp_path, monkeypatch): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + target = Path(design["preregistration"]) + original = Path.read_bytes + + def denied(path): + if path == target: + raise PermissionError("synthetic denied read") + return original(path) + + monkeypatch.setattr(Path, "read_bytes", denied) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + + +def test_archive_target_stat_failure_refuses(setup, monkeypatch): + provider = Provider() + with stage(setup, provider) as plan: + model, sid = nt.CHEAP[0], "h2" + folder = plan.households(model) + folder.mkdir(parents=True) + payload = { + "scenario_id": sid, + "status": "provider_error", + "error": "APIConnectionError", + "turns": [], + "attempts": [], + "totals": {"cost_usd": 0.1}, + } + (folder / f"{sid}.json").write_text(json.dumps(payload)) + plan.histories[model][sid] = {"final": payload, "entries": []} + target = folder / "attempts" / f"{sid}-1.json" + deny_stat(monkeypatch, target) + with pytest.raises(nt.StateError, match=str(target)): + plan.archive(model, sid) + assert (folder / f"{sid}.json").read_text() == json.dumps(payload) + assert provider.calls == [] + + +def test_board_exclusions_stat_failure_refuses(setup, tmp_path, monkeypatch): + write_board_fixture(setup, tmp_path / "board") + target = Path(setup["design"]["board"]) / "reference_exclusions.json" + deny_stat(monkeypatch, target) + provider = Provider() + with pytest.raises(nt.StateError, match=str(target)): + with nt.Stage(setup["design"], run_household=provider, out=io.StringIO()): + pass + assert provider.calls == [] From d31c1d7deb71e996900caabc26a616cde51384c7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 06:55:34 -0400 Subject: [PATCH 33/48] Confirmatory runner: bind decoded inputs and name evidence failures Enumerate saved histories and calculator evidence explicitly. Missing optional folders retain empty inventories; covered listing and stat failures must name their paths. Preserve sorted JSON histories, saved spend and attempt decisions. Guard exclusions metadata, model discovery, archive overwrite checks, summaries, calculator records and host logs, and the preregistration hash read. Read baseline exclusions and the eligible pool once each. Their hashes must bind the exact buffer decoded with strict UTF-8. Guard both calculator engine reads with the metadata sidecar path. Preflight retains its collected problems, and main prints every problem and returns 1. Valid board engine checks and provider execution retain their existing paths. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 68 +++++++++++++--- scripts/confirmatory_run.py | 120 +++++++++++++++++++++------- tests/test_confirmatory_no_tools.py | 8 +- 3 files changed, 148 insertions(+), 48 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index acdf0f88..cf67e653 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -40,6 +40,7 @@ import os import re import signal +import stat import sys import threading import time @@ -132,6 +133,51 @@ def _board_input(path: Path | str, read): raise StateError(f"{path}: {error}") from error +def saved_stat(path: Path): + """Only a missing path means absent evidence; other stat failures refuse.""" + + def read(): + try: + return path.stat() + except FileNotFoundError: + return None + + return _board_input(path, read) + + +def saved_exists(path: Path) -> bool: + return saved_stat(path) is not None + + +def saved_is_dir(path: Path) -> bool: + metadata = saved_stat(path) + return metadata is not None and stat.S_ISDIR(metadata.st_mode) + + +def saved_children(folder: Path) -> list[Path]: + """Enumerate explicitly so a failed scan cannot silently erase evidence.""" + metadata = saved_stat(folder) + if metadata is None: + return [] + if not stat.S_ISDIR(metadata.st_mode): + raise StateError(f"{folder}: evidence folder is not a directory") + + def read(): + try: + with os.scandir(folder) as entries: + return [folder / entry.name for entry in entries] + except FileNotFoundError: + return [] + + return _board_input(folder, read) + + +def saved_json_paths(folder: Path) -> list[Path]: + return sorted( + path for path in saved_children(folder) if path.name.endswith(".json") + ) + + def load_board(path: Path, engine: str) -> dict: """Check board inputs locally, then use the harness's unchanged loader.""" from policybench.reference_exclusions import ( @@ -176,7 +222,7 @@ def require_engine(): reference_path = path / "reference_outputs.csv" reference = _board_input(reference_path, lambda: csv_frame(reference_path)) exclusions_path = path / "reference_exclusions.json" - if _board_input(exclusions_path, exclusions_path.exists): + if saved_exists(exclusions_path): _board_input( exclusions_path, lambda: object_json(checked_bytes(exclusions_path).decode("utf-8")), @@ -390,12 +436,12 @@ def _attempts_agree(listed: list, entries: list) -> bool: def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], float]: """Validate a model's histories, including an interrupted archive, and sum spend.""" archives: dict[str, list[tuple[int, Path]]] = {} - for path in sorted((folder / "attempts").glob("*.json")): + for path in saved_json_paths(folder / "attempts"): match = re.fullmatch(r"(.+)-([1-9][0-9]*)\.json", path.name) if not match: raise StateError(f"{path}: invalid attempt number") archives.setdefault(match[1], []).append((int(match[2]), path)) - finals = {path.stem: path for path in sorted(folder.glob("*.json"))} + finals = {path.stem: path for path in saved_json_paths(folder)} states, spent = {}, 0.0 for sid in sorted(set(finals) | set(archives)): paths = sorted(archives.get(sid, [])) @@ -471,8 +517,7 @@ def stage_problems(design: dict) -> list[str]: folder, stage = component(spec["folder"]), component(spec["stage"]) draw_order = [component(sid) for sid in design["draw_order"]] models = set(CHEAP) - if root.is_dir(): - models.update(path.name for path in root.iterdir() if path.is_dir()) + models.update(path.name for path in saved_children(root) if saved_is_dir(path)) except (OSError, KeyError, TypeError, ValueError) as error: return [f"no-tools stage: {error}"] board = None @@ -590,9 +635,10 @@ def __init__( ): raise StateError("baseline exclusions need path and sha256") baseline_path = Path(baseline["path"]) - if sha256_file(baseline_path) != baseline.get("sha256"): + baseline_bytes = baseline_path.read_bytes() + if hashlib.sha256(baseline_bytes).hexdigest() != baseline.get("sha256"): raise StateError("baseline exclusions do not hash as declared") - payload = json.loads(baseline_path.read_text()) + payload = json.loads(baseline_bytes.decode("utf-8")) exclusions = ( payload.get("exclusions") if isinstance(payload, dict) else None ) @@ -644,11 +690,7 @@ def _load_state(self, board) -> None: ) self.histories = {} spent = 0.0 - folders = ( - {p.name for p in self.root.iterdir() if p.is_dir()} - if self.root.is_dir() - else set() - ) + folders = {p.name for p in saved_children(self.root) if saved_is_dir(p)} for model in sorted(folders | set(self.models)): states, cost = history( self.households(model), @@ -753,7 +795,7 @@ def archive(self, model: str, sid: str) -> bool: source = folder / f"{sid}.json" target = folder / "attempts" / f"{sid}-{n}.json" target.parent.mkdir(parents=True, exist_ok=True) - if target.exists(): + if saved_exists(target): raise StateError(f"{target}: would overwrite an archived attempt") link = entry(source, record, n) link["file"] = f"attempts/{target.name}" diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 3949f239..c6daacd1 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -1558,14 +1558,21 @@ def timeout(profile): } def summary(self) -> dict: + from confirmatory_no_tools import _board_input, saved_exists, saved_json_paths + records = [] for label, folder in self.households_dir.items(): - for path in sorted(folder.glob("*.json")) if folder.is_dir() else []: - records.append(json.loads(path.read_text())) + for path in saved_json_paths(folder): + records.append(_board_input(path, lambda: json.loads(path.read_text()))) lines = [] path = self.folder / "run-log.jsonl" - if path.exists(): - lines = [json.loads(x) for x in path.read_text().splitlines() if x.strip()] + if saved_exists(path): + lines = _board_input( + path, + lambda: [ + json.loads(x) for x in path.read_text().splitlines() if x.strip() + ], + ) attempts = [x for x in lines if "event" not in x] count: dict[str, int] = {} for x in attempts: @@ -2162,7 +2169,15 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: if missing: return [f"the design has no {', '.join(missing)}"] problems = [] - from confirmatory_no_tools import stage_problems + from confirmatory_no_tools import ( + BOARD_INPUT_ERRORS, + _board_input, + saved_children, + saved_exists, + saved_is_dir, + saved_json_paths, + stage_problems, + ) problems.extend(stage_problems(design)) try: @@ -2172,7 +2187,7 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: calculator_root ) or calculator_root.is_relative_to(no_tools_root): problems.append("the no-tools and calculator runs roots overlap") - except (KeyError, TypeError, OSError, ValueError) as error: + except (KeyError, TypeError, OSError, ValueError, RuntimeError) as error: problems.append(f"the no-tools runs root is invalid: {error}") command = design["anchor_command"] if ( @@ -2183,19 +2198,31 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: problems.append(f"the anchor command {command!r} is not a command here") if design.get("stated_engine_overrides"): problems.append("the confirmatory run states the board's engine: no overrides") - if sha256_file(Path(design["preregistration"])) != design["frozen_sha256"]: - problems.append("the frozen copy does not hash to frozen_sha256") - pool = design["eligible_pool"] - if sha256_file(Path(pool["path"])) != pool["sha256"]: - problems.append("the eligible pool does not hash as frozen") - else: - ids = [e["id"] for e in json.loads(Path(pool["path"]).read_text())] - if draw(master_seed(design["frozen_sha256"]), ids) != list( - design["draw_order"] + preregistration = Path(design["preregistration"]) + try: + if ( + _board_input(preregistration, lambda: sha256_file(preregistration)) + != design["frozen_sha256"] ): - problems.append( - "the draw, recomputed from the stream 'draw', is not the design's" - ) + problems.append("the frozen copy does not hash to frozen_sha256") + except ValueError as error: + problems.append(str(error)) + pool = design["eligible_pool"] + pool_path = Path(pool["path"]) + try: + pool_bytes = pool_path.read_bytes() + if sha256_bytes(pool_bytes) != pool["sha256"]: + problems.append(f"{pool_path}: the eligible pool does not hash as frozen") + else: + ids = [e["id"] for e in json.loads(pool_bytes.decode("utf-8"))] + if draw(master_seed(design["frozen_sha256"]), ids) != list( + design["draw_order"] + ): + problems.append( + "the draw, recomputed from the stream 'draw', is not the design's" + ) + except BOARD_INPUT_ERRORS as error: + problems.append(f"{pool_path}: cannot read the eligible pool: {error}") head, dirty = pilot._git("rev-parse", "HEAD"), pilot._git("status", "--porcelain") if head != design["run_commit"] or dirty: problems.append( @@ -2235,14 +2262,21 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: f"not {TREATMENT_COMMIT[:12]}" ) board = Path(design["board"]) - engine = pilot.board_engine(board) - if f"policyengine-us {engine}" != design["engine"]: + try: + engine = pilot.board_engine(board) + except BOARD_INPUT_ERRORS as error: problems.append( - f"the board's engine {engine} is not the design's {design['engine']}" + f"{board / 'reference_outputs.csv.meta.json'}: " + f"cannot read the board's engine: {error}" ) - installed = pilot._versions()["policyengine-us"] - if installed != engine: - problems.append(f"policyengine-us {installed} is installed, not {engine}") + else: + if f"policyengine-us {engine}" != design["engine"]: + problems.append( + f"the board's engine {engine} is not the design's {design['engine']}" + ) + installed = pilot._versions()["policyengine-us"] + if installed != engine: + problems.append(f"policyengine-us {installed} is installed, not {engine}") if set(design["arms"]) != set(ARMS) or set(design["frontier_arms"]) != set( FRONTIER_ARMS ): @@ -2262,12 +2296,27 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: if not os.environ.get(key): problems.append(f"{model}: {key} is not set") folder = Path(design["runs"]) / model - if (folder / "run-log.jsonl").exists() or any( - folder.glob("*/*/households/*.json") - ): - problems.append(f"{folder} already holds records; a run is never resumed") - if (Path(design["runs"]) / "host" / "host-log.jsonl").exists(): - problems.append("the runs folder already holds a host log") + try: + holds_records = saved_exists(folder / "run-log.jsonl") + if not holds_records: + holds_records = any( + saved_json_paths(stage / "households") + for arm in saved_children(folder) + if saved_is_dir(arm) + for stage in saved_children(arm) + if saved_is_dir(stage) + ) + if holds_records: + problems.append( + f"{folder} already holds records; a run is never resumed" + ) + except ValueError as error: + problems.append(str(error)) + try: + if saved_exists(Path(design["runs"]) / "host" / "host-log.jsonl"): + problems.append("the runs folder already holds a host log") + except ValueError as error: + problems.append(str(error)) ac, lid = host_power() if not (ac and lid): problems.append( @@ -2304,7 +2353,16 @@ def main(argv=None) -> int: return 1 models = CHEAP + FRONTIER problems = preflight(design, models) - pilot._require_engine(Path(design["board"])) + from confirmatory_no_tools import BOARD_INPUT_ERRORS + + board = Path(design["board"]) + try: + pilot._require_engine(board) + except BOARD_INPUT_ERRORS as error: + problems.append( + f"{board / 'reference_outputs.csv.meta.json'}: " + f"the board's engine check failed: {error}" + ) if problems: print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) return 1 diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 7bead2a2..b7da9487 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -1232,16 +1232,16 @@ def test_board_exclusions_exists_failure_refuses_with_path( write_board_fixture(setup, directory) exclusions = directory / "reference_exclusions.json" error = PermissionError(13, "synthetic unreadable board parent", str(exclusions)) - exists = Path.exists + stat = Path.stat checks = [] - def fail_exclusions_exists(path): + def fail_exclusions_stat(path, *args, **kwargs): if path == exclusions: checks.append(path) raise error - return exists(path) + return stat(path, *args, **kwargs) - monkeypatch.setattr(Path, "exists", fail_exclusions_exists) + monkeypatch.setattr(Path, "stat", fail_exclusions_stat) provider = Provider() try: with pytest.raises(nt.StateError) as caught: From cccc3817d030e7d7afeaf8583b73c8313391c70f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 07:05:12 -0400 Subject: [PATCH 34/48] Confirmatory runner: guard intermediate stage inventories Explicitly list each candidate calculator stage before checking its literal households child. A covered failure listing that parent must become a preflight problem naming the stage folder. Preserve the same household path, record filter and early refusal once saved calculator records are found. The final regression run exposed this missing parent listing after the first implementation: 39 cases passed and the denied-stage-listing case still failed. Keep the original failing regression and verify the corrected inventory and preflight paths. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index c6daacd1..9b8b6e08 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2284,6 +2284,12 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: for name in ("arms", "frontier_arms"): if len(set(design[name].values())) != len(design[name]): problems.append(f"two {name} share a folder") + + def household_paths(stage): + # Check the parent inventory even when its next component is literal. + saved_children(stage) + return saved_json_paths(stage / "households") + for model in models: if model not in design["caps"] or model not in design["reserves"]: problems.append(f"{model}: no cap or reserve") @@ -2300,7 +2306,7 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: holds_records = saved_exists(folder / "run-log.jsonl") if not holds_records: holds_records = any( - saved_json_paths(stage / "households") + household_paths(stage) for arm in saved_children(folder) if saved_is_dir(arm) for stage in saved_children(arm) From 4d93b4067e2a2fb1f9e4f36e3e9a1a4cab0ed5fb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 07:44:06 -0400 Subject: [PATCH 35/48] Confirmatory runner: name the new test modules for what they check The three test modules added for this round take descriptive names: test_confirmatory_unreadable_evidence.py holds the folder, listing and stat refusals; test_confirmatory_input_reads.py holds the single-buffer hash and decode checks for the baseline exclusions and the eligible pool, and the saved-variables control shared with the report; test_confirmatory_board_engine.py holds the board-engine refusals through preflight and main. The control's imported report module drops the round number from its name too. No test or production code changes. Co-Authored-By: Claude Opus 5.5 --- ...irmatory_r15_engine.py => test_confirmatory_board_engine.py} | 0 ...rmatory_r15_baseline.py => test_confirmatory_input_reads.py} | 2 +- ...ry_r15_files.py => test_confirmatory_unreadable_evidence.py} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename tests/{test_confirmatory_r15_engine.py => test_confirmatory_board_engine.py} (100%) rename tests/{test_confirmatory_r15_baseline.py => test_confirmatory_input_reads.py} (99%) rename tests/{test_confirmatory_r15_files.py => test_confirmatory_unreadable_evidence.py} (100%) diff --git a/tests/test_confirmatory_r15_engine.py b/tests/test_confirmatory_board_engine.py similarity index 100% rename from tests/test_confirmatory_r15_engine.py rename to tests/test_confirmatory_board_engine.py diff --git a/tests/test_confirmatory_r15_baseline.py b/tests/test_confirmatory_input_reads.py similarity index 99% rename from tests/test_confirmatory_r15_baseline.py rename to tests/test_confirmatory_input_reads.py index 8d4aceb9..10be2c16 100644 --- a/tests/test_confirmatory_r15_baseline.py +++ b/tests/test_confirmatory_input_reads.py @@ -134,7 +134,7 @@ def report_module(): "set CONFIRMATORY_REPORT for the separate report's acceptance control" ) path = Path(configured) - name = "confirmatory_report_r15_variables_control" + name = "confirmatory_report_variables_control" spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) with pytest.MonkeyPatch.context() as patch: diff --git a/tests/test_confirmatory_r15_files.py b/tests/test_confirmatory_unreadable_evidence.py similarity index 100% rename from tests/test_confirmatory_r15_files.py rename to tests/test_confirmatory_unreadable_evidence.py From 0ae6df56750bc6c720b8ec164085446cae99a8f4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 12:53:45 -0400 Subject: [PATCH 36/48] Confirmatory runner: refuse listed entries without metadata; one read per record Runner review r16 found two gaps. A no-tools model folder, a calculator arm or a calculator stage that a listing returned was dropped when its stat failed with ENOENT: a dangling link (an optional model's folder on an unmounted volume) or an entry removed after the listing. The report refuses such an entry. listed_is_dir now refuses every stat failure of a listed entry, at startup, in stage_problems and in the preflight. A missing root, households folder or attempts folder, which the code names itself, still reads as empty. history decoded each saved record with one read and hashed it with another, and Stage.archive hashed the runner's final again before the move. read_record now returns the digest of the bytes it decoded; an archive's link carries that digest, and the history keeps each final's digest (the bytes read at startup, or the bytes write_json wrote). Before the move, archive refuses a final whose bytes no longer hash to it. history now lists a model's finals before its attempts folder, as the report does, so a households folder that cannot be listed or searched is the folder both programs name (report review r15 finding 4). New tests: a dangling or vanished model entry (startup and preflight), a dangling or vanished calculator arm or stage (preflight), a file at or a stat failure on an optional model's households or attempts folder (startup and preflight), a second read of an archive or a final that differs from the hashed bytes, an archive link equal to the digest its final was read from, and finals changed after the runner read or wrote them. The board engine test now requires the preflight guard's own line and every saved-treatment line beside it. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 48 ++++---- scripts/confirmatory_run.py | 6 +- tests/test_confirmatory_board_engine.py | 11 +- tests/test_confirmatory_input_reads.py | 112 +++++++++++++++++- tests/test_confirmatory_no_tools.py | 2 +- tests/test_confirmatory_no_tools_records.py | 6 +- .../test_confirmatory_unreadable_evidence.py | 109 ++++++++++++++++- 7 files changed, 265 insertions(+), 29 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index cf67e653..b0877b17 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -83,9 +83,11 @@ def component(value: str) -> str: return value -def read_record(path: Path) -> dict: +def read_record(path: Path) -> tuple[dict, str]: + """Decode a saved record and hash the same bytes, so its link names them.""" try: - record = json.loads(path.read_text()) + data = path.read_bytes() + record = json.loads(data.decode("utf-8")) if not isinstance(record, dict): raise ValueError("record is not a JSON object") if "prior_attempts" in record: @@ -101,7 +103,7 @@ def read_record(path: Path) -> dict: cost = float(raw_cost) if not math.isfinite(cost) or cost < 0: raise ValueError("attempt cost must be finite and nonnegative") - return record + return record, hashlib.sha256(data).hexdigest() except ( OSError, ValueError, @@ -149,9 +151,10 @@ def saved_exists(path: Path) -> bool: return saved_stat(path) is not None -def saved_is_dir(path: Path) -> bool: - metadata = saved_stat(path) - return metadata is not None and stat.S_ISDIR(metadata.st_mode) +def listed_is_dir(path: Path) -> bool: + """A listing returned this entry, so every stat failure refuses, even ENOENT + (a dangling link, or an entry removed since the listing).""" + return stat.S_ISDIR(_board_input(path, path.stat).st_mode) def saved_children(folder: Path) -> list[Path]: @@ -411,12 +414,13 @@ def validate_treatment(path, record, model, board, engine): raise StateError(f"{path}: " + "; ".join(problems)) -def entry(path: Path, record: dict, n: int) -> dict: +def entry(path: Path, record: dict, n: int, sha256: str) -> dict: + """Link an archive by the digest of the bytes its record came from.""" cls, reason = classify(record, 0.0, 0.0, None) return { "n": n, "file": f"attempts/{path.name}", - "sha256": sha256_file(path), + "sha256": sha256, "status": record.get("status"), "class": cls, "reason": reason, @@ -435,13 +439,15 @@ def _attempts_agree(listed: list, entries: list) -> bool: def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], float]: """Validate a model's histories, including an interrupted archive, and sum spend.""" + # Finals first, as the report lists them: an unlistable households folder + # is named before the attempts folder inside it. + finals = {path.stem: path for path in saved_json_paths(folder)} archives: dict[str, list[tuple[int, Path]]] = {} for path in saved_json_paths(folder / "attempts"): match = re.fullmatch(r"(.+)-([1-9][0-9]*)\.json", path.name) if not match: raise StateError(f"{path}: invalid attempt number") archives.setdefault(match[1], []).append((int(match[2]), path)) - finals = {path.stem: path for path in saved_json_paths(folder)} states, spent = {}, 0.0 for sid in sorted(set(finals) | set(archives)): paths = sorted(archives.get(sid, [])) @@ -455,7 +461,7 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f raise StateError(f"{folder}: {sid} exceeds {MAX_RERUNS} reruns") entries = [] for n, path in paths: - record = read_record(path) + record, digest = read_record(path) if record.get("scenario_id") != sid: raise StateError(f"{path}: household differs from archive name {sid}") if validate is not None: @@ -470,12 +476,12 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f ) from error if action(cls, n) not in ("rerun", "pause-rerun"): raise StateError(f"{path}: archived attempt was ineligible for rerun") - entries = record["attempts"] + [entry(path, record, n)] + entries = record["attempts"] + [entry(path, record, n, digest)] spent += attempt_cost(record) - final = None + final = final_sha256 = None if sid in finals: path = finals[sid] - final = read_record(path) + final, final_sha256 = read_record(path) if drawn is not None: if sid not in drawn: raise StateError(f"{path}: cheap-model final is outside the draw") @@ -504,7 +510,7 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f ) from error entries = final["attempts"] spent += attempt_cost(final) - states[sid] = {"entries": entries, "final": final} + states[sid] = {"entries": entries, "final": final, "final_sha256": final_sha256} return states, spent @@ -517,7 +523,7 @@ def stage_problems(design: dict) -> list[str]: folder, stage = component(spec["folder"]), component(spec["stage"]) draw_order = [component(sid) for sid in design["draw_order"]] models = set(CHEAP) - models.update(path.name for path in saved_children(root) if saved_is_dir(path)) + models.update(path.name for path in saved_children(root) if listed_is_dir(path)) except (OSError, KeyError, TypeError, ValueError) as error: return [f"no-tools stage: {error}"] board = None @@ -690,7 +696,7 @@ def _load_state(self, board) -> None: ) self.histories = {} spent = 0.0 - folders = {p.name for p in saved_children(self.root) if saved_is_dir(p)} + folders = {p.name for p in saved_children(self.root) if listed_is_dir(p)} for model in sorted(folders | set(self.models)): states, cost = history( self.households(model), @@ -797,7 +803,9 @@ def archive(self, model: str, sid: str) -> bool: target.parent.mkdir(parents=True, exist_ok=True) if saved_exists(target): raise StateError(f"{target}: would overwrite an archived attempt") - link = entry(source, record, n) + if _board_input(source, lambda: sha256_file(source)) != state["final_sha256"]: + raise StateError(f"{source}: final changed after it was read or written") + link = entry(source, record, n, state["final_sha256"]) link["file"] = f"attempts/{target.name}" with self._admission_lock: if self.interrupted.is_set(): @@ -821,7 +829,7 @@ def run_model(self, model: str) -> None: self.states[model] = "interrupted" return state = self.histories[model].setdefault( - sid, {"entries": [], "final": None} + sid, {"entries": [], "final": None, "final_sha256": None} ) while True: if self.interrupted.is_set(): @@ -878,8 +886,8 @@ def no_tool(arguments): ) record["totals"]["cost_usd"] = spend.spent_usd record["attempts"] = list(state["entries"]) - write_json(self.households(model) / f"{sid}.json", record) - state["final"] = record + digest = write_json(self.households(model) / f"{sid}.json", record) + state["final"], state["final_sha256"] = record, digest n = len(state["entries"]) + 1 cls, _ = classify(record, 0.0, 0.0, None) outcome = action(cls, n) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 9b8b6e08..bfad199f 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2172,9 +2172,9 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: from confirmatory_no_tools import ( BOARD_INPUT_ERRORS, _board_input, + listed_is_dir, saved_children, saved_exists, - saved_is_dir, saved_json_paths, stage_problems, ) @@ -2308,9 +2308,9 @@ def household_paths(stage): holds_records = any( household_paths(stage) for arm in saved_children(folder) - if saved_is_dir(arm) + if listed_is_dir(arm) for stage in saved_children(arm) - if saved_is_dir(stage) + if listed_is_dir(stage) ) if holds_records: problems.append( diff --git a/tests/test_confirmatory_board_engine.py b/tests/test_confirmatory_board_engine.py index 11910e52..99dc3faf 100644 --- a/tests/test_confirmatory_board_engine.py +++ b/tests/test_confirmatory_board_engine.py @@ -62,7 +62,16 @@ def test_engine_metadata_refuses_full_calculator_start( w, design, metadata = engine_fault(world, tmp_path, monkeypatch, fault) if entrypoint == "preflight": problems = cr.preflight(design, (cr.CHEAP[0],)) - assert str(metadata) in "\n".join(problems) + # The preflight's own guard, beside the saved-treatment lines it keeps. + guard = f"{metadata}: cannot read the board's engine: " + assert sum(problem.startswith(guard) for problem in problems) == 1, problems + for model in cr.CHEAP: + assert any( + problem.startswith(f"no-tools {model}: ") + and "cannot check saved no-tools treatment" in problem + and str(metadata) in problem + for problem in problems + ), problems else: design_path = tmp_path / "design.json" design_path.write_text(json.dumps(design)) diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index 10be2c16..9236daad 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -2,15 +2,19 @@ from __future__ import annotations +import hashlib import importlib.util import io +import json import os +import re import sys from pathlib import Path +from types import SimpleNamespace import pytest -from tests.test_confirmatory_no_tools import Provider, nt +from tests.test_confirmatory_no_tools import Provider, nt, record, stage from tests.test_confirmatory_no_tools import setup as _setup_fixture from tests.test_confirmatory_no_tools_records import ( assert_boundary, @@ -91,6 +95,112 @@ def test_baseline_hash_and_decoder_share_one_valid_buffer(setup, monkeypatch): assert provider.calls == [] +def saved_history(setup, archive_cost=0.3, final_cost=0.1): + """Save one rerun-eligible archive and the final that links it.""" + root = Path(setup["design"]["no_tools"]["runs"]) + folder = root / nt.CHEAP[0] / "none/confirmatory/households" + profile = SimpleNamespace(name=nt.CHEAP[0], arm="none", model_id="synthetic") + archive = folder / "attempts/h2-1.json" + archive.parent.mkdir(parents=True) + prior = record( + "h2", + profile, + status="provider_error", + error="APIConnectionError", + cost=archive_cost, + ) + prior["attempts"] = [] + archive.write_text(json.dumps(prior)) + saved = record("h2", profile, cost=final_cost) + saved["attempts"] = [ + { + "n": 1, + "file": "attempts/h2-1.json", + "sha256": nt.sha256_file(archive), + "status": "provider_error", + "class": "provider", + "reason": "APIConnectionError", + } + ] + (folder / "h2.json").write_text(json.dumps(saved)) + return {"archive": archive, "final": folder / "h2.json"}, prior, saved + + +@pytest.mark.parametrize("which", ["archive", "final"]) +def test_saved_record_hash_and_decoder_share_one_buffer(setup, monkeypatch, which): + paths, prior, saved = saved_history(setup) + path = paths[which] + data = path.read_bytes() + # A second read would see a cheaper record than the hashed bytes hold. + replaced = dict(prior if which == "archive" else saved, totals={"cost_usd": 0.2}) + reads = replace_text_read(monkeypatch, path, json.dumps(replaced)) + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + assert plan.spend.spent_usd == pytest.approx(0.4) + state = plan.histories[nt.CHEAP[0]]["h2"] + assert state["entries"][0]["sha256"] == nt.sha256_file(paths["archive"]) + assert state["final_sha256"] == nt.sha256_file(paths["final"]) + assert reads == [("bytes", data)] + assert provider.calls == [] + + +def test_archive_links_the_digest_its_final_was_read_from(setup): + paths, _, saved = saved_history(setup) + saved["status"], saved["error"] = "provider_error", "APIConnectionError" + paths["final"].write_text(json.dumps(saved)) + data = paths["final"].read_bytes() + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + assert plan.archive(nt.CHEAP[0], "h2") + link = plan.histories[nt.CHEAP[0]]["h2"]["entries"][-1] + assert link["n"] == 2 and link["file"] == "attempts/h2-2.json" + assert link["sha256"] == hashlib.sha256(data).hexdigest() + assert (paths["archive"].parent / "h2-2.json").read_bytes() == data + assert provider.calls == [] + + +def test_archive_refuses_a_final_changed_after_it_was_read(setup): + paths, _, saved = saved_history(setup) + saved["status"], saved["error"] = "provider_error", "APIConnectionError" + paths["final"].write_text(json.dumps(saved)) + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + changed = dict(saved, totals={"cost_usd": 0.2}) + paths["final"].write_text(json.dumps(changed)) + with pytest.raises(nt.StateError, match=re.escape(str(paths["final"]))): + plan.archive(nt.CHEAP[0], "h2") + assert len(plan.histories[nt.CHEAP[0]]["h2"]["entries"]) == 1 + assert json.loads(paths["final"].read_text()) == changed + assert not (paths["archive"].parent / "h2-2.json").exists() + assert provider.calls == [] + + +def test_archive_refuses_a_final_changed_after_the_runner_wrote_it(setup, monkeypatch): + provider = Provider( + { + (nt.CHEAP[0], "h2", 1): { + "status": "provider_error", + "error": "APIConnectionError", + } + } + ) + original = nt.write_json + + def write_then_change(path, payload, *args, **kwargs): + digest = original(path, payload, *args, **kwargs) + path.write_bytes(path.read_bytes() + b" ") + return digest + + monkeypatch.setattr(nt, "write_json", write_then_change) + plan = stage(setup, provider, models=[nt.CHEAP[0]]) + assert plan.execute() == 1 + assert plan.states[nt.CHEAP[0]] == "error" + assert provider.calls == [(nt.CHEAP[0], "h2", 1)] + folder = plan.households(nt.CHEAP[0]) + assert json.loads((folder / "h2.json").read_text())["status"] == "provider_error" + assert not (folder / "attempts").exists() + + def pool_design(tmp_path, monkeypatch): monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) monkeypatch.setattr(cr, "host_power", lambda: (True, True)) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index b7da9487..85ff9ed0 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -1475,7 +1475,7 @@ def test_valid_saved_numeric_cost_preserves_spend(setup, cost): prior["attempts"] = [] archive.write_text(json.dumps(prior)) saved = record("h2", profile, cost=cost) - saved["attempts"] = [nt.entry(archive, prior, 1)] + saved["attempts"] = [nt.entry(archive, prior, 1, nt.sha256_file(archive))] (folder / "h2.json").write_text(json.dumps(saved)) provider = Provider() with stage(setup, provider) as plan: diff --git a/tests/test_confirmatory_no_tools_records.py b/tests/test_confirmatory_no_tools_records.py index c145012c..d9d7ac37 100644 --- a/tests/test_confirmatory_no_tools_records.py +++ b/tests/test_confirmatory_no_tools_records.py @@ -85,7 +85,7 @@ def save(path, payload): path = saved.folder / "attempts" / f"h2-{n}.json" save(path, payload) try: - link = nt.entry(path, payload, n) + link = nt.entry(path, payload, n, nt.sha256_file(path)) except (AttributeError, TypeError, ValueError, KeyError): # Malformed turn evidence must reach the startup reader, rather # than fail while this fixture refreshes otherwise valid links. @@ -450,5 +450,7 @@ def fake_household( assert not second.exists() payload = json.loads(final.read_text()) assert payload["status"] == "complete" - assert payload["attempts"] == [nt.entry(first, saved.archives[0], 1)] + assert payload["attempts"] == [ + nt.entry(first, saved.archives[0], 1, nt.sha256_file(first)) + ] assert nt.stage_problems(saved.design) == [] diff --git a/tests/test_confirmatory_unreadable_evidence.py b/tests/test_confirmatory_unreadable_evidence.py index 18a9d75a..bead81aa 100644 --- a/tests/test_confirmatory_unreadable_evidence.py +++ b/tests/test_confirmatory_unreadable_evidence.py @@ -4,6 +4,7 @@ import io import json import os +import re from pathlib import Path import pytest @@ -112,6 +113,108 @@ def test_model_inventory_failure_refuses( assert provider.calls == [] +def vanish(monkeypatch, target): + """A listing returned target, but its metadata read finds it gone.""" + original = os.stat + + def gone(path, *args, **kwargs): + if not isinstance(path, int) and Path(path) == target: + raise FileNotFoundError(errno.ENOENT, "synthetic vanished entry", str(path)) + return original(path, *args, **kwargs) + + monkeypatch.setattr(os, "stat", gone) + + +def plant_listed_entry(monkeypatch, target, fault, tmp_path): + if fault == "dangling": + # A model folder symlinked to an unmounted volume. + target.symlink_to(tmp_path / "unmounted-volume") + else: + target.mkdir() + vanish(monkeypatch, target) + + +@pytest.mark.parametrize("fault", ["dangling", "vanished"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_listed_model_entry_without_metadata_refuses( + setup, world, tmp_path, monkeypatch, fault, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + root = Path(design["no_tools"]["runs"]) + root.mkdir(parents=True, exist_ok=True) + target = root / "ghost-model" + plant_listed_entry(monkeypatch, target, fault, tmp_path) + if boundary == "startup": + with pytest.raises(nt.StateError, match=re.escape(str(target))): + with stage(setup, provider): + pass + else: + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any( + problem.startswith(f"no-tools stage: {target}:") for problem in problems + ), problems + assert not w.provider.count + assert provider.calls == [] + + +@pytest.mark.parametrize("fault", ["dangling", "vanished"]) +@pytest.mark.parametrize("level", ["arm", "stage"]) +def test_preflight_listed_calculator_entry_without_metadata_refuses( + world, tmp_path, monkeypatch, fault, level +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + folder = Path(design["runs"]) / cr.CHEAP[0] + parent = folder if level == "arm" else folder / "pe" + parent.mkdir(parents=True, exist_ok=True) + target = parent / "ghost" + plant_listed_entry(monkeypatch, target, fault, tmp_path) + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(problem.startswith(f"{target}:") for problem in problems), problems + assert not w.provider.count + + +@pytest.mark.parametrize("fault", ["file", "stat"]) +@pytest.mark.parametrize("which", ["finals", "attempts"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_broken_optional_history_folder_refuses( + setup, world, tmp_path, monkeypatch, fault, which, boundary +): + provider = Provider() + if boundary == "startup": + design = setup["design"] + else: + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) + spec = design["no_tools"] + folder = ( + Path(spec["runs"]) / "gpt-6-sol" / spec["folder"] / spec["stage"] / "households" + ) + target = folder if which == "finals" else folder / "attempts" + target.parent.mkdir(parents=True, exist_ok=True) + if fault == "file": + target.write_text("{}") + else: + target.mkdir() + deny_stat(monkeypatch, target) + if boundary == "startup": + with pytest.raises(nt.StateError, match=re.escape(str(target))): + with stage(setup, provider): + pass + else: + assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) + assert not w.provider.count + assert provider.calls == [] + + def state_for_loading(setup): # Isolate _load_state from the constructor's mkdir and lock operations. import threading @@ -229,7 +332,11 @@ def test_archive_target_stat_failure_refuses(setup, monkeypatch): "totals": {"cost_usd": 0.1}, } (folder / f"{sid}.json").write_text(json.dumps(payload)) - plan.histories[model][sid] = {"final": payload, "entries": []} + plan.histories[model][sid] = { + "final": payload, + "entries": [], + "final_sha256": nt.sha256_file(folder / f"{sid}.json"), + } target = folder / "attempts" / f"{sid}-1.json" deny_stat(monkeypatch, target) with pytest.raises(nt.StateError, match=str(target)): From c2ca2dc92440c102ddf6cb23472e5a245ac0e86c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 29 Oct 2026 22:14:22 -0400 Subject: [PATCH 37/48] Confirmatory runner: refuse an archive move before creating its folder In suites r10, three of 0ae6df56's new cases failed in all three runs: - test_archive_refuses_a_final_changed_after_the_runner_wrote_it: Stage.archive created the attempts folder before it checked the target and the final's digest, so a refused move left an empty attempts folder in a history that had none. Both refusals now come first, and the folder exists only when the move proceeds. - test_saved_record_hash_and_decoder_share_one_buffer[archive] and [final]: the test hashed the files itself after installing the read counter, so its own read counted as a second runner read. It now hashes them before counting. The runner read each record once. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 4 +++- tests/test_confirmatory_input_reads.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index b0877b17..b27baccd 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -800,11 +800,13 @@ def archive(self, model: str, sid: str) -> bool: folder = self.households(model) source = folder / f"{sid}.json" target = folder / "attempts" / f"{sid}-{n}.json" - target.parent.mkdir(parents=True, exist_ok=True) + # Both refusals come before the folder exists, so a refused move + # leaves the history as it was. if saved_exists(target): raise StateError(f"{target}: would overwrite an archived attempt") if _board_input(source, lambda: sha256_file(source)) != state["final_sha256"]: raise StateError(f"{source}: final changed after it was read or written") + target.parent.mkdir(parents=True, exist_ok=True) link = entry(source, record, n, state["final_sha256"]) link["file"] = f"attempts/{target.name}" with self._admission_lock: diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index 9236daad..798f6d3b 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -131,6 +131,9 @@ def test_saved_record_hash_and_decoder_share_one_buffer(setup, monkeypatch, whic paths, prior, saved = saved_history(setup) path = paths[which] data = path.read_bytes() + # Hashed before the reads are counted, so the count is the runner's alone. + archive_sha256 = nt.sha256_file(paths["archive"]) + final_sha256 = nt.sha256_file(paths["final"]) # A second read would see a cheaper record than the hashed bytes hold. replaced = dict(prior if which == "archive" else saved, totals={"cost_usd": 0.2}) reads = replace_text_read(monkeypatch, path, json.dumps(replaced)) @@ -138,8 +141,8 @@ def test_saved_record_hash_and_decoder_share_one_buffer(setup, monkeypatch, whic with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: assert plan.spend.spent_usd == pytest.approx(0.4) state = plan.histories[nt.CHEAP[0]]["h2"] - assert state["entries"][0]["sha256"] == nt.sha256_file(paths["archive"]) - assert state["final_sha256"] == nt.sha256_file(paths["final"]) + assert state["entries"][0]["sha256"] == archive_sha256 + assert state["final_sha256"] == final_sha256 assert reads == [("bytes", data)] assert provider.calls == [] From 274e1c7ea8d85c7a098fb02a41f9f88192560aa0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 01:10:45 -0400 Subject: [PATCH 38/48] Confirmatory runner: refuse unfinished optional histories and special files Review runner-r17 (sol61, REQUEST_CHANGES) found two gaps between the runner and the report and three smaller items. This commit fixes each. - The calculator preflight now refuses what the report refuses. stage_problems refuses any model's archive that no final lists, and any final whose action is rerun or pause-rerun. That covers optional models, and cheap-model archives outside the draw. Startup recovery still resumes both states. - Saved records now go through the report's descriptor guard. read_regular opens with O_NONBLOCK and refuses a descriptor that is not a regular file, then reads through that descriptor. Before this, a FIFO with no writer could hang startup while it held the invocation lock. The archive move's digest checks use the same reader. - The archive move hashes the moved bytes again after the rename. If a write landed between the first hash and the rename, it moves the bytes back and refuses. The docstring states the limit that remains: the runner cannot see a write to the same inode during either hash or after the second. Every later reader hashes each archive against its link. - New tests: - a FIFO or device in place of a final or an archive, at startup and preflight, bounded by a writer that releases a blocked open after 5 s; - unfinished optional and outside-draw histories; - a write between the digest check and the rename; - a target collision that keeps both files; - a digest refusal that leaves no attempts folder, and the missing folder after a target-stat refusal; - the changed-written-final case across every cheap model, which pins the other models' continuation. - The optional no-reply control now expects the preflight to refuse its final, which owes a rerun. - The read-counting helper also counts read_regular. It patches that reader only where it exists, so the same cases run against c2ca2dc9. Against c2ca2dc9, 14 of the new cases fail: 8 special-file, 1 move-back and 5 unfinished-history. The collision, folder and continuation cases pin existing behaviour. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 76 ++++++++++-- tests/test_confirmatory_input_reads.py | 57 ++++++++- tests/test_confirmatory_no_tools_records.py | 48 ++++++++ .../test_confirmatory_unreadable_evidence.py | 113 ++++++++++++++++++ 4 files changed, 279 insertions(+), 15 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index b27baccd..4be1f98a 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -83,10 +83,37 @@ def component(value: str) -> str: return value +def read_regular(path: Path) -> bytes: + """Read saved bytes as the report does: open with O_NONBLOCK, refuse a + descriptor that is not a regular file, then read through that descriptor. + A FIFO or device refuses at once, before any read can block under the + invocation lock. It follows file symlinks; a link to nothing refuses.""" + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + kind = { + stat.S_IFIFO: "FIFO", + stat.S_IFDIR: "directory", + stat.S_IFSOCK: "socket", + stat.S_IFCHR: "character device", + stat.S_IFBLK: "block device", + stat.S_IFLNK: "symlink", + }.get(stat.S_IFMT(metadata.st_mode), "unknown file type") + raise ValueError(f"expected a regular file, found {kind}") + os.set_blocking(descriptor, True) + chunks = [] + while chunk := os.read(descriptor, 1024 * 1024): + chunks.append(chunk) + return b"".join(chunks) + finally: + os.close(descriptor) + + def read_record(path: Path) -> tuple[dict, str]: """Decode a saved record and hash the same bytes, so its link names them.""" try: - data = path.read_bytes() + data = read_regular(path) record = json.loads(data.decode("utf-8")) if not isinstance(record, dict): raise ValueError("record is not a JSON object") @@ -151,6 +178,11 @@ def saved_exists(path: Path) -> bool: return saved_stat(path) is not None +def saved_sha256(path: Path) -> str: + """Hash a saved file through the same guard, naming it on failure.""" + return _board_input(path, lambda: hashlib.sha256(read_regular(path)).hexdigest()) + + def listed_is_dir(path: Path) -> bool: """A listing returned this entry, so every stat failure refuses, even ENOENT (a dangling link, or an entry removed since the listing).""" @@ -515,7 +547,12 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f def stage_problems(design: dict) -> list[str]: - """Check saved histories, treatment and standing without a provider.""" + """Check saved histories, treatment and standing without a provider. + + As the report does, refuse any model's archive that no final lists and any + final that owes a rerun, optional models included; each drawn cheap-model + household also needs a final that stands. Startup recovery still resumes + both unfinished states.""" problems = [] try: spec = design["no_tools"] @@ -543,18 +580,23 @@ def checked_board(): ), drawn=set(draw_order) if model in CHEAP else None, ) - if model not in CHEAP: - continue - for sid in draw_order: + drawn = draw_order if model in CHEAP else [] + for sid in drawn + sorted(set(states) - set(drawn)): state = states.get(sid) if state is None or state["final"] is None: - problems.append(f"no-tools {model}/{sid}: missing standing final") + problems.append( + f"no-tools {model}/{sid}: missing standing final" + if sid in drawn + else f"no-tools {model}/{sid}: archived attempts lack a final" + ) continue outcome = action( classify(state["final"], 0.0, 0.0, None)[0], len(state["entries"]) + 1, ) - if outcome != "stands": + if outcome in ("rerun", "pause-rerun") or ( + sid in drawn and outcome != "stands" + ): problems.append( f"no-tools {model}/{sid}: final action is {outcome}" ) @@ -790,7 +832,14 @@ def _dry_run(self) -> int: ) def archive(self, model: str, sid: str) -> bool: - """Atomically admit a prepared archive move, then extend its links.""" + """Atomically admit a prepared archive move, then extend its links. + + The target-exists and digest checks precede folder creation; either + refusal leaves the disk history unchanged. After the rename it hashes + the moved bytes again; if a write landed between the first hash and + the rename, it moves them back and refuses. It cannot see a write to + the same inode during either hash or after the second; every later + reader hashes each archive against its link and refuses a mismatch.""" state = self.histories[model][sid] record, entries = state["final"], state["entries"] n = len(entries) + 1 @@ -800,11 +849,9 @@ def archive(self, model: str, sid: str) -> bool: folder = self.households(model) source = folder / f"{sid}.json" target = folder / "attempts" / f"{sid}-{n}.json" - # Both refusals come before the folder exists, so a refused move - # leaves the history as it was. if saved_exists(target): raise StateError(f"{target}: would overwrite an archived attempt") - if _board_input(source, lambda: sha256_file(source)) != state["final_sha256"]: + if saved_sha256(source) != state["final_sha256"]: raise StateError(f"{source}: final changed after it was read or written") target.parent.mkdir(parents=True, exist_ok=True) link = entry(source, record, n, state["final_sha256"]) @@ -814,6 +861,13 @@ def archive(self, model: str, sid: str) -> bool: return False os.replace(source, target) state["final"] = None + if saved_sha256(target) != state["final_sha256"]: + os.replace(target, source) + state["final"] = record + raise StateError( + f"{source}: final changed during its archive move; " + f"the runner moved it back from {target}" + ) entries.append(link) return True diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index 798f6d3b..c4247ab1 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -34,8 +34,12 @@ def replace_text_read(monkeypatch, path, replacement): - """Simulate replacement after the first byte read, without changing disk.""" + """Simulate replacement after the first byte read, without changing disk. + Count both byte readers: Path.read_bytes, and the descriptor-guarded + read_regular that saved records go through (when the runner has it, so the + same cases run against an older runner).""" original_bytes, original_text = Path.read_bytes, Path.read_text + original_regular = getattr(nt, "read_regular", None) reads = [] def read_bytes(source): @@ -44,6 +48,15 @@ def read_bytes(source): reads.append(("bytes", data)) return data + def read_regular(source): + data = original_regular(source) + if Path(source) == path: + reads.append(("bytes", data)) + return data + + if original_regular is not None: + monkeypatch.setattr(nt, "read_regular", read_regular) + def read_text(source, *args, **kwargs): if source == path: reads.append(("text", replacement)) @@ -178,6 +191,34 @@ def test_archive_refuses_a_final_changed_after_it_was_read(setup): assert provider.calls == [] +def test_archive_moves_back_a_final_changed_before_its_rename(setup, monkeypatch): + paths, _, saved = saved_history(setup) + saved["status"], saved["error"] = "provider_error", "APIConnectionError" + paths["final"].write_text(json.dumps(saved)) + changed = dict(saved, totals={"cost_usd": 0.2}) + original = nt.os.replace + + def change_then_rename(source, target): + # The write lands after the digest check and before the rename. + if Path(source) == paths["final"]: + Path(source).write_text(json.dumps(changed)) + original(source, target) + + provider = Provider() + with stage(setup, provider, models=[nt.CHEAP[0]]) as plan: + monkeypatch.setattr(nt.os, "replace", change_then_rename) + with pytest.raises( + nt.StateError, + match=re.escape(f"{paths['final']}: final changed during its archive move"), + ): + plan.archive(nt.CHEAP[0], "h2") + state = plan.histories[nt.CHEAP[0]]["h2"] + assert len(state["entries"]) == 1 and state["final"] == saved + assert json.loads(paths["final"].read_text()) == changed + assert sorted(p.name for p in paths["archive"].parent.iterdir()) == ["h2-1.json"] + assert provider.calls == [] + + def test_archive_refuses_a_final_changed_after_the_runner_wrote_it(setup, monkeypatch): provider = Provider( { @@ -191,14 +232,22 @@ def test_archive_refuses_a_final_changed_after_the_runner_wrote_it(setup, monkey def write_then_change(path, payload, *args, **kwargs): digest = original(path, payload, *args, **kwargs) - path.write_bytes(path.read_bytes() + b" ") + if nt.CHEAP[0] in Path(path).parts: + path.write_bytes(path.read_bytes() + b" ") return digest monkeypatch.setattr(nt, "write_json", write_then_change) - plan = stage(setup, provider, models=[nt.CHEAP[0]]) + plan = stage(setup, provider) assert plan.execute() == 1 assert plan.states[nt.CHEAP[0]] == "error" - assert provider.calls == [(nt.CHEAP[0], "h2", 1)] + # The other models continue through both households. + assert all(plan.states[model] == "complete" for model in nt.CHEAP[1:]) + assert all( + plan.standing(model, sid) for model in nt.CHEAP[1:] for sid in ("h2", "h1") + ) + assert [call for call in provider.calls if call[0] == nt.CHEAP[0]] == [ + (nt.CHEAP[0], "h2", 1) + ] folder = plan.households(nt.CHEAP[0]) assert json.loads((folder / "h2.json").read_text())["status"] == "provider_error" assert not (folder / "attempts").exists() diff --git a/tests/test_confirmatory_no_tools_records.py b/tests/test_confirmatory_no_tools_records.py index d9d7ac37..45d9681b 100644 --- a/tests/test_confirmatory_no_tools_records.py +++ b/tests/test_confirmatory_no_tools_records.py @@ -364,11 +364,59 @@ def test_absent_predictions_without_reply_remain_accepted( no_reply(payload) payload.pop("predictions") save_records(saved) + if reader == "optional" and boundary == "preflight": + # Absent predictions raise nothing; the unanswered final still owes a + # rerun, which the report refuses for every model. + assert nt.stage_problems(saved.design) == [ + "no-tools gpt-6-sol/h2: final action is rerun" + ] + return assert_boundary( saved, boundary, None, expected_spend=0.6 if reader == "optional" else 0.5 ) +UNFINISHED = { + "optional-rerun": ("gpt-6-sol", "h2", "final action is rerun"), + "optional-pause": ("gpt-6-sol", "h2", "final action is pause-rerun"), + "optional-archive": ("gpt-6-sol", "h2", "archived attempts lack a final"), + "cheap-archive-outside-draw": ( + nt.CHEAP[1], + "outside", + "archived attempts lack a final", + ), +} + + +@pytest.mark.parametrize("unfinished", UNFINISHED) +def test_preflight_refuses_unfinished_histories_that_startup_resumes( + saved_records, unfinished +): + saved = saved_records + model, sid, needle = UNFINISHED[unfinished] + board = ( + add_board_household(saved, sid) + if sid == "outside" + else nt.pilot.load_board(Path(saved.design["board"])) + ) + payload = full_record(board, model, saved.design["engine"], sid) + error = ( + "AuthenticationError" + if unfinished == "optional-pause" + else "APIConnectionError" + ) + payload.update(status="provider_error", error=error) + households = saved.root / model / "none/confirmatory/households" + path = households / ( + f"{sid}.json" + if unfinished.endswith(("rerun", "pause")) + else f"attempts/{sid}-1.json" + ) + write_fixture_json(path, payload) + assert nt.stage_problems(saved.design) == [f"no-tools {model}/{sid}: {needle}"] + assert_boundary(saved, "startup", None, expected_spend=0.6) + + @pytest.mark.parametrize("reader", ["final", "archive"]) @pytest.mark.parametrize("boundary", ["startup", "preflight"]) def test_saved_call_classification_failure_refuses_with_path( diff --git a/tests/test_confirmatory_unreadable_evidence.py b/tests/test_confirmatory_unreadable_evidence.py index bead81aa..75b225ab 100644 --- a/tests/test_confirmatory_unreadable_evidence.py +++ b/tests/test_confirmatory_unreadable_evidence.py @@ -1,10 +1,13 @@ """File and folder failures must remain named refusals before providers.""" +import contextlib import errno import io import json import os import re +import threading +import time from pathlib import Path import pytest @@ -317,6 +320,115 @@ def denied(path): assert not w.provider.count +@contextlib.contextmanager +def fifo_released_after(path, seconds=5.0): + """Bound a reader that blocks on the FIFO: after `seconds`, open and close a + writer every 50 ms, so each blocked open sees end of file instead of hanging.""" + done = threading.Event() + + def release(): + if done.wait(seconds): + return + while not done.is_set(): + with contextlib.suppress(OSError): # ENXIO while no reader waits + os.close(os.open(path, os.O_WRONLY | os.O_NONBLOCK)) + done.wait(0.05) + + thread = threading.Thread(target=release, daemon=True) + thread.start() + try: + yield + finally: + done.set() + thread.join() + + +@pytest.mark.parametrize("kind", ["fifo", "device"]) +@pytest.mark.parametrize("which", ["final", "archive"]) +@pytest.mark.parametrize("boundary", ["startup", "preflight"]) +def test_saved_record_that_is_not_a_regular_file_refuses_at_once( + setup, boundary, which, kind +): + spec = setup["design"]["no_tools"] + folder = ( + Path(spec["runs"]) / nt.CHEAP[0] / spec["folder"] / spec["stage"] / "households" + ) + target = folder / ("h2.json" if which == "final" else "attempts/h2-1.json") + target.parent.mkdir(parents=True) + if kind == "fifo": + os.mkfifo(target) + else: + target.symlink_to("/dev/null") + provider = Provider() + start = time.monotonic() + with fifo_released_after(target) if kind == "fifo" else contextlib.nullcontext(): + if boundary == "startup": + with pytest.raises(nt.StateError) as caught: + with stage(setup, provider): + pytest.fail("a special file must refuse before execution") + message = str(caught.value) + else: + message = "\n".join(nt.stage_problems(setup["design"])) + found = "FIFO" if kind == "fifo" else "character device" + assert str(target) in message + assert f"expected a regular file, found {found}" in message + assert time.monotonic() - start < 5, "no read may block on the special file" + assert provider.calls == [] + + +def prepared_final(plan, model=nt.CHEAP[0], sid="h2"): + """Save a rerun-eligible final and its history, with no attempts folder.""" + folder = plan.households(model) + folder.mkdir(parents=True) + payload = { + "scenario_id": sid, + "status": "provider_error", + "error": "APIConnectionError", + "turns": [], + "attempts": [], + "totals": {"cost_usd": 0.1}, + } + source = folder / f"{sid}.json" + source.write_text(json.dumps(payload)) + plan.histories[model][sid] = { + "final": payload, + "entries": [], + "final_sha256": nt.sha256_file(source), + } + return folder, source, payload + + +def test_archive_refuses_an_existing_target_and_keeps_both_files(setup): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, payload = prepared_final(plan) + target = folder / "attempts" / "h2-1.json" + target.parent.mkdir() + target.write_text('{"synthetic": "an earlier archive"}') + with pytest.raises( + nt.StateError, match=re.escape(f"{target}: would overwrite") + ): + plan.archive(nt.CHEAP[0], "h2") + assert source.read_text() == json.dumps(payload) + assert target.read_text() == '{"synthetic": "an earlier archive"}' + assert sorted(p.name for p in target.parent.iterdir()) == ["h2-1.json"] + assert plan.histories[nt.CHEAP[0]]["h2"]["entries"] == [] + assert provider.calls == [] + + +def test_archive_digest_refusal_leaves_no_attempts_folder(setup): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, payload = prepared_final(plan) + source.write_text(json.dumps(dict(payload, totals={"cost_usd": 0.2}))) + with pytest.raises(nt.StateError, match=re.escape(f"{source}: final changed")): + plan.archive(nt.CHEAP[0], "h2") + assert json.loads(source.read_text())["totals"] == {"cost_usd": 0.2} + assert not (folder / "attempts").exists() + assert plan.histories[nt.CHEAP[0]]["h2"]["entries"] == [] + assert provider.calls == [] + + def test_archive_target_stat_failure_refuses(setup, monkeypatch): provider = Provider() with stage(setup, provider) as plan: @@ -342,6 +454,7 @@ def test_archive_target_stat_failure_refuses(setup, monkeypatch): with pytest.raises(nt.StateError, match=str(target)): plan.archive(model, sid) assert (folder / f"{sid}.json").read_text() == json.dumps(payload) + assert not (folder / "attempts").exists() assert provider.calls == [] From d3673c3ced995dec9252122076b0245e807864cf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 01:13:25 -0400 Subject: [PATCH 39/48] Confirmatory runner: format to CI's ruff 0.15.2 and wrap long lines CI runs `uv run ruff check .` and `ruff format --check .` with the locked ruff 0.15.2. CI has not run on this draft. Run locally on c2ca2dc9's tree, both fail on files this PR adds: four files needed formatting, and four lines ran past 88 characters. This commit formats those four files and rewraps the one string the formatter could not split. Every changed file parses to the same AST as before. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_anchor.py | 4 +++- scripts/confirmatory_run.py | 10 +++++----- tests/test_confirmatory_anchor.py | 12 +++++++----- tests/test_confirmatory_run.py | 14 +++++++------- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/scripts/confirmatory_anchor.py b/scripts/confirmatory_anchor.py index 8cd02865..18a56701 100644 --- a/scripts/confirmatory_anchor.py +++ b/scripts/confirmatory_anchor.py @@ -61,7 +61,9 @@ def main(argv=None) -> int: """Push a whole snapshot only while exclusively extending the clone's file.""" parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--repo", type=Path, required=True, help="the anchors clone") - parser.add_argument("anchors", type=Path, help="a snapshot of the run's anchors.jsonl") + parser.add_argument( + "anchors", type=Path, help="a snapshot of the run's anchors.jsonl" + ) args = parser.parse_args(argv) repo = args.repo.expanduser() target = repo / "anchors.jsonl" diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index bfad199f..7a9a5a0b 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -1771,8 +1771,10 @@ def _join_push(self) -> None: "ended": clock(), "outcome": "error", "returncode": None, - "output": "the anchor push thread did not end after its time limit; " - "its command may still be running", + "output": ( + "the anchor push thread did not end after its " + "time limit; its command may still be running" + ), } ) self._check_push_error() @@ -1788,9 +1790,7 @@ def _record_push(self, line: dict) -> None: self._push_pending = None self._last_push_outcome = line["outcome"] if line["outcome"] != "success": - self._next_anchor = min( - self._next_anchor, line["ended"] + ANCHOR_RETRY - ) + self._next_anchor = min(self._next_anchor, line["ended"] + ANCHOR_RETRY) try: path = Path(self.design["runs"]) / "anchor-pushes.jsonl" with path.open("a") as handle: diff --git a/tests/test_confirmatory_anchor.py b/tests/test_confirmatory_anchor.py index fbc5a367..3af7be87 100644 --- a/tests/test_confirmatory_anchor.py +++ b/tests/test_confirmatory_anchor.py @@ -987,7 +987,9 @@ def finish(): for line in (tmp_path / "anchor-pushes.jsonl").read_text().splitlines() ] assert len(rows) == 1 and rows[0]["outcome"] == "timeout" - assert "pipes" in rows[0]["output"] and "held open after the kill" in rows[0]["output"] + assert ( + "pipes" in rows[0]["output"] and "held open after the kill" in rows[0]["output"] + ) def test_finish_records_a_stuck_push_once_even_if_its_thread_later_ends( @@ -1051,7 +1053,9 @@ def short_join(timeout=None): live = (tmp_path / "anchors.jsonl").read_bytes() assert len(copies[Path(calls[0][-1])].splitlines()) == 1 assert len(copies[Path(calls[1][-1])].splitlines()) == 2 - assert all(copy.endswith(b"\n") and live.startswith(copy) for copy in copies.values()) + assert all( + copy.endswith(b"\n") and live.startswith(copy) for copy in copies.values() + ) def test_a_retired_command_cannot_shrink_the_remote_after_the_final_push( @@ -1074,9 +1078,7 @@ def test_a_retired_command_cannot_shrink_the_remote_after_the_final_push( ready_read, ready_write = os.pipe() release_read, release_write = os.pipe() real_popen, children, waits = subprocess.Popen, [], [] - run = bare_finish_run( - folder, [sys.executable, str(SCRIPT), "--repo", str(repo)] - ) + run = bare_finish_run(folder, [sys.executable, str(SCRIPT), "--repo", str(repo)]) def popen(command, **kwargs): """Delay only the first real helper, before it can acquire the clone lock.""" diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 10847f52..5ad1f156 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2511,9 +2511,9 @@ def tool(arguments): assert record["api"] == treatment["api"] == api assert record["contract"] == treatment["contract"] == contract assert record["status"] == treatment["status"] == "complete" - assert record["predictions"] == treatment["predictions"] == { - v: 0.0 for v in VARIABLES - } + assert ( + record["predictions"] == treatment["predictions"] == {v: 0.0 for v in VARIABLES} + ) assert len(sent) == len(requests) == 2 assert sent == requests and called == calculations and len(called) == 1 assert record["messages"] == treatment["messages"] @@ -3151,8 +3151,7 @@ def communicate(timeout): assert [p["outcome"] for p in pushes] == ["failed"] * 3 assert [p["ended"] - p["started"] for p in pushes] == [7.0, 11.0, 5.0] assert all( - b["started"] >= a["ended"] + cr.ANCHOR_RETRY - for a, b in zip(pushes, pushes[1:]) + b["started"] >= a["ended"] + cr.ANCHOR_RETRY for a, b in zip(pushes, pushes[1:]) ) @@ -3182,8 +3181,9 @@ def during_retry(seconds): assert [row["outcome"] for row in pushes] == ["failed", "success", "success"] for model in w.models: assert [row["cause"] for row in w.events(model, "stop")] == ["interrupt"] - assert anchors[-1]["models"][model]["run-log.jsonl"] == ( - w.run.lanes[model].heads()["run-log.jsonl"] + assert ( + anchors[-1]["models"][model]["run-log.jsonl"] + == (w.run.lanes[model].heads()["run-log.jsonl"]) ) assert anchors[-2]["models"][model]["run-log.jsonl"]["lines"] == ( len(w.lines(model)) - 1 From 0193811081a18be7e051757004a4047d904b3bae Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 01:15:57 -0400 Subject: [PATCH 40/48] Confirmatory runner: pin the archive move's descriptor guard A mutant check of 274e1c7e found one fault that the tests missed. Its digest read in Stage.archive used the blocking sha256_file. The new case replaces a prepared final with a FIFO after startup. The archive move must refuse it at once by name, within the 5 s bound, and leave no attempts folder. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_unreadable_evidence.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_confirmatory_unreadable_evidence.py b/tests/test_confirmatory_unreadable_evidence.py index 75b225ab..e3dbd174 100644 --- a/tests/test_confirmatory_unreadable_evidence.py +++ b/tests/test_confirmatory_unreadable_evidence.py @@ -429,6 +429,23 @@ def test_archive_digest_refusal_leaves_no_attempts_folder(setup): assert provider.calls == [] +def test_archive_refuses_a_final_replaced_by_a_fifo_at_once(setup): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, _ = prepared_final(plan) + source.unlink() + os.mkfifo(source) + start = time.monotonic() + with fifo_released_after(source): + with pytest.raises( + nt.StateError, match="expected a regular file, found FIFO" + ): + plan.archive(nt.CHEAP[0], "h2") + assert time.monotonic() - start < 5, "no digest read may block on a FIFO" + assert not (folder / "attempts").exists() + assert provider.calls == [] + + def test_archive_target_stat_failure_refuses(setup, monkeypatch): provider = Provider() with stage(setup, provider) as plan: From a8226ddb2aecaf0eed1b56b7b58089067dc0400a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 11:27:59 -0400 Subject: [PATCH 41/48] Confirmatory runner tests: hold the deadline test's second turn for the abandon suites-r12's first runner pass at 01938110 failed test_an_attempt_past_its_deadline_is_abandoned_and_stops_the_model with 0.02 == 0.01, and the next two passes held 730 passed. The test gave each turn 0.2 s against a 0.3 s deadline, so its second turn spent at 0.4 s, and it expected the waiting thread to abandon the attempt inside that 0.1 s. A wake 0.1 s late put the second turn's cost in the attempt's record, which is the spend so far that the runner's rule asks for. The runner was right and the test's expectation assumed a wake time. A plugin that delays the abandon by 0.25 s reproduces the same assertion at the same line. The first turn now spends at once and the second waits for the abandon itself, through a new `hold` key in FakeProvider. The deadline rises to 1 s and the other model's attempts to 0.2 s, which leaves 0.8 s of room against that model reaching its own deadline. The assertions are unchanged. The test passed 25 of 25 runs alone and 25 of 25 with the 0.25 s delay. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 5ad1f156..070bda02 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -649,7 +649,8 @@ def close(self): class FakeProvider: """Conversations a test plants. specs[(model, label, household, n)] holds a record's `status` and `error`, `turns` (default 1), `cost` per turn, - `sleep` per turn, `calls` (tool arguments, one call each, per turn), + `sleep` per turn, `hold` (an event each turn after the first waits for), + `calls` (tool arguments, one call each, per turn), `pressure` (high pressure during the attempt), `stall` (a host-log stall, seconds), `guard` (a guard pause over the attempt), `resume_after` (seconds after which the operator resumes), `raise` (the conversation @@ -694,6 +695,8 @@ def __call__(self, lanes, sid, label, tool_call, spend): rec["status"] = "budget_stop" break started = time.time() + if i and spec.get("hold"): + assert spec["hold"].wait(30) time.sleep(spec.get("sleep", 0.0)) completed = time.time() cost = spec.get("cost", COST) @@ -1905,16 +1908,26 @@ def test_an_attempt_past_its_deadline_is_abandoned_and_stops_the_model( """The conversation runs on in its own thread; at the deadline its record is a harness error with the spend so far, the model stops, and the conversation ends `budget_stop` at its next turn. Its spend after the - deadline is a late-cost event.""" - monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 0.3) + deadline is a late-cost event. The second turn waits for the abandon, so + the split does not depend on when the waiting thread wakes.""" + monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 1.0) + abandoned = threading.Event() + abandon = cr.AttemptSpend.abandon + + def abandon_then_release(spend): + spent = abandon(spend) + abandoned.set() + return spent + + monkeypatch.setattr(cr.AttemptSpend, "abandon", abandon_then_release) model = "gpt-6-luna" w = world(models=(model, "claude-haiku-5.5")) sid = w.draw_order[0] first = w.run.lanes[model].orders[sid][0] w.provider.specs = { - (model, first, sid, 1): {"turns": 4, "cost": 0.01, "sleep": 0.2}, + (model, first, sid, 1): {"turns": 4, "cost": 0.01, "hold": abandoned}, **{ - ("claude-haiku-5.5", label, s, 1): {"sleep": 0.05} + ("claude-haiku-5.5", label, s, 1): {"sleep": 0.2} for label in cr.ARMS for s in w.draw_order[:2] }, From 4441364256d1824dc82fb8c558bb3bd579b06da8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 18:28:18 -0400 Subject: [PATCH 42/48] Confirmatory runner: hold the preflight to the report's spend and baseline rules Answers review runner-r18. stage_problems dropped the cost history returned and never compared it with spent_before, and the preflight never read the baseline. Calculator spending could therefore start from an understated ceiling, or on a baseline the report refuses, and the report would then withhold the results. stage_problems now sums every model's validated cost and refuses a spent_before below that sum by more than MONEY_TOLERANCE, the report's value. It refuses an unusable value by the report's money rule: finite and nonnegative, with numeric text and booleans counting as numbers there too. check_baseline is the startup check moved unchanged, so startup and the preflight share one rule. Tests: a full preflight against spend in a final, an archive and an optional model; each unusable spent_before; and each baseline fault. The second hash after a rename gets its own FIFO case, because the earlier case attacks the first hash. Two execution cases fail the second hash and the move back, then check that the other models finish, the lock is released and startup recovers the archive. The deadline test now records the join's timeout: its hold on the second turn hides when the abandon came, so a half deadline passed it. The ceiling test waits for the crossing where it slept 0.3 s against an unbounded competitor. The archive docstring now limits later digest checks to surviving links: after an interrupt, startup rebuilds the last archive's digest from the bytes then present. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 114 +++++++++------ tests/test_confirmatory_input_reads.py | 132 ++++++++++++++++++ tests/test_confirmatory_no_tools.py | 1 + tests/test_confirmatory_no_tools_records.py | 1 + tests/test_confirmatory_run.py | 21 ++- .../test_confirmatory_unreadable_evidence.py | 80 +++++++++++ 6 files changed, 303 insertions(+), 46 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 4be1f98a..7ab908f1 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -62,6 +62,7 @@ ) NO_TOOLS_CAP = 5.0 +MONEY_TOLERANCE = 1e-6 # dollars, as the report compares spent_before with records START_GRACE_SECONDS = 10 NO_TOOLS_WORDING_SHA256 = ( "43b13e4e64463f4ee1654e9546672899f80ca5e1ce18e93cf2eb04091b3a2699" @@ -546,14 +547,54 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f return states, spent -def stage_problems(design: dict) -> list[str]: - """Check saved histories, treatment and standing without a provider. +def check_baseline(design: dict) -> None: + """Refuse a baseline inventory that fails its hash or schema, from one read.""" + baseline_path = None + try: + baseline = design["baseline_exclusions"] + if not isinstance(baseline, dict) or not isinstance(baseline.get("path"), str): + raise StateError("baseline exclusions need path and sha256") + baseline_path = Path(baseline["path"]) + baseline_bytes = baseline_path.read_bytes() + if hashlib.sha256(baseline_bytes).hexdigest() != baseline.get("sha256"): + raise StateError("baseline exclusions do not hash as declared") + payload = json.loads(baseline_bytes.decode("utf-8")) + exclusions = payload.get("exclusions") if isinstance(payload, dict) else None + if not isinstance(exclusions, list): + raise StateError("baseline exclusions must contain an exclusions list") + cells = [] + for exclusion in exclusions: + if not isinstance(exclusion, dict) or any( + not isinstance(exclusion.get(key), str) or not exclusion[key] + for key in ("scenario_id", "variable") + ): + raise StateError( + "baseline exclusion needs household and output strings" + ) + cells.append((exclusion["scenario_id"], exclusion["variable"])) + if len(cells) != len(set(cells)): + raise StateError("baseline exclusions repeat a cell") + except (OSError, KeyError, TypeError, ValueError, RecursionError) as error: + at = "baseline exclusions" + if baseline_path is not None: + at += f" ({baseline_path})" + raise StateError(f"{at}: {error}") from error + - As the report does, refuse any model's archive that no final lists and any - final that owes a rerun, optional models included; each drawn cheap-model - household also needs a final that stands. Startup recovery still resumes - both unfinished states.""" +def stage_problems(design: dict) -> list[str]: + """Check the baseline, saved histories, treatment, standing and spend + without a provider. + + As the report does, refuse a baseline that fails its hash or schema, any + model's archive that no final lists, any final that owes a rerun, optional + models included, and a spent_before below the saved records' spend; each + drawn cheap-model household also needs a final that stands. Startup + recovery still accepts both unfinished states.""" problems = [] + try: + check_baseline(design) + except StateError as error: + problems.append(str(error)) try: spec = design["no_tools"] root = Path(spec["runs"]) @@ -562,8 +603,9 @@ def stage_problems(design: dict) -> list[str]: models = set(CHEAP) models.update(path.name for path in saved_children(root) if listed_is_dir(path)) except (OSError, KeyError, TypeError, ValueError) as error: - return [f"no-tools stage: {error}"] + return [*problems, f"no-tools stage: {error}"] board = None + spent = 0.0 def checked_board(): nonlocal board @@ -573,13 +615,14 @@ def checked_board(): for model in sorted(models): try: - states, _ = history( + states, cost = history( root / model / folder / stage / "households", validate=lambda path, record: validate_treatment( path, record, model, checked_board, design["engine"] ), drawn=set(draw_order) if model in CHEAP else None, ) + spent += cost drawn = draw_order if model in CHEAP else [] for sid in drawn + sorted(set(states) - set(drawn)): state = states.get(sid) @@ -602,6 +645,20 @@ def checked_board(): ) except (OSError, TypeError, ValueError, AttributeError, KeyError) as error: problems.append(f"no-tools {model}: {error}") + try: + declared = float(design.get("spent_before")) + except (TypeError, ValueError, OverflowError): + declared = math.nan + if not 0 <= declared < math.inf: + problems.append( + "no-tools spend: spent_before must be a finite, nonnegative number, " + f"not {design.get('spent_before')!r}" + ) + elif declared < spent - MONEY_TOLERANCE: + problems.append( + f"no-tools spend: spent_before ${declared:.4f} is less than " + f"the saved records' ${spent:.4f}" + ) return problems @@ -675,40 +732,7 @@ def __init__( run_household if run_household is not None else pilot.run_household ) self._lock_file = None - baseline_path = None - try: - baseline = design["baseline_exclusions"] - if not isinstance(baseline, dict) or not isinstance( - baseline.get("path"), str - ): - raise StateError("baseline exclusions need path and sha256") - baseline_path = Path(baseline["path"]) - baseline_bytes = baseline_path.read_bytes() - if hashlib.sha256(baseline_bytes).hexdigest() != baseline.get("sha256"): - raise StateError("baseline exclusions do not hash as declared") - payload = json.loads(baseline_bytes.decode("utf-8")) - exclusions = ( - payload.get("exclusions") if isinstance(payload, dict) else None - ) - if not isinstance(exclusions, list): - raise StateError("baseline exclusions must contain an exclusions list") - cells = [] - for exclusion in exclusions: - if not isinstance(exclusion, dict) or any( - not isinstance(exclusion.get(key), str) or not exclusion[key] - for key in ("scenario_id", "variable") - ): - raise StateError( - "baseline exclusion needs household and output strings" - ) - cells.append((exclusion["scenario_id"], exclusion["variable"])) - if len(cells) != len(set(cells)): - raise StateError("baseline exclusions repeat a cell") - except (OSError, KeyError, TypeError, ValueError, RecursionError) as error: - at = "baseline exclusions" - if baseline_path is not None: - at += f" ({baseline_path})" - raise StateError(f"{at}: {error}") from error + check_baseline(design) self.root.mkdir(parents=True, exist_ok=True) self._lock_file = (self.root / ".confirmatory-no-tools.lock").open("a") try: @@ -838,8 +862,10 @@ def archive(self, model: str, sid: str) -> bool: refusal leaves the disk history unchanged. After the rename it hashes the moved bytes again; if a write landed between the first hash and the rename, it moves them back and refuses. It cannot see a write to - the same inode during either hash or after the second; every later - reader hashes each archive against its link and refuses a mismatch.""" + the same inode during either hash or after the second. Later readers + refuse an archive that differs from a surviving link; a saved record + holds that link only once the next attempt's final lands, so startup + rebuilds the last archive's digest from its bytes after an interrupt.""" state = self.histories[model][sid] record, entries = state["final"], state["entries"] n = len(entries) + 1 diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index c4247ab1..080296ed 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -17,7 +17,9 @@ from tests.test_confirmatory_no_tools import Provider, nt, record, stage from tests.test_confirmatory_no_tools import setup as _setup_fixture from tests.test_confirmatory_no_tools_records import ( + OPTIONAL_EVIDENCE, assert_boundary, + full_record, save_records, ) from tests.test_confirmatory_no_tools_records import ( @@ -288,6 +290,136 @@ def test_preflight_pool_hash_and_decoder_share_one_valid_buffer(tmp_path, monkey assert world.provider.count == {} +def no_tools_households(design, model): + spec = design["no_tools"] + return Path(spec["runs"]) / model / spec["folder"] / spec["stage"] / "households" + + +def saved_spend(world, design): + """Save $0.20 in a cheap model's final, $0.10 in the archive it replaced and + $0.05 in an optional model's final. Return their sum.""" + model, sid = cr.CHEAP[0], design["draw_order"][0] + folder = no_tools_households(design, model) + final = json.loads((folder / f"{sid}.json").read_text()) + failed = { + key: value for key, value in final.items() if key not in OPTIONAL_EVIDENCE + } + failed.update( + status="provider_error", + error="APIConnectionError", + predictions={variable: None for variable in final["variables"]}, + turns=[], + totals={"cost_usd": 0.1}, + ) + archive = folder / "attempts" / f"{sid}-1.json" + digest = cr.write_json(archive, failed) + final.update( + attempts=[nt.entry(archive, failed, 1, digest)], totals={"cost_usd": 0.2} + ) + cr.write_json(folder / f"{sid}.json", final) + optional = full_record(world.board, "gpt-6-sol", design["engine"], sid) + optional["totals"] = {"cost_usd": 0.05} + cr.write_json(no_tools_households(design, "gpt-6-sol") / f"{sid}.json", optional) + return 0.35 + + +@pytest.mark.parametrize( + "margin, refused", + [ + (1.0, False), + (0.0, False), + (-nt.MONEY_TOLERANCE / 2, False), + (-nt.MONEY_TOLERANCE * 2, True), + (-0.05, True), + (-0.35, True), + ], + ids=[ + "above", + "equal", + "inside-tolerance", + "past-tolerance", + "lacks-optional", + "zero", + ], +) +def test_preflight_compares_spent_before_with_all_saved_spend( + tmp_path, monkeypatch, margin, refused +): + world, design = pool_design(tmp_path, monkeypatch) + spent = saved_spend(world, design) + design["spent_before"] = spent + margin + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert [problem for problem in problems if problem.startswith("no-tools")] == ( + [ + f"no-tools spend: spent_before ${spent + margin:.4f} is less than " + f"the saved records' ${spent:.4f}" + ] + if refused + else [] + ) + assert world.provider.count == {} + + +@pytest.mark.parametrize( + "value", + [float("nan"), float("inf"), None, "unknown", -1e-7, 10**400], + ids=["nan", "infinite", "null", "text", "negative", "overflow"], +) +def test_preflight_refuses_an_unusable_spent_before(tmp_path, monkeypatch, value): + world, design = pool_design(tmp_path, monkeypatch) + design["spent_before"] = value + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any("finite, nonnegative" in problem for problem in problems), problems + assert world.provider.count == {} + + +@pytest.mark.parametrize( + "fault, needle", + [ + ("valid", None), + ("undeclared", "baseline_exclusions"), + ("missing", "No such file"), + ("changed", "do not hash as declared"), + ("malformed", "Expecting"), + ("schema", "must contain an exclusions list"), + ("repeated", "repeat a cell"), + ], +) +def test_preflight_applies_startup_s_baseline_check( + tmp_path, monkeypatch, fault, needle +): + world, design = pool_design(tmp_path, monkeypatch) + spec = design["baseline_exclusions"] + path = Path(spec["path"]) + cell = {"scenario_id": "h", "variable": "snap"} + if fault == "undeclared": + del design["baseline_exclusions"] + elif fault == "missing": + path.unlink() + elif fault == "changed": + path.write_text('{"exclusions": [], "changed": true}') + elif fault != "valid": + path.write_text( + { + "malformed": "{", + "schema": "{}", + "repeated": json.dumps({"exclusions": [cell, cell]}), + }[fault] + ) + spec["sha256"] = nt.sha256_file(path) + found = [ + problem + for problem in cr.preflight(design, (cr.CHEAP[0],)) + if "baseline" in problem + ] + if needle is None: + assert found == [] + else: + assert len(found) == 1 and needle in found[0], found + assert fault == "undeclared" or str(path) in found[0] + assert world.provider.count == {} + + @pytest.fixture(scope="module") def report_module(): configured = os.environ.get("CONFIRMATORY_REPORT") diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 85ff9ed0..2c56bb4a 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -32,6 +32,7 @@ def setup(tmp_path, monkeypatch): "board": "synthetic.csv", "engine": "policyengine-us 2.60.4", "draw_order": draw, + "spent_before": nt.NO_TOOLS_CAP, "baseline_exclusions": { "path": str(baseline), "sha256": nt.sha256_file(baseline), diff --git a/tests/test_confirmatory_no_tools_records.py b/tests/test_confirmatory_no_tools_records.py index 45d9681b..5c1b8ce3 100644 --- a/tests/test_confirmatory_no_tools_records.py +++ b/tests/test_confirmatory_no_tools_records.py @@ -110,6 +110,7 @@ def treatment_templates(tmp_path_factory): baseline.write_text('{"exclusions": []}\n') design = { "draw_order": ["h2", "outside"], + "spent_before": nt.NO_TOOLS_CAP, "baseline_exclusions": { "path": str(baseline), "sha256": nt.sha256_file(baseline), diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 070bda02..0616463b 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -1561,7 +1561,11 @@ def test_a_model_that_decides_nothing_still_has_both_logs(world): admit = held.admit def late(sid, lane): - time.sleep(0.3) + # A sleep here would race haiku's attempt on a loaded host. + deadline = time.monotonic() + 30 + while not w.run.ceiling.exhausted(): + assert time.monotonic() < deadline, "haiku never crossed the ceiling" + time.sleep(0.01) return admit(sid, lane) held.admit = late @@ -1909,10 +1913,19 @@ def test_an_attempt_past_its_deadline_is_abandoned_and_stops_the_model( is a harness error with the spend so far, the model stops, and the conversation ends `budget_stop` at its next turn. Its spend after the deadline is a late-cost event. The second turn waits for the abandon, so - the split does not depend on when the waiting thread wakes.""" + the split does not depend on when the waiting thread wakes, and the join's + own timeout pins the deadline, because that wait hides when the abandon came.""" monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 1.0) abandoned = threading.Event() abandon = cr.AttemptSpend.abandon + join = threading.Thread.join + waits = {} + + def recorded_join(thread, timeout=None): + waits.setdefault(thread.name, timeout) + return join(thread, timeout) + + monkeypatch.setattr(threading.Thread, "join", recorded_join) def abandon_then_release(spend): spent = abandon(spend) @@ -1933,6 +1946,7 @@ def abandon_then_release(spend): }, } w.execute() + assert waits[f"{model}-{sid}-{first}-1"] == 1.0 line = next( x for x in w.attempts(model) if x["household"] == sid and x["arm"] == first ) @@ -2215,9 +2229,12 @@ def complete_preflight_design(w, tmp_path): ) pool = tmp_path / "pool.json" pool.write_text(json.dumps([{"id": s} for s in POOL])) + baseline = tmp_path / "baseline-exclusions.json" + baseline.write_text('{"exclusions": []}\n') design = dict( w.design, eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, + baseline_exclusions={"path": str(baseline), "sha256": cr.sha256_file(baseline)}, treatment_harness=str(cr.ROOT), run_commit="0" * 40, no_tools={ diff --git a/tests/test_confirmatory_unreadable_evidence.py b/tests/test_confirmatory_unreadable_evidence.py index e3dbd174..f48be754 100644 --- a/tests/test_confirmatory_unreadable_evidence.py +++ b/tests/test_confirmatory_unreadable_evidence.py @@ -446,6 +446,86 @@ def test_archive_refuses_a_final_replaced_by_a_fifo_at_once(setup): assert provider.calls == [] +def test_archive_refuses_a_target_replaced_by_a_fifo_after_its_rename( + setup, monkeypatch +): + provider = Provider() + with stage(setup, provider) as plan: + folder, source, _ = prepared_final(plan) + target = folder / "attempts" / "h2-1.json" + original = nt.os.replace + + def rename_then_replace(old, new): + original(old, new) + Path(new).unlink() + os.mkfifo(new) + + monkeypatch.setattr(nt.os, "replace", rename_then_replace) + start = time.monotonic() + with fifo_released_after(target): + with pytest.raises( + nt.StateError, match="expected a regular file, found FIFO" + ): + plan.archive(nt.CHEAP[0], "h2") + assert time.monotonic() - start < 5, "the second hash may not block on a FIFO" + assert not source.exists() + assert plan.histories[nt.CHEAP[0]]["h2"]["entries"] == [] + assert provider.calls == [] + + +@pytest.mark.parametrize("fault", ["second-read", "move-back"]) +def test_a_failed_archive_stops_one_model_and_startup_recovers_it(setup, fault): + """The second hash or the move back fails: that model errors, the others + finish, the invocation releases its lock, and the archive stays with no + final and no link. Startup then rebuilds the link from the bytes present, + changed or not, which is the limit the draft states.""" + model = nt.CHEAP[0] + provider = Provider( + {(model, "h2", 1): {"status": "provider_error", "error": "APIConnectionError"}} + ) + plan = stage(setup, provider) + folder = plan.households(model) + source, target = folder / "h2.json", folder / "attempts" / "h2-1.json" + with pytest.MonkeyPatch.context() as patch: + if fault == "second-read": + read = nt.read_regular + + def unreadable(path): + if Path(path) == target: + raise PermissionError(errno.EACCES, "synthetic denied", str(path)) + return read(path) + + patch.setattr(nt, "read_regular", unreadable) + else: + replace = os.replace + + def failed_move_back(old, new): + if Path(old) == source: + Path(old).write_bytes(Path(old).read_bytes() + b" ") + elif Path(old) == target: + raise OSError(errno.EIO, "synthetic failed move", str(old)) + replace(old, new) + + patch.setattr(os, "replace", failed_move_back) + assert plan.execute() == 1 + assert plan.states[model] == "error" + assert str(target) in plan.out.getvalue() + assert all(plan.states[other] == "complete" for other in nt.CHEAP[1:]) + assert all( + plan.standing(other, sid) for other in nt.CHEAP[1:] for sid in ("h2", "h1") + ) + assert [call for call in provider.calls if call[0] == model] == [(model, "h2", 1)] + assert target.exists() and not source.exists() + digest = nt.sha256_file(target) + again = Provider() + resumed = stage(setup, again, models=[model]) # raises if the lock survived + assert resumed.histories[model]["h2"]["entries"][0]["sha256"] == digest + assert resumed.execute() == 0 + assert again.calls == [(model, "h2", 1), (model, "h1", 1)] + assert json.loads(source.read_text())["attempts"][0]["sha256"] == digest + assert nt.stage_problems(setup["design"]) == [] + + def test_archive_target_stat_failure_refuses(setup, monkeypatch): provider = Provider() with stage(setup, provider) as plan: From ceae93f4f095fd687512a4eb4a791a286b603b11 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 20:14:27 -0400 Subject: [PATCH 43/48] Confirmatory runner: sum saved spend exactly, as the report does Answers review runner-r19 and item 3 of confirmatory-report-r18. The preflight added each cost to a running total, and the report grouped its total by model, so the two programs rounded differently. A spent_before at the tolerance passed one and failed the other, and $2 of saved spend could round away beside a $1e16 cost. history now lists each saved cost, and exact_spend takes their correctly rounded sum, the same in any order and grouping. The report takes the same sum. An intermediate overflow means a total past every float, which the comparison refuses. Startup's spend uses the same sum. Tests: every order of two sets of costs at the tolerance, costs a running total rounds away, and a total past every float; a spent_before given as numeric text or a boolean, which counts as its number in both programs; and a design without spent_before. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 44 +++++++++----- tests/test_confirmatory_input_reads.py | 83 +++++++++++++++++++++++--- 2 files changed, 105 insertions(+), 22 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 7ab908f1..af728dad 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -470,8 +470,11 @@ def _attempts_agree(listed: list, entries: list) -> bool: ) -def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], float]: - """Validate a model's histories, including an interrupted archive, and sum spend.""" +def history( + folder: Path, validate=None, drawn=None +) -> tuple[dict[str, dict], list[float]]: + """Validate a model's histories, including an interrupted archive, and list + each saved cost.""" # Finals first, as the report lists them: an unlistable households folder # is named before the attempts folder inside it. finals = {path.stem: path for path in saved_json_paths(folder)} @@ -481,7 +484,7 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f if not match: raise StateError(f"{path}: invalid attempt number") archives.setdefault(match[1], []).append((int(match[2]), path)) - states, spent = {}, 0.0 + states, costs = {}, [] for sid in sorted(set(finals) | set(archives)): paths = sorted(archives.get(sid, [])) numbers = [n for n, _ in paths] @@ -510,7 +513,7 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f if action(cls, n) not in ("rerun", "pause-rerun"): raise StateError(f"{path}: archived attempt was ineligible for rerun") entries = record["attempts"] + [entry(path, record, n, digest)] - spent += attempt_cost(record) + costs.append(attempt_cost(record)) final = final_sha256 = None if sid in finals: path = finals[sid] @@ -542,9 +545,19 @@ def history(folder: Path, validate=None, drawn=None) -> tuple[dict[str, dict], f f"{path}: cannot classify saved final: {error}" ) from error entries = final["attempts"] - spent += attempt_cost(final) + costs.append(attempt_cost(final)) states[sid] = {"entries": entries, "final": final, "final_sha256": final_sha256} - return states, spent + return states, costs + + +def exact_spend(costs: list[float]) -> float: + """The correctly rounded sum of saved costs, the same in any order and + grouping, so that this reaches the report's total. Its intermediate overflow + means a total past every float.""" + try: + return math.fsum(costs) + except OverflowError: + return math.inf def check_baseline(design: dict) -> None: @@ -587,8 +600,8 @@ def stage_problems(design: dict) -> list[str]: As the report does, refuse a baseline that fails its hash or schema, any model's archive that no final lists, any final that owes a rerun, optional - models included, and a spent_before below the saved records' spend; each - drawn cheap-model household also needs a final that stands. Startup + models included, and a spent_before below the saved records' exact total; + each drawn cheap-model household also needs a final that stands. Startup recovery still accepts both unfinished states.""" problems = [] try: @@ -605,7 +618,7 @@ def stage_problems(design: dict) -> list[str]: except (OSError, KeyError, TypeError, ValueError) as error: return [*problems, f"no-tools stage: {error}"] board = None - spent = 0.0 + costs = [] def checked_board(): nonlocal board @@ -615,14 +628,14 @@ def checked_board(): for model in sorted(models): try: - states, cost = history( + states, saved = history( root / model / folder / stage / "households", validate=lambda path, record: validate_treatment( path, record, model, checked_board, design["engine"] ), drawn=set(draw_order) if model in CHEAP else None, ) - spent += cost + costs += saved drawn = draw_order if model in CHEAP else [] for sid in drawn + sorted(set(states) - set(drawn)): state = states.get(sid) @@ -645,6 +658,7 @@ def checked_board(): ) except (OSError, TypeError, ValueError, AttributeError, KeyError) as error: problems.append(f"no-tools {model}: {error}") + spent = exact_spend(costs) try: declared = float(design.get("spent_before")) except (TypeError, ValueError, OverflowError): @@ -761,10 +775,10 @@ def _load_state(self, board) -> None: f"drawn households missing from the board: {sorted(missing)}" ) self.histories = {} - spent = 0.0 + costs = [] folders = {p.name for p in saved_children(self.root) if listed_is_dir(p)} for model in sorted(folders | set(self.models)): - states, cost = history( + states, saved = history( self.households(model), validate=lambda path, record: validate_treatment( path, record, model, self.board, design["engine"] @@ -772,7 +786,7 @@ def _load_state(self, board) -> None: drawn=set(self.draw_order) if model in CHEAP else None, ) self.histories[model] = states - spent += cost + costs += saved # A saved ceiling failure is a stage-wide stop on every invocation. if any( state["final"] is not None @@ -784,7 +798,7 @@ def _load_state(self, board) -> None: for state in states.values() ): self.stopped.set() - self.spend = Spend(spent, self.stopped) + self.spend = Spend(exact_spend(costs), self.stopped) self.states = {model: "pending" for model in self.models} def close(self) -> None: diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index 080296ed..31894d88 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -5,7 +5,9 @@ import hashlib import importlib.util import io +import itertools import json +import math import os import re import sys @@ -295,9 +297,9 @@ def no_tools_households(design, model): return Path(spec["runs"]) / model / spec["folder"] / spec["stage"] / "households" -def saved_spend(world, design): - """Save $0.20 in a cheap model's final, $0.10 in the archive it replaced and - $0.05 in an optional model's final. Return their sum.""" +def saved_spend(world, design, final_cost=0.2, archive_cost=0.1, optional_cost=0.05): + """Save a cost in a cheap model's final, one in the archive it replaced and + one in an optional model's final. Return their sum.""" model, sid = cr.CHEAP[0], design["draw_order"][0] folder = no_tools_households(design, model) final = json.loads((folder / f"{sid}.json").read_text()) @@ -309,18 +311,19 @@ def saved_spend(world, design): error="APIConnectionError", predictions={variable: None for variable in final["variables"]}, turns=[], - totals={"cost_usd": 0.1}, + totals={"cost_usd": archive_cost}, ) archive = folder / "attempts" / f"{sid}-1.json" digest = cr.write_json(archive, failed) final.update( - attempts=[nt.entry(archive, failed, 1, digest)], totals={"cost_usd": 0.2} + attempts=[nt.entry(archive, failed, 1, digest)], + totals={"cost_usd": final_cost}, ) cr.write_json(folder / f"{sid}.json", final) optional = full_record(world.board, "gpt-6-sol", design["engine"], sid) - optional["totals"] = {"cost_usd": 0.05} + optional["totals"] = {"cost_usd": optional_cost} cr.write_json(no_tools_households(design, "gpt-6-sol") / f"{sid}.json", optional) - return 0.35 + return final_cost + archive_cost + optional_cost @pytest.mark.parametrize( @@ -360,6 +363,47 @@ def test_preflight_compares_spent_before_with_all_saved_spend( assert world.provider.count == {} +@pytest.mark.parametrize( + "costs", + sorted( + { + *itertools.permutations((0.1, 0.1, 0.4)), + *itertools.permutations((0.1, 0.2, 0.3)), + } + ), +) +def test_preflight_sums_saved_spend_exactly_in_any_order(tmp_path, monkeypatch, costs): + """Added one at a time, these costs land on either side of the tolerance + with their order; the report's exact total does not move.""" + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design, *costs) + design["spent_before"] = 0.599999 + problems = cr.preflight(design, (cr.CHEAP[0],)) + refused = 0.599999 < math.fsum(costs) - nt.MONEY_TOLERANCE + assert any(p.startswith("no-tools spend") for p in problems) is refused, problems + + +@pytest.mark.parametrize( + "declared, refused", [(1e16, True), (1e16 + 2, False)], ids=["short", "exact"] +) +def test_preflight_keeps_costs_a_running_total_rounds_away( + tmp_path, monkeypatch, declared, refused +): + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design, 1e16, 1.0, 1.0) + design["spent_before"] = declared + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(p.startswith("no-tools spend") for p in problems) is refused, problems + + +def test_preflight_refuses_a_saved_total_past_every_float(tmp_path, monkeypatch): + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design, 1.5e308, 0.0, 1.5e308) + design["spent_before"] = 1e308 + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any("the saved records' $inf" in problem for problem in problems), problems + + @pytest.mark.parametrize( "value", [float("nan"), float("inf"), None, "unknown", -1e-7, 10**400], @@ -373,6 +417,31 @@ def test_preflight_refuses_an_unusable_spent_before(tmp_path, monkeypatch, value assert world.provider.count == {} +def test_preflight_refuses_a_design_without_spent_before(tmp_path, monkeypatch): + world, design = pool_design(tmp_path, monkeypatch) + del design["spent_before"] + assert "the design has no spent_before" in cr.preflight(design, (cr.CHEAP[0],)) + assert any("finite, nonnegative" in p for p in nt.stage_problems(design)) + assert world.provider.count == {} + + +@pytest.mark.parametrize( + "declared, refused", + [("1", False), ("0.1", True), (True, False), (False, True)], + ids=["text-enough", "text-short", "true", "false"], +) +def test_preflight_counts_numeric_text_and_booleans_as_numbers( + tmp_path, monkeypatch, declared, refused +): + """As the report does: float() decides what a spent_before is worth.""" + world, design = pool_design(tmp_path, monkeypatch) + saved_spend(world, design) + design["spent_before"] = declared + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert any(p.startswith("no-tools spend") for p in problems) is refused, problems + assert not any("finite, nonnegative" in p for p in problems), problems + + @pytest.mark.parametrize( "fault, needle", [ From 3c80377ebe18b8b162c58b8a07933931f8859817 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 23:16:38 -0400 Subject: [PATCH 44/48] Confirmatory no-tools: pin startup's exact spend at the stage cap Review runner-r20 found that restoring a running total at startup alone would pass every test. A new case saves costs whose running total stays under the cap and whose exact sum reaches it, and asserts that no owed attempt starts and that the invocation lock comes free. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 4 ++-- tests/test_confirmatory_no_tools.py | 30 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index af728dad..ac39aab8 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -552,8 +552,8 @@ def history( def exact_spend(costs: list[float]) -> float: """The correctly rounded sum of saved costs, the same in any order and - grouping, so that this reaches the report's total. Its intermediate overflow - means a total past every float.""" + grouping, so that this reaches the report's total. An OverflowError gives + infinity, which every finite `spent_before` falls short of.""" try: return math.fsum(costs) except OverflowError: diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index 2c56bb4a..b7d184f8 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -3,8 +3,11 @@ from __future__ import annotations import ast +import functools import io import json +import math +import operator import os import signal import sys @@ -1487,6 +1490,33 @@ def test_valid_saved_numeric_cost_preserves_spend(setup, cost): assert provider.calls == [] +def test_startup_spend_takes_the_exact_sum_at_the_stage_cap(setup): + """A running total of these costs stays under the cap and their exact sum + reaches it, so no owed attempt starts and the invocation lock comes free.""" + costs = [math.nextafter(nt.NO_TOOLS_CAP, 0.0), 2**-52, 2**-52, 2**-52] + assert functools.reduce(operator.add, costs) < nt.NO_TOOLS_CAP + root = Path(setup["design"]["no_tools"]["runs"]) + saved = [ + (nt.CHEAP[0], "h2"), + (nt.CHEAP[0], "h1"), + (nt.CHEAP[1], "h2"), + (nt.CHEAP[2], "h2"), + ] + for (model, sid), cost in zip(saved, costs): + folder = root / model / "none/confirmatory/households" + folder.mkdir(parents=True, exist_ok=True) + profile = SimpleNamespace(name=model, arm="none", model_id="synthetic") + payload = dict(record(sid, profile, cost=cost), attempts=[]) + (folder / f"{sid}.json").write_text(json.dumps(payload)) + provider = Provider() + plan = stage(setup, provider) + assert plan.spend.spent_usd == nt.NO_TOOLS_CAP + assert plan.stopped.is_set() + assert plan.execute() == 1 + assert provider.calls == [] + assert plan._lock_file is None + + def run_no_tools_probe(setup, tmp_path, name, body): """Bound a signal or worker-shutdown regression to its own process.""" import textwrap From 58ba64be0241ff93be4c7fb3a7e3f4aa50645b75 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 00:54:05 -0400 Subject: [PATCH 45/48] No-tools startup: test the sum across models at the cap and the lock's release Review runner-r21 found two gaps in the startup test. Rounding each model's costs first escaped it, so a second test saves costs whose subtotals fall one float under the cap and whose exact sum reaches it. The test's lock assertion read bookkeeping, so both tests now take the lock from a second open. `exact_spend`'s docstring limits the claim to a flat list of costs, in any order, and to histories that pass validation. Production code changes only in that docstring. Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 7 ++-- tests/test_confirmatory_no_tools.py | 54 ++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index ac39aab8..d3891a8e 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -551,9 +551,10 @@ def history( def exact_spend(costs: list[float]) -> float: - """The correctly rounded sum of saved costs, the same in any order and - grouping, so that this reaches the report's total. An OverflowError gives - infinity, which every finite `spent_before` falls short of.""" + """The correctly rounded sum of a flat list of saved costs, the same in any + order. For histories that pass validation this reaches the report's total. + An OverflowError gives infinity, which every finite `spent_before` falls + short of.""" try: return math.fsum(costs) except OverflowError: diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index b7d184f8..e53abf47 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import fcntl import functools import io import json @@ -1490,19 +1491,11 @@ def test_valid_saved_numeric_cost_preserves_spend(setup, cost): assert provider.calls == [] -def test_startup_spend_takes_the_exact_sum_at_the_stage_cap(setup): - """A running total of these costs stays under the cap and their exact sum - reaches it, so no owed attempt starts and the invocation lock comes free.""" - costs = [math.nextafter(nt.NO_TOOLS_CAP, 0.0), 2**-52, 2**-52, 2**-52] - assert functools.reduce(operator.add, costs) < nt.NO_TOOLS_CAP +def require_a_stop_at_the_cap(setup, saved): + """Save these (model, household, cost) finals, and require that startup's sum + reaches the cap, no owed attempt starts, and another open takes the lock.""" root = Path(setup["design"]["no_tools"]["runs"]) - saved = [ - (nt.CHEAP[0], "h2"), - (nt.CHEAP[0], "h1"), - (nt.CHEAP[1], "h2"), - (nt.CHEAP[2], "h2"), - ] - for (model, sid), cost in zip(saved, costs): + for model, sid, cost in saved: folder = root / model / "none/confirmatory/households" folder.mkdir(parents=True, exist_ok=True) profile = SimpleNamespace(name=model, arm="none", model_id="synthetic") @@ -1514,7 +1507,42 @@ def test_startup_spend_takes_the_exact_sum_at_the_stage_cap(setup): assert plan.stopped.is_set() assert plan.execute() == 1 assert provider.calls == [] - assert plan._lock_file is None + with (root / ".confirmatory-no-tools.lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + pytest.fail(f"the invocation lock stayed held: {error}") + + +def test_startup_spend_takes_the_exact_sum_at_the_stage_cap(setup): + """A running total of these costs stays under the cap and their exact sum + reaches it.""" + costs = [math.nextafter(nt.NO_TOOLS_CAP, 0.0), 2**-52, 2**-52, 2**-52] + assert functools.reduce(operator.add, costs) < nt.NO_TOOLS_CAP + places = [ + (nt.CHEAP[0], "h2"), + (nt.CHEAP[0], "h1"), + (nt.CHEAP[1], "h2"), + (nt.CHEAP[2], "h2"), + ] + require_a_stop_at_the_cap(setup, [(*p, c) for p, c in zip(places, costs)]) + + +def test_startup_spend_sums_every_model_at_once_at_the_stage_cap(setup): + """Rounding each model's costs first falls one float under the cap, and the + exact sum of all three reaches it.""" + first = [3 * 2**-53, math.nextafter(nt.NO_TOOLS_CAP, 0.0)] + second = [2**-52] + subtotals = [nt.exact_spend(first), nt.exact_spend(second)] + assert nt.exact_spend(subtotals) < nt.NO_TOOLS_CAP + require_a_stop_at_the_cap( + setup, + [ + (nt.CHEAP[0], "h1", first[0]), + (nt.CHEAP[0], "h2", first[1]), + (nt.CHEAP[1], "h2", second[0]), + ], + ) def run_no_tools_probe(setup, tmp_path, name, body): From 384666600fb35be1e6b13720716739f19f10ae2c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 01:50:40 -0400 Subject: [PATCH 46/48] No-tools startup: test costs whose sum overflows Review runner-r22 found that startup's overflow rule had no test. Replacing `exact_spend(costs)` with `math.fsum(costs)` at startup alone would have raised a raw OverflowError and passed every test, because the existing overflow cases reach the preflight or refuse one invalid cost. A new test saves two valid 1.5e308 finals and requires that startup takes infinity for the total, stops every owed attempt and frees the lock. The helper takes the expected total as an argument. Production code doesn't change. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_no_tools.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_confirmatory_no_tools.py b/tests/test_confirmatory_no_tools.py index e53abf47..14c38fd9 100644 --- a/tests/test_confirmatory_no_tools.py +++ b/tests/test_confirmatory_no_tools.py @@ -1491,9 +1491,9 @@ def test_valid_saved_numeric_cost_preserves_spend(setup, cost): assert provider.calls == [] -def require_a_stop_at_the_cap(setup, saved): +def require_a_stop_at_the_cap(setup, saved, spent=nt.NO_TOOLS_CAP): """Save these (model, household, cost) finals, and require that startup's sum - reaches the cap, no owed attempt starts, and another open takes the lock.""" + reaches `spent`, no owed attempt starts, and another open takes the lock.""" root = Path(setup["design"]["no_tools"]["runs"]) for model, sid, cost in saved: folder = root / model / "none/confirmatory/households" @@ -1503,7 +1503,7 @@ def require_a_stop_at_the_cap(setup, saved): (folder / f"{sid}.json").write_text(json.dumps(payload)) provider = Provider() plan = stage(setup, provider) - assert plan.spend.spent_usd == nt.NO_TOOLS_CAP + assert plan.spend.spent_usd == spent assert plan.stopped.is_set() assert plan.execute() == 1 assert provider.calls == [] @@ -1545,6 +1545,19 @@ def test_startup_spend_sums_every_model_at_once_at_the_stage_cap(setup): ) +def test_startup_spend_stops_on_costs_whose_sum_overflows(setup): + """Each cost fits a float and `math.fsum` overflows on the pair, so startup + takes infinity for the total and stops.""" + costs = [1.5e308, 1.5e308] + with pytest.raises(OverflowError): + math.fsum(costs) + require_a_stop_at_the_cap( + setup, + [(nt.CHEAP[0], "h1", costs[0]), (nt.CHEAP[1], "h1", costs[1])], + spent=math.inf, + ) + + def run_no_tools_probe(setup, tmp_path, name, body): """Bound a signal or worker-shutdown regression to its own process.""" import textwrap From 2ee8acfc0bc9d66ec191077b6869dad620e764c9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 03:06:15 -0400 Subject: [PATCH 47/48] Stub the host facts in the confirmatory tests' World CI's test job failed 22 confirmatory tests on Linux at 38466660; they had passed on the Mac that ran every earlier suite. Fourteen reached a preflight check that compares the installed policyengine-us with the board's engine, and the lockfile installs 2.41.0 where the fixture board names 2.60.4. Eight reached host_power, which calls macOS's pmset. World already fakes the board's engine, so it now fakes the installed version and the power reading too. Its tests no longer read either from the host. A test that wants a different reading still sets it after World, as the lid-closed test does. The eight stubs in the unreadable-evidence tests and the ones in pool_design and engine_fault repeat World's and go. Production code does not change. A pytest plugin that hides pmset and reports 2.41.0 reproduced CI's 22 failures on this Mac at 38466660, by name. With this commit all 772 runner tests pass under that plugin. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_board_engine.py | 3 --- tests/test_confirmatory_input_reads.py | 1 - tests/test_confirmatory_run.py | 3 +++ tests/test_confirmatory_unreadable_evidence.py | 8 -------- 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/tests/test_confirmatory_board_engine.py b/tests/test_confirmatory_board_engine.py index 99dc3faf..e8dc7924 100644 --- a/tests/test_confirmatory_board_engine.py +++ b/tests/test_confirmatory_board_engine.py @@ -8,7 +8,6 @@ import pytest from tests.test_confirmatory_run import ( - ENGINE, complete_preflight_design, cr, pilot, @@ -32,8 +31,6 @@ def engine_fault(world, tmp_path, monkeypatch, fault): design = complete_preflight_design(w, tmp_path) metadata = Path(design["board"]) / "reference_outputs.csv.meta.json" monkeypatch.setattr(pilot, "board_engine", REAL_BOARD_ENGINE) - monkeypatch.setattr(pilot, "_versions", lambda: {"policyengine-us": ENGINE}) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) def no_run(*args, **kwargs): pytest.fail("a refused preflight must not construct or call a provider") diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index 31894d88..e85e7562 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -259,7 +259,6 @@ def write_then_change(path, payload, *args, **kwargs): def pool_design(tmp_path, monkeypatch): monkeypatch.setattr(cr, "HOUSEHOLDS", N_DRAW) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) world = World(tmp_path, monkeypatch, models=(cr.CHEAP[0],)) return world, complete_preflight_design(world, tmp_path) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 0616463b..44068cd8 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -789,6 +789,9 @@ def __init__( } monkeypatch.setattr(pilot, "load_board", lambda path: self.board) monkeypatch.setattr(pilot, "board_engine", lambda path=None: ENGINE) + installed = {**pilot._versions(), "policyengine-us": ENGINE} + monkeypatch.setattr(pilot, "_versions", lambda: installed) + monkeypatch.setattr(cr, "host_power", lambda: (True, True)) monkeypatch.setitem( pilot._STATED_ENGINE, "version", pilot._STATED_ENGINE["version"] ) diff --git a/tests/test_confirmatory_unreadable_evidence.py b/tests/test_confirmatory_unreadable_evidence.py index f48be754..cac7af15 100644 --- a/tests/test_confirmatory_unreadable_evidence.py +++ b/tests/test_confirmatory_unreadable_evidence.py @@ -72,7 +72,6 @@ def test_unlistable_optional_history_refuses( else: w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) spec = design["no_tools"] folder = ( Path(spec["runs"]) / "gpt-6-sol" / spec["folder"] / spec["stage"] / "households" @@ -101,7 +100,6 @@ def test_model_inventory_failure_refuses( else: w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) root = Path(design["no_tools"]["runs"]) (root / "gpt-6-sol").mkdir(parents=True, exist_ok=True) target = root / "gpt-6-sol" if which == "model-stat" else root @@ -148,7 +146,6 @@ def test_listed_model_entry_without_metadata_refuses( else: w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) root = Path(design["no_tools"]["runs"]) root.mkdir(parents=True, exist_ok=True) target = root / "ghost-model" @@ -173,7 +170,6 @@ def test_preflight_listed_calculator_entry_without_metadata_refuses( ): w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) folder = Path(design["runs"]) / cr.CHEAP[0] parent = folder if level == "arm" else folder / "pe" parent.mkdir(parents=True, exist_ok=True) @@ -196,7 +192,6 @@ def test_broken_optional_history_folder_refuses( else: w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) spec = design["no_tools"] folder = ( Path(spec["runs"]) / "gpt-6-sol" / spec["folder"] / spec["stage"] / "households" @@ -259,7 +254,6 @@ def test_calculator_preflight_inventory_failure_refuses( ): w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) folder = Path(design["runs"]) / cr.CHEAP[0] targets = { "model": folder, @@ -286,7 +280,6 @@ def test_calculator_preflight_directory_stat_failure_refuses( ): w = world(models=(cr.CHEAP[0],)) design = complete_preflight_design(w, tmp_path) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) folder = Path(design["runs"]) / cr.CHEAP[0] target = ( folder @@ -315,7 +308,6 @@ def denied(path): return original(path) monkeypatch.setattr(Path, "read_bytes", denied) - monkeypatch.setattr(cr, "host_power", lambda: (True, True)) assert str(target) in "\n".join(cr.preflight(design, (cr.CHEAP[0],))) assert not w.provider.count From 0ab2b1e25a1bdf3564d6ad2085fc774b80b98c63 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 03:10:45 -0400 Subject: [PATCH 48/48] Test the host facts that World stubs World now reports the installed engine and the power reading, so no test could see the checks that read them, and no earlier test had held those checks with the real values either. An installed engine other than the board's must now refuse in the preflight twice, in its own check and through the no-tools treatment check, and again in main's final check, which must keep the preflight's problems. Each test requires that no provider is called. host_power gets its first test, on the four mixes of power source and lid state, with the text pmset and ioreg print on this Mac. Production code does not change. Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_board_engine.py | 46 +++++++++++++++++++++++++ tests/test_confirmatory_run.py | 22 ++++++++++++ 2 files changed, 68 insertions(+) diff --git a/tests/test_confirmatory_board_engine.py b/tests/test_confirmatory_board_engine.py index e8dc7924..c7361703 100644 --- a/tests/test_confirmatory_board_engine.py +++ b/tests/test_confirmatory_board_engine.py @@ -8,6 +8,7 @@ import pytest from tests.test_confirmatory_run import ( + ENGINE, complete_preflight_design, cr, pilot, @@ -94,3 +95,48 @@ def test_main_names_engine_reread_failure_and_preserves_all_problems( assert "earlier preflight problem" in output assert str(metadata) in output assert not w.provider.count + + +def installed_elsewhere(world, tmp_path, monkeypatch): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + monkeypatch.setattr(pilot, "_versions", lambda: {"policyengine-us": "2.41.0"}) + + def no_run(*args, **kwargs): + pytest.fail("a refused preflight must not construct or call a provider") + + monkeypatch.setattr(cr, "Run", no_run) + monkeypatch.setattr(pilot, "run_household", no_run) + return w, design + + +def test_preflight_refuses_an_installed_engine_other_than_the_boards( + world, tmp_path, monkeypatch +): + w, design = installed_elsewhere(world, tmp_path, monkeypatch) + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert f"policyengine-us 2.41.0 is installed, not {ENGINE}" in problems + for model in cr.CHEAP: + assert any( + problem.startswith(f"no-tools {model}: ") + and f"policyengine-us 2.41.0 is installed; the pilot pins {ENGINE}" + in problem + for problem in problems + ), problems + assert not w.provider.count + + +def test_main_rechecks_the_installed_engine_and_preserves_all_problems( + world, tmp_path, monkeypatch, capsys +): + w, design = installed_elsewhere(world, tmp_path, monkeypatch) + # Isolate main's final engine check, which must retain preflight's problems. + monkeypatch.setattr(cr, "preflight", lambda *_: ["earlier preflight problem"]) + design_path = tmp_path / "design.json" + design_path.write_text(json.dumps(design)) + assert cr.main(["run", str(design_path)]) == 1 + output = capsys.readouterr().out + assert "The run cannot start" in output + assert "earlier preflight problem" in output + assert "the board's engine check failed: policyengine-us 2.41.0" in output + assert not w.provider.count diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 44068cd8..e2e924f1 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2280,6 +2280,28 @@ def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( ) +@pytest.mark.parametrize( + "drawing, clamshell, expected", + [ + ("AC Power", "No", (True, True)), + ("Battery Power", "No", (False, True)), + ("AC Power", "Yes", (True, False)), + ("Battery Power", "Yes", (False, False)), + ], +) +def test_host_power_reads_pmset_and_ioreg(monkeypatch, drawing, clamshell, expected): + output = { + "pmset": f"Now drawing from '{drawing}'\n", + "ioreg": f' "AppleClamshellState" = {clamshell}\n', + } + + def run(command, **kwargs): + return subprocess.CompletedProcess(command, 0, stdout=output[command[0]]) + + monkeypatch.setattr(cr.subprocess, "run", run) + assert cr.host_power() == expected + + def test_the_preflight_refuses_a_runs_folder_with_records(world, tmp_path): w = world(models=("gpt-6-luna",)).execute() design = complete_preflight_design(w, tmp_path)