From 152639a0e4d8b08059a05ff7341d89d632ad9f22 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 11:57:45 -0400 Subject: [PATCH 01/10] feat(jit): bind program reuse to the admitted record check and app environment The cache key was the task id and parameter schema only, so a changed record check or a new app version at the same origin silently reused the old program. The admission manifest now pins the record check's digest (contracts of every saving step, anchors left out so a repair keeps it) and the canary's environment (origin plus an optional environment_id the app reports). A run refuses a mismatch. A caller may also pin both in TaskSignature, which puts them in the key. Keys without pins are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../flow/openadapt_flow/jit/compile_agent.py | 95 ++++++++ packages/flow/openadapt_flow/jit/lifecycle.py | 21 +- packages/flow/openadapt_flow/jit/signature.py | 123 +++++++++- packages/flow/tests/test_jit_reuse_binding.py | 221 ++++++++++++++++++ 4 files changed, 448 insertions(+), 12 deletions(-) create mode 100644 packages/flow/tests/test_jit_reuse_binding.py diff --git a/packages/flow/openadapt_flow/jit/compile_agent.py b/packages/flow/openadapt_flow/jit/compile_agent.py index 407f637e77..f8b48db082 100644 --- a/packages/flow/openadapt_flow/jit/compile_agent.py +++ b/packages/flow/openadapt_flow/jit/compile_agent.py @@ -74,6 +74,50 @@ ) +def _reuse_refusal( + record: Mapping[str, Any], workflow: Any, task: TaskSignature, app: Any +) -> Optional[str]: + """Why an admitted program may not be reused for this task here, or None. + + Reuse is bound to intent, not just to the task id and parameter schema: + the record check must be the one admitted (and the one the caller pinned, + if any), and the app environment must be the one the canary ran in (and + the one the caller pinned, if any). Admissions from before these pins + existed carry neither; then only the caller's pins are checked. + """ + + from openadapt_flow.jit.signature import record_check_digest + + current = record_check_digest(workflow) + admitted = record.get("record_check_digest") + if admitted is not None and admitted != current: + return ( + f"the program's record check ({current[:12]}) is not the one admitted " + f"({str(admitted)[:12]}); admit it again" + ) + if task.record_check is not None and task.record_check != current: + return ( + f"the task pins record check {task.record_check[:12]}, the admitted " + f"program checks {current[:12]}; a changed record check is a new task" + ) + try: + environment = app.environment + except Exception as exc: # noqa: BLE001 - an unknown environment never matches + return f"the app's environment could not be read ({type(exc).__name__})" + canary_env = (record.get("canary") or {}).get("environment") + if canary_env is not None and canary_env != environment: + return ( + f"the program was admitted in environment {canary_env!r}, this run is " + f"in {environment!r}; run a canary in this environment" + ) + if task.environment is not None and task.environment != environment: + return ( + f"the task is pinned to environment {task.environment!r}, the app " + f"reports {environment!r}" + ) + return None + + def _print_warning(text: str) -> None: print(f"WARNING: {text}", file=sys.stderr) @@ -104,6 +148,11 @@ class WebApp: the compiler reads. ``None`` keeps the authoring default. Raise it for an app whose lookups take seconds, or the program inherits checks compiled from a still-loading screen. + environment_id: Optional. What identifies this app environment + beyond its origin, such as the app's version or build + (``"7.0.2"``), or a callable that reads it from the app before + each canary and run. A program admitted under one environment is + refused under another (see :attr:`environment`). """ url: str @@ -113,6 +162,7 @@ class WebApp: setup: Optional[Callable[[Any], None]] = None record_video_dir: Optional[str] = None capture_settle_timeout_s: Optional[float] = None + environment_id: Optional[str | Callable[[], str]] = None def open(self) -> tuple[Any, Callable[[], None]]: """Open the browser at ``url`` and run :attr:`setup`; returns ``(backend, close)``.""" @@ -139,10 +189,22 @@ def origin(self) -> str: def kind(self) -> str: return "web" + @property + def environment(self) -> str: + """The app environment's identity: the origin, plus ``#``.""" + + return _environment(self.origin, self.environment_id) + @property def entry_url(self) -> Optional[str]: return self.url + @property + def environment(self) -> str: + """As for :attr:`WebApp.environment`.""" + + return _environment(self.origin, self.environment_id) + @property def backend_hints(self) -> Optional[dict[str, Any]]: return None @@ -193,12 +255,14 @@ class SurfaceApp: open_backend: Builds the live backend from ``backend``. Defaults to :func:`openadapt_flow.backends.factory.build_backend`. Pass one to inject a client (for example a test fixture). + environment_id: As for :class:`WebApp`. """ backend: Any record_reader: Optional[Callable[[], Optional[list[dict[str, Any]]]]] = None reset: Optional[Callable[[], None]] = None open_backend: Optional[Callable[[Any], Any]] = field(default=None, repr=False) + environment_id: Optional[str | Callable[[], str]] = None def __post_init__(self) -> None: from openadapt_flow.deployment import BackendConfig @@ -272,6 +336,12 @@ def origin(self) -> str: assert origin is not None return origin + @property + def environment(self) -> str: + """As for :attr:`WebApp.environment`.""" + + return _environment(self.origin, self.environment_id) + @property def backend_hints(self) -> Optional[dict[str, Any]]: if self.kind not in ("rdp", "citrix"): @@ -316,6 +386,19 @@ def open(self) -> tuple[Any, Callable[[], None]]: App = WebApp | SurfaceApp +def _environment(origin: str, environment_id: Any) -> str: + """``origin`` or ``origin#id``. A callable id is read now; one that fails + or returns nothing raises, so an unknown environment never matches.""" + + value = environment_id() if callable(environment_id) else environment_id + if value is None: + return origin + text = str(value).strip() + if not text: + raise ValueError("the app's environment_id is empty") + return f"{origin}#{text}" + + @dataclass(frozen=True) class TaskRequest: """What the agent is asked to do on a cache miss.""" @@ -794,6 +877,14 @@ def canary(self, *, principal_id: Any, **params: Any) -> JitResult: candidate = self.candidate() if candidate is None or self.admitted: raise AdmissionError("a canary runs a DRAFT; there is none") + if ( + self.task.environment is not None + and self.task.environment != self.app.environment + ): + raise AdmissionError( + f"the task is pinned to environment {self.task.environment!r}; " + f"this app reports {self.app.environment!r}" + ) _, principal = lifecycle.resolve_actor( principal_id, self.program_dir, key_dir=self.key_dir ) @@ -838,6 +929,7 @@ def finish( report=report, readback=readback, target_origin=self.app.origin, + environment=self.app.environment, run_dir=run_dir, key_dir=self.key_dir, ) @@ -949,6 +1041,9 @@ def refused(detail: str) -> JitResult: f"the canary ran on {canary_origin}, not on {self.app.origin}; " "admit a canary on this app" ) + bound = _reuse_refusal(record, workflow, self.task, self.app) + if bound is not None: + return refused(bound) problems = lifecycle._readback_problems( lifecycle.readback_coverage(workflow, self.verifier) ) diff --git a/packages/flow/openadapt_flow/jit/lifecycle.py b/packages/flow/openadapt_flow/jit/lifecycle.py index 882c4f0cfa..c8a2e0f43d 100644 --- a/packages/flow/openadapt_flow/jit/lifecycle.py +++ b/packages/flow/openadapt_flow/jit/lifecycle.py @@ -981,11 +981,16 @@ def record_canary( target_origin: str, run_dir: Optional[Path], key_dir: Optional[Path] = None, + environment: Optional[str] = None, ) -> dict[str, Any]: """Record an attended canary run. It passes only when it ended ``VERIFIED`` with zero model calls and every write step had an independent read-back. Every run is also appended to ``canaries.jsonl``, - which an admission policy's ``min_clean_canaries`` counts.""" + which an admission policy's ``min_clean_canaries`` counts. + + ``environment`` is the app environment the canary ran in (the app's + origin plus, when the caller names one, its environment id). The + admission pins it, and a run in another environment is refused.""" program = Path(program_dir) _require_draft(program) @@ -1005,6 +1010,9 @@ def record_canary( "readback": list(readback), "passed": outcome == "VERIFIED" and model_calls == 0 and not problems, } + if environment is not None: + # Only written when known, so canaries recorded before it verify. + record["environment"] = environment record["signature"] = _sign_record(record, key_dir) _write(program / CANARY_FILE, record) with (program / CANARIES_FILE).open("a") as f: @@ -1181,8 +1189,13 @@ def _write_manifest( "target_origin", "outcome", "model_calls", + "environment", ) + if k in canary }, + # What the person approved, apart from the anchors: a repair that + # re-anchors a step keeps it, a changed record check does not. + "record_check_digest": _record_check_digest(program), "readback": canary["readback"], "recorded_by": recorder.get("recorded_by"), "agent_principal_id": recorder.get("agent_principal_id"), @@ -1208,6 +1221,12 @@ def _write_manifest( return manifest +def _record_check_digest(program: Path) -> str: + from openadapt_flow.jit.signature import record_check_digest + + return record_check_digest(_load_workflow(program / "bundle")) + + def admit_with_key( program_dir: Path | str, *, diff --git a/packages/flow/openadapt_flow/jit/signature.py b/packages/flow/openadapt_flow/jit/signature.py index 1b4e481a36..e4550fc7c5 100644 --- a/packages/flow/openadapt_flow/jit/signature.py +++ b/packages/flow/openadapt_flow/jit/signature.py @@ -3,6 +3,12 @@ There is no fuzzy matching. Two tasks share a compiled program only when the caller gave them the same ``task_id`` and the same parameter names and types. A changed schema is a different task and starts with a new demonstration. + +Reuse is also bound to intent. A caller can pin the record check it approved +(:func:`record_check_digest`) and the app environment it runs in; both enter +the key. Whether or not the caller pins them, the admission manifest records +both, and a run refuses an admitted program whose record check or environment +differs from the one admitted (see :mod:`openadapt_flow.jit.compile_agent`). """ from __future__ import annotations @@ -11,9 +17,16 @@ import json import re from dataclasses import dataclass, field -from typing import Any, Mapping +from typing import Any, Mapping, Optional -__all__ = ["PARAM_TYPES", "ParamError", "TaskSignature", "param_text"] +__all__ = [ + "PARAM_TYPES", + "ParamError", + "TaskSignature", + "param_text", + "record_check_digest", + "record_check_of", +] #: Parameter types a task can declare, with the Python types they accept. PARAM_TYPES: dict[str, tuple[type, ...]] = { @@ -25,6 +38,7 @@ _PY_TO_PARAM = {str: "string", int: "integer", float: "number", bool: "boolean"} _TASK_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") _PARAM_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,127}$") +_DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") class ParamError(ValueError): @@ -60,14 +74,25 @@ class TaskSignature: placeholders are filled from the run's parameters. params: Parameter name to type (``"string"``, ``"integer"``, ``"number"``, ``"boolean"``, or the matching Python type). + record_check: Optional. The digest of the record check the caller + approved (:func:`record_check_digest` of the admitted program). + A program whose record check differs is never reused for it. + environment: Optional. The app environment the task is for, as the + app reports it (for example ``"https://emr.example.test#7.0.2"``, + see ``WebApp.environment``). A program admitted in another + environment is never reused for it. The instruction is not part of the key: rewording it does not throw away - an admitted program. The task id and the schema are. + an admitted program. The task id and the schema are, and so are + ``record_check`` and ``environment`` when given. Leaving them out keeps + the key a task had before they existed. """ task_id: str instruction: str params: Mapping[str, Any] = field(default_factory=dict) + record_check: Optional[str] = None + environment: Optional[str] = None def __post_init__(self) -> None: if not _TASK_ID_RE.match(self.task_id): @@ -82,6 +107,12 @@ def __post_init__(self) -> None: ) typed[name] = _type_name(kind) object.__setattr__(self, "params", dict(sorted(typed.items()))) + if self.record_check is not None and not _DIGEST_RE.match(self.record_check): + raise ValueError( + "record_check must be a sha256 hex digest (record_check_digest)" + ) + if self.environment is not None and not self.environment.strip(): + raise ValueError("environment must not be empty") @property def schema(self) -> dict[str, str]: @@ -89,13 +120,15 @@ def schema(self) -> dict[str, str]: @property def key(self) -> str: - """``-<12 hex>``: the task id and a digest of its schema.""" - - canonical = json.dumps( - {"task_id": self.task_id, "params": self.schema}, - sort_keys=True, - separators=(",", ":"), - ) + """``-<12 hex>``: the task id and a digest of its schema, + and of the pinned record check and environment when given.""" + + body: dict[str, Any] = {"task_id": self.task_id, "params": self.schema} + if self.record_check is not None: + body["record_check"] = self.record_check + if self.environment is not None: + body["environment"] = self.environment + canonical = json.dumps(body, sort_keys=True, separators=(",", ":")) digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:12] return f"{self.task_id}-{digest}" @@ -130,4 +163,72 @@ def render_instruction(self, values: Mapping[str, Any]) -> str: return self.instruction.format(**{k: param_text(v) for k, v in values.items()}) def to_dict(self) -> dict[str, Any]: - return {"task_id": self.task_id, "params": self.schema, "key": self.key} + out: dict[str, Any] = { + "task_id": self.task_id, + "params": self.schema, + "key": self.key, + } + if self.record_check is not None: + out["record_check"] = self.record_check + if self.environment is not None: + out["environment"] = self.environment + return out + + +def record_check_of(workflow: Any) -> dict[str, Any]: + """The program's record check: what each saving step must leave behind. + + For every step the run gate treats as a save (fail-closed), its effect + contracts and its record identity contract, in step order. Anchors, + templates and locators are left out, so a repair that re-anchors a step + or adds a bridge before it keeps the same record check, while any change + to what is saved, or to which record, changes it. + """ + + from openadapt_flow.run_gate import is_consequential + + saves = [] + for step in workflow.steps: + if not is_consequential(step, workflow): + continue + saves.append( + { + "step_id": step.id, + "intent": step.intent or "", + "effects": [e.model_dump(mode="json") for e in step.effects], + "identity": _identity_of(step), + } + ) + return {"workflow": getattr(workflow, "name", None), "saves": saves} + + +def _identity_of(step: Any) -> Optional[dict[str, Any]]: + """The save's record identity contract: which record it must act on.""" + + anchor = getattr(step, "anchor", None) + if anchor is None: + return None + out = {} + for name in ("identity_template", "structured_identity", "context_text"): + value = getattr(anchor, name, None) + if value is not None: + out[name] = ( + value.model_dump(mode="json") if hasattr(value, "model_dump") else value + ) + return out or None + + +def record_check_digest(workflow: Any) -> str: + """sha256 hex of :func:`record_check_of`, step ids and intents left out. + + The contracts are what a person approved. A step id or the intent's + wording can change in a repair without changing what is saved or where. + """ + + check = record_check_of(workflow) + body = [ + {k: v for k, v in save.items() if k not in ("step_id", "intent")} + for save in check["saves"] + ] + canonical = json.dumps(body, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() diff --git a/packages/flow/tests/test_jit_reuse_binding.py b/packages/flow/tests/test_jit_reuse_binding.py new file mode 100644 index 0000000000..216d0c38c4 --- /dev/null +++ b/packages/flow/tests/test_jit_reuse_binding.py @@ -0,0 +1,221 @@ +"""A cached program is reused only for the record check and environment it was +admitted under. + +The task id and the parameter schema used to be the whole key, so a task +whose record check changed, or an app that moved to a new version at the same +origin, silently reused the old program. Now the admission pins the record +check's digest and the canary's environment, a caller can pin both in the +key, and a run refuses a mismatch. + +No browser and no model. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +from pathlib import Path +from typing import Any + +import pytest + +from openadapt_flow.ir import ActionKind, Anchor, Step, Workflow +from openadapt_flow.jit import AdmissionError, WebApp +from openadapt_flow.jit.compile_agent import _reuse_refusal +from openadapt_flow.jit.signature import ( + TaskSignature, + record_check_digest, + record_check_of, +) +from openadapt_flow.runtime.effects.effect import Effect, EffectKind +from tests.test_jit_lifecycle import ( # noqa: F401 (autouse fixture) + REVIEWER, + _admitted, + _jit, + _policy_passes, + _report, + _review, + _stub_runs, +) + +URL = "http://127.0.0.1:9/" + + +def _workflow(*, anchor_x: int = 10, effect_field: str = "note") -> Workflow: + return Workflow( + name="t", + steps=[ + Step(id="step_000", intent="type the note", action=ActionKind.TYPE, text="x"), + Step( + id="step_001", + intent="click Save", + action=ActionKind.CLICK, + anchor=Anchor( + template=f"templates/step_001_{anchor_x}.png", + region=(anchor_x, 10, 40, 20), + click_point=(anchor_x + 5, 15), + ), + effects=[ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={"type": "note"}, + field=effect_field, + ) + ], + ), + ], + ) + + +# -- the key --------------------------------------------------------------------- + + +def test_the_key_is_unchanged_when_nothing_is_pinned() -> None: + task = TaskSignature("t.add_note", "Add a note", {"note": str}) + canonical = json.dumps( + {"task_id": "t.add_note", "params": {"note": "string"}}, + sort_keys=True, + separators=(",", ":"), + ) + old = "t.add_note-" + hashlib.sha256(canonical.encode()).hexdigest()[:12] + assert task.key == old + assert "record_check" not in task.to_dict() + + +def test_pinning_a_record_check_or_environment_changes_the_key() -> None: + base = TaskSignature("t.add_note", "Add a note", {"note": str}) + check = TaskSignature( + "t.add_note", "Add a note", {"note": str}, record_check="a" * 64 + ) + other_check = dataclasses.replace(check, record_check="b" * 64) + env = TaskSignature( + "t.add_note", "Add a note", {"note": str}, environment=URL + "#7.0.2" + ) + other_env = dataclasses.replace(env, environment=URL + "#7.0.3") + keys = {base.key, check.key, other_check.key, env.key, other_env.key} + assert len(keys) == 5 + assert check.to_dict()["record_check"] == "a" * 64 + + +@pytest.mark.parametrize( + "kw", [{"record_check": "not-a-digest"}, {"environment": " "}] +) +def test_a_malformed_pin_is_refused(kw: dict) -> None: + with pytest.raises(ValueError): + TaskSignature("t.add_note", "Add a note", {}, **kw) + + +# -- the record check digest ----------------------------------------------------- + + +def test_a_re_anchored_save_keeps_its_record_check() -> None: + assert record_check_digest(_workflow(anchor_x=10)) == record_check_digest( + _workflow(anchor_x=300) + ) + + +def test_a_changed_contract_changes_the_record_check() -> None: + assert record_check_digest(_workflow()) != record_check_digest( + _workflow(effect_field="summary") + ) + + +def test_the_record_check_names_only_saving_steps() -> None: + check = record_check_of(_workflow()) + assert [s["step_id"] for s in check["saves"]] == ["step_001"] + assert check["saves"][0]["effects"][0]["field"] == "note" + + +# -- admission pins both, runs refuse a mismatch -------------------------------------- + + +def test_the_admission_pins_the_record_check_and_the_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + jit = _admitted(tmp_path, monkeypatch) + manifest = json.loads((jit.bundle_dir / "admission.json").read_text()) + assert manifest["record_check_digest"] == record_check_digest( + Workflow.load(jit.bundle_dir) + ) + assert manifest["canary"]["environment"] == "http://127.0.0.1:9" + assert jit.run().status == "VERIFIED" + + +def test_a_new_app_version_at_the_same_origin_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + jit = _jit(tmp_path) + jit = dataclasses.replace(jit, app=WebApp(url=URL, environment_id="7.0.2")) + _review(jit) + _stub_runs(jit, monkeypatch, _report()) + jit.canary(principal_id=REVIEWER) + from tests.test_jit_lifecycle import _ci + + assert jit.admit_with_key(_ci(jit, monkeypatch))["outcome"] == "admitted" + assert jit.run().status == "VERIFIED" + + upgraded = dataclasses.replace(jit, app=WebApp(url=URL, environment_id="7.0.3")) + _stub_runs(upgraded, monkeypatch, _report()) + result = upgraded.run() + assert result.status == "refused" + assert "admitted in environment" in (result.detail or "") + assert "7.0.3" in (result.detail or "") + + +def test_an_environment_that_cannot_be_read_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + jit = _admitted(tmp_path, monkeypatch) + + def broken() -> str: + raise RuntimeError("version endpoint down") + + other = dataclasses.replace(jit, app=WebApp(url=URL, environment_id=broken)) + _stub_runs(other, monkeypatch, _report()) + result = other.run() + assert result.status == "refused" + assert "could not be read" in (result.detail or "") + + +def test_a_pinned_record_check_the_program_does_not_have_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + task = TaskSignature("t.add_note", "Add a note", {}, record_check="0" * 64) + jit = _admitted(tmp_path, monkeypatch, task=task) + result = jit.run() + assert result.status == "refused" + assert "pins record check" in (result.detail or "") + + +def test_the_pinned_record_check_of_the_admitted_program_runs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + probe = _jit(tmp_path / "probe") + digest = record_check_digest(Workflow.load(probe.bundle_dir)) + task = TaskSignature("t.add_note", "Add a note", {}, record_check=digest) + jit = _admitted(tmp_path, monkeypatch, task=task) + assert jit.run().status == "VERIFIED" + + +def test_a_canary_outside_the_pinned_environment_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + task = TaskSignature("t.add_note", "Add a note", {}, environment=URL + "#8") + jit = _jit(tmp_path, task=task) + _review(jit) + _stub_runs(jit, monkeypatch, _report()) + with pytest.raises(AdmissionError, match="pinned to environment"): + jit.canary(principal_id=REVIEWER) + + +def test_an_admitted_record_check_that_differs_is_refused() -> None: + workflow = _workflow() + app = WebApp(url=URL) + task = TaskSignature("t.add_note", "Add a note", {}) + record = {"record_check_digest": "f" * 64, "canary": {"environment": app.origin}} + assert "not the one admitted" in (_reuse_refusal(record, workflow, task, app) or "") + record["record_check_digest"] = record_check_digest(workflow) + assert _reuse_refusal(record, workflow, task, app) is None + # An admission from before the pins existed: only the caller's pins apply. + assert _reuse_refusal({"canary": {}}, workflow, task, app) is None From 64fbe45fde67b31e02c8ed36d87a2778c272ff1d Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 12:17:41 -0400 Subject: [PATCH 02/10] feat(flow): repair every stop that wrote nothing, by default When a repair agent is configured (deploy.yaml repair section or --repair-agent), run hands every HALTED_BEFORE_EFFECT stop to it: the agent gets the stop, the screen evidence and the record check (literals masked, identity as digests), proposes a re-anchored step, a bridge or a standard change, Flow probes it and runs the replay and fault cases, asks one person unless a signed policy covers it, and resumes. RECONCILIATION_REQUIRED stops never go to repair. --no-auto-repair or repair.auto: false turns it off; a run that can't be repaired runs plainly instead of being refused. Every stop appends one row to /auto-repair/ledger.jsonl, and auto_repair.summarize reports the funnel from stops to resumed items. WorklistQueue writes the same rows for its items and test-record repairs. Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/JIT.md | 20 + packages/flow/docs/REPAIR_LIFECYCLE.md | 15 +- packages/flow/docs/SELF_REPAIR.md | 101 ++++- packages/flow/openadapt_flow/__main__.py | 11 +- .../flow/openadapt_flow/cli_self_repair.py | 117 +++++- packages/flow/openadapt_flow/deployment.py | 33 ++ .../flow/openadapt_flow/jit/compile_agent.py | 6 - packages/flow/openadapt_flow/jit/signature.py | 54 ++- .../flow/openadapt_flow/repair/auto_repair.py | 377 ++++++++++++++++++ .../openadapt_flow/repair/preflight_queue.py | 66 ++- .../flow/openadapt_flow/repair/self_repair.py | 9 + packages/flow/tests/test_auto_repair.py | 189 +++++++++ packages/flow/tests/test_auto_repair_cli.py | 189 +++++++++ packages/flow/tests/test_jit_reuse_binding.py | 26 +- 14 files changed, 1156 insertions(+), 57 deletions(-) create mode 100644 packages/flow/openadapt_flow/repair/auto_repair.py create mode 100644 packages/flow/tests/test_auto_repair.py create mode 100644 packages/flow/tests/test_auto_repair_cli.py diff --git a/packages/flow/docs/JIT.md b/packages/flow/docs/JIT.md index 45bccc0374..b075d67779 100644 --- a/packages/flow/docs/JIT.md +++ b/packages/flow/docs/JIT.md @@ -99,6 +99,7 @@ same tunnel port share it, so use a separate cache directory per environment. | A program is admitted | Checks the signed admission, runs the program under the standard profile, and reads the save back | `VERIFIED`, or `HALTED` and others | | The admitted bundle or its manifest changed | Nothing | `refused` | | The policy, the app's origin, or the verifier differs from what was admitted | Nothing | `refused` | +| The record check or the app environment differs from what was admitted, or from what the task pins | Nothing | `refused` | | The replay called a model | Sends the run to the inbox | `needs_review` | A run that does not end `VERIFIED` is written to the exception inbox @@ -112,6 +113,25 @@ The cache key is the task id plus the parameter names and types. The instruction text is not part of it, and nothing is matched fuzzily. Changing the schema makes a new task, which starts with a new demonstration. +Reuse is also bound to what the program checks and where it runs: + +- **The record check.** The admission records a digest of the program's + record check: the effect contracts and the record identity contract of every + step that saves (`openadapt_flow.jit.signature.record_check_digest`). + Anchors are left out, so a repair that re-anchors a step keeps it. A run + whose program checks something else is refused. +- **The app environment.** The canary records the app's environment: its + origin, plus `#` when you pass `environment_id` to `WebApp` or + `SurfaceApp`. Pass the app's version or build (`"7.0.2"`), or a function + that reads it before each run. A program admitted under `7.0.2` is refused + under `7.0.3` at the same origin, and the inbox entry says why; run a + canary on the new version. A function that fails is a refusal too. +- **Pins in the key.** `TaskSignature(..., record_check=, + environment=)` puts both in the cache key and refuses a + program that doesn't match them. Without pins the key is the same as before, + so existing caches keep working. Admissions made before these checks carry + no record-check digest or environment; for them only your pins apply. + ## The agent The agent is any object with a `dialect` and a `run(request, computer)` diff --git a/packages/flow/docs/REPAIR_LIFECYCLE.md b/packages/flow/docs/REPAIR_LIFECYCLE.md index 3c7d58f301..d52ca34478 100644 --- a/packages/flow/docs/REPAIR_LIFECYCLE.md +++ b/packages/flow/docs/REPAIR_LIFECYCLE.md @@ -389,16 +389,23 @@ payer names. ## Repairs proposed during a run (self-repair) -`openadapt-flow run --self-repair` and -`openadapt_flow.repair.self_repair.SelfRepair` handle a run that halts before -any effect because a screen changed. A repair agent proposes a fix, and Flow +`openadapt-flow run` and `openadapt_flow.repair.self_repair.SelfRepair` +handle a run that halts before any effect because a screen changed. This is +the default once a deployment names a repair agent (the `repair` section of +`deploy.yaml`, or `--repair-agent`): every stop that wrote nothing goes to the +agent without anyone starting it, and `--no-auto-repair` turns that off +(`openadapt_flow.repair.auto_repair`). A stop after a possible write +(`RECONCILIATION_REQUIRED`) never enters this lifecycle; it goes to +reconciliation. A repair agent proposes a fix, and Flow probes it up to the save without saving. A passing probe becomes a candidate here: source `model_suggestion`, provenance `repaired_by: self_repair`. The campaigns run on the evidence frames, or on live probes when the step's identity is read from the DOM. One person's answer records the review and the approval. The approved bundle is staged and enters its canary window, and its first canary run is the resumed item. The agent can't review or approve the -candidate. See [SELF_REPAIR.md](SELF_REPAIR.md). +candidate. Each stop, and what became of it, is one row in +`/auto-repair/ledger.jsonl`; `auto_repair.summarize` reports the +funnel from stops to resumed items. See [SELF_REPAIR.md](SELF_REPAIR.md). ## Standard changes: repairs a person approved in advance diff --git a/packages/flow/docs/SELF_REPAIR.md b/packages/flow/docs/SELF_REPAIR.md index 90e2e3c8b8..8351ae244b 100644 --- a/packages/flow/docs/SELF_REPAIR.md +++ b/packages/flow/docs/SELF_REPAIR.md @@ -13,9 +13,58 @@ form and the record are the same as in the recording. Then one person accepts the fix with one tap, on the phone or at a terminal, and the run resumes. Healthy runs never call the agent and record zero model calls. -Module: `openadapt_flow.repair.self_repair`. CLI: `openadapt-flow run ---self-repair`. Tests: `tests/test_self_repair.py`, -`tests/test_self_repair_cli.py`, `tests/test_self_repair_bridge.py`. +Module: `openadapt_flow.repair.self_repair`. CLI: `openadapt-flow run`, +on by default once a repair agent is configured. Tests: +`tests/test_self_repair.py`, `tests/test_self_repair_cli.py`, +`tests/test_self_repair_bridge.py`, `tests/test_auto_repair.py`, +`tests/test_auto_repair_cli.py`. + +## Repair is the default + +Once a deployment names a repair agent, every stop that wrote nothing goes to +it. Nobody has to start a repair. Module: `openadapt_flow.repair.auto_repair`. + +```yaml +# deploy.yaml +repair: + agent: mypkg.agents:make_agent + screen_access_by: ops-lead + fault_urls: ["https://emr.test/?fault=missing"] +``` + +Each stop is routed by what the run may have written: + +| The run ended | Where the stop goes | +| --- | --- | +| `VERIFIED` | Nowhere. The agent is never called. | +| `HALTED_BEFORE_EFFECT` (or a halt with no outcome word and no effect) | To the repair agent, then the probe, the replay and fault cases, one approval (or a standard change), and the resumed item. A halt self-repair doesn't handle, such as a wrong-record halt, goes to a person. | +| `RECONCILIATION_REQUIRED` | Never to repair. A write may have happened, so it goes to reconciliation: the change ledger in `REPORT.md` and `openadapt-flow undo list ` ([AFTER_A_STOP.md](AFTER_A_STOP.md)). | +| Anything else | To a person. | + +The agent is shown the stop (the halted step, its intent and why it halted), +the screen evidence (the live screen and the recorded crop, scrubbed unless +set otherwise) and the record check (`request.record_check`: each save's +effect contracts with literal values masked, and a digest of its record +identity contract). It proposes a re-anchored step, a bridge of up to three +steps, or a fix a signed policy pre-approves. A fix never changes the record +check: the probe, the cases and the resumed run check it again. For a +worklist, `WorklistQueue` holds the items that stopped the same way, repairs +on the test record, and releases them all with one approval. + +Every stop, repaired or not, appends one row to +`/auto-repair/ledger.jsonl`: the route, the halted step, the +fixes proposed, whether one passed the probe and the cases, who approved it +and how long the answer took, whether the item resumed and verified, and the +agent's model calls and cost. `openadapt_flow.repair.auto_repair.summarize` +turns the rows into the repair success rate, failures included. + +To turn it off for one run, pass `--no-auto-repair`. To turn it off for a +deployment, set `repair.auto: false`. A run that can't be repaired (for +example one pinned to `--pin-digest`) is not refused: it runs plainly and +prints why repair is off for it. `--self-repair` still asks for repair +explicitly, and then such a run is refused. + +See [Measured](#measured) for how often it works. ## What happens when a run halts @@ -253,30 +302,34 @@ class MyAgent: ## CLI ```bash -openadapt-flow run bundles/triage --config deploy.yaml --url "$APP_URL" --run-dir runs/item-17 --self-repair --repair-agent mypkg.agents:make_agent --repair-screen-access-by ops-lead --repair-fault-url "$FAULT_URL" +openadapt-flow run bundles/triage --config deploy.yaml --url "$APP_URL" --run-dir runs/item-17 --repair-agent mypkg.agents:make_agent --repair-screen-access-by ops-lead --repair-fault-url "$FAULT_URL" ``` -`run` admits the bundle as usual. The first run uses the same authorization -and approval gate as without `--self-repair`. Probes and the resumed run +With the deployment's `repair` section set, the flags aren't needed. A flag +wins over the deployment file. `run` admits the bundle as usual. The first +run uses the same authorization and approval gate as a run without repair. Probes and the resumed run admit the patched bundle through the same run gate settings, with a fresh authorization each. They don't use the deployment's approval mode: the person's answer to the repair request is the approval for the item's one save, and the resume refuses the save unless the form is the one they saw. -| Flag | Default | Meaning | -| --- | --- | --- | -| `--self-repair` | off | Turn on self-repair for this run. | -| `--repair-agent MODULE:NAME` | required | An agent object, class, or factory. | -| `--repair-screen-access-by NAME` | required | Who allowed the agent to see screens. | -| `--repair-screen-share-mode` | `scrubbed_only` | Or `allow_unscrubbed`. | -| `--repair-model-provider`, `--repair-model-key-file` | none | Load the customer's own key. | -| `--repair-store DIR` | `repair-store` | The repair store. | -| `--repair-attempts N` | 3 | Proposals per halt. | -| `--repair-fault-url URL` | none | A fault probe; repeatable. | -| `--repair-approval-wait SECONDS` | 900 | How long to wait for the person. | - -The CLI refuses to start self-repair when the run is bound to the exact -original bundle: +| Flag | `deploy.yaml` `repair:` key | Default | Meaning | +| --- | --- | --- | --- | +| `--no-auto-repair` | `auto: false` | on | Leave stops for a person. | +| `--self-repair` | none | off | Require repair: refuse a run that can't be repaired. | +| `--repair-agent MODULE:NAME` | `agent` | none | An agent object, class, or factory. Repair is on when one is set. | +| `--repair-screen-access-by NAME` | `screen_access_by` | required | Who allowed the agent to see screens. | +| `--repair-screen-share-mode` | `screen_share_mode` | `scrubbed_only` | Or `allow_unscrubbed`. | +| `--repair-model-provider`, `--repair-model-key-file` | `model_provider`, `model_key_file` | none | Load the customer's own key. | +| `--repair-store DIR` | `store` | `repair-store` | The repair store, and the stop ledger. | +| `--repair-attempts N` | `attempts` | 3 | Proposals per halt. | +| `--repair-fault-url URL` | `fault_urls` | none | A fault probe; repeatable. | +| `--repair-approval-wait SECONDS` | `approval_wait_s` | 900 | How long to wait for the person. | + +Repair is off for a run bound to the exact original bundle, and for a +worklist run (use `WorklistQueue`), because a stop there may come after +earlier items saved. With `--self-repair` such a run is refused; otherwise it +runs plainly. The bindings are: - a signed qualification admission (`--qualification-authority-file`); - a managed dispatch; @@ -301,6 +354,7 @@ the tutorial does. | `/item-approval/` | The request, the screenshot the person saw, and the answer. | | `/self-repair/preflight/` | The test record's run of the repaired bundle, when a fix applied without a person. | | The repair store | The candidate, its campaigns, its transitions, who reviewed and approved it, and its canary metrics. | +| `/auto-repair/ledger.jsonl` | One row per stop: its route, what the agent proposed, the probe, the approval and its wait, the resumed outcome, model calls and cost. | The run's own `model_calls` stays the count for the deterministic run, which is 0. The agent's calls are counted in `self_repair.model_calls`. @@ -525,6 +579,13 @@ live RDP or Citrix session. ## Limits +- Only a stop that wrote nothing is repaired. A stop after a step the run gate + counts as a save has acted is `RECONCILIATION_REQUIRED` and goes to + reconciliation, even when the app saved nothing; see [Measured](#measured) + for OpenEMR, where every recorded stop was of this kind. +- The CLI repairs a single run. A worklist that stops after earlier items + saved is not repaired by `run`; use `WorklistQueue`, which repairs on the + test record and releases the waiting items. - One halted click per run is repaired, by one corrected click or a bridge of up to three actions, in a linear workflow with exactly one save. When the patched run halts at another step, the probe fails with that reason. diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 76d68058c6..cf104a8a8a 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -3634,7 +3634,10 @@ def _item_authorization(rows, base=base_authorization): # Admitted. A deployment run is not the drift-demo; force it off and delegate # to the shared replay executor (which reads all deployment wiring itself). args.drift = None - if getattr(args, "self_repair", False): + from openadapt_flow.cli_self_repair import resolve_auto_repair + + repair_mode = resolve_auto_repair(args, cfg) + if repair_mode != "off": from openadapt_flow.cli_self_repair import cmd_run_self_repair exit_code = cmd_run_self_repair( @@ -3673,9 +3676,11 @@ def _item_authorization(rows, base=base_authorization): finish=_finish_replay, default_run_dir=_default_run_dir, attest_pre_state=_maybe_attest_pre_state, + mode=repair_mode, ) - _attach_cloud_seal(args) - return exit_code + if exit_code is not None: + _attach_cloud_seal(args) + return exit_code exit_code = _cmd_replay(args) _attach_cloud_seal(args) return exit_code diff --git a/packages/flow/openadapt_flow/cli_self_repair.py b/packages/flow/openadapt_flow/cli_self_repair.py index eb0a5e0aa1..eb30d8428d 100644 --- a/packages/flow/openadapt_flow/cli_self_repair.py +++ b/packages/flow/openadapt_flow/cli_self_repair.py @@ -1,4 +1,12 @@ -"""``openadapt-flow run --self-repair``: the CLI over :mod:`openadapt_flow.repair.self_repair`. +"""``openadapt-flow run`` with repair on: the CLI over :mod:`openadapt_flow.repair.self_repair`. + +Repair is the default. When a repair agent is configured (``--repair-agent`` +or ``repair.agent`` in the deployment file), every run hands a stop that wrote +nothing to it (:mod:`openadapt_flow.repair.auto_repair`), and each stop goes in +the repair store's ``auto-repair/ledger.jsonl``. ``--no-auto-repair`` (or +``repair.auto: false``) turns it off. ``--self-repair`` asks for it explicitly: +then a run that can't be repaired is refused; in the automatic mode it runs +plainly and says why repair is off for it. ``_cmd_run`` admits the bundle as usual and then hands the run here instead of to the plain replay. The first run is the ordinary run: the same @@ -23,7 +31,12 @@ if TYPE_CHECKING: from openadapt_flow.repair.self_repair import AppSession -__all__ = ["add_self_repair_flags", "cmd_run_self_repair", "load_agent"] +__all__ = [ + "add_self_repair_flags", + "cmd_run_self_repair", + "load_agent", + "resolve_auto_repair", +] def add_self_repair_flags(p: argparse.ArgumentParser) -> None: @@ -33,12 +46,22 @@ def add_self_repair_flags(p: argparse.ArgumentParser) -> None: "self-repair", "When the run halts before any effect because a screen changed, a " "repair agent proposes a fix, Flow probes it up to the save, and one " - "person approves it before the run resumes. See docs/SELF_REPAIR.md.", + "person approves it before the run resumes. On by default whenever a " + "repair agent is configured (here or in the deployment file's repair " + "section). See docs/SELF_REPAIR.md.", ) group.add_argument( "--self-repair", action="store_true", - help="Turn on self-repair for this run (web runs against --url).", + help=( + "Require self-repair for this run (web runs against --url): refuse " + "the run when it can't be repaired." + ), + ) + group.add_argument( + "--no-auto-repair", + action="store_true", + help="Do not hand stops to the repair agent; a stop waits for a person.", ) group.add_argument( "--repair-agent", @@ -56,7 +79,7 @@ def add_self_repair_flags(p: argparse.ArgumentParser) -> None: group.add_argument( "--repair-screen-share-mode", choices=["scrubbed_only", "allow_unscrubbed"], - default="scrubbed_only", + default=None, help="Scrub screens before the agent sees them (default) or not.", ) group.add_argument( @@ -71,13 +94,13 @@ def add_self_repair_flags(p: argparse.ArgumentParser) -> None: ) group.add_argument( "--repair-store", - default="repair-store", + default=None, help="The repair store the candidate enters (default repair-store/).", ) group.add_argument( "--repair-attempts", type=int, - default=3, + default=None, help="Proposals tried for one halt (default 3).", ) group.add_argument( @@ -94,12 +117,52 @@ def add_self_repair_flags(p: argparse.ArgumentParser) -> None: group.add_argument( "--repair-approval-wait", type=float, - default=900.0, + default=None, metavar="SECONDS", help="How long to wait for a person's answer (default 900).", ) +def resolve_auto_repair(args: argparse.Namespace, cfg: Any) -> str: + """Fill the repair flags from the deployment and say how repair runs. + + Returns ``explicit`` (``--self-repair``), ``auto`` (an agent is + configured and nothing turned it off) or ``off``. A flag on the command + line wins over the deployment's ``repair`` section. + """ + + section = getattr(cfg, "repair", None) + fill = { + "repair_agent": "agent", + "repair_screen_access_by": "screen_access_by", + "repair_screen_share_mode": "screen_share_mode", + "repair_store": "store", + "repair_attempts": "attempts", + "repair_approval_wait": "approval_wait_s", + "repair_model_provider": "model_provider", + "repair_model_key_file": "model_key_file", + } + defaults = { + "repair_screen_share_mode": "scrubbed_only", + "repair_store": "repair-store", + "repair_attempts": 3, + "repair_approval_wait": 900.0, + } + for flag, name in fill.items(): + if getattr(args, flag, None) is None: + value = getattr(section, name, None) if section is not None else None + setattr(args, flag, value if value is not None else defaults.get(flag)) + if not getattr(args, "repair_fault_url", None) and section is not None: + args.repair_fault_url = list(section.fault_urls) + if getattr(args, "self_repair", False): + return "explicit" + if getattr(args, "no_auto_repair", False): + return "off" + if section is not None and not section.auto: + return "off" + return "auto" if args.repair_agent else "off" + + def load_agent(spec: str) -> Any: """Import ``module:name``; instantiate it when it is a class or factory.""" @@ -280,10 +343,17 @@ def cmd_run_self_repair( finish: Callable[..., int], default_run_dir: Callable[[], Path], attest_pre_state: Optional[Callable[..., None]] = None, -) -> int: - """Run the admitted bundle with self-repair on. Returns the exit code.""" + mode: str = "explicit", +) -> Optional[int]: + """Run the admitted bundle with self-repair on. Returns the exit code. + + In ``auto`` mode a run that can't be repaired is not refused: this + prints why repair is off for it and returns ``None`` before anything + ran, and the caller runs it plainly. + """ from openadapt_flow.cli_hints import command + from openadapt_flow.repair.auto_repair import AutoRepair from openadapt_flow.repair.self_repair import ( ScreenAccess, SelfRepair, @@ -292,7 +362,23 @@ def cmd_run_self_repair( url = args.url or cfg.backend.url refusal = _refusal(args, backend_cfg, url) + if refusal is None and ( + worklists + or getattr(args, "workers", None) + or getattr(args, "readback_window", None) + ): + refusal = ( + "a worklist run halts after earlier items saved; repair a worklist " + "with WorklistQueue (docs/SELF_REPAIR.md)" + ) if refusal is not None: + if mode == "auto": + print( + f"Auto-repair is off for this run: {refusal}. A stop waits for a " + "person.", + flush=True, + ) + return None print(f"run REFUSED: --self-repair: {refusal}. Nothing was executed.") return 2 model_key = None @@ -347,14 +433,21 @@ def announce(run_dir: Path, request_id: str, request_sha256: str) -> None: on_approval_request=announce, ) except (SelfRepairRefused, ImportError, AttributeError, ValueError) as exc: + if mode == "auto": + print( + f"Auto-repair is off for this run: {exc}. A stop waits for a person.", + flush=True, + ) + return None print(f"run REFUSED: --self-repair: {exc}. Nothing was executed.") return 2 run_dir = Path(args.run_dir) if args.run_dir else default_run_dir() if attest_pre_state is not None: # Same point as a plain run: admitted, nothing actuated yet. attest_pre_state(run_dir, args) + auto = AutoRepair(repair) try: - result = repair.run(bundle, run_dir) + result = auto.run(bundle, run_dir) except SelfRepairRefused as exc: print(f"Self-repair stopped: {exc}. Nothing was saved by self-repair.") return 1 @@ -375,4 +468,6 @@ def announce(run_dir: Path, request_id: str, request_sha256: str) -> None: f" candidate {result.candidate.candidate_id} [{result.candidate.state}] " f"in {args.repair_store}" ) + if evidence.status != "not_needed": + print(f" stop logged in {auto.ledger.path}") return finish(result.run_dir, result.report, args, backend_kind="web") diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index 62f098fbda..f451de6604 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -957,6 +957,34 @@ class ProbeCopySection(BaseModel): UNRESOLVED = "UNRESOLVED" +class RepairSection(BaseModel): + """Automatic repair of stops that wrote nothing (docs/SELF_REPAIR.md). + + When ``agent`` is set, every ``run`` that halts before any effect hands + the stop to this repair agent, probes its fix up to the save, asks one + person (or applies a pre-approved standard change), and resumes. A stop + after a write is never repaired. ``auto: false`` or ``--no-auto-repair`` + turns it off. Omitted when serialized at its defaults. + """ + + model_config = ConfigDict(extra="forbid") + + #: ``module:name`` of the repair agent (an object, class or factory). + agent: Optional[str] = None + #: The person who allowed the agent to see screens (recorded). + screen_access_by: Optional[str] = None + screen_share_mode: Literal["scrubbed_only", "allow_unscrubbed"] = "scrubbed_only" + #: Repair every stop that wrote nothing, without being asked. + auto: bool = True + store: str = "repair-store" + attempts: int = Field(default=3, ge=1, le=10) + #: URLs that present a fault the repaired run must refuse before the save. + fault_urls: list[str] = Field(default_factory=list) + approval_wait_s: float = Field(default=900.0, ge=0.0) + model_provider: Optional[Literal["anthropic", "openai"]] = None + model_key_file: Optional[str] = None + + class DeploymentConfig(BaseModel): """The whole-deployment configuration (one ``deployment.yaml``).""" @@ -987,6 +1015,9 @@ class DeploymentConfig(BaseModel): #: Declares the target a resettable test copy (``induce --probe-on`` #: only). None (and omitted when serialized) otherwise. test_copy: Optional[ProbeCopySection] = None + #: Automatic repair of stops that wrote nothing. Omitted when serialized + #: at its defaults (no agent). + repair: RepairSection = Field(default_factory=RepairSection) @model_validator(mode="after") def _desktop_readback_is_its_own_session(self) -> "DeploymentConfig": @@ -1016,6 +1047,8 @@ def _serialize_compatible(self, handler: Any) -> dict[str, Any]: data.pop("worklist", None) if self.test_copy is None: data.pop("test_copy", None) + if self.repair == RepairSection(): + data.pop("repair", None) return data diff --git a/packages/flow/openadapt_flow/jit/compile_agent.py b/packages/flow/openadapt_flow/jit/compile_agent.py index f8b48db082..e1447e8f00 100644 --- a/packages/flow/openadapt_flow/jit/compile_agent.py +++ b/packages/flow/openadapt_flow/jit/compile_agent.py @@ -199,12 +199,6 @@ def environment(self) -> str: def entry_url(self) -> Optional[str]: return self.url - @property - def environment(self) -> str: - """As for :attr:`WebApp.environment`.""" - - return _environment(self.origin, self.environment_id) - @property def backend_hints(self) -> Optional[dict[str, Any]]: return None diff --git a/packages/flow/openadapt_flow/jit/signature.py b/packages/flow/openadapt_flow/jit/signature.py index e4550fc7c5..fb35824efd 100644 --- a/packages/flow/openadapt_flow/jit/signature.py +++ b/packages/flow/openadapt_flow/jit/signature.py @@ -175,14 +175,19 @@ def to_dict(self) -> dict[str, Any]: return out -def record_check_of(workflow: Any) -> dict[str, Any]: +def record_check_of(workflow: Any, *, literals: bool = True) -> dict[str, Any]: """The program's record check: what each saving step must leave behind. For every step the run gate treats as a save (fail-closed), its effect - contracts and its record identity contract, in step order. Anchors, + contracts (which name parameters, never values) and a digest of its + record identity contract, in step order. Anchors, templates and locators are left out, so a repair that re-anchors a step or adds a bridge before it keeps the same record check, while any change to what is saved, or to which record, changes it. + + ``literals=False`` replaces every literal value in a contract with + ``""`` and keeps parameter references: the view a repair agent + gets, since a contract may pin a value from a real record. """ from openadapt_flow.run_gate import is_consequential @@ -195,15 +200,44 @@ def record_check_of(workflow: Any) -> dict[str, Any]: { "step_id": step.id, "intent": step.intent or "", - "effects": [e.model_dump(mode="json") for e in step.effects], + "effects": [ + _contract(e.model_dump(mode="json"), literals) for e in step.effects + ], "identity": _identity_of(step), } ) return {"workflow": getattr(workflow, "name", None), "saves": saves} -def _identity_of(step: Any) -> Optional[dict[str, Any]]: - """The save's record identity contract: which record it must act on.""" +_VALUE_KEYS = ("match", "value", "new_records") + + +def _masked(value: Any) -> Any: + if isinstance(value, dict): + if set(value) <= {"literal", "param"}: + # A ValueExpr: keep a parameter reference, mask a literal. + if value.get("param") is not None: + return {"param": value["param"]} + return "" + return {k: _masked(v) for k, v in value.items()} + if isinstance(value, list): + return [_masked(v) for v in value] + return None if value is None else "" + + +def _contract(effect: dict[str, Any], literals: bool) -> dict[str, Any]: + if literals: + return effect + return {k: (_masked(v) if k in _VALUE_KEYS else v) for k, v in effect.items()} + + +def _identity_of(step: Any) -> Optional[dict[str, str]]: + """The save's record identity contract, as a sha256 per field. + + Older bundles carry the recorded identity as plain text, so only digests + leave this function: a repair agent shown the record check never sees a + name or a record number. + """ anchor = getattr(step, "anchor", None) if anchor is None: @@ -211,10 +245,12 @@ def _identity_of(step: Any) -> Optional[dict[str, Any]]: out = {} for name in ("identity_template", "structured_identity", "context_text"): value = getattr(anchor, name, None) - if value is not None: - out[name] = ( - value.model_dump(mode="json") if hasattr(value, "model_dump") else value - ) + if value is None: + continue + if hasattr(value, "model_dump"): + value = value.model_dump(mode="json") + canonical = json.dumps(value, sort_keys=True, separators=(",", ":")) + out[name] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() return out or None diff --git a/packages/flow/openadapt_flow/repair/auto_repair.py b/packages/flow/openadapt_flow/repair/auto_repair.py new file mode 100644 index 0000000000..412c5c26ed --- /dev/null +++ b/packages/flow/openadapt_flow/repair/auto_repair.py @@ -0,0 +1,377 @@ +"""Repair is the default response to a stop. + +A compiled program stops rather than guess when the app changes. Before this +module, a stop waited for a person to start self-repair. Now, whenever a repair +agent is configured, every stop that wrote nothing goes to it automatically: + +1. **Route the stop.** :func:`route_stop` sorts the run's outcome: + + - ``HALTED_BEFORE_EFFECT`` (or an unprofiled halt with no effect): repair. + - ``RECONCILIATION_REQUIRED``: never repaired. A write may have happened, + so the stop goes to reconciliation (the change ledger and declared undo, + ``docs/AFTER_A_STOP.md``) and a person. + - anything else that did not verify: a person. + +2. **Repair.** :class:`~openadapt_flow.repair.self_repair.SelfRepair` hands + the stop, the screen evidence and the record check to the configured + agent; the agent proposes a re-anchored step, a bridge of up to three + steps, or a change a signed policy pre-approves; Flow probes the fix up to + the save, replays it against the replay and fault cases with the same + read-back, asks one person (phone or Desktop) unless a standard change + covers it, and resumes. A worklist resumes through + :class:`~openadapt_flow.repair.preflight_queue.WorklistQueue`. + +3. **Count it.** Every stop, repaired or not, appends one row to + ``/auto-repair/ledger.jsonl``: the route, the halted step, how many + fixes the agent proposed, whether one passed the probe, who approved it + and how long they took, whether the item resumed and verified, and the + agent's model calls and cost. :func:`summarize` turns the rows into the + repair success rate, failures included. + +The off-switch is ``--no-auto-repair`` (or ``repair.auto: false`` in the +deployment file). Healthy runs never call the agent and add no row. +""" + +from __future__ import annotations + +import json +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import TYPE_CHECKING, Any, Iterable, Literal, Mapping, Optional + +if TYPE_CHECKING: + from openadapt_flow.ir import RunReport + from openadapt_flow.repair.self_repair import SelfRepair, SelfRepairResult + +__all__ = [ + "AutoRepair", + "LEDGER_NAME", + "RepairLedger", + "StopRoute", + "StopRow", + "ledger_row", + "route_stop", + "summarize", +] + +StopRoute = Literal["none", "repair", "reconciliation", "person", "off"] +LEDGER_NAME = "ledger.jsonl" + +#: Where a stop that may have written goes instead of repair. +RECONCILE_HINT = ( + "reconcile: read the change ledger in REPORT.md, then " + "`openadapt-flow undo list {run_dir}`" +) + + +def route_stop(report: "RunReport") -> tuple[StopRoute, str]: + """Where a run's stop goes: ``(route, why)``. + + ``none`` for a run that verified. ``repair`` only for a halt that wrote + nothing; self-repair still checks the halt itself (a wrong-record halt, + for example, is never repaired). ``reconciliation`` for any run that may + have written. ``person`` for the rest. + """ + + if report.success: + return "none", "" + outcome = report.transaction_outcome + if outcome == "RECONCILIATION_REQUIRED": + return ( + "reconciliation", + "a write may have happened before the stop; it is never repaired " + "automatically", + ) + if outcome not in (None, "HALTED_BEFORE_EFFECT"): + return "person", f"the run ended {outcome}" + if report.halt is None: + return "person", "the run did not halt at a step" + return "repair", "" + + +@dataclass +class StopRow: + """One stop and what happened to it. No record values, only ids.""" + + at: str + workflow: Optional[str] + run_dir: str + outcome: Optional[str] + route: StopRoute + route_reason: str = "" + halted_step_id: Optional[str] = None + repair_status: Optional[str] = None + repair_reason: str = "" + #: Fixes the agent proposed (each attempt that returned a fix). + proposed: int = 0 + #: True when one proposal passed the probe up to the save. + probe_passed: bool = False + #: True when the replay and fault cases both passed. + cases_passed: bool = False + #: ``person``, ``standard_change`` or ``None``. + approved_via: Optional[str] = None + approved_by: Optional[str] = None + #: Person decisions this stop took (approve or decline), 0 or 1. + person_decisions: int = 0 + #: Seconds from the approval request to the person's answer. + person_wait_s: Optional[float] = None + resumed: bool = False + resumed_outcome: Optional[str] = None + model_calls: int = 0 + est_model_cost_usd: float = 0.0 + repair_s: float = 0.0 + #: ``item``: a stop of a real item (the stops counted). ``test_record``: + #: a repair a worklist ran on its test record for items waiting on it. + scope: Literal["item", "test_record"] = "item" + #: For a worklist item released by a repair: that repair's candidate. + released_by: Optional[str] = None + #: Free-form labels the caller adds (for example the drift mode measured). + labels: dict[str, str] = field(default_factory=dict) + + def to_json(self) -> str: + return json.dumps(asdict(self), sort_keys=True) + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _parse(ts: str) -> Optional[datetime]: + try: + return datetime.fromisoformat(ts.replace("Z", "+00:00")) + except (TypeError, ValueError): + return None + + +def _person_wait(run_dir: Path, request_id: Optional[str]) -> Optional[float]: + if not request_id: + return None + from openadapt_flow.approval_mode.item_approval import ItemApprovalStore + + try: + store = ItemApprovalStore(run_dir) + request = store.read_request(request_id) + answer = store.read_answer(request_id) + except Exception: # noqa: BLE001 - timing is bookkeeping, never a decision + return None + if answer is None: + return None + asked, answered = _parse(request.created_at), _parse(answer.answered_at) + if asked is None or answered is None: + return None + return round(max((answered - asked).total_seconds(), 0.0), 3) + + +def outcome_of(report: Any) -> Optional[str]: + """The run's outcome word. A run with no named profile records none; then + ``VERIFIED`` only when it succeeded and the record check confirmed every + effect it read back, else ``UNVERIFIED`` or ``HALTED``.""" + + word = getattr(report, "execution_outcome", None) + if word: + return str(word) + if not getattr(report, "success", False): + return "HALTED" + verdicts = [ + evidence.final_verdict + for result in getattr(report, "results", []) or [] + for evidence in result.effect_evidence + ] + if verdicts and all(v == "confirmed" for v in verdicts): + return "VERIFIED" + return "UNVERIFIED" + + +def ledger_row( + report: "RunReport", + run_dir: Path | str, + *, + result: Optional["SelfRepairResult"] = None, + route: Optional[StopRoute] = None, + route_reason: str = "", + repair_s: float = 0.0, + labels: Optional[Mapping[str, str]] = None, +) -> StopRow: + """The ledger row for one stop. + + ``report`` is the halted run's report; ``result`` the self-repair result, + when self-repair ran. + """ + + if route is None: + route, route_reason = route_stop(report) + halt = report.halt + row = StopRow( + at=_now(), + workflow=getattr(report, "workflow_name", None), + run_dir=str(run_dir), + outcome=report.transaction_outcome or outcome_of(report), + route=route, + route_reason=route_reason, + halted_step_id=halt.state_id if halt is not None else None, + repair_s=round(repair_s, 3), + labels=dict(labels or {}), + ) + if result is None: + return row + evidence = result.evidence + row.repair_status = evidence.status + row.repair_reason = evidence.reason + row.halted_step_id = evidence.halted_step_id or row.halted_step_id + row.proposed = sum(1 for s in evidence.steps if s.stage == "propose" and s.passed) + row.probe_passed = any(s.stage == "probe" and s.passed for s in evidence.steps) + row.cases_passed = any( + s.stage == "campaign" and s.passed for s in evidence.steps + ) and not any(s.stage == "campaign" and not s.passed for s in evidence.steps) + approvals = [s for s in evidence.steps if s.stage == "approval"] + if evidence.standard_change is not None: + row.approved_via = "standard_change" + row.approved_by = evidence.approved_by + elif evidence.approved_by: + row.approved_via = "person" + row.approved_by = evidence.approved_by + if evidence.status == "declined" or row.approved_via == "person": + row.person_decisions = 1 + row.person_wait_s = _person_wait(Path(run_dir), result.approval_request_id) + elif approvals and evidence.status == "awaiting_approval": + row.person_decisions = 0 + row.resumed = evidence.status == "resumed" + if row.resumed: + row.resumed_outcome = outcome_of(result.report) + row.model_calls = evidence.model_calls + row.est_model_cost_usd = evidence.est_model_cost_usd + return row + + +class RepairLedger: + """``/auto-repair/ledger.jsonl``: one row per stop, append-only.""" + + def __init__(self, store_root: Path | str) -> None: + self.path = Path(store_root) / "auto-repair" / LEDGER_NAME + + def append(self, row: StopRow) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a") as f: + f.write(row.to_json() + "\n") + + def rows(self) -> list[dict[str, Any]]: + if not self.path.is_file(): + return [] + return [ + json.loads(line) + for line in self.path.read_text().splitlines() + if line.strip() + ] + + +def summarize(rows: Iterable[Mapping[str, Any] | StopRow]) -> dict[str, Any]: + """The repair funnel over ledger rows, failures included. + + ``stops`` counts the stops of real items. ``repair_runs`` counts the times + self-repair ran (on the item, or on the test record for a worklist); the + funnel ``repairs_proposed`` >= ``probe_passed`` >= ``approved`` is over + those. ``resumed_items`` counts stopped items that resumed and verified, + and ``repair_success_rate`` is that over the stops routed to repair + (``None`` when none were). + """ + + data = [asdict(r) if isinstance(r, StopRow) else dict(r) for r in rows] + items = [r for r in data if r.get("scope", "item") == "item"] + to_repair = [r for r in items if r["route"] == "repair"] + runs = [r for r in data if r.get("repair_status") not in (None, "released")] + verified = [ + r + for r in to_repair + if r.get("resumed") and r.get("resumed_outcome") == "VERIFIED" + ] + waits = [r["person_wait_s"] for r in data if r.get("person_wait_s") is not None] + statuses: dict[str, int] = {} + for r in runs: + key = str(r.get("repair_status")) + statuses[key] = statuses.get(key, 0) + 1 + return { + "stops": len(items), + "to_repair": len(to_repair), + "to_reconciliation": sum(1 for r in items if r["route"] == "reconciliation"), + "to_person": sum(1 for r in items if r["route"] == "person"), + "auto_repair_off": sum(1 for r in items if r["route"] == "off"), + "repair_runs": len(runs), + "repairs_proposed": sum(1 for r in runs if r.get("proposed", 0) > 0), + "proposals": sum(int(r.get("proposed", 0)) for r in runs), + "probe_passed": sum(1 for r in runs if r.get("probe_passed")), + "approved": sum(1 for r in runs if r.get("approved_via")), + "approved_by_person": sum(1 for r in runs if r.get("approved_via") == "person"), + "approved_by_standard_change": sum( + 1 for r in runs if r.get("approved_via") == "standard_change" + ), + "resumed_items": len(verified), + "resumed_not_verified": sum( + 1 + for r in to_repair + if r.get("resumed") and r.get("resumed_outcome") != "VERIFIED" + ), + "repair_success_rate": ( + round(len(verified) / len(to_repair), 4) if to_repair else None + ), + "person_decisions": sum(int(r.get("person_decisions", 0)) for r in data), + "person_wait_s_total": round(sum(waits), 3), + "model_calls": sum(int(r.get("model_calls", 0)) for r in data), + "est_model_cost_usd": round( + sum(float(r.get("est_model_cost_usd", 0.0)) for r in data), 6 + ), + "repair_status_counts": dict(sorted(statuses.items())), + } + + +class AutoRepair: + """Run one item with repair as the default response to a stop. + + Args: + self_repair: The configured :class:`SelfRepair` (agent, target, store, + screen access, fault probes, optional standard changes). + ledger: Where each stop's row goes; defaults to the repair store's + ``auto-repair/ledger.jsonl``. + labels: Added to every row (for example ``{"drift": "rename"}``). + """ + + def __init__( + self, + self_repair: "SelfRepair", + *, + ledger: Optional[RepairLedger] = None, + labels: Optional[Mapping[str, str]] = None, + ) -> None: + self.self_repair = self_repair + self.ledger = ledger or RepairLedger(self_repair.store.root) + self.labels = dict(labels or {}) + + def run(self, bundle_dir: Path | str, run_dir: Path | str) -> "SelfRepairResult": + """Run the item; repair its stop when it wrote nothing; log the stop.""" + + started = time.monotonic() + result = self.self_repair.run(bundle_dir, run_dir) + halted = result.halted_report or result.report + if result.status == "not_needed": + return result + route, why = route_stop(halted) + if route == "repair" and result.status == "not_repairable": + # The halt wrote nothing but is not one self-repair handles (a + # wrong-record halt, a declined item): it goes to a person. + route, why = "person", result.evidence.reason + row = ledger_row( + halted, + Path(run_dir), + result=result, + route=route, + route_reason=why, + repair_s=time.monotonic() - started, + labels=self.labels, + ) + if route == "reconciliation": + row.route_reason = (f"{why}; " + RECONCILE_HINT.format(run_dir=run_dir))[ + :1024 + ] + self.ledger.append(row) + return result diff --git a/packages/flow/openadapt_flow/repair/preflight_queue.py b/packages/flow/openadapt_flow/repair/preflight_queue.py index 2c111596e4..e708f301ac 100644 --- a/packages/flow/openadapt_flow/repair/preflight_queue.py +++ b/packages/flow/openadapt_flow/repair/preflight_queue.py @@ -26,7 +26,7 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Callable, Literal, Optional, Sequence +from typing import TYPE_CHECKING, Callable, Literal, Mapping, Optional, Sequence from openadapt_flow.ir import RunReport, Workflow from openadapt_flow.repair.standard_change import ( @@ -37,6 +37,7 @@ ) if TYPE_CHECKING: + from openadapt_flow.repair.auto_repair import RepairLedger from openadapt_flow.repair.lifecycle import RepairStore from openadapt_flow.repair.self_repair import ( SelfRepair, @@ -73,6 +74,9 @@ class ItemResult: candidate_id: Optional[str] = None runs: int = 0 reason: str = "" + #: The item's first run that did not verify, kept for the stop ledger. + stopped_report: Optional[RunReport] = None + stopped_run_dir: Optional[Path] = None @dataclass @@ -124,6 +128,10 @@ class WorklistQueue: for a target (the test record's). Without it a halt stops the queue for a person, as today. max_repairs: Repairs per queue run before it stops for a person. + ledger: Where each stop and each repair goes + (:class:`~openadapt_flow.repair.auto_repair.RepairLedger`); by + default the repair store's, once a repair has run. + labels: Added to every ledger row. """ def __init__( @@ -134,12 +142,16 @@ def __init__( *, self_repair: Optional[Callable[["SelfRepairTarget"], "SelfRepair"]] = None, max_repairs: int = 3, + ledger: Optional["RepairLedger"] = None, + labels: Optional[Mapping[str, str]] = None, ) -> None: self.bundle_dir = Path(bundle_dir) self.test_record = test_record self.root = Path(root) self.self_repair = self_repair self.max_repairs = max(0, int(max_repairs)) + self.ledger = ledger + self.labels = dict(labels or {}) self._store: Optional["RepairStore"] = None def run(self, items: Sequence[QueueItem]) -> QueueResult: @@ -152,9 +164,11 @@ def run(self, items: Sequence[QueueItem]) -> QueueResult: def done( status: QueueStatus, reason: str, preflight: PreflightResult ) -> QueueResult: - return QueueResult( + out = QueueResult( status, reason, preflight, list(results.values()), repairs, bundle ) + self._log(out) + return out preflight = run_preflight(self.test_record, bundle, self.root / "preflight") candidate_id: Optional[str] = None @@ -216,6 +230,52 @@ def done( # -- pieces ------------------------------------------------------------ + def _log(self, out: QueueResult) -> None: + """One ledger row per repair (on the test record) and per item stop.""" + + from openadapt_flow.repair.auto_repair import ( + RepairLedger, + ledger_row, + outcome_of, + route_stop, + ) + + ledger = self.ledger + if ledger is None and self._store is not None: + ledger = RepairLedger(self._store.root) + if ledger is None: + return + for repair in out.repairs: + row = ledger_row( + repair.halted_report or repair.report, + repair.run_dir, + result=repair, + labels=self.labels, + ) + row.scope = "test_record" + ledger.append(row) + for item in out.items: + if item.stopped_report is None: + continue + route, why = route_stop(item.stopped_report) + if route == "repair" and item.halt_code is None and item.runs <= 1: + route, why = "person", "a halt self-repair does not handle" + if route == "repair" and self.self_repair is None: + route, why = "off", "no repair agent for this queue" + row = ledger_row( + item.stopped_report, + item.stopped_run_dir or self.root, + route=route, + route_reason=why, + labels=self.labels, + ) + if item.runs > 1: + row.repair_status = "released" + row.released_by = item.candidate_id + row.resumed = True + row.resumed_outcome = outcome_of(item.report) if item.report else None + ledger.append(row) + def _still_in_use( self, candidate_id: Optional[str] ) -> Optional[tuple[QueueStatus, str]]: @@ -261,6 +321,8 @@ def _run_item( if _verified(report): result.status, result.reason = "verified", "" else: + if result.stopped_report is None: + result.stopped_report, result.stopped_run_dir = report, run_dir key = _halt_key(report, workflow) result.status = "halted" if report.halt is not None else "failed" result.reason = (report.halt.reason if report.halt else "not verified")[ diff --git a/packages/flow/openadapt_flow/repair/self_repair.py b/packages/flow/openadapt_flow/repair/self_repair.py index 2f5a221b93..ca85e1d671 100644 --- a/packages/flow/openadapt_flow/repair/self_repair.py +++ b/packages/flow/openadapt_flow/repair/self_repair.py @@ -194,6 +194,11 @@ class RepairRequest: expected_png: Optional[bytes] = None #: The item's parameters a bridge may type, by name. Secrets are left out. param_values: Mapping[str, str] = field(default_factory=dict, repr=False) + #: The program's record check: each saving step's effect contracts, with + #: literal values masked, and a digest of its record identity contract + #: (:func:`openadapt_flow.jit.signature.record_check_of`). A fix never + #: changes it; the probe and the resume check it again. + record_check: Mapping[str, Any] = field(default_factory=dict, repr=False) @dataclass(frozen=True) @@ -826,6 +831,9 @@ def _propose_until_probe_passes( if expected_png is not None: expected_png = self._shown(expected_png)[0] assert step.anchor is not None # _repairable checked it + from openadapt_flow.jit.signature import record_check_of + + record_check = record_check_of(workflow, literals=False) refusals: list[str] = [] current: Optional[AppSession] = session try: @@ -863,6 +871,7 @@ def _propose_until_probe_passes( ), expected_png=expected_png if mode == "after" else None, param_values=_typable_params(workflow, self._params()), + record_check=record_check, ) evidence.attempts = attempt try: diff --git a/packages/flow/tests/test_auto_repair.py b/packages/flow/tests/test_auto_repair.py new file mode 100644 index 0000000000..6097e4a277 --- /dev/null +++ b/packages/flow/tests/test_auto_repair.py @@ -0,0 +1,189 @@ +"""Repair as the default response to a stop: routing, the ledger, the funnel. + +The CLI end of it (on by default, ``--no-auto-repair``, the fallback when a +run can't be repaired) is in ``tests/test_auto_repair_cli.py``. No browser. +""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from openadapt_flow.ir import SelfRepairEvidence, SelfRepairStepRecord +from openadapt_flow.repair.auto_repair import ( + AutoRepair, + RepairLedger, + StopRow, + ledger_row, + route_stop, + summarize, +) + + +def _report( + *, + success: bool = False, + outcome: str | None = "HALTED_BEFORE_EFFECT", + halted_at: str | None = "step_005", +) -> Any: + return SimpleNamespace( + success=success, + transaction_outcome=outcome, + execution_outcome="VERIFIED" if success else "HALTED", + halt=SimpleNamespace(state_id=halted_at) if halted_at else None, + workflow_name="triage", + ) + + +@pytest.mark.parametrize( + ("report", "route"), + [ + (_report(success=True, outcome=None, halted_at=None), "none"), + (_report(), "repair"), + (_report(outcome=None), "repair"), + (_report(outcome="RECONCILIATION_REQUIRED"), "reconciliation"), + (_report(outcome="FAILED_AFTER_EFFECT"), "person"), + (_report(halted_at=None), "person"), + ], +) +def test_only_a_stop_that_wrote_nothing_goes_to_repair(report: Any, route: str) -> None: + assert route_stop(report)[0] == route + + +def _evidence(status: str, *, probe: bool, approved_by: str | None = None) -> Any: + steps = [ + SelfRepairStepRecord(stage="halt", passed=True, detail="x"), + SelfRepairStepRecord(stage="propose", passed=True, detail="x", attempt=1), + SelfRepairStepRecord(stage="probe", passed=probe, detail="x", attempt=1), + ] + if probe: + steps.append(SelfRepairStepRecord(stage="campaign", passed=True, detail="x")) + return SelfRepairEvidence( + status=status, # type: ignore[arg-type] + halted_step_id="step_005", + attempts=1, + model_calls=1, + est_model_cost_usd=0.01, + approved_by=approved_by, + steps=steps, + ) + + +def _result(status: str, evidence: Any, outcome: str = "VERIFIED") -> Any: + return SimpleNamespace( + status=status, + evidence=evidence, + report=SimpleNamespace(execution_outcome=outcome), + halted_report=_report(), + approval_request_id=None, + ) + + +def test_a_resumed_repair_fills_the_whole_funnel(tmp_path: Path) -> None: + result = _result( + "resumed", + _evidence("resumed", probe=True, approved_by="decision_portal:person"), + ) + row = ledger_row(_report(), tmp_path, result=result) + assert row.route == "repair" + assert row.proposed == 1 and row.probe_passed and row.cases_passed + assert row.approved_via == "person" and row.person_decisions == 1 + assert row.resumed and row.resumed_outcome == "VERIFIED" + summary = summarize([row]) + assert summary["stops"] == 1 and summary["resumed_items"] == 1 + assert summary["repair_success_rate"] == 1.0 + + +def test_failures_count_against_the_rate(tmp_path: Path) -> None: + rows = [ + ledger_row( + _report(), + tmp_path, + result=_result( + "resumed", + _evidence("resumed", probe=True, approved_by="decision_portal:person"), + ), + ), + ledger_row( + _report(), + tmp_path, + result=_result("no_fix", _evidence("no_fix", probe=False)), + ), + ledger_row(_report(outcome="RECONCILIATION_REQUIRED"), tmp_path), + ] + summary = summarize(rows) + assert summary["stops"] == 3 + assert summary["to_repair"] == 2 + assert summary["to_reconciliation"] == 1 + assert summary["repairs_proposed"] == 2 + assert summary["probe_passed"] == 1 + assert summary["approved"] == 1 + assert summary["resumed_items"] == 1 + assert summary["repair_success_rate"] == 0.5 + assert summary["repair_status_counts"] == {"no_fix": 1, "resumed": 1} + + +def test_worklist_rows_count_items_as_stops_and_test_record_runs_as_repairs( + tmp_path: Path, +) -> None: + repair = ledger_row( + _report(), + tmp_path, + result=_result( + "resumed", + _evidence("resumed", probe=True, approved_by="decision_portal:person"), + ), + ) + repair.scope = "test_record" + items = [] + for _ in range(3): + row = ledger_row(_report(), tmp_path) + row.repair_status, row.resumed, row.resumed_outcome = ( + "released", + True, + "VERIFIED", + ) + items.append(row) + summary = summarize([repair, *items]) + assert summary["stops"] == 3 + assert summary["repair_runs"] == 1 + assert summary["approved_by_person"] == 1 + assert summary["person_decisions"] == 1 + assert summary["resumed_items"] == 3 + assert summary["repair_success_rate"] == 1.0 + + +class _FakeRepair: + def __init__(self, root: Path, result: Any) -> None: + self.store = SimpleNamespace(root=root) + self._result = result + + def run(self, bundle_dir: Any, run_dir: Any) -> Any: + return self._result + + +def test_auto_repair_logs_each_stop_and_skips_healthy_runs(tmp_path: Path) -> None: + healthy = _result("not_needed", SelfRepairEvidence(status="not_needed")) + AutoRepair(_FakeRepair(tmp_path, healthy)).run("b", tmp_path / "r0") + assert RepairLedger(tmp_path).rows() == [] + + wrong_record = _result( + "not_repairable", + SelfRepairEvidence( + status="not_repairable", reason="a wrong-record halt is never repaired" + ), + ) + AutoRepair(_FakeRepair(tmp_path, wrong_record), labels={"drift": "x"}).run( + "b", tmp_path / "r1" + ) + wrote = _result("not_repairable", SelfRepairEvidence(status="not_repairable")) + wrote.halted_report = _report(outcome="RECONCILIATION_REQUIRED") + AutoRepair(_FakeRepair(tmp_path, wrote)).run("b", tmp_path / "r2") + rows = RepairLedger(tmp_path).rows() + assert [r["route"] for r in rows] == ["person", "reconciliation"] + assert rows[0]["labels"] == {"drift": "x"} + assert "undo list" in rows[1]["route_reason"] + assert StopRow(**rows[0]).route == "person" diff --git a/packages/flow/tests/test_auto_repair_cli.py b/packages/flow/tests/test_auto_repair_cli.py new file mode 100644 index 0000000000..59a4b1167d --- /dev/null +++ b/packages/flow/tests/test_auto_repair_cli.py @@ -0,0 +1,189 @@ +"""``openadapt-flow run`` repairs a stop by default once a repair agent is set. + +The deployment file names the repair agent; no ``--self-repair`` flag. The +redesigned save control halts the run before any effect, the agent proposes +the new control, one phone tap approves it, and the run resumes and verifies. +``--no-auto-repair`` leaves the same stop for a person, and a run that can't +be repaired runs plainly instead of being refused. Live MockMed, no model. +""" + +from __future__ import annotations + +import threading +from pathlib import Path + +import pytest + +from openadapt_flow.__main__ import main +from openadapt_flow.ir import RunReport +from openadapt_flow.repair.auto_repair import RepairLedger, summarize +from tests import test_self_repair_cli as cli +from tests.test_self_repair import ( # noqa: F401 - the shared MockMed fixture + QUERY, + _browser, + _center_of, + _records, + _reset, + mockmed, +) + +pytest.importorskip("playwright.sync_api") + + +def _config(tmp_path: Path, base_url: str, *, auto: bool = True) -> Path: + store = tmp_path / "repair-store" + config = tmp_path / "deploy.yaml" + config.write_text( + f"effects:\n kind: rest\n base_url: {base_url}\n" + "policy:\n policy: clinical-write\n" + "repair:\n" + " agent: tests.test_self_repair_cli:CliAgent\n" + " screen_access_by: ops-lead\n" + " screen_share_mode: allow_unscrubbed\n" + f" store: {store}\n" + f" auto: {'true' if auto else 'false'}\n" + " approval_wait_s: 120\n" + " fault_urls:\n" + f" - {base_url}/{QUERY}&drift=redesign,missing#tasks\n" + ) + return config + + +def _run(bundle: Path, config: Path, url: str, run_dir: Path, *extra: str) -> int: + """``main`` on a worker thread, for a run that ends on the plain replay + path: its Playwright sync API refuses the test thread, where the fixture + left an event loop set. (The self-repair path runs on the test thread.)""" + + out: list = [] + + def target() -> None: + try: + out.append(_main(bundle, config, url, run_dir, *extra)) + except BaseException as exc: # noqa: BLE001 - re-raised below + out.append(exc) + + thread = threading.Thread(target=target) + thread.start() + thread.join(timeout=600) + if isinstance(out[0], BaseException): + raise out[0] + return out[0] + + +def _main(bundle: Path, config: Path, url: str, run_dir: Path, *extra: str) -> int: + return main( + [ + "run", + str(bundle), + "--config", + str(config), + "--backend", + "web", + "--url", + url, + "--run-dir", + str(run_dir), + "--param", + "note=Synthetic follow-up in two weeks", + "--approval-mode", + "not_gated", + "--allow-unencrypted", + *extra, + ] + ) + + +def test_a_configured_agent_repairs_a_stop_without_being_asked( + mockmed, # noqa: F811 - the shared fixture + tmp_path, + capsys, +) -> None: + base_url, bundle = mockmed + _reset(base_url) + with _browser(): + cli.TARGET = _center_of(base_url, "#submit-encounter") + run_dir = tmp_path / "run" + stop = threading.Event() + answered: list[str] = [] + phone = threading.Thread( + target=cli._answer_on_phone, args=(run_dir, stop, answered), daemon=True + ) + phone.start() + try: + _main( + bundle, + _config(tmp_path, base_url), + f"{base_url}/{QUERY}&drift=redesign#tasks", + run_dir, + ) + finally: + stop.set() + phone.join(timeout=5) + cli.TARGET = None + out = capsys.readouterr().out + assert answered, out + assert "Self-repair: resumed" in out + resumed = RunReport.model_validate_json( + (run_dir / "self-repair" / "resume" / "report.json").read_text() + ) + assert resumed.execution_outcome == "VERIFIED" or resumed.success + assert len(_records(base_url)) == 1 + rows = RepairLedger(tmp_path / "repair-store").rows() + assert len(rows) == 1 + assert rows[0]["route"] == "repair" + assert rows[0]["approved_via"] == "person" + assert rows[0]["person_decisions"] == 1 + assert rows[0]["person_wait_s"] is not None + summary = summarize(rows) + assert summary["resumed_items"] == 1 and summary["repair_success_rate"] == 1.0 + + +def test_no_auto_repair_leaves_the_stop_for_a_person( + mockmed, # noqa: F811 - the shared fixture + tmp_path, + capsys, +) -> None: + base_url, bundle = mockmed + _reset(base_url) + run_dir = tmp_path / "run" + code = _run( + bundle, + _config(tmp_path, base_url), + f"{base_url}/{QUERY}&drift=redesign#tasks", + run_dir, + "--no-auto-repair", + ) + out = capsys.readouterr().out + assert code != 0 + assert "Self-repair" not in out + halted = RunReport.model_validate_json((run_dir / "report.json").read_text()) + assert halted.halt is not None and halted.self_repair is None + assert not (run_dir / "self-repair").exists() + assert _records(base_url) == [] + assert RepairLedger(tmp_path / "repair-store").rows() == [] + + +def test_a_run_that_cannot_be_repaired_runs_plainly( + mockmed, # noqa: F811 - the shared fixture + tmp_path, + capsys, +) -> None: + from openadapt_flow.ir import Workflow + + base_url, bundle = mockmed + _reset(base_url) + digest = Workflow.load(bundle).manifest.content_digest + run_dir = tmp_path / "run" + code = _run( + bundle, + _config(tmp_path, base_url), + f"{base_url}/{QUERY}#tasks", + run_dir, + "--pin-digest", + digest, + ) + out = capsys.readouterr().out + assert "Auto-repair is off for this run: --pin-digest" in out + assert code == 0, out + assert (run_dir / "report.json").is_file() + assert len(_records(base_url)) == 1 diff --git a/packages/flow/tests/test_jit_reuse_binding.py b/packages/flow/tests/test_jit_reuse_binding.py index 216d0c38c4..77024062b5 100644 --- a/packages/flow/tests/test_jit_reuse_binding.py +++ b/packages/flow/tests/test_jit_reuse_binding.py @@ -16,7 +16,6 @@ import hashlib import json from pathlib import Path -from typing import Any import pytest @@ -46,7 +45,9 @@ def _workflow(*, anchor_x: int = 10, effect_field: str = "note") -> Workflow: return Workflow( name="t", steps=[ - Step(id="step_000", intent="type the note", action=ActionKind.TYPE, text="x"), + Step( + id="step_000", intent="type the note", action=ActionKind.TYPE, text="x" + ), Step( id="step_001", intent="click Save", @@ -219,3 +220,24 @@ def test_an_admitted_record_check_that_differs_is_refused() -> None: assert _reuse_refusal(record, workflow, task, app) is None # An admission from before the pins existed: only the caller's pins apply. assert _reuse_refusal({"canary": {}}, workflow, task, app) is None + + +def test_the_agent_view_masks_literals_and_identity() -> None: + workflow = _workflow() + workflow.steps[1].effects[0].match = {"type": "note", "mrn": {"param": "mrn"}} + workflow.steps[1].anchor.context_text = "Synthetic Patient 1970-01-01" + view = record_check_of(workflow, literals=False) + effect = view["saves"][0]["effects"][0] + assert effect["match"] == {"type": "", "mrn": {"param": "mrn"}} + assert "Synthetic" not in json.dumps(view) + assert len(view["saves"][0]["identity"]["context_text"]) == 64 + + +def test_a_native_app_names_its_environment_the_same_way() -> None: + from openadapt_flow.jit import SurfaceApp + + app = SurfaceApp( + backend={"kind": "macos", "macos_app": "TextEdit"}, environment_id="1.19" + ) + assert app.environment == app.origin + "#1.19" + assert WebApp(url=URL).environment == "http://127.0.0.1:9" From 3718a32ccacd4a915c4371276421f0fade9e15ce Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 12:33:49 -0400 Subject: [PATCH 03/10] bench(flow): measure repair as the default response to a stop MockMed, all 22 drift modes, 3 trials, a real Claude repair agent and a scripted approver: 30 stops, 21 reached repair, 6 resumed and verified (relink and redesign 3 of 3 each), 0 wrong saves in 66 runs. Not repaired: a save moved into a closed menu, a new interstitial, a new required choice, and font/zoom false stops of a pixel postcondition. The worklist run (vendor change after the pre-flight): 10 stopped items, 2 repairs on the test record, 2 person decisions, 10 of 10 resumed and verified. Routed offline: the 8 recorded OpenEMR stops behind #1347 and the 3 openIMIS relabel stops of #1331 (before #1337) were all RECONCILIATION_REQUIRED, so none reached repair. Docs: SELF_REPAIR (repair is the default, Measured), REPAIR_LIFECYCLE, LIMITS, JIT (reuse binding), CAPABILITIES; llms-full.txt and the public artifact inventory regenerated with their scripts. Co-Authored-By: Claude Opus 5.5 --- CAPABILITIES.md | 2 + llms-full.txt | 96 ++- packages/flow/benchmark/auto_repair/README.md | 111 +++ .../benchmark/auto_repair/measure_mockmed.py | 545 ++++++++++++ .../auto_repair/measure_mockmed_worklist.py | 264 ++++++ .../results/mockmed-2026-10-19-a/calls.json | 677 +++++++++++++++ .../results/mockmed-2026-10-19-a/items.csv | 67 ++ .../results/mockmed-2026-10-19-a/ledger.jsonl | 30 + .../results/mockmed-2026-10-19-a/meta.json | 39 + .../results/mockmed-2026-10-19-a/summary.json | 797 ++++++++++++++++++ .../mockmed-worklist-2026-10-19-a/calls.json | 52 ++ .../mockmed-worklist-2026-10-19-a/items.csv | 11 + .../ledger.jsonl | 12 + .../mockmed-worklist-2026-10-19-a/meta.json | 15 + .../summary.json | 76 ++ .../results/openemr-1347/ledger.jsonl | 8 + .../results/openemr-1347/stops.csv | 9 + .../results/openemr-1347/summary.json | 28 + .../results/openimis-1331-before/ledger.jsonl | 3 + .../results/openimis-1331-before/stops.csv | 4 + .../results/openimis-1331-before/summary.json | 28 + .../auto_repair/route_recorded_stops.py | 136 +++ packages/flow/docs/LIMITS.md | 14 +- packages/flow/docs/SELF_REPAIR.md | 43 + .../flow/openadapt_flow/repair/auto_repair.py | 9 +- packages/flow/public-artifacts.json | 64 ++ packages/flow/tests/test_standard_change.py | 12 + 27 files changed, 3130 insertions(+), 22 deletions(-) create mode 100644 packages/flow/benchmark/auto_repair/README.md create mode 100644 packages/flow/benchmark/auto_repair/measure_mockmed.py create mode 100644 packages/flow/benchmark/auto_repair/measure_mockmed_worklist.py create mode 100644 packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/calls.json create mode 100644 packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/items.csv create mode 100644 packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/ledger.jsonl create mode 100644 packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/meta.json create mode 100644 packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/summary.json create mode 100644 packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/calls.json create mode 100644 packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/items.csv create mode 100644 packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/ledger.jsonl create mode 100644 packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/meta.json create mode 100644 packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/summary.json create mode 100644 packages/flow/benchmark/auto_repair/results/openemr-1347/ledger.jsonl create mode 100644 packages/flow/benchmark/auto_repair/results/openemr-1347/stops.csv create mode 100644 packages/flow/benchmark/auto_repair/results/openemr-1347/summary.json create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-1331-before/ledger.jsonl create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-1331-before/stops.csv create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-1331-before/summary.json create mode 100644 packages/flow/benchmark/auto_repair/route_recorded_stops.py diff --git a/CAPABILITIES.md b/CAPABILITIES.md index 8d9c26c249..4e998044d8 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -198,6 +198,7 @@ improvising or retrying when something is off. | Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | In review ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts) | `run --self-repair`, [SELF_REPAIR.md](packages/flow/docs/SELF_REPAIR.md) | | Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](packages/flow/benchmark/pixel_scale/README.md) | | Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | +| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#PR_NUM](https://github.com/OpenAdaptAI/OpenAdapt/pull/PR_NUM)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR and openIMIS stops | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md) | | Fleet drift patches: a verified fix with no patient data, shared across sites under policy | Released, local reference hub | Simulator | `openadapt-flow fleet`, [FLEET_DRIFT_PATCHES.md](packages/flow/docs/FLEET_DRIFT_PATCHES.md) | ## Computer-use agents @@ -205,6 +206,7 @@ improvising or retrying when something is off. | Capability | Status | Evidence | Where | | --- | --- | --- | --- | | Compile a computer-use agent's successful run into a program (JIT); later runs make no model calls | Released | Synthetic (a recorded Claude run on MockMed) | `openadapt_flow.jit`, [JIT.md](packages/flow/docs/JIT.md) | +| Reuse a compiled program only for the record check and app environment it was admitted under (a changed record check or a new app version at the same origin is refused) | In review ([#PR_NUM](https://github.com/OpenAdaptAI/OpenAdapt/pull/PR_NUM)) | Unit tests | `TaskSignature(record_check=, environment=)`, `WebApp(environment_id=)`, [JIT.md](packages/flow/docs/JIT.md) | | Code as action: the run-1 agent acts through one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program | In review ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | | Admit a draft by a person, an admitter key, or an agent under a person-signed policy, including self-admission when the policy allows it | Released | Unit tests | [JIT_ADMISSION.md](packages/flow/docs/JIT_ADMISSION.md) | | MCP server: your agent decides what to enter, OpenAdapt makes the entry and checks it | Released | Synthetic sandbox | `openadapt-agent`, [packages/agent](packages/agent/README.md) | diff --git a/llms-full.txt b/llms-full.txt index 623dfd5b6b..8cceebe92b 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -375,7 +375,8 @@ shipping in the next release; a pull request number means it is being merged. 1. **Learns from any successful run, a person's or an agent's.** A person records a task, or a computer-use agent does it once and OpenAdapt compiles the verified run into a program (`openadapt_flow.jit`, released; - prompt-to-program `compile-agent`, landing). Agents can drive the recording + prompt-to-program `compile-agent`, landing; the agent can act through code + instead of screenshots, [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)). Agents can drive the recording on web, native Windows, RDP and Citrix. 2. **Compiles to the fastest layer the app allows.** Page structure first (DOM, Windows UI Automation, macOS Accessibility), direct API calls where the app's @@ -414,7 +415,7 @@ shipping in the next release; a pull request number means it is being merged. review, campaign, canary, rollback), agent-demonstrated fixes for a halted run, and fleet drift patches shared with no patient data are released. Self-repair, where an agent proposes a fix and Flow probes it up to the save - without saving, is landing. + without saving, is on main; fixes of up to three new steps are in review. 9. **Is a safety layer underneath any agent.** An agent's MCP calls, a guarded save around any agent's write ([#1258](https://github.com/OpenAdaptAI/OpenAdapt/pull/1258)), `verify-run` @@ -435,10 +436,12 @@ shipping in the next release; a pull request number means it is being merged. | Uses page structure first on web, Windows, macOS and Linux, with pixels as the fallback for remote desktops. Benchmarks labelled "vision-only" ran a deliberately pixel-only arm. | [RESOLUTION_LADDER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/RESOLUTION_LADDER.md) | | Runs steps as direct API calls; compiling them from captured network traffic is being merged. | `runtime/actuators/api.py`, [#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269) | | Repairs itself when the app changes: governed repair, agent-demonstrated fixes, fleet patches, and self-repair probed up to the save. | [REPAIR_LIFECYCLE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/REPAIR_LIFECYCLE.md) | +| Repairs a change that needs new steps, such as a new required dialog or a control moved into a menu: up to three new steps, each with its own record check, probed up to the save before one person approves. | [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md), [#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325) | | Compiles a computer-use agent's verified run, with no human demonstration needed. | [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | +| The agent that does a task once can act through code, not a screenshot-and-click loop: it reads the page or accessibility tree, acts by role and name, keeps notes with a sandboxed read-only shell, and asks for a screenshot only when it needs one. It is arm 7 of the scoreboard; its figures are quoted from there once its rows are in. | `CODE_AS_ACTION.md` in [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305) | | Binds each approval to the exact screenshot and form state, and re-reads the form before saving. | [APPROVAL_MODES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/APPROVAL_MODES.md) | | Fills inputs from a referral PDF, fax image or email, and holds back anything unclear for a person. | `intake`, [INTAKE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/INTAKE.md) | -| Ships as one monorepo with separately installable packages (flow, agent, types, capture, and privacy and desktop moving in). | [README](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/README.md) | +| Ships as one monorepo with separately installable packages (flow, agent, types, capture, privacy and desktop), each released from the monorepo. | [README](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/README.md) | | Makes no model calls on healthy runs. Model training (openadapt-ml) is separate research. | [README](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/README.md) | ## The full map @@ -474,6 +477,7 @@ improvising or retrying when something is off. | Compile one or more demonstrations into a deterministic program | Released | Real app (OpenEMR referral, 62 timed runs) | `openadapt-flow record`, `compile`; [packages/flow](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/README.md) | | Right-record check before any write (structure, then OCR, then pixels) | Released | Real app and synthetic | [RESOLUTION_LADDER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/RESOLUTION_LADDER.md), `runtime/identity.py` | | Read the saved record back from the system of record (API, database, FHIR, separate session) | Released | Real app (OpenEMR 12 of 12 fields) | [VERIFICATION.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/VERIFICATION.md), `runtime/effects/` | +| Read values out of an app as a workflow step (READ), with a records format, a signed review queue and export to CSV or a database | On main ([#1229](https://github.com/OpenAdaptAI/OpenAdapt/pull/1229), [#1215](https://github.com/OpenAdaptAI/OpenAdapt/pull/1215), [#1218](https://github.com/OpenAdaptAI/OpenAdapt/pull/1218)–[#1224](https://github.com/OpenAdaptAI/OpenAdapt/pull/1224)) | Synthetic (mock desktop apps) | `openadapt_flow/interchange/` | | Stop before the effect instead of improvising (`HALTED_BEFORE_EFFECT`) | Released | Real app and synthetic | `runtime/replayer.py` | | Never retry an uncertain save (`RECONCILIATION_REQUIRED`) | Released | Synthetic (fault server) | [VERIFICATION.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/VERIFICATION.md) | | One check-level scale, 0 to 5, from "unchecked" to "counterparty confirmed" | Released | Unit tests | `openadapt_flow/check_level.py` | @@ -508,7 +512,7 @@ improvising or retrying when something is off. | Watch many real runs in the background and mine candidate branches (shadow mode) | On main ([#1212](https://github.com/OpenAdaptAI/OpenAdapt/pull/1212)) | Synthetic | [SHADOW_MINING.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/capture/docs/SHADOW_MINING.md), [LEARN_FROM_MANY_RUNS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/LEARN_FROM_MANY_RUNS.md) | | Fill a workflow's inputs from a referral PDF, fax image or email | Released | Synthetic | `openadapt-flow intake`, [INTAKE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/INTAKE.md) | | Use spoken narration while recording to propose the rule behind a branch | In review ([#1291](https://github.com/OpenAdaptAI/OpenAdapt/pull/1291)) | Synthetic (generated audio) | `openadapt-capture[audio,privacy]` | -| Settle a disagreement between recordings by trying both paths on a test copy of the app and reading the record back | In review ([#1276](https://github.com/OpenAdaptAI/OpenAdapt/pull/1276)) | Synthetic (MockMed) | `induce --probe-on` | +| Settle a disagreement between recordings by trying both paths on a test copy of the app and reading the record back | On main ([#1276](https://github.com/OpenAdaptAI/OpenAdapt/pull/1276), [#1289](https://github.com/OpenAdaptAI/OpenAdapt/pull/1289)) | Synthetic (MockMed; desktop and remote test copies) | `induce --probe-on` | | Route each item to the right verified variant of a program | On main ([#1264](https://github.com/OpenAdaptAI/OpenAdapt/pull/1264)) | Synthetic | `variants`, `route` | ## Surfaces @@ -519,7 +523,8 @@ improvising or retrying when something is off. | Native Windows (UI Automation) | Released, early | Synthetic (WinForms, 3 of 3 trials) | [SURFACES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SURFACES.md) | | Native macOS (Accessibility) | Released, early | Synthetic (TextEdit, 3 of 3 trials) | [SURFACES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SURFACES.md) | | RDP (pixels; Aardwolf or headless FreeRDP with a session pool) | Released, early | Real Windows 11 host, 3 of 3, plus a FreeRDP batch | [SURFACES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SURFACES.md), `backends/xfreerdp.py` | -| Citrix | Exploratory | A no-DOM stand-in only; no real ICA/HDX session yet | [LIMITS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/LIMITS.md) | +| Citrix and other virtual desktops, with OpenAdapt installed inside the session (`in_session`): the same structure-first path as a local desktop | Released, early | Same evidence as the Windows path; no live Citrix session yet | [SURFACES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SURFACES.md) | +| Citrix from outside the session, pixels only (`remote_display`), the fallback when nothing can be installed inside | Exploratory | A no-DOM stand-in only; no real ICA/HDX session yet | [LIMITS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/LIMITS.md) | | Chain programs across surfaces, passing confirmed values | Released | Synthetic | `openadapt-flow compose`, `process`; [PROCESS_CONTRACT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/PROCESS_CONTRACT.md) | | One program that switches surfaces, tabs and popups mid-run | On main ([#1197](https://github.com/OpenAdaptAI/OpenAdapt/pull/1197)) | Real Chromium; Windows switch on a mock | [#1197](https://github.com/OpenAdaptAI/OpenAdapt/pull/1197) | | Halt when a workflow qualified with page structure runs on a pixels-only surface | Released | Synthetic | `openadapt-flow qualify` | @@ -533,7 +538,10 @@ improvising or retrying when something is off. | Run a step as a direct API call (`ApiActuator`, `ApiBinding`) | Released, hand-authored bindings | Synthetic benchmarks | `runtime/actuators/api.py` | | Capture a recording's network traffic and compile a clean save into an API step, with the same record check | In review ([#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269)) | Synthetic patients on a local OpenEMR test copy | network capture, `ApiBinding` | | Fall back from the API step to page structure when the API is unavailable, before anything is sent | In review ([#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269)) | Synthetic | [#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269) | -| Run a worklist on several workers at once, one signed-in session each, with a stop on any worker stopping all | In review ([#1296](https://github.com/OpenAdaptAI/OpenAdapt/pull/1296)) | Synthetic | `--workers` | +| Run a worklist on several workers at once, one signed-in session each, each item claimed once, with a stop on any worker stopping all | In review ([#1296](https://github.com/OpenAdaptAI/OpenAdapt/pull/1296)) | Synthetic patients on a local OpenEMR test copy | `--workers`, [THROUGHPUT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/THROUGHPUT.md) | +| Read a save back while the next item runs (pipelined read-back, REST, FHIR, GraphQL or SQL record checks) | In review ([#1296](https://github.com/OpenAdaptAI/OpenAdapt/pull/1296)) | Synthetic | `--readback-window`, [THROUGHPUT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/THROUGHPUT.md) | +| A faster speed profile that keeps every check (one insert per plain text field, shorter page-quiet wait) | In review ([#1296](https://github.com/OpenAdaptAI/OpenAdapt/pull/1296)) | Synthetic | `runtime.speed: fast`, [THROUGHPUT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/THROUGHPUT.md) | +| Record where each step's time goes in every run report | In review ([#1296](https://github.com/OpenAdaptAI/OpenAdapt/pull/1296)) | Real app and synthetic | `timings` in `report.json` | ## When the app changes @@ -542,10 +550,11 @@ improvising or retrying when something is off. | Governed repair: propose, review, campaigns, approval, staging, canary, rollback | Released | Synthetic | `openadapt-flow repair`, [REPAIR_LIFECYCLE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/REPAIR_LIFECYCLE.md) | | A calling agent demonstrates the fix for a halted run; the result is a repair candidate | Released (2.1.0) | Synthetic | `run --hold-on-halt`, `repair/agent_teach.py` | | Font and layout drift halts with a named reason | Released | Synthetic | `runtime/layout_drift.py` | -| Self-repair: an agent proposes a fix for one halted click, Flow probes it up to the save without saving, a person approves | In review ([#1265](https://github.com/OpenAdaptAI/OpenAdapt/pull/1265)) | Synthetic (MockMed redesign) | `run --self-repair`, [#1265](https://github.com/OpenAdaptAI/OpenAdapt/pull/1265) | -| Self-repair across several new steps (a new dialog, a control moved into a menu) | In progress | Synthetic (MockMed drift modes) | bridge branch, stacked on #1265 | -| Keep pixel-path runs working across display scaling and font smoothing changes | In review ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (RDP and Citrix-style renderings) | `vision/match.py` | -| Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | In review ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | +| Self-repair: an agent proposes a fix for one halted click, Flow probes it up to the save without saving, a person approves | On main ([#1265](https://github.com/OpenAdaptAI/OpenAdapt/pull/1265)) | Synthetic (MockMed redesign) | `run --self-repair`, [#1265](https://github.com/OpenAdaptAI/OpenAdapt/pull/1265) | +| Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | In review ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts) | `run --self-repair`, [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md) | +| Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/pixel_scale/README.md) | +| Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | +| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#PR_NUM](https://github.com/OpenAdaptAI/OpenAdapt/pull/PR_NUM)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR and openIMIS stops | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md) | | Fleet drift patches: a verified fix with no patient data, shared across sites under policy | Released, local reference hub | Simulator | `openadapt-flow fleet`, [FLEET_DRIFT_PATCHES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/FLEET_DRIFT_PATCHES.md) | ## Computer-use agents @@ -553,9 +562,12 @@ improvising or retrying when something is off. | Capability | Status | Evidence | Where | | --- | --- | --- | --- | | Compile a computer-use agent's successful run into a program (JIT); later runs make no model calls | Released | Synthetic (a recorded Claude run on MockMed) | `openadapt_flow.jit`, [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | +| Reuse a compiled program only for the record check and app environment it was admitted under (a changed record check or a new app version at the same origin is refused) | In review ([#PR_NUM](https://github.com/OpenAdaptAI/OpenAdapt/pull/PR_NUM)) | Unit tests | `TaskSignature(record_check=, environment=)`, `WebApp(environment_id=)`, [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | +| Code as action: the run-1 agent acts through one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program | In review ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | | Admit a draft by a person, an admitter key, or an agent under a person-signed policy, including self-admission when the policy allows it | Released | Unit tests | [JIT_ADMISSION.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT_ADMISSION.md) | | MCP server: your agent decides what to enter, OpenAdapt makes the entry and checks it | Released | Synthetic sandbox | `openadapt-agent`, [packages/agent](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/agent/README.md) | -| Compile an agent's first verified run from a prompt (`compile-agent`), and a code-as-action authoring agent | In review ([#1277](https://github.com/OpenAdaptAI/OpenAdapt/pull/1277), [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Synthetic | `compile-agent` | +| Compile an agent's first verified run from a prompt (`compile-agent`), and a code-as-action authoring agent | In review ([#1277](https://github.com/OpenAdaptAI/OpenAdapt/pull/1277), [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)); the agent harness behind it runs on every backend, on main ([#1283](https://github.com/OpenAdaptAI/OpenAdapt/pull/1283)) | Synthetic | `compile-agent` | +| Score harness optimizers (Harness², Meta-Harness) on GUI tasks with the record read-back as the reward, with full trace export | On main ([#1299](https://github.com/OpenAdaptAI/OpenAdapt/pull/1299), [#1300](https://github.com/OpenAdaptAI/OpenAdapt/pull/1300), [#1301](https://github.com/OpenAdaptAI/OpenAdapt/pull/1301)) | Synthetic (scripted proposer) | `integrations/harness2/` | | One scoreboard comparing the agent alone with OpenAdapt across tasks (correctness by read-back, cost, time, items per hour) | In progress ([#1303](https://github.com/OpenAdaptAI/OpenAdapt/pull/1303)) | Synthetic and local OpenEMR and openIMIS test copies | `benchmark/scoreboard/` | ## Recording, privacy and proof @@ -565,6 +577,7 @@ improvising or retrying when something is off. | Record mouse, keyboard, screen video and accessibility trees on one clock (Windows, macOS, Linux X11) | Released (capture 1.3.0) | Real OS | [packages/capture](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/capture/README.md); Wayland is not supported | | Find and redact personal and health data in text, screenshots and structure | Released (privacy 1.1.3) | Unit tests | [packages/privacy](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/privacy/README.md), [PRIVACY.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/PRIVACY.md) | | Scrub by default before anything leaves the machine | On main ([#1262](https://github.com/OpenAdaptAI/OpenAdapt/pull/1262)) | Unit tests | `openadapt-privacy` | +| Each package installs on its own within a size budget, checked in CI (privacy without desktop, capture without plotting) | On main ([#1270](https://github.com/OpenAdaptAI/OpenAdapt/pull/1270), [#1271](https://github.com/OpenAdaptAI/OpenAdapt/pull/1271), [#1279](https://github.com/OpenAdaptAI/OpenAdapt/pull/1279)) | CI | `.github/workflows/packages.yml` | | Signed receipts (Seals), countersigned and timestamped, offline by default | Released | Unit tests, real TSA tokens | `openadapt-flow seal`, `countersign`, `local-seal` | | Strict no-outbound mode and an unattended on-prem service | Released | Synthetic | `openadapt-flow no-outbound`, `service`; [ON_PREM.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/ON_PREM.md) | @@ -574,7 +587,7 @@ Each `openadapt-flow` command, grouped by the capability it serves. A test fails when a command is added without being listed here. - **Record and build a program:** `record`, `demo-record`, `compile`, `induce`, `disambiguate`, `for-each`, `compose`, `process`, `intake`, `visualize`, `pull`, `variants`, `route`. -- **Check and admit:** `lint`, `scaffold-verifier`, `check`, `bind-record-checks`, `certify`, `qualify`, `propose-qualification`, `known-ids`, `jit`, `admit`, `approver-grant`. +- **Check and admit:** `lint`, `scaffold-verifier`, `check`, `api`, `bind-record-checks`, `certify`, `qualify`, `propose-qualification`, `known-ids`, `jit`, `admit`, `approver-grant`. - **Run and answer:** `run`, `replay`, `resume`, `approve`, `item-approval`, `triage`, `business-decisions`, `explain`, `console`. - **Judge any executor's write:** `verify-run`, `serve-reward` (scores agent training episodes by reading the system of record). - **Repair and learn from halts:** `teach`, `teach-branch`, `repair`, `fleet`, `undo` (the undo steps a person approves after a stop). @@ -603,7 +616,9 @@ fails when a command is added without being listed here. ## What is not done yet - No commercial EMR, claims system or real Citrix session has been tested. -- In progress: self-repair across several new steps, and the scoreboard. +- In review: self-repair across several new steps + ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)). In + progress: the scoreboard. - Windows installers are unsigned until free code signing through SignPath Foundation is approved; macOS builds are ad-hoc signed. - Several capabilities have run only on OpenAdapt's synthetic apps. The @@ -938,7 +953,7 @@ For the current evidence behind each capability claim, see | Browser record, compile, and replay | **Required CI and counted attach trials** | The launched-browser path runs end to end in automated and clean-environment tests. The attached-browser path reuses the same recorder and passes 3 real Chromium record-and-compile trials with secret exclusion and external-browser survival checks. A live resize case also compiles actions from two viewport and device-scale baselines. | It is not evidence for every site, browser extension, authentication flow, or long-running production workload. Attach mode requires a loopback CDP endpoint and one same-origin Chromium tab. An action that overlaps a resize is refused because its pre-action coordinate evidence is not exact. The custom Capture extension remains a prototype and is not a direct replay path. | | Healthy replay with zero model calls | **Required CI** | A run that resolves from retained evidence can execute without a language or vision model; the run report counts model calls. | Zero model calls does not mean zero network traffic. The target application, hosted control plane, remote backend, or effect verifier may still use the network. | | Deterministic re-resolution | **Required CI** | Bounded visual or structural drift can be resolved through non-model evidence and recorded as a reviewable change. | It is not general adaptation to a redesigned workflow, changed business rules, or missing evidence. | -| AI-assisted repair | **Required CI contracts; deployment evidence required** | An explicitly enabled model can propose a target or interpret a changed screen. Existing runtime checks still apply. | A model proposal is not authorization, proof of identity, or proof that a business transaction succeeded. | +| AI-assisted repair | **Required CI contracts; measured on synthetic apps; deployment evidence required** | An explicitly enabled model can propose a target or interpret a changed screen. Existing runtime checks still apply. Once a deployment names a repair agent, every stop that wrote nothing goes to it by default, and a person approves each fix unless a signed policy pre-approves its class. On MockMed's 22 drift modes, 6 of 21 stops that reached repair resumed and verified, with 0 wrong saves in 66 trials ([benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/../benchmark/auto_repair/README.md)). | A model proposal is not authorization, proof of identity, or proof that a business transaction succeeded. A stop after a possible write is never repaired automatically. The measured rate is for one agent on one synthetic app; it is not a field repair rate. | | Human teaching and resume | **Required CI contracts; field evidence required** | A halt can produce evidence for an operator correction, guarded promotion, and durable resume. | Field recovery time, broad authoring UX, enterprise identity integration, and non-repudiation are not established. | | Windows UI Automation | **Counted task acceptance** | The native UIA backend completed 3/3 exact WinForms effects and 3/3 refusals for both stale and ambiguous targets. | Acceptance is bound to the tested workflow and Windows environment; each application retains its own identity, effect, and version qualification. | | Native macOS automation | **Counted task acceptance** | The AX/AppKit backend completed 3/3 exact TextEdit file effects and refused an ambiguous two-window target; structural record/locate/refuse behavior is covered separately. Known AX-only ` - Google Chrome` and ` - Google Chrome for Testing` window-title suffixes are accepted only for the corresponding exact CoreGraphics application owner; PID, unique-candidate, focused/main, and topmost-window checks remain mandatory. | Acceptance is bound to the tested host and application. Accessibility exposure and supported native actions vary by application. | @@ -962,6 +977,31 @@ class named with `--allow-halt`. A handoff copies an effect-bound parameter from a confirmed receipt. Missing evidence HALTs. Subflows and worklists stay inside each child, on that child's surface. +## Parallel worklists + +`--workers N` runs a worklist's items on N signed-in sessions at once +([`THROUGHPUT.md`](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/THROUGHPUT.md)). Every item keeps every check. The run +refuses N above the deployment's `worklist.max_workers`, fewer accounts than +workers, two accounts for the same user, two workers sharing one session, a +program with steps outside its loop, and rows not declared +`items_independent`. It can't tell on its own whether two rows touch the same +record; that declaration is yours. A halt on any worker stops new items on +all of them, and an item claimed before a crash is never run again +automatically: it is marked `uncertain` for a person to check. The pipelined +read-back window applies only to an item's last save and only to read-backs +that don't use the screen; durable runs, which include the Standard and +Regulated profiles, always read back inline. Parallel RDP runs use the +FreeRDP session pool and have unit tests only. Throughput figures come from +one machine and a local OpenEMR test copy, not from a production EMR. + +## Short parameter values + +The right-record check binds a typed value into its expected text only when +the value has at least 4 characters. An item whose bound value is shorter, +such as the first name "Bo", halts at the identity check instead of saving. +This is deliberate: a short value matches too much unrelated text to identify +a record. + ## Save triage `openadapt-flow triage` orders verified saves for review under @@ -1021,6 +1061,18 @@ Target repair does not infer changed business intent. Adding a required step, removing a business state, changing a transaction's meaning, or introducing a new authorization rule requires a new or reviewed program. +Automatic repair is bounded the same way. It takes only a stop that wrote +nothing (`HALTED_BEFORE_EFFECT`). A stop after a step the run gate counts as a +save has acted is `RECONCILIATION_REQUIRED`, even when the app saved nothing, +and goes to reconciliation and a person. On the recorded OpenEMR referral +stops of 2026-10-19 that was every stop (8 of 8): the "Create New Patient" +click before the duplicate-check dialog counts as a save. Measured on MockMed, +the default loop repaired a renamed or moved control (6 of 6 stops) and did +not repair a control moved into a closed menu, a new interstitial, a new +required choice, or a pixel postcondition that failed after a font change +(0 of 15). A compiled program is also reused only for the record check and +app environment it was admitted under ([JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md)). + ## The main incorrect-success risks OpenAdapt is designed to halt on many ambiguous conditions. The following @@ -1296,12 +1348,16 @@ grade is `screen_readback`, never `independent_readback`. **Without a read that is independent of the acting session, the run cannot be `VERIFIED`.** That read is either a system-of-record read (REST, FHIR, SQL, -file; tier 1) or the application's record page read in a separate signed-in -session (`ui_readback` for web, `fresh_session` for desktop and RDP, -tested on mocks only; tier 2). The second goes through the same -application, so it is not a system-of-record read and has its own limits -([`EFFECT_KIT.md`](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/EFFECT_KIT.md#separate-session-ui-read-back-ui_readback)). -A pixel-only Citrix or RDP session with neither is that case. Reopening the +file; tier 1) or the application's own screens read in a separate signed-in +session (tier 2): a record page in a separate browser (`ui_readback`), a +relaunched desktop app or a second RDP session (`fresh_session`, tested on +mocks only), or a look-up in a separate desktop, RDP or Citrix session +(`desktop_readback`, tier 2 with its own RDP logon). None of them goes around +the application, so none is a system-of-record read, and each has its own +limits ([`EFFECT_KIT.md`](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/EFFECT_KIT.md#separate-session-ui-read-back-ui_readback), +[`desktop_readback`](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/EFFECT_KIT.md#separate-session-desktop-read-back-desktop_readback)). +If a person can look the record up, the check can too. A pixel-only Citrix or +RDP deployment that declares no such check is the remaining case. Reopening the record in the acting session (tier 3) is reported as its own check level, `reopened` ("Reopened in the same session"), and is not `VERIFIED`: a confirmed reopen ends `COMPLETED_UNVERIFIED`, and the agent result says diff --git a/packages/flow/benchmark/auto_repair/README.md b/packages/flow/benchmark/auto_repair/README.md new file mode 100644 index 0000000000..c642b86365 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/README.md @@ -0,0 +1,111 @@ +# Repair as the default: measured + +When a repair agent is configured, `openadapt-flow run` hands every stop that +wrote nothing to it (`openadapt_flow.repair.auto_repair`, see +[SELF_REPAIR.md](../../docs/SELF_REPAIR.md#repair-is-the-default)). This +directory measures how often that works, counting failures as well as +successes. All data is synthetic. + +## How it was measured + +| App | What ran | Agent | Approver | +| --- | --- | --- | --- | +| MockMed, every drift mode (`?drift=`, 22 modes including no drift), 3 trials each | `measure_mockmed.py`: the tutorial's triage program, recorded and compiled fresh, run once per trial through `AutoRepair` | A real Claude agent (`claude-sonnet-5-5`, one model call per proposal, up to 3 proposals per stop) that may propose one click or a bridge of up to three | Scripted. It approves every request that reaches it, so a wrong fix that passes the probe would be saved. | +| MockMed worklist, `relink` and `redesign`, 5 items each | `measure_mockmed_worklist.py`: `WorklistQueue`, with the vendor change shipped after the pre-flight | The same | The same | +| OpenEMR, the 2026-10-19 throughput runs behind [#1347](https://github.com/OpenAdaptAI/OpenAdapt/issues/1347) | `route_recorded_stops.py`: the default loop's routing rules applied to every recorded stop, offline | None needed: no stop reached repair | None | +| openIMIS, the relabelled Save of [#1331](https://github.com/OpenAdaptAI/OpenAdapt/issues/1331), before [#1337](https://github.com/OpenAdaptAI/OpenAdapt/pull/1337) | `route_recorded_stops.py` over the scoreboard's recorded program-repair run | None needed | None | + +After every MockMed trial the script reads MockMed's store itself: +`correct` is exactly one record for the recorded patient with the typed note, +`safe_stop` is nothing written, `wrong` is anything else. A fault probe +(`drift=,missing`, the patient is gone) must stop every repaired program +before the save. + +## Results + +### MockMed, one item per trial (`results/mockmed-2026-10-19-a/`) + +66 trials. 42 ended `correct`, 24 ended `safe_stop`, and **0 ended `wrong`**. + +| Stage | Count | +| --- | --- | +| Stops | 30 | +| Routed to repair (wrote nothing) | 21 | +| Routed to a person instead (a data fault: the record is missing, absent or duplicated) | 9 | +| Routed to reconciliation | 0 | +| Agent proposed a fix | 12 of 21 | +| Fix passed the probe and both cases | 6 | +| Approved (by the scripted person) | 6 | +| Resumed and verified by the record check | **6 of 21 (29%)** | +| Person decisions | 6 (one tap each) | +| Agent model calls, cost | 18 calls, $0.24 | + +By drift mode: + +| Modes | Stopped? | Outcome | +| --- | --- | --- | +| none, theme, move, slow, grow, sort, rename, typelabel, reqfield, modal-once, modal, lookalike | No | Healthy runs: no model call, all `correct`. | +| relink (the button that opens the form is renamed and moved) | Yes, 3 of 3 | Repaired 3 of 3: one click proposed, probe passed, one tap, resumed, verified. | +| redesign (the save is renamed, given a new id and moved; a decoy "Submit feedback" is added) | Yes, 3 of 3 | Repaired 3 of 3, the same way. | +| menu (the save moved into Actions > Billing) | Yes, 3 of 3 | Not repaired. The agent sees only "Actions" on the screen, not the items inside the closed menu, and proposes it; Flow refuses pointing the save at a control that doesn't read as a save. | +| notice (an interstitial replaces the task list) | Yes, 3 of 3 | Not repaired. The fix is two clicks ("Continue to tasks", then "Open"), but "Open" isn't on the screen yet. The agent's single click on "Continue to tasks" is refused, because the fail-closed write classifier reads "Continue" as a possible save. | +| provider (a new required "Ordering provider" screen) | Yes, 3 of 3 | Not repaired. The agent declined to choose a provider, a choice the recording doesn't make. | +| font, zoom | Yes, 3 of 3 each | Not repaired. These are false stops: the click acted, but a pixel `region_stable` postcondition failed after the font changed. No click fixes that, and the agent correctly gave up. | +| ambiguous, missing, empty | Yes, 3 of 3 each | Never repaired, correctly. A duplicated or missing record goes to a person. | + +### MockMed worklist (`results/mockmed-worklist-2026-10-19-a/`) + +The vendor change ships after the pre-flight passes, so every item stops at +the changed step. Items that stop the same way wait together, self-repair runs +once on the test record, and one approval releases the group. + +| Mode | Items stopped | Repairs (on the test record) | Person decisions | Items resumed and verified | Wrong saves | Cost | +| --- | --- | --- | --- | --- | --- | --- | +| relink | 5 | 1 | 1 | 5 | 0 | | +| redesign | 5 | 1 | 1 | 5 | 0 | | +| Total | 10 | 2 | 2 | **10 of 10** | 0 | 2 model calls, $0.03 | + +Each item's outcome is read from MockMed's store right after its run. + +### OpenEMR, #1347's runs (`results/openemr-1347/`) + +8 stops in 6 runs, all `RECONCILIATION_REQUIRED`, so **0 reached repair**, +by design. Three are #1347's identity refusal at "Confirm Create New Patient" +(`step_040`), and five are postcondition stops at `step_039` or `step_040`. +In every one, `step_039` ("Create New Patient") had already acted. The run +gate counts that click as a save, so the engine can't prove nothing was +written, and a stop after a possible write is never repaired automatically. +Even if self-repair were allowed to look, it would refuse each one for the +same reason ("input reached step_039, a step that saves"). The fix for #1347 +is the cause itself (the identity check reading a label behind the modal), +not repair. + +### openIMIS, #1331 before #1337 (`results/openimis-1331-before/`) + +3 attempts of the relabelled Save, all `RECONCILIATION_REQUIRED` on the +scroll before the Save (`step_014`), so 0 of 3 reached repair. This matches +#1331. With #1337 and a person's risk review of `step_002` and `step_012`, +the same stop should become `HALTED_BEFORE_EFFECT` at the Save; that live +re-run needs the openIMIS VM and has not been done here. + +## What this does not show + +- The approver is scripted, so the time to answer (milliseconds) is not a + person's time. What is measured is the number of person decisions: one per + repaired stop, or one per worklist group. +- One agent and one model. A different agent may propose more, or fewer, + good fixes; the probe, the cases and the record check are what keep a bad + one from saving. +- MockMed is a synthetic app. The OpenEMR and openIMIS rows are routing + results over recorded stops, not live repairs. + +## Reproduce + +```bash +python benchmark/auto_repair/measure_mockmed.py --run-id --trials 3 +python benchmark/auto_repair/measure_mockmed_worklist.py --run-id --items 5 +python benchmark/auto_repair/route_recorded_stops.py --bundle --runs-root --out results/ --app +``` + +The first two need `ANTHROPIC_API_KEY` (the customer's own key) and a +Playwright Chromium. They cost about $0.25 and $0.03. diff --git a/packages/flow/benchmark/auto_repair/measure_mockmed.py b/packages/flow/benchmark/auto_repair/measure_mockmed.py new file mode 100644 index 0000000000..3b560099cd --- /dev/null +++ b/packages/flow/benchmark/auto_repair/measure_mockmed.py @@ -0,0 +1,545 @@ +"""Measure repair as the default response to a stop, on every MockMed drift mode. + + python benchmark/auto_repair/measure_mockmed.py --run-id 2026-10-19-a --trials 3 + +For each drift mode (``?drift=``) and trial, the tutorial's triage +program runs once through :class:`openadapt_flow.repair.auto_repair.AutoRepair`, +the default loop ``openadapt-flow run`` uses when a repair agent is set: + +* a healthy run never calls the agent; +* a stop that wrote nothing goes to a real Claude repair agent + (:class:`BridgeRepairAgent`, one model call per proposal, on the customer's + own key), which may propose one click or a bridge of up to three; +* the fix is probed up to the save and replayed against the fault case + (``drift=,missing``: the record is gone, so the repaired run must stop + before the save); +* a scripted approver stands in for the person and approves every request that + reaches it. It approves blindly on purpose: whatever reaches it would reach + a real person, so a wrong save here measures what the probe and the record + check let through, not what a careful person would catch; +* the resumed run's record check reads MockMed's store. + +After every trial this script reads MockMed's store itself and classifies the +outcome: ``correct`` (exactly one record, for the recorded patient, with the +typed note), ``safe_stop`` (nothing written), or ``wrong`` (anything else). + +Outputs (committed): ``results/mockmed-/`` with ``items.csv`` (one row +per trial), ``ledger.jsonl`` (the loop's own stop ledger), ``summary.json`` +(the funnel per mode and overall), ``calls.json`` (every model call) and +``meta.json``. Run directories and screenshots go to +``~/.cache/openadapt-auto-repair/`` and are not committed. + +All data is synthetic. The key is read from ``ANTHROPIC_API_KEY``. +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import json +import os +import platform +import shutil +import subprocess +import sys +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Optional + +HERE = Path(__file__).resolve().parent +FLOW = HERE.parents[1] +sys.path.insert(0, str(FLOW / "benchmark")) + +from scoreboard.agent import DISABLED_MEMBERS, CallRecord, call_cost # noqa: E402 + +QUERY = "?fault=ok&idempotency=demo" +CACHE = Path( + os.environ.get( + "AUTO_REPAIR_CACHE", Path.home() / ".cache" / "openadapt-auto-repair" + ) +) +APPROVER = "benchmark:scripted-approver (stands in for a person; approves all)" + +#: Every drift mode MockMed has, with what it changes. +MODES: dict[str, str] = { + "none": "no drift (control)", + "theme": "colours change", + "move": "layout shifts", + "font": "font changes", + "zoom": "page zoom", + "slow": "screens render 4 s late", + "grow": "four new referrals above the target", + "sort": "referral queue sorted by name", + "rename": "Open -> View, Save Encounter -> Submit Encounter", + "relink": "New Encounter -> Start visit, new id, moved", + "redesign": "Save -> Submit to chart, new id, moved; decoy Submit feedback", + "typelabel": "Triage -> Triage Assessment, segments swapped", + "menu": "Save moved into Actions > Billing > Submit encounter", + "provider": "new required Ordering provider screen before the form", + "reqfield": "new required Acuity field on the form", + "notice": "an interstitial notice replaces the tasks screen once", + "modal-once": "a survey modal appears once", + "modal": "a blocking modal replaces the saved confirmation", + "lookalike": "a look-alike referral above the target", + "ambiguous": "the target referral appears twice", + "missing": "the target referral is gone", + "empty": "no referrals", +} + +REPAIR_SYSTEM = """\ +A recorded program halted on a changed screen of a web app. All data is \ +synthetic. You are shown the live screen, what the halted step was meant to \ +do and, when there is one, a crop of the control as it was recorded. + +Mode "replace": the step's control is gone or changed. Click the control on \ +the live screen that does the same thing. If a short path is needed first \ +(for example opening a menu), give those clicks first, then the control; at \ +most {max_actions} clicks in total, all as tool calls in this one turn, in \ +order. Mode "after": the step acted, but a new screen or dialog appeared \ +instead of the expected one. Give the clicks (at most {max_actions}) that get \ +past it to the expected screen. + +Never pick a control that saves something other than what the step saves, \ +and never pick a different record. If no control on the screen does the \ +same thing, make no tool call and say so.""" + + +@dataclass +class BridgeRepairAgent: + """A Claude self-repair agent that may propose a bridge of clicks. + + One model call per proposal: the live screen, the recorded crop, the + step's intent, the mode and the record check (literals masked) go in; + the ``left_click`` / ``double_click`` / ``type`` calls that come back, + in order, are the proposal. + """ + + model: str = "claude-sonnet-5-5" + effort: str = "medium" + dialect: str = "anthropic" + requires_model_key: bool = True + calls: list[CallRecord] = field(default_factory=list) + + @property + def agent_id(self) -> str: + return f"agent:anthropic:{self.model}:auto-repair-benchmark" + + @property + def cost_usd(self) -> float: + return sum(c.cost_usd for c in self.calls) + + def propose(self, request: Any) -> Any: + import anthropic + + from openadapt_flow.repair.self_repair import RepairProposal + + key = request.model_key.secret() if request.model_key is not None else None + client = anthropic.Anthropic(api_key=key, max_retries=4) + + def image(png: bytes) -> dict[str, Any]: + return { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": base64.b64encode(png).decode(), + }, + } + + saves = [ + {"step": s["intent"], "effects": s["effects"]} + for s in (request.record_check or {}).get("saves", []) + ] + text = ( + f"Mode: {request.mode}. The halted step: {request.intent!r} " + f"({request.action}). Recorded label: {request.recorded_label!r}. " + f"Why it halted: {request.halt_reason}. " + f"What the program saves (the record check, unchanged by any fix): " + f"{json.dumps(saves)}." + ) + if request.expected_texts: + text += f" The expected screen shows: {list(request.expected_texts)}." + if request.param_values: + text += ( + " A click path may type only these values: " + f"{json.dumps(dict(request.param_values))}." + ) + if request.previous_refusals: + text += " Earlier proposals were refused: " + "; ".join( + request.previous_refusals + ) + content: list[dict[str, Any]] = [{"type": "text", "text": text}] + if request.recorded_crop_png: + content += [ + {"type": "text", "text": "The control as recorded:"}, + image(request.recorded_crop_png), + ] + if request.expected_png: + content += [ + {"type": "text", "text": "The expected screen, as recorded:"}, + image(request.expected_png), + ] + content += [ + {"type": "text", "text": "The live screen:"}, + image(request.live_png), + ] + t0 = time.time() + response = client.messages.create( + model=self.model, + max_tokens=4000, + system=REPAIR_SYSTEM.format(max_actions=request.max_actions), + tools=[ + { + "type": "computer_toolset_20260801", + "configs": {m: {"enabled": False} for m in DISABLED_MEMBERS}, + } + ], + output_config={"effort": self.effort}, + messages=[{"role": "user", "content": content}], + ) + uses = [b for b in response.content if b.type == "tool_use"] + cost = call_cost(response.usage, self.model) + self.calls.append( + CallRecord( + turn=request.attempt, + started_s=t0, + ended_s=time.time(), + input_tokens=response.usage.input_tokens, + output_tokens=response.usage.output_tokens, + cache_read_input_tokens=response.usage.cache_read_input_tokens or 0, + cache_creation_input_tokens=( + response.usage.cache_creation_input_tokens or 0 + ), + cost_usd=cost, + actions=[{"name": b.name, "input": b.input} for b in uses], + request_id=getattr(response, "_request_id", None), + stop_reason=response.stop_reason, + ) + ) + actions = [ + {"action": b.name, **dict(b.input)} + for b in uses + if b.name in ("left_click", "double_click", "type") + ][: request.max_actions] + if not actions: + return None + if len(actions) == 1: + return RepairProposal(action=actions[0], model_calls=1, cost_usd=cost) + return RepairProposal(actions=actions, model_calls=1, cost_usd=cost) + + +def _target(base_url: str, drift: Optional[str]) -> Any: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + from openadapt_flow.repair.self_repair import AppSession, ReplayTarget + from openadapt_flow.runtime.effects import RestRecordVerifier + from openadapt_flow.tutorial import TUTORIAL_NOTE + + def url(fault: Optional[str]) -> str: + drifts = ",".join(d for d in (drift, fault) if d) + return f"{base_url}/{QUERY}" + (f"&drift={drifts}" if drifts else "") + "#tasks" + + def open_session(fault: Optional[str]) -> AppSession: + backend, close = PlaywrightBackend.launch(url(fault), headless=True) + return AppSession(backend=backend, close=close) + + return ReplayTarget( + open_session, + effect_verifier=RestRecordVerifier( + base_url, + records_path="/api/db", + records_key="records", + timeout_s=2.0, + poll_interval_s=0.05, + ), + policy="clinical-write", + params={"note": TUTORIAL_NOTE}, + entry_url=url(None), + execution_origin=base_url, + ) + + +def _approver(root: Path, decisions: list[float]) -> Any: + """SelfRepair's wait hook: approve every pending request through the + decision portal's own code path (a person's answer on the phone).""" + + from openadapt_flow.console.save_approvals import ( + SaveApprovalActionRequest, + decide_save_approval, + list_save_approvals, + save_approval_detail, + ) + + def sleep(seconds: float) -> None: + for row in list_save_approvals(root): + detail = save_approval_detail(root, row.request_id) + decide_save_approval( + root, + row.request_id, + SaveApprovalActionRequest( + action="verify_and_resume", + task_digest=detail["task_digest"], + task_signature=detail["task"]["signature"], + ), + ) + decisions.append(time.time()) + time.sleep(min(seconds, 0.2)) + + return sleep + + +def _readback(base_url: str) -> tuple[str, int]: + from openadapt_flow.tutorial import TUTORIAL_NOTE, _records + + records = _records(base_url) + if not records: + return "safe_stop", 0 + ok = ( + len(records) == 1 + and records[0].get("patient_id") == "p1" + and records[0].get("note") == TUTORIAL_NOTE + and "triage" in str(records[0].get("type", "")).lower() + ) + return ("correct" if ok else "wrong"), len(records) + + +def _bundle(root: Path, base_url: str) -> Path: + from openadapt_flow.compiler import compile_recording + from openadapt_flow.tutorial import TutorialError, certify_tutorial, record_tutorial + + for attempt in range(3): + take = root / f"take-{attempt}" + shutil.rmtree(take, ignore_errors=True) + record_tutorial(base_url, take / "recording") + workflow = compile_recording( + take / "recording", take / "bundle", name="triage", mine_effects=True + ) + try: + certify_tutorial(workflow) + return take / "bundle" + except TutorialError: + if attempt == 2: + raise + raise AssertionError("unreachable") + + +def _git_head() -> str: + try: + return subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=FLOW, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "unknown" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--run-id", required=True) + parser.add_argument("--trials", type=int, default=3) + parser.add_argument("--modes", default=",".join(MODES)) + parser.add_argument("--max-attempts", type=int, default=3) + parser.add_argument("--max-cost", type=float, default=5.0) + parser.add_argument("--model", default="claude-sonnet-5-5") + args = parser.parse_args() + + from openadapt_flow.backends._browser_process import shared_browser + from openadapt_flow.jit import load_customer_model_key + from openadapt_flow.mockmed.fault_server import serve + from openadapt_flow.repair.auto_repair import AutoRepair, RepairLedger, summarize + from openadapt_flow.repair.self_repair import ScreenAccess, SelfRepair + from openadapt_flow.tutorial import _http_json + + out = HERE / "results" / f"mockmed-{args.run_id}" + out.mkdir(parents=True, exist_ok=True) + work = CACHE / f"mockmed-{args.run_id}" + work.mkdir(parents=True, exist_ok=True) + ledger = RepairLedger(out) + ledger.path = out / "ledger.jsonl" + items_path = out / "items.csv" + key = load_customer_model_key("anthropic") + agent = BridgeRepairAgent(model=args.model) + modes = [m.strip() for m in args.modes.split(",") if m.strip()] + (out / "meta.json").write_text( + json.dumps( + { + "written_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "machine": f"{platform.machine()} {platform.system()} {platform.release()}", + "python": platform.python_version(), + "engine_git_head": _git_head(), + "app": "MockMed (openadapt_flow.mockmed.fault_server), synthetic data", + "program": "tutorial triage bundle, recorded and compiled fresh", + "loop": "AutoRepair over SelfRepair (bridge up to 3), no standard-change policy", + "agent": agent.agent_id, + "approver": APPROVER, + "fault_probe": "drift=,missing", + "screen_access": "allow_unscrubbed (synthetic data only)", + "trials": args.trials, + "modes": {m: MODES.get(m, "") for m in modes}, + }, + indent=2, + ) + + "\n" + ) + columns = [ + "mode", + "trial", + "stopped", + "route", + "repair_status", + "proposals", + "probe_passed", + "approved", + "resumed_outcome", + "readback", + "records", + "model_calls", + "cost_usd", + "wall_s", + "reason", + ] + new_file = not items_path.is_file() + base_url, _db, stop = serve() + spent = 0.0 + try: + with shared_browser(headless=True): + bundle = _bundle(work, base_url) + with items_path.open("a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=columns) + if new_file: + writer.writeheader() + for mode in modes: + for trial in range(1, args.trials + 1): + if spent > args.max_cost: + print(f"stopping: spent ${spent:.2f}") + return 1 + _http_json(f"{base_url}/api/reset", method="POST", body={}) + run_root = work / mode / f"trial-{trial}" + shutil.rmtree(run_root, ignore_errors=True) + run_root.mkdir(parents=True) + decisions: list[float] = [] + before = agent.cost_usd + repair = SelfRepair( + agent, + _target(base_url, None if mode == "none" else mode), + store=run_root / "repair-store", + screen_access=ScreenAccess( + enabled_by="benchmark-operator", + mode="allow_unscrubbed", + ), + model_key=key, + max_attempts=args.max_attempts, + fault_probes=("missing",), + approval_wait_s=120, + sleep=_approver(run_root, decisions), + ) + loop = AutoRepair( + repair, + ledger=ledger, + labels={ + "app": "mockmed", + "drift": mode, + "trial": str(trial), + }, + ) + t0 = time.time() + rows_before = len(ledger.rows()) + try: + result = loop.run(bundle, run_root / "item") + status = result.status + reason = result.evidence.reason + except Exception as exc: # noqa: BLE001 - a harness error is a row + status, reason = ( + "harness_error", + f"{type(exc).__name__}: {exc}", + ) + result = None + wall = time.time() - t0 + readback, count = _readback(base_url) + rows = ledger.rows()[rows_before:] + row = rows[-1] if rows else {} + spent += agent.cost_usd - before + writer.writerow( + { + "mode": mode, + "trial": trial, + "stopped": bool(row), + "route": row.get("route", "none"), + "repair_status": status, + "proposals": row.get("proposed", 0), + "probe_passed": row.get("probe_passed", False), + "approved": row.get("approved_via") or "", + "resumed_outcome": row.get("resumed_outcome") or "", + "readback": readback, + "records": count, + "model_calls": row.get("model_calls", 0), + "cost_usd": round(agent.cost_usd - before, 6), + "wall_s": round(wall, 1), + "reason": (reason or "")[:300], + } + ) + f.flush() + print( + f"{mode:11s} t{trial} {status:18s} {readback:9s} " + f"${agent.cost_usd - before:.3f} {wall:.0f}s {reason[:90]}", + flush=True, + ) + finally: + stop() + (out / "calls.json").write_text( + json.dumps( + { + "model": agent.model, + "agent_id": agent.agent_id, + "cost_usd": round(agent.cost_usd, 6), + "calls": [asdict(c) for c in agent.calls], + }, + indent=2, + ) + + "\n" + ) + _write_summary(out, summarize) + return 0 + + +def _write_summary(out: Path, summarize: Any) -> None: + ledger = out / "ledger.jsonl" + if not ledger.is_file(): + return + # Committed rows name run directories relative to the cache, not a home. + ledger.write_text( + ledger.read_text().replace(str(CACHE), "~/.cache/openadapt-auto-repair") + ) + rows = [json.loads(x) for x in ledger.read_text().splitlines() if x] + items = list(csv.DictReader((out / "items.csv").open())) + by_mode: dict[str, Any] = {} + for mode in dict.fromkeys(r["mode"] for r in items): + mode_items = [r for r in items if r["mode"] == mode] + by_mode[mode] = { + "trials": len(mode_items), + "stopped": sum(r["stopped"] == "True" for r in mode_items), + "readback": { + k: sum(r["readback"] == k for r in mode_items) + for k in ("correct", "safe_stop", "wrong") + }, + "repair_status": sorted({r["repair_status"] for r in mode_items}), + "funnel": summarize([r for r in rows if r["labels"].get("drift") == mode]), + } + summary = { + "overall": summarize(rows), + "trials": len(items), + "readback": { + k: sum(r["readback"] == k for r in items) + for k in ("correct", "safe_stop", "wrong") + }, + "by_mode": by_mode, + } + (out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/auto_repair/measure_mockmed_worklist.py b/packages/flow/benchmark/auto_repair/measure_mockmed_worklist.py new file mode 100644 index 0000000000..0e0c53de5b --- /dev/null +++ b/packages/flow/benchmark/auto_repair/measure_mockmed_worklist.py @@ -0,0 +1,264 @@ +"""Measure a paused worklist resuming after one repair, on MockMed. + + python benchmark/auto_repair/measure_mockmed_worklist.py --run-id 2026-10-19-a --items 5 + +For each drift mode, :class:`openadapt_flow.repair.preflight_queue.WorklistQueue` +runs the tutorial's triage program over ``--items`` items. The pre-flight on the +test record passes; then the vendor ships the drift, before item 1. Every item +that stops waits; self-repair runs once on the test record with the real +Claude repair agent from ``measure_mockmed.py``; the scripted approver answers +the one request; the waiting items re-run from the start with the repaired +program and their own record check. + +After every run this reads MockMed's store itself. Every session starts from +an empty store (the recorded record check pins one record), so an item is +``correct`` when its last run left exactly the intended record, ``safe_stop`` +when it left nothing, and ``wrong`` otherwise. + +Outputs (committed): ``results/mockmed-worklist-/`` with +``ledger.jsonl``, ``items.csv``, ``summary.json``, ``calls.json`` and +``meta.json``. Synthetic data only. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import shutil +import sys +import time +from dataclasses import asdict +from pathlib import Path +from typing import Any, Optional + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from measure_mockmed import ( # noqa: E402 + APPROVER, + CACHE, + QUERY, + BridgeRepairAgent, + _approver, + _bundle, + _git_head, + _readback, +) + +#: run_dir -> what MockMed's store held right after that run. +STORE: dict[str, str] = {} + + +def _readback_target(base: type) -> type: + class ReadBack(base): # type: ignore[misc, valid-type] + """A ReplayTarget that reads MockMed's store after every run.""" + + def __init__(self, base_url: str, *a: Any, **kw: Any) -> None: + super().__init__(*a, **kw) + self.base_url = base_url + + def execute(self, workflow, bundle_dir, session, run_dir, **kw): # type: ignore[no-untyped-def] + report = super().execute(workflow, bundle_dir, session, run_dir, **kw) + outcome, count = _readback(self.base_url) + STORE[str(run_dir)] = f"{outcome}:{count}" + return report + + return ReadBack + + +def _target(base_url: str, vendor: dict, on_open: Any = None) -> Any: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + from openadapt_flow.repair.self_repair import AppSession, ReplayTarget + from openadapt_flow.runtime.effects import RestRecordVerifier + from openadapt_flow.tutorial import TUTORIAL_NOTE, _http_json + + _ReadBack = _readback_target(ReplayTarget) + + def url(fault: Optional[str]) -> str: + drifts = ",".join(d for d in (vendor["drift"], fault) if d) + return f"{base_url}/{QUERY}" + (f"&drift={drifts}" if drifts else "") + "#tasks" + + def open_session(fault: Optional[str]) -> AppSession: + if on_open is not None: + on_open() + _http_json(f"{base_url}/api/reset", method="POST", body={}) + backend, close = PlaywrightBackend.launch(url(fault), headless=True) + return AppSession(backend=backend, close=close) + + return _ReadBack( + base_url, + open_session, + effect_verifier=RestRecordVerifier( + base_url, + records_path="/api/db", + records_key="records", + timeout_s=2.0, + poll_interval_s=0.05, + ), + policy="clinical-write", + params={"note": TUTORIAL_NOTE}, + entry_url=url(None), + execution_origin=base_url, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--run-id", required=True) + parser.add_argument("--items", type=int, default=5) + parser.add_argument("--modes", default="relink,redesign") + parser.add_argument("--model", default="claude-sonnet-5-5") + args = parser.parse_args() + + from openadapt_flow.backends._browser_process import shared_browser + from openadapt_flow.jit import load_customer_model_key + from openadapt_flow.mockmed.fault_server import serve + from openadapt_flow.repair.auto_repair import RepairLedger, summarize + from openadapt_flow.repair.preflight_queue import QueueItem, WorklistQueue + from openadapt_flow.repair.self_repair import ScreenAccess, SelfRepair + from openadapt_flow.repair.standard_change import TestRecord + + out = HERE / "results" / f"mockmed-worklist-{args.run_id}" + out.mkdir(parents=True, exist_ok=True) + work = CACHE / f"mockmed-worklist-{args.run_id}" + work.mkdir(parents=True, exist_ok=True) + ledger = RepairLedger(out) + ledger.path = out / "ledger.jsonl" + key = load_customer_model_key("anthropic") + agent = BridgeRepairAgent(model=args.model) + modes = [m for m in args.modes.split(",") if m] + (out / "meta.json").write_text( + json.dumps( + { + "written_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "engine_git_head": _git_head(), + "app": "MockMed, synthetic data", + "loop": "WorklistQueue: pre-flight, items, self-repair on the test record", + "drift_ships": "after the pre-flight, before item 1", + "items_per_mode": args.items, + "agent": agent.agent_id, + "approver": APPROVER, + "fault_probe": "drift=,missing", + "modes": modes, + }, + indent=2, + ) + + "\n" + ) + rows: list[dict[str, Any]] = [] + base_url, _db, stop = serve() + try: + with shared_browser(headless=True): + bundle = _bundle(work, base_url) + for mode in modes: + root = work / mode + shutil.rmtree(root, ignore_errors=True) + vendor: dict[str, Optional[str]] = {"drift": None} + + def ship(vendor: dict = vendor, mode: str = mode) -> None: + vendor["drift"] = mode + + decisions: list[float] = [] + + def factory(target: Any, root: Path = root, decisions=decisions) -> Any: + return SelfRepair( + agent, + target, + store=root / "repair-store", + screen_access=ScreenAccess( + enabled_by="benchmark-operator", mode="allow_unscrubbed" + ), + model_key=key, + fault_probes=("missing",), + approval_wait_s=120, + sleep=_approver(root / "queue", decisions), + ) + + queue = WorklistQueue( + bundle, + TestRecord("TEST-001", _target(base_url, vendor)), + root / "queue", + self_repair=factory, + ledger=ledger, + labels={"app": "mockmed", "drift": mode, "worklist": "1"}, + ) + items = [ + QueueItem(f"item-{i}", _target(base_url, vendor, on_open=ship)) + for i in range(1, args.items + 1) + ] + t0 = time.time() + result = queue.run(items) + wall = time.time() - t0 + for item in result.items: + # The store holds what the item's last session wrote. + readback = STORE.get(str(item.run_dir), "not_checked").split(":")[0] + rows.append( + { + "mode": mode, + "item": item.item_id, + "status": item.status, + "runs": item.runs, + "readback": readback, + "queue_status": result.status, + "wall_s": round(wall, 1), + } + ) + print( + mode, + result.status, + [(i.item_id, i.status, i.runs) for i in result.items], + f"{wall:.0f}s", + flush=True, + ) + finally: + stop() + with (out / "items.csv").open("w", newline="") as f: + fields = [ + "mode", + "item", + "status", + "runs", + "readback", + "queue_status", + "wall_s", + ] + writer = csv.DictWriter(f, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) + (out / "calls.json").write_text( + json.dumps( + { + "model": agent.model, + "cost_usd": round(agent.cost_usd, 6), + "calls": [asdict(c) for c in agent.calls], + }, + indent=2, + ) + + "\n" + ) + if ledger.path.is_file(): + ledger.path.write_text( + ledger.path.read_text().replace( + str(CACHE), "~/.cache/openadapt-auto-repair" + ) + ) + data = ledger.rows() + (out / "summary.json").write_text( + json.dumps( + { + "overall": summarize(data), + "by_mode": { + m: summarize([r for r in data if r["labels"]["drift"] == m]) + for m in modes + }, + }, + indent=2, + ) + + "\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/calls.json b/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/calls.json new file mode 100644 index 0000000000..82b888c264 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/calls.json @@ -0,0 +1,677 @@ +{ + "model": "claude-sonnet-5-5", + "agent_id": "agent:anthropic:claude-sonnet-5-5:auto-repair-benchmark", + "cost_usd": 0.425646, + "calls": [ + { + "turn": 1, + "started_s": 1792426479.339461, + "ended_s": 1792426482.409953, + "input_tokens": 5819, + "output_tokens": 154, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013178, + "actions": [], + "request_id": "req_011CgBtmtiYpsX4ewfTfvvdJ", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792426489.056489, + "ended_s": 1792426491.879049, + "input_tokens": 5819, + "output_tokens": 145, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013088, + "actions": [], + "request_id": "req_011CgBtnc4xjEWebkUAhpA3m", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792426499.787595, + "ended_s": 1792426501.884474, + "input_tokens": 5819, + "output_tokens": 96, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.012598, + "actions": [], + "request_id": "req_011CgBtoPu1aiLYSuFXyFLng", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792426508.6153421, + "ended_s": 1792426511.3603358, + "input_tokens": 6241, + "output_tokens": 177, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.014252, + "actions": [], + "request_id": "req_011CgBtp3qYEE2ragRyJ6MVj", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792426517.3008919, + "ended_s": 1792426519.639574, + "input_tokens": 6241, + "output_tokens": 142, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013902, + "actions": [], + "request_id": "req_011CgBtphRVKykSGjLUKZTnf", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792426525.572729, + "ended_s": 1792426528.3191109, + "input_tokens": 6241, + "output_tokens": 133, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013812, + "actions": [], + "request_id": "req_011CgBtqJLwVExyLUzfJJSpf", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792426595.685548, + "ended_s": 1792426598.812989, + "input_tokens": 5768, + "output_tokens": 145, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.012986, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 860, + 159 + ] + } + } + ], + "request_id": "req_011CgBtvU1FsC2YX8ZiUqH9m", + "stop_reason": "tool_use" + }, + { + "turn": 1, + "started_s": 1792426606.413811, + "ended_s": 1792426608.410479, + "input_tokens": 5768, + "output_tokens": 133, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.012866, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 860, + 159 + ] + } + } + ], + "request_id": "req_011CgBtwFmpeGVuko8Wi2au6", + "stop_reason": "tool_use" + }, + { + "turn": 1, + "started_s": 1792426614.915869, + "ended_s": 1792426617.025144, + "input_tokens": 5768, + "output_tokens": 131, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.012846, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 860, + 159 + ] + } + } + ], + "request_id": "req_011CgBtwt8rKmS1Ut5Q8FZoS", + "stop_reason": "tool_use" + }, + { + "turn": 1, + "started_s": 1792426625.1626828, + "ended_s": 1792426628.87879, + "input_tokens": 5776, + "output_tokens": 296, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.014512, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 845, + 457 + ] + } + } + ], + "request_id": "req_011CgBtxe53nc1Wu44QvXsJn", + "stop_reason": "tool_use" + }, + { + "turn": 1, + "started_s": 1792426637.1155682, + "ended_s": 1792426639.848324, + "input_tokens": 5776, + "output_tokens": 197, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013522, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 845, + 457 + ] + } + } + ], + "request_id": "req_011CgBtyX8MtrdRTbAMofbTS", + "stop_reason": "tool_use" + }, + { + "turn": 1, + "started_s": 1792426647.745773, + "ended_s": 1792426651.000404, + "input_tokens": 5776, + "output_tokens": 278, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.014332, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 845, + 457 + ] + } + } + ], + "request_id": "req_011CgBtzJZ6i2fkinhE4sBGL", + "stop_reason": "tool_use" + }, + { + "turn": 1, + "started_s": 1792426666.657011, + "ended_s": 1792426669.020239, + "input_tokens": 5776, + "output_tokens": 154, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013092, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 68, + 457 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgBu1hXVD5U1TNxBMYiFH", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792426669.251009, + "ended_s": 1792426672.4209569, + "input_tokens": 5832, + "output_tokens": 246, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.014124, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 68, + 457 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgBu1tVoS6Bnh2PeZAySo", + "stop_reason": "tool_use" + }, + { + "turn": 3, + "started_s": 1792426672.451106, + "ended_s": 1792426675.462482, + "input_tokens": 5876, + "output_tokens": 249, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.014242, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 68, + 457 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgBu286cQ9uBvBzYLyKLt", + "stop_reason": "tool_use" + }, + { + "turn": 1, + "started_s": 1792426677.7114182, + "ended_s": 1792426679.958957, + "input_tokens": 5776, + "output_tokens": 149, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013042, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 68, + 457 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgBu2WfAhuCVjVx2ZDeRf", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792426679.9910028, + "ended_s": 1792426683.1975172, + "input_tokens": 5832, + "output_tokens": 277, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.014434, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 68, + 457 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgBu2gSHbmnWiyzTKca6d", + "stop_reason": "tool_use" + }, + { + "turn": 3, + "started_s": 1792426683.227963, + "ended_s": 1792426686.1705668, + "input_tokens": 5876, + "output_tokens": 225, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.014002, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 68, + 457 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgBu2vRfA451eEJrtKo8h", + "stop_reason": "tool_use" + }, + { + "turn": 1, + "started_s": 1792426688.4510431, + "ended_s": 1792426690.7134569, + "input_tokens": 5776, + "output_tokens": 147, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013022, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 69, + 457 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgBu3Jbuaj3iFJGhpH3uQ", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792426690.745227, + "ended_s": 1792426694.087559, + "input_tokens": 5832, + "output_tokens": 247, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.014134, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 68, + 457 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgBu3UNn4xYjxjG4Aenxo", + "stop_reason": "tool_use" + }, + { + "turn": 3, + "started_s": 1792426694.1198301, + "ended_s": 1792426697.3117518, + "input_tokens": 5876, + "output_tokens": 275, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.014502, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 68, + 457 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgBu3iqg2CsLoCEYb2Zpr", + "stop_reason": "tool_use" + }, + { + "turn": 1, + "started_s": 1792426705.4009118, + "ended_s": 1792426711.2114382, + "input_tokens": 6150, + "output_tokens": 516, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.01746, + "actions": [], + "request_id": "req_011CgBu4Z4LhvrhFX9DS7saa", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792426717.5586638, + "ended_s": 1792426722.858582, + "input_tokens": 6150, + "output_tokens": 502, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.01732, + "actions": [], + "request_id": "req_011CgBu5SzynDuc8ZiBkjqPJ", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792426729.338506, + "ended_s": 1792426734.8588002, + "input_tokens": 6150, + "output_tokens": 478, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.01708, + "actions": [], + "request_id": "req_011CgBu6KT6GNkADDhkQgQKJ", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792426752.497749, + "ended_s": 1792426754.327499, + "input_tokens": 5741, + "output_tokens": 138, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.012862, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 129, + 235 + ] + } + } + ], + "request_id": "req_011CgBu82QUFngC6oeXFQVQc", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792426754.8020322, + "ended_s": 1792426759.026954, + "input_tokens": 5797, + "output_tokens": 439, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.015984, + "actions": [], + "request_id": "req_011CgBu8CHn886BoEMpQ51VQ", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792426769.3265831, + "ended_s": 1792426771.390105, + "input_tokens": 5741, + "output_tokens": 157, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013052, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 129, + 236 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgBu9GHv94hDNnnRhb2A6", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792426771.676735, + "ended_s": 1792426777.278472, + "input_tokens": 5797, + "output_tokens": 443, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.016024, + "actions": [], + "request_id": "req_011CgBu9SLR57QnARUSZLkBj", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792426787.551452, + "ended_s": 1792426789.690828, + "input_tokens": 5741, + "output_tokens": 123, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.012712, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 130, + 236 + ] + } + } + ], + "request_id": "req_011CgBuAcwQsBu9ZH5VoyqPw", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792426791.0171032, + "ended_s": 1792426795.804818, + "input_tokens": 5797, + "output_tokens": 507, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.016664, + "actions": [], + "request_id": "req_011CgBuAs8C6QL1qxGzeGF4S", + "stop_reason": "end_turn" + } + ] +} diff --git a/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/items.csv b/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/items.csv new file mode 100644 index 0000000000..44c90226d2 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/items.csv @@ -0,0 +1,67 @@ +mode,trial,stopped,route,repair_status,proposals,probe_passed,approved,resumed_outcome,readback,records,model_calls,cost_usd,wall_s,reason +none,1,False,none,not_needed,0,False,,,correct,1,0,0,4.5, +none,2,False,none,not_needed,0,False,,,correct,1,0,0,3.1, +none,3,False,none,not_needed,0,False,,,correct,1,0,0,3.0, +theme,1,False,none,not_needed,0,False,,,correct,1,0,0,3.0, +theme,2,False,none,not_needed,0,False,,,correct,1,0,0,3.6, +theme,3,False,none,not_needed,0,False,,,correct,1,0,0,3.8, +move,1,False,none,not_needed,0,False,,,correct,1,0,0,3.2, +move,2,False,none,not_needed,0,False,,,correct,1,0,0,3.3, +move,3,False,none,not_needed,0,False,,,correct,1,0,0,2.6, +font,1,True,repair,no_fix,0,False,,,safe_stop,0,0,0.013178,10.3,the agent gave up +font,2,True,repair,no_fix,0,False,,,safe_stop,0,0,0.013088,9.5,the agent gave up +font,3,True,repair,no_fix,0,False,,,safe_stop,0,0,0.012598,10.0,the agent gave up +zoom,1,True,repair,no_fix,0,False,,,safe_stop,0,0,0.014252,9.5,the agent gave up +zoom,2,True,repair,no_fix,0,False,,,safe_stop,0,0,0.013902,8.3,the agent gave up +zoom,3,True,repair,no_fix,0,False,,,safe_stop,0,0,0.013812,8.6,the agent gave up +slow,1,False,none,not_needed,0,False,,,correct,1,0,0.0,14.5, +slow,2,False,none,not_needed,0,False,,,correct,1,0,0.0,14.5, +slow,3,False,none,not_needed,0,False,,,correct,1,0,0.0,14.8, +grow,1,False,none,not_needed,0,False,,,correct,1,0,0.0,2.5, +grow,2,False,none,not_needed,0,False,,,correct,1,0,0.0,2.5, +grow,3,False,none,not_needed,0,False,,,correct,1,0,0.0,2.6, +sort,1,False,none,not_needed,0,False,,,correct,1,0,0.0,2.5, +sort,2,False,none,not_needed,0,False,,,correct,1,0,0.0,2.5, +sort,3,False,none,not_needed,0,False,,,correct,1,0,0.0,2.4, +rename,1,False,none,not_needed,0,False,,,correct,1,0,0.0,2.4, +rename,2,False,none,not_needed,0,False,,,correct,1,0,0.0,2.5, +rename,3,False,none,not_needed,0,False,,,correct,1,0,0.0,2.4, +relink,1,True,repair,resumed,1,True,person,VERIFIED,correct,1,1,0.012986,11.0, +relink,2,True,repair,resumed,1,True,person,VERIFIED,correct,1,1,0.012866,8.5, +relink,3,True,repair,resumed,1,True,person,VERIFIED,correct,1,1,0.012846,8.8, +redesign,1,True,repair,resumed,1,True,person,VERIFIED,correct,1,1,0.014512,12.2, +redesign,2,True,repair,resumed,1,True,person,VERIFIED,correct,1,1,0.013522,10.6, +redesign,3,True,repair,resumed,1,True,person,VERIFIED,correct,1,1,0.014332,11.0, +typelabel,1,False,none,not_needed,0,False,,,correct,1,0,0.0,2.7, +typelabel,2,False,none,not_needed,0,False,,,correct,1,0,0.0,2.5, +typelabel,3,False,none,not_needed,0,False,,,correct,1,0,0.0,2.5, +menu,1,True,repair,no_fix,3,False,,,safe_stop,0,3,0.041458,11.4,"step_005 saves, but the proposed control 'Actions' does not read as a save; refusing a patch that would point a save at another kind of control" +menu,2,True,repair,no_fix,3,False,,,safe_stop,0,3,0.041478,10.7,"step_005 saves, but the proposed control 'Actions' does not read as a save; refusing a patch that would point a save at another kind of control" +menu,3,True,repair,no_fix,3,False,,,safe_stop,0,3,0.041658,11.2,"step_005 saves, but the proposed control 'Actions' does not read as a save; refusing a patch that would point a save at another kind of control" +provider,1,True,repair,no_fix,0,False,,,safe_stop,0,0,0.01746,13.8,the agent gave up +provider,2,True,repair,no_fix,0,False,,,safe_stop,0,0,0.01732,11.6,the agent gave up +provider,3,True,repair,no_fix,0,False,,,safe_stop,0,0,0.01708,12.0,the agent gave up +reqfield,1,False,none,not_needed,0,False,,,correct,1,0,0.0,2.4, +reqfield,2,False,none,not_needed,0,False,,,correct,1,0,0.0,2.4, +reqfield,3,False,none,not_needed,0,False,,,correct,1,0,0.0,2.5, +notice,1,True,repair,no_fix,1,False,,,safe_stop,0,1,0.028846,16.8,the agent gave up +notice,2,True,repair,no_fix,1,False,,,safe_stop,0,1,0.029076,18.3,the agent gave up +notice,3,True,repair,no_fix,1,False,,,safe_stop,0,1,0.029376,18.5,the agent gave up +modal-once,1,False,none,not_needed,0,False,,,correct,1,0,0.0,4.2, +modal-once,2,False,none,not_needed,0,False,,,correct,1,0,0.0,3.3, +modal-once,3,False,none,not_needed,0,False,,,correct,1,0,0.0,3.1, +modal,1,False,none,not_needed,0,False,,,correct,1,0,0.0,2.6, +modal,2,False,none,not_needed,0,False,,,correct,1,0,0.0,2.5, +modal,3,False,none,not_needed,0,False,,,correct,1,0,0.0,2.6, +lookalike,1,False,none,not_needed,0,False,,,correct,1,0,0.0,2.4, +lookalike,2,False,none,not_needed,0,False,,,correct,1,0,0.0,2.4, +lookalike,3,False,none,not_needed,0,False,,,correct,1,0,0.0,2.4, +ambiguous,1,True,person,not_repairable,0,False,,,safe_stop,0,0,0.0,0.6,step_000 stopped before resolving its target for another reason; only a target the ladder could not find is repaired +ambiguous,2,True,person,not_repairable,0,False,,,safe_stop,0,0,0.0,0.2,step_000 stopped before resolving its target for another reason; only a target the ladder could not find is repaired +ambiguous,3,True,person,not_repairable,0,False,,,safe_stop,0,0,0.0,0.2,step_000 stopped before resolving its target for another reason; only a target the ladder could not find is repaired +missing,1,True,person,not_repairable,0,False,,,safe_stop,0,0,0.0,0.4,the run ended REJECTED_POLICY; self-repair handles only a halt before any effect +missing,2,True,person,not_repairable,0,False,,,safe_stop,0,0,0.0,0.4,the run ended REJECTED_POLICY; self-repair handles only a halt before any effect +missing,3,True,person,not_repairable,0,False,,,safe_stop,0,0,0.0,0.4,the run ended REJECTED_POLICY; self-repair handles only a halt before any effect +empty,1,True,person,not_repairable,0,False,,,safe_stop,0,0,0.0,1.2,the run ended REJECTED_POLICY; self-repair handles only a halt before any effect +empty,2,True,person,not_repairable,0,False,,,safe_stop,0,0,0.0,0.4,the run ended REJECTED_POLICY; self-repair handles only a halt before any effect +empty,3,True,person,not_repairable,0,False,,,safe_stop,0,0,0.0,0.4,the run ended REJECTED_POLICY; self-repair handles only a halt before any effect diff --git a/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/ledger.jsonl b/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/ledger.jsonl new file mode 100644 index 0000000000..e53c5db44e --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/ledger.jsonl @@ -0,0 +1,30 @@ +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:14:42+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_002", "labels": {"app": "mockmed", "drift": "font", "trial": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 10.312, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/font/trial-1/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:14:51+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_002", "labels": {"app": "mockmed", "drift": "font", "trial": "2"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 9.471, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/font/trial-2/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:15:01+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_002", "labels": {"app": "mockmed", "drift": "font", "trial": "3"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 9.997, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/font/trial-3/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:15:11+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "zoom", "trial": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 9.475, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/zoom/trial-1/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:15:19+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "zoom", "trial": "2"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 8.295, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/zoom/trial-2/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:15:28+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "zoom", "trial": "3"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 8.649, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/zoom/trial-3/item", "scope": "item", "workflow": "triage"} +{"approved_by": "decision_portal:person", "approved_via": "person", "at": "2026-10-19T16:16:45+00:00", "cases_passed": true, "est_model_cost_usd": 0.012986, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "relink", "trial": "1"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 1, "person_wait_s": 0.005, "probe_passed": true, "proposed": 1, "released_by": null, "repair_reason": "", "repair_s": 10.991, "repair_status": "resumed", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/relink/trial-1/item", "scope": "item", "workflow": "triage"} +{"approved_by": "decision_portal:person", "approved_via": "person", "at": "2026-10-19T16:16:53+00:00", "cases_passed": true, "est_model_cost_usd": 0.012866, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "relink", "trial": "2"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 1, "person_wait_s": 0.003, "probe_passed": true, "proposed": 1, "released_by": null, "repair_reason": "", "repair_s": 8.471, "repair_status": "resumed", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/relink/trial-2/item", "scope": "item", "workflow": "triage"} +{"approved_by": "decision_portal:person", "approved_via": "person", "at": "2026-10-19T16:17:02+00:00", "cases_passed": true, "est_model_cost_usd": 0.012846, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "relink", "trial": "3"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 1, "person_wait_s": 0.003, "probe_passed": true, "proposed": 1, "released_by": null, "repair_reason": "", "repair_s": 8.75, "repair_status": "resumed", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/relink/trial-3/item", "scope": "item", "workflow": "triage"} +{"approved_by": "decision_portal:person", "approved_via": "person", "at": "2026-10-19T16:17:14+00:00", "cases_passed": true, "est_model_cost_usd": 0.014512, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "redesign", "trial": "1"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 1, "person_wait_s": 0.003, "probe_passed": true, "proposed": 1, "released_by": null, "repair_reason": "", "repair_s": 12.176, "repair_status": "resumed", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/redesign/trial-1/item", "scope": "item", "workflow": "triage"} +{"approved_by": "decision_portal:person", "approved_via": "person", "at": "2026-10-19T16:17:25+00:00", "cases_passed": true, "est_model_cost_usd": 0.013522, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "redesign", "trial": "2"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 1, "person_wait_s": 0.004, "probe_passed": true, "proposed": 1, "released_by": null, "repair_reason": "", "repair_s": 10.562, "repair_status": "resumed", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/redesign/trial-2/item", "scope": "item", "workflow": "triage"} +{"approved_by": "decision_portal:person", "approved_via": "person", "at": "2026-10-19T16:17:36+00:00", "cases_passed": true, "est_model_cost_usd": 0.014332, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "redesign", "trial": "3"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 1, "person_wait_s": 0.004, "probe_passed": true, "proposed": 1, "released_by": null, "repair_reason": "", "repair_s": 11.044, "repair_status": "resumed", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/redesign/trial-3/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:17:55+00:00", "cases_passed": false, "est_model_cost_usd": 0.041458, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "menu", "trial": "1"}, "model_calls": 3, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 3, "released_by": null, "repair_reason": "step_005 saves, but the proposed control 'Actions' does not read as a save; refusing a patch that would point a save at another kind of control", "repair_s": 11.426, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/menu/trial-1/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:18:06+00:00", "cases_passed": false, "est_model_cost_usd": 0.041478, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "menu", "trial": "2"}, "model_calls": 3, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 3, "released_by": null, "repair_reason": "step_005 saves, but the proposed control 'Actions' does not read as a save; refusing a patch that would point a save at another kind of control", "repair_s": 10.702, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/menu/trial-2/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:18:17+00:00", "cases_passed": false, "est_model_cost_usd": 0.041658, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "menu", "trial": "3"}, "model_calls": 3, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 3, "released_by": null, "repair_reason": "step_005 saves, but the proposed control 'Actions' does not read as a save; refusing a patch that would point a save at another kind of control", "repair_s": 11.189, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/menu/trial-3/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:18:31+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "provider", "trial": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 13.843, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/provider/trial-1/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:18:42+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "provider", "trial": "2"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 11.648, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/provider/trial-2/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:18:54+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "provider", "trial": "3"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 11.983, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/provider/trial-3/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:19:19+00:00", "cases_passed": false, "est_model_cost_usd": 0.012862, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "notice", "trial": "1"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 16.801, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/notice/trial-1/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:19:37+00:00", "cases_passed": false, "est_model_cost_usd": 0.013052, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "notice", "trial": "2"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 18.251, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/notice/trial-2/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:19:55+00:00", "cases_passed": false, "est_model_cost_usd": 0.012712, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "notice", "trial": "3"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 18.523, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/notice/trial-3/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:20:21+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "ambiguous", "trial": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "step_000 stopped before resolving its target for another reason; only a target the ladder could not find is repaired", "repair_s": 0.628, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "person", "route_reason": "step_000 stopped before resolving its target for another reason; only a target the ladder could not find is repaired", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/ambiguous/trial-1/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:20:22+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "ambiguous", "trial": "2"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "step_000 stopped before resolving its target for another reason; only a target the ladder could not find is repaired", "repair_s": 0.242, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "person", "route_reason": "step_000 stopped before resolving its target for another reason; only a target the ladder could not find is repaired", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/ambiguous/trial-2/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:20:22+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "ambiguous", "trial": "3"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "step_000 stopped before resolving its target for another reason; only a target the ladder could not find is repaired", "repair_s": 0.238, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "person", "route_reason": "step_000 stopped before resolving its target for another reason; only a target the ladder could not find is repaired", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/ambiguous/trial-3/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:20:22+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "missing", "trial": "1"}, "model_calls": 0, "outcome": "REJECTED_POLICY", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the run ended REJECTED_POLICY; self-repair handles only a halt before any effect", "repair_s": 0.416, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "person", "route_reason": "the run ended REJECTED_POLICY", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/missing/trial-1/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:20:23+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "missing", "trial": "2"}, "model_calls": 0, "outcome": "REJECTED_POLICY", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the run ended REJECTED_POLICY; self-repair handles only a halt before any effect", "repair_s": 0.411, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "person", "route_reason": "the run ended REJECTED_POLICY", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/missing/trial-2/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:20:23+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "missing", "trial": "3"}, "model_calls": 0, "outcome": "REJECTED_POLICY", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the run ended REJECTED_POLICY; self-repair handles only a halt before any effect", "repair_s": 0.402, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "person", "route_reason": "the run ended REJECTED_POLICY", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/missing/trial-3/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:20:24+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "empty", "trial": "1"}, "model_calls": 0, "outcome": "REJECTED_POLICY", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the run ended REJECTED_POLICY; self-repair handles only a halt before any effect", "repair_s": 1.237, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "person", "route_reason": "the run ended REJECTED_POLICY", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/empty/trial-1/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:20:25+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "empty", "trial": "2"}, "model_calls": 0, "outcome": "REJECTED_POLICY", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the run ended REJECTED_POLICY; self-repair handles only a halt before any effect", "repair_s": 0.357, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "person", "route_reason": "the run ended REJECTED_POLICY", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/empty/trial-2/item", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:20:25+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_000", "labels": {"app": "mockmed", "drift": "empty", "trial": "3"}, "model_calls": 0, "outcome": "REJECTED_POLICY", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the run ended REJECTED_POLICY; self-repair handles only a halt before any effect", "repair_s": 0.397, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "person", "route_reason": "the run ended REJECTED_POLICY", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a/empty/trial-3/item", "scope": "item", "workflow": "triage"} diff --git a/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/meta.json b/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/meta.json new file mode 100644 index 0000000000..5a4363be69 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/meta.json @@ -0,0 +1,39 @@ +{ + "written_at": "2026-10-19T12:13:54-0400", + "machine": "arm64 Darwin 24.6.0", + "python": "3.12.7", + "engine_git_head": "152639a0e4d8b08059a05ff7341d89d632ad9f22", + "app": "MockMed (openadapt_flow.mockmed.fault_server), synthetic data", + "program": "tutorial triage bundle, recorded and compiled fresh", + "loop": "AutoRepair over SelfRepair (bridge up to 3), no standard-change policy", + "agent": "agent:anthropic:claude-sonnet-5-5:auto-repair-benchmark", + "approver": "benchmark:scripted-approver (stands in for a person; approves all)", + "fault_probe": "drift=,missing", + "screen_access": "allow_unscrubbed (synthetic data only)", + "trials": 3, + "modes": { + "none": "no drift (control)", + "theme": "colours change", + "move": "layout shifts", + "font": "font changes", + "zoom": "page zoom", + "slow": "screens render 4 s late", + "grow": "four new referrals above the target", + "sort": "referral queue sorted by name", + "rename": "Open -> View, Save Encounter -> Submit Encounter", + "relink": "New Encounter -> Start visit, new id, moved", + "redesign": "Save -> Submit to chart, new id, moved; decoy Submit feedback", + "typelabel": "Triage -> Triage Assessment, segments swapped", + "menu": "Save moved into Actions > Billing > Submit encounter", + "provider": "new required Ordering provider screen before the form", + "reqfield": "new required Acuity field on the form", + "notice": "an interstitial notice replaces the tasks screen once", + "modal-once": "a survey modal appears once", + "modal": "a blocking modal replaces the saved confirmation", + "lookalike": "a look-alike referral above the target", + "ambiguous": "the target referral appears twice", + "missing": "the target referral is gone", + "empty": "no referrals" + }, + "engine_note": "engine_git_head is the commit checked out; the run used the auto-repair loop as committed next in 64fbe45fd (then uncommitted). summary.json was rewritten from ledger.jsonl after summarize stopped counting not_repairable halts as repair runs." +} diff --git a/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/summary.json b/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/summary.json new file mode 100644 index 0000000000..db64d883ab --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/mockmed-2026-10-19-a/summary.json @@ -0,0 +1,797 @@ +{ + "overall": { + "stops": 30, + "to_repair": 21, + "to_reconciliation": 0, + "to_person": 9, + "auto_repair_off": 0, + "repair_runs": 21, + "repairs_proposed": 12, + "proposals": 18, + "probe_passed": 6, + "approved": 6, + "approved_by_person": 6, + "approved_by_standard_change": 0, + "resumed_items": 6, + "resumed_not_verified": 0, + "repair_success_rate": 0.2857, + "person_decisions": 6, + "person_wait_s_total": 0.022, + "model_calls": 18, + "est_model_cost_usd": 0.244284, + "repair_status_counts": { + "no_fix": 15, + "resumed": 6 + } + }, + "trials": 66, + "readback": { + "correct": 42, + "safe_stop": 24, + "wrong": 0 + }, + "by_mode": { + "none": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "theme": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "move": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "font": { + "trials": 3, + "stopped": 3, + "readback": { + "correct": 0, + "safe_stop": 3, + "wrong": 0 + }, + "repair_status": [ + "no_fix" + ], + "funnel": { + "stops": 3, + "to_repair": 3, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 3, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": 0.0, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0.0, + "repair_status_counts": { + "no_fix": 3 + } + } + }, + "zoom": { + "trials": 3, + "stopped": 3, + "readback": { + "correct": 0, + "safe_stop": 3, + "wrong": 0 + }, + "repair_status": [ + "no_fix" + ], + "funnel": { + "stops": 3, + "to_repair": 3, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 3, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": 0.0, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0.0, + "repair_status_counts": { + "no_fix": 3 + } + } + }, + "slow": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "grow": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "sort": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "rename": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "relink": { + "trials": 3, + "stopped": 3, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "resumed" + ], + "funnel": { + "stops": 3, + "to_repair": 3, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 3, + "repairs_proposed": 3, + "proposals": 3, + "probe_passed": 3, + "approved": 3, + "approved_by_person": 3, + "approved_by_standard_change": 0, + "resumed_items": 3, + "resumed_not_verified": 0, + "repair_success_rate": 1.0, + "person_decisions": 3, + "person_wait_s_total": 0.011, + "model_calls": 3, + "est_model_cost_usd": 0.038698, + "repair_status_counts": { + "resumed": 3 + } + } + }, + "redesign": { + "trials": 3, + "stopped": 3, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "resumed" + ], + "funnel": { + "stops": 3, + "to_repair": 3, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 3, + "repairs_proposed": 3, + "proposals": 3, + "probe_passed": 3, + "approved": 3, + "approved_by_person": 3, + "approved_by_standard_change": 0, + "resumed_items": 3, + "resumed_not_verified": 0, + "repair_success_rate": 1.0, + "person_decisions": 3, + "person_wait_s_total": 0.011, + "model_calls": 3, + "est_model_cost_usd": 0.042366, + "repair_status_counts": { + "resumed": 3 + } + } + }, + "typelabel": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "menu": { + "trials": 3, + "stopped": 3, + "readback": { + "correct": 0, + "safe_stop": 3, + "wrong": 0 + }, + "repair_status": [ + "no_fix" + ], + "funnel": { + "stops": 3, + "to_repair": 3, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 3, + "repairs_proposed": 3, + "proposals": 9, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": 0.0, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 9, + "est_model_cost_usd": 0.124594, + "repair_status_counts": { + "no_fix": 3 + } + } + }, + "provider": { + "trials": 3, + "stopped": 3, + "readback": { + "correct": 0, + "safe_stop": 3, + "wrong": 0 + }, + "repair_status": [ + "no_fix" + ], + "funnel": { + "stops": 3, + "to_repair": 3, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 3, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": 0.0, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0.0, + "repair_status_counts": { + "no_fix": 3 + } + } + }, + "reqfield": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "notice": { + "trials": 3, + "stopped": 3, + "readback": { + "correct": 0, + "safe_stop": 3, + "wrong": 0 + }, + "repair_status": [ + "no_fix" + ], + "funnel": { + "stops": 3, + "to_repair": 3, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 3, + "repairs_proposed": 3, + "proposals": 3, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": 0.0, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 3, + "est_model_cost_usd": 0.038626, + "repair_status_counts": { + "no_fix": 3 + } + } + }, + "modal-once": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "modal": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "lookalike": { + "trials": 3, + "stopped": 0, + "readback": { + "correct": 3, + "safe_stop": 0, + "wrong": 0 + }, + "repair_status": [ + "not_needed" + ], + "funnel": { + "stops": 0, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0, + "repair_status_counts": {} + } + }, + "ambiguous": { + "trials": 3, + "stopped": 3, + "readback": { + "correct": 0, + "safe_stop": 3, + "wrong": 0 + }, + "repair_status": [ + "not_repairable" + ], + "funnel": { + "stops": 3, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 3, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0.0, + "repair_status_counts": {} + } + }, + "missing": { + "trials": 3, + "stopped": 3, + "readback": { + "correct": 0, + "safe_stop": 3, + "wrong": 0 + }, + "repair_status": [ + "not_repairable" + ], + "funnel": { + "stops": 3, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 3, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0.0, + "repair_status_counts": {} + } + }, + "empty": { + "trials": 3, + "stopped": 3, + "readback": { + "correct": 0, + "safe_stop": 3, + "wrong": 0 + }, + "repair_status": [ + "not_repairable" + ], + "funnel": { + "stops": 3, + "to_repair": 0, + "to_reconciliation": 0, + "to_person": 3, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0.0, + "repair_status_counts": {} + } + } + } +} diff --git a/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/calls.json b/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/calls.json new file mode 100644 index 0000000000..148e11bb5a --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/calls.json @@ -0,0 +1,52 @@ +{ + "model": "claude-sonnet-5-5", + "cost_usd": 0.025158, + "calls": [ + { + "turn": 1, + "started_s": 1792427468.994693, + "ended_s": 1792427471.11069, + "input_tokens": 5768, + "output_tokens": 139, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.012926, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 860, + 159 + ] + } + } + ], + "request_id": "req_011CgBv2r4MJj9zTeSXYVcTy", + "stop_reason": "tool_use" + }, + { + "turn": 1, + "started_s": 1792427511.481853, + "ended_s": 1792427516.054673, + "input_tokens": 5776, + "output_tokens": 68, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.012232, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 845, + 457 + ] + } + } + ], + "request_id": "req_011CgBv5ySSPhV4GQPEQpRhE", + "stop_reason": "tool_use" + } + ] +} diff --git a/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/items.csv b/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/items.csv new file mode 100644 index 0000000000..ed237644b6 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/items.csv @@ -0,0 +1,11 @@ +mode,item,status,runs,readback,queue_status,wall_s +relink,item-1,verified,2,correct,done,33.2 +relink,item-2,verified,2,correct,done,33.2 +relink,item-3,verified,2,correct,done,33.2 +relink,item-4,verified,2,correct,done,33.2 +relink,item-5,verified,2,correct,done,33.2 +redesign,item-1,verified,2,correct,done,46.8 +redesign,item-2,verified,2,correct,done,46.8 +redesign,item-3,verified,2,correct,done,46.8 +redesign,item-4,verified,2,correct,done,46.8 +redesign,item-5,verified,2,correct,done,46.8 diff --git a/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/ledger.jsonl b/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/ledger.jsonl new file mode 100644 index 0000000000..acfc30c52b --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/ledger.jsonl @@ -0,0 +1,12 @@ +{"approved_by": "decision_portal:person", "approved_via": "person", "at": "2026-10-19T16:31:31+00:00", "cases_passed": true, "est_model_cost_usd": 0.012926, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "relink", "worklist": "1"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 1, "person_wait_s": null, "probe_passed": true, "proposed": 1, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": "resumed", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/relink/queue/repair-1/self-repair/resume", "scope": "test_record", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:31:31+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "relink", "worklist": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": "b13e845e304606e7", "repair_reason": "", "repair_s": 0.0, "repair_status": "released", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/relink/queue/items/item-1/run-1", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:31:31+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "relink", "worklist": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": "b13e845e304606e7", "repair_reason": "", "repair_s": 0.0, "repair_status": "released", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/relink/queue/items/item-2/run-1", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:31:31+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "relink", "worklist": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": "b13e845e304606e7", "repair_reason": "", "repair_s": 0.0, "repair_status": "released", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/relink/queue/items/item-3/run-1", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:31:31+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "relink", "worklist": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": "b13e845e304606e7", "repair_reason": "", "repair_s": 0.0, "repair_status": "released", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/relink/queue/items/item-4/run-1", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:31:31+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_001", "labels": {"app": "mockmed", "drift": "relink", "worklist": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": "b13e845e304606e7", "repair_reason": "", "repair_s": 0.0, "repair_status": "released", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/relink/queue/items/item-5/run-1", "scope": "item", "workflow": "triage"} +{"approved_by": "decision_portal:person", "approved_via": "person", "at": "2026-10-19T16:32:18+00:00", "cases_passed": true, "est_model_cost_usd": 0.012232, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "redesign", "worklist": "1"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 1, "person_wait_s": null, "probe_passed": true, "proposed": 1, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": "resumed", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/redesign/queue/repair-1/self-repair/resume", "scope": "test_record", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:32:18+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "redesign", "worklist": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": "7d6c6d3abfa96b8d", "repair_reason": "", "repair_s": 0.0, "repair_status": "released", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/redesign/queue/items/item-1/run-1", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:32:18+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "redesign", "worklist": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": "7d6c6d3abfa96b8d", "repair_reason": "", "repair_s": 0.0, "repair_status": "released", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/redesign/queue/items/item-2/run-1", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:32:18+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "redesign", "worklist": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": "7d6c6d3abfa96b8d", "repair_reason": "", "repair_s": 0.0, "repair_status": "released", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/redesign/queue/items/item-3/run-1", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:32:18+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "redesign", "worklist": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": "7d6c6d3abfa96b8d", "repair_reason": "", "repair_s": 0.0, "repair_status": "released", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/redesign/queue/items/item-4/run-1", "scope": "item", "workflow": "triage"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:32:18+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_005", "labels": {"app": "mockmed", "drift": "redesign", "worklist": "1"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": "7d6c6d3abfa96b8d", "repair_reason": "", "repair_s": 0.0, "repair_status": "released", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/mockmed-worklist-2026-10-19-a/redesign/queue/items/item-5/run-1", "scope": "item", "workflow": "triage"} diff --git a/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/meta.json b/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/meta.json new file mode 100644 index 0000000000..be3d77cd03 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/meta.json @@ -0,0 +1,15 @@ +{ + "written_at": "2026-10-19T12:30:49-0400", + "engine_git_head": "64fbe45fde67b31e02c8ed36d87a2778c272ff1d", + "app": "MockMed, synthetic data", + "loop": "WorklistQueue: pre-flight, items, self-repair on the test record", + "drift_ships": "after the pre-flight, before item 1", + "items_per_mode": 5, + "agent": "agent:anthropic:claude-sonnet-5-5:auto-repair-benchmark", + "approver": "benchmark:scripted-approver (stands in for a person; approves all)", + "fault_probe": "drift=,missing", + "modes": [ + "relink", + "redesign" + ] +} diff --git a/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/summary.json b/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/summary.json new file mode 100644 index 0000000000..de2266888a --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/summary.json @@ -0,0 +1,76 @@ +{ + "overall": { + "stops": 10, + "to_repair": 10, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 2, + "repairs_proposed": 2, + "proposals": 2, + "probe_passed": 2, + "approved": 2, + "approved_by_person": 2, + "approved_by_standard_change": 0, + "resumed_items": 10, + "resumed_not_verified": 0, + "repair_success_rate": 1.0, + "person_decisions": 2, + "person_wait_s_total": 0, + "model_calls": 2, + "est_model_cost_usd": 0.025158, + "repair_status_counts": { + "resumed": 2 + } + }, + "by_mode": { + "relink": { + "stops": 5, + "to_repair": 5, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 1, + "repairs_proposed": 1, + "proposals": 1, + "probe_passed": 1, + "approved": 1, + "approved_by_person": 1, + "approved_by_standard_change": 0, + "resumed_items": 5, + "resumed_not_verified": 0, + "repair_success_rate": 1.0, + "person_decisions": 1, + "person_wait_s_total": 0, + "model_calls": 1, + "est_model_cost_usd": 0.012926, + "repair_status_counts": { + "resumed": 1 + } + }, + "redesign": { + "stops": 5, + "to_repair": 5, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 1, + "repairs_proposed": 1, + "proposals": 1, + "probe_passed": 1, + "approved": 1, + "approved_by_person": 1, + "approved_by_standard_change": 0, + "resumed_items": 5, + "resumed_not_verified": 0, + "repair_success_rate": 1.0, + "person_decisions": 1, + "person_wait_s_total": 0, + "model_calls": 1, + "est_model_cost_usd": 0.012232, + "repair_status_counts": { + "resumed": 1 + } + } + } +} diff --git a/packages/flow/benchmark/auto_repair/results/openemr-1347/ledger.jsonl b/packages/flow/benchmark/auto_repair/results/openemr-1347/ledger.jsonl new file mode 100644 index 0000000000..bbb0d447f0 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openemr-1347/ledger.jsonl @@ -0,0 +1,8 @@ +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:12:59+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_039", "labels": {"app": "openemr", "run": "fast1"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": null, "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically", "run_dir": "fast1/pool/items/00026", "scope": "item", "workflow": "openemr-create-synthetic-patient"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:12:59+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_040", "labels": {"app": "openemr", "run": "fast2"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": null, "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically", "run_dir": "fast2/pool/items/00016", "scope": "item", "workflow": "openemr-create-synthetic-patient"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:12:59+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_040", "labels": {"app": "openemr", "run": "par8b"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": null, "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically", "run_dir": "par8b/pool/items/00002", "scope": "item", "workflow": "openemr-create-synthetic-patient"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:12:59+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_040", "labels": {"app": "openemr", "run": "par8b"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": null, "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically", "run_dir": "par8b/pool/items/00007", "scope": "item", "workflow": "openemr-create-synthetic-patient"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:12:59+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_040", "labels": {"app": "openemr", "run": "par8w1b"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": null, "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically", "run_dir": "par8w1b/pool/items/00009", "scope": "item", "workflow": "openemr-create-synthetic-patient"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:12:59+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_040", "labels": {"app": "openemr", "run": "par8w1b"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": null, "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically", "run_dir": "par8w1b/pool/items/00020", "scope": "item", "workflow": "openemr-create-synthetic-patient"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:12:59+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_039", "labels": {"app": "openemr", "run": "std1"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": null, "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically", "run_dir": "std1/pool/items/00026", "scope": "item", "workflow": "openemr-create-synthetic-patient"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:12:59+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_040", "labels": {"app": "openemr", "run": "std2"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": null, "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically", "run_dir": "std2/pool/items/00008", "scope": "item", "workflow": "openemr-create-synthetic-patient"} diff --git a/packages/flow/benchmark/auto_repair/results/openemr-1347/stops.csv b/packages/flow/benchmark/auto_repair/results/openemr-1347/stops.csv new file mode 100644 index 0000000000..151afd1a36 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openemr-1347/stops.csv @@ -0,0 +1,9 @@ +run,outcome,halted_step,halt_kind,identity,route,if_routed_to_repair +fast1/pool/items/00026,RECONCILIATION_REQUIRED,step_039,Postconditions failed,,reconciliation,"input reached step_039, a step that saves; never repaired" +fast2/pool/items/00016,RECONCILIATION_REQUIRED,step_040,Identity check failed,mismatch,reconciliation,"input reached step_039, a step that saves; never repaired" +par8b/pool/items/00002,RECONCILIATION_REQUIRED,step_040,Postconditions failed,verified,reconciliation,"input reached step_039, a step that saves; never repaired" +par8b/pool/items/00007,RECONCILIATION_REQUIRED,step_040,Identity check failed,mismatch,reconciliation,"input reached step_039, a step that saves; never repaired" +par8w1b/pool/items/00009,RECONCILIATION_REQUIRED,step_040,Postconditions failed,verified,reconciliation,"input reached step_039, a step that saves; never repaired" +par8w1b/pool/items/00020,RECONCILIATION_REQUIRED,step_040,Postconditions failed,verified,reconciliation,"input reached step_039, a step that saves; never repaired" +std1/pool/items/00026,RECONCILIATION_REQUIRED,step_039,Postconditions failed,,reconciliation,"input reached step_039, a step that saves; never repaired" +std2/pool/items/00008,RECONCILIATION_REQUIRED,step_040,Identity check failed,mismatch,reconciliation,"input reached step_039, a step that saves; never repaired" diff --git a/packages/flow/benchmark/auto_repair/results/openemr-1347/summary.json b/packages/flow/benchmark/auto_repair/results/openemr-1347/summary.json new file mode 100644 index 0000000000..d2111e5583 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openemr-1347/summary.json @@ -0,0 +1,28 @@ +{ + "app": "openemr", + "note": "Throughput runs of 2026-10-19 (#1296 harness, engine b82131e5), the runs #1347 reports", + "bundle_name": "openemr-create-synthetic-patient", + "reports_read": 110, + "funnel": { + "stops": 8, + "to_repair": 0, + "to_reconciliation": 8, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0.0, + "repair_status_counts": {} + } +} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-1331-before/ledger.jsonl b/packages/flow/benchmark/auto_repair/results/openimis-1331-before/ledger.jsonl new file mode 100644 index 0000000000..2c4dbad318 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-1331-before/ledger.jsonl @@ -0,0 +1,3 @@ +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:14:34+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_014", "labels": {"app": "openimis", "run": "item-050-1"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": null, "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically", "run_dir": "item-050-1/run", "scope": "item", "workflow": "openimis.claim_entry"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:14:34+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_014", "labels": {"app": "openimis", "run": "item-050-2"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": null, "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically", "run_dir": "item-050-2/run", "scope": "item", "workflow": "openimis.claim_entry"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T16:14:34+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_014", "labels": {"app": "openimis", "run": "item-050-3"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "", "repair_s": 0.0, "repair_status": null, "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically", "run_dir": "item-050-3/run", "scope": "item", "workflow": "openimis.claim_entry"} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-1331-before/stops.csv b/packages/flow/benchmark/auto_repair/results/openimis-1331-before/stops.csv new file mode 100644 index 0000000000..1be783c468 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-1331-before/stops.csv @@ -0,0 +1,4 @@ +run,outcome,halted_step,halt_kind,identity,route,if_routed_to_repair +item-050-1/run,RECONCILIATION_REQUIRED,step_014,"Step 'step_014' (scroll by (0, -600)): closed-loop scroll ex",,reconciliation,step_014 is not a click on an on-screen target +item-050-2/run,RECONCILIATION_REQUIRED,step_014,"Step 'step_014' (scroll by (0, -600)): closed-loop scroll ex",,reconciliation,step_014 is not a click on an on-screen target +item-050-3/run,RECONCILIATION_REQUIRED,step_014,"Step 'step_014' (scroll by (0, -600)): closed-loop scroll ex",,reconciliation,step_014 is not a click on an on-screen target diff --git a/packages/flow/benchmark/auto_repair/results/openimis-1331-before/summary.json b/packages/flow/benchmark/auto_repair/results/openimis-1331-before/summary.json new file mode 100644 index 0000000000..6cefb554b9 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-1331-before/summary.json @@ -0,0 +1,28 @@ +{ + "app": "openimis", + "note": "Scoreboard program_repair 2026-10-19-a: items 45-50, the relabelled Save (#1331) at item 50, 3 attempts; engine before #1337 and before the risk review", + "bundle_name": "openimis.claim_entry", + "reports_read": 8, + "funnel": { + "stops": 3, + "to_repair": 0, + "to_reconciliation": 3, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0.0, + "repair_status_counts": {} + } +} diff --git a/packages/flow/benchmark/auto_repair/route_recorded_stops.py b/packages/flow/benchmark/auto_repair/route_recorded_stops.py new file mode 100644 index 0000000000..2cf7c7ad22 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/route_recorded_stops.py @@ -0,0 +1,136 @@ +"""Route recorded stops through the default repair loop's rules, offline. + + python benchmark/auto_repair/route_recorded_stops.py \\ + --bundle BUNDLE --runs-root ROOT --out results/openemr-1347 --app openemr + +For every ``report.json`` under ``ROOT`` that halted, this applies what +:class:`openadapt_flow.repair.auto_repair.AutoRepair` would do with it: +:func:`~openadapt_flow.repair.auto_repair.route_stop` (repair, reconciliation +or a person) and, for a stop routed to repair, self-repair's own check of +whether it handles that halt. It calls no agent and opens no app: a stop that +is never handed to repair needs neither, and the answer for one that is +depends only on the halted report and the bundle. + +It also records what self-repair's check would say if the stop had been +routed to repair (``if_routed``), so a reader can see why the stop is or is +not repairable, not only where it went. + +Outputs: ``ledger.jsonl`` (the loop's own row per stop), ``stops.csv`` and +``summary.json``. Rows name the run and item directory relative to +``ROOT``; no record values are copied. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--bundle", required=True) + parser.add_argument("--runs-root", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--app", required=True) + parser.add_argument("--note", default="") + args = parser.parse_args() + + from openadapt_flow.ir import RunReport, Workflow + from openadapt_flow.repair.auto_repair import ( + RepairLedger, + ledger_row, + route_stop, + summarize, + ) + from openadapt_flow.repair.self_repair import _repairable + + workflow = Workflow.load(Path(args.bundle)) + root = Path(args.runs_root) + out = Path(args.out) + if not out.is_absolute(): + out = HERE / out + out.mkdir(parents=True, exist_ok=True) + ledger = RepairLedger(out) + ledger.path = out / "ledger.jsonl" + ledger.path.unlink(missing_ok=True) + rows = [] + reports = sorted(root.rglob("report.json")) + for path in reports: + report = RunReport.model_validate_json(path.read_text()) + if report.halt is None or report.success: + continue + name = str(path.parent.relative_to(root)) + route, why = route_stop(report) + verdict = _repairable(report, workflow) + if_routed = ( + verdict + if isinstance(verdict, str) + else f"repairable: {verdict[0].id} ({verdict[1]})" + ) + if route == "repair" and isinstance(verdict, str): + route, why = "person", verdict + row = ledger_row( + report, + name, + route=route, + route_reason=why, + labels={"app": args.app, "run": name.split("/")[0]}, + ) + ledger.append(row) + halted = next( + (r for r in reversed(report.results) if not r.ok and not r.skipped), None + ) + rows.append( + { + "run": name, + "outcome": report.transaction_outcome, + "halted_step": report.halt.state_id, + "halt_kind": (report.halt.reason or "").split(" for step")[0][:60], + "identity": ( + halted.identity.status + if halted is not None and halted.identity is not None + else "" + ), + "route": route, + "if_routed_to_repair": if_routed[:200], + } + ) + with (out / "stops.csv").open("w", newline="") as f: + writer = csv.DictWriter( + f, + fieldnames=[ + "run", + "outcome", + "halted_step", + "halt_kind", + "identity", + "route", + "if_routed_to_repair", + ], + ) + writer.writeheader() + writer.writerows(rows) + (out / "summary.json").write_text( + json.dumps( + { + "app": args.app, + "note": args.note, + "bundle_name": workflow.name, + "reports_read": len(reports), + "funnel": summarize(ledger.rows()), + }, + indent=2, + ) + + "\n" + ) + print(json.dumps(summarize(ledger.rows()), indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/flow/docs/LIMITS.md b/packages/flow/docs/LIMITS.md index 90f33aa90d..df15f1368f 100644 --- a/packages/flow/docs/LIMITS.md +++ b/packages/flow/docs/LIMITS.md @@ -21,7 +21,7 @@ For the current evidence behind each capability claim, see | Browser record, compile, and replay | **Required CI and counted attach trials** | The launched-browser path runs end to end in automated and clean-environment tests. The attached-browser path reuses the same recorder and passes 3 real Chromium record-and-compile trials with secret exclusion and external-browser survival checks. A live resize case also compiles actions from two viewport and device-scale baselines. | It is not evidence for every site, browser extension, authentication flow, or long-running production workload. Attach mode requires a loopback CDP endpoint and one same-origin Chromium tab. An action that overlaps a resize is refused because its pre-action coordinate evidence is not exact. The custom Capture extension remains a prototype and is not a direct replay path. | | Healthy replay with zero model calls | **Required CI** | A run that resolves from retained evidence can execute without a language or vision model; the run report counts model calls. | Zero model calls does not mean zero network traffic. The target application, hosted control plane, remote backend, or effect verifier may still use the network. | | Deterministic re-resolution | **Required CI** | Bounded visual or structural drift can be resolved through non-model evidence and recorded as a reviewable change. | It is not general adaptation to a redesigned workflow, changed business rules, or missing evidence. | -| AI-assisted repair | **Required CI contracts; deployment evidence required** | An explicitly enabled model can propose a target or interpret a changed screen. Existing runtime checks still apply. | A model proposal is not authorization, proof of identity, or proof that a business transaction succeeded. | +| AI-assisted repair | **Required CI contracts; measured on synthetic apps; deployment evidence required** | An explicitly enabled model can propose a target or interpret a changed screen. Existing runtime checks still apply. Once a deployment names a repair agent, every stop that wrote nothing goes to it by default, and a person approves each fix unless a signed policy pre-approves its class. On MockMed's 22 drift modes, 6 of 21 stops that reached repair resumed and verified, with 0 wrong saves in 66 trials ([benchmark/auto_repair](../benchmark/auto_repair/README.md)). | A model proposal is not authorization, proof of identity, or proof that a business transaction succeeded. A stop after a possible write is never repaired automatically. The measured rate is for one agent on one synthetic app; it is not a field repair rate. | | Human teaching and resume | **Required CI contracts; field evidence required** | A halt can produce evidence for an operator correction, guarded promotion, and durable resume. | Field recovery time, broad authoring UX, enterprise identity integration, and non-repudiation are not established. | | Windows UI Automation | **Counted task acceptance** | The native UIA backend completed 3/3 exact WinForms effects and 3/3 refusals for both stale and ambiguous targets. | Acceptance is bound to the tested workflow and Windows environment; each application retains its own identity, effect, and version qualification. | | Native macOS automation | **Counted task acceptance** | The AX/AppKit backend completed 3/3 exact TextEdit file effects and refused an ambiguous two-window target; structural record/locate/refuse behavior is covered separately. Known AX-only ` - Google Chrome` and ` - Google Chrome for Testing` window-title suffixes are accepted only for the corresponding exact CoreGraphics application owner; PID, unique-candidate, focused/main, and topmost-window checks remain mandatory. | Acceptance is bound to the tested host and application. Accessibility exposure and supported native actions vary by application. | @@ -129,6 +129,18 @@ Target repair does not infer changed business intent. Adding a required step, removing a business state, changing a transaction's meaning, or introducing a new authorization rule requires a new or reviewed program. +Automatic repair is bounded the same way. It takes only a stop that wrote +nothing (`HALTED_BEFORE_EFFECT`). A stop after a step the run gate counts as a +save has acted is `RECONCILIATION_REQUIRED`, even when the app saved nothing, +and goes to reconciliation and a person. On the recorded OpenEMR referral +stops of 2026-10-19 that was every stop (8 of 8): the "Create New Patient" +click before the duplicate-check dialog counts as a save. Measured on MockMed, +the default loop repaired a renamed or moved control (6 of 6 stops) and did +not repair a control moved into a closed menu, a new interstitial, a new +required choice, or a pixel postcondition that failed after a font change +(0 of 15). A compiled program is also reused only for the record check and +app environment it was admitted under ([JIT.md](JIT.md)). + ## The main incorrect-success risks OpenAdapt is designed to halt on many ambiguous conditions. The following diff --git a/packages/flow/docs/SELF_REPAIR.md b/packages/flow/docs/SELF_REPAIR.md index 8351ae244b..c6f6e243c9 100644 --- a/packages/flow/docs/SELF_REPAIR.md +++ b/packages/flow/docs/SELF_REPAIR.md @@ -577,6 +577,49 @@ it, and with WinForms-style UI Automation ids. The live tests (a repair, a pre-flight, a batch release) run on MockMed in a browser. No test drives a live RDP or Citrix session. +## Measured + +The default loop was measured with a real Claude repair agent +(`claude-sonnet-5-5`) and a scripted approver that approves every request +that reaches it, so a bad fix that passed the probe would have been saved. +Data, scripts and the full per-mode table are in +[benchmark/auto_repair](../benchmark/auto_repair/README.md). + +| Where | Stops | Reached repair | Agent proposed | Passed the probe | Approved | Resumed and verified | Wrong saves | Person decisions | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| MockMed, 22 drift modes, 3 trials each (66 runs) | 30 | 21 | 12 | 6 | 6 | **6 of 21** | 0 | 6 | +| MockMed worklist, `relink` and `redesign`, 5 items each (the change ships after the pre-flight) | 10 | 10 | 2 (one repair per group, on the test record) | 2 | 2 | **10 of 10** | 0 | 2 | +| OpenEMR referral, #1347's recorded runs | 8 | 0 (all `RECONCILIATION_REQUIRED`) | - | - | - | 0 | - | - | +| openIMIS relabelled Save (#1331), before #1337 | 3 | 0 (all `RECONCILIATION_REQUIRED`) | - | - | - | 0 | - | - | + +What worked and what didn't: + +- **Repaired:** a renamed and moved control that opens the form (`relink`, + 3 of 3) and a redesigned save with a decoy next to it (`redesign`, 3 of 3). + Each took one model call ($0.013 to $0.015) and one tap. +- **Not repaired, and nothing written:** a save moved into a closed menu + (`menu`: the agent sees only "Actions"), a new interstitial (`notice`: the + fix needs two clicks and the second control isn't on screen yet), a new + required choice (`provider`: the agent declined to choose), and pixel + postconditions that failed after a font or zoom change (`font`, `zoom`: + false stops no click can fix). These need a look at more than one screen + (the staging-copy bridge search of + [#1302](https://github.com/OpenAdaptAI/OpenAdapt/pull/1302)), or a fix to + the check itself. +- **Correctly never repaired:** a missing, absent or duplicated record (9 + stops) went to a person. +- **OpenEMR:** every recorded stop came after "Create New Patient", which the + run gate counts as a save, so all 8 went to reconciliation. Repair can't + help these; the fix for #1347 is the identity check's cause. +- **openIMIS:** before #1337 the relabelled Save stopped on the scroll before + it with `RECONCILIATION_REQUIRED`, 3 of 3. The live re-run with #1337 and + the risk review is not done yet. + +The approver was scripted, so its answer times (milliseconds) are not a +person's. The measure of a person's time here is the count of decisions: one +tap per repaired stop, or one per group of worklist items that stopped the +same way. + ## Limits - Only a stop that wrote nothing is repaired. A stop after a step the run gate diff --git a/packages/flow/openadapt_flow/repair/auto_repair.py b/packages/flow/openadapt_flow/repair/auto_repair.py index 412c5c26ed..21d4edfd3f 100644 --- a/packages/flow/openadapt_flow/repair/auto_repair.py +++ b/packages/flow/openadapt_flow/repair/auto_repair.py @@ -280,7 +280,14 @@ def summarize(rows: Iterable[Mapping[str, Any] | StopRow]) -> dict[str, Any]: data = [asdict(r) if isinstance(r, StopRow) else dict(r) for r in rows] items = [r for r in data if r.get("scope", "item") == "item"] to_repair = [r for r in items if r["route"] == "repair"] - runs = [r for r in data if r.get("repair_status") not in (None, "released")] + # Self-repair ran: it took the halt (a ``not_repairable`` halt was turned + # down before the agent was called) and this row is not a released item. + runs = [ + r + for r in data + if r.get("repair_status") + not in (None, "released", "not_repairable", "not_needed") + ] verified = [ r for r in to_repair diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 7d94b8a591..2fa21f2da3 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -147,6 +147,70 @@ "path": "benchmark/appliance_validation/results_supplement.json", "sha256": "d6594bf87583158de54935ae777eebbb6dc21f44636907921e79422ea5ddd59c" }, + { + "path": "benchmark/auto_repair/results/mockmed-2026-10-19-a/calls.json", + "sha256": "ac0559ac65873ab36d4b91535dfd8508608ce1872a9bc7f132942993f0e2b1e8" + }, + { + "path": "benchmark/auto_repair/results/mockmed-2026-10-19-a/items.csv", + "sha256": "dc4f6e1f117a917215e80841b9c47730562ad6dd3ea0f4cb1db90fb9169d1be9" + }, + { + "path": "benchmark/auto_repair/results/mockmed-2026-10-19-a/ledger.jsonl", + "sha256": "d3f8be0aadf37a4dc5b31824794cc5c37def2ae9721078a4845f2119236a6762" + }, + { + "path": "benchmark/auto_repair/results/mockmed-2026-10-19-a/meta.json", + "sha256": "3ddcae3f53d165c6e01271e58ebd0600401dc31dcb1b9404b3788dca09d6f030" + }, + { + "path": "benchmark/auto_repair/results/mockmed-2026-10-19-a/summary.json", + "sha256": "6b566bd44419a3c22bdb721f66c7eef195b34e89c17e0d5cdc3239d6a9341e6d" + }, + { + "path": "benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/calls.json", + "sha256": "222b7cdcf73450e14a62dcc9248e15d2503ec282fa39f47a6c0a01bd956d9c0d" + }, + { + "path": "benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/items.csv", + "sha256": "877a3709d26341f58ff3cbcf1a05f8739ef922723a9add26799a32d4e93cf62b" + }, + { + "path": "benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/ledger.jsonl", + "sha256": "d08b01e7e33112b2349045d6bcc59cc211df387a3585f7c637ff11e74781b77b" + }, + { + "path": "benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/meta.json", + "sha256": "1430dcbc1d22d68fdc81f159e25c0e64087811876e0c872f5f2fd4e5057f280d" + }, + { + "path": "benchmark/auto_repair/results/mockmed-worklist-2026-10-19-a/summary.json", + "sha256": "b051ea7bc2f419da4972bd8f32c113e0ad486fa921d35a06ed469131d99f9a6b" + }, + { + "path": "benchmark/auto_repair/results/openemr-1347/ledger.jsonl", + "sha256": "5cbf436dc9d9c4cec4e1210c5211862bca3c4410eb3c784646bd385a04c59016" + }, + { + "path": "benchmark/auto_repair/results/openemr-1347/stops.csv", + "sha256": "913661f9737d1eb221e56fd0bc90b1b6c9c4a7f69997d3cda06fd23938ec51ed" + }, + { + "path": "benchmark/auto_repair/results/openemr-1347/summary.json", + "sha256": "9df5d120a15664e1101e255827d1222a3808569b7b458b7d17fea18f8a62bb9d" + }, + { + "path": "benchmark/auto_repair/results/openimis-1331-before/ledger.jsonl", + "sha256": "900d3c3d1db85c53e5678e6c7d436a453bf80fd7fc45ceb0ddef11985c017680" + }, + { + "path": "benchmark/auto_repair/results/openimis-1331-before/stops.csv", + "sha256": "3819252b83e53295413a5ed5865523f663e750f98cae115ad93c4eebe27613c3" + }, + { + "path": "benchmark/auto_repair/results/openimis-1331-before/summary.json", + "sha256": "09c6cb8b9488563f7aed2f339241af549b2f3c6a0973e53fad6a86e7cd4ac6af" + }, { "path": "benchmark/canvas_ladder/results.json", "sha256": "4502affb014de71e5c584e48cef5bc6a7631983e2517551d5ea9c20c8a0668e2" diff --git a/packages/flow/tests/test_standard_change.py b/packages/flow/tests/test_standard_change.py index f36c01b1a9..ba0c911e33 100644 --- a/packages/flow/tests/test_standard_change.py +++ b/packages/flow/tests/test_standard_change.py @@ -407,6 +407,18 @@ def deploy() -> None: # the vendor ships between the pre-flight and item 1 ] assert {r.halt.state_id for r in first_runs} == {"step_005"} assert {r.transaction_outcome for r in first_runs} == {"HALTED_BEFORE_EFFECT"} + # The stop ledger: five stopped items, one repair on the test record, one + # person's decision, five items resumed and verified. + from openadapt_flow.repair.auto_repair import RepairLedger, summarize + + summary = summarize(RepairLedger(tmp_path / "repair-store").rows()) + assert summary["stops"] == 5 + assert summary["to_repair"] == 5 + assert summary["repair_runs"] == 1 + assert summary["approved_by_person"] == 1 + assert summary["person_decisions"] == 1 + assert summary["resumed_items"] == 5 + assert summary["repair_success_rate"] == 1.0 # -- refusals at the policy layer, on the recorded bundle --------------------------- From 2258d74a622d78e5cb30434bc575f1bf6518e368 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 12:36:42 -0400 Subject: [PATCH 04/10] docs: the bridge is on main; what repair by default has not shown yet Co-Authored-By: Claude Opus 5.5 --- CAPABILITIES.md | 9 +++++---- llms-full.txt | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CAPABILITIES.md b/CAPABILITIES.md index 4e998044d8..5bb4ac4293 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -195,7 +195,7 @@ improvising or retrying when something is off. | A calling agent demonstrates the fix for a halted run; the result is a repair candidate | Released (2.1.0) | Synthetic | `run --hold-on-halt`, `repair/agent_teach.py` | | Font and layout drift halts with a named reason | Released | Synthetic | `runtime/layout_drift.py` | | Self-repair: an agent proposes a fix for one halted click, Flow probes it up to the save without saving, a person approves | On main ([#1265](https://github.com/OpenAdaptAI/OpenAdapt/pull/1265)) | Synthetic (MockMed redesign) | `run --self-repair`, [#1265](https://github.com/OpenAdaptAI/OpenAdapt/pull/1265) | -| Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | In review ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts) | `run --self-repair`, [SELF_REPAIR.md](packages/flow/docs/SELF_REPAIR.md) | +| Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | On main ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts, scripted agent; a real one-shot agent repaired neither, see [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md)) | `run --self-repair`, [SELF_REPAIR.md](packages/flow/docs/SELF_REPAIR.md) | | Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](packages/flow/benchmark/pixel_scale/README.md) | | Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | | Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#PR_NUM](https://github.com/OpenAdaptAI/OpenAdapt/pull/PR_NUM)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR and openIMIS stops | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md) | @@ -260,9 +260,10 @@ fails when a command is added without being listed here. ## What is not done yet - No commercial EMR, claims system or real Citrix session has been tested. -- In review: self-repair across several new steps - ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)). In - progress: the scoreboard. +- Repair by default is measured only on MockMed. On the recorded OpenEMR + and openIMIS stops, none reached repair: each came after a step that may + have written (OpenEMR's "Create New Patient" click, openIMIS steps whose + risk nobody had reviewed). In progress: the scoreboard. - Windows installers are unsigned until free code signing through SignPath Foundation is approved; macOS builds are ad-hoc signed. - Several capabilities have run only on OpenAdapt's synthetic apps. The diff --git a/llms-full.txt b/llms-full.txt index 8cceebe92b..2bdcedac8b 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -551,7 +551,7 @@ improvising or retrying when something is off. | A calling agent demonstrates the fix for a halted run; the result is a repair candidate | Released (2.1.0) | Synthetic | `run --hold-on-halt`, `repair/agent_teach.py` | | Font and layout drift halts with a named reason | Released | Synthetic | `runtime/layout_drift.py` | | Self-repair: an agent proposes a fix for one halted click, Flow probes it up to the save without saving, a person approves | On main ([#1265](https://github.com/OpenAdaptAI/OpenAdapt/pull/1265)) | Synthetic (MockMed redesign) | `run --self-repair`, [#1265](https://github.com/OpenAdaptAI/OpenAdapt/pull/1265) | -| Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | In review ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts) | `run --self-repair`, [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md) | +| Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | On main ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts, scripted agent; a real one-shot agent repaired neither, see [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md)) | `run --self-repair`, [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md) | | Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/pixel_scale/README.md) | | Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | | Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#PR_NUM](https://github.com/OpenAdaptAI/OpenAdapt/pull/PR_NUM)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR and openIMIS stops | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md) | @@ -616,9 +616,10 @@ fails when a command is added without being listed here. ## What is not done yet - No commercial EMR, claims system or real Citrix session has been tested. -- In review: self-repair across several new steps - ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)). In - progress: the scoreboard. +- Repair by default is measured only on MockMed. On the recorded OpenEMR + and openIMIS stops, none reached repair: each came after a step that may + have written (OpenEMR's "Create New Patient" click, openIMIS steps whose + risk nobody had reviewed). In progress: the scoreboard. - Windows installers are unsigned until free code signing through SignPath Foundation is approved; macOS builds are ad-hoc signed. - Several capabilities have run only on OpenAdapt's synthetic apps. The From c65d90917ac45d676f463f9f03fc730a83060e99 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 12:37:27 -0400 Subject: [PATCH 05/10] docs(capabilities): link #1360 Co-Authored-By: Claude Opus 5.5 --- CAPABILITIES.md | 4 ++-- llms-full.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CAPABILITIES.md b/CAPABILITIES.md index 5bb4ac4293..4d54137913 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -198,7 +198,7 @@ improvising or retrying when something is off. | Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | On main ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts, scripted agent; a real one-shot agent repaired neither, see [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md)) | `run --self-repair`, [SELF_REPAIR.md](packages/flow/docs/SELF_REPAIR.md) | | Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](packages/flow/benchmark/pixel_scale/README.md) | | Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | -| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#PR_NUM](https://github.com/OpenAdaptAI/OpenAdapt/pull/PR_NUM)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR and openIMIS stops | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md) | +| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR and openIMIS stops | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md) | | Fleet drift patches: a verified fix with no patient data, shared across sites under policy | Released, local reference hub | Simulator | `openadapt-flow fleet`, [FLEET_DRIFT_PATCHES.md](packages/flow/docs/FLEET_DRIFT_PATCHES.md) | ## Computer-use agents @@ -206,7 +206,7 @@ improvising or retrying when something is off. | Capability | Status | Evidence | Where | | --- | --- | --- | --- | | Compile a computer-use agent's successful run into a program (JIT); later runs make no model calls | Released | Synthetic (a recorded Claude run on MockMed) | `openadapt_flow.jit`, [JIT.md](packages/flow/docs/JIT.md) | -| Reuse a compiled program only for the record check and app environment it was admitted under (a changed record check or a new app version at the same origin is refused) | In review ([#PR_NUM](https://github.com/OpenAdaptAI/OpenAdapt/pull/PR_NUM)) | Unit tests | `TaskSignature(record_check=, environment=)`, `WebApp(environment_id=)`, [JIT.md](packages/flow/docs/JIT.md) | +| Reuse a compiled program only for the record check and app environment it was admitted under (a changed record check or a new app version at the same origin is refused) | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Unit tests | `TaskSignature(record_check=, environment=)`, `WebApp(environment_id=)`, [JIT.md](packages/flow/docs/JIT.md) | | Code as action: the run-1 agent acts through one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program | In review ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | | Admit a draft by a person, an admitter key, or an agent under a person-signed policy, including self-admission when the policy allows it | Released | Unit tests | [JIT_ADMISSION.md](packages/flow/docs/JIT_ADMISSION.md) | | MCP server: your agent decides what to enter, OpenAdapt makes the entry and checks it | Released | Synthetic sandbox | `openadapt-agent`, [packages/agent](packages/agent/README.md) | diff --git a/llms-full.txt b/llms-full.txt index 2bdcedac8b..307624da2c 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -554,7 +554,7 @@ improvising or retrying when something is off. | Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | On main ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts, scripted agent; a real one-shot agent repaired neither, see [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md)) | `run --self-repair`, [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md) | | Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/pixel_scale/README.md) | | Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | -| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#PR_NUM](https://github.com/OpenAdaptAI/OpenAdapt/pull/PR_NUM)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR and openIMIS stops | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md) | +| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR and openIMIS stops | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md) | | Fleet drift patches: a verified fix with no patient data, shared across sites under policy | Released, local reference hub | Simulator | `openadapt-flow fleet`, [FLEET_DRIFT_PATCHES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/FLEET_DRIFT_PATCHES.md) | ## Computer-use agents @@ -562,7 +562,7 @@ improvising or retrying when something is off. | Capability | Status | Evidence | Where | | --- | --- | --- | --- | | Compile a computer-use agent's successful run into a program (JIT); later runs make no model calls | Released | Synthetic (a recorded Claude run on MockMed) | `openadapt_flow.jit`, [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | -| Reuse a compiled program only for the record check and app environment it was admitted under (a changed record check or a new app version at the same origin is refused) | In review ([#PR_NUM](https://github.com/OpenAdaptAI/OpenAdapt/pull/PR_NUM)) | Unit tests | `TaskSignature(record_check=, environment=)`, `WebApp(environment_id=)`, [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | +| Reuse a compiled program only for the record check and app environment it was admitted under (a changed record check or a new app version at the same origin is refused) | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Unit tests | `TaskSignature(record_check=, environment=)`, `WebApp(environment_id=)`, [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | | Code as action: the run-1 agent acts through one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program | In review ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | | Admit a draft by a person, an admitter key, or an agent under a person-signed policy, including self-admission when the policy allows it | Released | Unit tests | [JIT_ADMISSION.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT_ADMISSION.md) | | MCP server: your agent decides what to enter, OpenAdapt makes the entry and checks it | Released | Synthetic sandbox | `openadapt-agent`, [packages/agent](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/agent/README.md) | From a0b3328ad18dfb554a2a8af8c8bbc693151e403d Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 13:25:38 -0400 Subject: [PATCH 06/10] docs: refresh llms-full.txt after merging main Co-Authored-By: Claude Opus 5.5 --- llms-full.txt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/llms-full.txt b/llms-full.txt index 307624da2c..bbff517c7c 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -263,6 +263,7 @@ including the few still being merged. | Stopped before saving | `HALTED_BEFORE_EFFECT` | `needs_review` | Fix the cause, or show the program the fix, and run again. | | Check the record | `RECONCILIATION_REQUIRED` | `not_sure_if_saved` | A save may have gone through. A person checks the record before anything is retried. | | Finished, not checked | `COMPLETED_UNVERIFIED` | `not_sure_if_saved` | Fine for testing; not a production success. | +| Saved, waiting for its check | `SAVED_AWAITING_CHECK` | `not_sure_if_saved` | The save's check is a later report (a claim acknowledgment, an export). Don't rerun it. `openadapt-flow deferred settle` counts it as done only when the report confirms it. | | Didn't run | `REJECTED_POLICY`, `CANCELED`, `FAILED_PLATFORM` | `did_not_run` | Fix the gate or the environment and run again. | The console's last line can read `HALTED` for both the second and third rows. @@ -434,7 +435,11 @@ shipping in the next release; a pull request number means it is being merged. | Learns branching programs from several demonstrations, from repeated halts, and from many background runs. | `induce`, `teach-branch`, [SHADOW_MINING.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/capture/docs/SHADOW_MINING.md), [LEARN_FROM_MANY_RUNS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/LEARN_FROM_MANY_RUNS.md) | | Chains programs across surfaces, and one program can switch surfaces, tabs and popups mid-run. | `compose`, `process`, [#1197](https://github.com/OpenAdaptAI/OpenAdapt/pull/1197) | | Uses page structure first on web, Windows, macOS and Linux, with pixels as the fallback for remote desktops. Benchmarks labelled "vision-only" ran a deliberately pixel-only arm. | [RESOLUTION_LADDER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/RESOLUTION_LADDER.md) | -| Runs steps as direct API calls; compiling them from captured network traffic is being merged. | `runtime/actuators/api.py`, [#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269) | +| Runs steps as direct API calls, compiled from the network traffic captured while recording. | `api`, [#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269) | +| Builds the record check itself. After a save, the app usually reads the record back to show it; `scaffold-verifier` finds that read in the captured traffic or a read-only database, builds the check from it and tries to break it before a person reviews it. No API or SQL needs to be written by hand. | [EFFECT_KIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/EFFECT_KIT.md#a-record-check-built-from-the-apps-own-reads-scaffold-verifier) | +| Checks a save on desktop, RDP and Citrix apps with no API at all, by looking the record up in the app's own screens in a separate session. | [EFFECT_KIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/EFFECT_KIT.md#separate-session-desktop-read-back-desktop_readback) | +| Keeps pixel-path runs working across display scaling from 100% to 175% and font smoothing on or off, and stops rather than saves when the record changed. | [benchmark/pixel_scale](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/pixel_scale/README.md) | +| Lists every change made before a stop, and reverses them with undo steps a person approves. | `undo`, [AFTER_A_STOP.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/AFTER_A_STOP.md) | | Repairs itself when the app changes: governed repair, agent-demonstrated fixes, fleet patches, and self-repair probed up to the save. | [REPAIR_LIFECYCLE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/REPAIR_LIFECYCLE.md) | | Repairs a change that needs new steps, such as a new required dialog or a control moved into a menu: up to three new steps, each with its own record check, probed up to the save before one person approves. | [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md), [#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325) | | Compiles a computer-use agent's verified run, with no human demonstration needed. | [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | @@ -586,9 +591,9 @@ improvising or retrying when something is off. Each `openadapt-flow` command, grouped by the capability it serves. A test fails when a command is added without being listed here. -- **Record and build a program:** `record`, `demo-record`, `compile`, `induce`, `disambiguate`, `for-each`, `compose`, `process`, `intake`, `visualize`, `pull`, `variants`, `route`. +- **Record and build a program:** `record`, `demo-record`, `compile`, `induce`, `disambiguate`, `for-each`, `compose`, `process`, `intake`, `visualize`, `pull`, `variants`, `route`, `compile-agent`. - **Check and admit:** `lint`, `scaffold-verifier`, `check`, `api`, `bind-record-checks`, `certify`, `qualify`, `propose-qualification`, `known-ids`, `jit`, `admit`, `approver-grant`. -- **Run and answer:** `run`, `replay`, `resume`, `approve`, `item-approval`, `triage`, `business-decisions`, `explain`, `console`. +- **Run and answer:** `run`, `replay`, `resume`, `approve`, `item-approval`, `triage`, `business-decisions`, `explain`, `console`, `deferred`. - **Judge any executor's write:** `verify-run`, `serve-reward` (scores agent training episodes by reading the system of record). - **Repair and learn from halts:** `teach`, `teach-branch`, `repair`, `fleet`, `undo` (the undo steps a person approves after a stop). - **Give agents the program:** `emit-skill`, `emit-mcp`, `serve-execute`. @@ -1364,7 +1369,10 @@ record in the acting session (tier 3) is reported as its own check level, confirmed reopen ends `COMPLETED_UNVERIFIED`, and the agent result says `not_sure_if_saved` with `checked_below_minimum` ([`READBACK_LADDER.md`](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/READBACK_LADDER.md)). A workflow's `minimum_readback` -can raise the floor to `system_of_record`, never lower it. The on-screen +can raise the floor to `system_of_record`, never lower it. +A save that only a later report can confirm (a claim acknowledgment, a +nightly export) ends `SAVED_AWAITING_CHECK` and becomes `VERIFIED` only when +that report accepts it ([`deferred_report`](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/EFFECT_KIT.md#checked-later-deferred_report)). The on-screen oracle can still halt when the region is unreadable or wrong. If a consequential write may already have been delivered and the check did not confirm it, the production `transaction_outcome` is `RECONCILIATION_REQUIRED`, From 9d3b0aaba22e5b7bf1725c3b7ff6cdc24b4e1a64 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 13:53:08 -0400 Subject: [PATCH 07/10] fix(flow): repair a visual save and probe it from the CLI; measure openIMIS live The live openIMIS run of the relabelled Save (#1331, with #1337 merged locally and a scripted risk review) found two self-repair bugs: - a re-targeted save that carries a submit_binding got the DOM locator under the new point, and a step may not carry both, so the patched bundle failed to load even though the agent found the right control. The patch now keeps such a step visual and keeps its binding. - the halted session stays open while a probe opens another, and two Playwright drivers can't share a thread. Tests hid it behind their fixture's shared browser. run now shares one browser for every session of a repair. Measured: with the review, the stop reaches repair 3 of 3 (0 of 3 before); the agent found SAVE CLAIM 2 of 3; no fix passed the probe, because the replay sees the Save disabled where the template was cropped from the enabled button. Nothing was written. Co-Authored-By: Claude Opus 5.5 --- CAPABILITIES.md | 11 +- llms-full.txt | 16 +- packages/flow/benchmark/auto_repair/README.md | 60 ++++- .../benchmark/auto_repair/measure_openimis.py | 228 ++++++++++++++++++ .../openimis-reviewed-2026-10-19-a/calls.json | 91 +++++++ .../openimis-reviewed-2026-10-19-a/items.csv | 6 + .../ledger.jsonl | 3 + .../openimis-reviewed-2026-10-19-a/meta.json | 17 ++ .../summary.json | 31 +++ .../openimis-reviewed-2026-10-19-b/calls.json | 157 ++++++++++++ .../openimis-reviewed-2026-10-19-b/items.csv | 6 + .../ledger.jsonl | 3 + .../openimis-reviewed-2026-10-19-b/meta.json | 17 ++ .../summary.json | 31 +++ .../calls.json | 5 + .../items.csv | 3 + .../ledger.jsonl | 2 + .../meta.json | 17 ++ .../summary.json | 28 +++ packages/flow/docs/LIMITS.md | 5 +- packages/flow/docs/SELF_REPAIR.md | 16 +- .../flow/openadapt_flow/cli_self_repair.py | 9 +- .../flow/openadapt_flow/repair/self_repair.py | 18 +- packages/flow/public-artifacts.json | 60 +++++ packages/flow/tests/test_auto_repair_cli.py | 43 ++++ .../tests/test_self_repair_submit_binding.py | 71 ++++++ 26 files changed, 929 insertions(+), 25 deletions(-) create mode 100644 packages/flow/benchmark/auto_repair/measure_openimis.py create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/calls.json create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/items.csv create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/ledger.jsonl create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/meta.json create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/summary.json create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/calls.json create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/items.csv create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/ledger.jsonl create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/meta.json create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/summary.json create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/calls.json create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/items.csv create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/ledger.jsonl create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/meta.json create mode 100644 packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/summary.json create mode 100644 packages/flow/tests/test_self_repair_submit_binding.py diff --git a/CAPABILITIES.md b/CAPABILITIES.md index bd3b552a51..946af22723 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -202,7 +202,7 @@ improvising or retrying when something is off. | Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | On main ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts, scripted agent; a real one-shot agent repaired neither, see [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md)) | `run --self-repair`, [SELF_REPAIR.md](packages/flow/docs/SELF_REPAIR.md) | | Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](packages/flow/benchmark/pixel_scale/README.md) | | Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | -| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR and openIMIS stops | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md) | +| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR stops; live openIMIS relabel: 3 of 3 reached repair, 0 resumed | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md) | | Fleet drift patches: a verified fix with no patient data, shared across sites under policy | Released, local reference hub | Simulator | `openadapt-flow fleet`, [FLEET_DRIFT_PATCHES.md](packages/flow/docs/FLEET_DRIFT_PATCHES.md) | ## Computer-use agents @@ -264,10 +264,11 @@ fails when a command is added without being listed here. ## What is not done yet - No commercial EMR, claims system or real Citrix session has been tested. -- Repair by default is measured only on MockMed. On the recorded OpenEMR - and openIMIS stops, none reached repair: each came after a step that may - have written (OpenEMR's "Create New Patient" click, openIMIS steps whose - risk nobody had reviewed). In progress: the scoreboard. +- Repair by default has resumed items only on MockMed. On the recorded + OpenEMR stops none reached repair (each came after "Create New Patient", + which counts as a save). On openIMIS, with #1337 and a risk review, the + relabelled Save reached repair 3 of 3, but no fix passed the probe yet. + In progress: the scoreboard. - Windows installers are unsigned until free code signing through SignPath Foundation is approved; macOS builds are ad-hoc signed. - Several capabilities have run only on OpenAdapt's synthetic apps. The diff --git a/llms-full.txt b/llms-full.txt index bbff517c7c..256ae972ef 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -559,7 +559,7 @@ improvising or retrying when something is off. | Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | On main ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts, scripted agent; a real one-shot agent repaired neither, see [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md)) | `run --self-repair`, [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md) | | Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/pixel_scale/README.md) | | Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | -| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR and openIMIS stops | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md) | +| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR stops; live openIMIS relabel: 3 of 3 reached repair, 0 resumed | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md) | | Fleet drift patches: a verified fix with no patient data, shared across sites under policy | Released, local reference hub | Simulator | `openadapt-flow fleet`, [FLEET_DRIFT_PATCHES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/FLEET_DRIFT_PATCHES.md) | ## Computer-use agents @@ -621,10 +621,11 @@ fails when a command is added without being listed here. ## What is not done yet - No commercial EMR, claims system or real Citrix session has been tested. -- Repair by default is measured only on MockMed. On the recorded OpenEMR - and openIMIS stops, none reached repair: each came after a step that may - have written (OpenEMR's "Create New Patient" click, openIMIS steps whose - risk nobody had reviewed). In progress: the scoreboard. +- Repair by default has resumed items only on MockMed. On the recorded + OpenEMR stops none reached repair (each came after "Create New Patient", + which counts as a save). On openIMIS, with #1337 and a risk review, the + relabelled Save reached repair 3 of 3, but no fix passed the probe yet. + In progress: the scoreboard. - Windows installers are unsigned until free code signing through SignPath Foundation is approved; macOS builds are ad-hoc signed. - Several capabilities have run only on OpenAdapt's synthetic apps. The @@ -1076,7 +1077,10 @@ click before the duplicate-check dialog counts as a save. Measured on MockMed, the default loop repaired a renamed or moved control (6 of 6 stops) and did not repair a control moved into a closed menu, a new interstitial, a new required choice, or a pixel postcondition that failed after a font change -(0 of 15). A compiled program is also reused only for the record check and +(0 of 15). On openIMIS's relabelled Save, once a person reviewed the two +steps whose risk was unclassified, the stop reached repair 3 of 3, but no fix +passed the probe: the replay sees the Save disabled where the fix was cropped +from the enabled one. A compiled program is also reused only for the record check and app environment it was admitted under ([JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md)). ## The main incorrect-success risks diff --git a/packages/flow/benchmark/auto_repair/README.md b/packages/flow/benchmark/auto_repair/README.md index c642b86365..3427cfd86f 100644 --- a/packages/flow/benchmark/auto_repair/README.md +++ b/packages/flow/benchmark/auto_repair/README.md @@ -13,7 +13,7 @@ successes. All data is synthetic. | MockMed, every drift mode (`?drift=`, 22 modes including no drift), 3 trials each | `measure_mockmed.py`: the tutorial's triage program, recorded and compiled fresh, run once per trial through `AutoRepair` | A real Claude agent (`claude-sonnet-5-5`, one model call per proposal, up to 3 proposals per stop) that may propose one click or a bridge of up to three | Scripted. It approves every request that reaches it, so a wrong fix that passes the probe would be saved. | | MockMed worklist, `relink` and `redesign`, 5 items each | `measure_mockmed_worklist.py`: `WorklistQueue`, with the vendor change shipped after the pre-flight | The same | The same | | OpenEMR, the 2026-10-19 throughput runs behind [#1347](https://github.com/OpenAdaptAI/OpenAdapt/issues/1347) | `route_recorded_stops.py`: the default loop's routing rules applied to every recorded stop, offline | None needed: no stop reached repair | None | -| openIMIS, the relabelled Save of [#1331](https://github.com/OpenAdaptAI/OpenAdapt/issues/1331), before [#1337](https://github.com/OpenAdaptAI/OpenAdapt/pull/1337) | `route_recorded_stops.py` over the scoreboard's recorded program-repair run | None needed | None | +| openIMIS, the relabelled Save of [#1331](https://github.com/OpenAdaptAI/OpenAdapt/issues/1331) | Before [#1337](https://github.com/OpenAdaptAI/OpenAdapt/pull/1337): `route_recorded_stops.py` over the scoreboard's recorded run. With #1337: `measure_openimis.py`, live on the pinned openIMIS stack | The same Claude agent | Scripted, and a scripted risk reviewer for two steps | After every MockMed trial the script reads MockMed's store itself: `correct` is exactly one record for the recorded patient with the typed note, @@ -80,13 +80,49 @@ same reason ("input reached step_039, a step that saves"). The fix for #1347 is the cause itself (the identity check reading a label behind the modal), not repair. -### openIMIS, #1331 before #1337 (`results/openimis-1331-before/`) +### openIMIS, the relabelled Save (#1331) -3 attempts of the relabelled Save, all `RECONCILIATION_REQUIRED` on the -scroll before the Save (`step_014`), so 0 of 3 reached repair. This matches -#1331. With #1337 and a person's risk review of `step_002` and `step_012`, -the same stop should become `HALTED_BEFORE_EFFECT` at the Save; that live -re-run needs the openIMIS VM and has not been done here. +`measure_openimis.py` ran the scoreboard's claim-entry program (`launch-4`) +live on the pinned openIMIS stack, with the vendor change (the icon-only Save +becomes "SAVE CLAIM") switched on after the healthy items. The engine was this +branch with #1337 merged locally. + +| Run | Items | Reached repair | Agent proposed the Save | Probe passed | Resumed | Wrong saves | +| --- | --- | --- | --- | --- | --- | --- | +| Before #1337, no risk review (`results/openimis-1331-before/`, recorded earlier, routed offline) | 3 changed | 0 (`RECONCILIATION_REQUIRED` on the scroll) | - | - | 0 | 0 | +| #1337, no risk review (`results/openimis-unreviewed-2026-10-19-b/`) | 2 changed | 0 (`RECONCILIATION_REQUIRED`; the reason now names `step_002` and `step_012` and `qualify set-risk`) | - | - | 0 | 0 | +| #1337 and a scripted risk review of `step_002` and `step_012` (`results/openimis-reviewed-2026-10-19-b/`) | 2 healthy, 3 changed | **3 of 3** (`HALTED_BEFORE_EFFECT` at the Save) | 2 of 3 (one gave up) | 0 of 2 | 0 | 0 | + +The healthy items saved correctly with no model call. With the review in +place, the relabelled Save now halts on the Save itself, before any effect, +and goes to repair, which it never did before. The agent then pointed at +"SAVE CLAIM" in 2 of 3 items. No fix resumed, for a reason the probe found: +in the replay, the scroll before the Save runs while openIMIS still shows +the Save disabled (grey). The patched template was cropped from the enabled +(dark) button at the halt, so it never matches during the scroll, and the +probe stops before the save. Nothing was written. This is a false refusal, +not an unsafe one. + +The first live run (`results/openimis-reviewed-2026-10-19-a/`) found two bugs +in self-repair, both fixed in this PR: + +- **A re-targeted visual save failed validation.** openIMIS's Save is + recorded on the visual path, so its step carries a `submit_binding`, and a + step may not have both that and a usable structural locator. The patch put + the DOM locator under the new point on the anchor, so the patched bundle + failed to load (`ValidationError`), even though the agent had found the + right control. The patch now keeps such a step visual and keeps its + binding (`tests/test_self_repair_submit_binding.py`). +- **The probe couldn't open its session.** The halted session stays open + while the probe opens another, and two Playwright drivers can't run on one + thread. The tests never saw it because their fixture shares one browser. + `run` now shares one browser for every session of a repaired run + (`test_the_cli_repairs_without_a_shared_browser_from_its_caller`). + +A first attempt with `OPENADAPT_FLOW_BROWSER=bundled` halted every item, +healthy ones included, at `step_000`: the program was recorded in installed +Chrome, and the bundled Chromium renders the form a few pixels differently. +It wrote nothing and is not counted. ## What this does not show @@ -96,8 +132,9 @@ re-run needs the openIMIS VM and has not been done here. - One agent and one model. A different agent may propose more, or fewer, good fixes; the probe, the cases and the record check are what keep a bad one from saving. -- MockMed is a synthetic app. The OpenEMR and openIMIS rows are routing - results over recorded stops, not live repairs. +- MockMed is a synthetic app. The OpenEMR rows are routing results over + recorded stops, not live repairs. The openIMIS rows with #1337 are live, on + a synthetic dataset, with #1337 merged locally (it is not on main yet). ## Reproduce @@ -105,7 +142,12 @@ re-run needs the openIMIS VM and has not been done here. python benchmark/auto_repair/measure_mockmed.py --run-id --trials 3 python benchmark/auto_repair/measure_mockmed_worklist.py --run-id --items 5 python benchmark/auto_repair/route_recorded_stops.py --bundle --runs-root --out results/ --app +python benchmark/auto_repair/measure_openimis.py --run-id --bundle --label reviewed --healthy 2 --changed 3 ``` +The openIMIS run needs the pinned stack up (`benchmark/openimis_claims`, +with `OPENIMIS_STATE_DIR` pointing at the state the stack was created with), +`pg8000` for the record check, and installed Chrome. + The first two need `ANTHROPIC_API_KEY` (the customer's own key) and a Playwright Chromium. They cost about $0.25 and $0.03. diff --git a/packages/flow/benchmark/auto_repair/measure_openimis.py b/packages/flow/benchmark/auto_repair/measure_openimis.py new file mode 100644 index 0000000000..28b85ef8f2 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/measure_openimis.py @@ -0,0 +1,228 @@ +"""Measure the default repair loop on openIMIS's relabelled Save (#1331). + + python benchmark/auto_repair/measure_openimis.py --run-id 2026-10-19-a \\ + --bundle BUNDLE --label reviewed --healthy 2 --changed 3 + +Each item runs the compiled claim-entry program (``launch-4``) once through +:class:`openadapt_flow.repair.auto_repair.AutoRepair`, the loop ``run`` uses +by default when a repair agent is set. ``--healthy`` items run on the +unchanged app; then the vendor change is switched on (the icon-only Save +becomes "Save claim", ``scoreboard.tasks.openimis.ui_change``) and +``--changed`` items run on it. + +The program runs as the scoreboard's arm 3 runs it: the demo profile's plain +Replayer with the read-only SQL record check (``DemoTarget`` from +``scoreboard.repair_arm``). The repair agent is ``BridgeRepairAgent`` from +``measure_mockmed.py`` (one Claude call per proposal, the customer's key); +the approver is scripted and approves everything that reaches it. No fault +probe is configured, as in the scoreboard's repair arm. + +``--bundle`` is the program to run. ``--label`` names it in the results: the +bundle as compiled (``unreviewed``), or the same bundle after a scripted +reviewer classified its two unclassified steps with ``openadapt-flow qualify +set-risk`` (``reviewed``, the step #1337 points to). + +After every item this reads the claim table itself (``app.check``): +``correct``, ``missing``, ``duplicate`` or ``wrong``. + +Outputs (committed): ``/ledger.jsonl``, ``items.csv``, ``summary.json``, +``calls.json`` and ``meta.json``. Synthetic data only. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import shutil +import sys +import time +from dataclasses import asdict +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(HERE.parent)) + +from measure_mockmed import ( # noqa: E402 + APPROVER, + CACHE, + BridgeRepairAgent, + _approver, + _git_head, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--run-id", required=True) + parser.add_argument("--bundle", required=True) + parser.add_argument("--label", required=True) + parser.add_argument("--healthy", type=int, default=2) + parser.add_argument("--changed", type=int, default=3) + parser.add_argument("--start", type=int, default=200) + parser.add_argument("--tag", default="ar") + parser.add_argument("--out", default=None) + parser.add_argument("--model", default="claude-sonnet-5-5") + args = parser.parse_args() + + from scoreboard.repair_arm import DemoTarget + from scoreboard.tasks import openimis as app + + from openadapt_flow.jit import load_customer_model_key + from openadapt_flow.repair.auto_repair import AutoRepair, RepairLedger, summarize + from openadapt_flow.repair.self_repair import ScreenAccess, SelfRepair + + name = f"openimis-{args.label}-{args.run_id}" + out = Path(args.out) if args.out else HERE / "results" / name + out.mkdir(parents=True, exist_ok=True) + work = CACHE / name + work.mkdir(parents=True, exist_ok=True) + ledger = RepairLedger(out) + ledger.path = out / "ledger.jsonl" + key = load_customer_model_key("anthropic") + agent = BridgeRepairAgent(model=args.model) + (out / "meta.json").write_text( + json.dumps( + { + "written_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "engine_git_head": _git_head(), + "app": "openIMIS reference stack (pinned), synthetic data", + "program": "launch-4 claim-entry bundle (demo/cua_to_program)", + "bundle_label": args.label, + "risk_review": ( + "step_002 and step_012 classified state_changing by a scripted " + "reviewer (qualify set-risk), standing in for a person" + if args.label == "reviewed" + else "none" + ), + "ui_change": "icon-only Save relabelled 'Save claim' (#1331)", + "loop": "AutoRepair over SelfRepair; no fault probe; no policy", + "agent": agent.agent_id, + "approver": APPROVER, + "items": {"healthy": args.healthy, "changed": args.changed}, + }, + indent=2, + ) + + "\n" + ) + from openadapt_flow.backends._browser_process import shared_browser + + rows = [] + app.ui_change(False) + # The halted session stays open while a probe opens another; both need + # one shared browser process on this thread, as `run` shares one. + browser = shared_browser(headless=True) + browser.__enter__() + try: + total = args.healthy + args.changed + for n in range(total): + i = args.start + n + changed = n >= args.healthy + if changed != app.ui_change_is_on(): + app.ui_change(changed) + values = app.item_values(i, args.tag) + item_dir = work / f"item-{i:03d}" + shutil.rmtree(item_dir, ignore_errors=True) + decisions: list[float] = [] + before = agent.cost_usd + repair = SelfRepair( + agent, + DemoTarget(app, values), + store=item_dir / "repair-store", + screen_access=ScreenAccess( + enabled_by="benchmark-operator (synthetic data only)", + mode="allow_unscrubbed", + ), + model_key=key, + approval_wait_s=120, + sleep=_approver(item_dir, decisions), + ) + loop = AutoRepair( + repair, + ledger=ledger, + labels={ + "app": "openimis", + "bundle": args.label, + "ui_change": str(changed).lower(), + "item": str(i), + }, + ) + t0 = time.time() + try: + result = loop.run(Path(args.bundle), item_dir / "run") + status, reason = result.status, result.evidence.reason + except Exception as exc: # noqa: BLE001 - a harness error is a row + status, reason = "harness_error", f"{type(exc).__name__}: {exc}" + wall = time.time() - t0 + readback = app.check(values) + rows.append( + { + "item": i, + "ui_change": changed, + "repair_status": status, + "readback": readback, + "cost_usd": round(agent.cost_usd - before, 6), + "wall_s": round(wall, 1), + "reason": (reason or "")[:300], + } + ) + print( + f"{i} change={changed} {status:18s} {readback:9s} " + f"${agent.cost_usd - before:.3f} {wall:.0f}s {reason[:120]}", + flush=True, + ) + finally: + browser.__exit__(None, None, None) + app.ui_change(False) + with (out / "items.csv").open("w", newline="") as f: + writer = csv.DictWriter( + f, + fieldnames=[ + "item", + "ui_change", + "repair_status", + "readback", + "cost_usd", + "wall_s", + "reason", + ], + ) + writer.writeheader() + writer.writerows(rows) + (out / "calls.json").write_text( + json.dumps( + { + "model": agent.model, + "cost_usd": round(agent.cost_usd, 6), + "calls": [asdict(c) for c in agent.calls], + }, + indent=2, + ) + + "\n" + ) + if ledger.path.is_file(): + ledger.path.write_text( + ledger.path.read_text().replace( + str(CACHE), "~/.cache/openadapt-auto-repair" + ) + ) + (out / "summary.json").write_text( + json.dumps( + { + "funnel": summarize(ledger.rows()), + "readback": { + k: sum(r["readback"] == k for r in rows) + for k in sorted({r["readback"] for r in rows}) + }, + "ui_change_off_after_run": not app.ui_change_is_on(), + }, + indent=2, + ) + + "\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/calls.json b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/calls.json new file mode 100644 index 0000000000..f1fd5e2a8c --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/calls.json @@ -0,0 +1,91 @@ +{ + "model": "claude-sonnet-5-5", + "cost_usd": 0.089414, + "calls": [ + { + "turn": 1, + "started_s": 1792431229.755805, + "ended_s": 1792431236.859002, + "input_tokens": 6668, + "output_tokens": 201, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.015346, + "actions": [], + "request_id": "req_011CgBzp5MGfKRJWeMqgwwY4", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792431284.658227, + "ended_s": 1792431290.4752781, + "input_tokens": 6521, + "output_tokens": 596, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.019002, + "actions": [], + "request_id": "req_011CgBzt7tRpRJEkZM5DxSVo", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792431319.795273, + "ended_s": 1792431326.378529, + "input_tokens": 6521, + "output_tokens": 597, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.019012, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 1193, + 752 + ] + } + } + ], + "request_id": "req_011CgBzviLvGpKRcwBtbP9Ca", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792431326.678208, + "ended_s": 1792431330.611611, + "input_tokens": 6548, + "output_tokens": 396, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.017056, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 1193, + 752 + ] + } + } + ], + "request_id": "req_011CgBzwDh25KrwQ9HH2hYpU", + "stop_reason": "tool_use" + }, + { + "turn": 3, + "started_s": 1792431330.642976, + "ended_s": 1792431336.287853, + "input_tokens": 6564, + "output_tokens": 587, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.018998, + "actions": [], + "request_id": "req_011CgBzwWVKAUrPTdK2Trzub", + "stop_reason": "end_turn" + } + ] +} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/items.csv b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/items.csv new file mode 100644 index 0000000000..17e27a7581 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/items.csv @@ -0,0 +1,6 @@ +item,ui_change,repair_status,readback,cost_usd,wall_s,reason +240,False,not_needed,correct,0,22.7, +241,False,not_needed,correct,0,23.1, +242,True,no_fix,missing,0.015346,24.6,the agent gave up +243,True,no_fix,missing,0.019002,53.5,the agent gave up +244,True,no_fix,missing,0.055066,45.7,the agent gave up diff --git a/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/ledger.jsonl b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/ledger.jsonl new file mode 100644 index 0000000000..6236e46153 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/ledger.jsonl @@ -0,0 +1,3 @@ +{"approved_by": null, "approved_via": null, "at": "2026-10-19T17:33:57+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_005", "labels": {"app": "openimis", "bundle": "reviewed", "item": "242", "ui_change": "true"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 24.587, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/openimis-reviewed-2026-10-19-a/item-242/run", "scope": "item", "workflow": "openimis.claim_entry"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T17:34:51+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_015", "labels": {"app": "openimis", "bundle": "reviewed", "item": "243", "ui_change": "true"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 53.473, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/openimis-reviewed-2026-10-19-a/item-243/run", "scope": "item", "workflow": "openimis.claim_entry"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T17:35:37+00:00", "cases_passed": false, "est_model_cost_usd": 0.036068, "halted_step_id": "step_015", "labels": {"app": "openimis", "bundle": "reviewed", "item": "244", "ui_change": "true"}, "model_calls": 2, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 2, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 45.723, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/openimis-reviewed-2026-10-19-a/item-244/run", "scope": "item", "workflow": "openimis.claim_entry"} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/meta.json b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/meta.json new file mode 100644 index 0000000000..4d4e880010 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/meta.json @@ -0,0 +1,17 @@ +{ + "written_at": "2026-10-19T13:32:46-0400", + "engine_git_head": "50d5fa85b5bcb0473380d30ee42851a9326b36a4", + "app": "openIMIS reference stack (pinned), synthetic data", + "program": "launch-4 claim-entry bundle (demo/cua_to_program)", + "bundle_label": "reviewed", + "risk_review": "step_002 and step_012 classified state_changing by a scripted reviewer (qualify set-risk), standing in for a person", + "ui_change": "icon-only Save relabelled 'Save claim' (#1331)", + "loop": "AutoRepair over SelfRepair; no fault probe; no policy", + "agent": "agent:anthropic:claude-sonnet-5-5:auto-repair-benchmark", + "approver": "benchmark:scripted-approver (stands in for a person; approves all)", + "items": { + "healthy": 2, + "changed": 3 + }, + "engine_note": "Engine: this PR's branch with #1337 merged locally (measure/auto-repair-openimis 50d5fa85), before two fixes this run found: a re-targeted visual save lost validation (submit_binding with a structural locator), and a probe could not open a second Playwright session. Kept as evidence; superseded by -b. An earlier attempt with OPENADAPT_FLOW_BROWSER=bundled halted every item at step_000, healthy ones included (the program was recorded in installed Chrome); it wrote nothing and was discarded." +} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/summary.json b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/summary.json new file mode 100644 index 0000000000..6a5dbedcd5 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/summary.json @@ -0,0 +1,31 @@ +{ + "funnel": { + "stops": 3, + "to_repair": 3, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 3, + "repairs_proposed": 1, + "proposals": 2, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": 0.0, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 2, + "est_model_cost_usd": 0.036068, + "repair_status_counts": { + "no_fix": 3 + } + }, + "readback": { + "correct": 2, + "missing": 3 + }, + "ui_change_off_after_run": true +} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/calls.json b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/calls.json new file mode 100644 index 0000000000..da77bde83d --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/calls.json @@ -0,0 +1,157 @@ +{ + "model": "claude-sonnet-5-5", + "cost_usd": 0.126242, + "calls": [ + { + "turn": 1, + "started_s": 1792431796.3365571, + "ended_s": 1792431803.37988, + "input_tokens": 6521, + "output_tokens": 673, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.019772, + "actions": [], + "request_id": "req_011CgC1Xqhf7fBkX6WFsQwFW", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792431830.930938, + "ended_s": 1792431836.136097, + "input_tokens": 6521, + "output_tokens": 495, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.017992, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 1193, + 752 + ] + } + } + ], + "request_id": "req_011CgC1aPSfqeu2mPswFs6Nv", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792431857.9132988, + "ended_s": 1792431865.1740181, + "input_tokens": 6646, + "output_tokens": 659, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.019882, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 1193, + 752 + ] + } + } + ], + "request_id": "req_011CgC1cNqmi3SP9osZehDvj", + "stop_reason": "tool_use" + }, + { + "turn": 3, + "started_s": 1792431886.429514, + "ended_s": 1792431893.1243122, + "input_tokens": 6760, + "output_tokens": 567, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.01919, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 1193, + 752 + ] + } + } + ], + "request_id": "req_011CgC1eUhJER7ChbYHkDNJw", + "stop_reason": "tool_use" + }, + { + "turn": 1, + "started_s": 1792431960.688413, + "ended_s": 1792431962.870594, + "input_tokens": 6521, + "output_tokens": 69, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013732, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 1192, + 752 + ] + } + } + ], + "request_id": "req_011CgC1jxDekew8Fp8pGzBd6", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792432006.752075, + "ended_s": 1792432012.144477, + "input_tokens": 6624, + "output_tokens": 441, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.017658, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 1193, + 752 + ] + } + } + ], + "request_id": "req_011CgC1oMJJwYwZgXoAohHsj", + "stop_reason": "tool_use" + }, + { + "turn": 3, + "started_s": 1792432029.392721, + "ended_s": 1792432034.27016, + "input_tokens": 6738, + "output_tokens": 454, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.018016, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 1193, + 752 + ] + } + } + ], + "request_id": "req_011CgC1q28ByfG69p129SdSa", + "stop_reason": "tool_use" + } + ] +} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/items.csv b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/items.csv new file mode 100644 index 0000000000..e5ba4522a2 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/items.csv @@ -0,0 +1,6 @@ +item,ui_change,repair_status,readback,cost_usd,wall_s,reason +260,False,not_needed,correct,0,21.0, +261,False,not_needed,correct,0,19.0, +262,True,no_fix,missing,0.019772,36.3,the agent gave up +263,True,no_fix,missing,0.057064,113.0,"the probe stopped before reaching the save at step_014: Step 'step_014' (scroll by (0, -600)): closed-loop scroll exhausted its budget (1200px of 1500px allowed, 2.5x the recorded distance) without the anchor of step 'step_015' (click recorded visual target) resolving — target never came into view; " +264,True,no_fix,missing,0.049406,137.9,"the probe stopped before reaching the save at step_014: Step 'step_014' (scroll by (0, -600)): closed-loop scroll exhausted its budget (1200px of 1500px allowed, 2.5x the recorded distance) without the anchor of step 'step_015' (click recorded visual target) resolving — target never came into view; " diff --git a/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/ledger.jsonl b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/ledger.jsonl new file mode 100644 index 0000000000..d3c1241ed7 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/ledger.jsonl @@ -0,0 +1,3 @@ +{"approved_by": null, "approved_via": null, "at": "2026-10-19T17:43:23+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_015", "labels": {"app": "openimis", "bundle": "reviewed", "item": "262", "ui_change": "true"}, "model_calls": 0, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 36.346, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/openimis-reviewed-2026-10-19-b/item-262/run", "scope": "item", "workflow": "openimis.claim_entry"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T17:45:16+00:00", "cases_passed": false, "est_model_cost_usd": 0.057064, "halted_step_id": "step_015", "labels": {"app": "openimis", "bundle": "reviewed", "item": "263", "ui_change": "true"}, "model_calls": 3, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 3, "released_by": null, "repair_reason": "the probe stopped before reaching the save at step_014: Step 'step_014' (scroll by (0, -600)): closed-loop scroll exhausted its budget (1200px of 1500px allowed, 2.5x the recorded distance) without the anchor of step 'step_015' (click recorded visual target) resolving \u2014 target never came into view; run aborted", "repair_s": 113.013, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/openimis-reviewed-2026-10-19-b/item-263/run", "scope": "item", "workflow": "openimis.claim_entry"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T17:47:34+00:00", "cases_passed": false, "est_model_cost_usd": 0.049406, "halted_step_id": "step_015", "labels": {"app": "openimis", "bundle": "reviewed", "item": "264", "ui_change": "true"}, "model_calls": 3, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 3, "released_by": null, "repair_reason": "the probe stopped before reaching the save at step_014: Step 'step_014' (scroll by (0, -600)): closed-loop scroll exhausted its budget (1200px of 1500px allowed, 2.5x the recorded distance) without the anchor of step 'step_015' (click recorded visual target) resolving \u2014 target never came into view; run aborted", "repair_s": 137.874, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/openimis-reviewed-2026-10-19-b/item-264/run", "scope": "item", "workflow": "openimis.claim_entry"} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/meta.json b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/meta.json new file mode 100644 index 0000000000..e9c4ad267b --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/meta.json @@ -0,0 +1,17 @@ +{ + "written_at": "2026-10-19T13:42:05-0400", + "engine_git_head": "4c4c158cb7c2859a1f8c712ccbb7320a552b9462", + "app": "openIMIS reference stack (pinned), synthetic data", + "program": "launch-4 claim-entry bundle (demo/cua_to_program)", + "bundle_label": "reviewed", + "risk_review": "step_002 and step_012 classified state_changing by a scripted reviewer (qualify set-risk), standing in for a person", + "ui_change": "icon-only Save relabelled 'Save claim' (#1331)", + "loop": "AutoRepair over SelfRepair; no fault probe; no policy", + "agent": "agent:anthropic:claude-sonnet-5-5:auto-repair-benchmark", + "approver": "benchmark:scripted-approver (stands in for a person; approves all)", + "items": { + "healthy": 2, + "changed": 3 + }, + "engine_note": "Engine: this PR's branch with #1337 merged locally and both fixes applied (measure/auto-repair-openimis 4c4c158c). Installed Chrome, as the program was recorded." +} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/summary.json b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/summary.json new file mode 100644 index 0000000000..66e5d00803 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/summary.json @@ -0,0 +1,31 @@ +{ + "funnel": { + "stops": 3, + "to_repair": 3, + "to_reconciliation": 0, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 3, + "repairs_proposed": 2, + "proposals": 6, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": 0.0, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 6, + "est_model_cost_usd": 0.10647, + "repair_status_counts": { + "no_fix": 3 + } + }, + "readback": { + "correct": 2, + "missing": 3 + }, + "ui_change_off_after_run": true +} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/calls.json b/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/calls.json new file mode 100644 index 0000000000..a23d79ea4e --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/calls.json @@ -0,0 +1,5 @@ +{ + "model": "claude-sonnet-5-5", + "cost_usd": 0, + "calls": [] +} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/items.csv b/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/items.csv new file mode 100644 index 0000000000..fd1109ca67 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/items.csv @@ -0,0 +1,3 @@ +item,ui_change,repair_status,readback,cost_usd,wall_s,reason +270,True,not_repairable,missing,0,29.0,"the run ended RECONCILIATION_REQUIRED; self-repair handles only a halt before any effect (effect not settled at step_002, step_012); each acted with a risk nobody reviewed, so a write cannot be ruled out. A person can classify them with `openadapt-flow qualify set-risk`" +271,True,not_repairable,missing,0,27.7,"the run ended RECONCILIATION_REQUIRED; self-repair handles only a halt before any effect (effect not settled at step_002, step_012); each acted with a risk nobody reviewed, so a write cannot be ruled out. A person can classify them with `openadapt-flow qualify set-risk`" diff --git a/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/ledger.jsonl b/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/ledger.jsonl new file mode 100644 index 0000000000..cd1859ca50 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/ledger.jsonl @@ -0,0 +1,2 @@ +{"approved_by": null, "approved_via": null, "at": "2026-10-19T17:49:17+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_015", "labels": {"app": "openimis", "bundle": "unreviewed", "item": "270", "ui_change": "true"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the run ended RECONCILIATION_REQUIRED; self-repair handles only a halt before any effect (effect not settled at step_002, step_012); each acted with a risk nobody reviewed, so a write cannot be ruled out. A person can classify them with `openadapt-flow qualify set-risk`", "repair_s": 29.019, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically; reconcile: read the change ledger in REPORT.md, then `openadapt-flow undo list ~/.cache/openadapt-auto-repair/openimis-unreviewed-2026-10-19-b/item-270/run`", "run_dir": "~/.cache/openadapt-auto-repair/openimis-unreviewed-2026-10-19-b/item-270/run", "scope": "item", "workflow": "openimis.claim_entry"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T17:49:45+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_015", "labels": {"app": "openimis", "bundle": "unreviewed", "item": "271", "ui_change": "true"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the run ended RECONCILIATION_REQUIRED; self-repair handles only a halt before any effect (effect not settled at step_002, step_012); each acted with a risk nobody reviewed, so a write cannot be ruled out. A person can classify them with `openadapt-flow qualify set-risk`", "repair_s": 27.674, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically; reconcile: read the change ledger in REPORT.md, then `openadapt-flow undo list ~/.cache/openadapt-auto-repair/openimis-unreviewed-2026-10-19-b/item-271/run`", "run_dir": "~/.cache/openadapt-auto-repair/openimis-unreviewed-2026-10-19-b/item-271/run", "scope": "item", "workflow": "openimis.claim_entry"} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/meta.json b/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/meta.json new file mode 100644 index 0000000000..6838815ac1 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/meta.json @@ -0,0 +1,17 @@ +{ + "written_at": "2026-10-19T13:48:47-0400", + "engine_git_head": "4c4c158cb7c2859a1f8c712ccbb7320a552b9462", + "app": "openIMIS reference stack (pinned), synthetic data", + "program": "launch-4 claim-entry bundle (demo/cua_to_program)", + "bundle_label": "unreviewed", + "risk_review": "none", + "ui_change": "icon-only Save relabelled 'Save claim' (#1331)", + "loop": "AutoRepair over SelfRepair; no fault probe; no policy", + "agent": "agent:anthropic:claude-sonnet-5-5:auto-repair-benchmark", + "approver": "benchmark:scripted-approver (stands in for a person; approves all)", + "items": { + "healthy": 0, + "changed": 2 + }, + "engine_note": "Engine: as openimis-reviewed-2026-10-19-b. The bundle as compiled, no risk review." +} diff --git a/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/summary.json b/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/summary.json new file mode 100644 index 0000000000..2b5b998484 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/summary.json @@ -0,0 +1,28 @@ +{ + "funnel": { + "stops": 2, + "to_repair": 0, + "to_reconciliation": 2, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 0, + "repairs_proposed": 0, + "proposals": 0, + "probe_passed": 0, + "approved": 0, + "approved_by_person": 0, + "approved_by_standard_change": 0, + "resumed_items": 0, + "resumed_not_verified": 0, + "repair_success_rate": null, + "person_decisions": 0, + "person_wait_s_total": 0, + "model_calls": 0, + "est_model_cost_usd": 0.0, + "repair_status_counts": {} + }, + "readback": { + "missing": 2 + }, + "ui_change_off_after_run": true +} diff --git a/packages/flow/docs/LIMITS.md b/packages/flow/docs/LIMITS.md index 010e28d9a7..b949f083b6 100644 --- a/packages/flow/docs/LIMITS.md +++ b/packages/flow/docs/LIMITS.md @@ -138,7 +138,10 @@ click before the duplicate-check dialog counts as a save. Measured on MockMed, the default loop repaired a renamed or moved control (6 of 6 stops) and did not repair a control moved into a closed menu, a new interstitial, a new required choice, or a pixel postcondition that failed after a font change -(0 of 15). A compiled program is also reused only for the record check and +(0 of 15). On openIMIS's relabelled Save, once a person reviewed the two +steps whose risk was unclassified, the stop reached repair 3 of 3, but no fix +passed the probe: the replay sees the Save disabled where the fix was cropped +from the enabled one. A compiled program is also reused only for the record check and app environment it was admitted under ([JIT.md](JIT.md)). ## The main incorrect-success risks diff --git a/packages/flow/docs/SELF_REPAIR.md b/packages/flow/docs/SELF_REPAIR.md index c6f6e243c9..f53786bf16 100644 --- a/packages/flow/docs/SELF_REPAIR.md +++ b/packages/flow/docs/SELF_REPAIR.md @@ -591,6 +591,7 @@ Data, scripts and the full per-mode table are in | MockMed worklist, `relink` and `redesign`, 5 items each (the change ships after the pre-flight) | 10 | 10 | 2 (one repair per group, on the test record) | 2 | 2 | **10 of 10** | 0 | 2 | | OpenEMR referral, #1347's recorded runs | 8 | 0 (all `RECONCILIATION_REQUIRED`) | - | - | - | 0 | - | - | | openIMIS relabelled Save (#1331), before #1337 | 3 | 0 (all `RECONCILIATION_REQUIRED`) | - | - | - | 0 | - | - | +| openIMIS relabelled Save, live, with #1337 and a scripted risk review | 3 | **3** | 2 | 0 | 0 | 0 | 0 | 0 | What worked and what didn't: @@ -612,8 +613,15 @@ What worked and what didn't: run gate counts as a save, so all 8 went to reconciliation. Repair can't help these; the fix for #1347 is the identity check's cause. - **openIMIS:** before #1337 the relabelled Save stopped on the scroll before - it with `RECONCILIATION_REQUIRED`, 3 of 3. The live re-run with #1337 and - the risk review is not done yet. + it with `RECONCILIATION_REQUIRED`, 3 of 3. With #1337 and a risk review of + the two steps nobody had classified, it halts on the Save before any effect + and reaches repair, 3 of 3. The agent found "SAVE CLAIM" in 2 of 3, but no + fix passed the probe: in the replay the Save is still disabled (grey) while + the scroll looks for it, and the new template was cropped from the enabled + button. Nothing was written. The first live run also found two bugs, now + fixed: a re-targeted save that carries a `submit_binding` failed + validation, and the probe couldn't open a second browser session from the + CLI. The approver was scripted, so its answer times (milliseconds) are not a person's. The measure of a person's time here is the count of decisions: one @@ -622,6 +630,10 @@ same way. ## Limits +- A patch's new template is cropped from the screen at the halt. When the + app shows the control in another state during the replay (openIMIS shows + its Save disabled until the form validates), the probe doesn't find it and + refuses the fix. - Only a stop that wrote nothing is repaired. A stop after a step the run gate counts as a save has acted is `RECONCILIATION_REQUIRED` and goes to reconciliation, even when the app saved nothing; see [Measured](#measured) diff --git a/packages/flow/openadapt_flow/cli_self_repair.py b/packages/flow/openadapt_flow/cli_self_repair.py index eb30d8428d..78cef7ff77 100644 --- a/packages/flow/openadapt_flow/cli_self_repair.py +++ b/packages/flow/openadapt_flow/cli_self_repair.py @@ -445,9 +445,16 @@ def announce(run_dir: Path, request_id: str, request_sha256: str) -> None: if attest_pre_state is not None: # Same point as a plain run: admitted, nothing actuated yet. attest_pre_state(run_dir, args) + from openadapt_flow.backends._browser_process import shared_browser + auto = AutoRepair(repair) try: - result = auto.run(bundle, run_dir) + # The halted session stays open while each probe opens its own. Two + # Playwright drivers can't run on one thread, so every session of + # this run shares one browser process (each still gets a fresh + # context: no cookies or storage carry over). + with shared_browser(headless=not repair.target.headed): + result = auto.run(bundle, run_dir) except SelfRepairRefused as exc: print(f"Self-repair stopped: {exc}. Nothing was saved by self-repair.") return 1 diff --git a/packages/flow/openadapt_flow/repair/self_repair.py b/packages/flow/openadapt_flow/repair/self_repair.py index ca85e1d671..28734c14dd 100644 --- a/packages/flow/openadapt_flow/repair/self_repair.py +++ b/packages/flow/openadapt_flow/repair/self_repair.py @@ -1356,7 +1356,7 @@ def _retarget( anchor = event.new_anchor.model_copy( update={ "template": f"templates/{step.id}_repair.png", - "structural": structural, + "structural": _patched_structural(step, structural), "ocr_text": read, "landmarks": landmarks, } @@ -2151,6 +2151,22 @@ def _outline(png: bytes, region: tuple[int, int, int, int]) -> bytes: return out.getvalue() +def _patched_structural(step: Step, structural: Any) -> Any: + """The structural locator a re-targeted step's anchor keeps. + + A step with a ``submit_binding`` was recorded on the visual path: the + binding stands in for a record identity there, and a step may not carry + both it and a usable locator. Self-repair keeps that contract as it was + recorded, so the step stays visual (a new template crop) and keeps its + binding, which the save checks again at run time. Any other step takes + the locator found under the new point. + """ + + if step.submit_binding is not None: + return step.anchor.structural if step.anchor is not None else None + return structural + + def _recorded_crop(workflow: Workflow, bundle_dir: Path, step: Step) -> Optional[bytes]: if step.anchor is None: return None diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index edf2449f73..658aaee911 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -211,6 +211,66 @@ "path": "benchmark/auto_repair/results/openimis-1331-before/summary.json", "sha256": "09c6cb8b9488563f7aed2f339241af549b2f3c6a0973e53fad6a86e7cd4ac6af" }, + { + "path": "benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/calls.json", + "sha256": "e507e127310e692286cffa4646854983024e669b32eea0c0d1b896928c999d0e" + }, + { + "path": "benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/items.csv", + "sha256": "f57460b8bdd7bf422d0e547d690562357422311efa0480604980f73b3c575acd" + }, + { + "path": "benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/ledger.jsonl", + "sha256": "f817ef9fc416df05e401d6d50a36f69241757975e5d69fdd0e38815cf39af421" + }, + { + "path": "benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/meta.json", + "sha256": "894bfb2418a587c59c8110da8592d54d7c0582e6496273b5d57bb96009fc65a4" + }, + { + "path": "benchmark/auto_repair/results/openimis-reviewed-2026-10-19-a/summary.json", + "sha256": "0b06e1e738eecf8027825a5b45ea0bd51c08b0cffb2083d170a35f43477c84ce" + }, + { + "path": "benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/calls.json", + "sha256": "bcf7138d3479ff9dd16277ca49ec52896a48cfcf9e9b45599e69cecad8ab2f82" + }, + { + "path": "benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/items.csv", + "sha256": "19c1f39de75edaeb649c541325d8bb0b009afd628380ee3248e3518cd6bc72ea" + }, + { + "path": "benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/ledger.jsonl", + "sha256": "00d321e0df2cfe4f4f54f8d04655edd00cde8a060bf342a1f70cd2c5fbd11514" + }, + { + "path": "benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/meta.json", + "sha256": "193b4c3e4e8eb122a7072dcc4701404d28c0b3bb2a2b0914fc9a5388dcc22ebd" + }, + { + "path": "benchmark/auto_repair/results/openimis-reviewed-2026-10-19-b/summary.json", + "sha256": "c9f8ea5769e098d1e0c89642a5860fe09e696ed92eb93fd7881f50f6fb6ea96f" + }, + { + "path": "benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/calls.json", + "sha256": "7a88263a06fdaaa4b1a7eadc6ff4db102ddb12bc964e8d837ce67ac3c5e64e2c" + }, + { + "path": "benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/items.csv", + "sha256": "7ab7fa50cfeec823b570c768919e3408cb34a810d7a0262d7e4777bffb93fac0" + }, + { + "path": "benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/ledger.jsonl", + "sha256": "a2083aa13f54e729a437b52f8d3ef78a6d0f30bf896d8ceca304565536e25bd6" + }, + { + "path": "benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/meta.json", + "sha256": "1b77f548693f33ac3c095c77421effb238ee2ba0ebe28920a7934d7b7cb97ad2" + }, + { + "path": "benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/summary.json", + "sha256": "8ef178abba217091f9edf5f51fbe7c651bd7cc0451203965f837fcd0c7d5a013" + }, { "path": "benchmark/canvas_ladder/results.json", "sha256": "4502affb014de71e5c584e48cef5bc6a7631983e2517551d5ea9c20c8a0668e2" diff --git a/packages/flow/tests/test_auto_repair_cli.py b/packages/flow/tests/test_auto_repair_cli.py index 59a4b1167d..16d297c83c 100644 --- a/packages/flow/tests/test_auto_repair_cli.py +++ b/packages/flow/tests/test_auto_repair_cli.py @@ -187,3 +187,46 @@ def test_a_run_that_cannot_be_repaired_runs_plainly( assert code == 0, out assert (run_dir / "report.json").is_file() assert len(_records(base_url)) == 1 + + +def test_the_cli_repairs_without_a_shared_browser_from_its_caller( + mockmed, # noqa: F811 - the shared fixture + tmp_path, + capsys, +) -> None: + """The halted session stays open while the probe opens another. Outside + this module's fixture nothing shares one browser, and a second Playwright + driver on the same thread fails, so ``run`` must share one itself. Runs + on a fresh thread with the fixture's browser set aside, as a real + ``openadapt-flow run`` process would.""" + + from openadapt_flow.backends import _browser_process + + base_url, bundle = mockmed + _reset(base_url) + with _browser(): + cli.TARGET = _center_of(base_url, "#submit-encounter") + run_dir = tmp_path / "run" + stop = threading.Event() + answered: list[str] = [] + phone = threading.Thread( + target=cli._answer_on_phone, args=(run_dir, stop, answered), daemon=True + ) + phone.start() + saved = dict(_browser_process._SHARED_BROWSER) + _browser_process._SHARED_BROWSER.clear() + try: + _run( + bundle, + _config(tmp_path, base_url), + f"{base_url}/{QUERY}&drift=redesign#tasks", + run_dir, + ) + finally: + _browser_process._SHARED_BROWSER.update(saved) + stop.set() + phone.join(timeout=5) + cli.TARGET = None + out = capsys.readouterr().out + assert "Self-repair: resumed" in out, out + assert len(_records(base_url)) == 1 diff --git a/packages/flow/tests/test_self_repair_submit_binding.py b/packages/flow/tests/test_self_repair_submit_binding.py new file mode 100644 index 0000000000..16e56cf207 --- /dev/null +++ b/packages/flow/tests/test_self_repair_submit_binding.py @@ -0,0 +1,71 @@ +"""Self-repair keeps a visual save's submit binding (found on openIMIS, #1331). + +openIMIS's Save is an icon-only control recorded on the visual path, so its +step carries a ``submit_binding`` and no usable structural locator. When the +vendor relabelled it "Save claim", the agent proposed the right control, but +the patch put the DOM locator under the point on the anchor, and the patched +bundle no longer validated: a step may not carry both. The patch now keeps +such a step visual and keeps its binding. No browser. +""" + +from __future__ import annotations + +from openadapt_flow.ir import ( + ActionKind, + Anchor, + Step, + StructuralLocator, + SubmitBinding, + Workflow, +) +from openadapt_flow.repair.self_repair import _patched_structural + + +def _binding() -> SubmitBinding: + return SubmitBinding( + control_tag="button", + control_type="button", + fixed_position=True, + form_owner="fields_only", + submit_role="detached", + scope_kind="container", + scope_path="div>button", + filled_fields=3, + vertical="below", + horizontal="right", + ) + + +def _save(**fields) -> Step: + return Step( + id="step_015", + intent="click recorded visual target", + action=ActionKind.CLICK, + risk="irreversible", + anchor=Anchor( + template="templates/step_015.png", region=(0, 0, 10, 10), click_point=(5, 5) + ), + **fields, + ) + + +def _locator() -> StructuralLocator: + return StructuralLocator(selector="#save-claim", role="button", name="Save claim") + + +def test_a_visual_save_with_a_binding_stays_visual_and_keeps_it() -> None: + step = _save(submit_binding=_binding()) + kept = _patched_structural(step, _locator()) + assert kept is None + anchor = step.anchor.model_copy( + update={"template": "templates/step_015_repair.png", "structural": kept} + ) + patched = step.model_copy(update={"anchor": anchor}) + # The patched step validates, as the patched bundle must on load. + Workflow.model_validate({"name": "w", "steps": [patched.model_dump(mode="json")]}) + assert patched.submit_binding == step.submit_binding + + +def test_a_step_without_a_binding_takes_the_new_locator() -> None: + locator = _locator() + assert _patched_structural(_save(), locator) is locator From 2910720289a046b09d0fe797b75e71fa41a72255 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 13:57:19 -0400 Subject: [PATCH 08/10] fix(flow): count the model calls of a repair agent that gives up An agent that gave up returned None, so the call that decision took never reached the ledger: the MockMed run's ledger shows 18 calls and $0.24 where the agent's own log has 30 and $0.43. An agent can now give up with a proposal that has no action and still report its calls. The benchmark's agent does, and the README gives the numbers from the agent's log. Co-Authored-By: Claude Opus 5.5 --- packages/flow/benchmark/auto_repair/README.md | 11 +++++++-- .../benchmark/auto_repair/measure_mockmed.py | 3 ++- packages/flow/docs/SELF_REPAIR.md | 3 ++- .../flow/openadapt_flow/repair/self_repair.py | 11 ++++++++- packages/flow/tests/test_self_repair.py | 23 +++++++++++++++++++ 5 files changed, 46 insertions(+), 5 deletions(-) diff --git a/packages/flow/benchmark/auto_repair/README.md b/packages/flow/benchmark/auto_repair/README.md index 3427cfd86f..4ce9f9e673 100644 --- a/packages/flow/benchmark/auto_repair/README.md +++ b/packages/flow/benchmark/auto_repair/README.md @@ -38,7 +38,7 @@ before the save. | Approved (by the scripted person) | 6 | | Resumed and verified by the record check | **6 of 21 (29%)** | | Person decisions | 6 (one tap each) | -| Agent model calls, cost | 18 calls, $0.24 | +| Agent model calls, cost | 30 calls, $0.43 (from the agent's own log, `calls.json`) | By drift mode: @@ -53,6 +53,13 @@ By drift mode: | font, zoom | Yes, 3 of 3 each | Not repaired. These are false stops: the click acted, but a pixel `region_stable` postcondition failed after the font changed. No click fixes that, and the agent correctly gave up. | | ambiguous, missing, empty | Yes, 3 of 3 each | Never repaired, correctly. A duplicated or missing record goes to a person. | +The stop ledger in these results shows 18 calls and $0.24: when the agent +gave up, self-repair had no way to count the call that decision took. This PR +fixes that (an agent gives up with a proposal that has no action but reports +its calls), so new ledgers count every call; the numbers above come from the +agent's own log. The same applies to the openIMIS runs below: 7 calls and +$0.13 in run `b` by the agent's log. + ### MockMed worklist (`results/mockmed-worklist-2026-10-19-a/`) The vendor change ships after the pre-flight passes, so every item stops at @@ -150,4 +157,4 @@ with `OPENIMIS_STATE_DIR` pointing at the state the stack was created with), `pg8000` for the record check, and installed Chrome. The first two need `ANTHROPIC_API_KEY` (the customer's own key) and a -Playwright Chromium. They cost about $0.25 and $0.03. +Playwright Chromium. They cost about $0.45 and $0.03. diff --git a/packages/flow/benchmark/auto_repair/measure_mockmed.py b/packages/flow/benchmark/auto_repair/measure_mockmed.py index 3b560099cd..f770d4b8f2 100644 --- a/packages/flow/benchmark/auto_repair/measure_mockmed.py +++ b/packages/flow/benchmark/auto_repair/measure_mockmed.py @@ -224,7 +224,8 @@ def image(png: bytes) -> dict[str, Any]: if b.name in ("left_click", "double_click", "type") ][: request.max_actions] if not actions: - return None + # Give up, but report the call this decision took. + return RepairProposal(model_calls=1, cost_usd=cost) if len(actions) == 1: return RepairProposal(action=actions[0], model_calls=1, cost_usd=cost) return RepairProposal(actions=actions, model_calls=1, cost_usd=cost) diff --git a/packages/flow/docs/SELF_REPAIR.md b/packages/flow/docs/SELF_REPAIR.md index f53786bf16..e2fc111284 100644 --- a/packages/flow/docs/SELF_REPAIR.md +++ b/packages/flow/docs/SELF_REPAIR.md @@ -283,7 +283,8 @@ class MyAgent: - **One action.** A single-action proposal is one click or double click, matching the halted step's action. The agent may give a coordinate or, in the `playwright_mcp` dialect, a unique CSS selector. Anything else is refused. - Return `None` to give up. + To give up, return `RepairProposal(model_calls=..., cost_usd=...)` with no + action, so the calls it took are counted, or `None`. - **The customer's key.** An agent that calls a model must run on the customer's own key. Pass `model_key=load_customer_model_key("anthropic")` (see [JIT.md](JIT.md)). `SelfRepair` refuses to start without one unless the diff --git a/packages/flow/openadapt_flow/repair/self_repair.py b/packages/flow/openadapt_flow/repair/self_repair.py index 28734c14dd..932c8fa4b7 100644 --- a/packages/flow/openadapt_flow/repair/self_repair.py +++ b/packages/flow/openadapt_flow/repair/self_repair.py @@ -209,7 +209,8 @@ class RepairProposal: ``{"action": "left_click", "coordinate": [846, 457]}`` for ``anthropic`` or ``{"kind": "click", "x": 846, "y": 457}`` for ``generic``). ``actions`` is a bridge of up to ``request.max_actions`` such actions, - in order; give one or the other. Clicks may name a point or, where the + in order; give one or the other. Give neither to give up while still + reporting the model calls and cost it took to decide that. Clicks may name a point or, where the dialect allows, a CSS selector. A bridge may type only a parameter's value or text the screen shows at that moment. The agent reports the model calls it made and their cost; Flow records them and can't measure @@ -883,6 +884,14 @@ def _propose_until_probe_passes( detail = "the agent gave up" if proposal is None else "" calls = max(int(getattr(proposal, "model_calls", 0) or 0), 0) cost = max(float(getattr(proposal, "cost_usd", 0.0) or 0.0), 0.0) + if ( + proposal is not None + and proposal.action is None + and not proposal.actions + ): + # A proposal with no action gives up but still reports + # the model calls it took, so the ledger counts them. + proposal, detail = None, "the agent gave up" evidence.model_calls += calls evidence.est_model_cost_usd = round( evidence.est_model_cost_usd + cost, 6 diff --git a/packages/flow/tests/test_self_repair.py b/packages/flow/tests/test_self_repair.py index defbf4d4ad..7d9507c626 100644 --- a/packages/flow/tests/test_self_repair.py +++ b/packages/flow/tests/test_self_repair.py @@ -505,3 +505,26 @@ def geometry() -> Resolution: delivered = report(error=not_found) delivered.results[-1].delivery_attempted = None assert "may have reached" in _repairable(delivered, workflow) + + +class GivesUpWithCost(ScriptedAgent): + """Gives up on every attempt, reporting the model call each decision took.""" + + def propose(self, request: RepairRequest) -> Optional[RepairProposal]: + self.requests.append(request) + return RepairProposal(model_calls=1, cost_usd=0.02) + + +def test_a_give_up_still_counts_its_model_calls(mockmed, tmp_path) -> None: + base_url, bundle = mockmed + _reset(base_url) + agent = GivesUpWithCost() + with _browser(): + result = _self_repair(base_url, tmp_path, agent).run(bundle, tmp_path / "run") + assert result.status == "no_fix" + assert result.evidence.reason == "the agent gave up" + # The first give-up ends the search, and its call is in the ledger. + assert len(agent.requests) == 1 + assert result.evidence.model_calls == 1 + assert result.evidence.est_model_cost_usd == 0.02 + assert _records(base_url) == [] From 311129d02703860a9916be84d43e1fc098510c87 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 15:02:50 -0400 Subject: [PATCH 09/10] fix(flow): type the shared-browser choice by the CLI target, not the protocol Co-Authored-By: Claude Opus 5.5 --- .../flow/openadapt_flow/cli_self_repair.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/flow/openadapt_flow/cli_self_repair.py b/packages/flow/openadapt_flow/cli_self_repair.py index 78cef7ff77..84d7682615 100644 --- a/packages/flow/openadapt_flow/cli_self_repair.py +++ b/packages/flow/openadapt_flow/cli_self_repair.py @@ -403,24 +403,25 @@ def announce(run_dir: Path, request_id: str, request_sha256: str) -> None: flush=True, ) + target = CliTarget( + args, + bundle=bundle, + url=url, + cfg=cfg, + backend_cfg=backend_cfg, + gate_kwargs=gate_kwargs, + params=params, + worklists=worklists, + effect_verifier=effect_verifier, + api_actuator=api_actuator, + durable=durable, + allow_egress=allow_egress, + run_replayer=run_replayer, + approval_gate_for_run=approval_gate_for_run, + ) repair = SelfRepair( agent, - CliTarget( - args, - bundle=bundle, - url=url, - cfg=cfg, - backend_cfg=backend_cfg, - gate_kwargs=gate_kwargs, - params=params, - worklists=worklists, - effect_verifier=effect_verifier, - api_actuator=api_actuator, - durable=durable, - allow_egress=allow_egress, - run_replayer=run_replayer, - approval_gate_for_run=approval_gate_for_run, - ), + target, store=Path(args.repair_store), screen_access=ScreenAccess( enabled_by=args.repair_screen_access_by, @@ -453,7 +454,7 @@ def announce(run_dir: Path, request_id: str, request_sha256: str) -> None: # Playwright drivers can't run on one thread, so every session of # this run shares one browser process (each still gets a fresh # context: no cookies or storage carry over). - with shared_browser(headless=not repair.target.headed): + with shared_browser(headless=not target.headed): result = auto.run(bundle, run_dir) except SelfRepairRefused as exc: print(f"Self-repair stopped: {exc}. Nothing was saved by self-repair.") From 9aa29baf4bec69e67e7a769a9250c522dda34251 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 17:48:24 -0400 Subject: [PATCH 10/10] fix(flow): repair agent-compiled programs and stop false stops under page zoom Silent-saves 1.1.0 (#1380) stopped the compiled MockMed program on all 28 items of three UI-change blocks, and self-repair fixed none. Two causes are fixed here: - relink: a repaired copy of an admitted agent-compiled (JIT) program was refused by the run gate (DraftNotAdmitted), because its admission pins the digest every patch changes. A repair keeps the record check, so the admission now carries over: lifecycle.admit_repair writes a repair_probe admission (runs only behind a gate that stops before the save) and, after the person's approval or a signed standard change, a repair admission that embeds and re-verifies the prior one. A copy that changes the record check or parameters is refused. - zoom: at 125% a correct screen failed its region_stable postcondition (the content is 25% larger and further from the origin). The check now also looks at the page scale this step's own click showed, only when both axes agree, within one percent, at the same thresholds. A covered region fails. Measured on the same program and items: 18 of 28 now end correct (zoom 9 of 9 with no stop, relink 9 of 9 after one repair), 0 wrong saves. The What's New block (notice) still stops: its fix clicks "Continue to tasks", which the fail-closed write classifier reads as a possible save; the same classification causes #1377. Both need a person's classification. Co-Authored-By: Claude Opus 5.5 --- CAPABILITIES.md | 2 +- llms-full.txt | 9 +- packages/flow/benchmark/auto_repair/README.md | 54 ++ .../auto_repair/measure_silent_saves.py | 255 +++++++++ .../calls.json | 489 ++++++++++++++++++ .../silent-saves-1.1.0-2026-10-19-a/items.csv | 31 ++ .../ledger.jsonl | 13 + .../silent-saves-1.1.0-2026-10-19-a/meta.json | 25 + .../summary.json | 59 +++ packages/flow/docs/JIT.md | 7 + packages/flow/docs/LIMITS.md | 7 +- packages/flow/docs/SELF_REPAIR.md | 13 + packages/flow/openadapt_flow/ir.py | 5 + packages/flow/openadapt_flow/jit/lifecycle.py | 147 +++++- .../flow/openadapt_flow/repair/self_repair.py | 62 +++ .../flow/openadapt_flow/runtime/replayer.py | 140 ++++- packages/flow/public-artifacts.json | 20 + .../flow/tests/test_jit_repair_admission.py | 209 ++++++++ .../tests/test_page_scale_postcondition.py | 147 ++++++ 19 files changed, 1677 insertions(+), 17 deletions(-) create mode 100644 packages/flow/benchmark/auto_repair/measure_silent_saves.py create mode 100644 packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/calls.json create mode 100644 packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/items.csv create mode 100644 packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/ledger.jsonl create mode 100644 packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/meta.json create mode 100644 packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/summary.json create mode 100644 packages/flow/tests/test_jit_repair_admission.py create mode 100644 packages/flow/tests/test_page_scale_postcondition.py diff --git a/CAPABILITIES.md b/CAPABILITIES.md index 47ddf99fca..ac5471578e 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -203,7 +203,7 @@ improvising or retrying when something is off. | Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | On main ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts, scripted agent; a real one-shot agent repaired neither, see [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md)) | `run --self-repair`, [SELF_REPAIR.md](packages/flow/docs/SELF_REPAIR.md) | | Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](packages/flow/benchmark/pixel_scale/README.md) | | Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | -| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR stops; live openIMIS relabel: 3 of 3 reached repair, 0 resumed | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md) | +| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR stops; live openIMIS relabel: 3 of 3 reached repair, 0 resumed; silent-saves 1.1.0 UI-change blocks: 18 of 28 items that always stopped now correct, 0 wrong | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md) | | Fleet drift patches: a verified fix with no patient data, shared across sites under policy | Released, local reference hub | Simulator | `openadapt-flow fleet`, [FLEET_DRIFT_PATCHES.md](packages/flow/docs/FLEET_DRIFT_PATCHES.md) | ## Computer-use agents diff --git a/llms-full.txt b/llms-full.txt index 85db0dbf65..68dd3697ca 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -560,7 +560,7 @@ improvising or retrying when something is off. | Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | On main ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts, scripted agent; a real one-shot agent repaired neither, see [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md)) | `run --self-repair`, [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md) | | Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/pixel_scale/README.md) | | Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | -| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR stops; live openIMIS relabel: 3 of 3 reached repair, 0 resumed | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md) | +| Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | In review ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR stops; live openIMIS relabel: 3 of 3 reached repair, 0 resumed; silent-saves 1.1.0 UI-change blocks: 18 of 28 items that always stopped now correct, 0 wrong | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md) | | Fleet drift patches: a verified fix with no patient data, shared across sites under policy | Released, local reference hub | Simulator | `openadapt-flow fleet`, [FLEET_DRIFT_PATCHES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/FLEET_DRIFT_PATCHES.md) | ## Computer-use agents @@ -1081,7 +1081,12 @@ required choice, or a pixel postcondition that failed after a font change (0 of 15). On openIMIS's relabelled Save, once a person reviewed the two steps whose risk was unclassified, the stop reached repair 3 of 3, but no fix passed the probe: the replay sees the Save disabled where the fix was cropped -from the enabled one. A compiled program is also reused only for the record check and +from the enabled one. On silent-saves 1.1.0's UI-change blocks (#1380), 18 +of the 28 items that stopped every time now end correctly: a page zoom no +longer fails a correct screen's check, and a repaired agent-compiled program +keeps its admission. A What's New screen dismissed by "Continue to tasks" still +stops every item: "Continue" can submit, so a person must classify that +control first. A compiled program is also reused only for the record check and app environment it was admitted under ([JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md)). ## The main incorrect-success risks diff --git a/packages/flow/benchmark/auto_repair/README.md b/packages/flow/benchmark/auto_repair/README.md index 4ce9f9e673..b2c011a487 100644 --- a/packages/flow/benchmark/auto_repair/README.md +++ b/packages/flow/benchmark/auto_repair/README.md @@ -131,6 +131,60 @@ healthy ones included, at `step_000`: the program was recorded in installed Chrome, and the bundled Chromium renders the form a few pixels differently. It wrote nothing and is not counted. +### Silent-saves 1.1.0, the three UI-change blocks (`results/silent-saves-1.1.0-2026-10-19-a/`) + +In silent-saves 1.1.0 ([#1380](https://github.com/OpenAdaptAI/OpenAdapt/pull/1380)), +the compiled MockMed program (arms 3 and 3r) stopped on all 28 items with no +server fault in three blocks: display scaling (`zoom`, items 26-35), a What's +New screen (`notice`, 46-55) and a moved New Encounter control (`relink`, +76-85). Self-repair fixed none of them. `measure_silent_saves.py` re-ran those +30 items through the benchmark's own app, schedule and store judge, with the +default loop, the same admitted program, and the bridge-capable agent. + +| Block | Items with no server fault | Ended correct, #1380 (arm 3r) | Ended correct, now | How | +| --- | --- | --- | --- | --- | +| zoom | 9 | 0 | **9** | no stop: the postcondition check now finds its content at the page's scale | +| relink | 9 | 0 | **9** | 1 repair, approved once; the other 8 ran the repaired program | +| notice | 10 | 0 | 0 | not repaired: needs a person's classification of "Continue to tasks" | +| Total | 28 | 0 | **18** | 0 wrong saves | + +The two server-fault items in these blocks ended as before, without a silent +save: item 30 (`wrong_record`, flagged) and item 80 (`optimistic`, stopped). +The agent made 21 calls ($0.31): 10 of them on the `notice` items, one per +item, because a single run repairs only its own stop (a `WorklistQueue` asks +once per group). + +Why each block failed in #1380, and what changed: + +- **`relink`: a repaired agent-compiled program could never run.** The + program was compiled from an agent run (JIT), and its admission pins its + digest. Every patch changes the digest, so the run gate refused the + probe (`DraftNotAdmitted`), even though the agent proposed the right + control ("Start visit"). A repair keeps the record check, so the + admission now carries over (`jit.lifecycle.admit_repair`): a + `repair_probe` admission lets the copy run only behind a gate that stops + before the save, and a `repair` admission follows the person's approval. + A copy that changes the record check or the parameters is refused. +- **`zoom`: a false stop on a correct screen.** At 125% every element is 25% + larger and 25% further from the top-left corner, so a `region_stable` + postcondition's recorded content fell outside its search window and scale + band. In #1380's engine self-repair also declined the halt; with the + bridge (#1325) it took it, but no click can fix a correct screen. The + check now also looks at the scale this step's own click showed, only + when both axes agree (`_uniform_page_scale`), within one percent of it, + with the unchanged thresholds. A covered or missing region still fails + (`tests/test_page_scale_postcondition.py`). No model is asked. +- **`notice`: not fixed.** The fix is two clicks, "Continue to tasks" and + then "Open". The fail-closed write classifier reads "Continue" as a + possible save (a wizard's Continue often submits), so self-repair refuses + a patch that clicks it, and the probe could not pass it either. Clicking + it on a real record to find out is what the probe exists to avoid. The + same classification causes #1377: in arm 2, compiling a recording that + dismisses the screen gives that click a placeholder `record_written` + effect, so the canary refuses the program. Both need a person to classify + that control once as saving nothing (a reviewed risk classification, as + `qualify set-risk` does for openIMIS); neither is changed here. + ## What this does not show - The approver is scripted, so the time to answer (milliseconds) is not a diff --git a/packages/flow/benchmark/auto_repair/measure_silent_saves.py b/packages/flow/benchmark/auto_repair/measure_silent_saves.py new file mode 100644 index 0000000000..8a80684b0d --- /dev/null +++ b/packages/flow/benchmark/auto_repair/measure_silent_saves.py @@ -0,0 +1,255 @@ +"""Re-run silent-saves 1.1.0's UI-change blocks under the default repair loop. + + python benchmark/auto_repair/measure_silent_saves.py --run-id 2026-10-19-a + +Needs ``benchmark/silent_saves`` from #1380 (fault set 1.1.0) on the path, and +the admitted MockMed program arms 3 and 3r ran (``~/.cache/silent-saves/...``, +admitted on ``http://127.0.0.1:18765``, so the app is served on that port). + +Arms 3 and 3r of #1380 stopped on every item of three blocks: display scaling +(``zoom``, items 26-35), a What's New screen (``notice``, 46-55) and a moved +New Encounter control (``relink``, 76-85). This runs those 30 items (28 with +no server fault, plus one ``wrong_record`` and one ``optimistic`` item) the way +arm 3r does, through the benchmark's own app, schedule and store judge, but +with the loop ``openadapt-flow run`` now uses by default +(:class:`openadapt_flow.repair.auto_repair.AutoRepair`) and the bridge-capable +agent from ``measure_mockmed.py``. As in arm 3r, after a repair resumes and +verifies, later items run the repaired program. The approver is scripted. + +Outputs (committed): ``results/silent-saves-1.1.0-/`` with +``items.csv`` (one row per item, scored by ``silent_saves.judge``), +``ledger.jsonl``, ``summary.json``, ``calls.json`` and ``meta.json``. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import shutil +import sys +import time +from dataclasses import asdict +from pathlib import Path +from typing import Any, Optional + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(HERE.parent)) + +from measure_mockmed import ( # noqa: E402 + APPROVER, + CACHE, + BridgeRepairAgent, + _approver, + _git_head, +) + +BLOCKS = {"zoom": range(26, 36), "notice": range(46, 56), "relink": range(76, 86)} +PORT = 18765 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--run-id", required=True) + parser.add_argument("--blocks", default=",".join(BLOCKS)) + parser.add_argument("--model", default="claude-sonnet-5-5") + args = parser.parse_args() + + from silent_saves import judge, protocol + from silent_saves.app import MockMed + from silent_saves.repair_arm import DEFAULT_BUNDLE, SelfRepairProgram + + from openadapt_flow.backends._browser_process import shared_browser + from openadapt_flow.jit import load_customer_model_key + from openadapt_flow.repair.auto_repair import AutoRepair, RepairLedger, summarize + from openadapt_flow.repair.self_repair import ( + ReplayTarget, + ScreenAccess, + SelfRepair, + ) + from openadapt_flow.runtime.effects import RestRecordVerifier + + blocks = [b for b in args.blocks.split(",") if b] + wanted = {i for b in blocks for i in BLOCKS[b]} + items = [ + it for it in protocol.plan_change(protocol.load("1.1.0")) if it.item in wanted + ] + out = HERE / "results" / f"silent-saves-1.1.0-{args.run_id}" + out.mkdir(parents=True, exist_ok=True) + work = CACHE / f"silent-saves-{args.run_id}" + shutil.rmtree(work, ignore_errors=True) + ledger = RepairLedger(out) + ledger.path = out / "ledger.jsonl" + ledger.path.unlink(missing_ok=True) + key = load_customer_model_key("anthropic") + agent = BridgeRepairAgent(model=args.model) + (out / "meta.json").write_text( + json.dumps( + { + "written_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "engine_git_head": _git_head(), + "benchmark": "silent-saves 1.1.0 (#1380), UI-change blocks only", + "blocks": {b: [BLOCKS[b].start, BLOCKS[b].stop - 1] for b in blocks}, + "program": str(DEFAULT_BUNDLE).replace(str(Path.home()), "~"), + "loop": "AutoRepair over SelfRepair (bridge up to 3); target as arm 3r", + "agent": agent.agent_id, + "approver": APPROVER, + "fault_probe": "missing (Jane Sample's referral removed)", + }, + indent=2, + ) + + "\n" + ) + app = MockMed(port=PORT) + arm = SelfRepairProgram(app) + bundle = arm.original + rows: list[dict[str, Any]] = [] + try: + with shared_browser(headless=True): + for it in items: + app.prepare(it.fault, it.drift, it.slow_ms, it.idempotency_key) + before = app.records() + item_dir = work / f"item-{it.item:03d}" + item_dir.mkdir(parents=True) + arm._state = (app.fault, app.drift, app.slow_ms, app.idempotency_key) + target = ReplayTarget( + arm._open, + effect_verifier=RestRecordVerifier( + app.store_base, records_path="/api/db", records_key="records" + ), + policy="clinical-write", + profile="standard", + params=dict(it.params), + entry_url=app.url, + execution_origin=app.base, + ) + decisions: list[float] = [] + spent0 = agent.cost_usd + repair = SelfRepair( + agent, + target, + store=work / "repair-store", + screen_access=ScreenAccess( + enabled_by="benchmark-operator (synthetic data only)", + mode="allow_unscrubbed", + ), + model_key=key, + fault_probes=("missing",), + approval_wait_s=120, + sleep=_approver(item_dir, decisions), + ) + loop = AutoRepair( + repair, + ledger=ledger, + labels={ + "app": "mockmed", + "benchmark": "silent-saves-1.1.0", + "drift": ",".join(it.drift), + "fault": it.fault, + "item": str(it.item), + }, + ) + t0 = time.time() + status, reason, claimed = "harness_error", "", False + result: Optional[Any] = None + try: + result = loop.run(bundle, item_dir / "run") + status, reason = result.status, result.evidence.reason + claimed = bool(result.verified) + except Exception as exc: # noqa: BLE001 - a harness error is a row + reason = f"{type(exc).__name__}: {exc}" + finally: + app.fault, app.drift, app.slow_ms, app.idempotency_key = arm._state + wall = time.time() - t0 + time.sleep(0.5) + after = app.records() + verdict = judge.store_verdict(before, after, note=it.params["note"]) + outcome = judge.outcome(claimed, verdict, store_changed=before != after) + if ( + result is not None + and result.status == "resumed" + and result.verified + and result.patched_bundle + ): + bundle = Path(result.patched_bundle) + rows.append( + { + "item": it.item, + "drift": ",".join(it.drift), + "fault": it.fault, + "expect": it.expect, + "repair_status": status, + "claimed_success": claimed, + "store_verdict": verdict, + "outcome": outcome, + "ran_repaired_program": bundle != arm.original, + "cost_usd": round(agent.cost_usd - spent0, 6), + "wall_s": round(wall, 1), + "reason": (reason or "")[:300], + } + ) + print( + f"{it.item:3d} {','.join(it.drift):8s} {it.fault:12s} " + f"{status:16s} {outcome:14s} ${agent.cost_usd - spent0:.3f} " + f"{wall:.0f}s {reason[:100]}", + flush=True, + ) + finally: + app.close() + if rows: + with (out / "items.csv").open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + (out / "calls.json").write_text( + json.dumps( + { + "model": agent.model, + "cost_usd": round(agent.cost_usd, 6), + "calls": [asdict(c) for c in agent.calls], + }, + indent=2, + ) + + "\n" + ) + if ledger.path.is_file(): + ledger.path.write_text( + ledger.path.read_text().replace( + str(CACHE), "~/.cache/openadapt-auto-repair" + ) + ) + by_block = {} + for b in blocks: + mine = [r for r in rows if r["drift"] == b] + ok_items = [r for r in mine if r["fault"] == "ok"] + by_block[b] = { + "items": len(mine), + "ok_items": len(ok_items), + "ok_items_correct": sum(r["outcome"] == "correct" for r in ok_items), + "outcomes": { + o: sum(r["outcome"] == o for r in mine) + for o in sorted({r["outcome"] for r in mine}) + }, + } + (out / "summary.json").write_text( + json.dumps( + { + "ok_items": sum(b["ok_items"] for b in by_block.values()), + "ok_items_correct": sum( + b["ok_items_correct"] for b in by_block.values() + ), + "by_block": by_block, + "funnel": summarize(ledger.rows()) if ledger.path.is_file() else {}, + "agent_calls": len(agent.calls), + "agent_cost_usd": round(agent.cost_usd, 6), + }, + indent=2, + ) + + "\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/calls.json b/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/calls.json new file mode 100644 index 0000000000..c13fb18e30 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/calls.json @@ -0,0 +1,489 @@ +{ + "model": "claude-sonnet-5-5", + "cost_usd": 0.311728, + "calls": [ + { + "turn": 1, + "started_s": 1792445889.967665, + "ended_s": 1792445893.2962801, + "input_tokens": 5762, + "output_tokens": 170, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013224, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 130, + 235 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgCKSDqrFErsiJvPuZCPs", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792445893.586915, + "ended_s": 1792445898.4579499, + "input_tokens": 5818, + "output_tokens": 532, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.016956, + "actions": [], + "request_id": "req_011CgCKSV9Lt8uGK6eSzThHc", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792445909.7038631, + "ended_s": 1792445912.1232672, + "input_tokens": 5762, + "output_tokens": 169, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013214, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 130, + 236 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgCKTg3TSaqGgiATZR2iH", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792445912.349932, + "ended_s": 1792445916.865508, + "input_tokens": 5818, + "output_tokens": 486, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.016496, + "actions": [], + "request_id": "req_011CgCKTsMMtc6ws8EV7uSq5", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792445928.278692, + "ended_s": 1792445929.946601, + "input_tokens": 5762, + "output_tokens": 145, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.012974, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 129, + 235 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgCKV3Tqs7ZAyK7cmsHFb", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792445930.339893, + "ended_s": 1792445935.056223, + "input_tokens": 5818, + "output_tokens": 514, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.016776, + "actions": [], + "request_id": "req_011CgCKVCHAWoki1qoDtdun5", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792445946.624925, + "ended_s": 1792445948.6724758, + "input_tokens": 5762, + "output_tokens": 150, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013024, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 130, + 236 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgCKWPwvTkPhkJn5Zjcoa", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792445948.687715, + "ended_s": 1792445953.816659, + "input_tokens": 5818, + "output_tokens": 504, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.016676, + "actions": [], + "request_id": "req_011CgCKWYjW7NxjC4r1g8k6Q", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792445965.1404438, + "ended_s": 1792445967.960892, + "input_tokens": 5762, + "output_tokens": 166, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013184, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 130, + 236 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgCKXm7AztSsfuC2RYTce", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792445967.98344, + "ended_s": 1792445972.407252, + "input_tokens": 5818, + "output_tokens": 497, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.016606, + "actions": [], + "request_id": "req_011CgCKXyKPemneUaYAAv71F", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792445983.7640169, + "ended_s": 1792445985.495441, + "input_tokens": 5762, + "output_tokens": 156, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013084, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 129, + 235 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgCKZ8gyZmrtaGPk3G9VJ", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792445985.5144022, + "ended_s": 1792445991.727114, + "input_tokens": 5818, + "output_tokens": 538, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.017016, + "actions": [], + "request_id": "req_011CgCKZGEAbkTVhYxdv5Ar3", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792446003.038228, + "ended_s": 1792446004.829916, + "input_tokens": 5762, + "output_tokens": 167, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013194, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 129, + 235 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgCKaZ8S7omJ5hBa58xPL", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792446004.851601, + "ended_s": 1792446009.862905, + "input_tokens": 5818, + "output_tokens": 487, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.016506, + "actions": [], + "request_id": "req_011CgCKagp39KvBo8453FRue", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792446021.1567261, + "ended_s": 1792446023.0371342, + "input_tokens": 5762, + "output_tokens": 149, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013014, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 130, + 235 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgCKbtgwdeVWoweSR15xk", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792446023.058392, + "ended_s": 1792446027.832067, + "input_tokens": 5818, + "output_tokens": 527, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.016906, + "actions": [], + "request_id": "req_011CgCKc2itQwWQzfooNLR7x", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792446039.113469, + "ended_s": 1792446041.672343, + "input_tokens": 5762, + "output_tokens": 148, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013004, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 130, + 236 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgCKdDKQ67mAuSpdk7Uex", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792446041.691749, + "ended_s": 1792446046.180966, + "input_tokens": 5818, + "output_tokens": 525, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.016886, + "actions": [], + "request_id": "req_011CgCKdQNB2yCa14c4FSVfc", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792446057.485377, + "ended_s": 1792446059.925178, + "input_tokens": 5762, + "output_tokens": 152, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013044, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 129, + 235 + ] + } + }, + { + "name": "wait", + "input": { + "duration": 1 + } + }, + { + "name": "screenshot", + "input": {} + } + ], + "request_id": "req_011CgCKeZwQiMVVCXwR7fcKE", + "stop_reason": "tool_use" + }, + { + "turn": 2, + "started_s": 1792446059.9449291, + "ended_s": 1792446064.730947, + "input_tokens": 5818, + "output_tokens": 502, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.016656, + "actions": [], + "request_id": "req_011CgCKekadQeKeSrYFoaT4Q", + "stop_reason": "end_turn" + }, + { + "turn": 1, + "started_s": 1792446067.068005, + "ended_s": 1792446069.054962, + "input_tokens": 5789, + "output_tokens": 171, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cost_usd": 0.013288, + "actions": [ + { + "name": "left_click", + "input": { + "coordinate": [ + 860, + 159 + ] + } + } + ], + "request_id": "req_011CgCKfGr3FebHccYLcjHWv", + "stop_reason": "tool_use" + } + ] +} diff --git a/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/items.csv b/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/items.csv new file mode 100644 index 0000000000..9e19586a48 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/items.csv @@ -0,0 +1,31 @@ +item,drift,fault,expect,repair_status,claimed_success,store_verdict,outcome,ran_repaired_program,cost_usd,wall_s,reason +26,zoom,ok,save,not_needed,True,correct,correct,False,0,6.6, +27,zoom,ok,save,not_needed,True,correct,correct,False,0,6.1, +28,zoom,ok,save,not_needed,True,correct,correct,False,0,6.1, +29,zoom,ok,save,not_needed,True,correct,correct,False,0,6.1, +30,zoom,wrong_record,save,not_repairable,False,wrong_record,flagged_wrong,False,0,12.0,the run ended RECONCILIATION_REQUIRED; self-repair handles only a halt before any effect +31,zoom,ok,save,not_needed,True,correct,correct,False,0,6.1, +32,zoom,ok,save,not_needed,True,correct,correct,False,0,6.2, +33,zoom,ok,save,not_needed,True,correct,correct,False,0,6.1, +34,zoom,ok,save,not_needed,True,correct,correct,False,0,6.1, +35,zoom,ok,save,not_needed,True,correct,correct,False,0,6.2, +46,notice,ok,save,no_fix,False,missing,safe_stop,False,0.03018,20.2,the agent gave up +47,notice,ok,save,no_fix,False,missing,safe_stop,False,0.02971,17.9,the agent gave up +48,notice,ok,save,no_fix,False,missing,safe_stop,False,0.02975,17.7,the agent gave up +49,notice,ok,save,no_fix,False,missing,safe_stop,False,0.0297,18.2,the agent gave up +50,notice,ok,save,no_fix,False,missing,safe_stop,False,0.02979,18.1,the agent gave up +51,notice,ok,save,no_fix,False,missing,safe_stop,False,0.0301,18.8,the agent gave up +52,notice,ok,save,no_fix,False,missing,safe_stop,False,0.0297,17.6,the agent gave up +53,notice,ok,save,no_fix,False,missing,safe_stop,False,0.02992,17.5,the agent gave up +54,notice,ok,save,no_fix,False,missing,safe_stop,False,0.02989,17.8,the agent gave up +55,notice,ok,save,no_fix,False,missing,safe_stop,False,0.0297,18.0,the agent gave up +76,relink,ok,save,resumed,True,correct,correct,True,0.013288,14.3, +77,relink,ok,save,not_needed,True,correct,correct,True,0.0,5.9, +78,relink,ok,save,not_needed,True,correct,correct,True,0.0,5.9, +79,relink,ok,save,not_needed,True,correct,correct,True,0.0,5.8, +80,relink,optimistic,save,not_repairable,False,missing,safe_stop,True,0.0,11.4,the run ended RECONCILIATION_REQUIRED; self-repair handles only a halt before any effect +81,relink,ok,save,not_needed,True,correct,correct,True,0.0,5.9, +82,relink,ok,save,not_needed,True,correct,correct,True,0.0,5.9, +83,relink,ok,save,not_needed,True,correct,correct,True,0.0,5.7, +84,relink,ok,save,not_needed,True,correct,correct,True,0.0,5.9, +85,relink,ok,save,not_needed,True,correct,correct,True,0.0,5.9, diff --git a/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/ledger.jsonl b/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/ledger.jsonl new file mode 100644 index 0000000000..09a917554b --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/ledger.jsonl @@ -0,0 +1,13 @@ +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:37:24+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_005", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "zoom", "fault": "wrong_record", "item": "30"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the run ended RECONCILIATION_REQUIRED; self-repair handles only a halt before any effect", "repair_s": 11.981, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically; reconcile: read the change ledger in REPORT.md, then `openadapt-flow undo list ~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-030/run`", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-030/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:38:18+00:00", "cases_passed": false, "est_model_cost_usd": 0.03018, "halted_step_id": "step_000", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "notice", "fault": "ok", "item": "46"}, "model_calls": 2, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 20.221, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-046/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:38:36+00:00", "cases_passed": false, "est_model_cost_usd": 0.02971, "halted_step_id": "step_000", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "notice", "fault": "ok", "item": "47"}, "model_calls": 2, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 17.904, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-047/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:38:55+00:00", "cases_passed": false, "est_model_cost_usd": 0.02975, "halted_step_id": "step_000", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "notice", "fault": "ok", "item": "48"}, "model_calls": 2, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 17.686, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-048/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:39:13+00:00", "cases_passed": false, "est_model_cost_usd": 0.0297, "halted_step_id": "step_000", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "notice", "fault": "ok", "item": "49"}, "model_calls": 2, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 18.246, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-049/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:39:32+00:00", "cases_passed": false, "est_model_cost_usd": 0.02979, "halted_step_id": "step_000", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "notice", "fault": "ok", "item": "50"}, "model_calls": 2, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 18.097, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-050/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:39:51+00:00", "cases_passed": false, "est_model_cost_usd": 0.0301, "halted_step_id": "step_000", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "notice", "fault": "ok", "item": "51"}, "model_calls": 2, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 18.8, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-051/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:40:09+00:00", "cases_passed": false, "est_model_cost_usd": 0.0297, "halted_step_id": "step_000", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "notice", "fault": "ok", "item": "52"}, "model_calls": 2, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 17.632, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-052/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:40:27+00:00", "cases_passed": false, "est_model_cost_usd": 0.02992, "halted_step_id": "step_000", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "notice", "fault": "ok", "item": "53"}, "model_calls": 2, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 17.465, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-053/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:40:46+00:00", "cases_passed": false, "est_model_cost_usd": 0.02989, "halted_step_id": "step_000", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "notice", "fault": "ok", "item": "54"}, "model_calls": 2, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 17.835, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-054/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:41:04+00:00", "cases_passed": false, "est_model_cost_usd": 0.0297, "halted_step_id": "step_000", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "notice", "fault": "ok", "item": "55"}, "model_calls": 2, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 1, "released_by": null, "repair_reason": "the agent gave up", "repair_s": 18.047, "repair_status": "no_fix", "resumed": false, "resumed_outcome": null, "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-055/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": "decision_portal:person", "approved_via": "person", "at": "2026-10-19T21:41:19+00:00", "cases_passed": true, "est_model_cost_usd": 0.013288, "halted_step_id": "step_001", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "relink", "fault": "ok", "item": "76"}, "model_calls": 1, "outcome": "HALTED_BEFORE_EFFECT", "person_decisions": 1, "person_wait_s": 0.004, "probe_passed": true, "proposed": 1, "released_by": null, "repair_reason": "", "repair_s": 14.308, "repair_status": "resumed", "resumed": true, "resumed_outcome": "VERIFIED", "route": "repair", "route_reason": "", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-076/run", "scope": "item", "workflow": "mockmed.add_note"} +{"approved_by": null, "approved_via": null, "at": "2026-10-19T21:41:50+00:00", "cases_passed": false, "est_model_cost_usd": 0.0, "halted_step_id": "step_005", "labels": {"app": "mockmed", "benchmark": "silent-saves-1.1.0", "drift": "relink", "fault": "optimistic", "item": "80"}, "model_calls": 0, "outcome": "RECONCILIATION_REQUIRED", "person_decisions": 0, "person_wait_s": null, "probe_passed": false, "proposed": 0, "released_by": null, "repair_reason": "the run ended RECONCILIATION_REQUIRED; self-repair handles only a halt before any effect", "repair_s": 11.403, "repair_status": "not_repairable", "resumed": false, "resumed_outcome": null, "route": "reconciliation", "route_reason": "a write may have happened before the stop; it is never repaired automatically; reconcile: read the change ledger in REPORT.md, then `openadapt-flow undo list ~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-080/run`", "run_dir": "~/.cache/openadapt-auto-repair/silent-saves-2026-10-19-a/item-080/run", "scope": "item", "workflow": "mockmed.add_note"} diff --git a/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/meta.json b/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/meta.json new file mode 100644 index 0000000000..2d5cc36fdd --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/meta.json @@ -0,0 +1,25 @@ +{ + "written_at": "2026-10-19T17:36:44-0400", + "engine_git_head": "4e770401a669d4f36b207003dae280c720e03f96", + "benchmark": "silent-saves 1.1.0 (#1380), UI-change blocks only", + "blocks": { + "zoom": [ + 26, + 35 + ], + "notice": [ + 46, + 55 + ], + "relink": [ + 76, + 85 + ] + }, + "program": "~/.cache/silent-saves/program-2026-10-18-a/mockmed.add_note-48f60d6be84e/program/bundle", + "loop": "AutoRepair over SelfRepair (bridge up to 3); target as arm 3r", + "agent": "agent:anthropic:claude-sonnet-5-5:auto-repair-benchmark", + "approver": "benchmark:scripted-approver (stands in for a person; approves all)", + "fault_probe": "missing (Jane Sample's referral removed)", + "engine_note": "Engine: this PR's branch plus #1380's benchmark/silent_saves merged locally (measure/silent-saves-repair), with the carried JIT repair admission and the page-scale region_stable check applied. #1380's arms 3 and 3r ran the same program and items on an earlier engine." +} diff --git a/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/summary.json b/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/summary.json new file mode 100644 index 0000000000..e733231f00 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/summary.json @@ -0,0 +1,59 @@ +{ + "ok_items": 28, + "ok_items_correct": 18, + "by_block": { + "zoom": { + "items": 10, + "ok_items": 9, + "ok_items_correct": 9, + "outcomes": { + "correct": 9, + "flagged_wrong": 1 + } + }, + "notice": { + "items": 10, + "ok_items": 10, + "ok_items_correct": 0, + "outcomes": { + "safe_stop": 10 + } + }, + "relink": { + "items": 10, + "ok_items": 9, + "ok_items_correct": 9, + "outcomes": { + "correct": 9, + "safe_stop": 1 + } + } + }, + "funnel": { + "stops": 13, + "to_repair": 11, + "to_reconciliation": 2, + "to_person": 0, + "auto_repair_off": 0, + "repair_runs": 11, + "repairs_proposed": 11, + "proposals": 11, + "probe_passed": 1, + "approved": 1, + "approved_by_person": 1, + "approved_by_standard_change": 0, + "resumed_items": 1, + "resumed_not_verified": 0, + "repair_success_rate": 0.0909, + "person_decisions": 1, + "person_wait_s_total": 0.004, + "model_calls": 21, + "est_model_cost_usd": 0.311728, + "repair_status_counts": { + "no_fix": 10, + "resumed": 1 + } + }, + "agent_calls": 21, + "agent_cost_usd": 0.311728 +} diff --git a/packages/flow/docs/JIT.md b/packages/flow/docs/JIT.md index b697c5f537..0c897b0174 100644 --- a/packages/flow/docs/JIT.md +++ b/packages/flow/docs/JIT.md @@ -126,6 +126,13 @@ Reuse is also bound to what the program checks and where it runs: that reads it before each run. A program admitted under `7.0.2` is refused under `7.0.3` at the same origin, and the inbox entry says why; run a canary on the new version. A function that fails is a refusal too. +- **Repairs keep the admission.** A self-repair (`openadapt_flow.repair`) + changes the bundle, and so its digest, but never its record check. The + repaired copy gets an admission that embeds the prior one + (`lifecycle.admit_repair`): first a `repair_probe` admission, which lets it + run only as a probe that stops before the save, then, once a person (or a + signed standard change) approved the fix, a `repair` admission for every + run. A copy whose record check or parameters changed gets neither. - **Pins in the key.** `TaskSignature(..., record_check=, environment=)` puts both in the cache key and refuses a program that doesn't match them. Without pins the key is the same as before, diff --git a/packages/flow/docs/LIMITS.md b/packages/flow/docs/LIMITS.md index b949f083b6..01c05581bd 100644 --- a/packages/flow/docs/LIMITS.md +++ b/packages/flow/docs/LIMITS.md @@ -141,7 +141,12 @@ required choice, or a pixel postcondition that failed after a font change (0 of 15). On openIMIS's relabelled Save, once a person reviewed the two steps whose risk was unclassified, the stop reached repair 3 of 3, but no fix passed the probe: the replay sees the Save disabled where the fix was cropped -from the enabled one. A compiled program is also reused only for the record check and +from the enabled one. On silent-saves 1.1.0's UI-change blocks (#1380), 18 +of the 28 items that stopped every time now end correctly: a page zoom no +longer fails a correct screen's check, and a repaired agent-compiled program +keeps its admission. A What's New screen dismissed by "Continue to tasks" still +stops every item: "Continue" can submit, so a person must classify that +control first. A compiled program is also reused only for the record check and app environment it was admitted under ([JIT.md](JIT.md)). ## The main incorrect-success risks diff --git a/packages/flow/docs/SELF_REPAIR.md b/packages/flow/docs/SELF_REPAIR.md index e2fc111284..173bd2e28a 100644 --- a/packages/flow/docs/SELF_REPAIR.md +++ b/packages/flow/docs/SELF_REPAIR.md @@ -593,6 +593,7 @@ Data, scripts and the full per-mode table are in | OpenEMR referral, #1347's recorded runs | 8 | 0 (all `RECONCILIATION_REQUIRED`) | - | - | - | 0 | - | - | | openIMIS relabelled Save (#1331), before #1337 | 3 | 0 (all `RECONCILIATION_REQUIRED`) | - | - | - | 0 | - | - | | openIMIS relabelled Save, live, with #1337 and a scripted risk review | 3 | **3** | 2 | 0 | 0 | 0 | 0 | 0 | +| Silent-saves 1.1.0, the UI-change blocks that stopped arms 3 and 3r (#1380): `zoom`, `notice`, `relink`, 28 items with no server fault | 13 (before: 28) | 11 | 11 | 1 | 1 | **1 repaired, then 8 more items on the repaired program; 9 `zoom` items no longer stop: 18 of 28 correct** | 0 | 1 | What worked and what didn't: @@ -624,6 +625,15 @@ What worked and what didn't: validation, and the probe couldn't open a second browser session from the CLI. +- **Silent-saves 1.1.0:** 18 of the 28 items that stopped every time in #1380 + now end with the correct record. A repaired agent-compiled program now + keeps its admission (the `relink` repair could not even be probed before), + and a `region_stable` check now holds at the page scale its step's click + showed (`zoom` no longer stops). The What's New block still stops: its fix + clicks "Continue to tasks", which the write classifier reads as a possible + save, until a person classifies it. Details: + [benchmark/auto_repair](../benchmark/auto_repair/README.md). + The approver was scripted, so its answer times (milliseconds) are not a person's. The measure of a person's time here is the count of decisions: one tap per repaired stop, or one per group of worklist items that stopped the @@ -631,6 +641,9 @@ same way. ## Limits +- A fix that clicks a control the write classifier reads as a possible save + (for example "Continue") is refused, even on a dismiss screen, until a + person classifies that control. - A patch's new template is cropped from the screen at the halt. When the app shows the control in another state during the replay (openIMIS shows its Save disabled until the form validates), the probe doesn't find it and diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index 1dcce099a7..afca78798b 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -3926,6 +3926,11 @@ class StepResult(BaseModel): # (recorded for audit; empty unless an appliance is configured). postcondition_drift_rescues: list[str] = Field(default_factory=list) drift_oracle_calls: int = 0 # VLM state-verifier calls this step + #: The uniform page scale (for example 1.25 under 125% zoom) at which a + #: ``region_stable`` check found its recorded content, when it was found + #: only there. Set only when this step's own click showed the same scale. + #: Deterministic: no model is asked. ``None`` otherwise. + postcondition_page_scale: Optional[float] = None #: READ steps only: the value read, with its channels, grade and region. field_read: Optional[FieldRead] = None # The first identity check on this step mismatched WHILE the screen was diff --git a/packages/flow/openadapt_flow/jit/lifecycle.py b/packages/flow/openadapt_flow/jit/lifecycle.py index ddea31dbf8..817bffba36 100644 --- a/packages/flow/openadapt_flow/jit/lifecycle.py +++ b/packages/flow/openadapt_flow/jit/lifecycle.py @@ -112,7 +112,14 @@ CANARIES_FILE = "canaries.jsonl" #: Written when a draft is outside an agent's admission policy. NEEDS_PERSON_FILE = "needs_person.json" -ADMISSION_MODES = ("person", "admitter_key", "agent_policy") +#: An admission carried to a repaired copy of an admitted program, which keeps +#: the program's record check (see :func:`admit_repair`). ``repair_probe`` +#: lets the copy run only behind a gate that stops before the first save; +#: ``repair`` follows a person's approval, or a signed standard change. +REPAIR_MODES = ("repair", "repair_probe") +ADMISSION_MODES = ("person", "admitter_key", "agent_policy", *REPAIR_MODES) +#: How a standard change names itself as the approver of a repair. +STANDARD_CHANGE_APPROVER = "standard-change:" ADMISSION_SCHEMA = "openadapt-flow/jit-admission/v1" KEY_DIR_ENV = "OPENADAPT_JIT_KEY_DIR" @@ -321,6 +328,7 @@ def ensure_runnable( execution_origin: Optional[str] = None, effect_verifier: Any = None, at_run: bool = False, + stops_before_save: bool = False, ) -> None: """Refuse to execute a JIT draft that has no valid admission. @@ -338,6 +346,9 @@ def ensure_runnable( task's, ``execution_origin`` is the origin the canary ran on, and ``effect_verifier`` gives every write step an independent read-back. + A repaired copy whose admission is only a ``repair_probe`` runs only when + ``stops_before_save`` says the run's gate stops before the first save. + Raises: DraftNotAdmitted: the draft is not admitted, this run does not match its admission, or ``attended_by`` can't attend it. @@ -360,6 +371,11 @@ def ensure_runnable( f"admitted ({exc}). Review, canary and admit it first " "(openadapt-flow jit status)." ) from None + if manifest.get("admission_mode") == "repair_probe" and not stops_before_save: + raise DraftNotAdmitted( + f"refused: {bundle} is a repair a person has not approved yet; it runs " + "only as a probe that stops before the save. Nothing ran." + ) if not at_run: return try: @@ -1539,6 +1555,9 @@ def _verify_mode(body: Mapping[str, Any], keys: Path) -> None: mode = body.get("admission_mode", "person") if mode not in ADMISSION_MODES: raise AdmissionError(f"unknown admission mode {mode!r}") + if mode in REPAIR_MODES: + _verify_repair(body, keys) + return name = str(body.get("admitted_by_principal_id") or "") if mode != "agent_policy": _self_check(name, body, agent=False) @@ -1584,6 +1603,132 @@ def _verify_mode(body: Mapping[str, Any], keys: Path) -> None: ) +# -- repairs ------------------------------------------------------------------- + +#: Fields a repaired copy's admission must carry over from the prior one. +_REPAIR_KEPT = ("workflow_id", "task", "params", "policy", "canary", "readback") + + +def admit_repair( + prior_bundle: Path | str, + repaired_bundle: Path | str, + *, + candidate_id: str, + approved_by: Optional[str] = None, + route: Optional[str] = None, + probe_only: bool = False, + key_dir: Optional[Path] = None, +) -> dict[str, Any]: + """Carry an admitted program's admission to a repaired copy of it. + + A repair (``openadapt_flow.repair``) re-anchors steps or adds a bridge; + it never changes the record check. So the copy keeps the prior admission + when its record check (:func:`~openadapt_flow.jit.signature.record_check_digest`) + and parameters are the prior program's, and it is admitted for the same + task, policy, app and read-back. The new manifest embeds the prior one, + which must still verify. + + ``probe_only`` writes a ``repair_probe`` admission: the copy runs only + behind a gate that stops before the first save (see + :func:`ensure_runnable`). Otherwise ``approved_by`` is the person who + approved the repair, or ``standard-change:`` for a fix a + signed repair policy allowed. + + Raises: + AdmissionError: the prior program is not admitted, the copy changes + its record check or parameters, or no approver is named. + """ + + from openadapt_flow.execute.keys import ( + fingerprint_of, + load_or_create_private_key, + sign_bytes, + ) + from openadapt_flow.jit.signature import record_check_digest + + keys = Path(key_dir) if key_dir is not None else default_key_dir() + prior = verify_admission(prior_bundle, key_dir=keys) + if prior.get("admission_mode") == "repair_probe": + raise AdmissionError("a repair probe's admission can't carry to another copy") + prior_workflow = _load_workflow(Path(prior_bundle)) + repaired = _load_workflow(Path(repaired_bundle)) + if sorted(repaired.params) != sorted(prior.get("params") or []): + raise AdmissionError("the repair changes the program's parameters") + check = record_check_digest(repaired) + if check != record_check_digest(prior_workflow) or ( + prior.get("record_check_digest") not in (None, check) + ): + raise AdmissionError( + "the repair changes the program's record check; it needs a new admission" + ) + if probe_only: + name = "self-repair:probe" + else: + name = (approved_by or "").strip() + if not name: + raise AdmissionError("a repair's admission names who approved it") + if not name.startswith(STANDARD_CHANGE_APPROVER) and is_not_a_person_name(name): + raise AdmissionError(f"{name!r} is not a person; a model never approves") + private = load_or_create_private_key(keys) + body: dict[str, Any] = { + "schema": ADMISSION_SCHEMA, + "admission_mode": "repair_probe" if probe_only else "repair", + "state": BundleState.ADMITTED.value, + **{k: prior.get(k) for k in _REPAIR_KEPT}, + **{k: prior.get(k) for k in RECORDER_FIELDS}, + "digest": _digest(repaired), + "record_check_digest": check, + "admitted_by_principal_id": name, + "admitted_at": _now(), + "repair_of": { + "candidate_id": candidate_id, + "route": route, + "prior_digest": prior.get("digest"), + "prior_manifest": dict(prior), + }, + "signing_key": fingerprint_of(private.public_key()), + } + manifest = {**body, "signature": sign_bytes(private, _canonical(body))} + _write(Path(repaired_bundle) / ADMISSION_FILE, manifest) + return manifest + + +def _verify_repair(body: Mapping[str, Any], keys: Path) -> None: + """A repaired copy's admission: the prior one verifies, and nothing that + was admitted changed except the anchors and steps the repair touched.""" + + repair = body.get("repair_of") + if not isinstance(repair, Mapping): + raise AdmissionError("a repair admission names no prior admission") + prior = repair.get("prior_manifest") + if not isinstance(prior, Mapping) or not _record_signed(prior, keys): + raise AdmissionError("the prior admission's signature does not verify") + prior_body = {k: v for k, v in prior.items() if k != "signature"} + if prior_body.get("state") != BundleState.ADMITTED.value: + raise AdmissionError("the prior manifest does not admit its bundle") + if prior_body.get("admission_mode") == "repair_probe": + raise AdmissionError("a repair can't follow a repair probe") + _verify_mode(prior_body, keys) + if repair.get("prior_digest") != prior_body.get("digest"): + raise AdmissionError("the repair names another prior bundle") + for key in (*_REPAIR_KEPT, *RECORDER_FIELDS): + if body.get(key) != prior_body.get(key): + raise AdmissionError(f"the repair changed the admitted {key}") + pinned = prior_body.get("record_check_digest") + if not body.get("record_check_digest") or ( + pinned is not None and pinned != body.get("record_check_digest") + ): + raise AdmissionError("the repair changed the admitted record check") + if body.get("admission_mode") == "repair": + name = str(body.get("admitted_by_principal_id") or "") + if not name.startswith(STANDARD_CHANGE_APPROVER): + if is_not_a_person_name(name): + raise AdmissionError( + f"{name!r} is not a person; a model never approves" + ) + _self_check(name, body, agent=False) + + def reject( program_dir: Path | str, *, diff --git a/packages/flow/openadapt_flow/repair/self_repair.py b/packages/flow/openadapt_flow/repair/self_repair.py index 932c8fa4b7..4bf7ff1974 100644 --- a/packages/flow/openadapt_flow/repair/self_repair.py +++ b/packages/flow/openadapt_flow/repair/self_repair.py @@ -85,6 +85,7 @@ Step, Workflow, ) +from openadapt_flow.jit.lifecycle import STANDARD_CHANGE_APPROVER from openadapt_flow.repair.candidate import ( Actor, CampaignCaseResult, @@ -467,6 +468,9 @@ def __init__( super().__init__("not_gated") self._checks = checks self._resume_form_state = resume_form_state + #: In probe mode no save runs: the run gate admits a repaired copy + #: whose admission is only a repair probe (``jit.lifecycle``). + self.stops_before_first_save = resume_form_state is None self.outcome: Optional[ProbeOutcome] = None self._approved_result: Optional[int] = None @@ -1305,6 +1309,9 @@ def _patch( checked = tuple(s.id for s in new_steps if s.anchor is not None) + ( (step.id,) if mode == "replace" else () ) + _carry_admission( + bundle_dir, bundle_out, candidate.candidate_id, probe_only=True + ) return _Patch( attempt=attempt, step=shown_step, @@ -1640,6 +1647,15 @@ def _govern_and_resume( evidence=evidence, ) if standard is not None: + change = evidence.standard_change + _carry_admission( + bundle_dir, + patch.bundle_dir, + standard.candidate_id, + approved_by=STANDARD_CHANGE_APPROVER + + (change.policy_id if change is not None else "unknown"), + route="standard_change", + ) return self._resume( patch, probe, standard, halted_report, run_dir, work, evidence, None ) @@ -1720,6 +1736,13 @@ def _govern_and_resume( ) assert candidate.approval is not None evidence.approved_by = candidate.approval.approved_by + _carry_admission( + bundle_dir, + patch.bundle_dir, + candidate.candidate_id, + approved_by=candidate.approval.approved_by, + route=route, + ) evidence.steps.append( SelfRepairStepRecord( stage="approval", @@ -2160,6 +2183,45 @@ def _outline(png: bytes, region: tuple[int, int, int, int]) -> bytes: return out.getvalue() +def _carry_admission( + prior: Path, + repaired: Path, + candidate_id: str, + *, + approved_by: Optional[str] = None, + route: Optional[str] = None, + probe_only: bool = False, +) -> None: + """Carry an agent-compiled program's admission to its repaired copy. + + A program compiled from an agent run (``openadapt_flow.jit``) runs only + under its admission, which pins its digest; every patch changes the + digest. The repair keeps the record check, so the admission carries over: + first for the probe only, then, once a person (or a signed standard + change) approved the fix, for the resumed run and later items. Any other + program has nothing to carry. + """ + + from openadapt_flow.jit import lifecycle + + if not lifecycle.is_jit_bundle(prior): + return + try: + lifecycle.admit_repair( + prior, + repaired, + candidate_id=candidate_id, + approved_by=approved_by, + route=route, + probe_only=probe_only, + ) + except lifecycle.AdmissionError as exc: + raise SelfRepairRefused( + "the program was compiled from an agent run and admitted, and its " + f"admission can't carry to this repair: {exc}" + ) from None + + def _patched_structural(step: Step, structural: Any) -> Any: """The structural locator a re-targeted step's anchor keeps. diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 6dcfdf5fec..f96b0e5a03 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -297,6 +297,48 @@ def effect_field_checks(effect: Any, verdict: Any, tier: Any) -> list[Any]: return checks +#: A uniform page scale is inferred only from a click point at least this far +#: from both edges at the origin, only when both axes agree to within this +#: fraction, and only inside these bounds (and not within 5% of 1). +_PAGE_SCALE_MIN_COORD = 40 +_PAGE_SCALE_AXIS_AGREEMENT = 0.02 +_PAGE_SCALE_BOUNDS = (0.5, 2.0) +#: One pixel of click-point noise moves the estimate by a few tenths of a +#: percent, enough to drop a wide template below threshold; the check tries +#: this tight band around it. +_PAGE_SCALE_BAND = (0.99, 0.995, 1.0, 1.005, 1.01) + + +def _uniform_page_scale(step: Any, result: Any) -> Optional[float]: + """The page scale this step's click showed, or ``None``. + + The recorded click point and the point the ladder resolved this run, + divided axis by axis. A whole-page zoom multiplies both coordinates by + the same factor; a control that merely moved almost never does, so the + two axes must agree. + """ + + resolution = getattr(result, "resolution", None) + anchor = getattr(step, "anchor", None) + if resolution is None or anchor is None: + return None + try: + rx, ry = (float(v) for v in anchor.click_point) + lx, ly = (float(v) for v in resolution.point) + except (TypeError, ValueError): + return None + if min(rx, ry) < _PAGE_SCALE_MIN_COORD: + return None + sx, sy = lx / rx, ly / ry + if abs(sx - sy) > _PAGE_SCALE_AXIS_AGREEMENT * max(sx, sy): + return None + scale = (sx + sy) / 2 + low, high = _PAGE_SCALE_BOUNDS + if not low <= scale <= high or abs(scale - 1.0) < 0.05: + return None + return round(scale, 3) + + def _frame_viewport(frame_png: bytes) -> tuple[int, int]: """Return the viewport of THIS frame, read from the frame itself. @@ -1565,6 +1607,9 @@ def run( execution_origin=execution_origin or execution_target_origin, effect_verifier=self.effect_verifier, at_run=True, + stops_before_save=bool( + getattr(self.approval_gate, "stops_before_first_save", False) + ), ) bundle_dir = Path(bundle_dir) run_dir = Path(run_dir) @@ -15914,18 +15959,28 @@ def _check_postconditions( on, plus human-readable descriptions of the postconditions that failed the final check (empty when ok). """ - ok, frame_png = self._poll_postconditions( - step, frame_png, bundle_dir, start_state - ) - if ok: - return True, frame_png, [] - # One re-settle retry. - frame_png = self.vision.wait_settled(self.backend) - failed_pcs = [ - pc - for pc in step.expect - if not self._postcondition_passes(pc, frame_png, bundle_dir, start_state) - ] + self._pc_page_scale = _uniform_page_scale(step, result) + self._pc_scaled_match = False + try: + ok, frame_png = self._poll_postconditions( + step, frame_png, bundle_dir, start_state + ) + if ok: + self._note_page_scale(result) + return True, frame_png, [] + # One re-settle retry. + frame_png = self.vision.wait_settled(self.backend) + failed_pcs = [ + pc + for pc in step.expect + if not self._postcondition_passes( + pc, frame_png, bundle_dir, start_state + ) + ] + if not failed_pcs: + self._note_page_scale(result) + finally: + self._pc_page_scale = None if ( failed_pcs and self.state_verifier is not None @@ -15935,6 +15990,13 @@ def _check_postconditions( failed = [self._describe_postcondition(pc) for pc in failed_pcs] return not failed_pcs, frame_png, failed + def _note_page_scale(self, result: Optional[StepResult]) -> None: + """Record on ``result`` the page scale a passing check needed.""" + + scale = getattr(self, "_pc_page_scale", None) + if result is not None and scale is not None and self._pc_scaled_match: + result.postcondition_page_scale = scale + @staticmethod def _postcondition_confirms_a_write( step: Step, workflow: Optional[Workflow] @@ -16144,11 +16206,65 @@ def _postcondition_passes( ) if match is not None: return True + if self._scaled_region_match(frame_png, template_png, region): + return True live = self.vision.phash_png(frame_png, region=region) distance = self.vision.phash_distance(live, pc.phash) return distance <= pc.phash_tolerance return False + def _scaled_region_match( + self, frame_png: bytes, template_png: bytes, region: tuple[int, ...] + ) -> bool: + """The recorded content, at the page scale this step's click showed. + + A whole-page zoom (browser zoom, or an app's own scale) grows every + element and moves it away from the top-left corner by the same + factor, so the recorded region falls outside the search window and + its content outside the matcher's scale band. When this step's own + click landed at the recorded point times one uniform factor + (:func:`_uniform_page_scale`), look for the same template at that + factor (within one percent), at the recorded region times that + factor. Thresholds are the unscaled check's: a modal, a missing panel or another layout still + fails. + """ + + scale = getattr(self, "_pc_page_scale", None) + if scale is None: + return False + x, y, w, h = region + scaled = ( + int(round(x * scale)), + int(round(y * scale)), + int(round(w * scale)), + int(round(h * scale)), + ) + search = pad_region(scaled, PC_TEMPLATE_SEARCH_PAD, _frame_viewport(frame_png)) + matchers = [ + (self.vision.find_template, PC_TEMPLATE_THRESHOLD), + ( + getattr(self.vision, "find_structural_template", None), + PC_STRUCTURAL_TEMPLATE_THRESHOLD, + ), + ] + for matcher, threshold in matchers: + if not callable(matcher): + continue + try: + match = matcher( + frame_png, + template_png, + search_region=search, + scales=tuple(round(scale * f, 4) for f in _PAGE_SCALE_BAND), + threshold=threshold, + ) + except TypeError: # a vision stand-in with no scale band + return False + if match is not None: + self._pc_scaled_match = True + return True + return False + def _dom_text_visible(self, text: str) -> bool: """True only when the backend proves ``text`` is visibly rendered. diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 7ccc57e4cb..f3f4b00050 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -271,6 +271,26 @@ "path": "benchmark/auto_repair/results/openimis-unreviewed-2026-10-19-b/summary.json", "sha256": "8ef178abba217091f9edf5f51fbe7c651bd7cc0451203965f837fcd0c7d5a013" }, + { + "path": "benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/calls.json", + "sha256": "6cf4d4e9cac7cafb83f9c33fabd5006eea8fb04728f88e7b1efed6d029021970" + }, + { + "path": "benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/items.csv", + "sha256": "b0445ab5cbe789c996704641110e17054e589c072a4a230cc22ac9627e3de67a" + }, + { + "path": "benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/ledger.jsonl", + "sha256": "7a9de6c5672fc7f6d5cdd5eaec6b93a92065d8e9431a44b8fc5e1726d696b7d9" + }, + { + "path": "benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/meta.json", + "sha256": "c70dfd13b65dab4c355313e8485b4fd9876118be814b0693f0938ddec8d14083" + }, + { + "path": "benchmark/auto_repair/results/silent-saves-1.1.0-2026-10-19-a/summary.json", + "sha256": "51601d8aaa73d3746514c9ade40e37acc4ce3edeb5aad853a1c3f0fa9a7564d4" + }, { "path": "benchmark/canvas_ladder/results.json", "sha256": "4502affb014de71e5c584e48cef5bc6a7631983e2517551d5ea9c20c8a0668e2" diff --git a/packages/flow/tests/test_jit_repair_admission.py b/packages/flow/tests/test_jit_repair_admission.py new file mode 100644 index 0000000000..ee1cf3a081 --- /dev/null +++ b/packages/flow/tests/test_jit_repair_admission.py @@ -0,0 +1,209 @@ +"""A repaired copy of an admitted agent-compiled program keeps its admission. + +Silent-saves 1.1.0 (#1380) found that self-repair could never probe or resume +a program compiled from an agent run: every patch changes the bundle's +digest, and the admission pins it, so the run gate refused the patched copy +("DraftNotAdmitted"). A repair keeps the record check, so the admission now +carries over: for the probe only (a run that stops before the save), then, +after a person approves the fix, for every run. No browser and no model. +""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest + +from openadapt_flow.ir import Workflow +from openadapt_flow.jit import AdmissionError, lifecycle +from openadapt_flow.jit.lifecycle import DraftNotAdmitted, admit_repair, ensure_runnable +from tests.test_jit_lifecycle import ( # noqa: F401 (autouse fixture) + _admitted, + _policy_passes, +) + + +def _copy(jit, tmp_path: Path, name: str, edit) -> Path: + out = tmp_path / name + shutil.copytree(jit.bundle_dir, out) + (out / lifecycle.ADMISSION_FILE).unlink() + workflow = Workflow.load(out) + edit(workflow) + workflow.save(out) + return out + + +def _reanchor(workflow: Workflow) -> None: + # A repair re-targets steps; the record check (the save's contract) stays. + workflow.steps[0].text = "y" + + +def _recontract(workflow: Workflow) -> None: + workflow.steps[1].effects[0].field = "summary" + + +def test_the_patched_copy_is_refused_without_a_carried_admission( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + jit = _admitted(tmp_path, monkeypatch) + shutil.copytree(jit.bundle_dir, tmp_path / "copy") + workflow = Workflow.load(tmp_path / "copy") + _reanchor(workflow) + workflow.save(tmp_path / "copy") + with pytest.raises(DraftNotAdmitted, match="not admitted"): + ensure_runnable(tmp_path / "copy", key_dir=jit.key_dir) + + +def test_a_probe_admission_runs_only_behind_a_gate_that_stops_before_the_save( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + jit = _admitted(tmp_path, monkeypatch) + repaired = _copy(jit, tmp_path, "repaired", _reanchor) + manifest = admit_repair( + jit.bundle_dir, + repaired, + candidate_id="cand-1", + probe_only=True, + key_dir=jit.key_dir, + ) + assert manifest["admission_mode"] == "repair_probe" + assert ( + manifest["repair_of"]["prior_digest"] + == json.loads((jit.bundle_dir / lifecycle.ADMISSION_FILE).read_text())["digest"] + ) + with pytest.raises(DraftNotAdmitted, match="has not approved"): + ensure_runnable(repaired, key_dir=jit.key_dir) + ensure_runnable(repaired, key_dir=jit.key_dir, stops_before_save=True) + + +def test_a_person_approved_repair_runs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + jit = _admitted(tmp_path, monkeypatch) + repaired = _copy(jit, tmp_path, "repaired", _reanchor) + manifest = admit_repair( + jit.bundle_dir, + repaired, + candidate_id="cand-1", + approved_by="decision_portal:person", + route="decision_portal", + key_dir=jit.key_dir, + ) + assert manifest["admission_mode"] == "repair" + assert ( + manifest["record_check_digest"] + == manifest["repair_of"]["prior_manifest"]["record_check_digest"] + ) + ensure_runnable(repaired, key_dir=jit.key_dir) + assert ( + lifecycle.verify_admission(repaired, key_dir=jit.key_dir)["digest"] + != (manifest["repair_of"]["prior_digest"]) + ) + + +def test_a_repair_of_a_repair_keeps_the_chain( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + jit = _admitted(tmp_path, monkeypatch) + first = _copy(jit, tmp_path, "first", _reanchor) + admit_repair( + jit.bundle_dir, + first, + candidate_id="cand-1", + approved_by="Dana Reviewer", + key_dir=jit.key_dir, + ) + second = tmp_path / "second" + shutil.copytree(first, second) + workflow = Workflow.load(second) + workflow.steps[0].text = "z" + workflow.save(second) + admit_repair( + first, + second, + candidate_id="cand-2", + approved_by="standard-change:policy-1", + route="standard_change", + key_dir=jit.key_dir, + ) + ensure_runnable(second, key_dir=jit.key_dir) + + +def test_a_repair_that_changes_the_record_check_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + jit = _admitted(tmp_path, monkeypatch) + repaired = _copy(jit, tmp_path, "repaired", _recontract) + with pytest.raises(AdmissionError, match="record check"): + admit_repair( + jit.bundle_dir, + repaired, + candidate_id="cand-1", + approved_by="Dana Reviewer", + key_dir=jit.key_dir, + ) + + +@pytest.mark.parametrize("who", ["", "agent:repair-bot", "model:claude"]) +def test_only_a_person_or_a_standard_change_approves( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, who: str +) -> None: + jit = _admitted(tmp_path, monkeypatch) + repaired = _copy(jit, tmp_path, "repaired", _reanchor) + with pytest.raises(AdmissionError): + admit_repair( + jit.bundle_dir, + repaired, + candidate_id="cand-1", + approved_by=who, + key_dir=jit.key_dir, + ) + + +def test_an_edited_repair_admission_does_not_verify( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + jit = _admitted(tmp_path, monkeypatch) + repaired = _copy(jit, tmp_path, "repaired", _reanchor) + admit_repair( + jit.bundle_dir, + repaired, + candidate_id="cand-1", + approved_by="Dana Reviewer", + key_dir=jit.key_dir, + ) + path = repaired / lifecycle.ADMISSION_FILE + manifest = json.loads(path.read_text()) + manifest["repair_of"]["prior_manifest"]["params"] = ["anything"] + path.write_text(json.dumps(manifest)) + with pytest.raises(DraftNotAdmitted): + ensure_runnable(repaired, key_dir=jit.key_dir) + + +def test_a_probe_admission_cannot_be_carried_further( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + jit = _admitted(tmp_path, monkeypatch) + probe = _copy(jit, tmp_path, "probe", _reanchor) + admit_repair( + jit.bundle_dir, + probe, + candidate_id="cand-1", + probe_only=True, + key_dir=jit.key_dir, + ) + again = tmp_path / "again" + shutil.copytree(probe, again) + workflow = Workflow.load(again) + workflow.steps[0].text = "z" + workflow.save(again) + with pytest.raises(AdmissionError, match="probe"): + admit_repair( + probe, + again, + candidate_id="cand-2", + approved_by="Dana Reviewer", + key_dir=jit.key_dir, + ) diff --git a/packages/flow/tests/test_page_scale_postcondition.py b/packages/flow/tests/test_page_scale_postcondition.py new file mode 100644 index 0000000000..f1b9ba6bdc --- /dev/null +++ b/packages/flow/tests/test_page_scale_postcondition.py @@ -0,0 +1,147 @@ +"""A ``region_stable`` check holds under a uniform page zoom, and only then. + +Silent-saves 1.1.0 (#1380) stopped the compiled MockMed program on every item +of its display-scaling block: at 125% the recorded region's content sits 25% +further from the top-left corner and is 25% larger, outside the check's +search window and scale band, so a correct screen failed. The check now also +looks at the scale this step's own click showed, when both axes agree, with +the same thresholds. A covered or missing region still fails. + +No browser: synthetic frames and the real matcher. +""" + +from __future__ import annotations + +import io +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +from PIL import Image, ImageDraw + +import openadapt_flow.vision as vision +from openadapt_flow.ir import Anchor, Postcondition, PostconditionKind +from openadapt_flow.runtime.replayer import Replayer, _uniform_page_scale +from tests.test_replayer import FakeBackend + +VIEWPORT = (1280, 800) +REGION = (40, 120, 520, 260) + + +def _page(scale: float = 1.0, *, modal: bool = False) -> Image.Image: + """A form-like page; ``scale`` zooms it from the top-left corner.""" + + w, h = int(VIEWPORT[0] / scale) + 2, int(VIEWPORT[1] / scale) + 2 + page = Image.new("RGB", (w, h), (246, 248, 250)) + draw = ImageDraw.Draw(page) + rng = np.random.default_rng(7) + x0, y0, rw, rh = REGION + draw.rectangle((x0, y0, x0 + rw, y0 + rh), outline=(30, 60, 90), width=3) + for i in range(9): + y = y0 + 18 + i * 26 + width = int(rng.integers(120, rw - 40)) + draw.rectangle((x0 + 16, y, x0 + 16 + width, y + 12), fill=(40, 40, 40)) + draw.rectangle((x0 + rw - 90, y, x0 + rw - 20, y + 16), outline=(0, 90, 160)) + if scale != 1.0: + page = page.resize((round(w * scale), round(h * scale)), Image.BILINEAR) + page = page.crop((0, 0, *VIEWPORT)) + if modal: + sx = scale + ImageDraw.Draw(page).rectangle( + ( + int(x0 * sx) + 30, + int(y0 * sx) + 20, + int((x0 + rw) * sx), + int((y0 + rh) * sx), + ), + fill=(255, 255, 255), + outline=(0, 0, 0), + ) + return page + + +def _png(image: Image.Image) -> bytes: + out = io.BytesIO() + image.save(out, format="PNG") + return out.getvalue() + + +@pytest.fixture() +def check(tmp_path: Path): + base = _png(_page()) + x0, y0, rw, rh = REGION + crop = _page().crop((x0, y0, x0 + rw, y0 + rh)) + (tmp_path / "templates").mkdir() + (tmp_path / "templates" / "expect.png").write_bytes(_png(crop)) + pc = Postcondition( + kind=PostconditionKind.REGION_STABLE, + region=REGION, + phash=vision.phash_png(base, region=REGION), + template="templates/expect.png", + ) + replayer = Replayer(FakeBackend(), vision=vision) + + def passes(frame: bytes, scale) -> bool: + replayer._pc_page_scale = scale + replayer._pc_scaled_match = False + try: + return replayer._postcondition_passes(pc, frame, tmp_path) + finally: + replayer._pc_page_scale = None + + return passes + + +def test_the_recorded_page_passes_without_a_scale(check) -> None: + assert check(_png(_page()), None) + + +def test_a_zoomed_page_fails_without_its_scale(check) -> None: + assert not check(_png(_page(1.25)), None) + + +def test_a_zoomed_page_passes_at_the_scale_its_click_showed(check) -> None: + assert check(_png(_page(1.25)), 1.25) + # One pixel of click-point noise is within the band. + assert check(_png(_page(1.25)), 1.253) + + +def test_a_covered_region_still_fails_at_that_scale(check) -> None: + assert not check(_png(_page(1.25, modal=True)), 1.25) + + +def test_another_scale_does_not_stand_in(check) -> None: + assert not check(_png(_page(1.25)), 1.6) + + +def _step(click): + return SimpleNamespace( + anchor=Anchor(template="t.png", region=(0, 0, 10, 10), click_point=click) + ) + + +def _result(point): + return SimpleNamespace(resolution=SimpleNamespace(point=point)) + + +@pytest.mark.parametrize( + ("click", "point", "scale"), + [ + ((775, 186), (969, 232), 1.249), # MockMed at 125%, recorded vs live + ((343, 350), (430, 438), 1.253), + ((400, 300), (400, 300), None), # no zoom + ((400, 300), (520, 300), None), # moved right only: not a zoom + ((400, 300), (560, 330), None), # axes disagree + ((30, 300), (38, 375), None), # too close to the edge to tell + ((400, 300), (1000, 750), None), # beyond the bounds + ], +) +def test_a_page_scale_needs_both_axes_to_agree(click, point, scale) -> None: + assert _uniform_page_scale(_step(click), _result(point)) == scale + + +def test_no_resolution_means_no_scale() -> None: + assert ( + _uniform_page_scale(_step((400, 300)), SimpleNamespace(resolution=None)) is None + )