From bedf5439f462acd296a8056acdad8a58f4db2b31 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 24 Oct 2026 23:34:09 -0400 Subject: [PATCH 1/5] feat(flow): code as action is the default first-run agent; every save is read back - ClaudeAgent and compile-agent default to vision=code on claude-sonnet-5-5 - the harness reads the system of record after every save the gate lets through: another record's change stops run 1 (readback_wrong_record), no change is reported to the agent as not saved - MCP teach_run: the agent writes action scripts; with a person approving saves, the script stops before each save and the approval is bound to it - scoreboard arm 7: read-back wired, protocol fault items, --keep-program Co-Authored-By: Claude Opus 5.5 --- .github/workflows/packages.yml | 1 + packages/flow/benchmark/scoreboard/run.py | 65 +++++- packages/flow/openadapt_flow/agent/mcp.py | 5 +- packages/flow/openadapt_flow/agent/teach.py | 124 +++++++++- .../openadapt_flow/compile_agent/agents.py | 34 +-- .../flow/openadapt_flow/compile_agent/cli.py | 24 +- .../compile_agent/code_action.py | 120 +++++++++- .../openadapt_flow/compile_agent/driven.py | 85 ++++++- .../openadapt_flow/compile_agent/harness.py | 138 ++++++++++- .../openadapt_flow/compile_agent/pipeline.py | 26 ++- .../tests/e2e/test_code_agent_default_e2e.py | 216 ++++++++++++++++++ .../flow/tests/e2e/test_compile_agent_e2e.py | 4 +- packages/flow/tests/test_code_action.py | 34 +++ packages/flow/tests/test_compile_agent.py | 10 +- 14 files changed, 831 insertions(+), 55 deletions(-) create mode 100644 packages/flow/tests/e2e/test_code_agent_default_e2e.py diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 20556c037..91e13974c 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -455,6 +455,7 @@ jobs: tests/e2e/test_structural_action.py \ tests/e2e/test_compile_agent_e2e.py \ tests/e2e/test_code_action_e2e.py \ + tests/e2e/test_code_agent_default_e2e.py \ tests/e2e/test_mcp_teach_e2e.py \ tests/test_agent_guard.py \ tests/e2e/test_agent_guard_e2e.py \ diff --git a/packages/flow/benchmark/scoreboard/run.py b/packages/flow/benchmark/scoreboard/run.py index 3f84fba89..d2ba8a65f 100644 --- a/packages/flow/benchmark/scoreboard/run.py +++ b/packages/flow/benchmark/scoreboard/run.py @@ -703,6 +703,7 @@ def _program_check(jit: Any, trace: Any) -> dict[str, Any]: def arm_code_agent(args: argparse.Namespace) -> int: from openadapt_flow.compile_agent import ClaudeAgent, SaveGate + from openadapt_flow.compile_agent.harness import record_readback from openadapt_flow.compile_agent.pipeline import _HarnessedAgent task = task_module(args.task) @@ -720,9 +721,15 @@ def arm_code_agent(args: argparse.Namespace) -> int: max_cost_per_item=args.max_cost, on_screen=CODE_AGENT_ON_SCREEN[task.NAME], ) - # Arm 7 needs no admission bound to an origin, so it takes a free port. + # Arm 7 needs no admission bound to an origin, so it takes a free port, + # unless it keeps a program for arm 3: that runs on the fixed origin. sess = ( - Session(task, args.tag, faults=False, port=0) + Session( + task, + args.tag, + faults=args.faults, + port=None if args.keep_program else 0, + ) if task.NAME == "mockmed" else None ) @@ -747,12 +754,27 @@ def arm_code_agent(args: argparse.Namespace) -> int: ["note"], ) readback_fn = lambda: sess.readback(it) # noqa: E731 + # Read after every save in run 1 (the harness's read-back). The + # JIT resets the store before run 1, which leaves it empty. + save_readback = record_readback( + sess.server.records, + {"patient_id": "p1", "type": "Triage"}, + lambda: [], + ) + fault = it.fault else: values = task.item_values(OPENEMR_CODE_ITEM_BASE + i) canary_values = task.item_values(OPENEMR_CODE_ITEM_BASE + 100 + i) params = list(task.PARAMS) - before_ids = {p["id"] for p in task.patients()} + before = task.patients() + before_ids = {p["id"] for p in before} readback_fn = lambda: task.check(values, before_ids) # noqa: E731 + save_readback = record_readback( + task.patients, + {"email": values["email"], "lname": values["lname"]}, + lambda: before, + ) + fault = "none" inner = ClaudeAgent( vision="code", model=AGENT_MODEL, @@ -767,6 +789,7 @@ def arm_code_agent(args: argparse.Namespace) -> int: keep=[*on_screen, *values.values()], evidence_dir=cache / "gates", principal_id=inner.principal_id, + readback=save_readback, ) if sess is not None: jit = _mockmed_jit(sess, cache, it, agent, dialect="generic") @@ -802,6 +825,10 @@ def arm_code_agent(args: argparse.Namespace) -> int: } for g in (agent.harness.gate_log if agent.harness else []) ], + "readbacks": [ + {k: v for k, v in r.items() if k != "at"} + for r in (agent.harness.readback_log if agent.harness else []) + ], } if result is not None and result.candidate is not None: summary["program_check"] = _program_check(jit, trace) @@ -832,10 +859,10 @@ def arm_code_agent(args: argparse.Namespace) -> int: run_id=args.run_id, item=i, attempt=1, - fault="none", + fault=fault, reported_success=ok, readback=readback, - outcome=schema.classify(ok, readback, "none"), + outcome=schema.classify(ok, readback, fault), model_calls=ledger.model_calls if ledger is not None else 0, input_tokens=ledger.total("input_tokens") if ledger else 0, output_tokens=ledger.total("output_tokens") if ledger else 0, @@ -859,14 +886,30 @@ def arm_code_agent(args: argparse.Namespace) -> int: ), "canary_status": summary.get("canary_status"), "admitted": summary.get("admitted"), + "readbacks": [r["outcome"] for r in summary["readbacks"]], "error": f"{type(error).__name__}: {error}" if error else None, }, default=str, )[:300], ) schema.append(out / "items.csv", row) - # The admission keys and cache stay out of the repository. - shutil.rmtree(cache, ignore_errors=True) + keep = ( + args.keep_program + and i == args.start + and summary.get("admitted") + and sess is not None + ) + if keep: + # Arm 3 replays this program: `program --program-from code-`. + kept = schema.cache_dir(task.NAME, "first_run", f"code-{args.run_id}") + if kept.exists(): + shutil.rmtree(kept) + kept.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(cache), str(kept)) + print(f"kept the admitted program for arm 3: {kept}", flush=True) + else: + # The admission keys and cache stay out of the repository. + shutil.rmtree(cache, ignore_errors=True) print( f"{task.NAME} code_agent {i:3d} {row.outcome:13s} {row.model_calls} calls " f"{row.item_s:6.1f}s ${row.cost_usd:.3f} load {load0}->{load1} " @@ -899,6 +942,14 @@ def main() -> int: parser.add_argument("--no-faults", dest="faults", action="store_false") parser.add_argument("--program-from", default=None, help="first_run id") parser.add_argument("--pause-after-change-halt", action="store_true") + parser.add_argument( + "--keep-program", + action="store_true", + help=( + "code_agent on mockmed: keep the first item's admitted program (on the " + "fixed origin) for `program --program-from code-`" + ), + ) parser.add_argument( "--ui-change-at", type=int, diff --git a/packages/flow/openadapt_flow/agent/mcp.py b/packages/flow/openadapt_flow/agent/mcp.py index 4f7b0fd12..34f0e5f53 100644 --- a/packages/flow/openadapt_flow/agent/mcp.py +++ b/packages/flow/openadapt_flow/agent/mcp.py @@ -14,6 +14,7 @@ from __future__ import annotations +import functools import io import json import logging @@ -391,9 +392,9 @@ async def _call_tool( elif name == "admit" and authoring is not None and authoring.handles(name): await _confirm_admit(context, authoring) human_confirmed = True - elif teaching and name == "teach_act": + elif teaching and name in ("teach_act", "teach_run"): hold = await anyio.to_thread.run_sync( - teach.hold_needed, arguments or {} + functools.partial(teach.hold_needed, arguments or {}, tool=name) ) if hold: await _confirm_with_person( diff --git a/packages/flow/openadapt_flow/agent/teach.py b/packages/flow/openadapt_flow/agent/teach.py index f395e2b20..0321cd00d 100644 --- a/packages/flow/openadapt_flow/agent/teach.py +++ b/packages/flow/openadapt_flow/agent/teach.py @@ -6,8 +6,16 @@ Opens the operator's app (``--url`` or ``--config``) and starts run 1 of a task the agent names, with its inputs, the record fields the save must produce, and the record's identity as the screen shows it. +``teach_run`` (web apps; the default way to act) + The agent writes the action code: a short script that reads the page, + acts on elements by role and name, and reads the result back + (:mod:`openadapt_flow.compile_agent.code_action`). Every save goes through + the same gate, and is read back from the system of record before the + script goes on; a save on a record the task does not name ends run 1. + When a person approves saves, the script stops before each one and the + client's form asks the person; the approval covers that one line. ``teach_observe``, ``teach_act``, ``teach_screen`` - The agent does the task one action at a time. Every action goes through + Or the agent does the task one action at a time. Every action goes through the compile-agent harness (:mod:`openadapt_flow.compile_agent.harness`): on the web it checks that the right record is on screen before each save; on Windows, macOS, Linux, RDP and Citrix it can't, so a person approves @@ -52,6 +60,7 @@ TEACH_TOOLS = ( "teach_start", "teach_observe", + "teach_run", "teach_act", "teach_screen", "teach_finish", @@ -89,6 +98,12 @@ } +def _code_commands() -> str: + from openadapt_flow.compile_agent.code_action import COMMANDS + + return COMMANDS + + def _obj(properties: dict[str, Any], required: list[str]) -> dict[str, Any]: return { "type": "object", @@ -273,8 +288,9 @@ def list_tool_specs(self) -> list[ToolSpec]: description=( "Teach OpenAdapt a task by doing it once. OpenAdapt opens the " f"app this server was set up for ({self.backend_kind}) and " - "records what you do; you then call teach_observe and " - "teach_act until the task is done, and teach_finish. This " + "records what you do; you then call teach_run with short " + "scripts (or teach_observe and teach_act, one action at a " + "time) until the task is done, and teach_finish. This " "run makes a real entry in that app, so use test records. " "task is the instruction with {name} for each input; inputs " "are this run's values; record_fields are fields the saved " @@ -345,6 +361,48 @@ def list_tool_specs(self) -> list[ToolSpec]: input_schema=_obj({}, []), annotations=_READ_ONLY, ), + *( + [ + ToolSpec( + name="teach_run", + description=( + "Do the next part of run 1 by writing it as a short " + "script, one command per line: tree / find / text / " + "read to read the page; click, fill, select, press " + "and scroll to act on elements by role and name " + '(click role=button name="Save" near="Jane Sample"); ' + "sh for a read-only shell over files you saved. " + "Lines run in order and stop at the first error or " + "after a save. After every save OpenAdapt reads the " + "app's records back and tells you what landed; a " + "save on a record the task does not name ends run 1. " + + ( + "A person approves each save first: the script " + "stops before it with status needs_person, and " + "you send the rest of the script again." + if self.config.hold + else "Before any save the harness checks that the " + "record's identity is shown next to it." + ) + + "\n" + + _code_commands() + ), + input_schema=_obj( + { + "script": { + "type": "string", + "minLength": 1, + "maxLength": 8000, + } + }, + ["script"], + ), + annotations=_ACTS, + ) + ] + if on_web + else [] + ), ToolSpec( name="teach_act", description=( @@ -514,15 +572,19 @@ def _taught_spec(self, tool: str, workflow: str) -> ToolSpec: # -- what the MCP layer asks a person ------------------------------------------ - def hold_needed(self, arguments: Mapping[str, Any]) -> Optional[dict[str, Any]]: - """For ``teach_act``: when a person must approve this action first, - what it may commit, a description, and the binding the approval - covers (this run, this action, the app as it is now). Else None.""" + def hold_needed( + self, arguments: Mapping[str, Any], tool: str = "teach_act" + ) -> Optional[dict[str, Any]]: + """For ``teach_act`` and ``teach_run``: when a person must approve this + action (a script's first line) first, what it may commit, a + description, and the binding the approval covers (this run, this + action, the app as it is now). Else None.""" run = self._active() if not run.person_approves_saves: return None - answer = self._run_call("preview", arguments) + kind = "run_preview" if tool == "teach_run" else "preview" + answer = self._run_call(kind, arguments) if "error" in answer or not answer.get("commit"): return None # teach_act reports any error, performing nothing return answer @@ -595,6 +657,8 @@ def dispatch( } if name == "teach_act": return self._act(args, held_by_person=held_by_person or None) + if name == "teach_run": + return self._run_script(args, held_by_person=held_by_person or None) if name == "teach_confirm_checks": return self._confirm_checks(args, person) if name not in handlers or not self.handles(name): @@ -735,7 +799,12 @@ def _start(self, args: Mapping[str, Any]) -> dict[str, Any]: "person_approves_saves": run.person_approves_saves, "warning": RECORDING_WARNING, "observation": observation.get("observation"), - "next": "teach_act until the task is done, then teach_finish", + "next": ( + "teach_run with a script (start with: tree) until the task is " + "done, then teach_finish" + if compiler.backend_kind == "web" + else "teach_act until the task is done, then teach_finish" + ), } def _observe(self) -> dict[str, Any]: @@ -786,6 +855,43 @@ def _act( } return {"status": "ok", "observation": answer.get("observation")} + def _run_script( + self, args: Mapping[str, Any], *, held_by_person: Optional[str] + ) -> dict[str, Any]: + run = self._active() + if self.backend_kind != "web": + raise BridgeError("teach_run runs on web apps; on this app use teach_act") + payload = {"script": str(args.get("script") or "")} + if held_by_person: + payload["_held_by_person"] = held_by_person + answer = self._run_call("run", payload) + if "refused" in answer: + run.done.wait(120) + return { + "status": "refused", + "detail": answer["refused"], + "run_over": True, + "next": "teach_finish shows the result; nothing was compiled", + } + if "error" in answer: + return { + "status": "error", + "error": answer["error"], + "detail": answer.get("detail"), + "performed": False, + } + out: dict[str, Any] = { + "status": "needs_person" if answer.get("pending") else "ok", + "output": answer.get("output"), + } + if answer.get("pending"): + out["next"] = ( + "a person must approve the save on the first line of this " + "script; send it with teach_run again" + ) + out["script"] = answer["pending"]["script"] + return out + def _abort(self) -> dict[str, Any]: run = self._active() run.abort() diff --git a/packages/flow/openadapt_flow/compile_agent/agents.py b/packages/flow/openadapt_flow/compile_agent/agents.py index 3d5fd1dad..4727012ce 100644 --- a/packages/flow/openadapt_flow/compile_agent/agents.py +++ b/packages/flow/openadapt_flow/compile_agent/agents.py @@ -1,9 +1,16 @@ """Agents that drive the compile-agent harness on run 1. :class:`ClaudeAgent` is a live Claude computer-use loop on the customer's own -key, in one of two observation modes: +key, in one of three observation modes: -``selective`` (default) +``code`` (default) + Code as action (:mod:`.code_action`): one ``run`` tool that takes a short + script. The model queries the structure, acts by role and name, reads + back, keeps notes in a sandboxed working directory, and asks for a + screenshot only when it wants one. Several actions fit in one call. Every + save is read back from the system of record before the agent goes on. + Default model: ``claude-sonnet-5-5`` (:data:`CODE_DEFAULT_MODEL`). +``selective`` The model reads the page's structure (:mod:`.observe`) and acts on refs with ``click``, ``type_text`` and ``press_key``. A screenshot enters its context only when it calls ``screenshot``, and after an action that @@ -11,13 +18,7 @@ ``every_turn`` The baseline: Claude's computer toolset (``computer_toolset_20260801``), with a screenshot returned after every action. -``code`` - Code as action (:mod:`.code_action`): one ``run`` tool that takes a short - script. The model queries the structure, acts by role and name, reads - back, keeps notes in a sandboxed working directory, and asks for a - screenshot only when it wants one. Several actions fit in one call. - -Both modes perform every action through the same harness and capture computer, +All modes perform every action through the same harness and capture computer, so both are recorded and compiled the same way. Each model call's token usage and time go into an :class:`AgentLedger`. @@ -41,6 +42,7 @@ __all__ = [ "AgentLedger", + "CODE_DEFAULT_MODEL", "BudgetExceeded", "ClaudeAgent", "DEFAULT_MODEL", @@ -50,6 +52,8 @@ ] DEFAULT_MODEL = "claude-opus-5-5" +#: The model the code-as-action agent was measured with (scoreboard arm 7). +CODE_DEFAULT_MODEL = "claude-sonnet-5-5" #: Opus 5.5 and later take computer use only as this toolset (no beta header). COMPUTER_TOOLSET = "computer_toolset_20260801" @@ -389,8 +393,10 @@ class ClaudeAgent: """A live Claude computer-use loop on the customer's own key. Args: - vision: ``selective`` (structure first) or ``every_turn`` (baseline). - model: The model id. Default ``claude-opus-5-5``. + vision: ``code`` (code as action, the default), ``selective`` + (structure first) or ``every_turn`` (the screenshot baseline). + model: The model id. Default ``claude-sonnet-5-5`` for ``code``, + ``claude-opus-5-5`` for the others. effort: ``output_config.effort``. Opus 5.5 defaults to ``medium``. max_turns: Model calls before the run is ended. max_cost_usd: Estimated spend before the run is ended @@ -400,8 +406,8 @@ class ClaudeAgent: principal_id: The registered agent identity this loop records as. """ - vision: str = "selective" - model: str = DEFAULT_MODEL + vision: str = "code" + model: str = "" effort: Optional[str] = "medium" max_turns: int = 40 max_cost_usd: float = 2.0 @@ -416,6 +422,8 @@ class ClaudeAgent: def __post_init__(self) -> None: if self.vision not in VISION_MODES: raise ValueError(f"vision must be one of {VISION_MODES}") + if not self.model: + self.model = CODE_DEFAULT_MODEL if self.vision == "code" else DEFAULT_MODEL if self.prices is None and self.model not in PRICES_PER_MTOK: raise ValueError( f"no price for {self.model!r}, so the cost cap can't hold; pass " diff --git a/packages/flow/openadapt_flow/compile_agent/cli.py b/packages/flow/openadapt_flow/compile_agent/cli.py index 99c71ac13..45e7831d4 100644 --- a/packages/flow/openadapt_flow/compile_agent/cli.py +++ b/packages/flow/openadapt_flow/compile_agent/cli.py @@ -156,11 +156,19 @@ def add_parser(sub: argparse._SubParsersAction) -> None: ) p.add_argument( "--vision", - choices=["selective", "every_turn", "code"], - default="selective", - help="code: act through one terminal-style tool (docs/CODE_AS_ACTION.md)", + choices=["code", "selective", "every_turn"], + default="code", + help=( + "code (default): act through one terminal-style tool, every save read " + "back (docs/CODE_AS_ACTION.md); selective: the structural listing; " + "every_turn: a screenshot after every action" + ), + ) + p.add_argument( + "--model", + default=None, + help="model id (default claude-sonnet-5-5 for code, claude-opus-5-5 otherwise)", ) - p.add_argument("--model", default=None, help="model id (default claude-opus-5-5)") p.add_argument("--effort", default="medium") p.add_argument("--max-cost", type=float, default=2.0, help="USD cap for run 1") p.add_argument("--task-id", default=None) @@ -207,16 +215,12 @@ def _interactive() -> bool: def _agent(args: argparse.Namespace) -> Any: - from openadapt_flow.compile_agent.agents import ( - DEFAULT_MODEL, - ClaudeAgent, - ScriptedAgent, - ) + from openadapt_flow.compile_agent.agents import ClaudeAgent, ScriptedAgent if args.agent == "claude": return ClaudeAgent( vision=args.vision, - model=args.model or DEFAULT_MODEL, + model=args.model or "", effort=args.effort, max_cost_usd=args.max_cost, ) diff --git a/packages/flow/openadapt_flow/compile_agent/code_action.py b/packages/flow/openadapt_flow/compile_agent/code_action.py index d44ea18db..0c035b573 100644 --- a/packages/flow/openadapt_flow/compile_agent/code_action.py +++ b/packages/flow/openadapt_flow/compile_agent/code_action.py @@ -125,6 +125,15 @@ class ScriptError(ValueError): action that was performed is in the trace.""" +class NeedsPerson(Exception): + """The next action may commit a write and a person must approve it first. + Nothing in it was performed; the script stops before it.""" + + def __init__(self, control: str) -> None: + super().__init__(control) + self.control = control + + class LeftTheApp(RuntimeError): """An action took the browser to another origin. The run ends: the agent may act on the target app only.""" @@ -754,6 +763,11 @@ def seen_before(index: int) -> set[str]: "locator_agrees": sum(1 for s in recorded if s.get("agrees")), "locator_unique": sum(1 for s in recorded if s.get("unique")), "commits": len(commits), + "readbacks": { + outcome: sum(1 for st in self.steps if st.get("readback") == outcome) + for outcome in ("landed", "no_change", "unreadable") + if any(st.get("readback") == outcome for st in self.steps) + }, "identity_read_before_commit": ( sorted(seen_before(commits[0])) if commits else [] ), @@ -777,6 +791,10 @@ class ToolOutput: text: str images: list[bytes] = field(default_factory=list) committed: bool = False + #: Set when the script stopped before a save a person must approve: + #: ``line`` (not run), ``control`` (what it may commit), ``rest`` (the + #: lines from it on). + pending: Optional[dict[str, Any]] = None def blocks(self) -> list[dict[str, Any]]: from openadapt_flow.compile_agent.agents import _image @@ -866,14 +884,35 @@ def run(self, script: str) -> ToolOutput: chunks: list[str] = [] images: list[bytes] = [] committed = False + pending: Optional[dict[str, Any]] = None for number, line in enumerate(lines, 1): + self.line_number = number self.trace.lines += 1 gates = self.harness.stats.gate_checks + actions = self.harness.stats.actions try: text, image = self._line(line) except ScopeViolation: # Outside the task's declared scope: run 1 ends, no draft. raise + except NeedsPerson as exc: + acted = self.harness.stats.actions > actions + chunks.append( + f"$ {line}\nwaiting: this line may save ({self._out(exc.control)}), " + "and a person must approve it first. " + + ( + "Part of the line was done before the save; " + if acted + else "Nothing in this line was done; " + ) + + "send the script again from this line once they have." + ) + pending = { + "line": line, + "control": exc.control, + "rest": lines[number - 1 :], + } + break except (ScriptError, ShellRefused, UnsupportedAction, LookupError) as exc: self.trace.errors += 1 chunks.append(f"$ {line}\nerror: {self._out(str(exc))}") @@ -898,7 +937,7 @@ def run(self, script: str) -> ToolOutput: "run. Read the screen to check the result.)" ) break - out = ToolOutput("\n".join(chunks), images, committed) + out = ToolOutput("\n".join(chunks), images, committed, pending) self.trace.log.append( {"script": str(script), "output": out.text[:3000], "images": len(images)} ) @@ -1161,13 +1200,13 @@ def _act( ) ) event = self._event_index() - self._gates_before = len(self.harness.gate_log) + self._begin_step() self._do({"kind": "press", "key": key}, hint) pressed: dict[str, Any] = {"action": "press", "key": key} return self._finish_step(pressed, None, [event], line) element = self._target(positional, named) - self._gates_before = len(self.harness.gate_log) + self._begin_step() if element.disabled: raise ScriptError(f"@{element.ref} {element.name!r} is disabled") self._still_there(element) @@ -1298,7 +1337,7 @@ def _scroll( "area you want to scroll first" ) event = self._event_index() - self._gates_before = len(self.harness.gate_log) + self._begin_step() before = self._scroll_offsets() self._do({"kind": "scroll", "dx": 0, "dy": dy}, None) moved = before is None or self._scroll_offsets() != before @@ -1380,6 +1419,10 @@ def _click(self, element: Element, hint: CommitHint) -> Optional[dict[str, Any]] return recorded def _do(self, native: Mapping[str, Any], hint: Optional[CommitHint]) -> None: + if self.approve is not None: + commit = self.harness.commit_target(native, hint=hint) + if commit is not None and not self.approve(commit): + raise NeedsPerson(commit) self.harness.act(native, hint=hint) self._settle() origin = self.surface.origin() @@ -1440,6 +1483,13 @@ def _finish_step( text = f"ok: {what} {step.get('key')}" if step["gated"]: text += " (save gate passed)" + log = getattr(self.harness, "readback_log", []) + if len(log) > self._readbacks_before: + back = log[-1] + step["readback"] = back["outcome"] + text += "\nread back: " + back["detail"] + if back["outcome"] == "no_change": + text += " (whatever the screen says; do not report it as saved)" header = "" if self.surface.kind == "dom": page = self.harness.page @@ -1447,6 +1497,68 @@ def _finish_step( return self._out(text + header) _gates_before: int = 0 + _readbacks_before: int = 0 + #: The line of the running script, from 1. + line_number: int = 0 + #: Asked before an action the gate would stop: True lets it through to + #: the gate (a person approved it), False stops the script before it + #: (:class:`NeedsPerson`). None: the gate alone decides. + approve: Optional[Callable[[str], bool]] = None + + def _begin_step(self) -> None: + self._gates_before = len(self.harness.gate_log) + self._readbacks_before = len(getattr(self.harness, "readback_log", [])) + + def first_commit(self, script: str) -> Optional[str]: + """What the script's first line may commit, if it acts, or None. + Reads the screen; performs nothing (for a person's approval form).""" + + lines = [ + ln.strip() + for ln in str(script).splitlines() + if ln.strip() and not ln.strip().startswith("#") + ] + if not lines: + return None + try: + tokens = shlex.split(lines[0]) + except ValueError: + return None + command = tokens[0].lower() if tokens else "" + if command not in ("click", "fill", "select", "press"): + return None + positional, named = _parse_args(tokens[1:]) + try: + if command == "press": + if len(positional) != 1: + return None + key = positional[0] + focus = self.surface.focused() or {} + hint = ( + None + if not focus + else CommitHint( + name=str(focus.get("name") or "the focused element"), + scope=str(focus.get("scope") or ""), + key=key if key in ("Enter", "Space") else None, + ) + ) + return self.harness.commit_target( + {"kind": "press", "key": key}, hint=hint + ) + element = self._target(positional, named) + except (ScriptError, LookupError): + return None + editable = element.role in _FIELD_ROLES and not element.submits + hint = CommitHint( + name=element.name, + scope=element.scope or element.context, + editable=editable if command != "click" else element.role in _FIELD_ROLES, + commits=command == "select" and element.submits, + ) + return self.harness.commit_target( + {"kind": "click", "x": element.x, "y": element.y}, hint=hint + ) def _number(named: Mapping[str, str], key: str, default: int) -> int: diff --git a/packages/flow/openadapt_flow/compile_agent/driven.py b/packages/flow/openadapt_flow/compile_agent/driven.py index 446012c84..46143a716 100644 --- a/packages/flow/openadapt_flow/compile_agent/driven.py +++ b/packages/flow/openadapt_flow/compile_agent/driven.py @@ -33,11 +33,14 @@ from openadapt_flow.jit.compile_agent import TaskRequest from openadapt_flow.teach_scope import ScopeViolation -__all__ = ["DrivenAgent", "DrivenRun", "RunEnded", "ACTIONS"] +__all__ = ["DrivenAgent", "DrivenRun", "RunEnded", "ACTIONS", "SCRIPT_SURFACES"] #: Actions a caller sends. ``ref`` names a control in the latest observation. ACTIONS = ("click", "type", "press", "hotkey") +#: A code-action script sent with ``run`` (``teach_run``): web apps only. +SCRIPT_SURFACES = ("web",) + _STOP = object() @@ -77,6 +80,9 @@ def __init__(self, principal_id: Optional[str] = None) -> None: #: approval is bound to both (see :meth:`binding`). self.run_id = uuid.uuid4().hex self.performed = 0 + self._tool: Any = None + #: The code-action trace, saved with run 1 by the pipeline. + self.last_trace: Any = None # -- called by the gate, on the run thread ----------------------------------- @@ -122,8 +128,85 @@ def drive(self, request: TaskRequest, harness: Any) -> AgentLedger: self.ledger.wall_seconds = time.monotonic() - self.ledger.started_at return self.ledger + def _code_tool(self) -> Any: + """The code-action tool over this run's harness (web only).""" + + if self._tool is None: + from openadapt_flow.compile_agent.code_action import CodeActionTool, _watch + from openadapt_flow.compile_agent.harness import Harness + + if not isinstance(self.harness, Harness): + raise ValueError( + "scripts run on web apps only; on this app use teach_act" + ) + self._tool = CodeActionTool( + self.harness, watch=_watch(self.request, self.harness) + ) + self.last_trace = self._tool.trace + return self._tool + + def script_binding(self, script: str) -> str: + """What a person's approval of a script covers: this run, the actions + performed so far, and the script's first line (the save).""" + + first = next((ln.strip() for ln in str(script).splitlines() if ln.strip()), "") + actions = self.harness.stats.actions if self.harness is not None else 0 + return json.dumps([self.run_id, actions, first]) + + def _run_script(self, command: _Command) -> dict[str, Any]: + tool = self._code_tool() + script = str(command.payload.get("script") or "") + if command.kind == "run_preview": + commit = tool.first_commit(script) + first = next((ln.strip() for ln in script.splitlines() if ln.strip()), "") + return { + "commit": commit, + "binding": self.script_binding(script) if commit else None, + "action": first[:200], + } + held = command.payload.get("_held_by_person") + if held and held != self.script_binding(script): + raise ValueError( + "the app or the run changed after the person approved this save, " + "so nothing was done; call teach_run again" + ) + person_approves = self.harness.gate.hold is not None + approval = {"left": bool(held)} + + def approve(control: str) -> bool: + if not person_approves: + return True # the gate's own right-record check decides + # A person's approval covers one save, on the script's first line. + if approval["left"] and tool.line_number == 1: + approval["left"] = False + self._approved = control + return True + return False + + tool.approve = approve + before = self.harness.stats.actions + try: + out = tool.run(script) + finally: + tool.approve = None + self._approved = None + self.performed += self.harness.stats.actions - before + answer: dict[str, Any] = {"output": out.text, "committed": out.committed} + if out.images: + answer["output"] += "\n(screenshots are not shared here; use teach_screen)" + if out.pending is not None: + rest = "\n".join(out.pending["rest"]) + answer["pending"] = { + "save": out.pending["control"], + "script": rest, + "binding": self.script_binding(rest), + } + return answer + def _handle(self, command: _Command) -> dict[str, Any]: h = self.harness + if command.kind in ("run", "run_preview"): + return self._run_script(command) if command.kind == "observe": obs = h.observe() return {"observation": obs.text} diff --git a/packages/flow/openadapt_flow/compile_agent/harness.py b/packages/flow/openadapt_flow/compile_agent/harness.py index b1a44b0a1..2197f1550 100644 --- a/packages/flow/openadapt_flow/compile_agent/harness.py +++ b/packages/flow/openadapt_flow/compile_agent/harness.py @@ -21,6 +21,18 @@ This is the right-record check and the hold before saving from the guarded save design. It runs in the agent's run only; the compiled program has its own identity checks and reads every save back from the system of record. + +Read-back after every save. When the harness has a ``readback`` (the pipeline +always gives it one, built by :func:`record_readback` from the task's +``verify.yaml`` reader), every action that went through the gate is followed +by a read of the system of record, compared with the read taken when run 1 +started: + +* a change on any record other than the one the task names stops run 1 at + once (:class:`ReadBackRefused`); no draft is made; +* no change at all is reported to the agent ("nothing was saved"), so a screen + that says "saved" can't stand in for a save; +* a change on the named record is reported to the agent with its field names. """ from __future__ import annotations @@ -36,12 +48,15 @@ __all__ = [ "CommitHint", "GateRefused", + "ReadBack", + "ReadBackRefused", "GateRequest", "Harness", "HarnessStats", "SaveGate", "StaleRef", "SurfaceControl", + "record_readback", "SurfaceHarness", "SurfaceObservation", ] @@ -52,6 +67,107 @@ class GateRefused(RuntimeError): said no. Nothing in the refused action was performed.""" +class ReadBackRefused(GateRefused): + """A save went through, and the system of record shows a change on a record + the task does not name. Run 1 stops; no draft is made.""" + + +@dataclass(frozen=True) +class ReadBack: + """What the system of record showed after one save. + + ``outcome`` is ``landed`` (only the named record changed), ``no_change``, + ``wrong_record`` (another record changed) or ``unreadable``. + """ + + outcome: str + detail: str + #: Field names of the named record's change (values stay out of the log). + fields: tuple[str, ...] = () + + def to_dict(self) -> dict[str, Any]: + return { + "outcome": self.outcome, + "detail": self.detail, + "fields": list(self.fields), + } + + +def record_readback( + read: Callable[[], Optional[list[dict[str, Any]]]], + expect: Mapping[str, Any], + baseline: Callable[[], Optional[list[dict[str, Any]]]], +) -> Callable[[], ReadBack]: + """A read-back for the harness: ``read`` the system of record and compare + it with ``baseline()`` (the read when run 1 started). A created or changed + record is the named one when it has every ``expect`` field with that value. + """ + + wanted = {str(k): str(v) for k, v in expect.items()} + + def named(record: Mapping[str, Any]) -> bool: + return all(str(record.get(k)) == v for k, v in wanted.items()) + + def check() -> ReadBack: + from openadapt_flow.record_sor import diff_snapshots + + before, after = baseline(), read() + if before is None or after is None: + return ReadBack("unreadable", "the system of record could not be read") + diff = diff_snapshots(before, after) + touched = [*diff.created, *(c.after for c in diff.changed)] + if diff.removed: + return ReadBack("wrong_record", f"{len(diff.removed)} records disappeared") + others = [r for r in touched if not named(r)] + if others: + return ReadBack( + "wrong_record", + f"{len(others)} record(s) the task does not name changed", + ) + if not touched: + return ReadBack("no_change", "nothing was saved in the system of record") + fields = tuple(sorted({k for r in touched for k in r})) + return ReadBack( + "landed", + f"{len(touched)} record(s) with " + + ", ".join(f"{k}={v}" for k, v in wanted.items()) + + " saved", + fields, + ) + + return check + + +def _read_back(harness: Any, control: str) -> Optional[ReadBack]: + """Read the system of record after a save went through the gate; poll a + little for a write that lands late. Raises ReadBackRefused on another + record's change.""" + + check = getattr(harness, "readback", None) + if check is None: + return None + deadline = time.monotonic() + max(float(harness.readback_wait_s), 0.0) + while True: + try: + got = check() + except Exception as exc: # noqa: BLE001 - an unreadable read is reported + got = ReadBack("unreadable", f"{type(exc).__name__}: {exc}"[:200]) + if got.outcome != "no_change" or time.monotonic() >= deadline: + break + time.sleep(0.25) + harness.stats.readbacks += 1 + harness.readback_log.append( + {"control": control, **got.to_dict(), "at": time.time()} + ) + if got.outcome == "wrong_record": + harness.stats.gate_refusals += 1 + harness.refusal = ( + f"after {control!r}, the read-back shows {got.detail}; run 1 stopped" + ) + raise ReadBackRefused(harness.refusal) + return got + + class StaleRef(ValueError): """The ref is not in the latest observation, or the page changed under it.""" @@ -129,6 +245,8 @@ class HarnessStats: #: Kept as evidence with the run; not sent to the model. gate_evidence_screenshots: int = 0 gate_refusals: int = 0 + #: System-of-record reads after a save (see :func:`record_readback`). + readbacks: int = 0 def to_dict(self) -> dict[str, int]: return dict(self.__dict__) @@ -147,6 +265,10 @@ class Harness: #: One entry per gate decision, kept with the run. gate_log: list[dict[str, Any]] = field(default_factory=list) refusal: Optional[str] = None + #: Read after every save that went through the gate (see the module doc). + readback: Optional[Callable[[], ReadBack]] = None + readback_wait_s: float = 3.0 + readback_log: list[dict[str, Any]] = field(default_factory=list) _last: Optional[_observe.StructuralObservation] = None @property @@ -227,12 +349,17 @@ def native_for( return {"kind": "click", "x": control.x, "y": control.y}, control.ref def commit_target( - self, native: Mapping[str, Any], ref: Optional[str] = None + self, + native: Mapping[str, Any], + ref: Optional[str] = None, + *, + hint: Optional[CommitHint] = None, ) -> Optional[str]: """What may commit a write in ``native``, or None. Nothing is performed. ``ref`` is the text field a ``type`` will be typed into (it is clicked first), so the check looks at that field rather than today's focus. + ``hint`` is what a structure-aware tool resolved (see :meth:`act`). """ if ref is not None and native.get("kind") == "type": @@ -242,7 +369,7 @@ def commit_target( return "a typed line break" if any(c in text for c in "\r\n") else None hit = self._commit_target(native) if self.has_page else None if hit is None: - hit = self._hinted_commit(native, None) + hit = self._hinted_commit(native, hint) return hit[0] if hit is not None else None # -- every action ------------------------------------------------------------- @@ -277,6 +404,8 @@ def act( if commit is not None and self.has_page: # Let the app react before the agent looks at the result. _settle(self.page) + if commit is not None: + _read_back(self, commit[0]) return obs def _hinted_commit( @@ -567,6 +696,9 @@ class SurfaceHarness: stats: HarnessStats = field(default_factory=HarnessStats) gate_log: list[dict[str, Any]] = field(default_factory=list) refusal: Optional[str] = None + readback: Optional[Callable[[], ReadBack]] = None + readback_wait_s: float = 3.0 + readback_log: list[dict[str, Any]] = field(default_factory=list) _last: Optional[SurfaceObservation] = None _focused_role: Optional[str] = None @@ -679,6 +811,8 @@ def act(self, native: Mapping[str, Any], ref: Optional[str] = None) -> Observati self._check_gate(commit) obs = self.computer.execute(native) self.stats.actions += 1 + if commit is not None: + _read_back(self, commit) kind = native.get("kind") if kind in ("click", "double_click", "right_click"): control = self._ref_control(ref) diff --git a/packages/flow/openadapt_flow/compile_agent/pipeline.py b/packages/flow/openadapt_flow/compile_agent/pipeline.py index c3c69e993..ebe8cfa9e 100644 --- a/packages/flow/openadapt_flow/compile_agent/pipeline.py +++ b/packages/flow/openadapt_flow/compile_agent/pipeline.py @@ -45,6 +45,7 @@ Harness, SaveGate, SurfaceHarness, + record_readback, ) from openadapt_flow.jit import lifecycle from openadapt_flow.jit.compile_agent import ( @@ -236,7 +237,8 @@ class CompileResult: ``status`` is ``admitted`` (later runs make no model calls), ``draft`` (run 1 passed; checks or admission wait for a person), ``needs_person`` (outside the agent's admission policy), or why nothing can be admitted: - ``refused_wrong_record``, ``refused_by_hold``, ``record_check_failed``, + ``refused_wrong_record``, ``refused_by_hold``, ``readback_wrong_record`` + (a save landed on a record the task does not name), ``record_check_failed``, ``canary_failed``, ``capture_failed``, ``unsupported``, ``scope_refused`` (run 1 went outside the task's declared scope), ``readback_refused`` (the system of record did not show exactly the declared write). @@ -300,8 +302,11 @@ def __init__( keep: Sequence[str], evidence_dir: Path, principal_id: Optional[str], + readback: Optional[Callable[[], Any]] = None, ) -> None: self.inner = inner + #: Read after every save that goes through the gate (harness module). + self.readback = readback self.dialect = str(inner.dialect) self.gate = gate self.keep = list(keep) @@ -320,7 +325,11 @@ def run(self, request: TaskRequest, computer: Any) -> None: # window, an RDP or Citrix client window) gets the surface harness. harness_type = Harness if computer.page is not None else SurfaceHarness self.harness = harness_type( - computer, self.gate, keep=self.keep, evidence_dir=self.evidence_dir + computer, + self.gate, + keep=self.keep, + evidence_dir=self.evidence_dir, + readback=self.readback, ) try: self.ledger = self.inner.drive(request, self.harness) @@ -619,6 +628,13 @@ def record( keep=[*self.on_screen, *values.values(), *self.expect.values()], evidence_dir=attempt / "gates", principal_id=principal_id or getattr(agent, "principal_id", None), + # Every save in run 1 is read back from the system of record and + # compared with the read taken when run 1 started. + readback=record_readback( + lambda: self.observer.system_of_record, + self.expect, + lambda: self._baseline, + ), ) jit.agent = harnessed jit.dialect = harnessed.dialect @@ -635,6 +651,10 @@ def record( "agent": harnessed.ledger.to_dict() if harnessed.ledger else None, "harness": harness.stats.to_dict() if harness else None, "gates": harness.gate_log if harness else [], + "readbacks": [ + {k: v for k, v in r.items() if k != "at"} + for r in (harness.readback_log if harness else []) + ], "agent_error": harnessed.error, } trace_data: Optional[dict[str, Any]] = None @@ -668,6 +688,8 @@ def done(status: str, detail: Optional[str], **kw: Any) -> CompileResult: if decision == "refused_by_hold" else "refused_wrong_record" ) + if any(r["outcome"] == "wrong_record" for r in harness.readback_log): + status = "readback_wrong_record" if jit.candidate() is not None: # An agent that caught the refusal and went on still made a # draft; it must never be admitted. diff --git a/packages/flow/tests/e2e/test_code_agent_default_e2e.py b/packages/flow/tests/e2e/test_code_agent_default_e2e.py new file mode 100644 index 000000000..014aafc2d --- /dev/null +++ b/packages/flow/tests/e2e/test_code_agent_default_e2e.py @@ -0,0 +1,216 @@ +"""Code as action as the default first-run agent, with a read-back on every save. + +On MockMed, in a real browser, with no model calls: + +* compile-agent's run 1 reads each save back from the system of record and + tells the agent what landed; the read-backs are kept with run 1; +* a save that lands on another patient (``?fault=wrong_record``) stops run 1 + at the read-back, before the agent goes on; nothing can be admitted; +* a save the server drops while the screen says "saved" + (``?fault=optimistic``) is reported to the agent as "nothing was saved", + and run 1 fails its record check; +* the MCP teach path's ``teach_run`` does the task from scripts and drafts a + program; with a person approving saves, the script stops before the save, + and the approval (bound to that line and that moment) lets exactly that + line through. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Iterator + +import pytest + +from openadapt_flow.compile_agent import FirstRunCompiler, ScriptedCodeAgent +from openadapt_flow.mockmed.fault_server import serve +from openadapt_flow.tutorial import _http_json, _records + +PROMPT = ( + "Open the referral for Jane Sample, start a Triage encounter, and save " + "this note: {note}" +) +NOTE = "Synthetic follow-up in two weeks" +SCRIPTS = [ + 'click role=button name=Open near="Jane Sample"\ntext grep=MRN', + 'click name="New Encounter"\nclick name=Triage\nfill name=Note value="{note}"\n' + 'click name="Save Encounter"', + "text grep=saved", +] + + +@pytest.fixture(scope="module") +def mockmed() -> Iterator[str]: + base, _db, stop = serve() + yield base.rstrip("/") + stop() + + +def _reset(base: str): + return lambda: _http_json(f"{base}/api/reset", method="POST", body={}) + + +def _verify_yaml(tmp_path: Path, base: str) -> Path: + path = tmp_path / "verify.yaml" + path.write_text( + "verifier:\n" + " kind: rest\n" + f" base_url: {base}\n" + " records_path: /api/db\n" + " records_key: records\n" + " timeout_s: 2.0\n" + " poll_interval_s: 0.05\n" + ) + return path + + +def _compiler(tmp_path: Path, base: str, fault: str = "ok") -> FirstRunCompiler: + return FirstRunCompiler( + PROMPT, + params=["note"], + url=f"{base}/?fault={fault}&idempotency=demo#tasks", + verify=_verify_yaml(tmp_path, base), + expect={"patient_id": "p1", "type": "Triage"}, + on_screen=["Jane Sample", "MRN P1"], + task_id=f"mockmed.code_default.{fault}", + cache_dir=tmp_path / "cache", + key_dir=tmp_path / "keys", + reset=_reset(base), + on_warning=lambda text: None, + ) + + +def test_every_save_in_run_1_is_read_back(mockmed: str, tmp_path: Path) -> None: + agent = ScriptedCodeAgent(SCRIPTS) + result = _compiler(tmp_path, mockmed).record(agent, {"note": NOTE}) + assert result.status == "draft", result.detail + assert result.run1["readbacks"] == [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": ["id", "key", "note", "patient_id", "source", "type"], + } + ] + assert result.run1["harness"]["readbacks"] == 1 + assert "read back: 1 record(s) with patient_id=p1" in agent.outputs[1] + assert result.run1["code_action"]["readbacks"] == {"landed": 1} + + +def test_a_save_on_another_patient_stops_run_1_at_the_read_back( + mockmed: str, tmp_path: Path +) -> None: + agent = ScriptedCodeAgent(SCRIPTS) + compiler = _compiler(tmp_path, mockmed, fault="wrong_record") + result = compiler.record(agent, {"note": NOTE}) + assert result.status == "readback_wrong_record", result.detail + assert "the task does not name" in (result.detail or "") + assert result.run1["readbacks"][0]["outcome"] == "wrong_record" + # The gate saw the right patient on screen; the read-back caught the save. + assert result.run1["gates"][0]["decision"] == "passed_no_hold" + # The agent never got to its next script. + assert len(agent.outputs) == 1 + assert compiler._jit().candidate() is None + + +def test_a_dropped_save_is_reported_as_not_saved(mockmed: str, tmp_path: Path) -> None: + agent = ScriptedCodeAgent(SCRIPTS[:2]) + result = _compiler(tmp_path, mockmed, fault="optimistic").record( + agent, {"note": NOTE} + ) + assert result.run1["readbacks"][0]["outcome"] == "no_change" + assert "nothing was saved" in agent.outputs[1] + assert "do not report it as saved" in agent.outputs[1] + # Flow's own check after run 1 refuses it too: nothing landed. + assert result.status == "readback_refused", result.detail + assert _records(mockmed) == [] + + +# -- the MCP teach path ---------------------------------------------------------- + + +def _teach(tmp_path: Path, base: str, *, hold: bool) -> object: + from openadapt_flow.agent.teach import TeachBridge, TeachConfig + + return TeachBridge( + TeachConfig( + verify=_verify_yaml(tmp_path, base), + cache_dir=tmp_path / "teach", + url=f"{base}/?fault=ok&idempotency=demo#tasks", + key_dir=tmp_path / "keys", + hold=hold, + reset=_reset(base), + ) + ) + + +def _start(teach: object) -> dict: + return teach.dispatch( # type: ignore[attr-defined] + "teach_start", + { + "task": PROMPT, + "inputs": {"note": NOTE}, + "record_fields": {"patient_id": "p1", "type": "Triage"}, + "on_screen": ["Jane Sample", "MRN P1"], + "name": "add_note", + }, + ) + + +def test_teach_run_is_offered_and_drafts_a_program( + mockmed: str, tmp_path: Path +) -> None: + teach = _teach(tmp_path, mockmed, hold=False) + try: + names = [spec.name for spec in teach.list_tool_specs()] # type: ignore[attr-defined] + assert names.index("teach_run") < names.index("teach_act") + started = _start(teach) + assert started["started"] and "teach_run" in started["next"] + for script in SCRIPTS[:2]: + out = teach.dispatch( # type: ignore[attr-defined] + "teach_run", {"script": script.replace("{note}", NOTE)} + ) + assert out["status"] == "ok", out + assert "read back: 1 record(s)" in out["output"] + report = teach.dispatch("teach_finish", {}) # type: ignore[attr-defined] + assert report["status"] == "draft", report + finally: + teach.close() # type: ignore[attr-defined] + + +def test_teach_run_stops_before_a_save_a_person_must_approve( + mockmed: str, tmp_path: Path +) -> None: + teach = _teach(tmp_path, mockmed, hold=True) + try: + _start(teach) + out = teach.dispatch("teach_run", {"script": SCRIPTS[0]}) # type: ignore[attr-defined] + assert out["status"] == "ok", out + out = teach.dispatch( # type: ignore[attr-defined] + "teach_run", {"script": SCRIPTS[1].replace("{note}", NOTE)} + ) + assert out["status"] == "needs_person", out + assert out["script"] == 'click name="Save Encounter"' + assert _records(mockmed) == [] + # The MCP layer previews the first line, a person approves it in the + # client's form, and the approval is bound to it. + hold = teach.hold_needed({"script": out["script"]}, tool="teach_run") # type: ignore[attr-defined] + assert hold and hold["commit"] == "Save Encounter" + stale = teach.dispatch( # type: ignore[attr-defined] + "teach_run", + {"script": out["script"]}, + held_by_person=hold["binding"].replace('"click', '"press'), + ) + assert stale["status"] == "error" and _records(mockmed) == [] + done = teach.dispatch( # type: ignore[attr-defined] + "teach_run", {"script": out["script"]}, held_by_person=hold["binding"] + ) + assert done["status"] == "ok", done + assert "read back: 1 record(s)" in done["output"] + assert [(r["patient_id"], r["note"]) for r in _records(mockmed)] == [ + ("p1", NOTE) + ] + report = teach.dispatch("teach_finish", {}) # type: ignore[attr-defined] + assert report["status"] == "draft", report + finally: + teach.close() # type: ignore[attr-defined] diff --git a/packages/flow/tests/e2e/test_compile_agent_e2e.py b/packages/flow/tests/e2e/test_compile_agent_e2e.py index 10ce9b902..7133b6eff 100644 --- a/packages/flow/tests/e2e/test_compile_agent_e2e.py +++ b/packages/flow/tests/e2e/test_compile_agent_e2e.py @@ -214,7 +214,7 @@ def test_compile_agent_then_replay_with_no_model_calls( fake = FakeClaude("Jane Sample", NOTE) # The registered agent drives run 1 and admits it: self-admission, which # its signed policy allows. - agent = ClaudeAgent(client=fake, principal_id=AGENT_ID) + agent = ClaudeAgent(vision="selective", client=fake, principal_id=AGENT_ID) result = compile_first_run( PROMPT, params={"note": NOTE}, @@ -303,7 +303,7 @@ def test_wrong_patient_is_refused_at_the_save( result = compile_first_run( PROMPT, params={"note": NOTE}, - agent=ClaudeAgent(client=fake), + agent=ClaudeAgent(vision="selective", client=fake), confirm_checks="dana.reviewer", admitter=admitter, **_common(tmp_path, mockmed), diff --git a/packages/flow/tests/test_code_action.py b/packages/flow/tests/test_code_action.py index 3b52548d8..a1f77d62c 100644 --- a/packages/flow/tests/test_code_action.py +++ b/packages/flow/tests/test_code_action.py @@ -687,3 +687,37 @@ def test_the_claude_loop_in_code_mode(tmp_path: Path, monkeypatch: Any) -> None: assert ledger.to_dict()["images_in_requests"] == 0 assert agent.last_trace.summary()["commits"] == 1 assert harness.stats.gate_checks == 1 + + +def test_code_as_action_is_the_default_first_run_agent() -> None: + from openadapt_flow.__main__ import build_parser + from openadapt_flow.compile_agent.agents import CODE_DEFAULT_MODEL, DEFAULT_MODEL + from openadapt_flow.compile_agent.cli import _agent + + agent = ClaudeAgent() + assert agent.vision == "code" and agent.model == CODE_DEFAULT_MODEL + assert agent.dialect == "generic" + assert ClaudeAgent(vision="selective").model == DEFAULT_MODEL + args = build_parser().parse_args( + ["compile-agent", "Save the note {note}", "--url", "http://app.test/"] + ) + cli_agent = _agent(args) + assert cli_agent.vision == "code" and cli_agent.model == CODE_DEFAULT_MODEL + + +def test_record_readback_names_the_record_and_flags_any_other() -> None: + from openadapt_flow.compile_agent.harness import record_readback + + before = [{"id": 1, "patient_id": "p2", "note": "old"}] + state = {"now": list(before)} + check = record_readback(lambda: state["now"], {"patient_id": "p1"}, lambda: before) + assert check().outcome == "no_change" + state["now"] = before + [{"id": 2, "patient_id": "p1", "note": "x"}] + landed = check() + assert landed.outcome == "landed" and landed.fields == ("id", "note", "patient_id") + state["now"] = [{"id": 1, "patient_id": "p2", "note": "edited"}] + assert check().outcome == "wrong_record" + state["now"] = [] + assert check().outcome == "wrong_record" + unreadable = record_readback(lambda: None, {"patient_id": "p1"}, lambda: before) + assert unreadable().outcome == "unreadable" diff --git a/packages/flow/tests/test_compile_agent.py b/packages/flow/tests/test_compile_agent.py index 9ab412a0c..b9c8df8f2 100644 --- a/packages/flow/tests/test_compile_agent.py +++ b/packages/flow/tests/test_compile_agent.py @@ -225,7 +225,7 @@ def test_claude_loop_counts_tokens_and_the_gate_screenshot() -> None: ] ) harness = _Harness() - ledger = ClaudeAgent(client=client).drive(REQUEST, harness) # type: ignore[arg-type] + ledger = ClaudeAgent(vision="selective", client=client).drive(REQUEST, harness) # type: ignore[arg-type] assert ledger.finished and ledger.summary == "done" assert ledger.model_calls == 3 and ledger.tool_errors == 1 # the stale ref assert harness.clicks == ["e1"] @@ -252,13 +252,17 @@ def test_every_turn_mode_uses_the_computer_toolset() -> None: def test_a_refused_save_ends_the_loop() -> None: client = _Client([("click", {"ref": "e1"})]) with pytest.raises(GateRefused): - ClaudeAgent(client=client).drive(REQUEST, _Harness(refuse=True)) # type: ignore[arg-type] + ClaudeAgent(vision="selective", client=client).drive( + REQUEST, _Harness(refuse=True) + ) # type: ignore[arg-type] def test_the_cost_cap_ends_the_loop() -> None: client = _Client([("observe", {})] * 5, tokens=400_000) with pytest.raises(BudgetExceeded): - ClaudeAgent(client=client, max_cost_usd=1.0).drive(REQUEST, _Harness()) # type: ignore[arg-type] + ClaudeAgent(vision="selective", client=client, max_cost_usd=1.0).drive( + REQUEST, _Harness() + ) # type: ignore[arg-type] assert len(client.requests) == 1 From 7525b68540ecdcbd8dddb2d7b467ac95ed8559b1 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 00:21:38 -0400 Subject: [PATCH 2/5] feat(flow): measured code agent with read-back; recorder binds a late write to its save - the recorder's after-read waits for the system of record to change after a save the gate let through (Recorder.expect_write), so the compiler binds the write to that step (OpenEMR's chart lands after the screen settles) - scoreboard: arm 7 reports success only when its last read-back landed (Amendment 11); MockMed items 21-40 for arm 7 and the program compiled from it (Amendment 10); OpenEMR real-app check (Amendment 12) - docs: CODE_AS_ACTION, COMPILE_AGENT, agent README, CAPABILITIES, llms-full Co-Authored-By: Claude Opus 5.5 --- CAPABILITIES.md | 9 +- llms-full.txt | 10 +- packages/agent/README.md | 4 +- .../flow/benchmark/scoreboard/PROTOCOL.md | 58 +++ .../flow/benchmark/scoreboard/SCOREBOARD.md | 18 +- .../code_agent/2026-10-25-rb/HAND_REVIEW.md | 13 + .../code_agent/2026-10-25-rb/items.csv | 21 + .../2026-10-25-rb/ledgers/item-021.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-022.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-023.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-024.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-025.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-026.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-027.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-028.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-029.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-030.json | 181 ++++++++ .../2026-10-25-rb/ledgers/item-031.json | 188 ++++++++ .../2026-10-25-rb/ledgers/item-032.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-033.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-034.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-035.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-036.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-037.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-038.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-039.json | 169 ++++++++ .../2026-10-25-rb/ledgers/item-040.json | 117 +++++ .../code_agent/2026-10-25-rb/meta.json | 23 + .../mockmed/program/2026-10-25-rb/items.csv | 23 + .../mockmed/program/2026-10-25-rb/meta.json | 16 + .../code_agent/2026-10-25-rb-admit/items.csv | 2 + .../2026-10-25-rb-admit/ledgers/item-006.json | 360 +++++++++++++++ .../code_agent/2026-10-25-rb-admit/meta.json | 22 + .../code_agent/2026-10-25-rb/items.csv | 6 + .../2026-10-25-rb/ledgers/item-001.json | 346 +++++++++++++++ .../2026-10-25-rb/ledgers/item-002.json | 346 +++++++++++++++ .../2026-10-25-rb/ledgers/item-003.json | 374 ++++++++++++++++ .../2026-10-25-rb/ledgers/item-004.json | 319 ++++++++++++++ .../2026-10-25-rb/ledgers/item-005.json | 409 ++++++++++++++++++ .../code_agent/2026-10-25-rb/meta.json | 22 + .../flow/benchmark/scoreboard/raw/spend.json | 132 +++++- packages/flow/benchmark/scoreboard/run.py | 4 + .../flow/benchmark/scoreboard/scoreboard.json | 371 ++++++++++++++-- packages/flow/docs/CODE_AS_ACTION.md | 68 ++- packages/flow/docs/COMPILE_AGENT.md | 20 +- .../openadapt_flow/compile_agent/harness.py | 14 + packages/flow/openadapt_flow/recorder.py | 22 + 47 files changed, 6316 insertions(+), 75 deletions(-) create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/HAND_REVIEW.md create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/items.csv create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-021.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-022.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-023.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-024.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-025.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-026.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-027.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-028.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-029.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-030.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-031.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-032.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-033.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-034.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-035.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-036.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-037.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-038.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-039.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-040.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/meta.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/items.csv create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/meta.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/items.csv create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/ledgers/item-006.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/meta.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/items.csv create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-001.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-002.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-003.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-004.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-005.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/meta.json diff --git a/CAPABILITIES.md b/CAPABILITIES.md index e541d413f..e65713660 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -19,8 +19,9 @@ shipping in the next release. 1. **Learns from any successful run, a person's or an agent's.** A person records a task, or a computer-use agent does it once and OpenAdapt compiles the verified run into a program (`openadapt_flow.jit`, released; - prompt-to-program `compile-agent`, landing; the agent can act through code - instead of screenshots, [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)). Agents can drive the recording + prompt-to-program `compile-agent`, landing; the agent acts through code by default, not screenshots, and every + save in its run is read back, + [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)). Agents can drive the recording on web, native Windows, RDP and Citrix. An agent connected over MCP can teach itself a new workflow and then call it as a tool (`openadapt-agent serve --teach`, @@ -92,7 +93,7 @@ shipping in the next release. | Repairs itself when the app changes: governed repair, agent-demonstrated fixes, fleet patches, and self-repair probed up to the save. | [REPAIR_LIFECYCLE.md](packages/flow/docs/REPAIR_LIFECYCLE.md) | | Repairs a change that needs new steps, such as a new required dialog or a control moved into a menu: up to three new steps, each with its own record check, probed up to the save before one person approves. | [SELF_REPAIR.md](packages/flow/docs/SELF_REPAIR.md), [#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325) | | Compiles a computer-use agent's verified run, with no human demonstration needed. | [JIT.md](packages/flow/docs/JIT.md) | -| The agent that does a task once can act through code, not a screenshot-and-click loop: it reads the page or accessibility tree, acts by role and name, keeps notes with a sandboxed read-only shell, and asks for a screenshot only when it needs one. It is arm 7 of the scoreboard; its figures are quoted from there once its rows are in. | `CODE_AS_ACTION.md` in [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305) | +| The agent that does a task once acts through code by default, not a screenshot-and-click loop: it reads the page or accessibility tree, acts by role and name, keeps notes with a sandboxed read-only shell, and asks for a screenshot only when it needs one. Every save in that run is read back from the system of record before the agent goes on. Scoreboard arm 7, MockMed items 21 to 40: 18 of 18 normal items correct at $0.0093 an item, the wrong-patient save stopped by the read-back, no silent wrong save on hand review; the program compiled from it: 18 of 18 at $0. | `CODE_AS_ACTION.md` in [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305) | | Binds each approval to the exact screenshot and form state, and re-reads the form before saving. | [APPROVAL_MODES.md](packages/flow/docs/APPROVAL_MODES.md) | | Has a written threat model for prompt injection against the agent that teaches or repairs a program, with tests. | [THREAT_MODEL.md](packages/flow/docs/THREAT_MODEL.md) | | Fills inputs from a referral PDF, fax image or email, and holds back anything unclear for a person. | `intake`, [INTAKE.md](packages/flow/docs/INTAKE.md) | @@ -235,7 +236,7 @@ improvising or retrying when something is off. | --- | --- | --- | --- | | Compile a computer-use agent's successful run into a program (JIT); later runs make no model calls | Released | Synthetic (a recorded Claude run on MockMed) | `openadapt_flow.jit`, [JIT.md](packages/flow/docs/JIT.md) | | Reuse a compiled program only for the record check and app environment it was admitted under (a changed record check or a new app version at the same origin is refused) | On main ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Unit tests | `TaskSignature(record_check=, environment=)`, `WebApp(environment_id=)`, [JIT.md](packages/flow/docs/JIT.md) | -| Code as action: the run-1 agent acts through one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program | On main ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | +| Code as action is the default run-1 agent (compile-agent, and `teach_run` in the MCP teach tools): one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program, and every save is read back from the system of record | On main ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | | Admit a draft by a person, an admitter key, or an agent under a person-signed policy, including self-admission when the policy allows it | Released | Unit tests | [JIT_ADMISSION.md](packages/flow/docs/JIT_ADMISSION.md) | | MCP server: your agent decides what to enter, OpenAdapt makes the entry and checks it | Released | Synthetic sandbox | `openadapt-agent`, [packages/agent](packages/agent/README.md) | | Teach over MCP: an agent does a new task once through OpenAdapt; the run is read back, compiled and admitted (a person confirms the record check; the agent admits only under a person-signed policy), and becomes a tool it calls with no model calls. Web, and Windows, macOS, Linux, RDP and Citrix with a person approving each save | On main ([#1335](https://github.com/OpenAdaptAI/OpenAdapt/pull/1335)) | Synthetic (MockMed over MCP, and through the real RDP and Citrix backends) | `openadapt-agent serve --teach`, [packages/agent](packages/agent/README.md#teach-a-new-workflow-from-your-agent) | diff --git a/llms-full.txt b/llms-full.txt index 02ba1a00d..e65a56d52 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -416,8 +416,9 @@ shipping in the next release. 1. **Learns from any successful run, a person's or an agent's.** A person records a task, or a computer-use agent does it once and OpenAdapt compiles the verified run into a program (`openadapt_flow.jit`, released; - prompt-to-program `compile-agent`, landing; the agent can act through code - instead of screenshots, [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)). Agents can drive the recording + prompt-to-program `compile-agent`, landing; the agent acts through code by default, not screenshots, and every + save in its run is read back, + [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)). Agents can drive the recording on web, native Windows, RDP and Citrix. An agent connected over MCP can teach itself a new workflow and then call it as a tool (`openadapt-agent serve --teach`, @@ -489,7 +490,7 @@ shipping in the next release. | Repairs itself when the app changes: governed repair, agent-demonstrated fixes, fleet patches, and self-repair probed up to the save. | [REPAIR_LIFECYCLE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/REPAIR_LIFECYCLE.md) | | Repairs a change that needs new steps, such as a new required dialog or a control moved into a menu: up to three new steps, each with its own record check, probed up to the save before one person approves. | [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md), [#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325) | | Compiles a computer-use agent's verified run, with no human demonstration needed. | [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | -| The agent that does a task once can act through code, not a screenshot-and-click loop: it reads the page or accessibility tree, acts by role and name, keeps notes with a sandboxed read-only shell, and asks for a screenshot only when it needs one. It is arm 7 of the scoreboard; its figures are quoted from there once its rows are in. | `CODE_AS_ACTION.md` in [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305) | +| The agent that does a task once acts through code by default, not a screenshot-and-click loop: it reads the page or accessibility tree, acts by role and name, keeps notes with a sandboxed read-only shell, and asks for a screenshot only when it needs one. Every save in that run is read back from the system of record before the agent goes on. Scoreboard arm 7, MockMed items 21 to 40: 18 of 18 normal items correct at $0.0093 an item, the wrong-patient save stopped by the read-back, no silent wrong save on hand review; the program compiled from it: 18 of 18 at $0. | `CODE_AS_ACTION.md` in [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305) | | Binds each approval to the exact screenshot and form state, and re-reads the form before saving. | [APPROVAL_MODES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/APPROVAL_MODES.md) | | Has a written threat model for prompt injection against the agent that teaches or repairs a program, with tests. | [THREAT_MODEL.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/THREAT_MODEL.md) | | Fills inputs from a referral PDF, fax image or email, and holds back anything unclear for a person. | `intake`, [INTAKE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/INTAKE.md) | @@ -563,6 +564,7 @@ improvising or retrying when something is off. | Learn one program with branches from two or more recordings, or say which recordings are missing | Released | Synthetic | `openadapt-flow induce`, `disambiguate`; [CAPABILITIES.md (flow)](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/CAPABILITIES.md) | | Learn a guarded branch from several halts fixed the same way | Released | Synthetic | `openadapt-flow teach-branch`, `learning/branch_teach.py` | | Run a program over a worklist | Released | Real app | `openadapt-flow for-each` | +| Learn a worklist loop from recordings that search a record, open it, work it and go back to the list, even when the last row skips the return or rows differ (a field some rows lack, a write some rows get behind a screen check a person accepts). Each pass selects the row's own record by its values, verifies it, and checks that row's write. Certified and governed runs refuse it until qualification covers it | On main ([#1435](https://github.com/OpenAdaptAI/OpenAdapt/pull/1435)) | Synthetic (held-out rows run through the interpreter against an in-memory app) | `openadapt-flow induce`; `Step.target_param` | | Typed business decisions inside a run | Released | Synthetic | `business_decision` states, `openadapt-flow business-decisions` | | Watch many real runs in the background and mine candidate branches (shadow mode) | On main ([#1212](https://github.com/OpenAdaptAI/OpenAdapt/pull/1212)) | Synthetic | [SHADOW_MINING.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/capture/docs/SHADOW_MINING.md), [LEARN_FROM_MANY_RUNS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/LEARN_FROM_MANY_RUNS.md) | | Fill a workflow's inputs from a referral PDF, fax image or email | Released | Synthetic | `openadapt-flow intake`, [INTAKE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/INTAKE.md) | @@ -631,7 +633,7 @@ improvising or retrying when something is off. | --- | --- | --- | --- | | Compile a computer-use agent's successful run into a program (JIT); later runs make no model calls | Released | Synthetic (a recorded Claude run on MockMed) | `openadapt_flow.jit`, [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | | Reuse a compiled program only for the record check and app environment it was admitted under (a changed record check or a new app version at the same origin is refused) | On main ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Unit tests | `TaskSignature(record_check=, environment=)`, `WebApp(environment_id=)`, [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | -| Code as action: the run-1 agent acts through one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program | On main ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | +| Code as action is the default run-1 agent (compile-agent, and `teach_run` in the MCP teach tools): one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program, and every save is read back from the system of record | On main ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | | Admit a draft by a person, an admitter key, or an agent under a person-signed policy, including self-admission when the policy allows it | Released | Unit tests | [JIT_ADMISSION.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT_ADMISSION.md) | | MCP server: your agent decides what to enter, OpenAdapt makes the entry and checks it | Released | Synthetic sandbox | `openadapt-agent`, [packages/agent](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/agent/README.md) | | Teach over MCP: an agent does a new task once through OpenAdapt; the run is read back, compiled and admitted (a person confirms the record check; the agent admits only under a person-signed policy), and becomes a tool it calls with no model calls. Web, and Windows, macOS, Linux, RDP and Citrix with a person approving each save | On main ([#1335](https://github.com/OpenAdaptAI/OpenAdapt/pull/1335)) | Synthetic (MockMed over MCP, and through the real RDP and Citrix backends) | `openadapt-agent serve --teach`, [packages/agent](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/agent/README.md#teach-a-new-workflow-from-your-agent) | diff --git a/packages/agent/README.md b/packages/agent/README.md index 5b8056ada..ba74b7fb9 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -411,7 +411,7 @@ claude mcp add --scope user openadapt-teach -- uvx --python 3.12 --from 'openada Your agent then: 1. Calls `teach_start` with the task (`{note}` for each input), this run's `inputs`, the `record_fields` the saved record must have (such as its patient id), and `on_screen`, the record's identity as the screen shows it (a name and an MRN). -2. Calls `teach_observe` and `teach_act` until the task is done. On the web, `teach_observe` lists the page as short lines with a ref for each control, and before every save OpenAdapt checks that the record's identity is shown next to the control. A save on the wrong record is refused and ends the run. +2. Calls `teach_run` with short scripts (read the page, act on elements by role and name, read the result back; web apps), or `teach_observe` and `teach_act` one action at a time, until the task is done. After every save OpenAdapt reads the app's records back and tells the agent what landed; a save on a record the task does not name ends the run. On the web, `teach_observe` lists the page as short lines with a ref for each control, and before every save OpenAdapt checks that the record's identity is shown next to the control. A save on the wrong record is refused and ends the run. 3. Calls `teach_finish`. OpenAdapt compiles the run and reads the app's records: exactly one record must have appeared, with every `record_fields` value and every input. The result is the draft: its steps, its inputs, the proposed record check, and what still needs a person. 4. Calls `teach_confirm_checks`. Your MCP client shows a person the proposed check, and only their answer there, with their name, confirms it. Your agent has no argument that confirms it. 5. Calls `teach_admit` with `canary_inputs`, the values each canary saves. A canary is a real save on the same record, so use values that are safe there. @@ -428,7 +428,7 @@ It works on a web app (`--url`) and on the app a deployment file's `backend:` se | Flag | What it does | | --- | --- | -| `--teach VERIFY_YAML` | Adds `teach_start`, `teach_observe`, `teach_act`, `teach_finish`, `teach_abort`, `teach_status`, `teach_confirm_checks` and `teach_admit`, and turns on runs of taught workflows. | +| `--teach VERIFY_YAML` | Adds `teach_start`, `teach_run` (web apps), `teach_observe`, `teach_act`, `teach_finish`, `teach_abort`, `teach_status`, `teach_confirm_checks` and `teach_admit`, and turns on runs of taught workflows. | | `--teach-dir DIR` | Where drafts and taught programs live (default `RUNS_DIR/teach`). | | `--teach-key PEM`, `--teach-policy JSON` | The registered agent's encrypted key (passphrase in `OPENADAPT_JIT_KEY_PASSPHRASE`) and the signed admission policy. Only an agent key under a policy; an admitter key admits from CI with `openadapt-flow jit admit`. | | `--key-dir DIR` | The JIT key directory (default `OPENADAPT_JIT_KEY_DIR` or `~/.openadapt/jit`). | diff --git a/packages/flow/benchmark/scoreboard/PROTOCOL.md b/packages/flow/benchmark/scoreboard/PROTOCOL.md index a3728eaee..3c01f331f 100644 --- a/packages/flow/benchmark/scoreboard/PROTOCOL.md +++ b/packages/flow/benchmark/scoreboard/PROTOCOL.md @@ -350,3 +350,61 @@ read the receipt an earlier run left behind. Every claims script now uses a fresh directory per item and reads a receipt only after verify-run exits 0 or 1. Every row of the earlier claims runs has exit 0 or 1 and a fresh directory, so those runs were not affected. + +## Amendment 10 (2026-10-25): arm 7 with a read-back on every save, and the program it compiles + +Written while arm 7 was on item 25, before any result was summed. Code as +action became the default first-run agent, +and run 1 now reads every save back from the system of record before the +agent goes on (`compile_agent.harness.record_readback`; packages/flow +docs/CODE_AS_ACTION.md). Arm 7 measures that agent, and arm 3 measures the +program compiled from it: + +- MockMed items 21 to 40 (20 items, the protocol's own schedule, so item 30 + is the `optimistic` fault and item 40 the `wrong_record` fault). Run id + `2026-10-25-rb` for both arms. +- Arm 7: `run code_agent mockmed --start 21 --n 40 --keep-program`. Each item + is its own first run. The MockMed server runs on the fixed origin, because + item 21's admitted program is kept for arm 3. +- Arm 3: `run program mockmed --start 21 --n 40 --program-from + code-2026-10-25-rb`, the program compiled and admitted from arm 7's item 21. +- The outcome rules are unchanged. The read-back the agent sees is the + agent's evidence, not the scoreboard's: the scoreboard still reads the store + on its own after each item. +- Rows are timed only under the protocol's load rule. + +## Amendment 11 (2026-10-25, after Amendment 10's run): arm 7 reports what its read-back saw + +In Amendment 10's run, item 30 (the save the server drops while the screen +says "saved") came out `silent_wrong` by the word rule. The agent did not +claim the save: its final message says the save "did not take effect" and +that the read-back showed nothing was saved (`raw/mockmed/code_agent/ +2026-10-25-rb/ledgers/item-030.json`), but none of `FAILURE_WORDS` is in it. +By hand review the item is a safe stop (`HAND_REVIEW.md` next to the run). +Amendment 10's rows keep the word rule's verdict. + +From this amendment on, arm 7 reports success only when the word rule passes +and its own read-back after the last save shows the save landed. The read-back +is part of the arm (OpenAdapt reads every save back during run 1), so this is +what the arm reports, not the scoreboard's judgement: the scoreboard still +reads the store on its own after each item. + +## Amendment 12 (2026-10-25, written after the runs it describes): arm 7 on the OpenEMR referral with read-back + +A real-app check of the read-back, not a scored batch. Arm 7 on the +`realrun` OpenEMR fixture (pinned image, ports 9301/9300), items 1 to 5 +(`2026-10-25-rb`), then item 6 (`2026-10-25-rb-admit`) after a recorder fix. +Amendment 11's success rule applies. + +- Items 1 to 6: `correct`; each save's read-back showed the named chart + (matched by email and last name) landed. +- Admission refused all 6 before the canary ("a write step lacks an + independent read-back"). Items 1 to 5: the confirm-dialog save step had no + system-of-record change bound, because the chart lands after the recorder's + post-save read. The recorder now waits for the change after a save the gate + let through (`Recorder.expect_write`); item 6 and a scripted replay show + the save step bound (`record_written`). The remaining refusal is the + "Create New Patient" click before it: write-shaped, it opens the + duplicate-check dialog and writes nothing, so it has no read-back. Not + fixed here: binding a "nothing was written here" read-back to such a step + changes what admission accepts. diff --git a/packages/flow/benchmark/scoreboard/SCOREBOARD.md b/packages/flow/benchmark/scoreboard/SCOREBOARD.md index 2ee7fd99e..9072c77b4 100644 --- a/packages/flow/benchmark/scoreboard/SCOREBOARD.md +++ b/packages/flow/benchmark/scoreboard/SCOREBOARD.md @@ -8,11 +8,16 @@ Generated by `python -m scoreboard.build` from `raw/` (protocol 2026-10-18.1, se |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| | 1. Agent alone (Claude Sonnet 5.5 computer use) | claude-sonnet-5-5 | 100 | 47 of 47 (100%) | 98 of 100 | 2 | 0 / 0 of 53 | 0 | 0 | 5.0 | 0.0386 | 16.38 / 16.67 (n=3) | 20.1 / 20.22 | 181.1 | 0 | 8-524 | | 2. Agent's first run under OpenAdapt (recorded, read back, compiled) | claude-sonnet-5-5 | 1 | 1 of 1 (100%) | 1 of 1 | 0 | 0 / 0 of 0 | 0 | 0 | 5.0 | 0.0132 | - / - | - / - | - | 1 | 176-176 | -| 3. Compiled program driving the UI | none | 100 | 47 of 47 (100%) | 98 of 100 | 0 | 0 / 1 of 53 | 1 | 0 | 0.0 | 0.0 | - / - | - / - | - | 0 | 82-533 | -| 7. Code-as-action authoring agent, first run | claude-sonnet-5-5 | 5 | 5 of 5 (100%) | 5 of 5 | 0 | 0 / 0 of 0 | 0 | 0 | 5.0 | 0.0104 | 8.17 / 8.17 (n=1) | 9.78 / 9.78 | 368.0 | 5 | 12-14 | +| 3. Compiled program driving the UI | none | 120 | 65 of 65 (100%) | 116 of 120 | 0 | 0 / 2 of 55 | 2 | 0 | 0.0 | 0.0 | - / - | - / - | - | 0 | 39-533 | +| 7. Code-as-action authoring agent, first run | claude-sonnet-5-5 | 25 | 23 of 23 (100%) | 23 of 25 | 1 | 0 / 0 of 2 | 1 | 0 | 4.24 | 0.0095 | 8.17 / 8.17 (n=1) | 9.78 / 9.78 | 368.0 | 23 | 12-131 | - mockmed / agent: fault items {"optimistic": {"correct": 0, "silent_wrong": 1, "safe_stop": 0, "flagged_wrong": 0}, "ui_change": {"correct": 51, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 0}, "wrong_record": {"correct": 0, "silent_wrong": 1, "safe_stop": 0, "flagged_wrong": 0}} -- mockmed / program: fault items {"optimistic": {"correct": 0, "silent_wrong": 0, "safe_stop": 1, "flagged_wrong": 0}, "ui_change": {"correct": 51, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 0}, "wrong_record": {"correct": 0, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 1}} +- mockmed / program: fault items {"optimistic": {"correct": 0, "silent_wrong": 0, "safe_stop": 2, "flagged_wrong": 0}, "ui_change": {"correct": 51, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 0}, "wrong_record": {"correct": 0, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 2}} +- mockmed / code_agent: fault items {"optimistic": {"correct": 0, "silent_wrong": 1, "safe_stop": 0, "flagged_wrong": 0}, "wrong_record": {"correct": 0, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 1}} +- mockmed / program / run `2026-10-18-a` (engine 5881cb38): 100 rows, 47 of 47 correct, 0 paused, task s median - (timed rows only), items/hour - +- mockmed / program / run `2026-10-25-rb` (engine bedf5439): 20 rows, 18 of 18 correct, 0 paused, task s median - (timed rows only), items/hour - +- mockmed / code_agent / run `2026-10-19-timed` (engine a5627748): 5 rows, 5 of 5 correct, 0 paused, task s median 8.17 (timed rows only), items/hour 368.0 +- mockmed / code_agent / run `2026-10-25-rb` (engine bedf5439): 20 rows, 18 of 18 correct, 0 paused, task s median - (timed rows only), items/hour - - mockmed / agent: hand check of rows that are not correct: 2 of 2 checked, 2 outcomes confirmed; item 30 (optimistic): silent_wrong, ledgers/item-030.json final_text says the note was saved and that the encounter list shows it in full. The fault server rejected the write, and the read-back of /api/db after the item found no record (readback=missing). The agent reported a save that did not land; item 40 (wrong_record): silent_wrong, ledgers/item-040.json final_text says it saved the note on Jane Sample's referral. The fault server stored the write on another patient, and the read-back found no Triage record for p1 with this note (readback=wrong). The agent reported success on a record saved to the wrong patient (evidence for each in hand_check.json) ## openemr_referral @@ -25,7 +30,7 @@ Generated by `python -m scoreboard.build` from `raw/` (protocol 2026-10-18.1, se | 4. Compiled program, throughput fast path | none | 17 | 16 of 17 (94%) | 16 of 26 | 0 | 1 / 0 of 0 | 0 | 9 | 0.0 | 0.0 | 15.67 / 15.83 (n=4) | 15.84 / 16.0 | 229.5 | 0 | 10-15 | | 5. Compiled program with the API step | none | 20 | 18 of 18 (100%) | 18 of 20 | 0 | 0 / 2 of 2 | 0 | 0 | 0.0 | 0.0 | 0.6 / 0.67 (n=16) | 3.33 / 3.85 | 685.2 | 0 | 7-12 | | 6. Fast path or API step, N parallel workers | none | 31 | 26 of 31 (84%) | 26 of 52 | 0 | 5 / 0 of 0 | 0 | 21 | 0.0 | 0.0 | - / - | - / - | - | 0 | 13-240 | -| 7. Code-as-action authoring agent, first run | claude-sonnet-5-5 | 5 | 5 of 5 (100%) | 5 of 5 | 0 | 0 / 0 of 0 | 0 | 0 | 10.8 | 0.0429 | - / - | - / - | - | 0 | 11-32 | +| 7. Code-as-action authoring agent, first run | claude-sonnet-5-5 | 11 | 11 of 11 (100%) | 11 of 11 | 0 | 0 / 0 of 0 | 0 | 0 | 10.73 | 0.0419 | - / - | - / - | - | 0 | 11-256 | - openemr_referral / agent: fault items {"duplicate_check": {"correct": 0, "silent_wrong": 0, "safe_stop": 9, "flagged_wrong": 0}} - openemr_referral / program: fault items {"duplicate_check": {"correct": 0, "silent_wrong": 0, "safe_stop": 11, "flagged_wrong": 0}} @@ -40,6 +45,9 @@ Generated by `python -m scoreboard.build` from `raw/` (protocol 2026-10-18.1, se - self-loaded throughput: 304 correct items/hour (5 correct of 8 run in 59 s wall, 8 worker(s)); load at start 11.0, mean during 42.5 - openemr_referral / parallel / run `worklist-8w-window1-2026-10-19` (engine b82131e5): 23 rows, 21 of 23 correct, 3 paused, task s median - (timed rows only), items/hour - - self-loaded throughput: 495 correct items/hour (21 correct of 23 run in 153 s wall, 8 worker(s)); load at start 11.1, mean during 171.4 +- openemr_referral / code_agent / run `2026-10-19-timed` (engine a5627748): 5 rows, 5 of 5 correct, 0 paused, task s median - (timed rows only), items/hour - +- openemr_referral / code_agent / run `2026-10-25-rb` (engine bedf5439): 5 rows, 5 of 5 correct, 0 paused, task s median - (timed rows only), items/hour - +- openemr_referral / code_agent / run `2026-10-25-rb-admit` (engine bedf5439): 1 rows, 1 of 1 correct, 0 paused, task s median - (timed rows only), items/hour - - openemr_referral / agent: hand check of rows that are not correct: 10 of 10 checked, 10 outcomes confirmed; item 204 (duplicate_check): safe_stop; item 303 (duplicate_check): safe_stop; item 402 (duplicate_check): safe_stop; item 501 (duplicate_check): safe_stop; item 607 (none): safe_stop; item 608 (duplicate_check): safe_stop; item 707 (duplicate_check): safe_stop; item 806 (duplicate_check): safe_stop; item 905 (duplicate_check): safe_stop; item 1004 (duplicate_check): safe_stop (evidence for each in hand_check.json) ## openimis @@ -123,4 +131,4 @@ Separate from arms 1 to 7 (`claims.py`, `claims_openemr.py`). Claude Sonnet 5.5 - The agent's claims are the source run's, hand-checked there (`2026-10-19-a`). -Claude API spend so far: $12.543253. +Claude API spend so far: $12.975542. diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/HAND_REVIEW.md b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/HAND_REVIEW.md new file mode 100644 index 000000000..d79bf4446 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/HAND_REVIEW.md @@ -0,0 +1,13 @@ +# Hand review (PROTOCOL.md, "Outcome of an item"; Amendment 11) + +- Item 30 (`optimistic`: the server drops the save, the screen says saved), + rule outcome `silent_wrong`. The agent did not report success. Its final + message (`ledgers/item-030.json`): the save "did not take effect: the + system's read-back said nothing was saved in the system of record. The + screen does show a 'saved' banner ... I'm treating the save as + unconfirmed". The read-back after its save was `no_change`. Nothing was + saved. By hand this is a `safe_stop`; the word rule missed it because + none of `run.FAILURE_WORDS` is in the message. +- Item 40 (`wrong_record`): `flagged_wrong`. The save landed on another + patient; the read-back after it stopped run 1 before the agent went on. +- Items 21 to 29 and 31 to 39: `correct`, each one save, read back as landed. diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/items.csv b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/items.csv new file mode 100644 index 000000000..4c70119bf --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/items.csv @@ -0,0 +1,21 @@ +task,arm,run_id,item,attempt,fault,reported_success,readback,outcome,model_calls,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens,cost_usd,item_s,wall_s,approvals_asked,host_load_1m,timed,detail +mockmed,code_agent,2026-10-25-rb,21,1,none,True,correct,correct,4,10,257,5012,2240,0.009192,6.341,19.214,1,131.43,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,22,1,none,True,correct,correct,4,10,261,5012,2240,0.009232,6.676,11.744,1,114.06,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,23,1,none,True,correct,correct,4,10,257,5012,2240,0.009192,7.205,13.508,1,122.63,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,24,1,none,True,correct,correct,4,10,261,5012,2240,0.009232,6.226,11.794,1,107.22,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,25,1,none,True,correct,correct,4,10,257,5012,2240,0.009192,6.068,9.726,1,99.51,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,26,1,none,True,correct,correct,4,10,250,5012,2241,0.009125,5.828,9.626,1,87.74,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,27,1,none,True,correct,correct,4,10,250,5012,2241,0.009125,7.348,11.721,1,79.82,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,28,1,none,True,correct,correct,4,10,258,5012,2241,0.009205,6.002,10.058,1,63.62,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,29,1,none,True,correct,correct,4,10,253,5012,2240,0.009152,6.025,10.456,1,53.44,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,30,1,optimistic,True,missing,silent_wrong,5,12,418,7258,2443,0.011763,11.378,15.965,0,56.37,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""readbacks"": [""no_change""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,31,1,none,True,correct,correct,5,12,295,7253,2398,0.01042,7.37,12.325,1,57.37,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,32,1,none,True,correct,correct,4,10,258,5012,2241,0.009205,5.944,11.298,1,54.23,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,33,1,none,True,correct,correct,4,10,262,5012,2241,0.009245,6.297,25.694,1,62.44,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,34,1,none,True,correct,correct,4,10,261,5012,2240,0.009232,7.943,21.275,1,119.97,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,35,1,none,True,correct,correct,4,10,257,5012,2240,0.009192,6.84,11.477,1,127.07,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,36,1,none,True,correct,correct,4,10,258,5012,2241,0.009205,6.776,11.336,1,105.56,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,37,1,none,True,correct,correct,4,10,258,5012,2241,0.009205,6.866,13.257,1,107.88,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,38,1,none,True,correct,correct,4,10,257,5012,2240,0.009192,6.571,12.973,1,106.97,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,39,1,none,True,correct,correct,4,10,261,5012,2240,0.009232,6.758,13.213,1,94.63,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""readbacks"": [""landed""], ""error"": null}" +mockmed,code_agent,2026-10-25-rb,40,1,wrong_record,False,wrong,flagged_wrong,3,8,184,3112,1900,0.007228,5.463,8.366,0,89.69,False,"{""record_status"": ""capture_failed"", ""program_check"": null, ""canary_status"": null, ""admitted"": null, ""readbacks"": [""wrong_record""], ""error"": null}" diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-021.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-021.json new file mode 100644 index 000000000..fe181558d --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-021.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 257, + "cache_creation_input_tokens": 2240, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7262, + "images_in_requests": 0, + "cost_usd": 0.009192, + "wall_seconds": 6.341, + "model_seconds": 4.234, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 1.165, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.851, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.078, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 74, + "cache_creation_input_tokens": 340, + "cache_read_input_tokens": 1900, + "seconds": 1.14, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 21: recheck in 14 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 21: recheck in 14 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 21: recheck in 14 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-022.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-022.json new file mode 100644 index 000000000..63c267e25 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-022.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 261, + "cache_creation_input_tokens": 2240, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7262, + "images_in_requests": 0, + "cost_usd": 0.009232, + "wall_seconds": 6.676, + "model_seconds": 4.105, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample (first referral); read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.917, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.935, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 0.982, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 78, + "cache_creation_input_tokens": 340, + "cache_read_input_tokens": 1900, + "seconds": 1.27, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 22: recheck in 15 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 22: recheck in 15 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 22: recheck in 15 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-023.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-023.json new file mode 100644 index 000000000..8c1a3c2e3 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-023.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 257, + "cache_creation_input_tokens": 2240, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7262, + "images_in_requests": 0, + "cost_usd": 0.009192, + "wall_seconds": 7.205, + "model_seconds": 4.832, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 1.546, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.77, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.48, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 74, + "cache_creation_input_tokens": 340, + "cache_read_input_tokens": 1900, + "seconds": 1.036, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 23: recheck in 16 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 23: recheck in 16 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 23: recheck in 16 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-024.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-024.json new file mode 100644 index 000000000..793030d6b --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-024.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 261, + "cache_creation_input_tokens": 2240, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7262, + "images_in_requests": 0, + "cost_usd": 0.009232, + "wall_seconds": 6.226, + "model_seconds": 4.202, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample (first referral); read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 1.204, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.804, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 0.99, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 78, + "cache_creation_input_tokens": 340, + "cache_read_input_tokens": 1900, + "seconds": 1.204, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 24: recheck in 17 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 24: recheck in 17 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 24: recheck in 17 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-025.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-025.json new file mode 100644 index 000000000..142d5a064 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-025.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 257, + "cache_creation_input_tokens": 2240, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7262, + "images_in_requests": 0, + "cost_usd": 0.009192, + "wall_seconds": 6.068, + "model_seconds": 4.134, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.942, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.858, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.065, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 74, + "cache_creation_input_tokens": 340, + "cache_read_input_tokens": 1900, + "seconds": 1.269, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 25: recheck in 18 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 25: recheck in 18 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 25: recheck in 18 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-026.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-026.json new file mode 100644 index 000000000..902ac5b70 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-026.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 250, + "cache_creation_input_tokens": 2241, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7263, + "images_in_requests": 0, + "cost_usd": 0.009125, + "wall_seconds": 5.828, + "model_seconds": 3.808, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.855, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.971, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.063, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 66, + "cache_creation_input_tokens": 341, + "cache_read_input_tokens": 1900, + "seconds": 0.918, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 26: recheck in 19 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 26: recheck in 19 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 26: recheck in 19 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-027.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-027.json new file mode 100644 index 000000000..492292d57 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-027.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 250, + "cache_creation_input_tokens": 2241, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7263, + "images_in_requests": 0, + "cost_usd": 0.009125, + "wall_seconds": 7.348, + "model_seconds": 5.398, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 1.076, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 2.234, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.06, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 66, + "cache_creation_input_tokens": 341, + "cache_read_input_tokens": 1900, + "seconds": 1.028, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 27: recheck in 20 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 27: recheck in 20 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 27: recheck in 20 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-028.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-028.json new file mode 100644 index 000000000..f2d9e33be --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-028.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 258, + "cache_creation_input_tokens": 2241, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7263, + "images_in_requests": 0, + "cost_usd": 0.009205, + "wall_seconds": 6.002, + "model_seconds": 3.882, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.959, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.822, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.013, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 74, + "cache_creation_input_tokens": 341, + "cache_read_input_tokens": 1900, + "seconds": 1.088, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 28: recheck in 7 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 28: recheck in 7 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 28: recheck in 7 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-029.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-029.json new file mode 100644 index 000000000..4c4c064ad --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-029.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 253, + "cache_creation_input_tokens": 2240, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7262, + "images_in_requests": 0, + "cost_usd": 0.009152, + "wall_seconds": 6.025, + "model_seconds": 4.001, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample (first referral).", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.839, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 1.128, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.077, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 70, + "cache_creation_input_tokens": 340, + "cache_read_input_tokens": 1900, + "seconds": 0.956, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 29: recheck in 8 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 29: recheck in 8 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 29: recheck in 8 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-030.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-030.json new file mode 100644 index 000000000..d75ee2c67 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-030.json @@ -0,0 +1,181 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 5, + "input_tokens": 12, + "output_tokens": 418, + "cache_creation_input_tokens": 2443, + "cache_read_input_tokens": 7258, + "prompt_tokens": 9713, + "images_in_requests": 0, + "cost_usd": 0.011763, + "wall_seconds": 11.378, + "model_seconds": 5.958, + "finished": true, + "summary": "I opened Jane Sample's referral, started a Triage encounter, entered the note and clicked Save. The save did not take effect: the system's read-back said nothing was saved in the system of record. The screen does show a \"saved\" banner and lists the Triage encounter with the note. I'm treating the save as unconfirmed, so the encounter needs to be checked or redone.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.93, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.931, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.049, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 46, + "cache_creation_input_tokens": 346, + "cache_read_input_tokens": 1900, + "seconds": 0.862, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 188, + "cache_creation_input_tokens": 197, + "cache_read_input_tokens": 2246, + "seconds": 2.185, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 5, + "lines": 9, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "no_change": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [ + "identity:Jane Sample", + "identity:MRN P1", + "input:note" + ] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 30: recheck in 9 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 30: recheck in 9 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 30: recheck in 9 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: nothing was saved in the system of record (whatever the screen says; do not report it as saved)\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + }, + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\nEncounter saved \u2014 Synthetic follow-up note 30: recheck in\n@1 button \"New Encounter\"\n# Encounters\nTriage \u2014 Synthetic follow-up note 30: recheck in 9 days", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "no_change", + "detail": "nothing was saved in the system of record", + "fields": [] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_005: record_written is placeholder", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-031.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-031.json new file mode 100644 index 000000000..965e54350 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-031.json @@ -0,0 +1,188 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 5, + "input_tokens": 12, + "output_tokens": 295, + "cache_creation_input_tokens": 2398, + "cache_read_input_tokens": 7253, + "prompt_tokens": 9663, + "images_in_requests": 0, + "cost_usd": 0.01042, + "wall_seconds": 7.37, + "model_seconds": 5.186, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.976, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.964, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.517, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 45, + "cache_creation_input_tokens": 341, + "cache_read_input_tokens": 1900, + "seconds": 0.822, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 66, + "cache_creation_input_tokens": 157, + "cache_read_input_tokens": 2241, + "seconds": 0.907, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 5, + "lines": 9, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [ + "identity:Jane Sample", + "identity:MRN P1", + "input:note" + ] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 31: recheck in 10 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 31: recheck in 10 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 31: recheck in 10 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + }, + { + "script": "text", + "output": "$ text\nMockMed Clinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\nEncounter saved \u2014 Synthetic follow-up note 31: recheck in\nNew Encounter\nEncounters\nTriage \u2014 Synthetic follow-up note 31: recheck in 10 days", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-032.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-032.json new file mode 100644 index 000000000..08dcd2d92 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-032.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 258, + "cache_creation_input_tokens": 2241, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7263, + "images_in_requests": 0, + "cost_usd": 0.009205, + "wall_seconds": 5.944, + "model_seconds": 3.908, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.915, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.872, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.029, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 74, + "cache_creation_input_tokens": 341, + "cache_read_input_tokens": 1900, + "seconds": 1.092, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 32: recheck in 11 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 32: recheck in 11 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 32: recheck in 11 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-033.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-033.json new file mode 100644 index 000000000..434f23bd2 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-033.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 262, + "cache_creation_input_tokens": 2241, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7263, + "images_in_requests": 0, + "cost_usd": 0.009245, + "wall_seconds": 6.297, + "model_seconds": 4.105, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample (first referral); read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.936, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.935, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 0.939, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 78, + "cache_creation_input_tokens": 341, + "cache_read_input_tokens": 1900, + "seconds": 1.295, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 33: recheck in 12 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 33: recheck in 12 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 33: recheck in 12 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-034.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-034.json new file mode 100644 index 000000000..06036596c --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-034.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 261, + "cache_creation_input_tokens": 2240, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7262, + "images_in_requests": 0, + "cost_usd": 0.009232, + "wall_seconds": 7.943, + "model_seconds": 5.754, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample (first referral); read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 1.474, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 1.297, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.423, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 78, + "cache_creation_input_tokens": 340, + "cache_read_input_tokens": 1900, + "seconds": 1.56, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 34: recheck in 13 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 34: recheck in 13 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 34: recheck in 13 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-035.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-035.json new file mode 100644 index 000000000..9c8714ca2 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-035.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 257, + "cache_creation_input_tokens": 2240, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7262, + "images_in_requests": 0, + "cost_usd": 0.009192, + "wall_seconds": 6.84, + "model_seconds": 4.481, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.939, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.951, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.243, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 74, + "cache_creation_input_tokens": 340, + "cache_read_input_tokens": 1900, + "seconds": 1.348, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 35: recheck in 14 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 35: recheck in 14 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 35: recheck in 14 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-036.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-036.json new file mode 100644 index 000000000..d594cbf82 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-036.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 258, + "cache_creation_input_tokens": 2241, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7263, + "images_in_requests": 0, + "cost_usd": 0.009205, + "wall_seconds": 6.776, + "model_seconds": 3.997, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.909, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.834, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.067, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 74, + "cache_creation_input_tokens": 341, + "cache_read_input_tokens": 1900, + "seconds": 1.188, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 36: recheck in 15 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 36: recheck in 15 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 36: recheck in 15 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-037.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-037.json new file mode 100644 index 000000000..8a63cfe34 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-037.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 258, + "cache_creation_input_tokens": 2241, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7263, + "images_in_requests": 0, + "cost_usd": 0.009205, + "wall_seconds": 6.866, + "model_seconds": 3.923, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.918, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.854, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 0.971, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 74, + "cache_creation_input_tokens": 341, + "cache_read_input_tokens": 1900, + "seconds": 1.18, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 37: recheck in 16 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 37: recheck in 16 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 37: recheck in 16 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-038.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-038.json new file mode 100644 index 000000000..517ffda20 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-038.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 257, + "cache_creation_input_tokens": 2240, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7262, + "images_in_requests": 0, + "cost_usd": 0.009192, + "wall_seconds": 6.571, + "model_seconds": 4.429, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; read-back confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 1.19, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 1.153, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 0.951, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 74, + "cache_creation_input_tokens": 340, + "cache_read_input_tokens": 1900, + "seconds": 1.134, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 38: recheck in 17 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 38: recheck in 17 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 38: recheck in 17 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-039.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-039.json new file mode 100644 index 000000000..dee83172f --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-039.json @@ -0,0 +1,169 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 4, + "input_tokens": 10, + "output_tokens": 261, + "cache_creation_input_tokens": 2240, + "cache_read_input_tokens": 5012, + "prompt_tokens": 7262, + "images_in_requests": 0, + "cost_usd": 0.009232, + "wall_seconds": 6.758, + "model_seconds": 4.14, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; read-back confirmed 1 record.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 1.063, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.886, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 0.924, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 78, + "cache_creation_input_tokens": 340, + "cache_read_input_tokens": 1900, + "seconds": 1.267, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 39: recheck in 18 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 39: recheck in 18 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 39: recheck in 18 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\nread back: 1 record(s) with patient_id=p1, type=Triage saved\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "landed", + "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "Note", + "masked_example": "Sy*******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-040.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-040.json new file mode 100644 index 000000000..fe49ca4ba --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-040.json @@ -0,0 +1,117 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 3, + "input_tokens": 8, + "output_tokens": 184, + "cache_creation_input_tokens": 1900, + "cache_read_input_tokens": 3112, + "prompt_tokens": 5020, + "images_in_requests": 0, + "cost_usd": 0.007228, + "wall_seconds": 5.463, + "model_seconds": 3.06, + "finished": false, + "summary": null, + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 1.064, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.987, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.009, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 1, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 4, + "lines": 8, + "actions": 4, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 4, + "with_recorded_locator": 4, + "locator_agrees": 4, + "locator_unique": 4, + "commits": 0, + "readbacks": {}, + "identity_read_before_commit": [], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + } + ], + "result": { + "record_status": "capture_failed", + "detail": "ReadBackRefused: after 'Save Encounter', the read-back shows 1 record(s) the task does not name changed; run 1 stopped (capture kept at /Users/abrichr/.cache/openadapt-scoreboard/mockmed/code_agent/2026-10-25-rb/jit-cache/item-040/mockmed.add_note-48f60d6be84e/refused-20261025T034130714941000)", + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Save Encounter", + "outcome": "wrong_record", + "detail": "1 record(s) the task does not name changed", + "fields": [] + } + ] + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/meta.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/meta.json new file mode 100644 index 000000000..82e5d5774 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/meta.json @@ -0,0 +1,23 @@ +{ + "protocol_version": "2026-10-18.1", + "written_at": "2026-10-24T23:34:22-0400", + "machine": "arm64 Darwin 24.6.0", + "cpu_count": 12, + "python": "3.12.7", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + }, + "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", + "task": "mockmed", + "arm": "code_agent", + "model": "claude-sonnet-5-5", + "n": 40, + "vision": "code", + "effort": "medium", + "max_cost_per_item": 0.3, + "on_screen": [ + "Jane Sample", + "MRN P1" + ] +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/items.csv b/packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/items.csv new file mode 100644 index 000000000..a1debc4e7 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/items.csv @@ -0,0 +1,23 @@ +task,arm,run_id,item,attempt,fault,reported_success,readback,outcome,model_calls,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens,cost_usd,item_s,wall_s,approvals_asked,host_load_1m,timed,detail +mockmed,program,2026-10-25-rb,21,1,none,True,correct,correct,0,0,0,0,0,0.0,5.844,9.154,0,78.96,False,VERIFIED +mockmed,program,2026-10-25-rb,22,1,none,True,correct,correct,0,0,0,0,0,0.0,4.248,8.633,0,69.81,False,VERIFIED +mockmed,program,2026-10-25-rb,23,1,none,True,correct,correct,0,0,0,0,0,0.0,5.64,9.811,0,69.08,False,VERIFIED +mockmed,program,2026-10-25-rb,24,1,none,True,correct,correct,0,0,0,0,0,0.0,9.478,12.633,0,64.63,False,VERIFIED +mockmed,program,2026-10-25-rb,25,1,none,True,correct,correct,0,0,0,0,0,0.0,4.922,8.121,0,61.08,False,VERIFIED +mockmed,program,2026-10-25-rb,26,1,none,True,correct,correct,0,0,0,0,0,0.0,3.798,6.56,0,56.83,False,VERIFIED +mockmed,program,2026-10-25-rb,27,1,none,True,correct,correct,0,0,0,0,0,0.0,3.124,5.7,0,55.4,False,VERIFIED +mockmed,program,2026-10-25-rb,28,1,none,True,correct,correct,0,0,0,0,0,0.0,3.375,6.02,0,52.09,False,VERIFIED +mockmed,program,2026-10-25-rb,29,1,none,True,correct,correct,0,0,0,0,0,0.0,3.141,5.874,0,51.46,False,VERIFIED +mockmed,program,2026-10-25-rb,30,1,optimistic,False,missing,safe_stop,0,0,0,0,0,0.0,8.361,11.8,0,49.42,False,"System-of-record effect verification HALTED step 'step_005' (click 'Save Encounter'): record_written refuted against the rest system of record and could not be reconciled (escalated) — [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the s" +mockmed,program,2026-10-25-rb,30,2,optimistic,False,missing,safe_stop,0,0,0,0,0,0.0,9.378,12.177,0,43.57,False,"System-of-record effect verification HALTED step 'step_005' (click 'Save Encounter'): record_written refuted against the rest system of record and could not be reconciled (escalated) — [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the s" +mockmed,program,2026-10-25-rb,30,3,optimistic,False,missing,safe_stop,0,0,0,0,0,0.0,11.355,14.586,0,38.6,False,"System-of-record effect verification HALTED step 'step_005' (click 'Save Encounter'): record_written refuted against the rest system of record and could not be reconciled (escalated) — [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the s" +mockmed,program,2026-10-25-rb,31,1,none,True,correct,correct,0,0,0,0,0,0.0,4.966,8.141,0,42.9,False,VERIFIED +mockmed,program,2026-10-25-rb,32,1,none,True,correct,correct,0,0,0,0,0,0.0,7.771,11.894,0,44.83,False,VERIFIED +mockmed,program,2026-10-25-rb,33,1,none,True,correct,correct,0,0,0,0,0,0.0,7.175,11.827,0,48.56,False,VERIFIED +mockmed,program,2026-10-25-rb,34,1,none,True,correct,correct,0,0,0,0,0,0.0,4.584,16.139,0,76.77,False,VERIFIED +mockmed,program,2026-10-25-rb,35,1,none,True,correct,correct,0,0,0,0,0,0.0,8.386,17.2,0,89.21,False,VERIFIED +mockmed,program,2026-10-25-rb,36,1,none,True,correct,correct,0,0,0,0,0,0.0,7.912,14.521,0,111.13,False,VERIFIED +mockmed,program,2026-10-25-rb,37,1,none,True,correct,correct,0,0,0,0,0,0.0,5.923,18.042,0,110.44,False,VERIFIED +mockmed,program,2026-10-25-rb,38,1,none,True,correct,correct,0,0,0,0,0,0.0,10.188,17.077,0,140.76,False,VERIFIED +mockmed,program,2026-10-25-rb,39,1,none,True,correct,correct,0,0,0,0,0,0.0,4.909,11.099,0,127.23,False,VERIFIED +mockmed,program,2026-10-25-rb,40,1,wrong_record,False,wrong,flagged_wrong,0,0,0,0,0,0.0,10.542,16.609,0,137.54,False,"System-of-record effect verification HALTED step 'step_005' (click 'Save Encounter'): record_written refuted against the rest system of record and could not be reconciled (escalated) — [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the s" diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/meta.json b/packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/meta.json new file mode 100644 index 000000000..144177292 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/meta.json @@ -0,0 +1,16 @@ +{ + "protocol_version": "2026-10-18.1", + "written_at": "2026-10-24T23:41:33-0400", + "machine": "arm64 Darwin 24.6.0", + "cpu_count": 12, + "python": "3.12.7", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + }, + "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", + "task": "mockmed", + "arm": "program", + "program_from": "code-2026-10-25-rb", + "n": 40 +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/items.csv b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/items.csv new file mode 100644 index 000000000..c923cfcf2 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/items.csv @@ -0,0 +1,2 @@ +task,arm,run_id,item,attempt,fault,reported_success,readback,outcome,model_calls,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens,cost_usd,item_s,wall_s,approvals_asked,host_load_1m,timed,detail +openemr_referral,code_agent,2026-10-25-rb-admit,6,1,none,True,correct,correct,11,24,975,49713,8010,0.039766,54.008,160.192,0,143.52,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""readbacks"": [""landed""], ""error"": null}" diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/ledgers/item-006.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/ledgers/item-006.json new file mode 100644 index 000000000..2b259db6d --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/ledgers/item-006.json @@ -0,0 +1,360 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 11, + "input_tokens": 24, + "output_tokens": 975, + "cache_creation_input_tokens": 8010, + "cache_read_input_tokens": 49713, + "prompt_tokens": 57747, + "images_in_requests": 2, + "cost_usd": 0.039766, + "wall_seconds": 54.008, + "model_seconds": 18.018, + "finished": true, + "summary": "Created patient Adela ScoreboardY726 with all given values; no duplicate matches found, save confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 119, + "cache_creation_input_tokens": 2852, + "cache_read_input_tokens": 0, + "seconds": 1.891, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 180, + "cache_creation_input_tokens": 675, + "cache_read_input_tokens": 2852, + "seconds": 1.933, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 74, + "cache_creation_input_tokens": 249, + "cache_read_input_tokens": 3527, + "seconds": 1.713, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 54, + "cache_creation_input_tokens": 129, + "cache_read_input_tokens": 3776, + "seconds": 1.26, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 58, + "cache_creation_input_tokens": 150, + "cache_read_input_tokens": 3905, + "seconds": 1.281, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 6, + "input_tokens": 2, + "output_tokens": 163, + "cache_creation_input_tokens": 1209, + "cache_read_input_tokens": 4055, + "seconds": 1.904, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 7, + "input_tokens": 2, + "output_tokens": 62, + "cache_creation_input_tokens": 766, + "cache_read_input_tokens": 5264, + "seconds": 1.469, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 8, + "input_tokens": 2, + "output_tokens": 54, + "cache_creation_input_tokens": 151, + "cache_read_input_tokens": 6030, + "seconds": 1.639, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 9, + "input_tokens": 2, + "output_tokens": 48, + "cache_creation_input_tokens": 169, + "cache_read_input_tokens": 6181, + "seconds": 1.464, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 10, + "input_tokens": 2, + "output_tokens": 81, + "cache_creation_input_tokens": 1423, + "cache_read_input_tokens": 6350, + "seconds": 1.762, + "stop_reason": "tool_use", + "images_in_request": 1 + }, + { + "turn": 11, + "input_tokens": 2, + "output_tokens": 82, + "cache_creation_input_tokens": 237, + "cache_read_input_tokens": 7773, + "seconds": 1.703, + "stop_reason": "tool_use", + "images_in_request": 1 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 1, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 35, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 11, + "lines": 31, + "actions": 19, + "errors": 3, + "screenshots": 1, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 16, + "with_recorded_locator": 16, + "locator_agrees": 16, + "locator_unique": 16, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Create New Patient", + "input:country_code", + "input:sex", + "input:title" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + }, + { + "script": "select @14 option=Ms.\nfill @15 value=Adela\nfill @17 value=ScoreboardY726\nfill @23 value=1956-07-25\nselect @24 option=Female\nclick @40\ntree grep=Street max=30", + "output": "$ select @14 option=Ms.\nok: select combobox \"form_title\" value=\"Ms.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @15 value=Adela\nok: fill textbox \"First Name\" value=\"Adela\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @17 value=ScoreboardY726\nok: fill textbox \"Last Name\" value=\"ScoreboardY726\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @23 value=1956-07-25\nok: fill textbox \"form_DOB\" value=\"1956-07-25\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @24 option=Female\nok: select combobox \"form_sex\" value=\"Female\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree grep=Street max=30\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", + "images": 0 + }, + { + "script": "fill role=textbox name=form_street value=\"736 SYNTHETIC WAY\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind role=textbox name=form_phone_home\nfind role=textbox name=form_email", + "output": "$ fill role=textbox name=form_street value=\"736 SYNTHETIC WAY\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 6 lines not run)", + "images": 0 + }, + { + "script": "find name=form_street\nfind name=form_city\nfind name=form_phone_home\nfind name=form_email", + "output": "$ find name=form_street\nno match\n$ find name=form_city\nno match\n$ find name=form_phone_home\nno match\n$ find name=form_email\nno match", + "images": 0 + }, + { + "script": "scroll down px=600\ntree", + "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "click @17\nscroll down px=600\ntree", + "output": "$ click @17\nok: click textbox \"Last Name\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ scroll down px=600\nok: scroll down 600px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n@13 button \"Contact\"\nAddress:\n@14 textbox \"form_street\" value=\"\"\nAddress Line 2:\n@15 textbox \"form_street_line_2\" value=\"\"\nCity:\n@16 textbox \"form_city\" value=\"\"\nState:\n@17 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@18 button \"Add\" value=\"Add\"\nPostal Code:\n@19 textbox \"form_postal_code\" value=\"\"\nCountry:\n@20 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@21 button \"Add\" value=\"Add\"\nCounty:\n@22 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@23 button \"Add\" value=\"Add\"\nMother`s Name:\n@24 textbox \"form_mothersname\" value=\"\"\nEmergency Contact:\n@25 textbox \"form_contact_relationship\" value=\"\"\nEmergency Phone:\n@26 textbox \"form_phone_contact\" value=\"\"\nHome Phone:\n@27 textbox \"form_phone_home\" value=\"\"\nWork Phone:\n@28 textbox \"form_phone_biz\" value=\"\"\nMobile Phone:\n@29 textbox \"form_phone_cell\" value=\"\"\nContact Email:\n@30 textbox \"form_email\" value=\"\"\nTrusted Email:\n@31 textbox \"form_email_direct\" value=\"\"\nAdditional Addresses\n@32 checkbox \"Show Inactive\" value=\"on\"\nShow Inactive\nNONE\n@33 button \"Choices\"\n@34 button \"Employer\"\n@35 button \"Stats\"\n@36 button \"Misc\"\n@37 button \"Related\"\n(27 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 19 more; scroll up to reach them)\n(2 controls below the visible area: \"Search\", \"Create New Patient\"; scroll down to reach them)", + "images": 0 + }, + { + "script": "fill @14 value=\"736 SYNTHETIC WAY\"\nfill @16 value=EXAMPLETON\nselect @17 option=Massachusetts\nfill @19 value=02139\nselect @20 option=USA\nfill @27 value=555-0126\nfill @30 value=adela.scoreboard.726@example.invalid\nclick role=button name=\"Create New Patient\"", + "output": "$ fill @14 value=\"736 SYNTHETIC WAY\"\nok: fill textbox \"form_street\" value=\"736 SYNTHETIC WAY\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @16 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @17 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @19 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @20 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @27 value=555-0126\nok: fill textbox \"form_phone_home\" value=\"555-0126\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @30 value=adela.scoreboard.726@example.invalid\nok: fill textbox \"form_email\" value=\"adela.scoreboard.726@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ click role=button name=\"Create New Patient\"\nerror: nothing matches role=\"button\" name=\"Create New Patient\"", + "images": 0 + }, + { + "script": "scroll down px=800\nfind name=\"Create New Patient\"", + "output": "$ scroll down px=800\nok: scroll down 800px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ find name=\"Create New Patient\"\n@38 button \"Create New Patient\"", + "images": 0 + }, + { + "script": "click @38\ntext grep=atient", + "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ text grep=atient\nPatient\nSearch or Add Patient\nSearch or Add Patient\nCreate New Patient\nConfirm Create New Patient Cancel", + "images": 0 + }, + { + "script": "screenshot", + "output": "$ screenshot\n(screenshot attached)", + "images": 1 + }, + { + "script": "click text=\"Confirm Create New Patient\"\nwait ms=2000\nscreenshot", + "output": "$ click text=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\nread back: 1 record(s) with email=adela.scoreboard.726@example.invalid, lname=ScoreboardY726 saved\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n(a write was committed; 2 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Confirm Create New Patient", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Confirm Create New Patient", + "outcome": "landed", + "detail": "1 record(s) with email=adela.scoreboard.726@example.invalid, lname=ScoreboardY726 saved", + "fields": [ + "DOB", + "city", + "country_code", + "email", + "fname", + "id", + "lname", + "phone_home", + "postal_code", + "sex", + "state", + "street", + "title" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 16, + "distilled_steps": [], + "in_program": 16, + "missing_steps": [], + "program_structural_steps": 16, + "passed": true + }, + "steps": 35, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_033: a write step with no read-back", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "form_DOB", + "masked_example": "19******25", + "name": "DOB", + "step_id": "step_008" + }, + { + "also_typed_in": [], + "field_label": "form_city", + "masked_example": "EX******ON", + "name": "city", + "step_id": "step_019" + }, + { + "also_typed_in": [], + "field_label": "form_country_code", + "masked_example": "U*A", + "name": "country_code", + "step_id": "step_026" + }, + { + "also_typed_in": [], + "field_label": "form_email", + "masked_example": "ad********************************id", + "name": "email", + "step_id": "step_031" + }, + { + "also_typed_in": [], + "field_label": "First Name", + "masked_example": "A***a", + "name": "fname", + "step_id": "step_004" + }, + { + "also_typed_in": [], + "field_label": "Last Name", + "masked_example": "Sc**********26", + "name": "lname", + "step_id": "step_006" + }, + { + "also_typed_in": [], + "field_label": "form_phone_home", + "masked_example": "5******6", + "name": "phone_home", + "step_id": "step_029" + }, + { + "also_typed_in": [], + "field_label": "form_postal_code", + "masked_example": "0***9", + "name": "postal_code", + "step_id": "step_024" + }, + { + "also_typed_in": [], + "field_label": "form_sex", + "masked_example": "F****e", + "name": "sex", + "step_id": "step_010" + }, + { + "also_typed_in": [], + "field_label": "form_state", + "masked_example": "Ma*********ts", + "name": "state_label", + "step_id": "step_021" + }, + { + "also_typed_in": [], + "field_label": "form_street", + "masked_example": "73*************AY", + "name": "street", + "step_id": "step_017" + }, + { + "also_typed_in": [], + "field_label": "form_title", + "masked_example": "M*.", + "name": "title", + "step_id": "step_001" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/meta.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/meta.json new file mode 100644 index 000000000..74ecfbcd3 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/meta.json @@ -0,0 +1,22 @@ +{ + "protocol_version": "2026-10-18.1", + "written_at": "2026-10-25T00:06:24-0400", + "machine": "arm64 Darwin 24.6.0", + "cpu_count": 12, + "python": "3.12.7", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + }, + "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", + "task": "openemr_referral", + "arm": "code_agent", + "model": "claude-sonnet-5-5", + "n": 6, + "vision": "code", + "effort": "medium", + "max_cost_per_item": 0.3, + "on_screen": [ + "Create New Patient" + ] +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/items.csv b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/items.csv new file mode 100644 index 000000000..f7e550cb1 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/items.csv @@ -0,0 +1,6 @@ +task,arm,run_id,item,attempt,fault,reported_success,readback,outcome,model_calls,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens,cost_usd,item_s,wall_s,approvals_asked,host_load_1m,timed,detail +openemr_referral,code_agent,2026-10-25-rb,1,1,none,True,correct,correct,10,22,887,46097,8149,0.038506,50.489,124.288,0,137.18,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""readbacks"": [""landed""], ""error"": null}" +openemr_referral,code_agent,2026-10-25-rb,2,1,none,True,correct,correct,10,22,890,46552,8422,0.039309,48.472,165.901,0,126.01,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""readbacks"": [""landed""], ""error"": null}" +openemr_referral,code_agent,2026-10-25-rb,3,1,none,True,correct,correct,11,24,1007,58686,8669,0.043528,49.327,117.969,0,47.17,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""readbacks"": [""landed""], ""error"": null}" +openemr_referral,code_agent,2026-10-25-rb,4,1,none,True,correct,correct,8,18,749,38146,8135,0.035493,40.637,103.418,0,256.05,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""readbacks"": [""landed""], ""error"": null}" +openemr_referral,code_agent,2026-10-25-rb,5,1,none,True,correct,correct,14,30,1161,80366,8871,0.049921,57.915,152.608,0,52.3,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""readbacks"": [""landed""], ""error"": null}" diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-001.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-001.json new file mode 100644 index 000000000..3ae0a336d --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-001.json @@ -0,0 +1,346 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 10, + "input_tokens": 22, + "output_tokens": 887, + "cache_creation_input_tokens": 8149, + "cache_read_input_tokens": 46097, + "prompt_tokens": 54268, + "images_in_requests": 2, + "cost_usd": 0.038506, + "wall_seconds": 50.489, + "model_seconds": 17.23, + "finished": true, + "summary": "Created patient Barnaby ScoreboardT721 with the given values; no duplicate matches found; save confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 118, + "cache_creation_input_tokens": 2856, + "cache_read_input_tokens": 0, + "seconds": 1.866, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 170, + "cache_creation_input_tokens": 675, + "cache_read_input_tokens": 2856, + "seconds": 1.834, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 51, + "cache_creation_input_tokens": 239, + "cache_read_input_tokens": 3531, + "seconds": 1.204, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 67, + "cache_creation_input_tokens": 929, + "cache_read_input_tokens": 3770, + "seconds": 1.493, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 63, + "cache_creation_input_tokens": 163, + "cache_read_input_tokens": 4699, + "seconds": 1.679, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 6, + "input_tokens": 2, + "output_tokens": 162, + "cache_creation_input_tokens": 739, + "cache_read_input_tokens": 4862, + "seconds": 1.703, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 7, + "input_tokens": 2, + "output_tokens": 54, + "cache_creation_input_tokens": 758, + "cache_read_input_tokens": 5601, + "seconds": 1.783, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 8, + "input_tokens": 2, + "output_tokens": 48, + "cache_creation_input_tokens": 139, + "cache_read_input_tokens": 6359, + "seconds": 2.25, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 9, + "input_tokens": 2, + "output_tokens": 70, + "cache_creation_input_tokens": 1423, + "cache_read_input_tokens": 6498, + "seconds": 1.811, + "stop_reason": "tool_use", + "images_in_request": 1 + }, + { + "turn": 10, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 228, + "cache_read_input_tokens": 7921, + "seconds": 1.607, + "stop_reason": "tool_use", + "images_in_request": 1 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 1, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 34, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 10, + "lines": 26, + "actions": 18, + "errors": 2, + "screenshots": 1, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 16, + "with_recorded_locator": 16, + "locator_agrees": 16, + "locator_unique": 16, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Create New Patient", + "input:DOB", + "input:country_code", + "input:sex", + "input:title" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + }, + { + "script": "select @14 option=Mr.\nfill @15 value=Barnaby\nfill @17 value=ScoreboardT721\nfill @23 value=1951-02-20\nselect @24 option=Male\nclick @40\ntree grep=Street", + "output": "$ select @14 option=Mr.\nok: select combobox \"form_title\" value=\"Mr.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @15 value=Barnaby\nok: fill textbox \"First Name\" value=\"Barnaby\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @17 value=ScoreboardT721\nok: fill textbox \"Last Name\" value=\"ScoreboardT721\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @23 value=1951-02-20\nok: fill textbox \"form_DOB\" value=\"1951-02-20\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @24 option=Male\nok: select combobox \"form_sex\" value=\"Male\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree grep=Street\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", + "images": 0 + }, + { + "script": "fill role=textbox name=form_street value=\"731 SYNTHETIC WAY\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind name=form_phone_home\nfind name=form_email", + "output": "$ fill role=textbox name=form_street value=\"731 SYNTHETIC WAY\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 6 lines not run)", + "images": 0 + }, + { + "script": "tree grep=form_", + "output": "$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@14 combobox \"form_title\" value=\"Mr.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\n@23 textbox \"form_DOB\" value=\"1951-02-20\"\n@24 combobox \"form_sex\" value=\"Male\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"\n@38 combobox \"form_name_history[]\" value=\"\"\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", + "images": 0 + }, + { + "script": "scroll down px=800\ntree grep=form_", + "output": "$ scroll down px=800\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "click @37\nscroll down px=800\ntree grep=form_", + "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ scroll down px=800\nok: scroll down 800px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@13 textbox \"form_street\" value=\"\"\n@14 textbox \"form_street_line_2\" value=\"\"\n@15 textbox \"form_city\" value=\"\"\n@16 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@18 textbox \"form_postal_code\" value=\"\"\n@19 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@21 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@23 textbox \"form_mothersname\" value=\"\"\n@24 textbox \"form_contact_relationship\" value=\"\"\n@25 textbox \"form_phone_contact\" value=\"\"\n@26 textbox \"form_phone_home\" value=\"\"\n@27 textbox \"form_phone_biz\" value=\"\"\n@28 textbox \"form_phone_cell\" value=\"\"\n@29 textbox \"form_email\" value=\"\"\n@30 textbox \"form_email_direct\" value=\"\"\n(28 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 20 more; scroll up to reach them)", + "images": 0 + }, + { + "script": "fill @13 value=\"731 SYNTHETIC WAY\"\nfill @15 value=EXAMPLETON\nselect @16 option=Massachusetts\nfill @18 value=02139\nselect @19 option=USA\nfill @26 value=555-0121\nfill @29 value=barnaby.scoreboard.721@example.invalid\nfind name=\"Create New Patient\"", + "output": "$ fill @13 value=\"731 SYNTHETIC WAY\"\nok: fill textbox \"form_street\" value=\"731 SYNTHETIC WAY\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @15 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @16 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @18 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @19 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @26 value=555-0121\nok: fill textbox \"form_phone_home\" value=\"555-0121\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @29 value=barnaby.scoreboard.721@example.invalid\nok: fill textbox \"form_email\" value=\"barnaby.scoreboard.721@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ find name=\"Create New Patient\"\n@38 button \"Create New Patient\"", + "images": 0 + }, + { + "script": "click @38\ntext grep=atch", + "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ text grep=atch\nATCHISON\nNo matches were found.", + "images": 0 + }, + { + "script": "screenshot", + "output": "$ screenshot\n(screenshot attached)", + "images": 1 + }, + { + "script": "click text=\"Confirm Create New Patient\"\nscreenshot", + "output": "$ click text=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\nread back: 1 record(s) with email=barnaby.scoreboard.721@example.invalid, lname=ScoreboardT721 saved\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Confirm Create New Patient", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Confirm Create New Patient", + "outcome": "landed", + "detail": "1 record(s) with email=barnaby.scoreboard.721@example.invalid, lname=ScoreboardT721 saved", + "fields": [ + "DOB", + "city", + "country_code", + "email", + "fname", + "id", + "lname", + "phone_home", + "postal_code", + "sex", + "state", + "street", + "title" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 16, + "distilled_steps": [], + "in_program": 16, + "missing_steps": [], + "program_structural_steps": 16, + "passed": true + }, + "steps": 34, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_032: a write step with no read-back", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "form_DOB", + "masked_example": "19******20", + "name": "DOB", + "step_id": "step_008" + }, + { + "also_typed_in": [], + "field_label": "form_city", + "masked_example": "EX******ON", + "name": "city", + "step_id": "step_019" + }, + { + "also_typed_in": [], + "field_label": "form_country_code", + "masked_example": "U*A", + "name": "country_code", + "step_id": "step_026" + }, + { + "also_typed_in": [], + "field_label": "form_email", + "masked_example": "ba**********************************id", + "name": "email", + "step_id": "step_031" + }, + { + "also_typed_in": [], + "field_label": "First Name", + "masked_example": "B*****y", + "name": "fname", + "step_id": "step_004" + }, + { + "also_typed_in": [], + "field_label": "Last Name", + "masked_example": "Sc**********21", + "name": "lname", + "step_id": "step_006" + }, + { + "also_typed_in": [], + "field_label": "form_phone_home", + "masked_example": "5******1", + "name": "phone_home", + "step_id": "step_029" + }, + { + "also_typed_in": [], + "field_label": "form_postal_code", + "masked_example": "0***9", + "name": "postal_code", + "step_id": "step_024" + }, + { + "also_typed_in": [], + "field_label": "form_sex", + "masked_example": "M**e", + "name": "sex", + "step_id": "step_010" + }, + { + "also_typed_in": [], + "field_label": "form_state", + "masked_example": "Ma*********ts", + "name": "state_label", + "step_id": "step_021" + }, + { + "also_typed_in": [], + "field_label": "form_street", + "masked_example": "73*************AY", + "name": "street", + "step_id": "step_017" + }, + { + "also_typed_in": [], + "field_label": "form_title", + "masked_example": "M*.", + "name": "title", + "step_id": "step_001" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-002.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-002.json new file mode 100644 index 000000000..f616d1427 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-002.json @@ -0,0 +1,346 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 10, + "input_tokens": 22, + "output_tokens": 890, + "cache_creation_input_tokens": 8422, + "cache_read_input_tokens": 46552, + "prompt_tokens": 54996, + "images_in_requests": 2, + "cost_usd": 0.039309, + "wall_seconds": 48.472, + "model_seconds": 16.196, + "finished": true, + "summary": "Created patient Clarice ScoreboardU722; no duplicates found, save confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 117, + "cache_creation_input_tokens": 2854, + "cache_read_input_tokens": 0, + "seconds": 1.64, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 170, + "cache_creation_input_tokens": 672, + "cache_read_input_tokens": 2854, + "seconds": 1.771, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 51, + "cache_creation_input_tokens": 239, + "cache_read_input_tokens": 3526, + "seconds": 1.293, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 67, + "cache_creation_input_tokens": 929, + "cache_read_input_tokens": 3765, + "seconds": 1.439, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 63, + "cache_creation_input_tokens": 163, + "cache_read_input_tokens": 4694, + "seconds": 1.228, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 6, + "input_tokens": 2, + "output_tokens": 161, + "cache_creation_input_tokens": 739, + "cache_read_input_tokens": 4857, + "seconds": 1.728, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 7, + "input_tokens": 2, + "output_tokens": 70, + "cache_creation_input_tokens": 745, + "cache_read_input_tokens": 5596, + "seconds": 1.542, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 8, + "input_tokens": 2, + "output_tokens": 52, + "cache_creation_input_tokens": 376, + "cache_read_input_tokens": 6341, + "seconds": 1.249, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 9, + "input_tokens": 2, + "output_tokens": 63, + "cache_creation_input_tokens": 1485, + "cache_read_input_tokens": 6717, + "seconds": 2.675, + "stop_reason": "tool_use", + "images_in_request": 1 + }, + { + "turn": 10, + "input_tokens": 2, + "output_tokens": 76, + "cache_creation_input_tokens": 220, + "cache_read_input_tokens": 8202, + "seconds": 1.632, + "stop_reason": "tool_use", + "images_in_request": 1 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 1, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 35, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 10, + "lines": 27, + "actions": 19, + "errors": 2, + "screenshots": 1, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 16, + "with_recorded_locator": 16, + "locator_agrees": 16, + "locator_unique": 16, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Create New Patient", + "input:DOB", + "input:country_code", + "input:sex", + "input:title" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + }, + { + "script": "select @14 option=Ms.\nfill @15 value=Clarice\nfill @17 value=ScoreboardU722\nfill @23 value=1952-03-21\nselect @24 option=Female\nclick @40\ntree grep=Street", + "output": "$ select @14 option=Ms.\nok: select combobox \"form_title\" value=\"Ms.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @15 value=Clarice\nok: fill textbox \"First Name\" value=\"Clarice\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @17 value=ScoreboardU722\nok: fill textbox \"Last Name\" value=\"ScoreboardU722\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @23 value=1952-03-21\nok: fill textbox \"form_DOB\" value=\"1952-03-21\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @24 option=Female\nok: select combobox \"form_sex\" value=\"Female\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree grep=Street\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", + "images": 0 + }, + { + "script": "fill role=textbox name=form_street value=\"732 SYNTHETIC WAY\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind name=form_phone_home\nfind name=form_email", + "output": "$ fill role=textbox name=form_street value=\"732 SYNTHETIC WAY\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 6 lines not run)", + "images": 0 + }, + { + "script": "tree grep=form_", + "output": "$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@14 combobox \"form_title\" value=\"Ms.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\n@23 textbox \"form_DOB\" value=\"1952-03-21\"\n@24 combobox \"form_sex\" value=\"Female\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"\n@38 combobox \"form_name_history[]\" value=\"\"\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", + "images": 0 + }, + { + "script": "scroll down px=600\ntree grep=form_", + "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "click @37\nscroll down px=600\ntree grep=form_", + "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ scroll down px=600\nok: scroll down 600px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@14 textbox \"form_street\" value=\"\"\n@15 textbox \"form_street_line_2\" value=\"\"\n@16 textbox \"form_city\" value=\"\"\n@17 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@19 textbox \"form_postal_code\" value=\"\"\n@20 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@22 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@24 textbox \"form_mothersname\" value=\"\"\n@25 textbox \"form_contact_relationship\" value=\"\"\n@26 textbox \"form_phone_contact\" value=\"\"\n@27 textbox \"form_phone_home\" value=\"\"\n@28 textbox \"form_phone_biz\" value=\"\"\n@29 textbox \"form_phone_cell\" value=\"\"\n@30 textbox \"form_email\" value=\"\"\n@31 textbox \"form_email_direct\" value=\"\"\n(27 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 19 more; scroll up to reach them)", + "images": 0 + }, + { + "script": "fill @14 value=\"732 SYNTHETIC WAY\"\nfill @16 value=EXAMPLETON\nselect @17 option=Massachusetts\nfill @19 value=02139\nselect @20 option=USA\nfill @27 value=555-0122\nfill @30 value=clarice.scoreboard.722@example.invalid\nfind name=\"Create New Patient\"", + "output": "$ fill @14 value=\"732 SYNTHETIC WAY\"\nok: fill textbox \"form_street\" value=\"732 SYNTHETIC WAY\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @16 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @17 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @19 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @20 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @27 value=555-0122\nok: fill textbox \"form_phone_home\" value=\"555-0122\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @30 value=clarice.scoreboard.722@example.invalid\nok: fill textbox \"form_email\" value=\"clarice.scoreboard.722@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ find name=\"Create New Patient\"\nno match", + "images": 0 + }, + { + "script": "scroll down px=2000\nfind role=button", + "output": "$ scroll down px=2000\nok: scroll down 2000px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ find role=button\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\n@17 button \"Add\" value=\"Add\"\n@20 button \"Add\" value=\"Add\"\n@22 button \"Add\" value=\"Add\"\n@32 button \"Choices\"\n@33 button \"Employer\"\n@34 button \"Stats\"\n@35 button \"Misc\"\n@36 button \"Related\"\n@37 button \"Search\"\n@38 button \"Create New Patient\"", + "images": 0 + }, + { + "script": "click @38\nscreenshot", + "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ screenshot\n(screenshot attached)", + "images": 1 + }, + { + "script": "click text=\"Confirm Create New Patient\"\nscreenshot", + "output": "$ click text=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\nread back: 1 record(s) with email=clarice.scoreboard.722@example.invalid, lname=ScoreboardU722 saved\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Confirm Create New Patient", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Confirm Create New Patient", + "outcome": "landed", + "detail": "1 record(s) with email=clarice.scoreboard.722@example.invalid, lname=ScoreboardU722 saved", + "fields": [ + "DOB", + "city", + "country_code", + "email", + "fname", + "id", + "lname", + "phone_home", + "postal_code", + "sex", + "state", + "street", + "title" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 16, + "distilled_steps": [], + "in_program": 16, + "missing_steps": [], + "program_structural_steps": 16, + "passed": true + }, + "steps": 35, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_033: a write step with no read-back", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "form_DOB", + "masked_example": "19******21", + "name": "DOB", + "step_id": "step_008" + }, + { + "also_typed_in": [], + "field_label": "form_city", + "masked_example": "EX******ON", + "name": "city", + "step_id": "step_019" + }, + { + "also_typed_in": [], + "field_label": "form_country_code", + "masked_example": "U*A", + "name": "country_code", + "step_id": "step_026" + }, + { + "also_typed_in": [], + "field_label": "form_email", + "masked_example": "cl**********************************id", + "name": "email", + "step_id": "step_031" + }, + { + "also_typed_in": [], + "field_label": "First Name", + "masked_example": "C*****e", + "name": "fname", + "step_id": "step_004" + }, + { + "also_typed_in": [], + "field_label": "Last Name", + "masked_example": "Sc**********22", + "name": "lname", + "step_id": "step_006" + }, + { + "also_typed_in": [], + "field_label": "form_phone_home", + "masked_example": "5******2", + "name": "phone_home", + "step_id": "step_029" + }, + { + "also_typed_in": [], + "field_label": "form_postal_code", + "masked_example": "0***9", + "name": "postal_code", + "step_id": "step_024" + }, + { + "also_typed_in": [], + "field_label": "form_sex", + "masked_example": "F****e", + "name": "sex", + "step_id": "step_010" + }, + { + "also_typed_in": [], + "field_label": "form_state", + "masked_example": "Ma*********ts", + "name": "state_label", + "step_id": "step_021" + }, + { + "also_typed_in": [], + "field_label": "form_street", + "masked_example": "73*************AY", + "name": "street", + "step_id": "step_017" + }, + { + "also_typed_in": [], + "field_label": "form_title", + "masked_example": "M*.", + "name": "title", + "step_id": "step_001" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-003.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-003.json new file mode 100644 index 000000000..7931beb7c --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-003.json @@ -0,0 +1,374 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 11, + "input_tokens": 24, + "output_tokens": 1007, + "cache_creation_input_tokens": 8669, + "cache_read_input_tokens": 58686, + "prompt_tokens": 67379, + "images_in_requests": 0, + "cost_usd": 0.043528, + "wall_seconds": 49.327, + "model_seconds": 16.29, + "finished": true, + "summary": "Created patient Desmond ScoreboardV723 with all given values; no duplicate matches were found, and the save was confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 112, + "cache_creation_input_tokens": 2854, + "cache_read_input_tokens": 0, + "seconds": 1.672, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 180, + "cache_creation_input_tokens": 1937, + "cache_read_input_tokens": 2854, + "seconds": 1.929, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 67, + "cache_creation_input_tokens": 249, + "cache_read_input_tokens": 4791, + "seconds": 1.443, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 59, + "cache_creation_input_tokens": 110, + "cache_read_input_tokens": 5040, + "seconds": 1.42, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 63, + "cache_creation_input_tokens": 155, + "cache_read_input_tokens": 5150, + "seconds": 1.367, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 6, + "input_tokens": 2, + "output_tokens": 164, + "cache_creation_input_tokens": 739, + "cache_read_input_tokens": 5305, + "seconds": 1.71, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 7, + "input_tokens": 2, + "output_tokens": 102, + "cache_creation_input_tokens": 751, + "cache_read_input_tokens": 6044, + "seconds": 1.377, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 8, + "input_tokens": 2, + "output_tokens": 63, + "cache_creation_input_tokens": 250, + "cache_read_input_tokens": 6795, + "seconds": 1.198, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 9, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 154, + "cache_read_input_tokens": 7045, + "seconds": 1.247, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 10, + "input_tokens": 2, + "output_tokens": 60, + "cache_creation_input_tokens": 1264, + "cache_read_input_tokens": 7199, + "seconds": 1.422, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 11, + "input_tokens": 2, + "output_tokens": 87, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 8463, + "seconds": 1.505, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 35, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 12, + "lines": 31, + "actions": 19, + "errors": 2, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 16, + "with_recorded_locator": 16, + "locator_agrees": 16, + "locator_unique": 16, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Create New Patient", + "input:DOB", + "input:city", + "input:country_code", + "input:email", + "input:fname", + "input:lname", + "input:phone_home", + "input:postal_code", + "input:sex", + "input:state_label", + "input:street", + "input:title" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + }, + { + "script": "select @14 option=Mr.\nfill @15 value=Desmond\nfill @17 value=ScoreboardV723\nfill @23 value=1953-04-22\nselect @24 option=Male\nclick @40\ntree", + "output": "$ select @14 option=Mr.\nok: select combobox \"form_title\" value=\"Mr.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @15 value=Desmond\nok: fill textbox \"First Name\" value=\"Desmond\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @17 value=ScoreboardV723\nok: fill textbox \"Last Name\" value=\"ScoreboardV723\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @23 value=1953-04-22\nok: fill textbox \"form_DOB\" value=\"1953-04-22\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @24 option=Male\nok: select combobox \"form_sex\" value=\"Male\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Mr.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"Desmond\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"ScoreboardV723\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"1953-04-22\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Male\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"t", + "images": 0 + }, + { + "script": "fill role=textbox name=form_street value=\"733 SYNTHETIC WAY\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind role=textbox name=form_phone_home\nfind role=textbox name=form_email", + "output": "$ fill role=textbox name=form_street value=\"733 SYNTHETIC WAY\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 6 lines not run)", + "images": 0 + }, + { + "script": "find name=form_street\nfind name=form_phone_home\nfind name=form_email", + "output": "$ find name=form_street\nno match\n$ find name=form_phone_home\nno match\n$ find name=form_email\nno match", + "images": 0 + }, + { + "script": "scroll down px=600\ntree grep=form_", + "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "click @37\nscroll down px=600\ntree grep=form_", + "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ scroll down px=600\nok: scroll down 600px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@14 textbox \"form_street\" value=\"\"\n@15 textbox \"form_street_line_2\" value=\"\"\n@16 textbox \"form_city\" value=\"\"\n@17 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@19 textbox \"form_postal_code\" value=\"\"\n@20 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@22 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@24 textbox \"form_mothersname\" value=\"\"\n@25 textbox \"form_contact_relationship\" value=\"\"\n@26 textbox \"form_phone_contact\" value=\"\"\n@27 textbox \"form_phone_home\" value=\"\"\n@28 textbox \"form_phone_biz\" value=\"\"\n@29 textbox \"form_phone_cell\" value=\"\"\n@30 textbox \"form_email\" value=\"\"\n@31 textbox \"form_email_direct\" value=\"\"\n(27 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 19 more; scroll up to reach them)", + "images": 0 + }, + { + "script": "fill @14 value=\"733 SYNTHETIC WAY\"\nfill @16 value=EXAMPLETON\nselect @17 option=Massachusetts\nfill @19 value=02139\nselect @20 option=USA\nfill @27 value=555-0123\nfill @30 value=desmond.scoreboard.723@example.invalid\nfind role=button name=\"Create New Patient\"", + "output": "$ fill @14 value=\"733 SYNTHETIC WAY\"\nok: fill textbox \"form_street\" value=\"733 SYNTHETIC WAY\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @16 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @17 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @19 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @20 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @27 value=555-0123\nok: fill textbox \"form_phone_home\" value=\"555-0123\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @30 value=desmond.scoreboard.723@example.invalid\nok: fill textbox \"form_email\" value=\"desmond.scoreboard.723@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ find role=button name=\"Create New Patient\"\nno match", + "images": 0 + }, + { + "script": "find name=\"Create New Patient\"", + "output": "$ find name=\"Create New Patient\"\nno match", + "images": 0 + }, + { + "script": "tree grep=Create", + "output": "$ tree grep=Create\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n(2 controls below the visible area: \"Search\", \"Create New Patient\"; scroll down to reach them)", + "images": 0 + }, + { + "script": "scroll down px=2000\nfind name=\"Create New Patient\"", + "output": "$ scroll down px=2000\nok: scroll down 2000px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ find name=\"Create New Patient\"\n@38 button \"Create New Patient\"", + "images": 0 + }, + { + "script": "click @38\ntree", + "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\nAddress:\n@13 textbox \"form_street\" value=\"733 SYNTHETIC WAY\"\nAddress Line 2:\n@14 textbox \"form_street_line_2\" value=\"\"\nCity:\n@15 textbox \"form_city\" value=\"EXAMPLETON\"\nState:\n@16 combobox \"form_state\" value=\"Massachusetts\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@17 button \"Add\" value=\"Add\"\nPostal Code:\n@18 textbox \"form_postal_code\" value=\"02139\"\nCountry:\n@19 combobox \"form_country_code\" value=\"USA\" options=[\"Unassigned\", \"USA\"]\n@20 button \"Add\" value=\"Add\"\nCounty:\n@21 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@22 button \"Add\" value=\"Add\"\nMother`s Name:\n@23 textbox \"form_mothersname\" value=\"\"\nEmergency Contact:\n@24 textbox \"form_contact_relationship\" value=\"\"\nEmergency Phone:\n@25 textbox \"form_phone_contact\" value=\"\"\nHome Phone:\n@26 textbox \"form_phone_home\" value=\"555-0123\"\nWork Phone:\n@27 textbox \"form_phone_biz\" value=\"\"\nMobile Phone:\n@28 textbox \"form_phone_cell\" value=\"\"\nContact Email:\n@29 textbox \"form_email\" value=\"desmond.scoreboard.723@example.invalid\"\nTrusted Email:\n@30 textbox \"form_email_direct\" value=\"\"\nAdditional Addresses\n@31 checkbox \"Show Inactive\" value=\"on\"\nShow Inactive\nNONE\n@32 button \"Choices\"\n@33 button \"Employer\"\n@34 button \"Stats\"\n@35 button \"Misc\"\n@36 button \"Related\"\n@37 button \"Search\"\n@38 button \"Create New Patient\"\n(28 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 20 more; scroll up to reach them)\n# frame 7: \nrow: 1 - 0 of 0\nrow: Hits | Name | Preferred Name | Birth Name | DOB | Birth Sex | External ID | Previous Names\nrow: # No matches were found.\n@39 button \"Confirm Create New Patient\"\n@40 button \"Cancel\"", + "images": 0 + }, + { + "script": "click @39\ntext", + "output": "$ click @39\nok: click button \"Confirm Create New Patient\" (save gate passed)\nread back: 1 record(s) with email=desmond.scoreboard.723@example.invalid, lname=ScoreboardV723 saved\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Confirm Create New Patient", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Confirm Create New Patient", + "outcome": "landed", + "detail": "1 record(s) with email=desmond.scoreboard.723@example.invalid, lname=ScoreboardV723 saved", + "fields": [ + "DOB", + "city", + "country_code", + "email", + "fname", + "id", + "lname", + "phone_home", + "postal_code", + "sex", + "state", + "street", + "title" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 16, + "distilled_steps": [], + "in_program": 16, + "missing_steps": [], + "program_structural_steps": 16, + "passed": true + }, + "steps": 35, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_033: a write step with no read-back", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "form_DOB", + "masked_example": "19******22", + "name": "DOB", + "step_id": "step_008" + }, + { + "also_typed_in": [], + "field_label": "form_city", + "masked_example": "EX******ON", + "name": "city", + "step_id": "step_019" + }, + { + "also_typed_in": [], + "field_label": "form_country_code", + "masked_example": "U*A", + "name": "country_code", + "step_id": "step_026" + }, + { + "also_typed_in": [], + "field_label": "form_email", + "masked_example": "de**********************************id", + "name": "email", + "step_id": "step_031" + }, + { + "also_typed_in": [], + "field_label": "First Name", + "masked_example": "D*****d", + "name": "fname", + "step_id": "step_004" + }, + { + "also_typed_in": [], + "field_label": "Last Name", + "masked_example": "Sc**********23", + "name": "lname", + "step_id": "step_006" + }, + { + "also_typed_in": [], + "field_label": "form_phone_home", + "masked_example": "5******3", + "name": "phone_home", + "step_id": "step_029" + }, + { + "also_typed_in": [], + "field_label": "form_postal_code", + "masked_example": "0***9", + "name": "postal_code", + "step_id": "step_024" + }, + { + "also_typed_in": [], + "field_label": "form_sex", + "masked_example": "M**e", + "name": "sex", + "step_id": "step_010" + }, + { + "also_typed_in": [], + "field_label": "form_state", + "masked_example": "Ma*********ts", + "name": "state_label", + "step_id": "step_021" + }, + { + "also_typed_in": [], + "field_label": "form_street", + "masked_example": "73*************AY", + "name": "street", + "step_id": "step_017" + }, + { + "also_typed_in": [], + "field_label": "form_title", + "masked_example": "M*.", + "name": "title", + "step_id": "step_001" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-004.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-004.json new file mode 100644 index 000000000..f71ea6e38 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-004.json @@ -0,0 +1,319 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 8, + "input_tokens": 18, + "output_tokens": 749, + "cache_creation_input_tokens": 8135, + "cache_read_input_tokens": 38146, + "prompt_tokens": 46299, + "images_in_requests": 0, + "cost_usd": 0.035493, + "wall_seconds": 40.637, + "model_seconds": 12.34, + "finished": true, + "summary": "Created patient Edwina ScoreboardW724 with all the given values. The duplicate check found no matches, and the save was confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 112, + "cache_creation_input_tokens": 2854, + "cache_read_input_tokens": 0, + "seconds": 1.65, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 103, + "cache_creation_input_tokens": 1937, + "cache_read_input_tokens": 2854, + "seconds": 1.558, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 59, + "cache_creation_input_tokens": 192, + "cache_read_input_tokens": 4791, + "seconds": 1.361, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 63, + "cache_creation_input_tokens": 155, + "cache_read_input_tokens": 4983, + "seconds": 1.398, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 164, + "cache_creation_input_tokens": 739, + "cache_read_input_tokens": 5138, + "seconds": 1.622, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 6, + "input_tokens": 2, + "output_tokens": 49, + "cache_creation_input_tokens": 761, + "cache_read_input_tokens": 5877, + "seconds": 1.419, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 7, + "input_tokens": 2, + "output_tokens": 110, + "cache_creation_input_tokens": 1227, + "cache_read_input_tokens": 6638, + "seconds": 1.725, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 8, + "input_tokens": 2, + "output_tokens": 89, + "cache_creation_input_tokens": 270, + "cache_read_input_tokens": 7865, + "seconds": 1.606, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 34, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 8, + "lines": 28, + "actions": 18, + "errors": 1, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 16, + "with_recorded_locator": 16, + "locator_agrees": 16, + "locator_unique": 16, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Create New Patient", + "input:DOB", + "input:country_code", + "input:fname", + "input:lname", + "input:sex", + "input:state_label", + "input:title" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + }, + { + "script": "select @14 option=Ms.\nfill @15 value=Edwina\nfill @17 value=ScoreboardW724\nfill @23 value=1954-05-23\nselect @24 option=Female\nclick @40\ntree", + "output": "$ select @14 option=Ms.\nok: select combobox \"form_title\" value=\"Ms.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @15 value=Edwina\nok: fill textbox \"First Name\" value=\"Edwina\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @17 value=ScoreboardW724\nok: fill textbox \"Last Name\" value=\"ScoreboardW724\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @23 value=1954-05-23\nok: fill textbox \"form_DOB\" value=\"1954-05-23\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @24 option=Female\nok: select combobox \"form_sex\" value=\"Female\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Ms.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"Edwina\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"ScoreboardW724\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"1954-05-23\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Female\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\",", + "images": 0 + }, + { + "script": "find name=form_street\nfind role=textbox name=form_phone_home\nfind role=textbox name=form_email\nfind role=combobox name=form_state\nfind role=combobox name=form_country_code", + "output": "$ find name=form_street\nno match\n$ find role=textbox name=form_phone_home\nno match\n$ find role=textbox name=form_email\nno match\n$ find role=combobox name=form_state\nno match\n$ find role=combobox name=form_country_code\nno match", + "images": 0 + }, + { + "script": "scroll down px=600\ntree grep=form_", + "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "click @37\nscroll down px=700\ntree grep=form_", + "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ scroll down px=700\nok: scroll down 700px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@13 textbox \"form_street\" value=\"\"\n@14 textbox \"form_street_line_2\" value=\"\"\n@15 textbox \"form_city\" value=\"\"\n@16 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@18 textbox \"form_postal_code\" value=\"\"\n@19 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@21 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@23 textbox \"form_mothersname\" value=\"\"\n@24 textbox \"form_contact_relationship\" value=\"\"\n@25 textbox \"form_phone_contact\" value=\"\"\n@26 textbox \"form_phone_home\" value=\"\"\n@27 textbox \"form_phone_biz\" value=\"\"\n@28 textbox \"form_phone_cell\" value=\"\"\n@29 textbox \"form_email\" value=\"\"\n@30 textbox \"form_email_direct\" value=\"\"\n(28 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 20 more; scroll up to reach them)", + "images": 0 + }, + { + "script": "fill @13 value=\"734 SYNTHETIC WAY\"\nfill @15 value=EXAMPLETON\nselect @16 option=Massachusetts\nfill @18 value=02139\nselect @19 option=USA\nfill @26 value=555-0124\nfill @29 value=edwina.scoreboard.724@example.invalid\nfind role=button name=\"Create New Patient\"", + "output": "$ fill @13 value=\"734 SYNTHETIC WAY\"\nok: fill textbox \"form_street\" value=\"734 SYNTHETIC WAY\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @15 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @16 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @18 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @19 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @26 value=555-0124\nok: fill textbox \"form_phone_home\" value=\"555-0124\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @29 value=edwina.scoreboard.724@example.invalid\nok: fill textbox \"form_email\" value=\"edwina.scoreboard.724@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ find role=button name=\"Create New Patient\"\n@38 button \"Create New Patient\"", + "images": 0 + }, + { + "script": "click @38\ntext", + "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ text\nCalendar\nFinder\nFlow\nRecalls\nMessages\nPatient\nFees\nModules\nProcedures\nAdmin\nReports\nMiscellaneous\nPopups\nCalendar\nMessage Center\nSearch or Add Patient\nSearch or Add Patient\nWho\nTitle:\nUnassigned\nMr.\nMrs.\nMs.\nDr.\nName:\nPreferred Name:\nBirth Name:\nDOB:\nBirth Sex:\nUnassigned\nFemale\nMale\nUnknown\nGender Identity:\nUnassigned\nIdentifies as Male\nIdentifies as Female\nFemale-to-Male (FTM)/Transgender Male/Trans Man\nMale-to-Female (MTF)/Transgender Female/Trans Woman\nGenderqueer, neither exclusively male nor female\nAdditional gender category or other, please specify\nChoose not to disclose\nSex:\nUnassigned\nMale\nFemale\nIdentifies as nonbinary gender (finding)\nAsked But Declined\nunknown\nSexual Orientation:\nUnassigned\nStraight or heterosexual\nLesbian, gay or homosexual\nBisexual\nSomething else, please describe\nDon't know\nChoose not to disclose\nPronouns:\nUnassigned\nhe/him/his/his/himself\nshe/her/her/hers/herself\nthey/them/their/theirs/themselves\nAsk me\nDecline to answer\nOther\nExternal ID:\nS.S.:\nLicense/ID:\nMarital Status:\nUnassigned\nMarried\nSingle\nDivorced\nWidowed\nSeparated\nDomestic Partner\nUser Defined:\nBilling Note:\nPrevious Names:\nAdd\nContact\nAddress:\nAddress Line 2:\nCity:\nState:\nUnassigned\nAlabama\nAlaska\nArizona\nArkansas\nCalifornia\nColorado\nConnecticut\nDelaware\nDistrict of Columbia\nFlorida\nGeorgia\nHawaii\nIdaho\nIllinois\nIndiana\nIowa\nKansas\nKentucky\nLouisiana\nMaine\nMaryland\nMassachusetts\nMichigan\nMinnesota\nMississippi\nMissouri\nMontana\nNebraska\nNevada\nNew Hampshire\nNew Jersey\nNew Mexico\nNew York\nNorth Carolina\nNorth Dakota\nOhio\nOklahoma\nOregon\nPennsylvania\nPuerto Rico\nRhode Island\nSouth Carolina\nSouth Dakota\nTennessee\nTexas\nUtah\nVermont\nVirginia\nWashington\nWest Virginia\nWisconsin\nWyoming\nPostal Code:\nCountry:\nUnassigned\nUSA\nCounty:\nUnassigned\nADAIR\nANDREW\nATCHISON\nAUDRAIN\nBARRY\nMother`s Name:\nEmergency Contact:\nEmergency Phone:\nHome Phone:\nWork Phone:\nMobile Phone:\nContact Email:\nTrusted Email:\nAdditional Addresses\nShow Inactive\nNONE\nChoices\nEmployer\nStats\nMisc\nRelated\nSearch\nCreate New Patient\n1 - 0 of 0\nHits\tName\t\tPreferred Name\tBirth Name\t\t\tDOB\tBirth Sex\tExternal ID\tPrevious Names\nNo matches were found.\nConfirm Create New Patient Cancel", + "images": 0 + }, + { + "script": "click role=button name=\"Confirm Create New Patient\"\ntext grep=Edwina", + "output": "$ click role=button name=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\nread back: 1 record(s) with email=edwina.scoreboard.724@example.invalid, lname=ScoreboardW724 saved\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Confirm Create New Patient", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Confirm Create New Patient", + "outcome": "landed", + "detail": "1 record(s) with email=edwina.scoreboard.724@example.invalid, lname=ScoreboardW724 saved", + "fields": [ + "DOB", + "city", + "country_code", + "email", + "fname", + "id", + "lname", + "phone_home", + "postal_code", + "sex", + "state", + "street", + "title" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 16, + "distilled_steps": [], + "in_program": 16, + "missing_steps": [], + "program_structural_steps": 16, + "passed": true + }, + "steps": 34, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_032: a write step with no read-back", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "form_DOB", + "masked_example": "19******23", + "name": "DOB", + "step_id": "step_008" + }, + { + "also_typed_in": [], + "field_label": "form_city", + "masked_example": "EX******ON", + "name": "city", + "step_id": "step_019" + }, + { + "also_typed_in": [], + "field_label": "form_country_code", + "masked_example": "U*A", + "name": "country_code", + "step_id": "step_026" + }, + { + "also_typed_in": [], + "field_label": "form_email", + "masked_example": "ed*********************************id", + "name": "email", + "step_id": "step_031" + }, + { + "also_typed_in": [], + "field_label": "First Name", + "masked_example": "E****a", + "name": "fname", + "step_id": "step_004" + }, + { + "also_typed_in": [], + "field_label": "Last Name", + "masked_example": "Sc**********24", + "name": "lname", + "step_id": "step_006" + }, + { + "also_typed_in": [], + "field_label": "form_phone_home", + "masked_example": "5******4", + "name": "phone_home", + "step_id": "step_029" + }, + { + "also_typed_in": [], + "field_label": "form_postal_code", + "masked_example": "0***9", + "name": "postal_code", + "step_id": "step_024" + }, + { + "also_typed_in": [], + "field_label": "form_sex", + "masked_example": "F****e", + "name": "sex", + "step_id": "step_010" + }, + { + "also_typed_in": [], + "field_label": "form_state", + "masked_example": "Ma*********ts", + "name": "state_label", + "step_id": "step_021" + }, + { + "also_typed_in": [], + "field_label": "form_street", + "masked_example": "73*************AY", + "name": "street", + "step_id": "step_017" + }, + { + "also_typed_in": [], + "field_label": "form_title", + "masked_example": "M*.", + "name": "title", + "step_id": "step_001" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-005.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-005.json new file mode 100644 index 000000000..61cc8b580 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/ledgers/item-005.json @@ -0,0 +1,409 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 14, + "input_tokens": 30, + "output_tokens": 1161, + "cache_creation_input_tokens": 8871, + "cache_read_input_tokens": 80366, + "prompt_tokens": 89267, + "images_in_requests": 0, + "cost_usd": 0.049921, + "wall_seconds": 57.915, + "model_seconds": 22.121, + "finished": true, + "summary": "Created patient Fitzroy ScoreboardX725 with all given values; no existing matches found, save confirmed.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 123, + "cache_creation_input_tokens": 2858, + "cache_read_input_tokens": 0, + "seconds": 2.016, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 49, + "cache_creation_input_tokens": 626, + "cache_read_input_tokens": 2858, + "seconds": 1.284, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 180, + "cache_creation_input_tokens": 1450, + "cache_read_input_tokens": 3484, + "seconds": 2.563, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 59, + "cache_creation_input_tokens": 249, + "cache_read_input_tokens": 4934, + "seconds": 1.541, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 63, + "cache_creation_input_tokens": 155, + "cache_read_input_tokens": 5183, + "seconds": 1.311, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 6, + "input_tokens": 2, + "output_tokens": 166, + "cache_creation_input_tokens": 739, + "cache_read_input_tokens": 5338, + "seconds": 1.812, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 7, + "input_tokens": 2, + "output_tokens": 54, + "cache_creation_input_tokens": 757, + "cache_read_input_tokens": 6077, + "seconds": 1.378, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 8, + "input_tokens": 2, + "output_tokens": 68, + "cache_creation_input_tokens": 74, + "cache_read_input_tokens": 6834, + "seconds": 1.454, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 9, + "input_tokens": 2, + "output_tokens": 51, + "cache_creation_input_tokens": 171, + "cache_read_input_tokens": 6908, + "seconds": 1.296, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 10, + "input_tokens": 2, + "output_tokens": 47, + "cache_creation_input_tokens": 116, + "cache_read_input_tokens": 7079, + "seconds": 1.419, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 11, + "input_tokens": 2, + "output_tokens": 45, + "cache_creation_input_tokens": 109, + "cache_read_input_tokens": 7195, + "seconds": 1.311, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 12, + "input_tokens": 2, + "output_tokens": 126, + "cache_creation_input_tokens": 1165, + "cache_read_input_tokens": 7304, + "seconds": 1.727, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 13, + "input_tokens": 2, + "output_tokens": 47, + "cache_creation_input_tokens": 234, + "cache_read_input_tokens": 8469, + "seconds": 1.499, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 14, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 168, + "cache_read_input_tokens": 8703, + "seconds": 1.509, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 35, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0, + "readbacks": 1 + }, + "trace": { + "surface": "dom", + "scripts": 14, + "lines": 31, + "actions": 19, + "errors": 3, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 16, + "with_recorded_locator": 16, + "locator_agrees": 16, + "locator_unique": 16, + "commits": 1, + "readbacks": { + "landed": 1 + }, + "identity_read_before_commit": [ + "identity:Create New Patient", + "input:DOB", + "input:country_code", + "input:fname", + "input:lname", + "input:sex", + "input:state_label", + "input:title" + ], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + }, + { + "script": "select @14 option=Mr.\nfill @15 value=Fitzroy\nfill @17 value=ScoreboardX725\nfill @23 value=1955-06-24\nselect @24 option=Male\nclick @40\ntree grep=Contact max=60", + "output": "$ select @14 option=Mr.\nok: select combobox \"form_title\" value=\"Mr.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @15 value=Fitzroy\nok: fill textbox \"First Name\" value=\"Fitzroy\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @17 value=ScoreboardX725\nok: fill textbox \"Last Name\" value=\"ScoreboardX725\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @23 value=1955-06-24\nok: fill textbox \"form_DOB\" value=\"1955-06-24\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @24 option=Male\nok: select combobox \"form_sex\" value=\"Male\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree grep=Contact max=60\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@40 button \"Contact\" (focused)", + "images": 0 + }, + { + "script": "tree max=120", + "output": "$ tree max=120\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Mr.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"Fitzroy\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"ScoreboardX725\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"1955-06-24\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Male\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\" (focused)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", ", + "images": 0 + }, + { + "script": "fill role=textbox name=form_street value=\"735 SYNTHETIC WAY\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind role=textbox name=form_phone_home\nfind role=textbox name=form_email", + "output": "$ fill role=textbox name=form_street value=\"735 SYNTHETIC WAY\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 6 lines not run)", + "images": 0 + }, + { + "script": "scroll down px=600\ntree grep=form_", + "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "click @37\nscroll down px=600\ntree grep=form_", + "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ scroll down px=600\nok: scroll down 600px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@14 textbox \"form_street\" value=\"\"\n@15 textbox \"form_street_line_2\" value=\"\"\n@16 textbox \"form_city\" value=\"\"\n@17 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@19 textbox \"form_postal_code\" value=\"\"\n@20 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@22 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@24 textbox \"form_mothersname\" value=\"\"\n@25 textbox \"form_contact_relationship\" value=\"\"\n@26 textbox \"form_phone_contact\" value=\"\"\n@27 textbox \"form_phone_home\" value=\"\"\n@28 textbox \"form_phone_biz\" value=\"\"\n@29 textbox \"form_phone_cell\" value=\"\"\n@30 textbox \"form_email\" value=\"\"\n@31 textbox \"form_email_direct\" value=\"\"\n(27 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 19 more; scroll up to reach them)", + "images": 0 + }, + { + "script": "fill @14 value=\"735 SYNTHETIC WAY\"\nfill @16 value=EXAMPLETON\nselect @17 option=Massachusetts\nfill @19 value=02139\nselect @20 option=USA\nfill @27 value=555-0125\nfill @30 value=fitzroy.scoreboard.725@example.invalid\nfind role=button name=\"Create New Patient\"", + "output": "$ fill @14 value=\"735 SYNTHETIC WAY\"\nok: fill textbox \"form_street\" value=\"735 SYNTHETIC WAY\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @16 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @17 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @19 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ select @20 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @27 value=555-0125\nok: fill textbox \"form_phone_home\" value=\"555-0125\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ fill @30 value=fitzroy.scoreboard.725@example.invalid\nok: fill textbox \"form_email\" value=\"fitzroy.scoreboard.725@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ find role=button name=\"Create New Patient\"\nno match", + "images": 0 + }, + { + "script": "find name=\"Create New Patient\"", + "output": "$ find name=\"Create New Patient\"\nno match", + "images": 0 + }, + { + "script": "scroll down px=3000\nfind name~=\"Create\"\ntree grep=Create", + "output": "$ scroll down px=3000\nok: scroll down 2000px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n$ find name~=\"Create\"\nerror: find needs role=, name= or near=\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "tree grep=Create", + "output": "$ tree grep=Create\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@38 button \"Create New Patient\"", + "images": 0 + }, + { + "script": "click @38", + "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)", + "images": 0 + }, + { + "script": "text", + "output": "$ text\nCalendar\nFinder\nFlow\nRecalls\nMessages\nPatient\nFees\nModules\nProcedures\nAdmin\nReports\nMiscellaneous\nPopups\nCalendar\nMessage Center\nSearch or Add Patient\nSearch or Add Patient\nWho\nTitle:\nUnassigned\nMr.\nMrs.\nMs.\nDr.\nName:\nPreferred Name:\nBirth Name:\nDOB:\nBirth Sex:\nUnassigned\nFemale\nMale\nUnknown\nGender Identity:\nUnassigned\nIdentifies as Male\nIdentifies as Female\nFemale-to-Male (FTM)/Transgender Male/Trans Man\nMale-to-Female (MTF)/Transgender Female/Trans Woman\nGenderqueer, neither exclusively male nor female\nAdditional gender category or other, please specify\nChoose not to disclose\nSex:\nUnassigned\nMale\nFemale\nIdentifies as nonbinary gender (finding)\nAsked But Declined\nunknown\nSexual Orientation:\nUnassigned\nStraight or heterosexual\nLesbian, gay or homosexual\nBisexual\nSomething else, please describe\nDon't know\nChoose not to disclose\nPronouns:\nUnassigned\nhe/him/his/his/himself\nshe/her/her/hers/herself\nthey/them/their/theirs/themselves\nAsk me\nDecline to answer\nOther\nExternal ID:\nS.S.:\nLicense/ID:\nMarital Status:\nUnassigned\nMarried\nSingle\nDivorced\nWidowed\nSeparated\nDomestic Partner\nUser Defined:\nBilling Note:\nPrevious Names:\nAdd\nContact\nAddress:\nAddress Line 2:\nCity:\nState:\nUnassigned\nAlabama\nAlaska\nArizona\nArkansas\nCalifornia\nColorado\nConnecticut\nDelaware\nDistrict of Columbia\nFlorida\nGeorgia\nHawaii\nIdaho\nIllinois\nIndiana\nIowa\nKansas\nKentucky\nLouisiana\nMaine\nMaryland\nMassachusetts\nMichigan\nMinnesota\nMississippi\nMissouri\nMontana\nNebraska\nNevada\nNew Hampshire\nNew Jersey\nNew Mexico\nNew York\nNorth Carolina\nNorth Dakota\nOhio\nOklahoma\nOregon\nPennsylvania\nPuerto Rico\nRhode Island\nSouth Carolina\nSouth Dakota\nTennessee\nTexas\nUtah\nVermont\nVirginia\nWashington\nWest Virginia\nWisconsin\nWyoming\nPostal Code:\nCountry:\nUnassigned\nUSA\nCounty:\nUnassigned\nADAIR\nANDREW\nATCHISON\nAUDRAIN\nBARRY\nMother`s Name:\nEmergency Contact:\nEmergency Phone:\nHome Phone:\nWork Phone:\nMobile Phone:\nContact Email:\nTrusted Email:\nAdditional Addresses\nShow Inactive\nNONE\nChoices\nEmployer\nStats\nMisc\nRelated\nSearch\nCreate New Patient\n1 - 0 of 0\nHits\tName\t\tPreferred Name\tBirth Name\t\t\tDOB\tBirth Sex\tExternal ID\tPrevious Names\nNo matches were found.\nConfirm Create New Patient Cancel", + "images": 0 + }, + { + "script": "find role=button name=\"Confirm Create New Patient\"\ntree grep=Confirm", + "output": "$ find role=button name=\"Confirm Create New Patient\"\n@39 button \"Confirm Create New Patient\"\n$ tree grep=Confirm\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)\n@39 button \"Confirm Create New Patient\"", + "images": 0 + }, + { + "script": "click @39", + "output": "$ click @39\nok: click button \"Confirm Create New Patient\" (save gate passed)\nread back: 1 record(s) with email=fitzroy.scoreboard.725@example.invalid, lname=ScoreboardX725 saved\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?...)", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Confirm Create New Patient", + "missing": [], + "decision": "passed_no_hold" + } + ], + "readbacks": [ + { + "control": "Confirm Create New Patient", + "outcome": "landed", + "detail": "1 record(s) with email=fitzroy.scoreboard.725@example.invalid, lname=ScoreboardX725 saved", + "fields": [ + "DOB", + "city", + "country_code", + "email", + "fname", + "id", + "lname", + "phone_home", + "postal_code", + "sex", + "state", + "street", + "title" + ] + } + ], + "program_check": { + "trace_steps_with_locator": 16, + "distilled_steps": [], + "in_program": 16, + "missing_steps": [], + "program_structural_steps": 16, + "passed": true + }, + "steps": 35, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_033: a write step with no read-back", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "also_typed_in": [], + "field_label": "form_DOB", + "masked_example": "19******24", + "name": "DOB", + "step_id": "step_008" + }, + { + "also_typed_in": [], + "field_label": "form_city", + "masked_example": "EX******ON", + "name": "city", + "step_id": "step_019" + }, + { + "also_typed_in": [], + "field_label": "form_country_code", + "masked_example": "U*A", + "name": "country_code", + "step_id": "step_026" + }, + { + "also_typed_in": [], + "field_label": "form_email", + "masked_example": "fi**********************************id", + "name": "email", + "step_id": "step_031" + }, + { + "also_typed_in": [], + "field_label": "First Name", + "masked_example": "F*****y", + "name": "fname", + "step_id": "step_004" + }, + { + "also_typed_in": [], + "field_label": "Last Name", + "masked_example": "Sc**********25", + "name": "lname", + "step_id": "step_006" + }, + { + "also_typed_in": [], + "field_label": "form_phone_home", + "masked_example": "5******5", + "name": "phone_home", + "step_id": "step_029" + }, + { + "also_typed_in": [], + "field_label": "form_postal_code", + "masked_example": "0***9", + "name": "postal_code", + "step_id": "step_024" + }, + { + "also_typed_in": [], + "field_label": "form_sex", + "masked_example": "M**e", + "name": "sex", + "step_id": "step_010" + }, + { + "also_typed_in": [], + "field_label": "form_state", + "masked_example": "Ma*********ts", + "name": "state_label", + "step_id": "step_021" + }, + { + "also_typed_in": [], + "field_label": "form_street", + "masked_example": "73*************AY", + "name": "street", + "step_id": "step_017" + }, + { + "also_typed_in": [], + "field_label": "form_title", + "masked_example": "M*.", + "name": "title", + "step_id": "step_001" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/meta.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/meta.json new file mode 100644 index 000000000..19530df25 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/meta.json @@ -0,0 +1,22 @@ +{ + "protocol_version": "2026-10-18.1", + "written_at": "2026-10-24T23:47:00-0400", + "machine": "arm64 Darwin 24.6.0", + "cpu_count": 12, + "python": "3.12.7", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + }, + "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", + "task": "openemr_referral", + "arm": "code_agent", + "model": "claude-sonnet-5-5", + "n": 5, + "vision": "code", + "effort": "medium", + "max_cost_per_item": 0.3, + "on_screen": [ + "Create New Patient" + ] +} diff --git a/packages/flow/benchmark/scoreboard/raw/spend.json b/packages/flow/benchmark/scoreboard/raw/spend.json index 09b9cce1d..e71d8b50c 100644 --- a/packages/flow/benchmark/scoreboard/raw/spend.json +++ b/packages/flow/benchmark/scoreboard/raw/spend.json @@ -1,5 +1,5 @@ { - "total_usd": 12.543253, + "total_usd": 12.975542, "entries": [ { "usd": 0.577, @@ -1234,6 +1234,136 @@ "usd": 0.042055, "note": "openemr_referral code_agent 2026-10-19-timed item 725 (run from the claude/scoreboard-arm7 checkout)", "at": 1792435252.3616168 + }, + { + "usd": 0.009192, + "note": "mockmed code_agent 2026-10-25-rb item 21", + "at": 1792899281.865619 + }, + { + "usd": 0.009232, + "note": "mockmed code_agent 2026-10-25-rb item 22", + "at": 1792899302.969949 + }, + { + "usd": 0.009192, + "note": "mockmed code_agent 2026-10-25-rb item 23", + "at": 1792899328.5665998 + }, + { + "usd": 0.009232, + "note": "mockmed code_agent 2026-10-25-rb item 24", + "at": 1792899351.1901581 + }, + { + "usd": 0.009192, + "note": "mockmed code_agent 2026-10-25-rb item 25", + "at": 1792899367.119232 + }, + { + "usd": 0.009125, + "note": "mockmed code_agent 2026-10-25-rb item 26", + "at": 1792899382.452783 + }, + { + "usd": 0.009125, + "note": "mockmed code_agent 2026-10-25-rb item 27", + "at": 1792899400.154708 + }, + { + "usd": 0.009205, + "note": "mockmed code_agent 2026-10-25-rb item 28", + "at": 1792899416.541518 + }, + { + "usd": 0.009152, + "note": "mockmed code_agent 2026-10-25-rb item 29", + "at": 1792899432.968076 + }, + { + "usd": 0.011763, + "note": "mockmed code_agent 2026-10-25-rb item 30", + "at": 1792899456.0796869 + }, + { + "usd": 0.01042, + "note": "mockmed code_agent 2026-10-25-rb item 31", + "at": 1792899468.9014761 + }, + { + "usd": 0.009205, + "note": "mockmed code_agent 2026-10-25-rb item 32", + "at": 1792899488.583463 + }, + { + "usd": 0.009245, + "note": "mockmed code_agent 2026-10-25-rb item 33", + "at": 1792899522.400672 + }, + { + "usd": 0.009232, + "note": "mockmed code_agent 2026-10-25-rb item 34", + "at": 1792899556.2135549 + }, + { + "usd": 0.009192, + "note": "mockmed code_agent 2026-10-25-rb item 35", + "at": 1792899577.582444 + }, + { + "usd": 0.009205, + "note": "mockmed code_agent 2026-10-25-rb item 36", + "at": 1792899598.211006 + }, + { + "usd": 0.009205, + "note": "mockmed code_agent 2026-10-25-rb item 37", + "at": 1792899620.2384691 + }, + { + "usd": 0.009192, + "note": "mockmed code_agent 2026-10-25-rb item 38", + "at": 1792899649.332273 + }, + { + "usd": 0.009232, + "note": "mockmed code_agent 2026-10-25-rb item 39", + "at": 1792899670.250988 + }, + { + "usd": 0.007228, + "note": "mockmed code_agent 2026-10-25-rb item 40", + "at": 1792899692.904334 + }, + { + "usd": 0.038506, + "note": "openemr_referral code_agent 2026-10-25-rb item 1", + "at": 1792900146.4762108 + }, + { + "usd": 0.039309, + "note": "openemr_referral code_agent 2026-10-25-rb item 2", + "at": 1792900377.1045408 + }, + { + "usd": 0.043528, + "note": "openemr_referral code_agent 2026-10-25-rb item 3", + "at": 1792900562.074008 + }, + { + "usd": 0.035493, + "note": "openemr_referral code_agent 2026-10-25-rb item 4", + "at": 1792900722.0964842 + }, + { + "usd": 0.049921, + "note": "openemr_referral code_agent 2026-10-25-rb item 5", + "at": 1792900928.3733308 + }, + { + "usd": 0.039766, + "note": "openemr_referral code_agent 2026-10-25-rb-admit item 6", + "at": 1792901346.200465 } ] } diff --git a/packages/flow/benchmark/scoreboard/run.py b/packages/flow/benchmark/scoreboard/run.py index d2ba8a65f..0142c509e 100644 --- a/packages/flow/benchmark/scoreboard/run.py +++ b/packages/flow/benchmark/scoreboard/run.py @@ -841,6 +841,10 @@ def arm_code_agent(args: argparse.Namespace) -> int: and ledger is not None and ledger.finished and not any(w in text.lower() for w in FAILURE_WORDS) + # Amendment 11: the arm reports success only when its own + # read-back after the last save shows the save landed. + and bool(summary["readbacks"]) + and summary["readbacks"][-1]["outcome"] == "landed" ) ledger_doc = { "ledger": ledger.to_dict() if ledger is not None else None, diff --git a/packages/flow/benchmark/scoreboard/scoreboard.json b/packages/flow/benchmark/scoreboard/scoreboard.json index 4a3599bba..b5f95606a 100644 --- a/packages/flow/benchmark/scoreboard/scoreboard.json +++ b/packages/flow/benchmark/scoreboard/scoreboard.json @@ -1,7 +1,7 @@ { "protocol_version": "2026-10-18.1", "timed_load_max": 12.0, - "spend_usd": 12.543253, + "spend_usd": 12.975542, "tasks": { "mockmed": { "agent": { @@ -304,8 +304,8 @@ }, "program": { "label": "3. Compiled program driving the UI", - "items": 100, - "attempts": 102, + "items": 120, + "attempts": 124, "runs": { "2026-10-18-a": { "items": 100, @@ -358,19 +358,65 @@ "min": 82.45, "max": 532.93 } + }, + "2026-10-25-rb": { + "items": 20, + "attempts": 22, + "runs": [ + "2026-10-25-rb" + ], + "normal_items": 18, + "correct": 18, + "correct_rate": 1.0, + "silent_wrong": 0, + "safe_stops_normal": 0, + "fault_items": 2, + "fault_items_by_kind": { + "optimistic": { + "correct": 0, + "silent_wrong": 0, + "safe_stop": 1, + "flagged_wrong": 0 + }, + "wrong_record": { + "correct": 0, + "silent_wrong": 0, + "safe_stop": 0, + "flagged_wrong": 1 + } + }, + "planned_items": 20, + "done_correctly_all_items": 18, + "flagged_wrong": 1, + "paused": 0, + "model_calls_per_item": 0.0, + "tokens_per_item": 0, + "cost_usd_total": 0.0, + "cost_usd_per_item": 0.0, + "approvals_asked": 0, + "timed_correct_items": 0, + "item_s_median": null, + "item_s_p90": null, + "wall_s_median": null, + "wall_s_p90": null, + "items_per_hour": null, + "host_load_1m": { + "min": 38.6, + "max": 140.76 + } } }, - "normal_items": 47, - "correct": 47, + "normal_items": 65, + "correct": 65, "correct_rate": 1.0, "silent_wrong": 0, "safe_stops_normal": 0, - "fault_items": 53, + "fault_items": 55, "fault_items_by_kind": { "optimistic": { "correct": 0, "silent_wrong": 0, - "safe_stop": 1, + "safe_stop": 2, "flagged_wrong": 0 }, "ui_change": { @@ -383,12 +429,12 @@ "correct": 0, "silent_wrong": 0, "safe_stop": 0, - "flagged_wrong": 1 + "flagged_wrong": 2 } }, - "planned_items": 100, - "done_correctly_all_items": 98, - "flagged_wrong": 1, + "planned_items": 120, + "done_correctly_all_items": 116, + "flagged_wrong": 2, "paused": 0, "model_calls_per_item": 0.0, "tokens_per_item": 0, @@ -402,7 +448,7 @@ "wall_s_p90": null, "items_per_hour": null, "host_load_1m": { - "min": 82.45, + "min": 38.6, "max": 532.93 }, "run_meta": { @@ -422,10 +468,27 @@ "arm": "program", "program_from": "2026-10-18-a", "n": 100 + }, + "2026-10-25-rb": { + "run": "2026-10-25-rb", + "protocol_version": "2026-10-18.1", + "written_at": "2026-10-24T23:41:33-0400", + "machine": "arm64 Darwin 24.6.0", + "cpu_count": 12, + "python": "3.12.7", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + }, + "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", + "task": "mockmed", + "arm": "program", + "program_from": "code-2026-10-25-rb", + "n": 40 } }, "hand_check": { - "rows_not_correct": 4, + "rows_not_correct": 8, "checked": 0, "confirmed": 0, "items": [] @@ -437,13 +500,20 @@ "path": "/Users/abrichr/oa/src/OpenAdapt/.claude/worktrees/launch-demo-cua-to-program-8fa902/packages/flow/openadapt_flow", "git_head": "5881cb38b24b3d24402c45dcec86af4934a99cc2" } + }, + { + "run": "2026-10-25-rb", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + } } ] }, "code_agent": { "label": "7. Code-as-action authoring agent, first run", - "items": 5, - "attempts": 5, + "items": 25, + "attempts": 25, "runs": { "2026-10-19-timed": { "items": 5, @@ -477,24 +547,83 @@ "min": 11.78, "max": 14.29 } + }, + "2026-10-25-rb": { + "items": 20, + "attempts": 20, + "runs": [ + "2026-10-25-rb" + ], + "normal_items": 18, + "correct": 18, + "correct_rate": 1.0, + "silent_wrong": 1, + "safe_stops_normal": 0, + "fault_items": 2, + "fault_items_by_kind": { + "optimistic": { + "correct": 0, + "silent_wrong": 1, + "safe_stop": 0, + "flagged_wrong": 0 + }, + "wrong_record": { + "correct": 0, + "silent_wrong": 0, + "safe_stop": 0, + "flagged_wrong": 1 + } + }, + "planned_items": 20, + "done_correctly_all_items": 18, + "flagged_wrong": 1, + "paused": 0, + "model_calls_per_item": 4.05, + "tokens_per_item": 7656, + "cost_usd_total": 0.1858, + "cost_usd_per_item": 0.0093, + "approvals_asked": 18, + "timed_correct_items": 0, + "item_s_median": null, + "item_s_p90": null, + "wall_s_median": null, + "wall_s_p90": null, + "items_per_hour": null, + "host_load_1m": { + "min": 53.44, + "max": 131.43 + } } }, - "normal_items": 5, - "correct": 5, + "normal_items": 23, + "correct": 23, "correct_rate": 1.0, - "silent_wrong": 0, + "silent_wrong": 1, "safe_stops_normal": 0, - "fault_items": 0, - "fault_items_by_kind": {}, - "planned_items": 5, - "done_correctly_all_items": 5, - "flagged_wrong": 0, + "fault_items": 2, + "fault_items_by_kind": { + "optimistic": { + "correct": 0, + "silent_wrong": 1, + "safe_stop": 0, + "flagged_wrong": 0 + }, + "wrong_record": { + "correct": 0, + "silent_wrong": 0, + "safe_stop": 0, + "flagged_wrong": 1 + } + }, + "planned_items": 25, + "done_correctly_all_items": 23, + "flagged_wrong": 1, "paused": 0, - "model_calls_per_item": 5.0, - "tokens_per_item": 9913, - "cost_usd_total": 0.0522, - "cost_usd_per_item": 0.0104, - "approvals_asked": 5, + "model_calls_per_item": 4.24, + "tokens_per_item": 8108, + "cost_usd_total": 0.2379, + "cost_usd_per_item": 0.0095, + "approvals_asked": 23, "timed_correct_items": 1, "item_s_median": 8.17, "item_s_p90": 8.17, @@ -503,7 +632,7 @@ "items_per_hour": 368.0, "host_load_1m": { "min": 11.78, - "max": 14.29 + "max": 131.43 }, "run_meta": { "2026-10-19-timed": { @@ -554,10 +683,34 @@ ] } ] + }, + "2026-10-25-rb": { + "run": "2026-10-25-rb", + "protocol_version": "2026-10-18.1", + "written_at": "2026-10-24T23:34:22-0400", + "machine": "arm64 Darwin 24.6.0", + "cpu_count": 12, + "python": "3.12.7", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + }, + "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", + "task": "mockmed", + "arm": "code_agent", + "model": "claude-sonnet-5-5", + "n": 40, + "vision": "code", + "effort": "medium", + "max_cost_per_item": 0.3, + "on_screen": [ + "Jane Sample", + "MRN P1" + ] } }, "hand_check": { - "rows_not_correct": 0, + "rows_not_correct": 2, "checked": 0, "confirmed": 0, "items": [] @@ -570,6 +723,14 @@ "path": "/Users/abrichr/oa/worktrees/scoreboard-arm7/packages/flow/openadapt_flow", "git_head": "a5627748b9c9334fc2401c5636c57e7c858101d5" } + }, + { + "run": "2026-10-25-rb", + "model": "claude-sonnet-5-5", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + } } ] } @@ -1691,8 +1852,8 @@ }, "code_agent": { "label": "7. Code-as-action authoring agent, first run", - "items": 5, - "attempts": 5, + "items": 11, + "attempts": 11, "runs": { "2026-10-19-timed": { "items": 5, @@ -1726,23 +1887,89 @@ "min": 10.84, "max": 32.3 } + }, + "2026-10-25-rb": { + "items": 5, + "attempts": 5, + "runs": [ + "2026-10-25-rb" + ], + "normal_items": 5, + "correct": 5, + "correct_rate": 1.0, + "silent_wrong": 0, + "safe_stops_normal": 0, + "fault_items": 0, + "fault_items_by_kind": {}, + "planned_items": 5, + "done_correctly_all_items": 5, + "flagged_wrong": 0, + "paused": 0, + "model_calls_per_item": 10.6, + "tokens_per_item": 63381, + "cost_usd_total": 0.2068, + "cost_usd_per_item": 0.0414, + "approvals_asked": 0, + "timed_correct_items": 0, + "item_s_median": null, + "item_s_p90": null, + "wall_s_median": null, + "wall_s_p90": null, + "items_per_hour": null, + "host_load_1m": { + "min": 47.17, + "max": 256.05 + } + }, + "2026-10-25-rb-admit": { + "items": 1, + "attempts": 1, + "runs": [ + "2026-10-25-rb-admit" + ], + "normal_items": 1, + "correct": 1, + "correct_rate": 1.0, + "silent_wrong": 0, + "safe_stops_normal": 0, + "fault_items": 0, + "fault_items_by_kind": {}, + "planned_items": 1, + "done_correctly_all_items": 1, + "flagged_wrong": 0, + "paused": 0, + "model_calls_per_item": 11.0, + "tokens_per_item": 58722, + "cost_usd_total": 0.0398, + "cost_usd_per_item": 0.0398, + "approvals_asked": 0, + "timed_correct_items": 0, + "item_s_median": null, + "item_s_p90": null, + "wall_s_median": null, + "wall_s_p90": null, + "items_per_hour": null, + "host_load_1m": { + "min": 143.52, + "max": 143.52 + } } }, - "normal_items": 5, - "correct": 5, + "normal_items": 11, + "correct": 11, "correct_rate": 1.0, "silent_wrong": 0, "safe_stops_normal": 0, "fault_items": 0, "fault_items_by_kind": {}, - "planned_items": 5, - "done_correctly_all_items": 5, + "planned_items": 11, + "done_correctly_all_items": 11, "flagged_wrong": 0, "paused": 0, - "model_calls_per_item": 10.8, - "tokens_per_item": 65772, - "cost_usd_total": 0.2144, - "cost_usd_per_item": 0.0429, + "model_calls_per_item": 10.73, + "tokens_per_item": 64044, + "cost_usd_total": 0.4609, + "cost_usd_per_item": 0.0419, "approvals_asked": 0, "timed_correct_items": 0, "item_s_median": null, @@ -1752,7 +1979,7 @@ "items_per_hour": null, "host_load_1m": { "min": 10.84, - "max": 32.3 + "max": 256.05 }, "run_meta": { "2026-10-19-timed": { @@ -1801,6 +2028,52 @@ ] } ] + }, + "2026-10-25-rb": { + "run": "2026-10-25-rb", + "protocol_version": "2026-10-18.1", + "written_at": "2026-10-24T23:47:00-0400", + "machine": "arm64 Darwin 24.6.0", + "cpu_count": 12, + "python": "3.12.7", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + }, + "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", + "task": "openemr_referral", + "arm": "code_agent", + "model": "claude-sonnet-5-5", + "n": 5, + "vision": "code", + "effort": "medium", + "max_cost_per_item": 0.3, + "on_screen": [ + "Create New Patient" + ] + }, + "2026-10-25-rb-admit": { + "run": "2026-10-25-rb-admit", + "protocol_version": "2026-10-18.1", + "written_at": "2026-10-25T00:06:24-0400", + "machine": "arm64 Darwin 24.6.0", + "cpu_count": 12, + "python": "3.12.7", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + }, + "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", + "task": "openemr_referral", + "arm": "code_agent", + "model": "claude-sonnet-5-5", + "n": 6, + "vision": "code", + "effort": "medium", + "max_cost_per_item": 0.3, + "on_screen": [ + "Create New Patient" + ] } }, "hand_check": { @@ -1817,6 +2090,22 @@ "path": "/Users/abrichr/oa/worktrees/scoreboard-arm7/packages/flow/openadapt_flow", "git_head": "a5627748b9c9334fc2401c5636c57e7c858101d5" } + }, + { + "run": "2026-10-25-rb", + "model": "claude-sonnet-5-5", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + } + }, + { + "run": "2026-10-25-rb-admit", + "model": "claude-sonnet-5-5", + "engine": { + "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" + } } ] } diff --git a/packages/flow/docs/CODE_AS_ACTION.md b/packages/flow/docs/CODE_AS_ACTION.md index 6f595b450..5893a869b 100644 --- a/packages/flow/docs/CODE_AS_ACTION.md +++ b/packages/flow/docs/CODE_AS_ACTION.md @@ -1,8 +1,10 @@ # Code as action: the run-1 agent works through a terminal Before OpenAdapt compiles a task, an agent does it once -([COMPILE_AGENT.md](COMPILE_AGENT.md)). With `vision="code"` that agent does -not look at a screenshot after every click. It gets one terminal-style tool, +([COMPILE_AGENT.md](COMPILE_AGENT.md)). Code as action is the default for that +first run: `ClaudeAgent()` and `openadapt-flow compile-agent` use +`vision="code"` on `claude-sonnet-5-5`, and the MCP teach tools offer +`teach_run`. The agent does not look at a screenshot after every click. It gets one terminal-style tool, `run`, and writes short scripts: read the app's structure, act on elements by role and name, read the result back, keep notes in a working directory, and take a screenshot only when it asks for one. One model call can carry several @@ -18,19 +20,45 @@ from openadapt_flow.compile_agent import ClaudeAgent, compile_first_run result = compile_first_run( "Open the referral for Jane Sample, start a Triage encounter, and save this note: {note}", params={"note": "Follow up in two weeks"}, - agent=ClaudeAgent(vision="code", model="claude-sonnet-5-5"), + agent=ClaudeAgent(), # vision="code", claude-sonnet-5-5 url="http://localhost:8080/#tasks", verify="verify.yaml", expect={"patient_id": "p1"}, on_screen=["Jane Sample", "MRN P1"], ) ``` ```bash -openadapt-flow compile-agent "..." --vision code --url ... --verify verify.yaml ... +openadapt-flow compile-agent "..." --url ... --verify verify.yaml ... # --vision code is the default ``` `--agent scripts.json` with `{"scripts": ["tree", "click name=Open near=\"Jane Sample\""]}` runs the same tool from fixed scripts with no model calls (tests, CI, demos). +## Every save is read back + +After every action the save gate lets through, the harness reads the system of +record (the `verifier:` in `verify.yaml`) and compares it with the read taken +when run 1 started: + +- a change on any record the task does not name stops run 1 at once + (`readback_wrong_record`); nothing can be admitted; +- no change is told to the agent as "nothing was saved in the system of + record (whatever the screen says; do not report it as saved)", so a screen + that says "saved" can't stand in for a save; +- a change on the named record is told to the agent: "1 record(s) with + patient_id=p1, type=Triage saved". + +Each read-back is kept with run 1 (`run1["readbacks"]`). It runs for every +agent in the harness, not only the code agent, and on every surface. + +## In the MCP teach tools + +`openadapt-agent serve --teach VERIFY_YAML` adds `teach_run` on web apps: the +client's model sends a script, and the server runs it through the same tool. +When a person approves saves (`--teach-hold`), the script stops +before each save with `status: needs_person` and the rest of the script; the +client's form asks the person, and the approval covers that one line on the +app as it was when they answered. `teach_act` (one action per call) stays. + ## Why Three 2026 harness results point the same way. @@ -198,12 +226,32 @@ Each one wrongly refused, or failed to gate, a step on OpenEMR: ## Measured -**Scoreboard, arm 7** is the quotable source -(`benchmark/scoreboard`, protocol 2026-10-18.1, -[#1307](https://github.com/OpenAdaptAI/OpenAdapt/pull/1307)). Its rows are run -by the scoreboard's single runner under the load gate; until they are in -`scoreboard.json`, there is no arm 7 figure to quote. Trial runs made while -building the arm are set aside in `benchmark/scoreboard/superseded/`. +**Scoreboard, arm 7** is the quotable source (`benchmark/scoreboard`, +`python -m scoreboard.build` over committed rows; PROTOCOL.md Amendments 10 +and 11). + +Agent first runs against the program compiled from one of them, MockMed items +21 to 40 (20 items; item 30 is a save the server drops while the screen says +saved, item 40 a save that lands on another patient). claude-sonnet-5-5, +effort medium. All rows untimed: the host's load was above 12 at every item. + +| 20 items, final attempt | Agent first runs (code, read-back on every save) | Compiled program | +| --- | ---: | ---: | +| Correct, on the 18 normal items | 18 | 18 | +| Item 30, dropped save | `silent_wrong` by the word rule; a safe stop on hand review | `safe_stop` | +| Item 40, wrong patient | `flagged_wrong`: the read-back stopped run 1 | `flagged_wrong` | +| Silent wrong saves (hand review) | 0 | 0 | +| Model calls per item (median) | 4 | 0 | +| Cost per item | $0.0093 | $0 | +| Task seconds per correct item (median, untimed) | 6.6 | 5.3 | + +On item 30 the agent wrote that the save "did not take effect" because the +read-back showed nothing was saved, though the screen showed a "saved" +banner. The scoreboard's word rule did not catch that wording, so the row +keeps the rule's verdict and `HAND_REVIEW.md` next to it records the review. +From Amendment 11 on, arm 7 reports success only when its last read-back +shows the save landed. The program for arm 3 was compiled and admitted from +arm 7's item 21. **Exploratory measurement** (before the scoreboard; its own runner): diff --git a/packages/flow/docs/COMPILE_AGENT.md b/packages/flow/docs/COMPILE_AGENT.md index babd72ef8..e3588845b 100644 --- a/packages/flow/docs/COMPILE_AGENT.md +++ b/packages/flow/docs/COMPILE_AGENT.md @@ -112,14 +112,18 @@ except for the values the operator put in the task: the parameters, the `--expect` values and the `--on-screen` identity. The agent needs those to find the right row. -`--vision every_turn` is the baseline for comparison: Claude's computer -toolset (`computer_toolset_20260801`) with a screenshot after every action. - -`--vision code` gives the agent one terminal-style tool instead: it writes -short scripts that read the structure, act by role and name, read back, and -keep notes in a sandboxed working directory, with a screenshot only when it -asks. Every action becomes a structured step that is checked against the -compiled program. See [CODE_AS_ACTION.md](CODE_AS_ACTION.md). +The default is `--vision code` on `claude-sonnet-5-5`: the agent gets one +terminal-style tool and writes short scripts that read the structure, act by +role and name, read back, and keep notes in a sandboxed working directory, +with a screenshot only when it asks. Every action becomes a structured step +that is checked against the compiled program, and every save is read back +from the system of record before the agent goes on. See +[CODE_AS_ACTION.md](CODE_AS_ACTION.md). + +`--vision selective` gives the structural listing after every action (the +earlier default, on `claude-opus-5-5`). `--vision every_turn` is the baseline +for comparison: Claude's computer toolset (`computer_toolset_20260801`) with a +screenshot after every action. ### The save gate diff --git a/packages/flow/openadapt_flow/compile_agent/harness.py b/packages/flow/openadapt_flow/compile_agent/harness.py index 2197f1550..4e81195c8 100644 --- a/packages/flow/openadapt_flow/compile_agent/harness.py +++ b/packages/flow/openadapt_flow/compile_agent/harness.py @@ -138,6 +138,18 @@ def check() -> ReadBack: return check +def _expect_write(harness: Any) -> None: + """Tell the recorder the next action may write, so the event's after-read + of the system of record waits for the change (and the compiler binds the + write to this step).""" + + session = getattr(harness.computer, "session", None) + recorder = getattr(session, "_recorder", None) + expect = getattr(recorder, "expect_write", None) + if callable(expect): + expect(harness.readback_wait_s) + + def _read_back(harness: Any, control: str) -> Optional[ReadBack]: """Read the system of record after a save went through the gate; poll a little for a write that lands late. Raises ReadBackRefused on another @@ -399,6 +411,7 @@ def act( commit = self._hinted_commit(native, hint) if commit is not None: self._check_gate(*commit) + _expect_write(self) obs = self.computer.execute(native) self.stats.actions += 1 if commit is not None and self.has_page: @@ -809,6 +822,7 @@ def act(self, native: Mapping[str, Any], ref: Optional[str] = None) -> Observati commit = self.commit_target(native, ref) if commit is not None: self._check_gate(commit) + _expect_write(self) obs = self.computer.execute(native) self.stats.actions += 1 if commit is not None: diff --git a/packages/flow/openadapt_flow/recorder.py b/packages/flow/openadapt_flow/recorder.py index f27a6b6a9..9d7f59589 100644 --- a/packages/flow/openadapt_flow/recorder.py +++ b/packages/flow/openadapt_flow/recorder.py @@ -201,6 +201,8 @@ def __init__( # backend must never advertise an authoritative business-effect oracle # merely because the recorder can query a separate read-only source. self._system_of_record_reader = system_of_record_reader + #: Set by :meth:`expect_write` for the next event only. + self._write_wait_s = 0.0 self._settle_interval_s = settle_interval_s self._settle_stable_frames = settle_stable_frames self._settle_timeout_s = settle_timeout_s @@ -655,6 +657,13 @@ def event_count(self) -> int: return self._i + def expect_write(self, wait_s: float = 3.0) -> None: + """The next action may write to the system of record: its after-read + waits up to ``wait_s`` for the records to change (see :meth:`_commit`). + Changes nothing else about the event.""" + + self._write_wait_s = max(float(wait_s), 0.0) + # -- internals ----------------------------------------------------------- def _record(self, event: dict[str, Any], act: Callable[[], None]) -> None: @@ -906,6 +915,19 @@ def _commit( line[f"{key}_before"] = value if structural_after is None: structural_after = self._structural_state() + wait_s, self._write_wait_s = self._write_wait_s, 0.0 + if wait_s > 0 and "sor" in structural_before: + # The caller said this action may write (a save the gate let + # through). A system of record can show the write a moment + # after the screen settles; read again until it changes, so + # the write is bound to this event. + deadline = time.monotonic() + wait_s + while ( + structural_after.get("sor") == structural_before.get("sor") + and time.monotonic() < deadline + ): + time.sleep(0.25) + structural_after = self._structural_state() for key, value in structural_after.items(): line[f"{key}_after"] = value line["t"] = round(time.monotonic() - self._t0, 3) From fe7df6e912bd19cc6d43a90c41b99de45ea876d7 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 00:36:00 -0400 Subject: [PATCH 3/5] fix(flow): review fixes for read-back and teach_run approvals - each save is judged against a read just before it (a second save that writes nothing is told so); a record overwritten to match the task, or an edit shown as removed+created on a store without ids, is handled - a teach_run approval is bound to the save the person was shown and is re-read before it is used; comment lines are not shown as the action - a line that already saved and stops before a second save is not resent - typing on a native drop-down is not a save (a select no longer loops in hold mode); the recorder's write wait is taken back if the action fails Co-Authored-By: Claude Opus 5.5 --- .../flow/benchmark/scoreboard/PROTOCOL.md | 3 + .../compile_agent/code_action.py | 18 ++- .../openadapt_flow/compile_agent/driven.py | 53 ++++--- .../openadapt_flow/compile_agent/harness.py | 138 +++++++++++++----- packages/flow/tests/test_code_action.py | 75 ++++++++++ 5 files changed, 235 insertions(+), 52 deletions(-) diff --git a/packages/flow/benchmark/scoreboard/PROTOCOL.md b/packages/flow/benchmark/scoreboard/PROTOCOL.md index 3c01f331f..a76b07abe 100644 --- a/packages/flow/benchmark/scoreboard/PROTOCOL.md +++ b/packages/flow/benchmark/scoreboard/PROTOCOL.md @@ -372,6 +372,9 @@ program compiled from it: agent's evidence, not the scoreboard's: the scoreboard still reads the store on its own after each item. - Rows are timed only under the protocol's load rule. +- Arm 7 now follows the runner's `--no-faults` switch like arms 1 and 3 + (faults on by default). Its earlier runs were items 1 to 5, before any + fault item, so they are unaffected. ## Amendment 11 (2026-10-25, after Amendment 10's run): arm 7 reports what its read-back saw diff --git a/packages/flow/openadapt_flow/compile_agent/code_action.py b/packages/flow/openadapt_flow/compile_agent/code_action.py index 0c035b573..1195f2919 100644 --- a/packages/flow/openadapt_flow/compile_agent/code_action.py +++ b/packages/flow/openadapt_flow/compile_agent/code_action.py @@ -897,6 +897,17 @@ def run(self, script: str) -> ToolOutput: raise except NeedsPerson as exc: acted = self.harness.stats.actions > actions + if self.harness.stats.gate_checks > gates: + # A save on this line already went through; sending the + # line again would save twice. Stop without a pending line. + committed = True + chunks.append( + f"$ {line}\nstopped: a save on this line went through, and " + f"the next action ({self._out(exc.control)}) needs a " + "person's approval. Do not send this line again; read " + "the screen and go on from what it shows." + ) + break chunks.append( f"$ {line}\nwaiting: this line may save ({self._out(exc.control)}), " "and a person must approve it first. " @@ -1556,9 +1567,14 @@ def first_commit(self, script: str) -> Optional[str]: editable=editable if command != "click" else element.role in _FIELD_ROLES, commits=command == "select" and element.submits, ) - return self.harness.commit_target( + commit = self.harness.commit_target( {"kind": "click", "x": element.x, "y": element.y}, hint=hint ) + value = named.get("value", "") + if commit is None and command == "fill" and any(c in value for c in "\r\n"): + # The field is focused by then; only a line break can submit. + return "a typed line break" + return commit def _number(named: Mapping[str, str], key: str, default: int) -> int: diff --git a/packages/flow/openadapt_flow/compile_agent/driven.py b/packages/flow/openadapt_flow/compile_agent/driven.py index 46143a716..ab293b6e9 100644 --- a/packages/flow/openadapt_flow/compile_agent/driven.py +++ b/packages/flow/openadapt_flow/compile_agent/driven.py @@ -145,39 +145,60 @@ def _code_tool(self) -> Any: self.last_trace = self._tool.trace return self._tool - def script_binding(self, script: str) -> str: + @staticmethod + def _first_line(script: str) -> str: + return next( + ( + ln.strip() + for ln in str(script).splitlines() + if ln.strip() and not ln.strip().startswith("#") + ), + "", + ) + + def script_binding(self, script: str, commit: Optional[str]) -> str: """What a person's approval of a script covers: this run, the actions - performed so far, and the script's first line (the save).""" + performed so far, the script's first line, and the save it previews.""" - first = next((ln.strip() for ln in str(script).splitlines() if ln.strip()), "") actions = self.harness.stats.actions if self.harness is not None else 0 - return json.dumps([self.run_id, actions, first]) + return json.dumps([self.run_id, actions, self._first_line(script), commit]) def _run_script(self, command: _Command) -> dict[str, Any]: tool = self._code_tool() script = str(command.payload.get("script") or "") if command.kind == "run_preview": commit = tool.first_commit(script) - first = next((ln.strip() for ln in script.splitlines() if ln.strip()), "") return { "commit": commit, - "binding": self.script_binding(script) if commit else None, - "action": first[:200], + "binding": self.script_binding(script, commit) if commit else None, + "action": tool._out(self._first_line(script)[:200]), } held = command.payload.get("_held_by_person") - if held and held != self.script_binding(script): - raise ValueError( - "the app or the run changed after the person approved this save, " - "so nothing was done; call teach_run again" - ) + approved_commit: Optional[str] = None + if held: + # The approval covers the save the person was shown, on the app as + # it is now: read it again and require the same. + approved_commit = tool.first_commit(script) + if approved_commit is None or held != self.script_binding( + script, approved_commit + ): + raise ValueError( + "the app or the run changed after the person approved this " + "save, so nothing was done; call teach_run again" + ) person_approves = self.harness.gate.hold is not None approval = {"left": bool(held)} def approve(control: str) -> bool: if not person_approves: return True # the gate's own right-record check decides - # A person's approval covers one save, on the script's first line. - if approval["left"] and tool.line_number == 1: + # A person's approval covers one save: the one previewed, on the + # script's first line. + if ( + approval["left"] + and tool.line_number == 1 + and control == approved_commit + ): approval["left"] = False self._approved = control return True @@ -195,11 +216,9 @@ def approve(control: str) -> bool: if out.images: answer["output"] += "\n(screenshots are not shared here; use teach_screen)" if out.pending is not None: - rest = "\n".join(out.pending["rest"]) answer["pending"] = { "save": out.pending["control"], - "script": rest, - "binding": self.script_binding(rest), + "script": "\n".join(out.pending["rest"]), } return answer diff --git a/packages/flow/openadapt_flow/compile_agent/harness.py b/packages/flow/openadapt_flow/compile_agent/harness.py index 4e81195c8..8d970d26c 100644 --- a/packages/flow/openadapt_flow/compile_agent/harness.py +++ b/packages/flow/openadapt_flow/compile_agent/harness.py @@ -56,6 +56,7 @@ "SaveGate", "StaleRef", "SurfaceControl", + "RecordReadBack", "record_readback", "SurfaceHarness", "SurfaceObservation", @@ -93,61 +94,115 @@ def to_dict(self) -> dict[str, Any]: } -def record_readback( - read: Callable[[], Optional[list[dict[str, Any]]]], - expect: Mapping[str, Any], - baseline: Callable[[], Optional[list[dict[str, Any]]]], -) -> Callable[[], ReadBack]: - """A read-back for the harness: ``read`` the system of record and compare - it with ``baseline()`` (the read when run 1 started). A created or changed - record is the named one when it has every ``expect`` field with that value. - """ - - wanted = {str(k): str(v) for k, v in expect.items()} +class RecordReadBack: + """A read-back for the harness (see :func:`record_readback`). - def named(record: Mapping[str, Any]) -> bool: - return all(str(record.get(k)) == v for k, v in wanted.items()) + :meth:`mark` takes a read just before a save; calling the object then + reads again. A record that changed or disappeared since run 1 started and + is not the named record is the wrong record. Whether this save landed is + judged against the read :meth:`mark` took, so a second save that writes + nothing is told so. + """ - def check() -> ReadBack: + def __init__( + self, + read: Callable[[], Optional[list[dict[str, Any]]]], + expect: Mapping[str, Any], + baseline: Callable[[], Optional[list[dict[str, Any]]]], + ) -> None: + self.read = read + self.baseline = baseline + self.wanted = {str(k): str(v) for k, v in expect.items()} + self._before: Optional[list[dict[str, Any]]] = None + + def named(self, record: Mapping[str, Any]) -> bool: + return all(str(record.get(k)) == v for k, v in self.wanted.items()) + + def mark(self) -> None: + try: + self._before = self.read() + except Exception: # noqa: BLE001 - judged against the run's start then + self._before = None + + def _others(self, diff: Any) -> list[str]: + """Changes on records the task does not name.""" + + bad: list[str] = [] + named_created = [r for r in diff.created if self.named(r)] + for record in diff.created: + if not self.named(record): + bad.append("a record the task does not name was created") + for change in diff.changed: + # The named record before and after: overwriting another record + # until it matches the task is not the named record. + if not (self.named(change.before) and self.named(change.after)): + bad.append("a record the task does not name changed") + for record in diff.removed: + # A store without record ids shows an edit as removed + created. + if not (self.named(record) and named_created): + bad.append("a record disappeared") + return bad + + def __call__(self) -> ReadBack: from openadapt_flow.record_sor import diff_snapshots - before, after = baseline(), read() - if before is None or after is None: + start, after = self.baseline(), self.read() + if start is None or after is None: return ReadBack("unreadable", "the system of record could not be read") - diff = diff_snapshots(before, after) - touched = [*diff.created, *(c.after for c in diff.changed)] - if diff.removed: - return ReadBack("wrong_record", f"{len(diff.removed)} records disappeared") - others = [r for r in touched if not named(r)] - if others: + bad = self._others(diff_snapshots(start, after)) + if bad: return ReadBack( - "wrong_record", - f"{len(others)} record(s) the task does not name changed", + "wrong_record", f"{len(bad)} change(s) the task does not name: {bad[0]}" ) + before = self._before if self._before is not None else start + self._before = None + step = diff_snapshots(before, after) + touched = [*step.created, *(c.after for c in step.changed)] if not touched: return ReadBack("no_change", "nothing was saved in the system of record") fields = tuple(sorted({k for r in touched for k in r})) return ReadBack( "landed", f"{len(touched)} record(s) with " - + ", ".join(f"{k}={v}" for k, v in wanted.items()) + + ", ".join(f"{k}={v}" for k, v in self.wanted.items()) + " saved", fields, ) - return check +def record_readback( + read: Callable[[], Optional[list[dict[str, Any]]]], + expect: Mapping[str, Any], + baseline: Callable[[], Optional[list[dict[str, Any]]]], +) -> RecordReadBack: + """A read-back for the harness: ``read`` the system of record after each + save and compare it with ``baseline()`` (the read when run 1 started) and + with the read just before the save. A record is the named one when it has + every ``expect`` field with that value.""" + + return RecordReadBack(read, expect, baseline) -def _expect_write(harness: Any) -> None: + +def _expect_write(harness: Any, wait_s: Optional[float] = None) -> None: """Tell the recorder the next action may write, so the event's after-read of the system of record waits for the change (and the compiler binds the - write to this step).""" + write to this step). ``wait_s=0`` takes it back.""" session = getattr(harness.computer, "session", None) recorder = getattr(session, "_recorder", None) expect = getattr(recorder, "expect_write", None) if callable(expect): - expect(harness.readback_wait_s) + expect(harness.readback_wait_s if wait_s is None else wait_s) + + +def _before_write(harness: Any) -> None: + """Just before a save the gate let through: the read-back's before-read, + and the recorder's wait for the write.""" + + mark = getattr(harness.readback, "mark", None) + if callable(mark): + mark() + _expect_write(harness) def _read_back(harness: Any, control: str) -> Optional[ReadBack]: @@ -411,8 +466,13 @@ def act( commit = self._hinted_commit(native, hint) if commit is not None: self._check_gate(*commit) - _expect_write(self) - obs = self.computer.execute(native) + _before_write(self) + try: + obs = self.computer.execute(native) + except BaseException: + if commit is not None: + _expect_write(self, 0.0) + raise self.stats.actions += 1 if commit is not None and self.has_page: # Let the app react before the agent looks at the result. @@ -500,7 +560,12 @@ def _commit_target( elif op.kind == "type" and " " in (op.text or ""): # A typed space on a focused button presses it. focused = _observe.focused_element(self.page) or {} - if not focused.get("editable"): + # Typing on a native drop-down picks an option, unless its + # change submits the form. + on_select = focused.get("tag") == "select" and not focused.get( + "submits" + ) + if not focused.get("editable") and not on_select: label = str(focused.get("name") or "the focused element") return (f"typing a space on {label}", focused.get("scope")) elif op.kind == "hotkey": @@ -822,8 +887,13 @@ def act(self, native: Mapping[str, Any], ref: Optional[str] = None) -> Observati commit = self.commit_target(native, ref) if commit is not None: self._check_gate(commit) - _expect_write(self) - obs = self.computer.execute(native) + _before_write(self) + try: + obs = self.computer.execute(native) + except BaseException: + if commit is not None: + _expect_write(self, 0.0) + raise self.stats.actions += 1 if commit is not None: _read_back(self, commit) diff --git a/packages/flow/tests/test_code_action.py b/packages/flow/tests/test_code_action.py index a1f77d62c..f78d36d51 100644 --- a/packages/flow/tests/test_code_action.py +++ b/packages/flow/tests/test_code_action.py @@ -721,3 +721,78 @@ def test_record_readback_names_the_record_and_flags_any_other() -> None: assert check().outcome == "wrong_record" unreadable = record_readback(lambda: None, {"patient_id": "p1"}, lambda: before) assert unreadable().outcome == "unreadable" + + +def test_each_save_is_judged_against_the_read_just_before_it() -> None: + from openadapt_flow.compile_agent.harness import record_readback + + state = {"now": []} + check = record_readback(lambda: state["now"], {"patient_id": "p1"}, lambda: []) + check.mark() + state["now"] = [{"id": 1, "patient_id": "p1", "note": "a"}] + assert check().outcome == "landed" + # A second save that writes nothing is told so. + check.mark() + assert check().outcome == "no_change" + + +def test_an_edit_on_a_store_without_ids_is_the_named_record() -> None: + from openadapt_flow.compile_agent.harness import record_readback + + before = [{"patient_id": "p1", "note": "a"}, {"patient_id": "p2", "note": "b"}] + state = {"now": list(before)} + check = record_readback(lambda: state["now"], {"patient_id": "p1"}, lambda: before) + check.mark() + state["now"] = [{"patient_id": "p1", "note": "edited"}, before[1]] + assert check().outcome == "landed" + + +def test_overwriting_another_record_to_match_the_task_is_the_wrong_record() -> None: + from openadapt_flow.compile_agent.harness import record_readback + + before = [{"id": 2, "patient_id": "p2", "note": "b"}] + state = {"now": list(before)} + check = record_readback(lambda: state["now"], {"patient_id": "p1"}, lambda: before) + check.mark() + state["now"] = [{"id": 2, "patient_id": "p1", "note": "b"}] + assert check().outcome == "wrong_record" + + +def test_typing_on_a_native_drop_down_is_not_a_save() -> None: + class Page: + url = "http://app.test/" + + def evaluate(self, script: str, arg: Any = None) -> Any: + return { + "tag": "select", + "name": "Visit type", + "scope": "", + "editable": False, + } + + computer = SimpleNamespace( + dialect="generic", + page=Page(), + pointer=None, + executed=[], + screenshot=lambda: b"png", + ) + computer.execute = lambda native: computer.executed.append(native) + harness = Harness(computer, SaveGate(["MRN P1"])) # type: ignore[arg-type] + harness.act({"kind": "type", "text": "Office visit"}) + assert harness.stats.gate_checks == 0 and computer.executed + + +def test_a_failed_action_takes_back_the_recorders_wait(tmp_path: Path) -> None: + waits: list[float] = [] + tool, computer, harness = _tool(tmp_path) + computer.session._recorder = SimpleNamespace(expect_write=waits.append) + tool.run('click role=button name=Open near="Jane Sample"') + + def boom(native: Any) -> Any: + raise RuntimeError("no") + + computer.execute = boom + with pytest.raises(RuntimeError): + tool.run('click name="Save note"') + assert waits[-2:] == [harness.readback_wait_s, 0.0] From 4d74a9d0ea22d61c362b41bf08c45e32bddd1064 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 01:33:47 -0400 Subject: [PATCH 4/5] fix(flow): read-back review: only new records count, duplicates, late writes, failed reads - a record that existed when run 1 started never counts as the named one: any change to or removal of it stops the run; a second named record stops the run as a duplicate (counted, so stores without ids are covered) - a save with no new record is unconfirmed; the next save is refused if the first landed late; a failed read stops the run, and run 1 does not start without a starting read - a save that raises is still read back, and the result is in the error text - landed requires every input value in the new record - the tool waits briefly for a target that is still loading - wording: the read-back covers saves the gate detects; the wrong-patient write landed; the word rule's silent_wrong is stated with HAND_REVIEW.md - data: no local paths in the new runs; meta.json records the code each run used; llms-full.txt keeps only this PR's change; PROTOCOL Amendment 13 Co-Authored-By: Claude Opus 5.5 --- CAPABILITIES.md | 6 +- llms-full.txt | 7 +- packages/agent/README.md | 2 +- .../flow/benchmark/scoreboard/PROTOCOL.md | 14 + .../2026-10-25-rb/ledgers/item-040.json | 2 +- .../code_agent/2026-10-25-rb/meta.json | 8 +- .../mockmed/program/2026-10-25-rb/meta.json | 8 +- .../code_agent/2026-10-25-rb-admit/meta.json | 8 +- .../code_agent/2026-10-25-rb/meta.json | 8 +- packages/flow/benchmark/scoreboard/run.py | 3 + packages/flow/benchmark/scoreboard/schema.py | 8 +- packages/flow/docs/CODE_AS_ACTION.md | 45 ++- packages/flow/docs/COMPILE_AGENT.md | 4 +- packages/flow/openadapt_flow/agent/teach.py | 7 +- .../compile_agent/code_action.py | 28 +- .../openadapt_flow/compile_agent/harness.py | 278 ++++++++++++++---- .../openadapt_flow/compile_agent/pipeline.py | 10 +- .../tests/e2e/test_code_agent_default_e2e.py | 14 +- packages/flow/tests/test_code_action.py | 198 +++++++++---- 19 files changed, 490 insertions(+), 168 deletions(-) diff --git a/CAPABILITIES.md b/CAPABILITIES.md index e65713660..a148313a9 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -20,7 +20,7 @@ shipping in the next release. records a task, or a computer-use agent does it once and OpenAdapt compiles the verified run into a program (`openadapt_flow.jit`, released; prompt-to-program `compile-agent`, landing; the agent acts through code by default, not screenshots, and every - save in its run is read back, + save the gate detects in its run is read back, [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)). Agents can drive the recording on web, native Windows, RDP and Citrix. An agent connected over MCP can teach itself a new workflow and then call it as a tool @@ -93,7 +93,7 @@ shipping in the next release. | Repairs itself when the app changes: governed repair, agent-demonstrated fixes, fleet patches, and self-repair probed up to the save. | [REPAIR_LIFECYCLE.md](packages/flow/docs/REPAIR_LIFECYCLE.md) | | Repairs a change that needs new steps, such as a new required dialog or a control moved into a menu: up to three new steps, each with its own record check, probed up to the save before one person approves. | [SELF_REPAIR.md](packages/flow/docs/SELF_REPAIR.md), [#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325) | | Compiles a computer-use agent's verified run, with no human demonstration needed. | [JIT.md](packages/flow/docs/JIT.md) | -| The agent that does a task once acts through code by default, not a screenshot-and-click loop: it reads the page or accessibility tree, acts by role and name, keeps notes with a sandboxed read-only shell, and asks for a screenshot only when it needs one. Every save in that run is read back from the system of record before the agent goes on. Scoreboard arm 7, MockMed items 21 to 40: 18 of 18 normal items correct at $0.0093 an item, the wrong-patient save stopped by the read-back, no silent wrong save on hand review; the program compiled from it: 18 of 18 at $0. | `CODE_AS_ACTION.md` in [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305) | +| The agent that does a task once acts through code by default, not a screenshot-and-click loop: it reads the page or accessibility tree, acts by role and name, keeps notes with a sandboxed read-only shell, and asks for a screenshot only when it needs one. Every save the save gate detects is read back from the system of record before the agent goes on. Scoreboard arm 7, MockMed items 21 to 40: 18 of 18 normal items correct at $0.0093 an item; the wrong-patient save landed and the read-back then stopped the run; the scoreboard counts 1 silent wrong save by its word rule, a safe stop on hand review ([HAND_REVIEW.md](packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/HAND_REVIEW.md)). The program compiled from it: 18 of 18 at $0. | `CODE_AS_ACTION.md` in [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305) | | Binds each approval to the exact screenshot and form state, and re-reads the form before saving. | [APPROVAL_MODES.md](packages/flow/docs/APPROVAL_MODES.md) | | Has a written threat model for prompt injection against the agent that teaches or repairs a program, with tests. | [THREAT_MODEL.md](packages/flow/docs/THREAT_MODEL.md) | | Fills inputs from a referral PDF, fax image or email, and holds back anything unclear for a person. | `intake`, [INTAKE.md](packages/flow/docs/INTAKE.md) | @@ -236,7 +236,7 @@ improvising or retrying when something is off. | --- | --- | --- | --- | | Compile a computer-use agent's successful run into a program (JIT); later runs make no model calls | Released | Synthetic (a recorded Claude run on MockMed) | `openadapt_flow.jit`, [JIT.md](packages/flow/docs/JIT.md) | | Reuse a compiled program only for the record check and app environment it was admitted under (a changed record check or a new app version at the same origin is refused) | On main ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Unit tests | `TaskSignature(record_check=, environment=)`, `WebApp(environment_id=)`, [JIT.md](packages/flow/docs/JIT.md) | -| Code as action is the default run-1 agent (compile-agent, and `teach_run` in the MCP teach tools): one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program, and every save is read back from the system of record | On main ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | +| Code as action is the default run-1 agent (compile-agent, and `teach_run` in the MCP teach tools): one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program, and every save the gate detects is read back from the system of record | On main ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | | Admit a draft by a person, an admitter key, or an agent under a person-signed policy, including self-admission when the policy allows it | Released | Unit tests | [JIT_ADMISSION.md](packages/flow/docs/JIT_ADMISSION.md) | | MCP server: your agent decides what to enter, OpenAdapt makes the entry and checks it | Released | Synthetic sandbox | `openadapt-agent`, [packages/agent](packages/agent/README.md) | | Teach over MCP: an agent does a new task once through OpenAdapt; the run is read back, compiled and admitted (a person confirms the record check; the agent admits only under a person-signed policy), and becomes a tool it calls with no model calls. Web, and Windows, macOS, Linux, RDP and Citrix with a person approving each save | On main ([#1335](https://github.com/OpenAdaptAI/OpenAdapt/pull/1335)) | Synthetic (MockMed over MCP, and through the real RDP and Citrix backends) | `openadapt-agent serve --teach`, [packages/agent](packages/agent/README.md#teach-a-new-workflow-from-your-agent) | diff --git a/llms-full.txt b/llms-full.txt index e65a56d52..682e49be6 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -417,7 +417,7 @@ shipping in the next release. records a task, or a computer-use agent does it once and OpenAdapt compiles the verified run into a program (`openadapt_flow.jit`, released; prompt-to-program `compile-agent`, landing; the agent acts through code by default, not screenshots, and every - save in its run is read back, + save the gate detects in its run is read back, [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)). Agents can drive the recording on web, native Windows, RDP and Citrix. An agent connected over MCP can teach itself a new workflow and then call it as a tool @@ -490,7 +490,7 @@ shipping in the next release. | Repairs itself when the app changes: governed repair, agent-demonstrated fixes, fleet patches, and self-repair probed up to the save. | [REPAIR_LIFECYCLE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/REPAIR_LIFECYCLE.md) | | Repairs a change that needs new steps, such as a new required dialog or a control moved into a menu: up to three new steps, each with its own record check, probed up to the save before one person approves. | [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md), [#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325) | | Compiles a computer-use agent's verified run, with no human demonstration needed. | [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | -| The agent that does a task once acts through code by default, not a screenshot-and-click loop: it reads the page or accessibility tree, acts by role and name, keeps notes with a sandboxed read-only shell, and asks for a screenshot only when it needs one. Every save in that run is read back from the system of record before the agent goes on. Scoreboard arm 7, MockMed items 21 to 40: 18 of 18 normal items correct at $0.0093 an item, the wrong-patient save stopped by the read-back, no silent wrong save on hand review; the program compiled from it: 18 of 18 at $0. | `CODE_AS_ACTION.md` in [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305) | +| The agent that does a task once acts through code by default, not a screenshot-and-click loop: it reads the page or accessibility tree, acts by role and name, keeps notes with a sandboxed read-only shell, and asks for a screenshot only when it needs one. Every save the save gate detects is read back from the system of record before the agent goes on. Scoreboard arm 7, MockMed items 21 to 40: 18 of 18 normal items correct at $0.0093 an item; the wrong-patient save landed and the read-back then stopped the run; the scoreboard counts 1 silent wrong save by its word rule, a safe stop on hand review ([HAND_REVIEW.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/HAND_REVIEW.md)). The program compiled from it: 18 of 18 at $0. | `CODE_AS_ACTION.md` in [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305) | | Binds each approval to the exact screenshot and form state, and re-reads the form before saving. | [APPROVAL_MODES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/APPROVAL_MODES.md) | | Has a written threat model for prompt injection against the agent that teaches or repairs a program, with tests. | [THREAT_MODEL.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/THREAT_MODEL.md) | | Fills inputs from a referral PDF, fax image or email, and holds back anything unclear for a person. | `intake`, [INTAKE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/INTAKE.md) | @@ -564,7 +564,6 @@ improvising or retrying when something is off. | Learn one program with branches from two or more recordings, or say which recordings are missing | Released | Synthetic | `openadapt-flow induce`, `disambiguate`; [CAPABILITIES.md (flow)](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/CAPABILITIES.md) | | Learn a guarded branch from several halts fixed the same way | Released | Synthetic | `openadapt-flow teach-branch`, `learning/branch_teach.py` | | Run a program over a worklist | Released | Real app | `openadapt-flow for-each` | -| Learn a worklist loop from recordings that search a record, open it, work it and go back to the list, even when the last row skips the return or rows differ (a field some rows lack, a write some rows get behind a screen check a person accepts). Each pass selects the row's own record by its values, verifies it, and checks that row's write. Certified and governed runs refuse it until qualification covers it | On main ([#1435](https://github.com/OpenAdaptAI/OpenAdapt/pull/1435)) | Synthetic (held-out rows run through the interpreter against an in-memory app) | `openadapt-flow induce`; `Step.target_param` | | Typed business decisions inside a run | Released | Synthetic | `business_decision` states, `openadapt-flow business-decisions` | | Watch many real runs in the background and mine candidate branches (shadow mode) | On main ([#1212](https://github.com/OpenAdaptAI/OpenAdapt/pull/1212)) | Synthetic | [SHADOW_MINING.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/capture/docs/SHADOW_MINING.md), [LEARN_FROM_MANY_RUNS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/LEARN_FROM_MANY_RUNS.md) | | Fill a workflow's inputs from a referral PDF, fax image or email | Released | Synthetic | `openadapt-flow intake`, [INTAKE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/INTAKE.md) | @@ -633,7 +632,7 @@ improvising or retrying when something is off. | --- | --- | --- | --- | | Compile a computer-use agent's successful run into a program (JIT); later runs make no model calls | Released | Synthetic (a recorded Claude run on MockMed) | `openadapt_flow.jit`, [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | | Reuse a compiled program only for the record check and app environment it was admitted under (a changed record check or a new app version at the same origin is refused) | On main ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Unit tests | `TaskSignature(record_check=, environment=)`, `WebApp(environment_id=)`, [JIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT.md) | -| Code as action is the default run-1 agent (compile-agent, and `teach_run` in the MCP teach tools): one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program, and every save is read back from the system of record | On main ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | +| Code as action is the default run-1 agent (compile-agent, and `teach_run` in the MCP teach tools): one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program, and every save the gate detects is read back from the system of record | On main ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | | Admit a draft by a person, an admitter key, or an agent under a person-signed policy, including self-admission when the policy allows it | Released | Unit tests | [JIT_ADMISSION.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT_ADMISSION.md) | | MCP server: your agent decides what to enter, OpenAdapt makes the entry and checks it | Released | Synthetic sandbox | `openadapt-agent`, [packages/agent](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/agent/README.md) | | Teach over MCP: an agent does a new task once through OpenAdapt; the run is read back, compiled and admitted (a person confirms the record check; the agent admits only under a person-signed policy), and becomes a tool it calls with no model calls. Web, and Windows, macOS, Linux, RDP and Citrix with a person approving each save | On main ([#1335](https://github.com/OpenAdaptAI/OpenAdapt/pull/1335)) | Synthetic (MockMed over MCP, and through the real RDP and Citrix backends) | `openadapt-agent serve --teach`, [packages/agent](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/agent/README.md#teach-a-new-workflow-from-your-agent) | diff --git a/packages/agent/README.md b/packages/agent/README.md index ba74b7fb9..fbf1f8831 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -411,7 +411,7 @@ claude mcp add --scope user openadapt-teach -- uvx --python 3.12 --from 'openada Your agent then: 1. Calls `teach_start` with the task (`{note}` for each input), this run's `inputs`, the `record_fields` the saved record must have (such as its patient id), and `on_screen`, the record's identity as the screen shows it (a name and an MRN). -2. Calls `teach_run` with short scripts (read the page, act on elements by role and name, read the result back; web apps), or `teach_observe` and `teach_act` one action at a time, until the task is done. After every save OpenAdapt reads the app's records back and tells the agent what landed; a save on a record the task does not name ends the run. On the web, `teach_observe` lists the page as short lines with a ref for each control, and before every save OpenAdapt checks that the record's identity is shown next to the control. A save on the wrong record is refused and ends the run. +2. Calls `teach_run` with short scripts (read the page, act on elements by role and name, read the result back; web apps), or `teach_observe` and `teach_act` one action at a time, until the task is done. After every save the gate detects, OpenAdapt reads the app's records back and tells the agent what landed; a write on a record the task does not name ends the run after it lands. On the web, `teach_observe` lists the page as short lines with a ref for each control, and before every save OpenAdapt checks that the record's identity is shown next to the control. A save on the wrong record is refused and ends the run. 3. Calls `teach_finish`. OpenAdapt compiles the run and reads the app's records: exactly one record must have appeared, with every `record_fields` value and every input. The result is the draft: its steps, its inputs, the proposed record check, and what still needs a person. 4. Calls `teach_confirm_checks`. Your MCP client shows a person the proposed check, and only their answer there, with their name, confirms it. Your agent has no argument that confirms it. 5. Calls `teach_admit` with `canary_inputs`, the values each canary saves. A canary is a real save on the same record, so use values that are safe there. diff --git a/packages/flow/benchmark/scoreboard/PROTOCOL.md b/packages/flow/benchmark/scoreboard/PROTOCOL.md index a76b07abe..f06a7550e 100644 --- a/packages/flow/benchmark/scoreboard/PROTOCOL.md +++ b/packages/flow/benchmark/scoreboard/PROTOCOL.md @@ -411,3 +411,17 @@ Amendment 11's success rule applies. duplicate-check dialog and writes nothing, so it has no read-back. Not fixed here: binding a "nothing was written here" read-back to such a step changes what admission accepts. + +## Amendment 13 (2026-10-25, after review): the read-back's rules changed after Amendments 10 to 12's runs + +After review, run 1's read-back counts only a record created during run 1 as +the named one (any change to or removal of a starting record stops the run), +stops on a second named record (a duplicate, counted even without record +ids), treats a save with no new record as unconfirmed and stops the next save +if it lands late, stops when a read fails, and calls a save `landed` only +when the new record holds every input value (`compile_agent.harness. +RecordReadBack`). Amendments 10 to 12's runs used the earlier rules. Not +re-run; by the rules, their verdicts are the same: every item made one save, +on MockMed's empty store or next to OpenEMR records it left untouched; item +40's save created a record on another patient (wrong record either way) and +item 30's showed no new record (unconfirmed, the earlier "no change"). diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-040.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-040.json index fe49ca4ba..3b05da9c0 100644 --- a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-040.json +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/ledgers/item-040.json @@ -97,7 +97,7 @@ ], "result": { "record_status": "capture_failed", - "detail": "ReadBackRefused: after 'Save Encounter', the read-back shows 1 record(s) the task does not name changed; run 1 stopped (capture kept at /Users/abrichr/.cache/openadapt-scoreboard/mockmed/code_agent/2026-10-25-rb/jit-cache/item-040/mockmed.add_note-48f60d6be84e/refused-20261025T034130714941000)", + "detail": "ReadBackRefused: after 'Save Encounter', the read-back shows 1 record(s) the task does not name changed; run 1 stopped (capture kept at /mockmed/code_agent/2026-10-25-rb/jit-cache/item-040/mockmed.add_note-48f60d6be84e/refused-20261025T034130714941000)", "gates": [ { "control": "Save Encounter", diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/meta.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/meta.json index 82e5d5774..803c25553 100644 --- a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/meta.json +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/meta.json @@ -5,7 +5,7 @@ "cpu_count": 12, "python": "3.12.7", "engine": { - "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "path": "packages/flow/openadapt_flow", "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" }, "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", @@ -19,5 +19,9 @@ "on_screen": [ "Jane Sample", "MRN P1" - ] + ], + "code_at_run": { + "commit": "bedf5439f462acd296a8056acdad8a58f4db2b31", + "note": "run on bedf5439 with no uncommitted engine or runner change" + } } diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/meta.json b/packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/meta.json index 144177292..f9acadb5d 100644 --- a/packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/meta.json +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/program/2026-10-25-rb/meta.json @@ -5,12 +5,16 @@ "cpu_count": 12, "python": "3.12.7", "engine": { - "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "path": "packages/flow/openadapt_flow", "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" }, "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", "task": "mockmed", "arm": "program", "program_from": "code-2026-10-25-rb", - "n": 40 + "n": 40, + "code_at_run": { + "commit": "bedf5439f462acd296a8056acdad8a58f4db2b31", + "note": "run on bedf5439 with no uncommitted engine or runner change" + } } diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/meta.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/meta.json index 74ecfbcd3..229d96422 100644 --- a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/meta.json +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb-admit/meta.json @@ -5,7 +5,7 @@ "cpu_count": 12, "python": "3.12.7", "engine": { - "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "path": "packages/flow/openadapt_flow", "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" }, "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", @@ -18,5 +18,9 @@ "max_cost_per_item": 0.3, "on_screen": [ "Create New Patient" - ] + ], + "code_at_run": { + "commit": "7525b6854", + "note": "run on bedf5439 plus Recorder.expect_write and the Amendment 11 rule, uncommitted at the time and committed in 7525b685" + } } diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/meta.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/meta.json index 19530df25..765031bb6 100644 --- a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/meta.json +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-25-rb/meta.json @@ -5,7 +5,7 @@ "cpu_count": 12, "python": "3.12.7", "engine": { - "path": "/Users/abrichr/oa/worktrees/code-agent-default/packages/flow/openadapt_flow", + "path": "packages/flow/openadapt_flow", "git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31" }, "benchmark_git_head": "bedf5439f462acd296a8056acdad8a58f4db2b31", @@ -18,5 +18,9 @@ "max_cost_per_item": 0.3, "on_screen": [ "Create New Patient" - ] + ], + "code_at_run": { + "commit": "7525b6854", + "note": "run on bedf5439 plus the Amendment 11 success rule in run.py, uncommitted at the time and committed in 7525b685" + } } diff --git a/packages/flow/benchmark/scoreboard/run.py b/packages/flow/benchmark/scoreboard/run.py index 0142c509e..2b9942db2 100644 --- a/packages/flow/benchmark/scoreboard/run.py +++ b/packages/flow/benchmark/scoreboard/run.py @@ -760,6 +760,7 @@ def arm_code_agent(args: argparse.Namespace) -> int: sess.server.records, {"patient_id": "p1", "type": "Triage"}, lambda: [], + values=values, ) fault = it.fault else: @@ -773,6 +774,8 @@ def arm_code_agent(args: argparse.Namespace) -> int: task.patients, {"email": values["email"], "lname": values["lname"]}, lambda: before, + # The state is typed as its label and stored as its code. + values={k: v for k, v in values.items() if k != "state_label"}, ) fault = "none" inner = ClaudeAgent( diff --git a/packages/flow/benchmark/scoreboard/schema.py b/packages/flow/benchmark/scoreboard/schema.py index 7ffafee7d..bdbf32771 100644 --- a/packages/flow/benchmark/scoreboard/schema.py +++ b/packages/flow/benchmark/scoreboard/schema.py @@ -127,7 +127,13 @@ def engine() -> dict[str, str]: import openadapt_flow root = Path(openadapt_flow.__file__).resolve().parent - return {"path": str(root), "git_head": git_head(root)} + # Relative to the repository, so no local path lands in committed data. + repo = HERE.parents[3] + try: + shown = str(root.relative_to(repo)) + except ValueError: + shown = root.name + return {"path": shown, "git_head": git_head(root)} def write_meta(out: Path, **extra: Any) -> None: diff --git a/packages/flow/docs/CODE_AS_ACTION.md b/packages/flow/docs/CODE_AS_ACTION.md index 5893a869b..63be5b25a 100644 --- a/packages/flow/docs/CODE_AS_ACTION.md +++ b/packages/flow/docs/CODE_AS_ACTION.md @@ -33,19 +33,30 @@ openadapt-flow compile-agent "..." --url ... --verify verify.yaml ... # --visi `--agent scripts.json` with `{"scripts": ["tree", "click name=Open near=\"Jane Sample\""]}` runs the same tool from fixed scripts with no model calls (tests, CI, demos). -## Every save is read back - -After every action the save gate lets through, the harness reads the system of -record (the `verifier:` in `verify.yaml`) and compares it with the read taken -when run 1 started: - -- a change on any record the task does not name stops run 1 at once - (`readback_wrong_record`); nothing can be admitted; -- no change is told to the agent as "nothing was saved in the system of - record (whatever the screen says; do not report it as saved)", so a screen - that says "saved" can't stand in for a save; -- a change on the named record is told to the agent: "1 record(s) with - patient_id=p1, type=Triage saved". +## Every save the gate detects is read back + +The read-back covers the actions the save gate treats as saves (see the gate +in [COMPILE_AGENT.md](COMPILE_AGENT.md)); a write the gate does not detect is +caught only by the record check after run 1. After each of those actions the +harness reads the system of record (the `verifier:` in `verify.yaml`) and +compares it with the read taken when run 1 started and with a read taken just +before the save: + +- only a record created during run 1 can be the one the task names. A new + record the task does not name, or any change to or removal of a record that + was there when run 1 started, stops run 1 (`readback_wrong_record`). The + wrong write has landed by then; the read-back stops anything after it, and + nothing can be admitted; +- a second new record the task names stops run 1 as a duplicate, counted even + in a store without record ids; +- no new record is told to the agent as "unconfirmed ... Do not save again". + Before the next save a fresh read must show nothing changed since; if the + first save landed late, run 1 stops before the second (`readback_refused`); +- a new named record that lacks an input value is `incomplete`; one with every + input is `landed`; +- a read that fails stops run 1, and run 1 does not start when the starting + read fails. A save that raises is still read back, and the result is added + to the error the agent sees. Each read-back is kept with run 1 (`run1["readbacks"]`). It runs for every agent in the harness, not only the code agent, and on every surface. @@ -239,8 +250,8 @@ effort medium. All rows untimed: the host's load was above 12 at every item. | --- | ---: | ---: | | Correct, on the 18 normal items | 18 | 18 | | Item 30, dropped save | `silent_wrong` by the word rule; a safe stop on hand review | `safe_stop` | -| Item 40, wrong patient | `flagged_wrong`: the read-back stopped run 1 | `flagged_wrong` | -| Silent wrong saves (hand review) | 0 | 0 | +| Item 40, wrong patient | `flagged_wrong`: the save landed on the other patient, and the read-back stopped run 1 | `flagged_wrong` | +| Silent wrong saves | 1 by the scoreboard's word rule (item 30), 0 on hand review | 0 | | Model calls per item (median) | 4 | 0 | | Cost per item | $0.0093 | $0 | | Task seconds per correct item (median, untimed) | 6.6 | 5.3 | @@ -248,7 +259,9 @@ effort medium. All rows untimed: the host's load was above 12 at every item. On item 30 the agent wrote that the save "did not take effect" because the read-back showed nothing was saved, though the screen showed a "saved" banner. The scoreboard's word rule did not catch that wording, so the row -keeps the rule's verdict and `HAND_REVIEW.md` next to it records the review. +keeps the rule's verdict and +[HAND_REVIEW.md](../benchmark/scoreboard/raw/mockmed/code_agent/2026-10-25-rb/HAND_REVIEW.md) +records the review. From Amendment 11 on, arm 7 reports success only when its last read-back shows the save landed. The program for arm 3 was compiled and admitted from arm 7's item 21. diff --git a/packages/flow/docs/COMPILE_AGENT.md b/packages/flow/docs/COMPILE_AGENT.md index e3588845b..43d187fc8 100644 --- a/packages/flow/docs/COMPILE_AGENT.md +++ b/packages/flow/docs/COMPILE_AGENT.md @@ -116,8 +116,8 @@ The default is `--vision code` on `claude-sonnet-5-5`: the agent gets one terminal-style tool and writes short scripts that read the structure, act by role and name, read back, and keep notes in a sandboxed working directory, with a screenshot only when it asks. Every action becomes a structured step -that is checked against the compiled program, and every save is read back -from the system of record before the agent goes on. See +that is checked against the compiled program, and every save the gate +detects is read back from the system of record before the agent goes on. See [CODE_AS_ACTION.md](CODE_AS_ACTION.md). `--vision selective` gives the structural listing after every action (the diff --git a/packages/flow/openadapt_flow/agent/teach.py b/packages/flow/openadapt_flow/agent/teach.py index 0321cd00d..bd26a4c4b 100644 --- a/packages/flow/openadapt_flow/agent/teach.py +++ b/packages/flow/openadapt_flow/agent/teach.py @@ -373,9 +373,10 @@ def list_tool_specs(self) -> list[ToolSpec]: '(click role=button name="Save" near="Jane Sample"); ' "sh for a read-only shell over files you saved. " "Lines run in order and stop at the first error or " - "after a save. After every save OpenAdapt reads the " - "app's records back and tells you what landed; a " - "save on a record the task does not name ends run 1. " + "after a save. After every save the gate detects, " + "OpenAdapt reads the app's records back and tells you " + "what landed; a write on a record the task does not " + "name ends run 1. " + ( "A person approves each save first: the script " "stops before it with status needs_person, and " diff --git a/packages/flow/openadapt_flow/compile_agent/code_action.py b/packages/flow/openadapt_flow/compile_agent/code_action.py index 1195f2919..a5084c824 100644 --- a/packages/flow/openadapt_flow/compile_agent/code_action.py +++ b/packages/flow/openadapt_flow/compile_agent/code_action.py @@ -114,6 +114,8 @@ } _MAX_WAIT_MS = 2000 +#: How long a target named by role and name may take to appear on the web. +_TARGET_WAIT_S = 3.0 #: Roles a click only focuses or opens: it commits nothing by itself. _FIELD_ROLES = ("textbox", "searchbox", "spinbutton", "combobox", "listbox") _MAX_LINES = 40 @@ -765,7 +767,15 @@ def seen_before(index: int) -> set[str]: "commits": len(commits), "readbacks": { outcome: sum(1 for st in self.steps if st.get("readback") == outcome) - for outcome in ("landed", "no_change", "unreadable") + for outcome in ( + "landed", + "incomplete", + "unconfirmed", + "wrong_record", + "duplicate", + "landed_late", + "unreadable", + ) if any(st.get("readback") == outcome for st in self.steps) }, "identity_read_before_commit": ( @@ -890,6 +900,7 @@ def run(self, script: str) -> ToolOutput: self.trace.lines += 1 gates = self.harness.stats.gate_checks actions = self.harness.stats.actions + readbacks = len(getattr(self.harness, "readback_log", [])) try: text, image = self._line(line) except ScopeViolation: @@ -927,6 +938,9 @@ def run(self, script: str) -> ToolOutput: except (ScriptError, ShellRefused, UnsupportedAction, LookupError) as exc: self.trace.errors += 1 chunks.append(f"$ {line}\nerror: {self._out(str(exc))}") + log = getattr(self.harness, "readback_log", []) + if len(log) > readbacks: + chunks.append("read back after the error: " + log[-1]["detail"]) if self.harness.stats.gate_checks > gates: committed = True chunks.append( @@ -1132,6 +1146,14 @@ def _target(self, positional: Sequence[str], named: Mapping[str, str]) -> Elemen raise ScriptError("a target needs @N, or name= (with role= or near=)") self._read() hits = self._match(described, exact=True) + if not hits and self.surface.kind == "dom": + # A page that is still loading: look again for a moment before + # calling the target missing. + deadline = time.monotonic() + _TARGET_WAIT_S + while not hits and time.monotonic() < deadline: + time.sleep(0.25) + self._read() + hits = self._match(described, exact=True) if not hits and self.surface.kind == "tree" and described.get("role"): located = self.surface.locate(described["role"], described["name"]) if located is not None: @@ -1499,8 +1521,8 @@ def _finish_step( back = log[-1] step["readback"] = back["outcome"] text += "\nread back: " + back["detail"] - if back["outcome"] == "no_change": - text += " (whatever the screen says; do not report it as saved)" + if back["outcome"] == "unconfirmed": + text += " (whatever the screen says)" header = "" if self.surface.kind == "dom": page = self.harness.page diff --git a/packages/flow/openadapt_flow/compile_agent/harness.py b/packages/flow/openadapt_flow/compile_agent/harness.py index 8d970d26c..91fdb69be 100644 --- a/packages/flow/openadapt_flow/compile_agent/harness.py +++ b/packages/flow/openadapt_flow/compile_agent/harness.py @@ -40,6 +40,7 @@ import time from dataclasses import dataclass, field from pathlib import Path +from types import MappingProxyType from typing import Any, Callable, Mapping, Optional, Sequence from openadapt_flow.compile_agent import observe as _observe @@ -69,21 +70,28 @@ class GateRefused(RuntimeError): class ReadBackRefused(GateRefused): - """A save went through, and the system of record shows a change on a record - the task does not name. Run 1 stops; no draft is made.""" + """The read-back stopped run 1: a record the task does not name changed, a + starting record changed or disappeared, the named record was saved twice, + an earlier save landed late, or the system of record could not be read. + No draft is made.""" + + +#: Read-back outcomes that stop run 1. +STOP_OUTCOMES = frozenset({"wrong_record", "duplicate", "landed_late", "unreadable"}) @dataclass(frozen=True) class ReadBack: """What the system of record showed after one save. - ``outcome`` is ``landed`` (only the named record changed), ``no_change``, - ``wrong_record`` (another record changed) or ``unreadable``. + ``outcome`` is ``landed`` (one new record the task names, with every + input value), ``incomplete`` (that record, missing an input value), + ``unconfirmed`` (no new record yet), or one of :data:`STOP_OUTCOMES`. """ outcome: str detail: str - #: Field names of the named record's change (values stay out of the log). + #: Field names of the new record (values stay out of the log). fields: tuple[str, ...] = () def to_dict(self) -> dict[str, Any]: @@ -94,14 +102,52 @@ def to_dict(self) -> dict[str, Any]: } +class ReadBackError(RuntimeError): + """A read the read-back needs failed, or showed a late write.""" + + def __init__(self, outcome: str, detail: str) -> None: + super().__init__(detail) + self.outcome = outcome + + +def _canonical(record: Mapping[str, Any]) -> str: + import json + + return json.dumps(record, sort_keys=True, default=str) + + +def _added_removed( + old: Sequence[Mapping[str, Any]], new: Sequence[Mapping[str, Any]] +) -> tuple[list[Mapping[str, Any]], int]: + """Records in ``new`` not in ``old`` (counted, so two equal records are + two), and how many of ``old`` are no longer there as they were.""" + + from collections import Counter + + before = Counter(_canonical(r) for r in old) + after = Counter(_canonical(r) for r in new) + added: list[Mapping[str, Any]] = [] + seen: Counter[str] = Counter() + for record in new: + key = _canonical(record) + seen[key] += 1 + if seen[key] > before.get(key, 0): + added.append(record) + removed = sum((before - after).values()) + return added, removed + + class RecordReadBack: """A read-back for the harness (see :func:`record_readback`). - :meth:`mark` takes a read just before a save; calling the object then - reads again. A record that changed or disappeared since run 1 started and - is not the named record is the wrong record. Whether this save landed is - judged against the read :meth:`mark` took, so a second save that writes - nothing is told so. + Only a record created during run 1 can be the one the task names; any + change to or removal of a record that was there when run 1 started stops + the run, and so does a second new record the task names (a duplicate save, + counted even in a store without record ids). :meth:`mark` takes a read + just before each save, and whether this save landed is judged against it, + never against an older read. A save that shows no new record is + unconfirmed: before the next save a fresh read must show nothing changed + since, or the run stops (the first save landed late). """ def __init__( @@ -109,63 +155,113 @@ def __init__( read: Callable[[], Optional[list[dict[str, Any]]]], expect: Mapping[str, Any], baseline: Callable[[], Optional[list[dict[str, Any]]]], + values: Mapping[str, Any] = MappingProxyType({}), ) -> None: self.read = read self.baseline = baseline self.wanted = {str(k): str(v) for k, v in expect.items()} + self.values = {str(k): str(v) for k, v in values.items() if str(v)} self._before: Optional[list[dict[str, Any]]] = None + self._unconfirmed: Optional[list[dict[str, Any]]] = None def named(self, record: Mapping[str, Any]) -> bool: return all(str(record.get(k)) == v for k, v in self.wanted.items()) - def mark(self) -> None: + def _read(self, what: str) -> list[dict[str, Any]]: try: - self._before = self.read() - except Exception: # noqa: BLE001 - judged against the run's start then - self._before = None - - def _others(self, diff: Any) -> list[str]: - """Changes on records the task does not name.""" - - bad: list[str] = [] - named_created = [r for r in diff.created if self.named(r)] - for record in diff.created: - if not self.named(record): - bad.append("a record the task does not name was created") - for change in diff.changed: - # The named record before and after: overwriting another record - # until it matches the task is not the named record. - if not (self.named(change.before) and self.named(change.after)): - bad.append("a record the task does not name changed") - for record in diff.removed: - # A store without record ids shows an edit as removed + created. - if not (self.named(record) and named_created): - bad.append("a record disappeared") - return bad + records = self.read() + except Exception as exc: # noqa: BLE001 - reported, and the run stops + raise ReadBackError( + "unreadable", + f"the system of record could not be read {what}: {type(exc).__name__}", + ) from None + if records is None: + raise ReadBackError( + "unreadable", f"the system of record could not be read {what}" + ) + return records - def __call__(self) -> ReadBack: - from openadapt_flow.record_sor import diff_snapshots + def start(self) -> None: + """Before run 1's first action: the starting read must exist.""" + + if self.baseline() is None: + raise ReadBackError( + "unreadable", + "the system of record could not be read when run 1 started", + ) + + def mark(self) -> None: + """Just before a save. Raises ReadBackError when the read fails, or + when an earlier unconfirmed save has landed since.""" + + now = self._read("before this save") + if self._unconfirmed is not None: + late, gone = _added_removed(self._unconfirmed, now) + if late or gone: + raise ReadBackError( + "landed_late", + "an earlier unconfirmed save landed late (the records " + "changed after it showed nothing); this save was not made, " + "so nothing is saved twice", + ) + self._unconfirmed = None + self._before = now - start, after = self.baseline(), self.read() - if start is None or after is None: - return ReadBack("unreadable", "the system of record could not be read") - bad = self._others(diff_snapshots(start, after)) - if bad: + def __call__(self) -> ReadBack: + start = self.baseline() + if start is None: + return ReadBack("unreadable", "the starting read is missing") + try: + after = self._read("after this save") + except ReadBackError as exc: + return ReadBack(exc.outcome, str(exc)) + added, removed = _added_removed(start, after) + if removed: + return ReadBack( + "wrong_record", + f"{removed} record(s) that existed when run 1 started changed or " + "disappeared", + ) + foreign = [r for r in added if not self.named(r)] + if foreign: return ReadBack( - "wrong_record", f"{len(bad)} change(s) the task does not name: {bad[0]}" + "wrong_record", + f"{len(foreign)} new record(s) the task does not name were saved", ) - before = self._before if self._before is not None else start + if len(added) > 1: + return ReadBack( + "duplicate", + f"{len(added)} new records the task names were saved; it saves one", + ) + before = self._before self._before = None - step = diff_snapshots(before, after) - touched = [*step.created, *(c.after for c in step.changed)] - if not touched: - return ReadBack("no_change", "nothing was saved in the system of record") - fields = tuple(sorted({k for r in touched for k in r})) + if before is None: + return ReadBack("unreadable", "no read was taken just before this save") + new, _gone = _added_removed(before, after) + if not new: + self._unconfirmed = after + return ReadBack( + "unconfirmed", + "unconfirmed: the system of record shows no new record after this " + "save. Do not save again; report the save as not confirmed", + ) + record = new[0] + fields = tuple(sorted(record)) + held = {str(v) for v in record.values() if v is not None} + missing = sorted(k for k, v in self.values.items() if v not in held) + if missing: + return ReadBack( + "incomplete", + "1 record saved for " + + ", ".join(f"{k}={v}" for k, v in self.wanted.items()) + + f", without the input(s) {', '.join(missing)}", + fields, + ) return ReadBack( "landed", - f"{len(touched)} record(s) with " + "1 record saved for " + ", ".join(f"{k}={v}" for k, v in self.wanted.items()) - + " saved", + + ", with every input", fields, ) @@ -174,13 +270,15 @@ def record_readback( read: Callable[[], Optional[list[dict[str, Any]]]], expect: Mapping[str, Any], baseline: Callable[[], Optional[list[dict[str, Any]]]], + values: Mapping[str, Any] = MappingProxyType({}), ) -> RecordReadBack: """A read-back for the harness: ``read`` the system of record after each save and compare it with ``baseline()`` (the read when run 1 started) and - with the read just before the save. A record is the named one when it has - every ``expect`` field with that value.""" + with the read just before the save. A new record is the named one when it + has every ``expect`` field with that value; it landed when it also holds + every ``values`` input.""" - return RecordReadBack(read, expect, baseline) + return RecordReadBack(read, expect, baseline, values) def _expect_write(harness: Any, wait_s: Optional[float] = None) -> None: @@ -195,38 +293,82 @@ def _expect_write(harness: Any, wait_s: Optional[float] = None) -> None: expect(harness.readback_wait_s if wait_s is None else wait_s) -def _before_write(harness: Any) -> None: - """Just before a save the gate let through: the read-back's before-read, - and the recorder's wait for the write.""" +def _stop_run(harness: Any, control: str, outcome: str, detail: str) -> None: + harness.stats.gate_refusals += 1 + harness.readback_log.append( + { + "control": control, + "outcome": outcome, + "detail": detail, + "fields": [], + "at": time.time(), + } + ) + harness.refusal = f"at {control!r}, the read-back shows: {detail}; run 1 stopped" + raise ReadBackRefused(harness.refusal) + + +def _start_readback(harness: Any) -> None: + """Before run 1's first action: refuse to start without a starting read.""" + + start = getattr(harness.readback, "start", None) + if callable(start): + try: + start() + except ReadBackError as exc: + _stop_run(harness, "(run 1 start)", exc.outcome, str(exc)) + + +def _before_write(harness: Any, control: str) -> None: + """Just before a save the gate let through: the read-back's before-read + (nothing is performed if it fails or shows a late write), and the + recorder's wait for the write.""" mark = getattr(harness.readback, "mark", None) if callable(mark): - mark() + try: + mark() + except ReadBackError as exc: + _stop_run(harness, control, exc.outcome, str(exc)) _expect_write(harness) +def _read_back_after_error(harness: Any, control: str, exc: BaseException) -> None: + """After a save raised: read back (a stop outcome raises ReadBackRefused + from ``exc``), and keep the result in the log for the error text.""" + + try: + _read_back(harness, control) + except ReadBackRefused as refused: + raise refused from exc + + def _read_back(harness: Any, control: str) -> Optional[ReadBack]: """Read the system of record after a save went through the gate; poll a - little for a write that lands late. Raises ReadBackRefused on another - record's change.""" + little for a write that lands late. Raises ReadBackRefused on any outcome + in :data:`STOP_OUTCOMES`.""" check = getattr(harness, "readback", None) if check is None: return None deadline = time.monotonic() + max(float(harness.readback_wait_s), 0.0) + before = getattr(check, "_before", None) while True: try: got = check() - except Exception as exc: # noqa: BLE001 - an unreadable read is reported + except Exception as exc: # noqa: BLE001 - an unreadable read stops the run got = ReadBack("unreadable", f"{type(exc).__name__}: {exc}"[:200]) - if got.outcome != "no_change" or time.monotonic() >= deadline: + if got.outcome != "unconfirmed" or time.monotonic() >= deadline: break + # Not landed yet: poll again against the same read from before the save. + if isinstance(check, RecordReadBack): + check._before, check._unconfirmed = before, None time.sleep(0.25) harness.stats.readbacks += 1 harness.readback_log.append( {"control": control, **got.to_dict(), "at": time.time()} ) - if got.outcome == "wrong_record": + if got.outcome in STOP_OUTCOMES: harness.stats.gate_refusals += 1 harness.refusal = ( f"after {control!r}, the read-back shows {got.detail}; run 1 stopped" @@ -466,12 +608,15 @@ def act( commit = self._hinted_commit(native, hint) if commit is not None: self._check_gate(*commit) - _before_write(self) + _before_write(self, commit[0]) try: obs = self.computer.execute(native) - except BaseException: + except BaseException as exc: if commit is not None: _expect_write(self, 0.0) + # The save may have gone out before the error: read it back, + # so a wrong or duplicate write still stops run 1. + _read_back_after_error(self, commit[0], exc) raise self.stats.actions += 1 if commit is not None and self.has_page: @@ -887,12 +1032,15 @@ def act(self, native: Mapping[str, Any], ref: Optional[str] = None) -> Observati commit = self.commit_target(native, ref) if commit is not None: self._check_gate(commit) - _before_write(self) + _before_write(self, commit) try: obs = self.computer.execute(native) - except BaseException: + except BaseException as exc: if commit is not None: _expect_write(self, 0.0) + # The save may have gone out before the error: read it back, + # so a wrong or duplicate write still stops run 1. + _read_back_after_error(self, commit, exc) raise self.stats.actions += 1 if commit is not None: diff --git a/packages/flow/openadapt_flow/compile_agent/pipeline.py b/packages/flow/openadapt_flow/compile_agent/pipeline.py index ebe8cfa9e..198d99c72 100644 --- a/packages/flow/openadapt_flow/compile_agent/pipeline.py +++ b/packages/flow/openadapt_flow/compile_agent/pipeline.py @@ -332,6 +332,10 @@ def run(self, request: TaskRequest, computer: Any) -> None: readback=self.readback, ) try: + if self.readback is not None: + from openadapt_flow.compile_agent.harness import _start_readback + + _start_readback(self.harness) self.ledger = self.inner.drive(request, self.harness) except BaseException as exc: self.error = f"{type(exc).__name__}: {exc}" @@ -634,6 +638,7 @@ def record( lambda: self.observer.system_of_record, self.expect, lambda: self._baseline, + values=values, ), ) jit.agent = harnessed @@ -688,8 +693,11 @@ def done(status: str, detail: Optional[str], **kw: Any) -> CompileResult: if decision == "refused_by_hold" else "refused_wrong_record" ) - if any(r["outcome"] == "wrong_record" for r in harness.readback_log): + stops = [r["outcome"] for r in harness.readback_log] + if any(o in ("wrong_record", "duplicate") for o in stops): status = "readback_wrong_record" + elif any(o in ("landed_late", "unreadable") for o in stops): + status = "readback_refused" if jit.candidate() is not None: # An agent that caught the refusal and went on still made a # draft; it must never be admitted. diff --git a/packages/flow/tests/e2e/test_code_agent_default_e2e.py b/packages/flow/tests/e2e/test_code_agent_default_e2e.py index 014aafc2d..ec8262fd0 100644 --- a/packages/flow/tests/e2e/test_code_agent_default_e2e.py +++ b/packages/flow/tests/e2e/test_code_agent_default_e2e.py @@ -88,12 +88,12 @@ def test_every_save_in_run_1_is_read_back(mockmed: str, tmp_path: Path) -> None: { "control": "Save Encounter", "outcome": "landed", - "detail": "1 record(s) with patient_id=p1, type=Triage saved", + "detail": "1 record saved for patient_id=p1, type=Triage, with every input", "fields": ["id", "key", "note", "patient_id", "source", "type"], } ] assert result.run1["harness"]["readbacks"] == 1 - assert "read back: 1 record(s) with patient_id=p1" in agent.outputs[1] + assert "read back: 1 record saved for patient_id=p1" in agent.outputs[1] assert result.run1["code_action"]["readbacks"] == {"landed": 1} @@ -118,9 +118,9 @@ def test_a_dropped_save_is_reported_as_not_saved(mockmed: str, tmp_path: Path) - result = _compiler(tmp_path, mockmed, fault="optimistic").record( agent, {"note": NOTE} ) - assert result.run1["readbacks"][0]["outcome"] == "no_change" - assert "nothing was saved" in agent.outputs[1] - assert "do not report it as saved" in agent.outputs[1] + assert result.run1["readbacks"][0]["outcome"] == "unconfirmed" + assert "unconfirmed" in agent.outputs[1] + assert "Do not save again" in agent.outputs[1] # Flow's own check after run 1 refuses it too: nothing landed. assert result.status == "readback_refused", result.detail assert _records(mockmed) == [] @@ -171,7 +171,7 @@ def test_teach_run_is_offered_and_drafts_a_program( "teach_run", {"script": script.replace("{note}", NOTE)} ) assert out["status"] == "ok", out - assert "read back: 1 record(s)" in out["output"] + assert "read back: 1 record saved" in out["output"] report = teach.dispatch("teach_finish", {}) # type: ignore[attr-defined] assert report["status"] == "draft", report finally: @@ -206,7 +206,7 @@ def test_teach_run_stops_before_a_save_a_person_must_approve( "teach_run", {"script": out["script"]}, held_by_person=hold["binding"] ) assert done["status"] == "ok", done - assert "read back: 1 record(s)" in done["output"] + assert "read back: 1 record saved" in done["output"] assert [(r["patient_id"], r["note"]) for r in _records(mockmed)] == [ ("p1", NOTE) ] diff --git a/packages/flow/tests/test_code_action.py b/packages/flow/tests/test_code_action.py index f78d36d51..45245c857 100644 --- a/packages/flow/tests/test_code_action.py +++ b/packages/flow/tests/test_code_action.py @@ -27,6 +27,7 @@ from openadapt_flow.compile_agent.harness import CommitHint, Harness, SaveGate from openadapt_flow.compile_agent.sandbox import Sandbox, ShellRefused from openadapt_flow.ir import StructuralLocator +from openadapt_flow.jit.adapters import UnsupportedAction from openadapt_flow.jit.compile_agent import TaskRequest # -- the shell ---------------------------------------------------------------------- @@ -705,59 +706,6 @@ def test_code_as_action_is_the_default_first_run_agent() -> None: assert cli_agent.vision == "code" and cli_agent.model == CODE_DEFAULT_MODEL -def test_record_readback_names_the_record_and_flags_any_other() -> None: - from openadapt_flow.compile_agent.harness import record_readback - - before = [{"id": 1, "patient_id": "p2", "note": "old"}] - state = {"now": list(before)} - check = record_readback(lambda: state["now"], {"patient_id": "p1"}, lambda: before) - assert check().outcome == "no_change" - state["now"] = before + [{"id": 2, "patient_id": "p1", "note": "x"}] - landed = check() - assert landed.outcome == "landed" and landed.fields == ("id", "note", "patient_id") - state["now"] = [{"id": 1, "patient_id": "p2", "note": "edited"}] - assert check().outcome == "wrong_record" - state["now"] = [] - assert check().outcome == "wrong_record" - unreadable = record_readback(lambda: None, {"patient_id": "p1"}, lambda: before) - assert unreadable().outcome == "unreadable" - - -def test_each_save_is_judged_against_the_read_just_before_it() -> None: - from openadapt_flow.compile_agent.harness import record_readback - - state = {"now": []} - check = record_readback(lambda: state["now"], {"patient_id": "p1"}, lambda: []) - check.mark() - state["now"] = [{"id": 1, "patient_id": "p1", "note": "a"}] - assert check().outcome == "landed" - # A second save that writes nothing is told so. - check.mark() - assert check().outcome == "no_change" - - -def test_an_edit_on_a_store_without_ids_is_the_named_record() -> None: - from openadapt_flow.compile_agent.harness import record_readback - - before = [{"patient_id": "p1", "note": "a"}, {"patient_id": "p2", "note": "b"}] - state = {"now": list(before)} - check = record_readback(lambda: state["now"], {"patient_id": "p1"}, lambda: before) - check.mark() - state["now"] = [{"patient_id": "p1", "note": "edited"}, before[1]] - assert check().outcome == "landed" - - -def test_overwriting_another_record_to_match_the_task_is_the_wrong_record() -> None: - from openadapt_flow.compile_agent.harness import record_readback - - before = [{"id": 2, "patient_id": "p2", "note": "b"}] - state = {"now": list(before)} - check = record_readback(lambda: state["now"], {"patient_id": "p1"}, lambda: before) - check.mark() - state["now"] = [{"id": 2, "patient_id": "p1", "note": "b"}] - assert check().outcome == "wrong_record" - - def test_typing_on_a_native_drop_down_is_not_a_save() -> None: class Page: url = "http://app.test/" @@ -796,3 +744,147 @@ def boom(native: Any) -> Any: with pytest.raises(RuntimeError): tool.run('click name="Save note"') assert waits[-2:] == [harness.readback_wait_s, 0.0] + + +# -- the read-back after every save (review of #1542) --------------------------- + + +def _rb(state: dict, values: Optional[dict] = None, start: Any = "same") -> Any: + from openadapt_flow.compile_agent.harness import record_readback + + first = list(state["now"]) if start == "same" else start + return record_readback( + lambda: state["now"], + {"patient_id": "p1"}, + lambda: first, + values=values or {}, + ) + + +def test_a_save_lands_only_as_a_new_record_with_every_input() -> None: + state = {"now": [{"id": 1, "patient_id": "p2", "note": "old"}]} + check = _rb(state, {"note": "x"}) + check.mark() + state["now"] = state["now"] + [{"id": 2, "patient_id": "p1", "note": "x"}] + landed = check() + assert landed.outcome == "landed" and "with every input" in landed.detail + check.mark() + assert check().outcome == "unconfirmed" # a second save that wrote nothing + + +def test_a_save_without_an_input_value_is_incomplete() -> None: + state = {"now": []} + check = _rb(state, {"note": "x"}) + check.mark() + state["now"] = [{"id": 1, "patient_id": "p1", "note": ""}] + got = check() + assert got.outcome == "incomplete" and "note" in got.detail + + +@pytest.mark.parametrize( + "after", + [ + # Overwriting a starting record until it matches the task. + [{"id": 1, "patient_id": "p1", "note": "old"}], + # Deleting a starting record and creating the named one. + [{"id": 2, "patient_id": "p1", "note": "x"}], + # Editing a starting record in a store without ids. + [{"patient_id": "p2", "note": "edited"}, {"patient_id": "p1", "note": "x"}], + ], +) +def test_any_change_to_a_starting_record_is_the_wrong_record(after: list) -> None: + state = {"now": [{"id": 1, "patient_id": "p2", "note": "old"}]} + check = _rb(state) + check.mark() + state["now"] = after + assert check().outcome == "wrong_record" + + +@pytest.mark.parametrize("with_ids", [True, False]) +def test_a_duplicate_save_stops_the_run(with_ids: bool) -> None: + state = {"now": []} + check = _rb(state) + one = {"patient_id": "p1", "note": "x"} + check.mark() + state["now"] = [{"id": 1, **one} if with_ids else dict(one)] + assert check().outcome == "landed" + check.mark() + state["now"] = state["now"] + [{"id": 2, **one} if with_ids else dict(one)] + assert check().outcome == "duplicate" + + +def test_an_unreadable_read_or_a_missing_start_read_stops_the_run() -> None: + from openadapt_flow.compile_agent.harness import ReadBackError, record_readback + + state: dict = {"now": []} + check = _rb(state) + check.mark() + state["now"] = None + assert check().outcome == "unreadable" + with pytest.raises(ReadBackError): + check.mark() # no fallback to an older read + no_start = record_readback(lambda: [], {"patient_id": "p1"}, lambda: None) + with pytest.raises(ReadBackError): + no_start.start() + + +def test_a_save_that_lands_late_stops_the_next_save(tmp_path: Path) -> None: + from openadapt_flow.compile_agent.harness import ReadBackRefused + + tool, computer, harness = _tool(tmp_path) + state: dict = {"now": []} + harness.readback = _rb(state, start=[]) + harness.readback_wait_s = 0.0 + tool.run('click role=button name=Open near="Jane Sample"') + out = tool.run('click name="Save note"').text + assert "unconfirmed" in out and "Do not save again" in out + # The write shows up after the read-back gave up. + state["now"] = [{"id": 1, "patient_id": "p1", "note": ""}] + computer.surface.screen = "form" + saves = len([n for n in computer.executed if n["kind"] == "click"]) + with pytest.raises(ReadBackRefused, match="landed late"): + tool.run('click name="Save note"') + assert len([n for n in computer.executed if n["kind"] == "click"]) == saves + assert harness.readback_log[-1]["outcome"] == "landed_late" + + +def test_a_save_that_raises_is_still_read_back(tmp_path: Path) -> None: + from openadapt_flow.compile_agent.harness import ReadBackRefused + + tool, computer, harness = _tool(tmp_path) + state: dict = {"now": [{"id": 9, "patient_id": "p2"}]} + harness.readback = _rb(state) + harness.readback_wait_s = 0.0 + tool.run('click role=button name=Open near="Jane Sample"') + + def save_then_fail(native: Any) -> Any: + state["now"] = [] # the save went out and touched another record + raise UnsupportedAction("generic", "click", "the page went away") + + computer.execute = save_then_fail + with pytest.raises(ReadBackRefused): + tool.run('click name="Save note"') + assert harness.readback_log[-1]["outcome"] == "wrong_record" + # Without a wrong write, the error text carries the read-back. + tool2, computer2, harness2 = _tool(tmp_path / "b") + harness2.readback = _rb({"now": []}) + harness2.readback_wait_s = 0.0 + tool2.run('click role=button name=Open near="Jane Sample"') + computer2.execute = lambda native: (_ for _ in ()).throw( + UnsupportedAction("generic", "click", "the page went away") + ) + out = tool2.run('click name="Save note"').text + assert "read back after the error: unconfirmed" in out + + +def test_run_1_does_not_start_without_a_starting_read(tmp_path: Path) -> None: + from openadapt_flow.compile_agent.harness import ( + ReadBackRefused, + _start_readback, + record_readback, + ) + + _tool_, _computer, harness = _tool(tmp_path) + harness.readback = record_readback(lambda: [], {"patient_id": "p1"}, lambda: None) + with pytest.raises(ReadBackRefused, match="when run 1 started"): + _start_readback(harness) From 75980cacfee15b94130b0788be6d9cd1f324f0f4 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 04:08:16 -0400 Subject: [PATCH 5/5] test(flow): a wrong-record save over MCP teach now stops at the save's read-back Co-Authored-By: Claude Opus 5.5 --- packages/flow/tests/e2e/test_mcp_teach_e2e.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/flow/tests/e2e/test_mcp_teach_e2e.py b/packages/flow/tests/e2e/test_mcp_teach_e2e.py index 0e63c661c..c83fd278c 100644 --- a/packages/flow/tests/e2e/test_mcp_teach_e2e.py +++ b/packages/flow/tests/e2e/test_mcp_teach_e2e.py @@ -380,9 +380,10 @@ async def scenario() -> None: server, teach, _ = _server(tmp_path, mockmed, fault="wrong_record") async with connect(server) as client: result = await _teach_once(client) - # The teach scope's own read-back of run 1 refuses it first. - assert result["status"] == "readback_refused", result - assert "readback_wrong_record" in result["detail"] + # The read-back right after the save stops run 1 at the save, + # before the teach scope's read-back at the end of run 1. + assert result["status"] == "readback_wrong_record", result + assert "the task does not name" in result["detail"] assert result["needs_person"] == [] status = await client.call("teach_status", {"task_key": result["task_key"]}) assert status["status"] == "no_draft"