diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index ec22b92b9..64c3da981 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -363,7 +363,7 @@ jobs: python-version: "3.12" cache: pip - name: Install - run: pip install -e ../types -e '.[dev]' + run: pip install -e ../types -e '.[dev,agent]' - name: Resolve Playwright version id: pw run: | @@ -399,6 +399,9 @@ jobs: tests/e2e/test_structural_action.py \ tests/e2e/test_compile_agent_e2e.py \ tests/e2e/test_code_action_e2e.py \ + tests/e2e/test_mcp_teach_e2e.py \ + tests/test_agent_guard.py \ + tests/e2e/test_agent_guard_e2e.py \ --basetemp=runs/ci-critical --junitxml=runs/critical-junit.xml - name: Fail if any critical test was skipped or none ran run: | diff --git a/.gitleaksignore b/.gitleaksignore index 572fb59ec..98b8c559f 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -48,6 +48,10 @@ eb78575c688f06d7edc04afc69389aad10e1f5fa:packages/flow/tests/fixtures/terminal_v # branch commit that added them. The lines carry gitleaks:allow from the next commit on. 3f0df2191a649540e79a621ac5406ebe2ff228e2:packages/flow/openadapt_flow/jit/authority.py:generic-api-key:101 3f0df2191a649540e79a621ac5406ebe2ff228e2:packages/flow/openadapt_flow/jit/authority.py:generic-api-key:390 +# MCP teach (PR #1335): the generic-api-key rule matched the variable name `key_dir` +# in a principal_from_key(...) call in two tests. No credential. +e8ec96725c4451aa64cb97230c18d04b55d98503:packages/flow/tests/test_mcp_teach_surfaces.py:generic-api-key:147 +e8ec96725c4451aa64cb97230c18d04b55d98503:packages/flow/tests/e2e/test_mcp_teach_e2e.py:generic-api-key:122 # PR #1367 (network capture redaction): the first commit's synthetic test values in # test_network_capture_redact_json.py; the next commit builds them at run time. 52502adf407793c211fb3c0ac5aba37556206d8c:packages/flow/tests/test_network_capture_redact_json.py:generic-api-key:17 diff --git a/CAPABILITIES.md b/CAPABILITIES.md index c42ee9af9..1f64b3f53 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -21,7 +21,10 @@ shipping in the next release; a pull request number means it is being merged. the verified run into a program (`openadapt_flow.jit`, released; prompt-to-program `compile-agent`, landing; the agent can act through code instead of screenshots, [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)). Agents can drive the recording - on web, native Windows, RDP and Citrix. + on web, native Windows, RDP and Citrix. An agent connected over MCP can + teach itself a new workflow and then call it as a tool + (`openadapt-agent serve --teach`, + [#1335](https://github.com/OpenAdaptAI/OpenAdapt/pull/1335)). 2. **Compiles to the fastest layer the app allows.** Page structure first (DOM, Windows UI Automation, macOS Accessibility), direct API calls where the app's own traffic allows them (`ApiActuator`, released; compiled from captured @@ -63,7 +66,9 @@ shipping in the next release; a pull request number means it is being merged. 9. **Is a safety layer underneath any agent.** An agent's MCP calls, a task ledger that keeps a long job's rules, items, questions and checked saves on the server ([#1332](https://github.com/OpenAdaptAI/OpenAdapt/pull/1332)), a guarded save around any agent's write - ([#1258](https://github.com/OpenAdaptAI/OpenAdapt/pull/1258)), `verify-run` + ([#1258](https://github.com/OpenAdaptAI/OpenAdapt/pull/1258)), a guard for + an agent you already run that holds its browser's saves with no change to + the agent ([#1357](https://github.com/OpenAdaptAI/OpenAdapt/pull/1357)), `verify-run` to judge any executor's write from the system of record, and Agent Skills and a CLI for coding agents. Released except where a PR is named. 10. **Runs locally and keeps data private.** Healthy runs make no model calls @@ -218,6 +223,8 @@ improvising or retrying when something is off. | Code as action: the run-1 agent acts through one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program | In review ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | | Admit a draft by a person, an admitter key, or an agent under a person-signed policy, including self-admission when the policy allows it | Released | Unit tests | [JIT_ADMISSION.md](packages/flow/docs/JIT_ADMISSION.md) | | MCP server: your agent decides what to enter, OpenAdapt makes the entry and checks it | Released | Synthetic sandbox | `openadapt-agent`, [packages/agent](packages/agent/README.md) | +| Teach over MCP: an agent does a new task once through OpenAdapt; the run is read back, compiled and admitted (a person confirms the record check; the agent admits only under a person-signed policy), and becomes a tool it calls with no model calls. Web, and Windows, macOS, Linux, RDP and Citrix with a person approving each save | In review ([#1335](https://github.com/OpenAdaptAI/OpenAdapt/pull/1335)) | Synthetic (MockMed over MCP, and through the real RDP and Citrix backends) | `openadapt-agent serve --teach`, [packages/agent](packages/agent/README.md#teach-a-new-workflow-from-your-agent) | +| Guard an agent you already run: attach to its own Chromium over CDP (one added line), hold each save until the record on screen is the one the task names, block it and tell the agent why when it isn't, read the save back; a run that reads back correctly becomes a draft program with a record check from scaffold-verifier, which a person confirms and admits, then a `taught_*` tool with no model calls. `--observe-only` never blocks | In review ([#1357](https://github.com/OpenAdaptAI/OpenAdapt/pull/1357), stacked on [#1335](https://github.com/OpenAdaptAI/OpenAdapt/pull/1335)) | Synthetic (MockMed, scripted Playwright agent, `effect_e2e` faults) | `openadapt-flow guard`, [GUARD_AN_AGENT.md](packages/flow/docs/GUARD_AN_AGENT.md) | | Task ledger per job: rules locked when the job opens and enforced on every run, item status set only from OpenAdapt's own results, and yes-or-no questions a person answers on the phone or in Desktop, including when a run stops | In review ([#1332](https://github.com/OpenAdaptAI/OpenAdapt/pull/1332)) | Synthetic (30-item MockMed worklist, scripted agent, no model calls) | MCP tools `open_job`, `get_job`, `add_items`, `record_evidence`, `ask_user`, `get_answer`, `change_constraint`, `close_job`; [Track a long job](packages/agent/README.md#track-a-long-job) | | Compile an agent's first verified run from a prompt (`compile-agent`), and a code-as-action authoring agent | In review ([#1277](https://github.com/OpenAdaptAI/OpenAdapt/pull/1277), [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)); the agent harness behind it runs on every backend, on main ([#1283](https://github.com/OpenAdaptAI/OpenAdapt/pull/1283)) | Synthetic | `compile-agent` | | Score harness optimizers (Harness², Meta-Harness) on GUI tasks with the record read-back as the reward, with full trace export | On main ([#1299](https://github.com/OpenAdaptAI/OpenAdapt/pull/1299), [#1300](https://github.com/OpenAdaptAI/OpenAdapt/pull/1300), [#1301](https://github.com/OpenAdaptAI/OpenAdapt/pull/1301)) | Synthetic (scripted proposer) | `integrations/harness2/` | @@ -246,7 +253,7 @@ A test fails when a command has no file or this list is out of date. **Record and build a program:** -- `compile-agent` +- `compile-agent`: With `admit TASK_DIR`, a person confirms and admits a waiting draft, such as one an agent taught over MCP (`openadapt-agent serve --teach`). - `compile` - `compose` - `demo-record` @@ -341,6 +348,10 @@ A test fails when a command has no file or this list is out of date. - `demo` - `tutorial` +**Guard an agent you already run:** + +- `guard`: Attaches to the agent's own Chromium over CDP, holds each save until the right record is on screen, reads it back, and hands a verified run to the teach dir. + ## Measured results @@ -352,6 +363,16 @@ A test fails when a command has no file or this list is out of date. took 38.6 s, slower than the agent. Raw data and figures: [#1251](https://github.com/OpenAdaptAI/OpenAdapt/pull/1251) (`benchmark/openemr_referral_timing/` once merged). +- **Guard an agent you already run, MockMed.** A scripted Playwright agent + in its own browser, 90 runs at a high shared load (median load average 29). + Of the 8 `effect_e2e` faults that leave a wrong record, the agent alone + reported 6 as saved; with the guard, the read-back caught 7 and 1 (a stray + write to a table the REST read doesn't cover) got through; the compiled + program caught all 8. The guard blocked the saves it can see: a save on the + wrong patient and a second submit. Same-load pairs: the guarded agent took + 6.3 s (6 actions, each a model call for a real agent), the compiled tool + 4.4 s with no model calls: + [GUARD_AN_AGENT.md](packages/flow/docs/GUARD_AN_AGENT.md#measured-on-mockmed). - **Fault test.** We injected 8 kinds of bad save, 9 runs each, alongside clean runs: 72 runs left a wrong record behind. A check that trusts the screen's "Saved" banner reported 54 of those as success; reading the record back over the app's REST API missed 9; the diff --git a/llms-full.txt b/llms-full.txt index 7f4752fa6..43f1b3cb7 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -378,7 +378,10 @@ shipping in the next release; a pull request number means it is being merged. the verified run into a program (`openadapt_flow.jit`, released; prompt-to-program `compile-agent`, landing; the agent can act through code instead of screenshots, [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)). Agents can drive the recording - on web, native Windows, RDP and Citrix. + on web, native Windows, RDP and Citrix. An agent connected over MCP can + teach itself a new workflow and then call it as a tool + (`openadapt-agent serve --teach`, + [#1335](https://github.com/OpenAdaptAI/OpenAdapt/pull/1335)). 2. **Compiles to the fastest layer the app allows.** Page structure first (DOM, Windows UI Automation, macOS Accessibility), direct API calls where the app's own traffic allows them (`ApiActuator`, released; compiled from captured @@ -416,10 +419,13 @@ shipping in the next release; a pull request number means it is being merged. review, campaign, canary, rollback), agent-demonstrated fixes for a halted run, and fleet drift patches shared with no patient data are released. Self-repair, where an agent proposes a fix and Flow probes it up to the save - without saving, is on main; fixes of up to three new steps are in review. -9. **Is a safety layer underneath any agent.** An agent's MCP calls, a guarded - save around any agent's write - ([#1258](https://github.com/OpenAdaptAI/OpenAdapt/pull/1258)), `verify-run` + without saving, is on main, and so are fixes of up to three new steps. +9. **Is a safety layer underneath any agent.** An agent's MCP calls, a task + ledger that keeps a long job's rules, items, questions and checked saves on + the server ([#1332](https://github.com/OpenAdaptAI/OpenAdapt/pull/1332)), a guarded save around any agent's write + ([#1258](https://github.com/OpenAdaptAI/OpenAdapt/pull/1258)), a guard for + an agent you already run that holds its browser's saves with no change to + the agent ([#1357](https://github.com/OpenAdaptAI/OpenAdapt/pull/1357)), `verify-run` to judge any executor's write from the system of record, and Agent Skills and a CLI for coding agents. Released except where a PR is named. 10. **Runs locally and keeps data private.** Healthy runs make no model calls @@ -574,6 +580,9 @@ improvising or retrying when something is off. | Code as action: the run-1 agent acts through one terminal tool (structure first, actions by role and name, a sandboxed read-only shell, screenshots on request); every action is a structured step checked against the compiled program | In review ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | Real app (local OpenEMR) and synthetic | `ClaudeAgent(vision="code")`, `CODE_AS_ACTION.md` ([#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)) | | Admit a draft by a person, an admitter key, or an agent under a person-signed policy, including self-admission when the policy allows it | Released | Unit tests | [JIT_ADMISSION.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT_ADMISSION.md) | | MCP server: your agent decides what to enter, OpenAdapt makes the entry and checks it | Released | Synthetic sandbox | `openadapt-agent`, [packages/agent](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/agent/README.md) | +| Teach over MCP: an agent does a new task once through OpenAdapt; the run is read back, compiled and admitted (a person confirms the record check; the agent admits only under a person-signed policy), and becomes a tool it calls with no model calls. Web, and Windows, macOS, Linux, RDP and Citrix with a person approving each save | In review ([#1335](https://github.com/OpenAdaptAI/OpenAdapt/pull/1335)) | Synthetic (MockMed over MCP, and through the real RDP and Citrix backends) | `openadapt-agent serve --teach`, [packages/agent](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/agent/README.md#teach-a-new-workflow-from-your-agent) | +| Guard an agent you already run: attach to its own Chromium over CDP (one added line), hold each save until the record on screen is the one the task names, block it and tell the agent why when it isn't, read the save back; a run that reads back correctly becomes a draft program with a record check from scaffold-verifier, which a person confirms and admits, then a `taught_*` tool with no model calls. `--observe-only` never blocks | In review ([#1357](https://github.com/OpenAdaptAI/OpenAdapt/pull/1357), stacked on [#1335](https://github.com/OpenAdaptAI/OpenAdapt/pull/1335)) | Synthetic (MockMed, scripted Playwright agent, `effect_e2e` faults) | `openadapt-flow guard`, [GUARD_AN_AGENT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/GUARD_AN_AGENT.md) | +| Task ledger per job: rules locked when the job opens and enforced on every run, item status set only from OpenAdapt's own results, and yes-or-no questions a person answers on the phone or in Desktop, including when a run stops | In review ([#1332](https://github.com/OpenAdaptAI/OpenAdapt/pull/1332)) | Synthetic (30-item MockMed worklist, scripted agent, no model calls) | MCP tools `open_job`, `get_job`, `add_items`, `record_evidence`, `ask_user`, `get_answer`, `change_constraint`, `close_job`; [Track a long job](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/agent/README.md#track-a-long-job) | | Compile an agent's first verified run from a prompt (`compile-agent`), and a code-as-action authoring agent | In review ([#1277](https://github.com/OpenAdaptAI/OpenAdapt/pull/1277), [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)); the agent harness behind it runs on every backend, on main ([#1283](https://github.com/OpenAdaptAI/OpenAdapt/pull/1283)) | Synthetic | `compile-agent` | | Score harness optimizers (Harness², Meta-Harness) on GUI tasks with the record read-back as the reward, with full trace export | On main ([#1299](https://github.com/OpenAdaptAI/OpenAdapt/pull/1299), [#1300](https://github.com/OpenAdaptAI/OpenAdapt/pull/1300), [#1301](https://github.com/OpenAdaptAI/OpenAdapt/pull/1301)) | Synthetic (scripted proposer) | `integrations/harness2/` | | One scoreboard comparing the agent alone with OpenAdapt across tasks (correctness by read-back, cost, time, items per hour) | In progress ([#1303](https://github.com/OpenAdaptAI/OpenAdapt/pull/1303)) | Synthetic and local OpenEMR and openIMIS test copies | `benchmark/scoreboard/` | @@ -591,19 +600,116 @@ improvising or retrying when something is off. ## Every command, by capability -Each `openadapt-flow` command, grouped by the capability it serves. A test -fails when a command is added without being listed here. - -- **Record and build a program:** `record`, `demo-record`, `compile`, `induce`, `disambiguate`, `for-each`, `compose`, `process`, `intake`, `visualize`, `pull`, `variants`, `route`, `compile-agent`. -- **Check and admit:** `lint`, `scaffold-verifier`, `check`, `api`, `bind-record-checks`, `certify`, `qualify`, `propose-qualification`, `known-ids`, `jit`, `admit`, `approver-grant`. -- **Run and answer:** `run`, `replay`, `resume`, `approve`, `item-approval`, `triage`, `business-decisions`, `explain`, `console`, `deferred`. -- **Judge any executor's write:** `verify-run`, `serve-reward` (scores agent training episodes by reading the system of record). -- **Repair and learn from halts:** `teach`, `teach-branch`, `repair`, `fleet`, `undo` (the undo steps a person approves after a stop). -- **Give agents the program:** `emit-skill`, `emit-mcp`, `serve-execute`. -- **Proof:** `seal`, `local-seal`, `countersign`, `attach-seal`. -- **Run on premises:** `service`, `no-outbound`. -- **Share safely with OpenAdapt Cloud:** `sanitize`, `review-sanitized`, `approve-sanitized`, `materialize-approved`, `push`, `validate-hosted`, `report-break`, `report-run`, `connect`, `login`, `connector`. -- **Try it and measure it:** `demo`, `tutorial`, `bench`, `benchmark`. +Each `openadapt-flow` command, grouped by the capability it serves. Each +command has one file under +[`packages/flow/docs/commands/`](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/commands/): to add a +command, add `/.md` and run `python scripts/build_command_map.py`. +A test fails when a command has no file or this list is out of date. + + + +**Record and build a program:** + +- `compile-agent`: With `admit TASK_DIR`, a person confirms and admits a waiting draft, such as one an agent taught over MCP (`openadapt-agent serve --teach`). +- `compile` +- `compose` +- `demo-record` +- `disambiguate` +- `for-each` +- `induce` +- `intake` +- `process` +- `pull` +- `record` +- `route` +- `variants` +- `visualize` + +**Check and admit:** + +- `admit` +- `api` +- `approver-grant` +- `bind-record-checks` +- `certify` +- `check` +- `jit` +- `known-ids` +- `lint` +- `propose-qualification` +- `qualify` +- `scaffold-verifier` + +**Run and answer:** + +- `approve` +- `business-decisions` +- `console` +- `deferred` +- `explain` +- `item-approval` +- `replay` +- `resume` +- `run` +- `triage` + +**Judge any executor's write:** + +- `serve-reward`: Scores agent training episodes by reading the system of record. +- `verify-run` + +**Repair and learn from halts:** + +- `fleet` +- `repair` +- `teach-branch` +- `teach` +- `undo`: The undo steps a person approves after a stop. + +**Give agents the program:** + +- `emit-mcp` +- `emit-skill` +- `serve-execute` + +**Proof:** + +- `attach-seal` +- `countersign` +- `local-seal` +- `seal` + +**Run on premises:** + +- `no-outbound` +- `service` + +**Share safely with OpenAdapt Cloud:** + +- `approve-sanitized` +- `connect` +- `connector` +- `login` +- `materialize-approved` +- `push` +- `report-break` +- `report-run` +- `review-sanitized` +- `sanitize` +- `validate-hosted` + +**Try it and measure it:** + +- `bench` +- `benchmark` +- `demo` +- `tutorial` + +**Guard an agent you already run:** + +- `guard`: Attaches to the agent's own Chromium over CDP, holds each save until the right record is on screen, reads it back, and hands a verified run to the teach dir. + + ## Measured results @@ -614,6 +720,16 @@ fails when a command is added without being listed here. took 38.6 s, slower than the agent. Raw data and figures: [#1251](https://github.com/OpenAdaptAI/OpenAdapt/pull/1251) (`benchmark/openemr_referral_timing/` once merged). +- **Guard an agent you already run, MockMed.** A scripted Playwright agent + in its own browser, 90 runs at a high shared load (median load average 29). + Of the 8 `effect_e2e` faults that leave a wrong record, the agent alone + reported 6 as saved; with the guard, the read-back caught 7 and 1 (a stray + write to a table the REST read doesn't cover) got through; the compiled + program caught all 8. The guard blocked the saves it can see: a save on the + wrong patient and a second submit. Same-load pairs: the guarded agent took + 6.3 s (6 actions, each a model call for a real agent), the compiled tool + 4.4 s with no model calls: + [GUARD_AN_AGENT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/GUARD_AN_AGENT.md#measured-on-mockmed). - **Fault test.** We injected 8 kinds of bad save, 9 runs each, alongside clean runs: 72 runs left a wrong record behind. A check that trusts the screen's "Saved" banner reported 54 of those as success; reading the record back over the app's REST API missed 9; the @@ -1128,6 +1244,30 @@ halts with a named reason when the confirmation is missing before and after numbers are in [`benchmark/known_ids/KNOWN_IDS.md`](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/../benchmark/known_ids/KNOWN_IDS.md). +Before a step stops on a look-alike identifier, the replayer takes extra looks +that change nothing +([`openadapt_flow/runtime/identity_looks.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/../openadapt_flow/runtime/identity_looks.py)): +the same read on two fresh screenshots, the 2x re-read, the known-ID list when +one is configured, and, where the surface allows it, a select-and-copy of the +identifier's text box (`runtime.identity_copy_look`). OCR looks can't settle +the record on their own, because a look-alike twin can read the same on every +frame and at every size. Only two looks can: the known-ID list, and a copied +identifier that equals a run parameter in the identity band. A look that names +another record stops the run. Each look is recorded on the step's identity +check. + +The copy look has limits. It needs a surface where selecting the identifier +and copying it changes nothing, and a clipboard that reaches the machine Flow +runs on (RDP or Citrix clipboard redirection). It drags across only the one +text box that holds the identifier, and the run stops if the screen changes +anywhere else during the look. A canvas with no selectable text has no copy +look. A recording's own identifier is OCR text and can be a misread (the RDP +kiosk recorded `A1001` as `Alo01`), so in a workflow that always opens the +recorded record, the copy can stop a run but never confirm one. It confirms +only a record chosen by a run parameter. Governed and qualification runs never +take it. The check just before the click takes no input; it reuses the copy +only when the identifier's pixels are unchanged since the look. + Permissive `replay` may proceed on an unreadable reversible target and reports that condition. Governed `run` requires an affirmative live verdict for every identity-required admitted step; mismatch, unreadable, or abstain halts before @@ -1156,6 +1296,39 @@ irreversible, and qualification requires its record check. An icon-only click that only navigates after data entry is therefore over-classified until an operator reclassifies it. +On a pixel surface an icon-only click carries no text that names its control. +When the recorder can move the pointer without clicking +(`Recorder(capture_hover=True)` on a backend with `hover`), it rests the +pointer on each click target first and keeps that frame. For a click with no +other identity evidence, the compiler stores what changed near the target, such +as the control's highlight or tooltip, as the step's hover evidence +([`openadapt_flow/runtime/hover_evidence.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/../openadapt_flow/runtime/hover_evidence.py)). +At replay, on the pixel rungs, the runtime rests the pointer on the resolved +target, waits the same 0.8 s, and compares. A different response, or a +response where the recording saw none, stops the run before the click. A +missing response, or one that can't be compared (another render with no +readable text), is recorded on the step and changes nothing. A tooltip that +opens later than the wait looks like a different response and stops the run. +Controls that don't answer the pointer, such as most text fields, get no +evidence this way. The browser, remote-display (Citrix and RDP client window) +and FreeRDP backends can hover. The Aardwolf RDP transport and the native +Windows, macOS and Linux backends can't move the pointer without a button +yet, so they take no hover look. + +A pixel step is checked on one frame and clicked a moment later. Consequential +identity-gated clicks were already bound to the exact checked frame. Every +other pixel click now gets a check-to-click guard (`Replayer._pixel_recheck`): +right before the click, the runtime takes a fresh frame and compares the +target region, the identity region and the hover response with the frame the +step was checked on. If one changed (a dialog opened, the list re-sorted), it +doesn't click. It re-resolves the target and re-runs the identity check on +the fresh frame, with no input, and clicks only when the next frame matches +that one. After two re-observations that still see a change, or when the +target or its identity can't be confirmed again, it refuses the click. Each +change is recorded on the step (`StepResult.check_to_click`). A change smaller +than 4 pixels, or under 24 levels per channel, doesn't count. Structured +targets are bound by element tokens instead and aren't rechecked. + A consequential click must be tied to a record before Flow delivers it. Inside a table row, the row is that record. An icon-only Save that floats beside a form (openIMIS's claim Save has no text, accessible name, or tooltip) has no diff --git a/packages/agent/README.md b/packages/agent/README.md index 800c75c27..a97322a8b 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -238,6 +238,8 @@ asyncio.run(main()) The SDK starts the server with only a few environment variables. To pass others, such as `OPENADAPT_BUNDLE_KEY`, add `env=` to `StdioServerParameters`. +To let your agent add workflows itself, see [Teach a new workflow from your agent](#teach-a-new-workflow-from-your-agent). + [scripts/smoke_client.py](scripts/smoke_client.py) is a fuller example that runs every sandbox case. From a clone of this repository, run `pip install -e .`, then: ```bash @@ -384,6 +386,58 @@ ChatGPT and Claude in a web browser can't start a server on your computer. To re It claims the job, polls openadapt.ai over outgoing HTTPS, and asks you in the terminal before a chat account may drive it. Don't confuse it with the launcher's `openadapt connect`, which pairs a computer with OpenAdapt Cloud. [docs/MAILBOX_CLI.md](docs/MAILBOX_CLI.md) has the details. +### Guard an agent you already run + +If your agent already drives a Chromium (a Playwright script, Playwright MCP, browser-use, or a computer-use agent with a browser), you don't have to move it onto these tools. Start its browser with a DevTools port, which is the one line you add, and run the guard next to it for each task: + +```bash +openadapt-flow guard --cdp http://127.0.0.1:9222 --config guard.yaml --task task.json --out runs/task-0001 +``` + +The guard holds every save the agent's tab sends until the record on the screen is the one `task.json` names. When it isn't, the save is blocked and the agent's page says why. A save that goes through is read back from the system of record (`VERIFIED`, `REFUTED` or `RECONCILIATION_REQUIRED`). `--observe-only` never blocks and shows what the guard would have done. + +When a run reads back correctly, it becomes a draft in the teach dir that `guard.yaml` names, with a record check built from the app's own reads. A person confirms the check and admits the draft (`openadapt-flow compile-agent admit`); then `serve --teach verify.yaml --teach-dir ` serves it as a `taught_` tool that makes no model calls. [GUARD_AN_AGENT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/GUARD_AN_AGENT.md) has the files, what counts as a save, and the measurements. + +### Teach a new workflow from your agent + +`serve --teach` lets your agent add a workflow it doesn't have yet, without leaving the conversation. It does the task once through OpenAdapt; OpenAdapt checks the record it saved, compiles the run, and once the run is admitted your agent calls it as a tool. Every later call runs the compiled program with no model calls and reads the save back. + +```bash +claude mcp add --scope user openadapt-teach -- uvx --python 3.12 --from 'openadapt-agent[tutorial]' openadapt-agent serve --teach verify.yaml --url https://emr-test.example.org --runs-dir ~/.openadapt/runs +``` + +`verify.yaml` needs only a `verifier:` section that reads the app's records (`rest`, `sql`, `fhir`, `graphql` or `file`), the same file `openadapt-flow record --verify` uses. The first run makes a real entry in the app, so point `--url` at a test copy with test records. + +Your agent then: + +1. Calls `teach_start` with the task (`{note}` for each input), this run's `inputs`, the `record_fields` the saved record must have (such as its patient id), and `on_screen`, the record's identity as the screen shows it (a name and an MRN). +2. Calls `teach_observe` and `teach_act` until the task is done. On the web, `teach_observe` lists the page as short lines with a ref for each control, and before every save OpenAdapt checks that the record's identity is shown next to the control. A save on the wrong record is refused and ends the run. +3. Calls `teach_finish`. OpenAdapt compiles the run and reads the app's records: exactly one record must have appeared, with every `record_fields` value and every input. The result is the draft: its steps, its inputs, the proposed record check, and what still needs a person. +4. Calls `teach_confirm_checks`. Your MCP client shows a person the proposed check, and only their answer there, with their name, confirms it. Your agent has no argument that confirms it. +5. Calls `teach_admit` with `canary_inputs`, the values each canary saves. A canary is a real save on the same record, so use values that are safe there. + +`teach_admit` admits only under the rules in [JIT_ADMISSION.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/JIT_ADMISSION.md). The operator starts the server with `--teach-key agent.pem --teach-policy signed-policy.json`: the key of the agent registered with `openadapt-flow jit register --role agent`, and an admission policy a person signed with `openadapt-flow jit sign-policy`. The server records the run as that agent, so admitting it is self-admission, which the policy must allow (`allow_self_admission`). Without a key and a policy, `teach_admit` refuses and returns the command a person runs at a terminal instead: + +```bash +openadapt-flow compile-agent admit --admit-as "" --canary-param note="Canary test note" +``` + +Once the program is admitted, it is in `list_workflows` and `run_workflow` at once and has its own `taught_` tool, and the server sends `notifications/tools/list_changed` to clients that support it. A server started later lists every program taught in its `--teach-dir`. + +It works on a web app (`--url`) and on the app a deployment file's `backend:` section names (`--config deploy.yaml`): a native Windows, macOS or Linux window, or a local RDP or Citrix client window. Off the web, OpenAdapt can't read which record a form shows, so a person approves, in your client's form, every action that might save: any click except on a text field, a check box, a tab or a control that plainly navigates (Open, New, Cancel), any click at x,y, Enter, typing outside a text field, and shortcuts. Each answer covers that one action on the screen as it was; if anything changes first, the action isn't done. A client without forms can't save. RDP and Citrix have no control tree, and the Windows, macOS and Linux backends don't list controls yet, so off the web your agent needs `--teach-screens scrubbed` to see the screen, and clicks x,y pixels. Admission there fails closed for now: a pixel-only click carries no checked record identity, so the canary is refused and says why. Network RDP (`rdp_host`) is refused. + +| Flag | What it does | +| --- | --- | +| `--teach VERIFY_YAML` | Adds `teach_start`, `teach_observe`, `teach_act`, `teach_finish`, `teach_abort`, `teach_status`, `teach_confirm_checks` and `teach_admit`, and turns on runs of taught workflows. | +| `--teach-dir DIR` | Where drafts and taught programs live (default `RUNS_DIR/teach`). | +| `--teach-key PEM`, `--teach-policy JSON` | The registered agent's encrypted key (passphrase in `OPENADAPT_JIT_KEY_PASSPHRASE`) and the signed admission policy. Only an agent key under a policy; an admitter key admits from CI with `openadapt-flow jit admit`. | +| `--key-dir DIR` | The JIT key directory (default `OPENADAPT_JIT_KEY_DIR` or `~/.openadapt/jit`). | +| `--teach-hold` | On the web, a person also approves every save in the first run. Without it, your agent names the identity the save gate checks (`on_screen`); a value every form shows proves nothing, and then only the record check after the run and the person confirming it catch a wrong record. | +| `--teach-screens scrubbed` | Adds `teach_screen`: the app's screen with identifiers scrubbed, and nothing shared if scrubbing fails. `unscrubbed` skips scrubbing. Off by default. | +| `--mode production --bundles DIR` | Serves your other workflows next to the taught ones. | + +[COMPILE_AGENT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/COMPILE_AGENT.md) describes the pipeline underneath: the harness, the record check, the canaries and the files it keeps. + ## Track a long job On long work, agents lose track of the rules they were given, miss information that arrives partway through, guess instead of asking, and skip checking their work ([OSWorld 2.0](https://arxiv.org/abs/2606.29537)). The task ledger keeps that state on the server, so your agent reads it instead of rebuilding it from its own history. It's on whenever runs or guarded saves are; `serve --no-ledger` turns it off. diff --git a/packages/agent/docs/DESIGN.md b/packages/agent/docs/DESIGN.md index a0859150c..506a5911e 100644 --- a/packages/agent/docs/DESIGN.md +++ b/packages/agent/docs/DESIGN.md @@ -219,6 +219,7 @@ repeats the text. Every declared input stays required unless | `--mode production --bundles DIR` (or `--allow-run`) | production | the above plus `run_workflow` and deprecated `run_` | | `--mode attended --bundles DIR` (or `--allow-run --allow-attended-actions`) | attended | the above plus Reject, Teach, Escalate; Continue and Skip with a qualified `--config` | | `--bundles DIR --allow-attended-actions` (older flags, no runs) | attended, read-only | the read-only tools plus the same decisions on paused runs | +| `--teach VERIFY_YAML` with `--url` or `--config` | production (taught programs) | the production tools plus `teach_start`, `teach_observe`, `teach_act`, `teach_finish`, `teach_abort`, `teach_status`, `teach_confirm_checks`, `teach_admit`, `teach_screen` (with `--teach-screens`), and `taught_` for each admitted program | The read-only tools return PHI-safe projections: workflow cards, opaque ids, availability, status, and count or boolean metrics. They do not @@ -227,6 +228,14 @@ observed text, local paths, subprocess output, or exception text. The attention tools use the same boundary plus typed categories, artifact ids, and non-authorizing capability metadata. +The teach tools are the exception. While the agent does run 1 of a task, +`teach_observe` and `teach_act` return what it needs to find the right record: +on the web, the page's text and controls, scrubbed by +`openadapt_flow.privacy` except for the values the agent put in the task. Run +1 makes a real entry, so the operator points it at a test copy. The draft +report, the record check and the admission return the same kind of facts as +the other tools, plus the proposed record check's field names and values. + The deprecated per-workflow `run_` tools keep their schemas and legacy `status` field and add the contract fields. They take no `request_id`, so their descriptions say a retry can write twice. Missing diff --git a/packages/flow/benchmark/agent_guard/RESULTS.md b/packages/flow/benchmark/agent_guard/RESULTS.md new file mode 100644 index 000000000..c3df62906 --- /dev/null +++ b/packages/flow/benchmark/agent_guard/RESULTS.md @@ -0,0 +1,99 @@ +# Guard an agent you already run: MockMed results + +Date: 2026-10-19. Synthetic MockMed, localhost only, no model calls anywhere. 2 trials per scenario and arm, one browser at a time. The outside agent is a scripted Playwright agent with 1.0 s between actions. Every run is judged by the `effect_e2e` ground truth, which reads the SQLite file directly. + +**Load.** The host was shared with other work. The 1-minute load average during the runs was 15.4 to 249.4 (median 27.6); every row in `results.json` has its own. Read the times as run 1 against run 2 under the same load, not as absolute speed. + +## Setup the developer adds + +| What | Lines | +| --- | --- | +| Agent code (the DevTools port on the browser it launches) | 1 | +| The guard command | 1 | +| `guard.yaml`, once per app | 9 | +| `verify.yaml`, once per app (the read verify-run uses) | 16 | +| Task file, per task, from whoever assigns the work | 12 | + +## What happened to each save + +| Scenario | What goes wrong | agent_alone | guard | observe_only | +| --- | --- | --- | --- | --- | +| `ok` (effect_e2e) | a clean accepted write | clean | clean | clean | +| `no_persist` (effect_e2e) | 2xx, nothing persisted | **silent wrong** | caught by read-back | caught by read-back | +| `partial` (effect_e2e) | row persisted, note dropped | **silent wrong** | caught by read-back | caught by read-back | +| `duplicate` (effect_e2e) | one request lands twice | **silent wrong** | caught by read-back | caught by read-back | +| `wrong_record` (effect_e2e) | the write lands on p2 | **silent wrong** | caught by read-back | caught by read-back | +| `stale` (effect_e2e) | last write wins over a concurrent row | **silent wrong** | caught by read-back | caught by read-back | +| `collateral_unaudited` (effect_e2e) | right row, plus a stray billing row | **silent wrong** | **got through** | **got through** | +| `optimistic` (effect_e2e) | server refuses with 409 | error on screen | caught by read-back | caught by read-back | +| `session` (effect_e2e) | session expired (401) | error on screen | caught by read-back | caught by read-back | +| `timeout` (effect_e2e) | commits, answers after 1.5 s | clean | clean | clean | +| `page_duplicate` (page) | the page sends the save twice | **silent wrong** | **blocked** | caught by read-back | +| `page_optimistic` (page) | the page paints success, the server refuses | **silent wrong** | caught by read-back | caught by read-back | +| `page_timeout` (page) | the page gives up after 1.2 s; the row commits | false alarm | clean | clean | +| `agent_wrong_patient` (agent) | the agent opens p2 | **silent wrong** | **blocked** | caught by read-back | +| `agent_wrong_value` (agent) | the agent types another note | **silent wrong** | caught by read-back | caught by read-back | + +Outcomes: clean: right save, read back VERIFIED; **blocked**: blocked; nothing wrong saved; caught by read-back: wrong or missing save, reported (not VERIFIED); **got through**: wrong save, reported VERIFIED; **silent wrong**: wrong save; the screen said saved; error on screen: wrong or missing save; the screen showed an error; false alarm: right save, reported as not verified + +## The effect-test faults, counted + +The nine `effect_e2e` fault classes other than `ok`, injected at the store. `timeout` leaves the right record (it commits, then answers late), so it is a wrong save only if it is reported as one. + +| Arm | Blocked by the guard | Caught by verify-run | Got through | Right save, verified | Right save, flagged | +| --- | --- | --- | --- | --- | --- | +| agent_alone | n/a | n/a | 6 silent, 2 with an error on screen | 1 | 0 | +| guard | 0 | 7 | 1 | 1 | 0 | +| observe_only | 0 | 7 | 1 | 1 | 0 | + +## Run 1 against run 2 + +| | Wall time (median) | Model calls | Agent actions | Load (1 min) | +| --- | --- | --- | --- | --- | +| Run 1, agent_alone | 6.24 s | 0 (scripted; a model-driven agent makes about one call per action) | 6.0 | 100.12 | +| Run 1, guard | 6.24 s | 0 (scripted; a model-driven agent makes about one call per action) | 6.0 | 103.96 | +| Run 1, observe_only | 6.22 s | 0 (scripted; a model-driven agent makes about one call per action) | 6.0 | 114.31 | +| Run 2, the `taught_*` tool over MCP | 3.71 s | 0 | 0 | 24.3 | + +**Same load.** Then 5 pairs, back to back: a guarded run 1 of `ok` by the agent, then one call of the run-2 tool, with the load taken at each. + +| Pair | Run 1 (agent, guarded) | Run 2 (tool) | Run 2 model calls | Load (1 min) | +| --- | --- | --- | --- | --- | +| 1 | 6.256 s (clean) | 4.14 s (done) | 0 | 18.65 | +| 2 | 6.299 s (clean) | 4.456 s (done) | 0 | 22.64 | +| 3 | 6.241 s (clean) | 4.041 s (done) | 0 | 20.27 | +| 4 | 6.234 s (clean) | 4.393 s (done) | 0 | 19.75 | +| 5 | 6.369 s (clean) | 6.404 s (done) | 0 | 20.19 | +| Median | 6.26 s | 4.39 s | 0 | 20.19 | + +Run 1's time is mostly the scripted agent's 1 s pause before each of its 6 actions; a model-driven agent spends a model call there instead, so its run 1 is longer and costs about 6 model calls. Run 2 makes none. + +Run 1 is the agent's time from its first action to seeing the result, with the guard (if any) holding the save: 1.0 s of thinking before each action stands in for a model. The guard held the save for its checks for a median 7.4 ms and read it back in 41.2 ms. Compiling run 1 into the draft (when the guard has a `teach:` section) happens after that, in the guard process: 8.33 s for the whole guard process on the run that was handed over. Run 2 includes starting Flow's own headless browser, the replay and its read-back. + +## Run 2 against the same faults + +After a person confirmed the record check and admitted the program (`compile-agent admit`, one attended canary, 8.293 s), the program was called as an MCP tool once per store fault. + +Its record check came from scaffold-verifier (used; caught 33 of 33 faults in its own test). Reads: the rest verifier section, the rest verifier section, compared by content, table banner, table banner, compared by content, table billing, table encounters, table encounters, compared by content. + +| Fault | Tool outcome | Ground truth | Result | Seconds | +| --- | --- | --- | --- | --- | +| `ok` | done | correct | clean | 3.713 | +| `no_persist` | not_sure_if_saved | absent | caught by read-back | 19.643 | +| `partial` | not_sure_if_saved | partial | caught by read-back | 19.407 | +| `duplicate` | not_sure_if_saved | duplicate | caught by read-back | 19.197 | +| `wrong_record` | not_sure_if_saved | wrong_record | caught by read-back | 19.226 | +| `stale` | not_sure_if_saved | collateral_loss | caught by read-back | 20.908 | +| `collateral_unaudited` | not_sure_if_saved | collateral_write | caught by read-back | 9.608 | +| `optimistic` | not_sure_if_saved | absent | caught by read-back | 21.771 | +| `session` | not_sure_if_saved | absent | caught by read-back | 27.908 | +| `timeout` | done | correct | clean | 5.458 | + +API step: not proposed (the recording has no parameters to map into a request). The program runs its GUI steps. + +## Reproduce + +```bash +python -m benchmark.agent_guard.run --n 2 --pairs 5 +python -m benchmark.agent_guard.report +``` diff --git a/packages/flow/benchmark/agent_guard/agent.py b/packages/flow/benchmark/agent_guard/agent.py new file mode 100644 index 000000000..988755d90 --- /dev/null +++ b/packages/flow/benchmark/agent_guard/agent.py @@ -0,0 +1,98 @@ +"""The outside agent: a scripted Playwright "agent" that OpenAdapt doesn't control. + +It stands in for a developer's own browser agent (a Playwright script, +Playwright MCP, browser-use, a computer-use agent with a Chromium). It opens +MockMed, opens a patient, starts an encounter, picks Triage, types the note +and clicks Save, with ``think_s`` between actions where a model-driven agent +would be thinking. It calls no model. + +The only change a developer makes for the guard is the DevTools port on the +browser their agent already launches, marked ``# OpenAdapt guard`` below. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any, Optional + +__all__ = ["AgentRun", "ScriptedAgent"] + + +@dataclass +class AgentRun: + actions: int = 0 + seconds: float = 0.0 + #: What the page said after Save: ``saved`` (the app's banner), ``error`` + #: (the app's error text), ``blocked`` (the guard's notice) or ``nothing``. + screen_after: str = "nothing" + notice: Optional[str] = None + log: list[str] = field(default_factory=list) + + +class ScriptedAgent: + """One browser, one tab, driven by a fixed script.""" + + def __init__(self, *, port: int, headless: bool = True, think_s: float = 1.0): + from playwright.sync_api import sync_playwright + + self.think_s = think_s + self._pw = sync_playwright().start() + self.browser = self._pw.chromium.launch( + headless=headless, + args=[f"--remote-debugging-port={port}"], # OpenAdapt guard + ) + self.page = self.browser.new_page(viewport={"width": 1280, "height": 800}) + + def open(self, url: str) -> None: + self.page.goto(url) + self.page.wait_for_selector("#tasks-table") + + def _act(self, run: AgentRun, what: str, fn: Any) -> None: + time.sleep(self.think_s) + fn() + run.actions += 1 + run.log.append(what) + + def add_triage_note(self, *, patient: str, note: str) -> AgentRun: + """Open ``patient``, write a Triage encounter with ``note``, Save.""" + + page = self.page + run = AgentRun() + started = time.monotonic() + self._act(run, f"open {patient}", lambda: page.click(f"#open-{patient}")) + self._act(run, "new encounter", lambda: page.click("#new-encounter")) + self._act(run, "triage", lambda: page.click("#type-triage")) + # Click the field, then type, as a computer-use agent does. (A bare + # ``fill`` records no focus step; see GUARD_AN_AGENT.md, Limits.) + self._act(run, "click note", lambda: page.click("#note")) + self._act(run, "type note", lambda: page.fill("#note", note)) + self._act(run, "save", lambda: page.click("#save-encounter")) + # Look at the result, as an agent would before reporting back. + deadline = time.monotonic() + 4.0 + while time.monotonic() < deadline: + notice = page.query_selector("#openadapt-guard-notice") + if notice is not None: + run.notice = notice.inner_text() + if "blocked" in run.notice: + run.screen_after = "blocked" + break + if page.query_selector("#saved-banner"): + run.screen_after = "saved" + break + error = page.query_selector("#save-error") + if error is not None and error.inner_text().strip(): + run.screen_after = "error" + break + page.wait_for_timeout(100) + notice = page.query_selector("#openadapt-guard-notice") + if notice is not None: + run.notice = notice.inner_text() + run.seconds = round(time.monotonic() - started, 3) + return run + + def close(self) -> None: + try: + self.browser.close() + finally: + self._pw.stop() diff --git a/packages/flow/benchmark/agent_guard/app.py b/packages/flow/benchmark/agent_guard/app.py new file mode 100644 index 000000000..d5b1ad2a8 --- /dev/null +++ b/packages/flow/benchmark/agent_guard/app.py @@ -0,0 +1,153 @@ +"""MockMed's page in front of the effect test's record store. + +The browser loads the MockMed single-page app (``openadapt_flow/mockmed/ +static``). Its Save posts ``/api/encounter`` (the page's fault path, on with +``?fault=``), and the write lands in the SQLite system of record of +``benchmark/effect_e2e`` (encounters, plus the billing table no encounters +read covers), through that harness's own fault hook. So every +``effect_e2e`` fault class can be put under a real browser agent, and the +independent ground truth (:mod:`benchmark.effect_e2e.ground_truth`) reads the +file directly. + +Two ways to inject a fault: + +* **At the store** (``POST /api/fault {"fault": ...}``): every write gets that + fault, whatever the page asked for. These are the ``effect_e2e`` classes, + with their meaning there (``duplicate``: one request lands twice). +* **By the page** (``?fault=duplicate|optimistic|timeout`` with no store + fault): MockMed's page sends the save twice, paints success before the + server answers, or gives up after 1.2 s. Each request then lands once + (``duplicate``), is refused with 409 (``optimistic``), or commits and + answers late (``timeout``). + +Routes: ``GET /api/records`` (the encounters read verify-run uses), +``GET /api/db`` (the same, in MockMed's shape), ``POST /api/reset``, +``POST /api/fault``, ``POST /api/encounter``, the ``/chart/`` pages, and the +static app. Localhost only; all data is synthetic. +""" + +from __future__ import annotations + +import json +import threading +from dataclasses import dataclass +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Callable, Optional +from urllib.parse import parse_qs, urlparse + +from benchmark.effect_e2e.record_service import TARGET_TYPE, _init_db, _Store +from openadapt_flow.mockmed.chart_pages import ChartPages +from openadapt_flow.mockmed.fault_server import STATIC_DIR + +#: Page-delivered faults: what the store does with each request the page sends. +_PAGE_FAULT_AT_STORE = {"duplicate": "ok", "double": "ok", "idempotent": "ok"} + + +@dataclass +class App: + base_url: str + db_path: Path + store: _Store + stop: Callable[[], None] + #: The fault every write gets, or None to follow the page's ``?fault=``. + fault: Optional[str] = None + writes: int = 0 + + +def _handler(app_ref: dict, directory: str, chart: ChartPages): + class Handler(SimpleHTTPRequestHandler): + def log_message(self, fmt: str, *args: object) -> None: # noqa: A002 + pass + + def _json(self, status: int, payload: dict) -> None: + body = json.dumps(payload).encode("utf-8") + try: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + pass # the page gave up (the timeout fault) + + def _body(self) -> dict: + length = int(self.headers.get("Content-Length", 0) or 0) + if not length: + return {} + try: + return json.loads(self.rfile.read(length).decode("utf-8")) + except (ValueError, UnicodeDecodeError): + return {} + + def do_GET(self) -> None: # noqa: N802 + app: App = app_ref["app"] + if chart.handle_get(self): + return + path = urlparse(self.path).path + if path in ("/api/records", "/api/db"): + self._json(200, {"records": app.store.read_encounters()}) + return + super().do_GET() + + def do_POST(self) -> None: # noqa: N802 + app: App = app_ref["app"] + if chart.handle_post(self): + return + parsed = urlparse(self.path) + body = self._body() + if parsed.path == "/api/reset": + app.store.reset(seed_concurrent=bool(body.get("seed_concurrent"))) + app.fault = body.get("fault") or None + app.writes = 0 + self._json(200, {"ok": True}) + return + if parsed.path == "/api/fault": + app.fault = body.get("fault") or None + self._json(200, {"ok": True, "fault": app.fault}) + return + if parsed.path == "/api/encounter": + page_fault = (parse_qs(parsed.query).get("fault", [""])[0]).strip() + fault = app.fault + if fault is None: + fault = _PAGE_FAULT_AT_STORE.get(page_fault, page_fault) + app.writes += 1 + status = app.store.apply_write( + fault, + patient_id=str(body.get("patient_id", "")), + enc_type=str(body.get("type", "") or TARGET_TYPE), + note=str(body.get("note", "")), + key=body.get("key"), + ) + self._json(status, {"ok": 200 <= status < 300}) + return + self._json(404, {"ok": False}) + + return partial(Handler, directory=directory) + + +def serve(db_path: Path, *, host: str = "127.0.0.1", port: int = 0) -> App: + """Serve the app on a loopback port; the database is created fresh.""" + + _init_db(db_path) + store = _Store(db_path) + chart = ChartPages(rows_for=store.rows_for_patient) + ref: dict = {} + httpd = ThreadingHTTPServer((host, port), _handler(ref, str(STATIC_DIR), chart)) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + + def stop() -> None: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=5) + + app = App( + base_url=f"http://{host}:{httpd.server_address[1]}", + db_path=db_path, + store=store, + stop=stop, + ) + ref["app"] = app + return app diff --git a/packages/flow/benchmark/agent_guard/report.py b/packages/flow/benchmark/agent_guard/report.py new file mode 100644 index 000000000..7ffef7e47 --- /dev/null +++ b/packages/flow/benchmark/agent_guard/report.py @@ -0,0 +1,282 @@ +"""Write ``RESULTS.md`` from ``results.json``. + +python -m benchmark.agent_guard.report +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from statistics import median +from typing import Any, Optional + +from benchmark.agent_guard.run import SCENARIOS + +HERE = Path(__file__).resolve().parent + +WORDS = { + "clean": "right save, read back VERIFIED", + "blocked": "blocked; nothing wrong saved", + "caught_by_readback": "wrong or missing save, reported (not VERIFIED)", + "got_through": "wrong save, reported VERIFIED", + "silent_wrong": "wrong save; the screen said saved", + "visible_error": "wrong or missing save; the screen showed an error", + "false_alarm": "right save, reported as not verified", +} +SHORT = { + "clean": "clean", + "blocked": "**blocked**", + "caught_by_readback": "caught by read-back", + "got_through": "**got through**", + "silent_wrong": "**silent wrong**", + "visible_error": "error on screen", + "false_alarm": "false alarm", +} + + +def _cell(outcomes: list[str]) -> str: + kinds = sorted(set(outcomes)) + text = " / ".join(SHORT.get(k, k) for k in kinds) + return text if len(kinds) == 1 else f"mixed: {text}" + + +def _med(values: list[Any]) -> Optional[float]: + values = [v for v in values if v is not None] + return round(median(values), 2) if values else None + + +def render(results: dict[str, Any]) -> str: + rows = results.get("runs") or [] + arms = results.get("arms") or [] + out: list[str] = [] + out.append("# Guard an agent you already run: MockMed results\n") + out.append( + f"Date: {results.get('date')}. Synthetic MockMed, localhost only, no model " + f"calls anywhere. {results.get('n')} trials per scenario and arm, one " + "browser at a time. The outside agent is a scripted Playwright agent " + f"with {results.get('think_s')} s between actions. Every run is judged by " + "the `effect_e2e` ground truth, which reads the SQLite file directly.\n" + ) + loads = [r.get("load_avg_1m") for r in rows] + [ + r.get("load_avg_1m") for r in (results.get("run2") or {}).get("runs") or [] + ] + loads = [v for v in loads if v is not None] + if loads: + out.append( + f"**Load.** The host was shared with other work. The 1-minute load " + f"average during the runs was {min(loads):.1f} to {max(loads):.1f} " + f"(median {median(loads):.1f}); every row in `results.json` has its " + "own. Read the times as run 1 against run 2 under the same load, not " + "as absolute speed.\n" + ) + + lines = results.get("setup_lines") or {} + out.append("## Setup the developer adds\n") + out.append("| What | Lines |\n| --- | --- |") + out.append( + f"| Agent code (the DevTools port on the browser it launches) | " + f"{lines.get('agent_code')} |" + ) + out.append(f"| The guard command | {lines.get('command')} |") + out.append(f"| `guard.yaml`, once per app | {lines.get('guard_yaml')} |") + out.append( + f"| `verify.yaml`, once per app (the read verify-run uses) | " + f"{lines.get('verify_yaml')} |" + ) + out.append( + f"| Task file, per task, from whoever assigns the work | " + f"{lines.get('task_json_per_task')} |\n" + ) + + out.append("## What happened to each save\n") + head = "| Scenario | What goes wrong | " + " | ".join(arms) + " |" + out.append(head) + out.append("| --- | --- | " + " | ".join("---" for _ in arms) + " |") + for scenario in SCENARIOS: + cells = [] + for arm in arms: + got = [ + r["outcome"] + for r in rows + if r["scenario"] == scenario.name and r["arm"] == arm + ] + cells.append(_cell(got) if got else "") + out.append( + f"| `{scenario.name}` ({scenario.family}) | {scenario.blurb} | " + + " | ".join(cells) + + " |" + ) + out.append("") + out.append("Outcomes: " + "; ".join(f"{SHORT[k]}: {v}" for k, v in WORDS.items())) + out.append("") + + out.append("## The effect-test faults, counted\n") + wrong = [s for s in SCENARIOS if s.family == "effect_e2e" and s.name != "ok"] + out.append( + "The nine `effect_e2e` fault classes other than `ok`, injected at the store. " + "`timeout` leaves the right record (it commits, then answers late), so it " + "is a wrong save only if it is reported as one.\n" + ) + out.append( + "| Arm | Blocked by the guard | Caught by verify-run | Got through | " + "Right save, verified | Right save, flagged |" + ) + out.append("| --- | --- | --- | --- | --- | --- |") + for arm in arms: + counts = {k: 0 for k in WORDS} + for scenario in wrong: + got = [ + r["outcome"] + for r in rows + if r["scenario"] == scenario.name and r["arm"] == arm + ] + if got: + # Deterministic in practice; a mixed scenario counts its worst. + order = [ + "got_through", + "silent_wrong", + "false_alarm", + "visible_error", + "caught_by_readback", + "blocked", + "clean", + ] + worst = next(k for k in order if k in got) + counts[worst] += 1 + if arm == "agent_alone": + out.append( + f"| {arm} | n/a | n/a | {counts['silent_wrong']} silent, " + f"{counts['visible_error']} with an error on screen | " + f"{counts['clean']} | {counts['false_alarm']} |" + ) + else: + out.append( + f"| {arm} | {counts['blocked']} | {counts['caught_by_readback']} | " + f"{counts['got_through']} | {counts['clean']} | " + f"{counts['false_alarm']} |" + ) + out.append("") + + out.append("## Run 1 against run 2\n") + ok = [r for r in rows if r["scenario"] == "ok"] + run2 = results.get("run2") or {} + ok2 = [r for r in run2.get("runs") or [] if r["scenario"] == "ok"] + out.append("| | Wall time (median) | Model calls | Agent actions | Load (1 min) |") + out.append("| --- | --- | --- | --- | --- |") + for arm in arms: + mine = [r for r in ok if r["arm"] == arm] + if not mine: + continue + out.append( + f"| Run 1, {arm} | {_med([r['agent_seconds'] for r in mine])} s | 0 " + f"(scripted; a model-driven agent makes about one call per action) | " + f"{_med([r['agent_actions'] for r in mine])} | " + f"{_med([r.get('load_avg_1m') for r in mine])} |" + ) + if ok2: + out.append( + f"| Run 2, the `taught_*` tool over MCP | {_med([r['seconds'] for r in ok2])}" + f" s | {ok2[0].get('model_calls')} | 0 | " + f"{_med([r.get('load_avg_1m') for r in ok2])} |" + ) + out.append("") + pairs = run2.get("paired") or [] + if pairs: + out.append( + f"**Same load.** Then {len(pairs)} pairs, back to back: a guarded run 1 of " + "`ok` by the agent, then one call of the run-2 tool, with the load " + "taken at each.\n" + ) + out.append( + "| Pair | Run 1 (agent, guarded) | Run 2 (tool) | Run 2 model calls | " + "Load (1 min) |" + ) + out.append("| --- | --- | --- | --- | --- |") + for k, pair in enumerate(pairs): + out.append( + f"| {k + 1} | {pair['run1_agent_seconds']} s ({pair['run1_outcome']}) | " + f"{pair['run2_seconds']} s ({pair['run2_outcome']}) | " + f"{pair['run2_model_calls']} | {pair['run1_load_avg_1m']} |" + ) + out.append( + f"| Median | {_med([p['run1_agent_seconds'] for p in pairs])} s | " + f"{_med([p['run2_seconds'] for p in pairs])} s | 0 | " + f"{_med([p['run1_load_avg_1m'] for p in pairs])} |\n" + ) + out.append( + "Run 1's time is mostly the scripted agent's 1 s pause before each of " + "its 6 actions; a model-driven agent spends a model call there " + "instead, so its run 1 is longer and costs about 6 model calls. Run 2 " + "makes none.\n" + ) + guarded = [r for r in ok if r["arm"] == "guard"] + holds = [h for r in guarded for h in r.get("hold_ms") or []] + reads = [r.get("readback_ms") for r in guarded] + out.append( + "Run 1 is the agent's time from its first action to seeing the result, " + "with the guard (if any) holding the save: " + f"{results.get('think_s')} s of thinking before each action stands in for " + "a model. The guard held the save for its " + f"checks for a median {_med(holds)} ms and read it back in {_med(reads)} ms. " + "Compiling run 1 into the draft (when the guard has a `teach:` section) " + "happens after that, in the guard process: " + f"{_med([r.get('guard_seconds') for r in guarded if r.get('handoff')])} s " + "for the whole guard process on the run that was handed over. Run 2 " + "includes starting Flow's own headless browser, the replay and its " + "read-back.\n" + ) + if run2: + out.append("## Run 2 against the same faults\n") + out.append( + "After a person confirmed the record check and admitted the program " + f"(`compile-agent admit`, one attended canary, {run2.get('admit_seconds')}" + " s), the program was called as an MCP tool once per store fault.\n" + ) + check = ((results.get("handoff") or {}).get("proposed_check")) or {} + if check: + out.append( + f"Its record check came from scaffold-verifier " + f"({'used' if check.get('proposed') else 'not used'}; " + f"caught {check.get('faults_caught')} of {check.get('faults')} faults " + f"in its own test). Reads: {', '.join(check.get('reads') or [])}.\n" + ) + out.append("| Fault | Tool outcome | Ground truth | Result | Seconds |") + out.append("| --- | --- | --- | --- | --- |") + for r in run2.get("runs") or []: + out.append( + f"| `{r['scenario']}` | {r.get('tool_outcome')} | " + f"{r['truth']['fault_class']} | {SHORT.get(r['outcome'], r['outcome'])} " + f"| {r['seconds']} |" + ) + out.append("") + api = (results.get("handoff") or {}).get("api_step") or {} + if api: + out.append( + "API step: " + + ( + "proposed." + if api.get("proposed") + else "not proposed (" + + "; ".join(api.get("reasons") or []) + + "). The program runs its GUI steps." + ) + + "\n" + ) + out.append("## Reproduce\n") + out.append( + "```bash\npython -m benchmark.agent_guard.run --n 2 --pairs 5\npython -m benchmark.agent_guard.report\n```\n" + ) + return "\n".join(out) + + +def main(argv: Optional[list[str]] = None) -> int: + path = Path((argv or sys.argv[1:] or [str(HERE / "results.json")])[0]) + results = json.loads(path.read_text()) + (path.parent / "RESULTS.md").write_text(render(results)) + print(path.parent / "RESULTS.md") + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/packages/flow/benchmark/agent_guard/results.json b/packages/flow/benchmark/agent_guard/results.json new file mode 100644 index 000000000..881891356 --- /dev/null +++ b/packages/flow/benchmark/agent_guard/results.json @@ -0,0 +1,3446 @@ +{ + "arms": [ + "agent_alone", + "guard", + "observe_only" + ], + "date": "2026-10-19", + "handoff": { + "api_step": { + "proposed": false, + "reasons": [ + "the recording has no parameters to map into a request" + ], + "step": null + }, + "detail": "run 1 read back correctly; a person confirms the record check, then admits", + "expect": { + "patient_id": "p1", + "type": "Triage" + }, + "inputs": [ + "note" + ], + "next_steps": [ + "openadapt-flow compile-agent admit /teach/triage_note-1ffb76035f26 --admit-as '' --canary-param note=", + "openadapt-agent serve --teach /verify.yaml --teach-dir /teach (serves the admitted program as a tool)" + ], + "proposed_check": { + "faults": 33, + "faults_caught": 33, + "file": "/teach/triage_note-1ffb76035f26/compiled/scaffold/record_check.proposed.yaml", + "misses": [], + "proof": "/teach/triage_note-1ffb76035f26/compiled/scaffold/record_check.proof.json", + "proposed": true, + "proposed_file": "/teach/triage_note-1ffb76035f26/compiled/verify.proposed.yaml", + "reads": [ + "the rest verifier section", + "the rest verifier section, compared by content", + "table banner", + "table banner, compared by content", + "table billing", + "table encounters", + "table encounters, compared by content" + ], + "source": "scaffold-verifier", + "working": true + }, + "record_check": { + "changed": 0, + "created": 1, + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ], + "passed": true, + "reasons": [], + "removed": 0 + }, + "status": "draft", + "task_dir": "/teach/triage_note-1ffb76035f26", + "task_key": "triage_note-1ffb76035f26" + }, + "model_calls": 0, + "n": 2, + "run2": { + "admission": { + "admitted_by": "Bench Reviewer", + "canaries": [ + { + "halt_reason": null, + "model_calls": 0, + "receipt": "/teach/triage_note-1ffb76035f26/receipts/canary-1-20261019T161158-10a247/effect-receipt.json", + "receipt_outcome": "VERIFIED", + "status": "VERIFIED" + } + ], + "digest": "66a792ff1862b0b9ec2fe159fe3c518ad07121b58a839aa22e6e76e15e4a1382", + "mode": "person", + "policy_id": null + }, + "admit_seconds": 8.293, + "admitted": true, + "paired": [ + { + "run1_agent_seconds": 6.256, + "run1_load_avg_1m": 18.65, + "run1_outcome": "clean", + "run2_load_avg_1m": 18.65, + "run2_model_calls": 0, + "run2_outcome": "done", + "run2_seconds": 4.14 + }, + { + "run1_agent_seconds": 6.299, + "run1_load_avg_1m": 22.64, + "run1_outcome": "clean", + "run2_load_avg_1m": 22.64, + "run2_model_calls": 0, + "run2_outcome": "done", + "run2_seconds": 4.456 + }, + { + "run1_agent_seconds": 6.241, + "run1_load_avg_1m": 20.27, + "run1_outcome": "clean", + "run2_load_avg_1m": 20.27, + "run2_model_calls": 0, + "run2_outcome": "done", + "run2_seconds": 4.041 + }, + { + "run1_agent_seconds": 6.234, + "run1_load_avg_1m": 19.75, + "run1_outcome": "clean", + "run2_load_avg_1m": 19.75, + "run2_model_calls": 0, + "run2_outcome": "done", + "run2_seconds": 4.393 + }, + { + "run1_agent_seconds": 6.369, + "run1_load_avg_1m": 20.19, + "run1_outcome": "clean", + "run2_load_avg_1m": 20.19, + "run2_model_calls": 0, + "run2_outcome": "done", + "run2_seconds": 6.404 + } + ], + "proposed": { + "effects": [ + { + "kind": "record_written", + "match": { + "key": "mockmed-triage-p1-v1", + "note": "{param: note}", + "patient_id": "p1", + "type": "Triage" + } + }, + { + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": "-" + } + }, + { + "kind": "record_written", + "match": { + "note": "{param: note}", + "patient_id": "p1", + "type": "Triage" + } + }, + { + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": "-" + } + }, + { + "expected_count": 0, + "kind": "record_written", + "match": {} + }, + { + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": "-" + } + }, + { + "kind": "record_written", + "match": { + "key": "mockmed-triage-p1-v1", + "note": "{param: note}", + "patient_id": "p1", + "type": "Triage" + } + }, + { + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": "-" + } + } + ], + "file": "/teach/triage_note-1ffb76035f26/compiled/verify.proposed.yaml", + "params": [ + "note" + ], + "reads": [ + "banner", + "banner.content", + "billing", + "encounters", + "encounters.content", + "rest_read", + "rest_read.content" + ] + }, + "runs": [ + { + "load_avg_1m": 24.3, + "model_calls": 0, + "outcome": "clean", + "scenario": "ok", + "seconds": 3.713, + "tool": "taught_triage_note", + "tool_outcome": "done", + "tool_reply": { + "outcome": "done", + "reason": "saved_and_checked" + }, + "truth": { + "correct": true, + "fault_class": "correct" + } + }, + { + "load_avg_1m": 21.18, + "model_calls": 0, + "outcome": "caught_by_readback", + "scenario": "no_persist", + "seconds": 19.643, + "tool": "taught_triage_note", + "tool_outcome": "not_sure_if_saved", + "tool_reply": { + "outcome": "not_sure_if_saved", + "reason": "save_not_confirmed" + }, + "truth": { + "correct": false, + "fault_class": "absent" + } + }, + { + "load_avg_1m": 18.94, + "model_calls": 0, + "outcome": "caught_by_readback", + "scenario": "partial", + "seconds": 19.407, + "tool": "taught_triage_note", + "tool_outcome": "not_sure_if_saved", + "tool_reply": { + "outcome": "not_sure_if_saved", + "reason": "save_not_confirmed" + }, + "truth": { + "correct": false, + "fault_class": "partial" + } + }, + { + "load_avg_1m": 16.81, + "model_calls": 0, + "outcome": "caught_by_readback", + "scenario": "duplicate", + "seconds": 19.197, + "tool": "taught_triage_note", + "tool_outcome": "not_sure_if_saved", + "tool_reply": { + "outcome": "not_sure_if_saved", + "reason": "save_not_confirmed" + }, + "truth": { + "correct": false, + "fault_class": "duplicate" + } + }, + { + "load_avg_1m": 15.38, + "model_calls": 0, + "outcome": "caught_by_readback", + "scenario": "wrong_record", + "seconds": 19.226, + "tool": "taught_triage_note", + "tool_outcome": "not_sure_if_saved", + "tool_reply": { + "outcome": "not_sure_if_saved", + "reason": "save_not_confirmed" + }, + "truth": { + "correct": false, + "fault_class": "wrong_record" + } + }, + { + "load_avg_1m": 16.96, + "model_calls": 0, + "outcome": "caught_by_readback", + "scenario": "stale", + "seconds": 20.908, + "tool": "taught_triage_note", + "tool_outcome": "not_sure_if_saved", + "tool_reply": { + "outcome": "not_sure_if_saved", + "reason": "save_not_confirmed" + }, + "truth": { + "correct": false, + "fault_class": "collateral_loss" + } + }, + { + "load_avg_1m": 16.68, + "model_calls": 0, + "outcome": "caught_by_readback", + "scenario": "collateral_unaudited", + "seconds": 9.608, + "tool": "taught_triage_note", + "tool_outcome": "not_sure_if_saved", + "tool_reply": { + "outcome": "not_sure_if_saved", + "reason": "save_not_confirmed" + }, + "truth": { + "correct": false, + "fault_class": "collateral_write" + } + }, + { + "load_avg_1m": 18.73, + "model_calls": 0, + "outcome": "caught_by_readback", + "scenario": "optimistic", + "seconds": 21.771, + "tool": "taught_triage_note", + "tool_outcome": "not_sure_if_saved", + "tool_reply": { + "outcome": "not_sure_if_saved", + "reason": "save_not_confirmed" + }, + "truth": { + "correct": false, + "fault_class": "absent" + } + }, + { + "load_avg_1m": 17.0, + "model_calls": 0, + "outcome": "caught_by_readback", + "scenario": "session", + "seconds": 27.908, + "tool": "taught_triage_note", + "tool_outcome": "not_sure_if_saved", + "tool_reply": { + "outcome": "not_sure_if_saved", + "reason": "save_not_confirmed" + }, + "truth": { + "correct": false, + "fault_class": "absent" + } + }, + { + "load_avg_1m": 20.12, + "model_calls": 0, + "outcome": "clean", + "scenario": "timeout", + "seconds": 5.458, + "tool": "taught_triage_note", + "tool_outcome": "done", + "tool_reply": { + "outcome": "done", + "reason": "saved_and_checked" + }, + "truth": { + "correct": true, + "fault_class": "correct" + } + } + ] + }, + "run2_rerun_date": "2026-10-19 12:11", + "runs": [ + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.241, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 175.11, + "model_calls": 0, + "outcome": "clean", + "readback_ms": null, + "run1_seconds": 6.241, + "scenario": "ok", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.25, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 8.327, + "handoff": { + "api_step": { + "proposed": false, + "reasons": [ + "the recording has no parameters to map into a request" + ], + "step": null + }, + "detail": "run 1 read back correctly; a person confirms the record check, then admits", + "expect": { + "patient_id": "p1", + "type": "Triage" + }, + "inputs": [ + "note" + ], + "next_steps": [ + "openadapt-flow compile-agent admit /teach/triage_note-1ffb76035f26 --admit-as '' --canary-param note=", + "openadapt-agent serve --teach /verify.yaml --teach-dir /teach (serves the admitted program as a tool)" + ], + "proposed_check": { + "faults": 33, + "faults_caught": 33, + "file": "/teach/triage_note-1ffb76035f26/compiled/scaffold/record_check.proposed.yaml", + "misses": [], + "proof": "/teach/triage_note-1ffb76035f26/compiled/scaffold/record_check.proof.json", + "proposed": true, + "proposed_file": "/teach/triage_note-1ffb76035f26/compiled/verify.proposed.yaml", + "reads": [ + "the rest verifier section", + "the rest verifier section, compared by content", + "table banner", + "table banner, compared by content", + "table billing", + "table encounters", + "table encounters, compared by content" + ], + "source": "scaffold-verifier", + "working": true + }, + "record_check": { + "changed": 0, + "created": 1, + "fields": [ + "id", + "key", + "note", + "patient_id", + "source", + "type" + ], + "passed": true, + "reasons": [], + "removed": 0 + }, + "status": "draft", + "task_dir": "/teach/triage_note-1ffb76035f26", + "task_key": "triage_note-1ffb76035f26" + }, + "hold_ms": [ + 6.7 + ], + "load_avg_1m": 184.69, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 40.9, + "run1_seconds": 12.462, + "scenario": "ok", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.236, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 8.4, + "handoff": null, + "hold_ms": [ + 7.5 + ], + "load_avg_1m": 205.09, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 38.9, + "run1_seconds": 7.474, + "scenario": "ok", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.223, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 202.51, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.223, + "scenario": "no_persist", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.276, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.377, + "handoff": null, + "hold_ms": [ + 21.1 + ], + "load_avg_1m": 220.18, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1084.0, + "run1_seconds": 8.516, + "scenario": "no_persist", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.279, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.371, + "handoff": null, + "hold_ms": [ + 10.8 + ], + "load_avg_1m": 227.78, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1079.9, + "run1_seconds": 8.515, + "scenario": "no_persist", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.255, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 229.39, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.255, + "scenario": "partial", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.245, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.315, + "handoff": null, + "hold_ms": [ + 7.5 + ], + "load_avg_1m": 227.21, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1074.3, + "run1_seconds": 8.496, + "scenario": "partial", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.229, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.413, + "handoff": null, + "hold_ms": [ + 17.0 + ], + "load_avg_1m": 222.99, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1062.5, + "run1_seconds": 8.428, + "scenario": "partial", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.26, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 221.39, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.26, + "scenario": "duplicate", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "duplicate" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.251, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.371, + "handoff": null, + "hold_ms": [ + 7.4 + ], + "load_avg_1m": 205.98, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1070.8, + "run1_seconds": 8.498, + "scenario": "duplicate", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "duplicate" + }, + "verdict": "REFUTED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.405, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.772, + "handoff": null, + "hold_ms": [ + 7.5 + ], + "load_avg_1m": 220.17, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1077.2, + "run1_seconds": 8.651, + "scenario": "duplicate", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "duplicate" + }, + "verdict": "REFUTED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.259, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 225.52, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.259, + "scenario": "wrong_record", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "wrong_record" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.264, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 10.315, + "handoff": null, + "hold_ms": [ + 6.6 + ], + "load_avg_1m": 249.41, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1071.4, + "run1_seconds": 8.664, + "scenario": "wrong_record", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "wrong_record" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.207, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.426, + "handoff": null, + "hold_ms": [ + 5.3 + ], + "load_avg_1m": 236.9, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1061.6, + "run1_seconds": 8.461, + "scenario": "wrong_record", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "wrong_record" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.22, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 203.18, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.22, + "scenario": "stale", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "collateral_loss" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.232, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.426, + "handoff": null, + "hold_ms": [ + 7.3 + ], + "load_avg_1m": 174.87, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1062.4, + "run1_seconds": 8.446, + "scenario": "stale", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "collateral_loss" + }, + "verdict": "REFUTED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.239, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.337, + "handoff": null, + "hold_ms": [ + 6.0 + ], + "load_avg_1m": 174.28, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1066.8, + "run1_seconds": 8.475, + "scenario": "stale", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "collateral_loss" + }, + "verdict": "REFUTED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.206, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 180.34, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.206, + "scenario": "collateral_unaudited", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "collateral_write" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.254, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 8.292, + "handoff": null, + "hold_ms": [ + 7.0 + ], + "load_avg_1m": 161.9, + "model_calls": 0, + "outcome": "got_through", + "readback_ms": 42.0, + "run1_seconds": 7.429, + "scenario": "collateral_unaudited", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "collateral_write" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.209, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 8.267, + "handoff": null, + "hold_ms": [ + 6.4 + ], + "load_avg_1m": 140.78, + "model_calls": 0, + "outcome": "got_through", + "readback_ms": 36.6, + "run1_seconds": 7.413, + "scenario": "collateral_unaudited", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "collateral_write" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.205, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 121.02, + "model_calls": 0, + "outcome": "visible_error", + "readback_ms": null, + "run1_seconds": 6.205, + "scenario": "optimistic", + "screen_after": "error", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.265, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.323, + "handoff": null, + "hold_ms": [ + 7.8 + ], + "load_avg_1m": 105.51, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1087.4, + "run1_seconds": 8.456, + "scenario": "optimistic", + "screen_after": "error", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.226, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.282, + "handoff": null, + "hold_ms": [ + 5.9 + ], + "load_avg_1m": 93.95, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1064.7, + "run1_seconds": 8.426, + "scenario": "optimistic", + "screen_after": "error", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 10.183, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 88.06, + "model_calls": 0, + "outcome": "visible_error", + "readback_ms": null, + "run1_seconds": 10.183, + "scenario": "session", + "screen_after": "nothing", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": "OpenAdapt read the record back: The system of record doesn't show this save, or couldn't be read. It may still land. Don't retry. A person must check the record.", + "agent_seconds": 10.219, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.371, + "handoff": null, + "hold_ms": [ + 8.0 + ], + "load_avg_1m": 77.46, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1063.3, + "run1_seconds": 10.219, + "scenario": "session", + "screen_after": "nothing", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": "OpenAdapt read the record back: the system of record doesn't show this save, or couldn't be read. A person must check the record.", + "agent_seconds": 10.127, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.277, + "handoff": null, + "hold_ms": [ + 5.9 + ], + "load_avg_1m": 69.75, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1062.6, + "run1_seconds": 10.127, + "scenario": "session", + "screen_after": "nothing", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.692, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 67.19, + "model_calls": 0, + "outcome": "clean", + "readback_ms": null, + "run1_seconds": 7.692, + "scenario": "timeout", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.685, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 9.73, + "handoff": null, + "hold_ms": [ + 6.5 + ], + "load_avg_1m": 61.61, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 39.5, + "run1_seconds": 8.895, + "scenario": "timeout", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.703, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 9.783, + "handoff": null, + "hold_ms": [ + 6.4 + ], + "load_avg_1m": 56.89, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 37.9, + "run1_seconds": 8.945, + "scenario": "timeout", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.269, + "arm": "agent_alone", + "blocked": [], + "family": "page", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 54.98, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.269, + "scenario": "page_duplicate", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "duplicate" + }, + "verdict": null, + "would_block": [], + "writes": 2 + }, + { + "agent_actions": 6, + "agent_notice": "OpenAdapt blocked this save: this task was already committed and its result is not verified; a person must check the record before anything else is written. Nothing was sent to the app. Don't retry this save.", + "agent_seconds": 6.268, + "arm": "guard", + "blocked": [ + "needs_reconciliation" + ], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 8.262, + "handoff": null, + "hold_ms": [ + 7.8 + ], + "load_avg_1m": 54.82, + "model_calls": 0, + "outcome": "blocked", + "readback_ms": 40.8, + "run1_seconds": 7.429, + "scenario": "page_duplicate", + "screen_after": "blocked", + "trial": 0, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.25, + "arm": "observe_only", + "blocked": [], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.326, + "handoff": null, + "hold_ms": [ + 6.3 + ], + "load_avg_1m": 51.24, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1060.6, + "run1_seconds": 8.464, + "scenario": "page_duplicate", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "duplicate" + }, + "verdict": "REFUTED", + "would_block": [ + "already_committed" + ], + "writes": 2 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.223, + "arm": "agent_alone", + "blocked": [], + "family": "page", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 49.38, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.223, + "scenario": "page_optimistic", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.234, + "arm": "guard", + "blocked": [], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.317, + "handoff": null, + "hold_ms": [ + 7.2 + ], + "load_avg_1m": 48.21, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1059.0, + "run1_seconds": 8.467, + "scenario": "page_optimistic", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.229, + "arm": "observe_only", + "blocked": [], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.286, + "handoff": null, + "hold_ms": [ + 5.9 + ], + "load_avg_1m": 43.46, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1067.5, + "run1_seconds": 8.496, + "scenario": "page_optimistic", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.323, + "arm": "agent_alone", + "blocked": [], + "family": "page", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 40.78, + "model_calls": 0, + "outcome": "false_alarm", + "readback_ms": null, + "run1_seconds": 7.323, + "scenario": "page_timeout", + "screen_after": "error", + "trial": 0, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.43, + "arm": "guard", + "blocked": [], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 9.452, + "handoff": null, + "hold_ms": [ + 9.3 + ], + "load_avg_1m": 33.86, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 41.7, + "run1_seconds": 8.597, + "scenario": "page_timeout", + "screen_after": "error", + "trial": 0, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.425, + "arm": "observe_only", + "blocked": [], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 9.441, + "handoff": null, + "hold_ms": [ + 6.5 + ], + "load_avg_1m": 30.51, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 42.0, + "run1_seconds": 8.578, + "scenario": "page_timeout", + "screen_after": "error", + "trial": 0, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.239, + "arm": "agent_alone", + "blocked": [], + "family": "agent", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 28.86, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.239, + "scenario": "agent_wrong_patient", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "wrong_record" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": "OpenAdapt blocked this save: the open record does not match the task (not found: name, patient_id); nothing may be written. Nothing was sent to the app. Open the record the task names and save again, or stop and tell a person.", + "agent_seconds": 6.26, + "arm": "guard", + "blocked": [ + "identity_mismatch" + ], + "family": "agent", + "guard_ended": "stopped", + "guard_exit": 3, + "guard_notes": [], + "guard_seconds": 15.261, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 25.09, + "model_calls": 0, + "outcome": "blocked", + "readback_ms": null, + "run1_seconds": 14.389, + "scenario": "agent_wrong_patient", + "screen_after": "blocked", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": null, + "would_block": [], + "writes": 0 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.255, + "arm": "observe_only", + "blocked": [], + "family": "agent", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.342, + "handoff": null, + "hold_ms": [ + 6.4 + ], + "load_avg_1m": 22.87, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1070.9, + "run1_seconds": 8.483, + "scenario": "agent_wrong_patient", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "wrong_record" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [ + "identity_mismatch" + ], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.289, + "arm": "agent_alone", + "blocked": [], + "family": "agent", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 23.2, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.289, + "scenario": "agent_wrong_value", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.29, + "arm": "guard", + "blocked": [], + "family": "agent", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.392, + "handoff": null, + "hold_ms": [ + 8.7 + ], + "load_avg_1m": 22.64, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1074.0, + "run1_seconds": 8.545, + "scenario": "agent_wrong_value", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.242, + "arm": "observe_only", + "blocked": [], + "family": "agent", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.412, + "handoff": null, + "hold_ms": [ + 7.0 + ], + "load_avg_1m": 26.27, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1065.9, + "run1_seconds": 8.465, + "scenario": "agent_wrong_value", + "screen_after": "saved", + "trial": 0, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.24, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 25.12, + "model_calls": 0, + "outcome": "clean", + "readback_ms": null, + "run1_seconds": 6.24, + "scenario": "ok", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.221, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 8.255, + "handoff": null, + "hold_ms": [ + 8.1 + ], + "load_avg_1m": 23.23, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 41.5, + "run1_seconds": 7.39, + "scenario": "ok", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.207, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 8.239, + "handoff": null, + "hold_ms": [ + 5.9 + ], + "load_avg_1m": 23.53, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 39.1, + "run1_seconds": 7.394, + "scenario": "ok", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.254, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 22.37, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.254, + "scenario": "no_persist", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.246, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.299, + "handoff": null, + "hold_ms": [ + 7.7 + ], + "load_avg_1m": 21.22, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1061.8, + "run1_seconds": 8.431, + "scenario": "no_persist", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.245, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.283, + "handoff": null, + "hold_ms": [ + 5.8 + ], + "load_avg_1m": 21.58, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1072.7, + "run1_seconds": 8.441, + "scenario": "no_persist", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.201, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 23.69, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.201, + "scenario": "partial", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.314, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.441, + "handoff": null, + "hold_ms": [ + 10.3 + ], + "load_avg_1m": 36.42, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1128.3, + "run1_seconds": 8.616, + "scenario": "partial", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.243, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.353, + "handoff": null, + "hold_ms": [ + 9.9 + ], + "load_avg_1m": 39.88, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1069.7, + "run1_seconds": 8.488, + "scenario": "partial", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.244, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 43.87, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.244, + "scenario": "duplicate", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "duplicate" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.254, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.508, + "handoff": null, + "hold_ms": [ + 6.7 + ], + "load_avg_1m": 40.48, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1192.8, + "run1_seconds": 8.624, + "scenario": "duplicate", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "duplicate" + }, + "verdict": "REFUTED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.22, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.273, + "handoff": null, + "hold_ms": [ + 6.1 + ], + "load_avg_1m": 36.03, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1067.9, + "run1_seconds": 8.416, + "scenario": "duplicate", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "duplicate" + }, + "verdict": "REFUTED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.236, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 31.72, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.236, + "scenario": "wrong_record", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "wrong_record" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.206, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.258, + "handoff": null, + "hold_ms": [ + 6.2 + ], + "load_avg_1m": 28.95, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1063.2, + "run1_seconds": 8.41, + "scenario": "wrong_record", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "wrong_record" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.216, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.404, + "handoff": null, + "hold_ms": [ + 5.7 + ], + "load_avg_1m": 26.16, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1064.8, + "run1_seconds": 8.46, + "scenario": "wrong_record", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "wrong_record" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.238, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 24.46, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.238, + "scenario": "stale", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "collateral_loss" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.217, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.3, + "handoff": null, + "hold_ms": [ + 6.7 + ], + "load_avg_1m": 21.26, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1066.3, + "run1_seconds": 8.442, + "scenario": "stale", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "collateral_loss" + }, + "verdict": "REFUTED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.217, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.289, + "handoff": null, + "hold_ms": [ + 6.2 + ], + "load_avg_1m": 19.75, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1060.2, + "run1_seconds": 8.431, + "scenario": "stale", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "collateral_loss" + }, + "verdict": "REFUTED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.222, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 18.89, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.222, + "scenario": "collateral_unaudited", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "collateral_write" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.255, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 8.319, + "handoff": null, + "hold_ms": [ + 8.2 + ], + "load_avg_1m": 16.91, + "model_calls": 0, + "outcome": "got_through", + "readback_ms": 37.8, + "run1_seconds": 7.472, + "scenario": "collateral_unaudited", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "collateral_write" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.255, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 8.343, + "handoff": null, + "hold_ms": [ + 6.2 + ], + "load_avg_1m": 16.66, + "model_calls": 0, + "outcome": "got_through", + "readback_ms": 40.3, + "run1_seconds": 7.455, + "scenario": "collateral_unaudited", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "collateral_write" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.256, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 23.11, + "model_calls": 0, + "outcome": "visible_error", + "readback_ms": null, + "run1_seconds": 6.256, + "scenario": "optimistic", + "screen_after": "error", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.269, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.386, + "handoff": null, + "hold_ms": [ + 7.5 + ], + "load_avg_1m": 22.3, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1087.6, + "run1_seconds": 8.484, + "scenario": "optimistic", + "screen_after": "error", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.255, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.347, + "handoff": null, + "hold_ms": [ + 8.7 + ], + "load_avg_1m": 20.59, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1068.8, + "run1_seconds": 8.456, + "scenario": "optimistic", + "screen_after": "error", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 10.211, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 21.03, + "model_calls": 0, + "outcome": "visible_error", + "readback_ms": null, + "run1_seconds": 10.211, + "scenario": "session", + "screen_after": "nothing", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": "OpenAdapt read the record back: The system of record doesn't show this save, or couldn't be read. It may still land. Don't retry. A person must check the record.", + "agent_seconds": 10.198, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.656, + "handoff": null, + "hold_ms": [ + 11.3 + ], + "load_avg_1m": 21.28, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1111.2, + "run1_seconds": 10.199, + "scenario": "session", + "screen_after": "nothing", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": "OpenAdapt read the record back: the system of record doesn't show this save, or couldn't be read. A person must check the record.", + "agent_seconds": 10.247, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.705, + "handoff": null, + "hold_ms": [ + 7.0 + ], + "load_avg_1m": 20.62, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1078.9, + "run1_seconds": 10.247, + "scenario": "session", + "screen_after": "nothing", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.733, + "arm": "agent_alone", + "blocked": [], + "family": "effect_e2e", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 24.9, + "model_calls": 0, + "outcome": "clean", + "readback_ms": null, + "run1_seconds": 7.733, + "scenario": "timeout", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.801, + "arm": "guard", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 10.768, + "handoff": null, + "hold_ms": [ + 15.9 + ], + "load_avg_1m": 33.02, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 53.2, + "run1_seconds": 8.97, + "scenario": "timeout", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.735, + "arm": "observe_only", + "blocked": [], + "family": "effect_e2e", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 10.197, + "handoff": null, + "hold_ms": [ + 11.9 + ], + "load_avg_1m": 32.42, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 39.9, + "run1_seconds": 8.936, + "scenario": "timeout", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.204, + "arm": "agent_alone", + "blocked": [], + "family": "page", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 28.89, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.204, + "scenario": "page_duplicate", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "duplicate" + }, + "verdict": null, + "would_block": [], + "writes": 2 + }, + { + "agent_actions": 6, + "agent_notice": "OpenAdapt blocked this save: this task was already committed and its result is not verified; a person must check the record before anything else is written. Nothing was sent to the app. Don't retry this save.", + "agent_seconds": 6.248, + "arm": "guard", + "blocked": [ + "needs_reconciliation" + ], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 8.281, + "handoff": null, + "hold_ms": [ + 7.3 + ], + "load_avg_1m": 25.9, + "model_calls": 0, + "outcome": "blocked", + "readback_ms": 40.6, + "run1_seconds": 7.43, + "scenario": "page_duplicate", + "screen_after": "blocked", + "trial": 1, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.247, + "arm": "observe_only", + "blocked": [], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 10.139, + "handoff": null, + "hold_ms": [ + 5.9 + ], + "load_avg_1m": 23.72, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1129.2, + "run1_seconds": 8.544, + "scenario": "page_duplicate", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "duplicate" + }, + "verdict": "REFUTED", + "would_block": [ + "already_committed" + ], + "writes": 2 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.272, + "arm": "agent_alone", + "blocked": [], + "family": "page", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 23.18, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.272, + "scenario": "page_optimistic", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.225, + "arm": "guard", + "blocked": [], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.433, + "handoff": null, + "hold_ms": [ + 7.3 + ], + "load_avg_1m": 23.6, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1066.3, + "run1_seconds": 8.465, + "scenario": "page_optimistic", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.243, + "arm": "observe_only", + "blocked": [], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.302, + "handoff": null, + "hold_ms": [ + 6.4 + ], + "load_avg_1m": 21.27, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1060.4, + "run1_seconds": 8.436, + "scenario": "page_optimistic", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.38, + "arm": "agent_alone", + "blocked": [], + "family": "page", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 20.77, + "model_calls": 0, + "outcome": "false_alarm", + "readback_ms": null, + "run1_seconds": 7.38, + "scenario": "page_timeout", + "screen_after": "error", + "trial": 1, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.443, + "arm": "guard", + "blocked": [], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 9.656, + "handoff": null, + "hold_ms": [ + 8.5 + ], + "load_avg_1m": 19.44, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 41.0, + "run1_seconds": 8.586, + "scenario": "page_timeout", + "screen_after": "error", + "trial": 1, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 7.377, + "arm": "observe_only", + "blocked": [], + "family": "page", + "guard_ended": "read_back", + "guard_exit": 0, + "guard_notes": [], + "guard_seconds": 9.323, + "handoff": null, + "hold_ms": [ + 5.9 + ], + "load_avg_1m": 17.83, + "model_calls": 0, + "outcome": "clean", + "readback_ms": 39.2, + "run1_seconds": 8.461, + "scenario": "page_timeout", + "screen_after": "error", + "trial": 1, + "truth": { + "correct": true, + "fault_class": "correct" + }, + "verdict": "VERIFIED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.212, + "arm": "agent_alone", + "blocked": [], + "family": "agent", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 18.16, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.212, + "scenario": "agent_wrong_patient", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "wrong_record" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": "OpenAdapt blocked this save: the open record does not match the task (not found: name, patient_id); nothing may be written. Nothing was sent to the app. Open the record the task names and save again, or stop and tell a person.", + "agent_seconds": 6.207, + "arm": "guard", + "blocked": [ + "identity_mismatch" + ], + "family": "agent", + "guard_ended": "stopped", + "guard_exit": 3, + "guard_notes": [], + "guard_seconds": 15.571, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 16.57, + "model_calls": 0, + "outcome": "blocked", + "readback_ms": null, + "run1_seconds": 14.372, + "scenario": "agent_wrong_patient", + "screen_after": "blocked", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "absent" + }, + "verdict": null, + "would_block": [], + "writes": 0 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.229, + "arm": "observe_only", + "blocked": [], + "family": "agent", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.384, + "handoff": null, + "hold_ms": [ + 6.6 + ], + "load_avg_1m": 16.9, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1067.3, + "run1_seconds": 8.433, + "scenario": "agent_wrong_patient", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "wrong_record" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [ + "identity_mismatch" + ], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.221, + "arm": "agent_alone", + "blocked": [], + "family": "agent", + "guard_ended": null, + "guard_exit": null, + "guard_notes": [], + "guard_seconds": null, + "handoff": null, + "hold_ms": [], + "load_avg_1m": 16.1, + "model_calls": 0, + "outcome": "silent_wrong", + "readback_ms": null, + "run1_seconds": 6.221, + "scenario": "agent_wrong_value", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": null, + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.214, + "arm": "guard", + "blocked": [], + "family": "agent", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.314, + "handoff": null, + "hold_ms": [ + 6.5 + ], + "load_avg_1m": 21.66, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1067.0, + "run1_seconds": 8.445, + "scenario": "agent_wrong_value", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + }, + { + "agent_actions": 6, + "agent_notice": null, + "agent_seconds": 6.273, + "arm": "observe_only", + "blocked": [], + "family": "agent", + "guard_ended": "read_back", + "guard_exit": 4, + "guard_notes": [], + "guard_seconds": 9.618, + "handoff": null, + "hold_ms": [ + 14.9 + ], + "load_avg_1m": 21.96, + "model_calls": 0, + "outcome": "caught_by_readback", + "readback_ms": 1077.5, + "run1_seconds": 8.494, + "scenario": "agent_wrong_value", + "screen_after": "saved", + "trial": 1, + "truth": { + "correct": false, + "fault_class": "partial" + }, + "verdict": "RECONCILIATION_REQUIRED", + "would_block": [], + "writes": 1 + } + ], + "schema": "openadapt.agent-guard-benchmark/v1", + "setup_files": { + "guard": "schema_version: openadapt.agent-guard/v1\napp_url: http://127.0.0.1:52449/?fault=ok&idempotency=demo#tasks\nverify: verify.yaml\nscreen_selector: \"#patient-banner\"\nteach:\n dir: /teach\n name: triage_note\n db: /app.sqlite\n key_dir: /keys\n", + "task": "{\n \"schema_version\": \"openadapt.guard-task/v1\",\n \"task_id\": \"triage-p1-0001\",\n \"expected_record\": {\n \"patient_id\": \"p1\",\n \"name\": \"Jane Sample\"\n },\n \"intended_fields\": {\n \"type\": \"Triage\",\n \"note\": \"Agent guard benchmark triage note\"\n }\n}\n", + "verify": "schema_version: openadapt.verify/v1\nverifier:\n kind: rest\n base_url: http://127.0.0.1:52449\n records_path: /api/records\n records_key: records\neffects:\n - kind: record_written\n match:\n patient_id: {param: patient_id}\n type: {param: type}\n note: {param: note}\n expected_count: 1\n count_new_only: true\n forbid_collateral_loss: true\n timeout_s: 1.0\n" + }, + "setup_lines": { + "agent_code": 1, + "command": 1, + "guard_yaml": 9, + "task_json_per_task": 12, + "verify_yaml": 16 + }, + "summary": { + "counts": { + "agent_alone:agent": { + "silent_wrong": 2 + }, + "agent_alone:effect_e2e": { + "clean": 1, + "silent_wrong": 6, + "visible_error": 2 + }, + "agent_alone:page": { + "false_alarm": 1, + "silent_wrong": 2 + }, + "guard:agent": { + "blocked": 1, + "caught_by_readback": 1 + }, + "guard:effect_e2e": { + "caught_by_readback": 7, + "clean": 1, + "got_through": 1 + }, + "guard:page": { + "blocked": 1, + "caught_by_readback": 1, + "clean": 1 + }, + "observe_only:agent": { + "caught_by_readback": 2 + }, + "observe_only:effect_e2e": { + "caught_by_readback": 7, + "clean": 1, + "got_through": 1 + }, + "observe_only:page": { + "caught_by_readback": 2, + "clean": 1 + } + }, + "per_scenario": { + "agent_wrong_patient": { + "agent_alone": [ + "silent_wrong" + ], + "guard": [ + "blocked" + ], + "observe_only": [ + "caught_by_readback" + ] + }, + "agent_wrong_value": { + "agent_alone": [ + "silent_wrong" + ], + "guard": [ + "caught_by_readback" + ], + "observe_only": [ + "caught_by_readback" + ] + }, + "collateral_unaudited": { + "agent_alone": [ + "silent_wrong" + ], + "guard": [ + "got_through" + ], + "observe_only": [ + "got_through" + ] + }, + "duplicate": { + "agent_alone": [ + "silent_wrong" + ], + "guard": [ + "caught_by_readback" + ], + "observe_only": [ + "caught_by_readback" + ] + }, + "no_persist": { + "agent_alone": [ + "silent_wrong" + ], + "guard": [ + "caught_by_readback" + ], + "observe_only": [ + "caught_by_readback" + ] + }, + "ok": { + "agent_alone": [ + "clean" + ], + "guard": [ + "clean" + ], + "observe_only": [ + "clean" + ] + }, + "optimistic": { + "agent_alone": [ + "visible_error" + ], + "guard": [ + "caught_by_readback" + ], + "observe_only": [ + "caught_by_readback" + ] + }, + "page_duplicate": { + "agent_alone": [ + "silent_wrong" + ], + "guard": [ + "blocked" + ], + "observe_only": [ + "caught_by_readback" + ] + }, + "page_optimistic": { + "agent_alone": [ + "silent_wrong" + ], + "guard": [ + "caught_by_readback" + ], + "observe_only": [ + "caught_by_readback" + ] + }, + "page_timeout": { + "agent_alone": [ + "false_alarm" + ], + "guard": [ + "clean" + ], + "observe_only": [ + "clean" + ] + }, + "partial": { + "agent_alone": [ + "silent_wrong" + ], + "guard": [ + "caught_by_readback" + ], + "observe_only": [ + "caught_by_readback" + ] + }, + "session": { + "agent_alone": [ + "visible_error" + ], + "guard": [ + "caught_by_readback" + ], + "observe_only": [ + "caught_by_readback" + ] + }, + "stale": { + "agent_alone": [ + "silent_wrong" + ], + "guard": [ + "caught_by_readback" + ], + "observe_only": [ + "caught_by_readback" + ] + }, + "timeout": { + "agent_alone": [ + "clean" + ], + "guard": [ + "clean" + ], + "observe_only": [ + "clean" + ] + }, + "wrong_record": { + "agent_alone": [ + "silent_wrong" + ], + "guard": [ + "caught_by_readback" + ], + "observe_only": [ + "caught_by_readback" + ] + } + }, + "run2": { + "admitted": true, + "ok_model_calls": 0, + "ok_seconds": 3.713, + "per_scenario": { + "collateral_unaudited": "caught_by_readback", + "duplicate": "caught_by_readback", + "no_persist": "caught_by_readback", + "ok": "clean", + "optimistic": "caught_by_readback", + "partial": "caught_by_readback", + "session": "caught_by_readback", + "stale": "caught_by_readback", + "timeout": "clean", + "wrong_record": "caught_by_readback" + } + }, + "timing_ok": { + "agent_alone": { + "agent_actions": 6.0, + "hold_ms_median": null, + "readback_ms_median": null, + "run1_seconds_median": 6.24 + }, + "guard": { + "agent_actions": 6.0, + "hold_ms_median": 7.4, + "readback_ms_median": 41.2, + "run1_seconds_median": 9.926 + }, + "observe_only": { + "agent_actions": 6.0, + "hold_ms_median": 6.7, + "readback_ms_median": 39.0, + "run1_seconds_median": 7.434 + } + } + }, + "think_s": 1.0 +} diff --git a/packages/flow/benchmark/agent_guard/run.py b/packages/flow/benchmark/agent_guard/run.py new file mode 100644 index 000000000..8f33324b2 --- /dev/null +++ b/packages/flow/benchmark/agent_guard/run.py @@ -0,0 +1,783 @@ +"""Guard an agent you already run: measured on MockMed, synthetic, no model calls. + +The outside agent is a scripted Playwright "agent" (:mod:`.agent`) in its own +Chromium. The only change to it is the DevTools port. For each scenario and +arm it adds a Triage note to patient p1 in MockMed (:mod:`.app`): + +* ``agent_alone``: no OpenAdapt. What lands, and what the agent's screen says. +* ``guard``: ``openadapt-flow guard`` (a separate process, as a developer runs + it) attached over CDP, fail-closed. +* ``observe_only``: the same guard with ``--observe-only``. + +Scenarios: the ten ``benchmark/effect_e2e`` fault classes injected at the +store; three that MockMed's page delivers (it sends the save twice, paints +success before the server answers, gives up after 1.2 s); and two the agent +makes (it opens the wrong patient, or types another note than the task's). +Every run is judged by the independent ground truth of ``effect_e2e``, which +reads the SQLite file directly. The page runs with ``idempotency=demo`` (a +stable key on the save), as #1335's teach test does: the ``clinical-write`` +policy refuses to run a compiled save without one. + +Run 2: the first ``ok`` guarded run is handed to the teach dir, a person (a +named stand-in here; this is the step a person takes) confirms the proposed +record check and admits the draft after an attended canary, and the program +is called as a ``taught_*`` tool over an MCP session for each store fault. + + python -m benchmark.agent_guard.run --n 3 + +Writes ``results.json`` and ``RESULTS.md`` next to this file. Localhost only. +""" + +from __future__ import annotations + +import argparse +import json +import os +import queue +import signal +import subprocess +import sys +import tempfile +import threading +import time +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +from benchmark.agent_guard.agent import ScriptedAgent +from benchmark.agent_guard.app import App, serve +from benchmark.effect_e2e import ground_truth + +__all__ = ["SCENARIOS", "Scenario", "outcome", "run_benchmark"] + +HERE = Path(__file__).resolve().parent +NOTE = "Agent guard benchmark triage note" +OTHER_NOTE = "A note that belongs to another task" +PERSON = "Bench Reviewer" +ARMS = ("agent_alone", "guard", "observe_only") + + +@dataclass(frozen=True) +class Scenario: + name: str + family: str + blurb: str + store_fault: Optional[str] = "ok" + page_fault: str = "ok" + seed: bool = False + patient: str = "p1" + note: str = NOTE + + +SCENARIOS: tuple[Scenario, ...] = ( + Scenario("ok", "effect_e2e", "a clean accepted write"), + Scenario("no_persist", "effect_e2e", "2xx, nothing persisted", "no_persist"), + Scenario("partial", "effect_e2e", "row persisted, note dropped", "partial"), + Scenario("duplicate", "effect_e2e", "one request lands twice", "duplicate"), + Scenario("wrong_record", "effect_e2e", "the write lands on p2", "wrong_record"), + Scenario( + "stale", + "effect_e2e", + "last write wins over a concurrent row", + "stale", + seed=True, + ), + Scenario( + "collateral_unaudited", + "effect_e2e", + "right row, plus a stray billing row", + "collateral_unaudited", + ), + Scenario("optimistic", "effect_e2e", "server refuses with 409", "optimistic"), + Scenario("session", "effect_e2e", "session expired (401)", "session"), + Scenario("timeout", "effect_e2e", "commits, answers after 1.5 s", "timeout"), + Scenario( + "page_duplicate", + "page", + "the page sends the save twice", + store_fault=None, + page_fault="duplicate", + ), + Scenario( + "page_optimistic", + "page", + "the page paints success, the server refuses", + store_fault=None, + page_fault="optimistic", + ), + Scenario( + "page_timeout", + "page", + "the page gives up after 1.2 s; the row commits", + store_fault=None, + page_fault="timeout", + ), + Scenario("agent_wrong_patient", "agent", "the agent opens p2", patient="p2"), + Scenario( + "agent_wrong_value", "agent", "the agent types another note", note=OTHER_NOTE + ), +) + + +# -- files the developer writes ------------------------------------------------- + + +def write_setup( + work: Path, + base: str, + *, + teach_dir: Optional[Path], + db_path: Optional[Path] = None, +) -> dict[str, Path]: + """verify.yaml, guard.yaml and task.json, as a developer writes them.""" + + work.mkdir(parents=True, exist_ok=True) + verify = work / "verify.yaml" + verify.write_text( + "schema_version: openadapt.verify/v1\n" + "verifier:\n" + " kind: rest\n" + f" base_url: {base}\n" + " records_path: /api/records\n" + " records_key: records\n" + "effects:\n" + " - kind: record_written\n" + " match:\n" + " patient_id: {param: patient_id}\n" + " type: {param: type}\n" + " note: {param: note}\n" + " expected_count: 1\n" + " count_new_only: true\n" + " forbid_collateral_loss: true\n" + " timeout_s: 1.0\n" + ) + guard = work / "guard.yaml" + teach = ( + "teach:\n" + f" dir: {teach_dir}\n" + " name: triage_note\n" + + (f" db: {db_path}\n" if db_path else "") + + f" key_dir: {work / 'keys'}\n" + if teach_dir + else "" + ) + guard.write_text( + "schema_version: openadapt.agent-guard/v1\n" + f"app_url: {base}/?fault=ok&idempotency=demo#tasks\n" + "verify: verify.yaml\n" + 'screen_selector: "#patient-banner"\n' + teach + ) + task = work / "task.json" + task.write_text( + json.dumps( + { + "schema_version": "openadapt.guard-task/v1", + "task_id": "triage-p1-0001", + "expected_record": {"patient_id": "p1", "name": "Jane Sample"}, + "intended_fields": {"type": "Triage", "note": NOTE}, + }, + indent=2, + ) + + "\n" + ) + return {"verify": verify, "guard": guard, "task": task} + + +def setup_lines(files: dict[str, Path]) -> dict[str, int]: + def count(path: Path) -> int: + return sum(1 for line in path.read_text().splitlines() if line.strip()) + + agent_src = (HERE / "agent.py").read_text().splitlines() + return { + # The one line the agent's browser launch changes (marked in agent.py). + "agent_code": sum( + 1 + for line in agent_src + if "# OpenAdapt guard" in line and "--remote-debugging-port" in line + ), + "guard_yaml": count(files["guard"]), + "verify_yaml": count(files["verify"]), + "task_json_per_task": count(files["task"]), + "command": 1, + } + + +# -- one run --------------------------------------------------------------------- + + +def _post(base: str, path: str, body: dict) -> None: + request = urllib.request.Request( + base + path, + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + ) + urllib.request.urlopen(request, timeout=10).read() + + +class _GuardProcess: + """``openadapt-flow guard`` in its own process, as a developer starts it.""" + + def __init__(self, args: list[str]) -> None: + self.proc = subprocess.Popen( + [sys.executable, "-m", "openadapt_flow", "guard", *args, "--json"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=os.environ.copy(), + ) + self.lines: "queue.Queue[str]" = queue.Queue() + self.stderr: list[str] = [] + threading.Thread(target=self._read, daemon=True).start() + + def _read(self) -> None: + assert self.proc.stderr is not None + for line in self.proc.stderr: + self.stderr.append(line) + self.lines.put(line) + + def wait_attached(self, timeout_s: float = 90.0) -> None: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + try: + line = self.lines.get(timeout=0.2) + except queue.Empty: + if self.proc.poll() is not None: + break + continue + if "guard: attached" in line: + return + raise RuntimeError("the guard did not attach:\n" + "".join(self.stderr)) + + def finish(self, wait_s: float) -> dict[str, Any]: + try: + out, _ = self.proc.communicate(timeout=wait_s) + except subprocess.TimeoutExpired: + # No save went through (it was blocked): stop it as a person would. + self.proc.send_signal(signal.SIGINT) + try: + out, _ = self.proc.communicate(timeout=60) + except subprocess.TimeoutExpired: + self.proc.kill() + out, _ = self.proc.communicate() + try: + return {"exit": self.proc.returncode, **json.loads(out)} + except ValueError: + return {"exit": self.proc.returncode, "error": "".join(self.stderr)[-2000:]} + + +def outcome(row: dict[str, Any]) -> str: + gt_correct = row["truth"]["correct"] + if row["arm"] == "agent_alone": + saved = row["screen_after"] == "saved" + if gt_correct: + return "clean" if saved else "false_alarm" + if row["truth"]["fault_class"] == "absent" and not saved: + return "visible_error" + return "silent_wrong" if saved else "visible_error" + blocked = row["blocked"] + if blocked and ( + gt_correct or (row["truth"]["fault_class"] == "absent" and row["writes"] == 0) + ): + return "blocked" + if gt_correct: + return "clean" if row["verdict"] == "VERIFIED" else "false_alarm" + return "got_through" if row["verdict"] == "VERIFIED" else "caught_by_readback" + + +def run_one( + app: App, + scenario: Scenario, + arm: str, + work: Path, + port: int, + *, + think_s: float, + files: dict[str, Path], + teach: bool, +) -> dict[str, Any]: + _post( + app.base_url, + "/api/reset", + {"seed_concurrent": scenario.seed, "fault": scenario.store_fault}, + ) + before = ground_truth.capture(app.db_path) + agent = ScriptedAgent(port=port, think_s=think_s) + guard: Optional[_GuardProcess] = None + try: + agent.open( + f"{app.base_url}/?fault={scenario.page_fault}&idempotency=demo#tasks" + ) + if arm != "agent_alone": + args = [ + "--cdp", + f"http://127.0.0.1:{port}", + "--config", + str(files["guard"]), + "--task", + str(files["task"]), + "--out", + str(work), + "--timeout", + "60", + ] + if arm == "observe_only": + args.append("--observe-only") + if not teach: + args.append("--no-teach") + guard = _GuardProcess(args) + guard.wait_attached() + started = time.monotonic() + acted = agent.add_triage_note(patient=scenario.patient, note=scenario.note) + # A handed-over run compiles the draft after the read-back. + wait_s = 180.0 if teach else 8.0 + result = guard.finish(wait_s=wait_s) if guard is not None else {} + run1_s = round(time.monotonic() - started, 3) + finally: + agent.close() + if guard is not None and guard.proc.poll() is None: + guard.proc.kill() + time.sleep(0.2) + after = ground_truth.capture(app.db_path) + truth = ground_truth.judge(before, after, intended_note=NOTE) + summary = (result.get("run") or {}) if result else {} + row = { + "scenario": scenario.name, + "family": scenario.family, + "arm": arm, + "truth": {"correct": truth.correct, "fault_class": truth.fault_class}, + "screen_after": acted.screen_after, + "agent_notice": acted.notice, + "agent_actions": acted.actions, + "agent_seconds": acted.seconds, + "run1_seconds": run1_s, + "writes": app.writes, + "verdict": summary.get("verdict"), + "blocked": [b["code"] for b in summary.get("blocked") or []], + "would_block": [b["code"] for b in summary.get("would_block") or []], + "hold_ms": summary.get("hold_ms") or [], + "readback_ms": summary.get("readback_ms"), + "guard_exit": result.get("exit") if result else None, + "guard_seconds": summary.get("seconds"), + "guard_ended": summary.get("ended"), + "guard_notes": summary.get("notes") or [], + "model_calls": 0, + "load_avg_1m": round(os.getloadavg()[0], 2), + "handoff": result.get("handoff") if result else None, + } + if result and "error" in result: + row["guard_error"] = result["error"] + row["outcome"] = outcome(row) + return row + + +# -- run 2 ------------------------------------------------------------------------- + + +def _person_admits(task_dir: Path) -> dict[str, Any]: + """The person's terminal path: ``compile-agent admit``. They give their + name, confirm the proposed record check, review the steps, attend the + canary and admit. A named stand-in types the answers here, as in #1335's + tests; a person's admission needs a terminal, which a script has none of. + """ + + import contextlib + import io + from unittest import mock + + from openadapt_flow.__main__ import build_parser + from openadapt_flow.compile_agent.cli import cmd_compile_agent + + answers = iter([PERSON, "ok", "ok", "ok"]) + args = build_parser().parse_args( + [ + "compile-agent", + "admit", + str(task_dir), + "--admit-as", + PERSON, + "--drop-field", + "key", + "--drop-field", + "source", + "--canary-param", + "note=Canary triage note", + "--json", + ] + ) + out = io.StringIO() + with ( + mock.patch("openadapt_flow.jit.lifecycle._has_terminal", lambda: True), + contextlib.redirect_stdout(out), + ): + code = cmd_compile_agent( + args, ask=lambda _prompt: next(answers), interactive=lambda: True + ) + text = out.getvalue() + try: + result = json.loads(text[text.index("\n{") + 1 :]) + except ValueError: + result = {"status": "error", "output": text[-3000:]} + result["exit"] = code + return result + + +def run2( + app: App, + task_dir: Path, + verify: Path, + teach_dir: Path, + runs: Path, + *, + pairs: int = 0, + run1: Optional[Any] = None, +) -> dict: + """A person confirms and admits; then the program is an MCP tool. + + With ``pairs``, ``run1()`` (a guarded run 1 of ``ok``) and one tool call + alternate that many times, so the two are timed under the same load. + """ + + import anyio + + from openadapt_flow.agent.bridge import AgentBridge + from openadapt_flow.agent.mcp import build_server + from openadapt_flow.agent.runner import RunnerConfig + from openadapt_flow.agent.teach import TeachBridge, TeachConfig + from openadapt_flow.compile_agent.pipeline import FirstRunCompiler + + _post(app.base_url, "/api/reset", {"fault": None}) + compiler = FirstRunCompiler.load(task_dir) + proposed = compiler.proposed() + started = time.monotonic() + admitted = _person_admits(task_dir) + admit_s = round(time.monotonic() - started, 3) + if admitted.get("status") != "admitted": + return {"admitted": False, "detail": admitted, "proposed": proposed} + runs.mkdir(parents=True, exist_ok=True) + workflows = AgentBridge( + None, RunnerConfig(runs_dir=runs), allow_run=True, mode="production" + ) + teach = TeachBridge( + TeachConfig(verify=verify, cache_dir=teach_dir, url=compiler.url), + workflows=workflows, + ) + server = build_server(workflows, teach=teach) + rows: list[dict[str, Any]] = [] + + async def scenario() -> None: + from mcp import ClientSession + from mcp.shared.memory import create_client_server_memory_streams + + async with create_client_server_memory_streams() as (client, transport): + async with anyio.create_task_group() as tasks: + tasks.start_soon( + server.run, *transport, server.create_initialization_options() + ) + async with ClientSession(*client) as session: + await session.initialize() + tools = [t.name for t in (await session.list_tools()).tools] + (tool,) = [t for t in tools if t.startswith("taught_")] + for item in SCENARIOS: + if item.family != "effect_e2e": + continue + _post( + app.base_url, + "/api/reset", + {"seed_concurrent": item.seed, "fault": item.store_fault}, + ) + before = ground_truth.capture(app.db_path) + t0 = time.monotonic() + res = await session.call_tool( + tool, + { + "inputs": {"note": NOTE}, + "request_id": f"bench-{item.name}-0001", + }, + ) + seconds = round(time.monotonic() - t0, 3) + payload = json.loads(res.content[0].text) + await anyio.sleep(0.2) + truth = ground_truth.judge( + before, + ground_truth.capture(app.db_path), + intended_note=NOTE, + ) + done = payload.get("outcome") == "done" + if truth.correct: + result = "clean" if done else "false_alarm" + else: + result = "got_through" if done else "caught_by_readback" + rows.append( + { + "scenario": item.name, + "load_avg_1m": round(os.getloadavg()[0], 2), + "tool": tool, + "seconds": seconds, + "tool_outcome": payload.get("outcome"), + "tool_reply": { + k: payload.get(k) + for k in ("outcome", "reason", "error", "status") + if payload.get(k) is not None + }, + "model_calls": payload.get("model_calls"), + "truth": { + "correct": truth.correct, + "fault_class": truth.fault_class, + }, + "outcome": result, + } + ) + for k in range(pairs): + assert run1 is not None + first = await anyio.to_thread.run_sync(run1) + _post(app.base_url, "/api/reset", {"fault": None}) + load = round(os.getloadavg()[0], 2) + t0 = time.monotonic() + res = await session.call_tool( + tool, + { + "inputs": {"note": NOTE}, + "request_id": f"bench-pair-{k:04d}", + }, + ) + payload = json.loads(res.content[0].text) + paired.append( + { + "run1_agent_seconds": first["agent_seconds"], + "run1_outcome": first["outcome"], + "run1_load_avg_1m": first["load_avg_1m"], + "run2_seconds": round(time.monotonic() - t0, 3), + "run2_outcome": payload.get("outcome"), + "run2_model_calls": payload.get("model_calls"), + "run2_load_avg_1m": load, + } + ) + print(f"pair {k}: {paired[-1]}", flush=True) + tasks.cancel_scope.cancel() + + paired: list[dict[str, Any]] = [] + anyio.run(scenario) + _post(app.base_url, "/api/reset", {"fault": None}) + return { + "admitted": True, + "admit_seconds": admit_s, + "proposed": proposed, + "admission": admitted.get("admission"), + "runs": rows, + "paired": paired, + } + + +# -- the whole study --------------------------------------------------------------- + + +def run_benchmark( + n: int, + think_s: float, + arms: tuple[str, ...], + port: int, + *, + pairs: int = 0, + previous: Optional[dict[str, Any]] = None, +) -> dict: + """The matrix, then run 2. With ``previous`` (an earlier results.json), + the matrix is kept from it and only run 2 (and the pairs) run again.""" + + results: dict[str, Any] = { + "schema": "openadapt.agent-guard-benchmark/v1", + "date": time.strftime("%Y-%m-%d"), + "n": n, + "think_s": think_s, + "arms": list(arms), + "model_calls": 0, + } + with tempfile.TemporaryDirectory(prefix="agent-guard-bench-") as tmp: + work = Path(tmp) + app = serve(work / "app.sqlite") + try: + teach_dir = work / "teach" + files = write_setup( + work, app.base_url, teach_dir=teach_dir, db_path=app.db_path + ) + plain = write_setup(work / "plain", app.base_url, teach_dir=None) + results["setup_lines"] = setup_lines(files) + results["setup_files"] = { + name: path.read_text() for name, path in files.items() + } + rows: list[dict[str, Any]] = [] + handoff: Optional[dict[str, Any]] = None + plan = ( + [(0, s, "guard") for s in SCENARIOS if s.name == "ok"] + if previous is not None + else [(t, s, a) for t in range(n) for s in SCENARIOS for a in arms] + ) + for trial, scenario, arm in plan: + port += 1 + teach = handoff is None and arm == "guard" and scenario.name == "ok" + row = run_one( + app, + scenario, + arm, + work / "runs" / f"{scenario.name}-{arm}-{trial}", + port, + think_s=think_s, + files=files if teach else plain, + teach=teach, + ) + row["trial"] = trial + if teach: + handoff = row.get("handoff") + rows.append(row) + print( + f"{trial} {scenario.name:22s} {arm:13s} {row['outcome']:20s}" + f" {row['run1_seconds']:6.2f}s", + flush=True, + ) + if previous is not None: + results.update( + { + k: previous[k] + for k in ("n", "think_s", "arms", "date") + if k in previous + } + ) + results["runs"] = previous["runs"] + results["run2_rerun_date"] = time.strftime("%Y-%m-%d %H:%M") + else: + results["runs"] = rows + results["handoff"] = handoff + counter = {"port": port + 1000} + + def guarded_ok() -> dict[str, Any]: + counter["port"] += 1 + ok = next(s for s in SCENARIOS if s.name == "ok") + return run_one( + app, + ok, + "guard", + work / "pairs" / f"ok-{counter['port']}", + counter["port"], + think_s=think_s, + files=plain, + teach=False, + ) + + if handoff and handoff.get("status") == "draft": + results["run2"] = run2( + app, + Path(handoff["task_dir"]), + files["verify"], + teach_dir, + work / "taught-runs", + pairs=pairs, + run1=guarded_ok, + ) + finally: + app.stop() + # Paths under the temporary folder name nothing anyone can open later. + results = json.loads( + json.dumps(results, default=str).replace(str(work), "") + ) + results["summary"] = summarize(results) + return results + + +def _median(values: list[float]) -> Optional[float]: + values = sorted(v for v in values if v is not None) + if not values: + return None + mid = len(values) // 2 + return ( + values[mid] + if len(values) % 2 + else round((values[mid - 1] + values[mid]) / 2, 3) + ) + + +def summarize(results: dict[str, Any]) -> dict[str, Any]: + rows = results.get("runs") or [] + table: dict[str, dict[str, Any]] = {} + for row in rows: + cell = table.setdefault(row["scenario"], {}).setdefault(row["arm"], []) + cell.append(row["outcome"]) + per_scenario = { + s: {arm: sorted(set(v)) for arm, v in arms.items()} for s, arms in table.items() + } + counts: dict[str, dict[str, int]] = {} + for scenario in SCENARIOS: + if scenario.name == "ok": + continue + for arm, outcomes in per_scenario.get(scenario.name, {}).items(): + label = outcomes[0] if len(outcomes) == 1 else "mixed:" + "/".join(outcomes) + key = f"{arm}:{scenario.family}" + counts.setdefault(key, {}) + counts[key][label] = counts[key].get(label, 0) + 1 + ok = [r for r in rows if r["scenario"] == "ok"] + timing = { + arm: { + "run1_seconds_median": _median( + [r["run1_seconds"] for r in ok if r["arm"] == arm] + ), + "agent_actions": _median( + [r["agent_actions"] for r in ok if r["arm"] == arm] + ), + "hold_ms_median": _median( + [h for r in ok if r["arm"] == arm for h in r["hold_ms"]] + ), + "readback_ms_median": _median( + [r["readback_ms"] for r in ok if r["arm"] == arm] + ), + } + for arm in results.get("arms", []) + } + run2 = results.get("run2") or {} + run2_rows = run2.get("runs") or [] + ok2 = [r for r in run2_rows if r["scenario"] == "ok"] + return { + "per_scenario": per_scenario, + "counts": counts, + "timing_ok": timing, + "run2": { + "admitted": run2.get("admitted"), + "ok_seconds": ok2[0]["seconds"] if ok2 else None, + "ok_model_calls": ok2[0]["model_calls"] if ok2 else None, + "per_scenario": {r["scenario"]: r["outcome"] for r in run2_rows}, + }, + } + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--n", type=int, default=3) + parser.add_argument("--think", type=float, default=1.0) + parser.add_argument("--arms", default=",".join(ARMS)) + parser.add_argument("--port", type=int, default=9400) + parser.add_argument("--out", type=Path, default=HERE) + parser.add_argument( + "--pairs", + type=int, + default=0, + help="alternate a guarded run 1 and a run-2 tool call this many times", + ) + parser.add_argument( + "--keep-matrix", + type=Path, + help="an earlier results.json: keep its matrix, run only run 2 again", + ) + args = parser.parse_args(argv) + arms = tuple(a for a in args.arms.split(",") if a) + previous = json.loads(args.keep_matrix.read_text()) if args.keep_matrix else None + results = run_benchmark( + args.n, args.think, arms, args.port, pairs=args.pairs, previous=previous + ) + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "results.json").write_text( + json.dumps(results, indent=2, sort_keys=True, default=str) + "\n" + ) + print(json.dumps(results["summary"], indent=2, default=str)) + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/packages/flow/docs/BACKEND_PARITY.md b/packages/flow/docs/BACKEND_PARITY.md index 24e2df2ff..b59cb3046 100644 --- a/packages/flow/docs/BACKEND_PARITY.md +++ b/packages/flow/docs/BACKEND_PARITY.md @@ -99,6 +99,7 @@ drives an app). | `effects.kind: desktop_readback` ([#1280](https://github.com/OpenAdaptAI/OpenAdapt/pull/1280)) | refused: "a web record page uses effects.kind: ui_readback" | tested (UI Automation, WinForms stand-in) | works (OCR; Accessibility tree not read yet) | works (OCR; AT-SPI tree not read yet) | tested (network RDP, OCR) | works (OCR) | | `openadapt-agent serve --authoring` (pinning a window) | tested | tested | tested (frontmost window only) | **refused** | tested | tested | | `openadapt-agent` run tools | tested | works (surface from `--config`) | works | works | works | works | +| `openadapt-agent serve --teach` (record, check, compile, admit over MCP) | tested (`tests/e2e/test_mcp_teach_e2e.py`) | works (a person approves every save) | works (same) | works (same; needs `linux_allow_physical_input`) | tested; admission refused: pixel identity | tested; admission refused: pixel identity | - `SurfaceApp`: `tests/test_jit_surface_app.py` (every kind: validation, origin binding, Citrix readiness refusal) and @@ -128,6 +129,16 @@ drives an app). `--authoring --config deploy.yaml --authoring-from-config` for every surface except network RDP, which it refuses (each run's own session would disconnect the recording's). +- `serve --teach`: `tests/test_mcp_teach_surfaces.py` teaches MockMed over + MCP through the real `RemoteDisplayBackend` and `CitrixWorkspaceBackend`: + run 1 records, passes its record check and a person confirms it; the canary + is refused with the identity-coverage reason above, so nothing is admitted. + `tests/test_compile_agent_surface_harness.py` covers the surface harness on + a native window (what is gated, nothing performed when refused). Off the + web the harness can't read which record a form shows, so a person approves + every gated action in the client's form. Native backends list no controls + yet (the `authoring_tree()` gap below), so the agent reads the screen + (`--teach-screens`) and clicks pixels there too. - An agent clicks at window coordinates. Native Linux refuses coordinate input unless the section sets `linux_allow_physical_input: true` ("coordinate input is disabled for native Linux"), so agent-driven @@ -165,5 +176,5 @@ are not app URLs. | `route` on a desktop surface: no test | [#1264](https://github.com/OpenAdaptAI/OpenAdapt/pull/1264) | [#1288](https://github.com/OpenAdaptAI/OpenAdapt/pull/1288) | | `induce --probe-on` on a desktop test copy: no test | [#1276](https://github.com/OpenAdaptAI/OpenAdapt/pull/1276) | [#1289](https://github.com/OpenAdaptAI/OpenAdapt/pull/1289) | | `run --self-repair` refuses non-web: "only browser runs can be repaired for now" | [#1265](https://github.com/OpenAdaptAI/OpenAdapt/pull/1265) | refused with a reason; desktop support after it merges | -| `compile-agent` takes `--backend`/`--config` but refuses every non-web target (exit 2, before anything runs); its observe step and save gate read the DOM | [#1277](https://github.com/OpenAdaptAI/OpenAdapt/pull/1277) | refused with a reason; desktop support (via `jit.SurfaceApp`) after it merges | +| `compile-agent` takes `--backend`/`--config` but refuses every non-web target (exit 2, before anything runs); its Claude agent reads the DOM listing | [#1277](https://github.com/OpenAdaptAI/OpenAdapt/pull/1277) | refused with a reason; the same pipeline runs on every surface through `serve --teach` (`FirstRunCompiler(surface=...)` with the surface harness) | | The GTK3 AT-SPI qualification job is in `packages/flow/.github/`, which GitHub does not run in the monorepo | `.github/workflows/` | follow-up | diff --git a/packages/flow/docs/COMPILE_AGENT.md b/packages/flow/docs/COMPILE_AGENT.md index 262f6b8ff..0786a6a62 100644 --- a/packages/flow/docs/COMPILE_AGENT.md +++ b/packages/flow/docs/COMPILE_AGENT.md @@ -194,7 +194,49 @@ later.verified, later.model_calls, later.receipt_outcome # True, 0, "VERIFIED" `FirstRunCompiler` exposes the same steps one at a time (`record`, `confirm_checks`, `admit`, `emit`, `run`) for a flow where a person confirms -later. +later. A draft waiting for a person is admitted at a terminal with: + +```bash +openadapt-flow compile-agent admit --admit-as "Dana Reviewer" \ + --canary-param note="Canary test note" +``` + +It shows the person the proposed checks (unless someone confirmed them +already), the steps, and the canary result, and asks them to confirm each. +`--admit-key` admits with a registered key instead, once the checks are +confirmed. + +## An agent over MCP + +`openadapt-agent serve --teach verify.yaml` puts this pipeline behind MCP +tools, so an agent connected to the server (Claude Code, Codex, any MCP +client) teaches a task without leaving its conversation: `teach_start`, +`teach_observe` and `teach_act` for run 1, `teach_finish` for the record check +and the draft, `teach_confirm_checks` for a person to confirm the checks in the +client's form, and `teach_admit` for the canaries and the admission. The agent +is the inner agent here: `compile_agent.driven.DrivenRun` runs +`FirstRunCompiler.record` on a thread of its own and hands each tool call's +action to the harness. Admission follows [JIT_ADMISSION.md](JIT_ADMISSION.md): +the server records as the registered agent whose key the operator gave it, +so admitting is self-admission and needs a policy with +`allow_self_admission`; without a key and policy, a person admits with the +command above. An admitted program joins the server's `list_workflows` and +gets a `taught_` tool. See the +[agent README](../../agent/README.md#teach-a-new-workflow-from-your-agent). + +The MCP tools also run on Windows, macOS, Linux, RDP and Citrix: pass +`FirstRunCompiler(surface=)`. Off the web the +harness is `SurfaceHarness`. It reads the authoring session's +accessibility tree (no values, no window titles). It can't read which form or +row a control is in, so it gates more than the web harness: every click except +one, named by its ref, on a text field, a check box, a tab or a control whose +label plainly navigates (Open, New, Cancel), and every click at x,y, Enter, +typing outside a text field, single-letter keys and shortcuts. A person +approves each gated action (the hold); without a hold, it is refused. Over MCP +the approval is bound to the run, the exact action and the number of actions +performed so far, so it covers only what the person was shown. On +RDP and Citrix the canary is refused until pixel identity is qualified +([BACKEND_PARITY.md](BACKEND_PARITY.md)). Any agent works if it has a `dialect` (an `openadapt_flow.jit` action format) and `drive(request, harness)`. `ScriptedAgent` follows a list of steps by @@ -235,13 +277,12 @@ object with `messages.create`; the tests pass a fake. ## Limits -- Web apps only so far. The JIT captures and runs through a Playwright - browser, and the harness reads the page's DOM. A Windows, macOS, Linux, RDP - or Citrix target (`--backend`, or a `--config` whose `backend.kind` names - one) and any URL that is not http(s) are refused before anything runs; there - is no fallback to a browser. Those surfaces need the JIT to drive the - backends `record --backend` uses and an observer over UI Automation, - Accessibility, AT-SPI or pixels. +- The `compile-agent` command runs web apps only so far: its Claude agent + reads the page's DOM listing. A Windows, macOS, Linux, RDP or Citrix target + (`--backend`, or a `--config` whose `backend.kind` names one) and any URL + that is not http(s) are refused before anything runs; there is no fallback + to a browser. The MCP teach tools run the same pipeline on those surfaces + (above). - The save gate recognizes a click that commits by the control's label. An icon with no label, or a label that names no write ("OK"), is not gated in run 1; the record check after run 1 still refuses a wrong record, but only diff --git a/packages/flow/docs/GUARD_AN_AGENT.md b/packages/flow/docs/GUARD_AN_AGENT.md new file mode 100644 index 000000000..541d09249 --- /dev/null +++ b/packages/flow/docs/GUARD_AN_AGENT.md @@ -0,0 +1,253 @@ +# Guard an agent you already run + +Preview. The command, the file formats and the tool names may change before a +release. + +If your agent already drives a Chromium (a Playwright script, Playwright MCP, +browser-use, or a computer-use agent with a browser), you keep it. You start +its browser with a DevTools port and run `openadapt-flow guard` next to it. +The agent doesn't call OpenAdapt and doesn't know it is there. + +- **Run 1, guarded.** The guard holds every save the agent's tab sends until + the record on the screen is the one the task names. If it isn't, the guard + blocks the save and the agent's page says why. After a save goes through, + the guard reads the record back from the system of record. +- **Run 2, compiled.** When run 1 reads back correctly, it becomes a draft + program with a record check built from the app's own reads. A person + confirms the check and admits the draft. Then it is a tool your agent calls, + and that tool makes no model calls. + +Nothing leaves your computer, and the guard makes no model calls. + +## Set it up + +1. Add a DevTools port to the browser your agent launches. This is the only + change to the agent: + + | Agent | The line | + |---|---| + | Playwright (Python) | `chromium.launch(args=["--remote-debugging-port=9222"])` | + | Playwright MCP | `"args": ["@playwright/mcp", "--cdp-endpoint", "http://127.0.0.1:9222"]` with Chromium started on that port | + | browser-use | pass `cdp_url="http://127.0.0.1:9222"` to its browser, with Chromium started on that port | + | Your own Chromium | `--remote-debugging-port=9222` on its command line | + + The endpoint must be on this computer (`127.0.0.1` or `localhost`). + +2. Write `guard.yaml` once per app: + + ```yaml + schema_version: openadapt.agent-guard/v1 + app_url: http://127.0.0.1:8765/#tasks # the guard holds saves to this origin + verify: verify.yaml # how the save is read back + screen_selector: "#patient-banner" # optional: where the record is shown + not_saves: # optional: writes that aren't the save + - {method: POST, path: /chart/signin} + approval: not_gated # each_item: a person approves each save + teach: # optional: make run 2 a program + dir: .openadapt-teach + name: triage_note + ``` + + `verify.yaml` is the file [`verify-run`](VERIFY_RUN.md) reads: the reader + for the system of record and the record the save must produce. See + [GUARDED_SAVE.md](GUARDED_SAVE.md) for an example. + +3. For each task, whoever assigns the work writes the task file. The agent + never writes it, so it can't change what it is checked against: + + ```json + { + "schema_version": "openadapt.guard-task/v1", + "task_id": "triage-p1-0001", + "expected_record": {"patient_id": "p1", "name": "Jane Sample"}, + "intended_fields": {"type": "Triage", "note": "Knee pain, refer"} + } + ``` + +4. Start the guard before the agent starts the task: + + ```bash + openadapt-flow guard --cdp http://127.0.0.1:9222 --config guard.yaml --task task.json --out runs/triage-p1-0001 + ``` + + The guard waits for a tab on `app_url`'s origin, attaches, and prints + `guard: attached`. Then run your agent as usual. The guard ends after the + save is read back. + +## What the guard does with each request + +The guard pauses the tab's requests with the CDP `Fetch` domain: documents +(form posts), `fetch`, XHR, beacons and others. Images, event streams, +scripts, styles, fonts and media aren't paused. + +| Request | What happens | +|---|---| +| `GET`, `HEAD`, `OPTIONS`, a GraphQL `query` | Goes on at once. | +| A write listed in `not_saves` | Goes on. | +| `POST`, `PUT`, `PATCH`, `DELETE`, another method, or a GraphQL `mutation` to the app's origin | Held. This is a save. | +| A write to another origin | Blocked, unless it is in `not_saves`. | + +A GraphQL body that Flow can't read, such as a persisted query with no text, +counts as a save. + +For a save, the guard runs the [guarded save](GUARDED_SAVE.md) contract on +the agent's behalf: + +1. It reads the text of the record on the screen (`screen_selector`, or the + whole page) and checks that every `expected_record` value is there as a + whole word. It reads the system of record for a baseline. +2. It commits the task once. A second save of the same task is blocked. + With `approval: each_item`, the save waits for a person + (`openadapt-flow item-approval approve ...`, Desktop or the phone portal). +3. It lets the request go on. When the request finishes, it reads the record + back through `verify-run`: `VERIFIED`, `REFUTED` or + `RECONCILIATION_REQUIRED`, with a receipt. + +A blocked save gets a `409` response with an `X-OpenAdapt-Guard` header. A +`fetch` or XHR gets a JSON body that names the reason. A form post gets a +page that says why. The guard also puts a notice at the top of the agent's +page (`role="alert"`), so an agent that reads the screen or the accessibility +tree sees it. The notice names the fields that didn't match, not their +values. + +### Observe only + +`--observe-only` (or `observe_only: true`) never blocks. The guard runs the +same checks, records what it would have blocked, and still reads the save +back. Use it to see what the guard would do before you turn it on. + +### Fail-closed + +The default blocks when the guard can't tell that a save is safe: the record +isn't on the screen, the system of record can't be read before the write, a +second save of the task, a write to another origin, or an error in the guard. +If the guard stops while a save is paused, it blocks that save before it lets +go of the tab, because Chromium sends paused requests on when the session +that paused them goes away. + +## Run 2: a program from run 1 + +With a `teach:` section, a run that reads back `VERIFIED` is handed to the +teach dir, the same place the MCP teach tools +([packages/agent](../../agent/README.md#teach-a-new-workflow-from-your-agent)) +keep their drafts: + +1. The recording (with its network capture) is compiled. When the save is a + form post that replays cleanly, `compile` proposes an API step + (`api_proposal.json` says which step, or why not). The GUI steps stay as + the fallback. +2. Run 1's record change is checked again: exactly one new record, nothing + else changed, the record the task named, every input in it. +3. `scaffold-verifier` builds the record check from the app's own reads (the + reads the page made after the save, `teach.db` for a database read path, or + the `verify.yaml` read) and fault-tests it. If it misses a fault, it is + kept for review and the check from the before and after reads is proposed + instead. +4. The draft is recorded as made by the attached agent, with no agent + principal. Only a person or a registered admitter key can admit it. + +Then a person confirms the check and admits: + +```bash +openadapt-flow compile-agent admit .openadapt-teach/ --admit-as "Dana Reviewer" --canary-param note="Canary note" +openadapt-agent serve --teach verify.yaml --teach-dir .openadapt-teach +``` + +The admitted program is the `taught_` tool. Each call runs it with no +model calls and reads the save back. + +## Measured on MockMed + +Synthetic MockMed in front of the `effect_e2e` record store, localhost only, +no model calls anywhere. The outside agent is a scripted Playwright agent in +its own Chromium, with a 1 s pause before each of its 6 actions where a +model-driven agent would think. 15 scenarios, 3 arms, 2 trials each (90 runs, +the same outcome in both trials), one browser at a time, each judged by the +`effect_e2e` ground truth, which reads the SQLite file directly. Full tables +and raw rows: [benchmark/agent_guard/RESULTS.md](../benchmark/agent_guard/RESULTS.md). + +The host was shared with other work: the 1-minute load average was 16 to 249 +during the runs (median 29). The times below compare run 1 with run 2 under +the same load; they aren't absolute speeds. + +**Setup the developer adds:** 1 line in the agent (the DevTools port on the +browser it launches), 1 command, a 9-line `guard.yaml` and a 16-line +`verify.yaml` once per app, and a 12-line task file per task from whoever +assigns the work. + +**The `effect_e2e` faults**, injected in the app's store. Eight of the nine +leave a wrong or missing record: + +| Of the 8 | Blocked by the guard | Caught by the read-back | Got through | +|---|---|---|---| +| Agent alone | n/a | n/a | 8: 6 with the screen saying saved, 2 with an error on screen | +| Guard, run 1 | 0 | 7 | 1 (`collateral_unaudited`) | +| Program, run 2 | 0 | 8 | 0 | + +The guard can't block a fault in the app's server; it reads the record back +and reports it. Run 1's read is the app's REST record read, which doesn't +cover the billing table, so the stray billing write gets through, as it does +for the same read in [EFFECT_E2E.md](../benchmark/effect_e2e/EFFECT_E2E.md). +Run 2's record check came from scaffold-verifier with the database read path +(`teach.db`), so it guards the billing table too. + +`timeout` (the save commits, then the server answers after 1.5 s) is a right +save: the agent alone sees it saved, and the guard and run 2 read it back +`VERIFIED`. When MockMed's page gives up +after 1.2 s, the agent alone sees an error for a save that landed; the guard +reads it back `VERIFIED`. + +**What the guard blocks** is the agent's side: a save sent while the wrong +patient was on the screen (blocked with `identity_mismatch`; nothing was +written; the agent's page named the fields that didn't match), and the second +save when the page sent it twice (blocked; one record). Observe-only lets +both through and reports them on the read-back. + +**Run 1 against run 2**, 5 pairs back to back at a load of 19 to 23: + +| | Median time | Model calls | +|---|---|---| +| Run 1: the agent, guarded | 6.26 s | 0 here (scripted); about 6 for a model-driven agent, one per action | +| Run 2: the `taught_triage_note` tool over MCP | 4.39 s | 0 | + +The guard held each save for its checks for a median 7.4 ms and read it back +in 41 ms; the agent's time with and without the guard was the same (6.24 s). +Run 1's time here is mostly the scripted pauses, so with a real model run 1 is +longer and run 2 is unchanged. Compiling run 1 into the draft took about 8 s +after the read-back. A person's admission (`compile-agent admit`, one attended +canary) took 8 s. + +Run 2 ran under the JIT standard profile: durable, with governed +authorization. No API step was proposed. The compiler's recorded reason is +"the recording has no parameters to map into a request" (the attached +recorder marks no parameters; inputs are lifted after compile). MockMed's +save is also a JSON request built by script, which isn't proposed either, and +an API step doesn't run in a durable or governed run. So the program ran its +GUI steps, and the run-2 times are GUI times. + +## Limits + +- One task per guard session, from when the guard attaches until the save is + read back. A save sent before the guard attaches isn't seen. Start the + guard before the agent starts the task. +- One tab: the tab on the app's origin. A save from another tab or a popup + isn't guarded. +- Saves sent over a WebSocket aren't seen. CDP `Fetch` doesn't pause them. +- The record check is text on the screen. If the screen shows the expected + values for two records (a list), the check can pass on the wrong row. Set + `screen_selector` to the element that shows the open record. +- The guard can't stop a fault in the app's server. It reads the record back + after the save, so it reports those faults; it doesn't prevent them. +- The agent must click a field before it types into it. An agent that fills a + field without clicking it (Playwright's `fill`, which Playwright MCP's + typing tool uses) leaves no focus step in the recording, and the compiled + program refuses at the canary because it can't confirm what it typed. Run 1 + is still guarded and read back. +- Compiling run 1 needs the recording. The recorder refuses a run it can't + keep exactly (an agent that acts faster than a frame can be kept). The + guard keeps guarding; only run 2 is missing. +- API steps are proposed only for form posts + (`application/x-www-form-urlencoded`). A save built by script as JSON, like + MockMed's, stays on GUI steps. Programs from the teach dir run as durable + JIT runs, and API steps don't run in durable runs yet, so a taught program + runs its GUI steps. diff --git a/packages/flow/docs/commands/01-record-and-build/compile-agent.md b/packages/flow/docs/commands/01-record-and-build/compile-agent.md index e69de29bb..825e7db09 100644 --- a/packages/flow/docs/commands/01-record-and-build/compile-agent.md +++ b/packages/flow/docs/commands/01-record-and-build/compile-agent.md @@ -0,0 +1 @@ +With `admit TASK_DIR`, a person confirms and admits a waiting draft, such as one an agent taught over MCP (`openadapt-agent serve --teach`). diff --git a/packages/flow/docs/commands/11-guard-an-agent-you-already-run/README.md b/packages/flow/docs/commands/11-guard-an-agent-you-already-run/README.md new file mode 100644 index 000000000..d1ec0994d --- /dev/null +++ b/packages/flow/docs/commands/11-guard-an-agent-you-already-run/README.md @@ -0,0 +1 @@ +# Guard an agent you already run diff --git a/packages/flow/docs/commands/11-guard-an-agent-you-already-run/guard.md b/packages/flow/docs/commands/11-guard-an-agent-you-already-run/guard.md new file mode 100644 index 000000000..6bc864889 --- /dev/null +++ b/packages/flow/docs/commands/11-guard-an-agent-you-already-run/guard.md @@ -0,0 +1 @@ +Attaches to the agent's own Chromium over CDP, holds each save until the right record is on screen, reads it back, and hands a verified run to the teach dir. diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 36db4af05..f1e539e75 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -10989,6 +10989,10 @@ def _repair_store_flag(rp: argparse.ArgumentParser) -> None: _add_compile_agent_parser(sub) + from openadapt_flow.agent_guard.cli import add_parser as _add_guard_parser + + _add_guard_parser(sub) + from openadapt_flow.cli_fleet import register_fleet_parser register_fleet_parser(sub) diff --git a/packages/flow/openadapt_flow/agent/bridge.py b/packages/flow/openadapt_flow/agent/bridge.py index c8e52e5de..c2b1a4688 100644 --- a/packages/flow/openadapt_flow/agent/bridge.py +++ b/packages/flow/openadapt_flow/agent/bridge.py @@ -242,6 +242,18 @@ def __init__( entry.legacy_id: entry for entry in self.catalog.values() if entry.legacy_id } + def add_catalog_entry(self, entry: CatalogEntry) -> None: + """Add a workflow while the server runs (a program taught over MCP). + + The run service sees it at once; ``list_workflows`` and the + ``run_workflow`` schema list it from the next call. + """ + if entry.name in self.catalog: + raise BridgeError("a workflow with that name already exists") + self.catalog[entry.name] = entry + if self._service is not None: + self._service.catalog[entry.name] = entry + @property def service(self) -> RunService: """The run service, created on first use so read-only servers stay inert.""" diff --git a/packages/flow/openadapt_flow/agent/cli.py b/packages/flow/openadapt_flow/agent/cli.py index 7500f3776..ca7603413 100644 --- a/packages/flow/openadapt_flow/agent/cli.py +++ b/packages/flow/openadapt_flow/agent/cli.py @@ -204,6 +204,68 @@ def build_parser() -> argparse.ArgumentParser: "save must produce. Needs --guard-tasks and --guard-state." ), ) + p.add_argument( + "--teach", + default=None, + metavar="VERIFY_YAML", + help=( + "Add the teach tools: the agent does a task once in the app at --url " + "(or the --config backend: Windows, macOS, Linux, RDP, Citrix), " + "Flow checks the saved record through VERIFY_YAML's verifier, " + "compiles the run, and once it is admitted the agent calls it as a " + "tool with no model calls. Run 1 makes a real entry: use a test copy." + ), + ) + p.add_argument( + "--teach-dir", + default=None, + metavar="DIR", + help="With --teach: where drafts and taught programs live (default RUNS_DIR/teach)", + ) + p.add_argument( + "--teach-key", + default=None, + metavar="PEM", + help=( + "With --teach and --teach-policy: the encrypted key of the registered " + "agent that may admit what it taught (passphrase from " + "$OPENADAPT_JIT_KEY_PASSPHRASE). Without it, a person admits at a " + "terminal (openadapt-flow compile-agent admit)." + ), + ) + p.add_argument( + "--teach-policy", + default=None, + metavar="JSON", + help=( + "With --teach-key: the admission policy a person signed " + "(openadapt-flow jit sign-policy). The agent admits its own draft " + "only if it sets allow_self_admission." + ), + ) + p.add_argument( + "--key-dir", + default=None, + metavar="DIR", + help="With --teach: the JIT key directory (default $OPENADAPT_JIT_KEY_DIR or ~/.openadapt/jit)", + ) + p.add_argument( + "--teach-hold", + action="store_true", + help=( + "With --teach on a web app: a person also approves every save in " + "the client's form (always on for other surfaces)." + ), + ) + p.add_argument( + "--teach-screens", + choices=("off", "scrubbed", "unscrubbed"), + default="off", + help=( + "With --teach: add teach_screen so the agent sees the app (RDP and " + "Citrix need it). scrubbed shares nothing if scrubbing fails." + ), + ) p.add_argument( "--guard-tasks", default=None, @@ -596,6 +658,17 @@ def _cmd_serve(args: argparse.Namespace) -> int: if args.guard: return _serve_guard(args, serve) + if args.teach: + return _serve_teach(args, serve) + if ( + args.teach_key + or args.teach_policy + or args.teach_dir + or args.teach_hold + or args.teach_screens != "off" + ): + print("serve: the --teach-* flags need --teach VERIFY_YAML", file=sys.stderr) + return 2 desktop_pin = bool( args.windows_agent_url or args.remote_window @@ -920,6 +993,131 @@ def _serve_guard(args: argparse.Namespace, serve) -> int: return 0 +def _teach_app(args: argparse.Namespace) -> tuple[Optional[str], Any]: + """The app to teach in: (url, None) for a web app, (None, backend section) + for any other surface.""" + + if not args.config: + if not args.url: + raise ValueError("--teach needs the app: --url, or --config deploy.yaml") + return args.url, None + from openadapt_flow.deployment import load_deployment + + section = load_deployment(args.config).backend + if (section.kind or "web").strip().lower() == "web": + url = args.url or getattr(section, "url", None) + if not url: + raise ValueError("the deployment names a web app; pass --url") + return url, None + if args.url: + raise ValueError("--url names a web app; the deployment names another surface") + if section.kind == "rdp" and section.rdp_host: + raise ValueError( + "network RDP (rdp_host) can't be taught; use the local client window " + "(rdp_window), as for authoring" + ) + return None, section + + +def _serve_teach(args: argparse.Namespace, serve) -> int: + """Serve the teach tools next to the workflow tools over local stdio.""" + from openadapt_flow.agent.bridge import AgentBridge + from openadapt_flow.agent.runner import default_flow_cli + from openadapt_flow.agent.teach import TeachBridge, TeachConfig + + if args.authoring or args.tutorial or args.mode in ("sandbox", "attended"): + print( + "serve: --teach has its own recording tools; it can't be combined " + "with --authoring, --tutorial, or --mode sandbox/attended", + file=sys.stderr, + ) + return 2 + if bool(args.teach_key) != bool(args.teach_policy): + print("serve: --teach-key and --teach-policy go together", file=sys.stderr) + return 2 + if args.bundles and args.mode != "production": + print( + "serve: with --teach, --bundles needs --mode production", + file=sys.stderr, + ) + return 2 + try: + from openadapt_flow.record_sor import load_observer + + url, surface = _teach_app(args) + # Read the verifier now: a typo fails here, not at the agent's first task. + load_observer(Path(args.teach).expanduser().resolve()) + key_dir = Path(args.key_dir).expanduser().resolve() if args.key_dir else None + admitter = None + if args.teach_key: + from openadapt_flow.jit import authority, lifecycle + + admitter = authority.principal_from_key( + authority.load_encrypted_key(Path(args.teach_key)), + key_dir or lifecycle.default_key_dir(), + policy=authority.SignedAdmissionPolicy.load(args.teach_policy), + ) + runs_dir = ( + (Path(args.runs_dir) if args.runs_dir else default_runs_dir()) + .expanduser() + .resolve() + ) + config = TeachConfig( + verify=Path(args.teach).expanduser().resolve(), + cache_dir=( + Path(args.teach_dir).expanduser().resolve() + if args.teach_dir + else runs_dir / "teach" + ), + url=url, + surface=surface, + key_dir=key_dir, + admitter=admitter, + hold=args.teach_hold, + headless=not args.headed, + screens=args.teach_screens, + ) + except Exception as exc: # noqa: BLE001 - every setup problem is a refusal + print(f"serve: {exc}", file=sys.stderr) + return 2 + runner_config = RunnerConfig( + flow_cli=( + tuple(shlex.split(args.flow_cli)) if args.flow_cli else default_flow_cli() + ), + runs_dir=runs_dir, + url=args.url, + deployment_config=args.config, + policy=args.policy, + timeout_s=args.timeout, + approval_mode=args.approval_mode, + ) + bridge = AgentBridge( + Path(args.bundles) if args.bundles else None, + runner_config, + allow_run=True, + mode="production", + ) + _attach_ledger(args, bridge, runs_dir) + teach = TeachBridge(config, workflows=bridge) + print( + f"openadapt-agent {__version__}: teach tools over local stdio on the " + f"{teach.backend_kind} app; admission " + + ( + f"by agent {admitter.principal_id} under policy " + f"{admitter.policy.policy.policy_id}" + if admitter is not None + else "by a person at a terminal" + ) + + f"; {len(teach.taught)} taught workflow(s); runs folder {runs_dir}", + file=sys.stderr, + ) + try: + serve(bridge, teach=teach) + finally: + teach.close() + return 0 + + def _attach_ledger(args: argparse.Namespace, bridge: Any, runs_dir: Path) -> None: """Give the bridge a task ledger one level inside its runs folder. diff --git a/packages/flow/openadapt_flow/agent/mcp.py b/packages/flow/openadapt_flow/agent/mcp.py index 621708c5e..4f7b0fd12 100644 --- a/packages/flow/openadapt_flow/agent/mcp.py +++ b/packages/flow/openadapt_flow/agent/mcp.py @@ -122,6 +122,50 @@ async def _confirm_with_person( raise BridgeError(declined) +async def _ask_person_name(context: Any, message: str) -> str: + """Ask the person for their name and a confirmation in the client's form.""" + session = context.session + if not client_supports_form_elicitation(session): + raise BridgeError( + "confirming the record check needs a person in an MCP client with " + "form elicitation; nothing was confirmed. A person can also run " + "`openadapt-flow compile-agent admit TASK_DIR` at a terminal" + ) + result = await session.elicit_form( + message, + { + "type": "object", + "properties": { + "name": {"type": "string", "title": "Your name"}, + "confirmed": { + "type": "boolean", + "title": "I am a person and I confirm this record check", + }, + }, + "required": ["name", "confirmed"], + }, + related_request_id=context.request_id, + ) + content = result.content or {} + name = content.get("name") + if ( + result.action != "accept" + or content.get("confirmed") is not True + or not isinstance(name, str) + or not name.strip() + ): + raise BridgeError("the person declined or cancelled; nothing was confirmed") + return name.strip() + + +async def _notify_tools_changed(context: Any) -> None: + """Tell a client that supports it that the tool list changed.""" + try: + await context.session.send_tool_list_changed() + except Exception: # noqa: BLE001 - the next tools/list shows it anyway + _LOG.debug("could not send tools/list_changed", exc_info=True) + + async def _confirm_attended_action(context: Any, name: str) -> None: """Require a second, protocol-native human confirmation before mutation.""" await _confirm_with_person( @@ -181,6 +225,7 @@ def _server_instructions( bridge: AgentBridge | None = None, guard: Any = None, ledger: Any = None, + teach: Any = None, ) -> str: text = "" if bridge is not None: @@ -237,6 +282,16 @@ def _server_instructions( from openadapt_flow.agent.guard_tools import GUARD_INSTRUCTIONS text += " " + GUARD_INSTRUCTIONS + if teach is not None: + text += ( + " To add a workflow, teach it: call teach_start with the task, then " + "teach_observe and teach_act until it is done, then teach_finish. " + "Run 1 makes a real entry in the app; use test records. A person " + "confirms the record check (teach_confirm_checks shows them a form) " + "and teach_admit admits it only under the operator's signed policy. " + "Once admitted, it is in list_workflows and has a taught_ tool " + "that runs with no model calls." + ) if ledger is not None: from openadapt_flow.agent.ledger import LEDGER_INSTRUCTIONS @@ -249,14 +304,17 @@ def build_server( authoring: AuthoringBridge | None = None, guard: Any = None, ledger: Any = None, + teach: Any = None, ) -> Server: - """Wrap workflow, authoring and/or guard bridges in an MCP Server (no I/O started). + """Wrap workflow, authoring, guard and/or teach bridges in an MCP Server + (no I/O started). - ``guard`` is a :class:`~openadapt_flow.agent.guard_tools.GuardBridge`. + ``guard`` is a :class:`~openadapt_flow.agent.guard_tools.GuardBridge`; + ``teach`` is a :class:`~openadapt_flow.agent.teach.TeachBridge`. ``ledger`` is a :class:`~openadapt_flow.agent.ledger.TaskLedger`; it adds the job tools and reads every run and guard result this server returns. """ - if bridge is None and authoring is None and guard is None: + if bridge is None and authoring is None and guard is None and teach is None: raise ValueError( "MCP server requires a workflow, authoring or guarded-save bridge" ) @@ -269,6 +327,7 @@ def _tool_specs(): *(bridge.list_tool_specs() if bridge is not None else ()), *(authoring.list_tool_specs() if authoring is not None else ()), *(guard.list_tool_specs() if guard is not None else ()), + *(teach.list_tool_specs() if teach is not None else ()), *(ledger.list_tool_specs() if ledger is not None else ()), ) @@ -315,6 +374,9 @@ async def _call_tool( or "tool arguments do not match the input schema" ) from None human_confirmed = False + teaching = teach is not None and teach.handles(name) + held_by_person: str | None = None + person: str | None = None admission_by_person = ( name == "decide_admission" and bridge is not None @@ -329,6 +391,27 @@ async def _call_tool( elif name == "admit" and authoring is not None and authoring.handles(name): await _confirm_admit(context, authoring) human_confirmed = True + elif teaching and name == "teach_act": + hold = await anyio.to_thread.run_sync( + teach.hold_needed, arguments or {} + ) + if hold: + await _confirm_with_person( + context, + teach.hold_message(hold), + title="I am a person and I approve this save", + unsupported=( + "this save needs a person's approval in an MCP client " + "with form elicitation; nothing was done" + ), + declined="the person did not approve the save; nothing was done", + ) + held_by_person = hold["binding"] + elif teaching and name == "teach_confirm_checks": + message = await anyio.to_thread.run_sync( + teach.confirm_checks_message, arguments or {} + ) + person = await _ask_person_name(context, message) def call() -> dict[str, Any]: payload = dict(arguments or {}) @@ -336,13 +419,17 @@ def call() -> dict[str, Any]: return dispatch(payload) if ledger.handles(name): return ledger.dispatch(name, payload) - if name == "run_workflow": + # A taught workflow's own tool is a run_workflow call. + taught_run = teaching and name in teach.taught + if name == "run_workflow" or taught_run: ledger.check_run(payload) result = dispatch(payload) # Item status comes only from this server's own results. A # ledger failure never hides a run's result from the caller. try: - ledger.observe(name, payload, result) + ledger.observe( + "run_workflow" if taught_run else name, payload, result + ) except Exception: _LOG.exception("could not record a result in the task ledger") return result @@ -350,6 +437,10 @@ def call() -> dict[str, Any]: def dispatch(payload: dict[str, Any]) -> dict[str, Any]: if guard is not None and guard.handles(name): return guard.dispatch(name, payload) + if teaching: + return teach.dispatch( + name, payload, held_by_person=held_by_person, person=person + ) if authoring is not None and authoring.handles(name): if human_confirmed: return authoring.dispatch(name, payload, human_confirmed=True) @@ -366,6 +457,12 @@ def dispatch(payload: dict[str, Any]) -> dict[str, Any]: # actions synchronously submit to their own non-async backend-owner # thread, so every MCP call can leave the event loop responsive. result = await anyio.to_thread.run_sync(call) + if ( + teaching + and name == "teach_admit" + and result.get("status") == "admitted" + ): + await _notify_tools_changed(context) except BridgeError as exc: return types.CallToolResult( content=[ @@ -406,7 +503,9 @@ def dispatch(payload: dict[str, Any]) -> dict[str, Any]: server = Server( SERVER_NAME, version=__version__, - instructions=_server_instructions(authoring, bridge, guard, ledger), + instructions=_server_instructions( + authoring, bridge, guard, ledger=ledger, teach=teach + ), ) async def _list_v1() -> list[types.Tool]: @@ -431,10 +530,29 @@ async def _call_v2(context: Any, params: types.CallToolRequestParams): server = Server( SERVER_NAME, version=__version__, - instructions=_server_instructions(authoring, bridge, guard, ledger), + instructions=_server_instructions( + authoring, bridge, guard, ledger=ledger, teach=teach + ), on_list_tools=_list_v2, on_call_tool=_call_v2, ) + if teach is not None: + # A taught workflow adds a tool while the server runs: say so in the + # capabilities, so clients listen for tools/list_changed. + from mcp.server.lowlevel.server import NotificationOptions + + initialization_options = server.create_initialization_options + + def _with_tools_changed( + notification_options: Any = None, *args: Any, **kwargs: Any + ): + return initialization_options( + notification_options or NotificationOptions(tools_changed=True), + *args, + **kwargs, + ) + + server.create_initialization_options = _with_tools_changed # type: ignore[method-assign] return server @@ -468,8 +586,9 @@ async def _run_stdio( bridge: AgentBridge | None, authoring: AuthoringBridge | None = None, guard: Any = None, + teach: Any = None, ) -> None: - server = build_server(bridge, authoring=authoring, guard=guard) + server = build_server(bridge, authoring=authoring, guard=guard, teach=teach) with _protected_stdout() as wire: kwargs = {"stdout": wire} if wire is not None else {} async with stdio_server(**kwargs) as (read_stream, write_stream): @@ -482,8 +601,12 @@ def serve( bridge: AgentBridge | None = None, authoring: AuthoringBridge | None = None, guard: Any = None, + teach: Any = None, ) -> None: """Serve the bridge over stdio until the client disconnects.""" + if teach is not None: + anyio.run(_run_stdio, bridge, authoring, guard, teach) + return if guard is None: anyio.run(_run_stdio, bridge, authoring) return diff --git a/packages/flow/openadapt_flow/agent/teach.py b/packages/flow/openadapt_flow/agent/teach.py new file mode 100644 index 000000000..ad4f8207d --- /dev/null +++ b/packages/flow/openadapt_flow/agent/teach.py @@ -0,0 +1,1013 @@ +"""Teach tools: an MCP agent turns a task it did once into a workflow it calls. + +``openadapt-agent serve --teach VERIFY_YAML`` adds these tools to the server: + +``teach_start`` + Opens the operator's app (``--url`` or ``--config``) and starts run 1 of a + task the agent names, with its inputs, the record fields the save must + produce, and the record's identity as the screen shows it. +``teach_observe``, ``teach_act``, ``teach_screen`` + The agent does the task one action at a time. Every action goes through + the compile-agent harness (:mod:`openadapt_flow.compile_agent.harness`): + on the web it checks that the right record is on screen before each save; + on Windows, macOS, Linux, RDP and Citrix it can't, so a person approves + each save in the client's form. +``teach_finish`` + Ends run 1. Flow compiles it, reads the system of record back, and reports + the draft: its steps, its parameters, the proposed record check, and what + still needs a person. +``teach_confirm_checks`` + A person confirms the proposed record check in the client's form. An + agent can't: the form asks the person for their name. +``teach_admit`` + Runs the canaries and admits the draft under the JIT rules + (``packages/flow/docs/JIT_ADMISSION.md``): the agent admits only as the + registered agent the operator configured, under a policy a person signed. + It records as that agent, so admitting is self-admission, which the policy + must allow (``allow_self_admission``). Without a key and a policy, a + person admits at a terminal. + +An admitted program joins ``list_workflows`` and ``run_workflow`` at once, and +gets its own ``taught_`` tool; the server tells clients that support it +that its tool list changed. Each run executes the program with no model calls +and reads every save back. + +The pipeline is :class:`~openadapt_flow.compile_agent.FirstRunCompiler`. This +module only adapts it to MCP calls; it decides nothing about admission. +""" + +from __future__ import annotations + +import base64 +import re +import threading +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Mapping, Optional + +from openadapt_flow.agent.bridge import BridgeError, ToolSpec + +__all__ = ["TEACH_TOOLS", "TaughtEngine", "TeachBridge", "TeachConfig"] + +TEACH_TOOLS = ( + "teach_start", + "teach_observe", + "teach_act", + "teach_screen", + "teach_finish", + "teach_abort", + "teach_status", + "teach_confirm_checks", + "teach_admit", +) +#: Tools that change state only after a person answers the client's form. +TEACH_PERSON_TOOLS = frozenset({"teach_confirm_checks"}) +TAUGHT_PREFIX = "taught_" + +_TASK_KEY = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_NAME = re.compile(r"^[a-z][a-z0-9_]{0,31}$") + +_READ_ONLY = { + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False, +} +_ACTS = { + "readOnlyHint": False, + "destructiveHint": True, + "idempotentHint": False, + "openWorldHint": True, +} +_LOCAL = { + "readOnlyHint": False, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False, +} + + +def _obj(properties: dict[str, Any], required: list[str]) -> dict[str, Any]: + return { + "type": "object", + "properties": properties, + "required": required, + "additionalProperties": False, + } + + +_TASK_KEY_PROP = { + "type": "string", + "description": "task_key from teach_start or teach_finish.", +} + + +@dataclass +class TeachConfig: + """What the operator fixed at server start. The agent changes none of it.""" + + verify: Path + cache_dir: Path + url: Optional[str] = None + #: A deployment ``backend:`` section for a native or remote app. + surface: Any = None + key_dir: Optional[Path] = None + #: A registered agent principal under a signed admission policy, or None. + admitter: Any = None + #: Ask a person before every save on the web too (always on elsewhere). + hold: bool = False + policy: str = "clinical-write" + headless: bool = True + #: Whether the agent sees the screen (``teach_screen``): ``off``, + #: ``scrubbed`` (identifiers scrubbed; nothing is shared if scrubbing + #: fails) or ``unscrubbed``. Off unless the operator turns it on. + screens: str = "off" + #: Tests only: the screen scrubber (default: openadapt-privacy's). + scrubber_factory: Optional[Callable[[], Any]] = None + #: Tests only: reset a demo app before every run. + reset: Optional[Callable[[], None]] = None + #: Tests only: build the live backend for a surface. + open_backend: Optional[Callable[[Any], Any]] = None + #: How long a taught run waits for run 1 or a canary to finish. + app_wait_s: float = 600.0 + + #: One writer at a time on the app: run 1, the canaries and taught runs + #: would otherwise count each other's records in their checks. + app_lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + + def __post_init__(self) -> None: + self.cache_dir = Path(self.cache_dir).resolve() + if self.screens not in ("off", "scrubbed", "unscrubbed"): + raise ValueError("screens is off, scrubbed or unscrubbed") + if (self.url is None) == (self.surface is None): + raise ValueError("teach needs the app: a --url, or a --config backend") + if self.admitter is not None: + if getattr(self.admitter, "kind", None) != "agent" or ( + getattr(self.admitter, "policy", None) is None + ): + raise ValueError( + "the teach server admits only as a registered agent under a " + "signed admission policy (--teach-key with --teach-policy); " + "an admitter key admits from CI with openadapt-flow jit admit" + ) + + +def _off_thread(fn: Callable[[], Any]) -> Any: + """Run ``fn`` on a fresh thread: browsers and native backends it opens + live and close there, away from the server's worker pool.""" + + box: dict[str, Any] = {} + + def main() -> None: + try: + box["value"] = fn() + except BaseException as exc: # noqa: BLE001 - re-raised below + box["error"] = exc + + thread = threading.Thread(target=main, name="openadapt-teach", daemon=True) + thread.start() + thread.join() + if "error" in box: + raise box["error"] + return box.get("value") + + +class TaughtEngine: + """Runs an admitted taught program for :class:`RunService`: no model + calls, every save read back, a verify-run receipt.""" + + def __init__(self, task_dir: Path, config: TeachConfig) -> None: + self.task_dir = Path(task_dir) + self.config = config + + def execute( + self, entry: Any, run_id: str, inputs: dict[str, str], raw: Mapping[str, Any] + ) -> Any: + from openadapt_flow.agent.service import EngineResult + from openadapt_flow.compile_agent import FirstRunCompiler + + compiler = FirstRunCompiler.load( + self.task_dir, + reset=self.config.reset, + open_backend=self.config.open_backend, + ) + if not self.config.app_lock.acquire(timeout=self.config.app_wait_s): + # A run 1 or a canary is still writing to the app. Nothing ran. + return EngineResult(reason="not_ready_to_run") + try: + replay = compiler.run(**inputs) + finally: + self.config.app_lock.release() + if replay.verified: + reason = "saved_and_checked" + elif replay.status == "not_admitted": + reason = "workflow_unavailable" + elif replay.status == "refused": + reason = "not_ready_to_run" + elif replay.receipt_outcome == "HALTED_BEFORE_EFFECT": + reason = "stopped_for_review" + else: + # Anything else may have written: a person checks the record. + reason = "save_not_confirmed" + return EngineResult( + reason=reason, + execution_outcome=replay.status if replay.status.isupper() else None, + transaction_outcome=replay.receipt_outcome, + model_calls=replay.model_calls, + seconds=replay.seconds, + run_dir=str(replay.run_dir) if replay.run_dir else None, + bundle_dir=str(self.task_dir / "program" / "bundle"), + ) + + def refresh(self, entry: Any, run_id: str) -> Any: + return None + + +class TeachBridge: + """The teach tools over :class:`FirstRunCompiler`, one run 1 at a time.""" + + def __init__(self, config: TeachConfig, workflows: Any = None) -> None: + self.config = config + self.workflows = workflows + self.config.cache_dir.mkdir(parents=True, exist_ok=True) + self._run: Any = None + self._compiler: Any = None + self._lock = threading.Lock() + #: taught tool name -> catalog workflow name + self.taught: dict[str, str] = {} + for task_dir in sorted(self.config.cache_dir.iterdir()): + if self._is_admitted(task_dir): + self._register(task_dir) + + # -- tool surface ---------------------------------------------------------- + + def handles(self, name: str) -> bool: + if name == "teach_screen": + return self.config.screens != "off" + return name in TEACH_TOOLS or name in self.taught + + @property + def backend_kind(self) -> str: + if self.config.surface is None: + return "web" + from openadapt_flow.jit.compile_agent import SurfaceApp + + return SurfaceApp(self.config.surface).kind + + def list_tool_specs(self) -> list[ToolSpec]: + on_web = self.backend_kind == "web" + specs = [ + ToolSpec( + name="teach_start", + description=( + "Teach OpenAdapt a task by doing it once. OpenAdapt opens the " + f"app this server was set up for ({self.backend_kind}) and " + "records what you do; you then call teach_observe and " + "teach_act until the task is done, and teach_finish. This " + "run makes a real entry in that app, so use test records. " + "task is the instruction with {name} for each input; inputs " + "are this run's values; record_fields are fields the saved " + "record must have (such as its patient id); on_screen is " + "the record's identity as the screen shows it (a name, an " + "MRN), checked before every save. If the task was already " + "taught, nothing runs and you get its status." + ), + input_schema=_obj( + { + "task": {"type": "string", "minLength": 4, "maxLength": 500}, + "inputs": { + "type": "object", + "additionalProperties": {"type": "string"}, + }, + "record_fields": { + "type": "object", + "minProperties": 1, + "additionalProperties": {"type": "string"}, + }, + "on_screen": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1}, + }, + "name": { + "type": "string", + "pattern": _NAME.pattern, + "description": "Short name for the workflow (a-z, 0-9, _).", + }, + }, + ["task", "inputs", "record_fields", "on_screen"], + ), + annotations=_ACTS, + ), + ToolSpec( + name="teach_observe", + description=( + "Read the app again. " + + ( + "You get the page as short lines: headings, text, table " + "rows, and every control with a ref such as [e3]." + if on_web + else "You get one line per control with a ref such as " + "[n_1a2b3c4d]; a remote session has none, so use " + "teach_screen and click x,y." + ) + + " Refs change after every action." + ), + input_schema=_obj({}, []), + annotations=_READ_ONLY, + ), + ToolSpec( + name="teach_act", + description=( + "Do one action in run 1: click (ref, or x and y), type " + "(text, or input to type a task input's value; ref to " + "focus a field first), press (key: Enter, Tab, ...) or " + "hotkey (key and modifiers). Returns the app's new state. " + + ( + "Before any save the harness checks that the record's " + "identity is shown next to the control; a save on the " + "wrong record is refused and ends run 1." + if on_web and not self.config.hold + else "A person approves every save in your client's form " + "before it happens; a save they don't approve ends run 1." + ) + ), + input_schema=_obj( + { + "action": { + "type": "string", + "enum": ["click", "type", "press", "hotkey"], + }, + "ref": {"type": "string", "maxLength": 64}, + "x": {"type": "integer", "minimum": 0}, + "y": {"type": "integer", "minimum": 0}, + "text": {"type": "string", "maxLength": 4000}, + "input": {"type": "string", "maxLength": 64}, + "key": {"type": "string", "maxLength": 32}, + "modifiers": { + "type": "array", + "items": {"type": "string", "maxLength": 16}, + "maxItems": 4, + }, + }, + ["action"], + ), + annotations=_ACTS, + ), + ToolSpec( + name="teach_finish", + description=( + "End run 1 when the task is done. OpenAdapt compiles what you " + "did, reads the app's records to check that exactly the " + "record the task named was saved, and returns the draft: " + "its steps, its parameters, the proposed record check, and " + "what still needs a person." + ), + input_schema=_obj({}, []), + annotations=_LOCAL, + ), + ToolSpec( + name="teach_abort", + description="Stop run 1 now. Nothing is compiled.", + input_schema=_obj({}, []), + annotations=_LOCAL, + ), + ToolSpec( + name="teach_status", + description=( + "The draft of a taught task: steps, parameters, the proposed " + "record check, and what still needs a person." + ), + input_schema=_obj({"task_key": _TASK_KEY_PROP}, ["task_key"]), + annotations=_READ_ONLY, + ), + ToolSpec( + name="teach_confirm_checks", + description=( + "Ask a person to confirm the proposed record check. Your " + "client shows them a form with the check, and only their " + "answer there confirms it; there is no argument you can " + "send to confirm it. drop_fields are fields that change on " + "every save (an id, a time stamp) to leave out of the check." + ), + input_schema=_obj( + { + "task_key": _TASK_KEY_PROP, + "drop_fields": { + "type": "array", + "items": {"type": "string", "maxLength": 64}, + "maxItems": 20, + }, + }, + ["task_key"], + ), + annotations=_LOCAL, + ), + ToolSpec( + name="teach_admit", + description=( + "Admit the draft so it can run. OpenAdapt runs canaries (real " + "saves with canary_inputs, on the record the task names, so " + "use values that are safe there) and admits only under a " + "signed admission policy the operator set up for you. " + "Without one, it tells you the command a person runs at a " + "terminal. Once admitted, the workflow is in list_workflows " + "and has its own taught_ tool." + ), + input_schema=_obj( + { + "task_key": _TASK_KEY_PROP, + "canary_inputs": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "items": { + "type": "object", + "additionalProperties": {"type": "string"}, + }, + }, + }, + ["task_key", "canary_inputs"], + ), + annotations=_ACTS, + ), + ] + if self.config.screens != "off": + specs.insert( + 2, + ToolSpec( + name="teach_screen", + description=( + "The app's screen as a PNG" + + ( + ", with identifiers scrubbed (nothing is shared if " + "scrubbing fails)" + if self.config.screens == "scrubbed" + else "" + ) + + ". Use it when teach_observe is not enough (a remote " + "session has no control tree)." + ), + input_schema=_obj({}, []), + annotations=_READ_ONLY, + ), + ) + for tool, workflow in sorted(self.taught.items()): + specs.append(self._taught_spec(tool, workflow)) + return specs + + def _taught_spec(self, tool: str, workflow: str) -> ToolSpec: + from openadapt_flow.agent.cards import inputs_schema + + entry = self.workflows.catalog[workflow] + return ToolSpec( + name=tool, + description=( + f"Run the taught workflow {workflow}: {entry.card.purpose} It " + "runs a compiled program with no model calls and reads every " + "save back. Same outcomes as run_workflow; reuse request_id on " + "a retry." + ), + input_schema=_obj( + { + "inputs": inputs_schema(entry.card), + "request_id": {"type": "string", "minLength": 4, "maxLength": 128}, + }, + ["inputs", "request_id"], + ), + annotations={ + "readOnlyHint": False, + "destructiveHint": True, + "idempotentHint": True, + "openWorldHint": True, + }, + ) + + # -- what the MCP layer asks a person ------------------------------------------ + + def hold_needed(self, arguments: Mapping[str, Any]) -> Optional[dict[str, Any]]: + """For ``teach_act``: when a person must approve this action first, + what it may commit, a description, and the binding the approval + covers (this run, this action, the app as it is now). Else None.""" + + run = self._active() + if not run.person_approves_saves: + return None + answer = self._run_call("preview", arguments) + if "error" in answer or not answer.get("commit"): + return None # teach_act reports any error, performing nothing + return answer + + def hold_message(self, hold: Mapping[str, Any]) -> str: + commit = str(hold.get("commit")) + task = self._compiler.task if self._compiler is not None else None + identity = ", ".join(self._compiler.on_screen) if self._compiler else "" + return ( + "The agent is teaching OpenAdapt a task" + + (f" ({task.task_id})" if task is not None else "") + + f" and is about to {hold.get('action')} ({commit}), which may " + f"save a record. Look at the app: is the record {identity} on " + "screen, and should this happen? It is a real entry in this app. " + "Your answer covers only this action on the screen as it is now." + ) + + def confirm_checks_message(self, arguments: Mapping[str, Any]) -> str: + compiler = self._load(arguments.get("task_key")) + proposed = compiler.proposed() + drop = set(arguments.get("drop_fields") or []) + lines = [] + for effect in proposed["effects"]: + fields = ", ".join( + f"{name} = {value}" + for name, value in effect["match"].items() + if name not in drop + ) + lines.append(f"{effect['kind']}: {fields}") + params = ", ".join(proposed["params"]) or "none" + return ( + "Every later run of this taught workflow must pass this record " + "check, read from the app's records: " + + "; ".join(lines) + + f". Inputs that change on each run: {params}." + + (f" Left out: {', '.join(sorted(drop))}." if drop else "") + + " Confirm only if this is the record the task should write. " + "Enter your own name; an agent or model name is refused." + ) + + # -- dispatch ----------------------------------------------------------------- + + def dispatch( + self, + name: str, + arguments: Optional[Mapping[str, Any]] = None, + *, + held_by_person: Optional[str] = None, + person: Optional[str] = None, + ) -> dict[str, Any]: + """Run one teach tool. + + ``held_by_person`` (the control a person approved for this + ``teach_act``) and ``person`` (the name a person gave to confirm the + record check) are set only by the MCP layer after a person answered + the client's form; no argument stands in for them. + """ + + args = dict(arguments or {}) + if name in self.taught: + return self._run_taught(name, args) + handlers = { + "teach_start": self._start, + "teach_observe": lambda a: self._observe(), + "teach_screen": lambda a: self._screen(), + "teach_finish": lambda a: self._finish(), + "teach_abort": lambda a: self._abort(), + "teach_status": lambda a: self._status(a.get("task_key")), + "teach_admit": self._admit, + } + if name == "teach_act": + return self._act(args, held_by_person=held_by_person or None) + if name == "teach_confirm_checks": + return self._confirm_checks(args, person) + if name not in handlers or not self.handles(name): + raise BridgeError("unknown tool name") + return handlers[name](args) + + # -- run 1 ---------------------------------------------------------------------- + + def _active(self) -> Any: + run = self._run + if run is None or run.done.is_set(): + raise BridgeError( + "no run 1 is in progress; call teach_start (or teach_finish to " + "see how the last one ended)" + ) + return run + + def _run_call(self, kind: str, payload: Optional[Mapping[str, Any]] = None) -> dict: + from openadapt_flow.compile_agent.driven import RunEnded + + run = self._active() + try: + return run.call(kind, payload or {}) + except RunEnded as exc: + raise BridgeError(f"{exc}; call teach_finish for the result") from None + + def _compiler_for(self, args: Mapping[str, Any]) -> Any: + from openadapt_flow.compile_agent import FirstRunCompiler + + inputs = {str(k): str(v) for k, v in (args.get("inputs") or {}).items()} + for key in inputs: + if not _NAME.fullmatch(key): + raise BridgeError(f"input names are a-z, 0-9 and _ ({key!r})") + name = args.get("name") + try: + return FirstRunCompiler( + str(args["task"]), + params=sorted(inputs), + url=self.config.url, + surface=self.config.surface, + open_backend=self.config.open_backend, + verify=self.config.verify, + expect=dict(args["record_fields"]), + on_screen=list(args["on_screen"]), + task_id=f"taught.{name}" if name else None, + policy=self.config.policy, + cache_dir=self.config.cache_dir, + key_dir=self.config.key_dir, + reset=self.config.reset, + headless=self.config.headless, + on_warning=lambda text: None, + ), inputs + except ValueError as exc: + raise BridgeError(str(exc)) from None + + def _start(self, args: Mapping[str, Any]) -> dict[str, Any]: + from openadapt_flow.compile_agent.driven import DrivenRun + from openadapt_flow.jit.compile_agent import RECORDING_WARNING + + with self._lock: + if self._run is not None and not self._run.done.is_set(): + raise BridgeError( + "run 1 of another task is in progress; teach_finish or " + "teach_abort it first" + ) + compiler, inputs = self._compiler_for(args) + jit = compiler._jit() + if jit.admitted or jit.candidate() is not None: + return {"started": False, **self._status(compiler.task.key)} + if not self.config.app_lock.acquire(blocking=False): + raise BridgeError( + "a canary or a taught run is writing to the app; try " + "teach_start again when it is done" + ) + admitter = self.config.admitter + run = DrivenRun( + compiler, + inputs, + principal_id=admitter.principal_id if admitter is not None else None, + hold=self.config.hold, + on_done=self.config.app_lock.release, + ) + # Kept even if it fails to start: until its thread ends, no other + # run 1 may start on the app. + self._run, self._compiler = run, compiler + try: + run.start() + except Exception as exc: + # A run that opens late must not wait for actions forever. + run.abort() + raise BridgeError( + f"run 1 did not start: {exc}; it is being stopped" + ) from None + observation = self._run_call("observe") + return { + "started": True, + "status": "recording", + "task_key": compiler.task.key, + "surface": compiler.backend_kind, + "person_approves_saves": run.person_approves_saves, + "warning": RECORDING_WARNING, + "observation": observation.get("observation"), + "next": "teach_act until the task is done, then teach_finish", + } + + def _observe(self) -> dict[str, Any]: + return {"observation": self._run_call("observe").get("observation")} + + def _screen(self) -> dict[str, Any]: + png = self._run_call("screenshot").get("png") or b"" + scrubbed = False + if self.config.screens == "scrubbed": + from openadapt_flow.agent.repair import default_scrubber, scrub_png + + try: + png, _evidence = scrub_png( + png, self.config.scrubber_factory or default_scrubber + ) + except Exception: # noqa: BLE001 - fail closed: share nothing + raise BridgeError( + "the screen could not be scrubbed, so no screen was shared" + ) from None + scrubbed = True + return { + "scrubbed": scrubbed, + "screen_png_base64": base64.b64encode(png).decode("ascii"), + } + + def _act( + self, args: Mapping[str, Any], *, held_by_person: Optional[str] + ) -> dict[str, Any]: + run = self._active() + payload = dict(args) + if held_by_person: + payload["_held_by_person"] = held_by_person + answer = self._run_call("act", payload) + if "refused" in answer: + run.done.wait(120) + return { + "status": "refused", + "detail": answer["refused"], + "run_over": True, + "next": "teach_finish shows the result; nothing was saved or compiled", + } + if "error" in answer: + return { + "status": "error", + "error": answer["error"], + "detail": answer.get("detail"), + "performed": False, + } + return {"status": "ok", "observation": answer.get("observation")} + + def _abort(self) -> dict[str, Any]: + run = self._active() + run.abort() + run.done.wait(120) + return {"status": "aborted", "compiled": False} + + def _finish(self) -> dict[str, Any]: + run = self._run + if run is None: + raise BridgeError("no run 1 to finish; call teach_start") + try: + result = run.finish() + except Exception as exc: # noqa: BLE001 - reported to the agent + return { + "status": "capture_failed", + "detail": f"{type(exc).__name__}: {exc}", + } + report = self._report(self._compiler, result.status, result.detail) + report["record_check"] = result.record_check + report["save_gate"] = [ + {"control": g.get("control"), "decision": g.get("decision")} + for g in (result.run1.get("gates") or []) + ] + return report + + # -- the draft --------------------------------------------------------------------- + + def _load(self, task_key: Any) -> Any: + from openadapt_flow.compile_agent import FirstRunCompiler + + if not isinstance(task_key, str) or not _TASK_KEY.fullmatch(task_key): + raise BridgeError("unknown task_key") + task_dir = self.config.cache_dir / task_key + if not (task_dir / "compile_agent.json").is_file(): + raise BridgeError("unknown task_key") + return FirstRunCompiler.load( + task_dir, reset=self.config.reset, open_backend=self.config.open_backend + ) + + def _status(self, task_key: Any) -> dict[str, Any]: + compiler = self._load(task_key) + jit = compiler._jit() + if jit.admitted: + status = "admitted" + elif jit.candidate() is not None: + status = "draft" + else: + status = "no_draft" + return self._report(compiler, status, None) + + def _report( + self, compiler: Any, status: str, detail: Optional[str] + ) -> dict[str, Any]: + from openadapt_flow.compile_agent.pipeline import CONFIRMED_FILE + + jit = compiler._jit() + report: dict[str, Any] = { + "status": status, + "task_key": compiler.task.key, + "surface": compiler.backend_kind, + } + if detail: + report["detail"] = detail + if status == "admitted" or jit.admitted: + report["status"] = "admitted" + report["workflow"] = self._workflow_name(compiler.dir) + report["needs_person"] = [] + return report + candidate = jit.candidate() + if candidate is None: + report["needs_person"] = [] + if status not in ("draft", "no_draft"): + report["next"] = ( + "nothing can be admitted from this run; fix the cause and " + "call teach_start again" + ) + return report + lifted = candidate.get("params") or {} + report["steps"] = [ + {"id": s["id"], "action": s["action"]} for s in jit.status()["steps"] + ] + report["parameters"] = { + "inputs": sorted(compiler.task.params), + "lifted": [p.get("name") for p in lifted.get("lifted") or []], + "not_lifted": dict(lifted.get("unlifted") or {}), + } + try: + proposed = compiler.proposed() + report["proposed_record_check"] = proposed["effects"] + except (OSError, KeyError): + report["proposed_record_check"] = None + confirmed = (compiler.out_dir / CONFIRMED_FILE).is_file() + report["record_check_confirmed"] = confirmed + needs: list[str] = [] + if not confirmed: + needs.append( + "A person confirms the proposed record check: teach_confirm_checks " + "asks them in your client's form, or they run " + f"`{self._terminal_command(compiler)}` at a terminal." + ) + if self.config.admitter is None: + needs.append( + "A person admits the program at a terminal: " + f"`{self._terminal_command(compiler)}`. This server has no " + "admission policy, so teach_admit can't admit it." + ) + report["needs_person"] = needs + report["next"] = ( + "teach_confirm_checks" + if not confirmed + else ( + "teach_admit" if self.config.admitter is not None else "a person admits" + ) + ) + return report + + def _terminal_command(self, compiler: Any) -> str: + params = " ".join( + f"--canary-param {p}=" for p in sorted(compiler.task.params) + ) + return ( + f"openadapt-flow compile-agent admit {compiler.dir} --admit-as " + + params + ).strip() + + def _confirm_checks(self, args: Mapping[str, Any], person: Optional[str]) -> dict: + from openadapt_flow.jit.lifecycle import AdmissionError + + if not person: + raise BridgeError( + "a person confirms the record check in the client's form; " + "nothing was confirmed" + ) + compiler = self._load(args.get("task_key")) + try: + compiler.confirm_checks( + principal_id=person, drop_fields=list(args.get("drop_fields") or []) + ) + except (AdmissionError, ValueError, OSError) as exc: + raise BridgeError(f"not confirmed: {exc}") from None + return {**self._report(compiler, "draft", None), "confirmed_by": person} + + def _admit(self, args: Mapping[str, Any]) -> dict[str, Any]: + from openadapt_flow.jit.lifecycle import AdmissionError + + compiler = self._load(args.get("task_key")) + if self.config.admitter is None: + return { + **self._report(compiler, "draft", None), + "status": "needs_person", + "detail": ( + "this server has no admission policy, so an agent can't " + "admit; a person admits at a terminal" + ), + } + canaries = [ + {str(k): str(v) for k, v in row.items()} + for row in args.get("canary_inputs") or [] + ] + + def admit_and_emit() -> Any: + if not self.config.app_lock.acquire(blocking=False): + raise BridgeError( + "run 1 or a taught run is writing to the app; admit when it is done" + ) + try: + result = compiler.admit( + admitter=self.config.admitter, canary_params=canaries + ) + finally: + self.config.app_lock.release() + if result.admitted: + result.emitted = compiler.emit() + return result + + try: + result = _off_thread(admit_and_emit) + except (AdmissionError, ValueError) as exc: + raise BridgeError(f"not admitted: {exc}") from None + out: dict[str, Any] = { + "status": result.status, + "task_key": compiler.task.key, + "detail": result.detail, + "canaries": [ + { + "status": c.get("status"), + "receipt": c.get("receipt_outcome"), + "model_calls": c.get("model_calls"), + } + for c in (result.admission or {}).get("canaries") or [] + ], + } + if result.admitted: + tool = self._register(compiler.dir) + out.update( + { + "workflow": self.taught[tool], + "tool": tool, + "mode": (result.admission or {}).get("mode"), + "admitted_by": (result.admission or {}).get("admitted_by"), + "skill": (result.emitted or {}).get("skill"), + "next": f"call {tool}, or run_workflow with this workflow", + } + ) + return out + + def close(self, timeout_s: float = 30.0) -> None: + """Stop a run 1 the agent never finished (server exit). No draft.""" + + run = self._run + if run is not None and not run.done.is_set(): + run.abort() + run.done.wait(timeout_s) + + # -- admitted programs as workflows --------------------------------------------------- + + @staticmethod + def _is_admitted(task_dir: Path) -> bool: + return (task_dir / "compile_agent.json").is_file() and ( + task_dir / "program" / "bundle" / "admission.json" + ).is_file() + + def _workflow_name(self, task_dir: Path) -> Optional[str]: + for workflow in self.taught.values(): + entry = self.workflows.catalog.get(workflow) if self.workflows else None + engine = getattr(entry, "engine", None) + if isinstance(engine, TaughtEngine) and engine.task_dir == task_dir: + return workflow + return None + + def _register(self, task_dir: Path) -> str: + """Add an admitted program to the workflow catalog; returns its tool.""" + + import json + + from openadapt_flow.agent.cards import ( + DEFAULT_DONE_MEANS, + WorkflowCard, + _check_text, + _default_description, + default_purpose, + ) + from openadapt_flow.agent.service import CatalogEntry + + if self.workflows is None: + raise BridgeError("this server has no workflow tools to add the program to") + existing = self._workflow_name(task_dir) + if existing is not None: + return next(t for t, w in self.taught.items() if w == existing) + spec = json.loads((task_dir / "compile_agent.json").read_text()) + task = spec["task"] + base = re.sub(r"[^a-z0-9]+", "_", str(task["task_id"]).lower()).strip("_") + base = ( + base.removeprefix("taught_").removeprefix("compile_agent_")[:40] or "task" + ) + name = base + suffix = 2 + while name in self.workflows.catalog or TAUGHT_PREFIX + name in self.taught: + name = f"{base}_{suffix}" + suffix += 1 + try: + purpose = _check_text("purpose", f"Taught: {task['instruction']}") + except Exception: # noqa: BLE001 - an unusable instruction gets the default + purpose = default_purpose() + params = sorted(task.get("params") or []) + card = WorkflowCard( + name=name, + purpose=purpose, + done_means=DEFAULT_DONE_MEANS, + inputs={ + p: {"type": "string", "description": _default_description(p)} + for p in params + }, + required=params, + changes_records=True, + ) + self.workflows.add_catalog_entry( + CatalogEntry(card=card, engine=TaughtEngine(task_dir, self.config)) + ) + tool = TAUGHT_PREFIX + name + self.taught[tool] = name + return tool + + def _run_taught(self, tool: str, args: Mapping[str, Any]) -> dict[str, Any]: + return self.workflows.dispatch( + "run_workflow", + { + "workflow": self.taught[tool], + "inputs": args.get("inputs") or {}, + "request_id": args.get("request_id"), + }, + ) diff --git a/packages/flow/openadapt_flow/agent_guard/__init__.py b/packages/flow/openadapt_flow/agent_guard/__init__.py new file mode 100644 index 000000000..2924e88d0 --- /dev/null +++ b/packages/flow/openadapt_flow/agent_guard/__init__.py @@ -0,0 +1,33 @@ +"""Guard an agent you already run. + +A developer whose agent already drives a Chromium (Playwright MCP, +browser-use, a computer-use agent with a browser, their own Playwright +script) starts that browser with a DevTools port and runs +``openadapt-flow guard`` next to it. The agent doesn't change. + +* **Run 1, guarded.** The guard attaches to the agent's tab over CDP, holds + every save (:mod:`.requests`) until the record on screen is the one the task + names (:mod:`.commit_guard`, over :mod:`openadapt_flow.guard`), blocks it + and tells the agent why when it isn't, and reads the save back with + ``verify-run``. ``observe_only`` never blocks. +* **Run 2, compiled.** A run that reads back correctly becomes a draft + program (:mod:`.handoff`) with a record check built from the app's own + reads. A person confirms the check and admits it; then it is an MCP tool + that runs with no model calls (``openadapt-agent serve --teach``). + +See ``docs/GUARD_AN_AGENT.md``. +""" + +from openadapt_flow.agent_guard.commit_guard import CommitGuard, GuardSummary +from openadapt_flow.agent_guard.config import GuardConfig, load_config +from openadapt_flow.agent_guard.requests import HeldRequest, NotASave, classify + +__all__ = [ + "CommitGuard", + "GuardConfig", + "GuardSummary", + "HeldRequest", + "NotASave", + "classify", + "load_config", +] diff --git a/packages/flow/openadapt_flow/agent_guard/cdp.py b/packages/flow/openadapt_flow/agent_guard/cdp.py new file mode 100644 index 000000000..4ee64ad02 --- /dev/null +++ b/packages/flow/openadapt_flow/agent_guard/cdp.py @@ -0,0 +1,269 @@ +"""The commit guard's hands in the agent's tab: the CDP ``Fetch`` domain. + +:class:`CdpGuardPage` opens its own DevTools session on the tab the recorder +attached to, pauses the tab's requests at the request stage, and follows +``Network`` events to learn when a save finished. Playwright's sync API +delivers CDP events only while it waits on the wire, so the handlers only +queue them; :meth:`CdpGuardPage.drain` hands them to the +:class:`~openadapt_flow.agent_guard.commit_guard.CommitGuard` from the +session's loop. + +Paused resource types: documents (form posts), ``fetch``, XHR, beacons +(``Ping``) and ``Other``. Images, scripts, stylesheets, fonts, event streams +and media are GETs and aren't paused. A save sent over a WebSocket isn't seen +(CDP ``Fetch`` doesn't intercept WebSocket frames). + +When the session stops with a save still paused (waiting for a person, or an +error in the guard), the save is blocked before the DevTools session +detaches. Chromium lets paused requests go on when the session that paused +them goes away, so detaching first would let them through unchecked. +""" + +from __future__ import annotations + +import base64 +import json +from typing import Any, Optional + +from openadapt_flow.agent_guard.commit_guard import BLOCK_STATUS, GUARD_HEADER +from openadapt_flow.agent_guard.requests import HeldRequest + +__all__ = ["CdpGuardPage", "PAUSED_TYPES"] + +PAUSED_TYPES = ("Document", "XHR", "Fetch", "Ping", "Other") + +NOTICE_ID = "openadapt-guard-notice" + +_READ_SCREEN = """ +(selector) => { + const old = document.getElementById(%(notice)s); + if (old) { old.remove(); } + const root = selector ? document.querySelector(selector) : document.body; + return root ? (root.innerText || root.textContent || '') : null; +} +""" + +_NOTIFY = """ +(message, level) => { + let box = document.getElementById(%(notice)s); + if (!box) { + box = document.createElement('div'); + box.id = %(notice)s; + box.setAttribute('role', 'alert'); + box.setAttribute('aria-live', 'assertive'); + box.setAttribute('data-openadapt-guard', '1'); + box.style.cssText = 'position:fixed;left:0;right:0;top:0;z-index:2147483647;' + + 'padding:10px 14px;font:14px/1.4 system-ui,sans-serif;color:#fff;'; + (document.body || document.documentElement).appendChild(box); + } + box.style.background = level === 'ok' ? '#1f6f43' + : (level === 'info' ? '#1d4f91' : '#a1271b'); + box.textContent = message; + return true; +} +""" + +_CLEAR = """ +() => { + const old = document.getElementById(%(notice)s); + if (old) { old.remove(); } + return true; +} +""" + + +def _call(source: str) -> str: + return source % {"notice": json.dumps(NOTICE_ID)} + + +class CdpGuardPage: + """A :class:`~openadapt_flow.agent_guard.commit_guard.GuardPage` over CDP.""" + + def __init__(self, page: Any, *, screen_selector: Optional[str] = None) -> None: + self.page = page + self.screen_selector = screen_selector + self.cdp = page.context.new_cdp_session(page) + self._paused: list[dict[str, Any]] = [] + self._done: list[Optional[str]] = [] + #: Requests this session paused and hasn't answered yet. + self._open: dict[str, HeldRequest] = {} + # Playwright tags each handler, so a bound builtin won't do. + self.cdp.on("Fetch.requestPaused", lambda p: self._paused.append(p)) + self.cdp.on( + "Network.loadingFinished", lambda p: self._done.append(p.get("requestId")) + ) + self.cdp.on( + "Network.loadingFailed", lambda p: self._done.append(p.get("requestId")) + ) + self.cdp.send("Network.enable", {}) + self.cdp.send( + "Fetch.enable", + { + "patterns": [ + {"urlPattern": "*", "resourceType": kind, "requestStage": "Request"} + for kind in PAUSED_TYPES + ] + }, + ) + + # -- the loop ------------------------------------------------------------------ + + def drain(self, guard: Any) -> int: + """Hand queued requests and completions to ``guard``; returns how many + requests it decided.""" + + decided = 0 + while self._paused: + held = HeldRequest.from_cdp(self._paused.pop(0)) + self._open[held.request_id] = held + try: + guard.on_request(held) + except Exception: # noqa: BLE001 - fail closed on any guard error + self._block_open(held.request_id, "guard_error") + raise + decided += 1 + while self._done: + guard.on_request_done(self._done.pop(0)) + guard.poll() + return decided + + def release(self) -> None: + """Block every request still paused, then stop pausing and detach.""" + + for request_id in list(self._open): + self._block_open(request_id, "guard_stopped") + for params in self._paused: + self._block_open(str(params.get("requestId") or ""), "guard_stopped") + self._paused.clear() + for method in ("Fetch.disable",): + try: + self.cdp.send(method, {}) + except Exception: # noqa: BLE001 - the tab may be gone + pass + try: + self.cdp.detach() + except Exception: # noqa: BLE001 + pass + + def _block_open(self, request_id: str, code: str) -> None: + body = json.dumps( + { + "openadapt_guard": { + "blocked": True, + "code": code, + "message": "the guard stopped before this save was checked", + } + } + ) + self.block_request( + request_id, + status=BLOCK_STATUS, + body=body, + content_type="application/json", + headers={GUARD_HEADER: f"blocked; code={code}"}, + ) + + # -- GuardPage ------------------------------------------------------------------- + + def continue_request(self, request_id: str) -> None: + self._open.pop(request_id, None) + try: + self.cdp.send("Fetch.continueRequest", {"requestId": request_id}) + except Exception: # noqa: BLE001 - the page cancelled it + pass + + def block_request( + self, + request_id: str, + *, + status: int, + body: str, + content_type: str, + headers: dict[str, str], + ) -> None: + self._open.pop(request_id, None) + response_headers = [{"name": "Content-Type", "value": content_type}] + [ + {"name": k, "value": v} for k, v in headers.items() + ] + try: + self.cdp.send( + "Fetch.fulfillRequest", + { + "requestId": request_id, + "responseCode": status, + "responseHeaders": response_headers, + "body": base64.b64encode(body.encode("utf-8")).decode("ascii"), + }, + ) + except Exception: # noqa: BLE001 - the page cancelled it; nothing was sent + try: + self.cdp.send( + "Fetch.failRequest", + {"requestId": request_id, "errorReason": "BlockedByClient"}, + ) + except Exception: # noqa: BLE001 + pass + + def screen_text(self) -> Optional[str]: + try: + out = self.cdp.send( + "Runtime.callFunctionOn", + { + "functionDeclaration": _call(_READ_SCREEN), + "arguments": [{"value": self.screen_selector}], + "executionContextId": self._context_id(), + "returnByValue": True, + }, + ) + except Exception: # noqa: BLE001 - an unreadable screen is checked as such + return None + value = (out.get("result") or {}).get("value") + return value if isinstance(value, str) else None + + def screenshot(self) -> Optional[bytes]: + try: + out = self.cdp.send("Page.captureScreenshot", {"format": "png"}) + return base64.b64decode(out["data"]) + except Exception: # noqa: BLE001 + return None + + def notify(self, message: str, *, level: str) -> None: + try: + self.cdp.send( + "Runtime.callFunctionOn", + { + "functionDeclaration": _call(_NOTIFY), + "arguments": [{"value": message}, {"value": level}], + "executionContextId": self._context_id(), + "returnByValue": True, + }, + ) + except Exception: # noqa: BLE001 - a notice is best effort; the block isn't + pass + + def clear_notice(self) -> None: + try: + self.cdp.send( + "Runtime.callFunctionOn", + { + "functionDeclaration": _call(_CLEAR), + "executionContextId": self._context_id(), + "returnByValue": True, + }, + ) + except Exception: # noqa: BLE001 - best effort + pass + + def _context_id(self) -> int: + """A fresh isolated world on the tab's current document: it shares the + DOM, so it reads what is shown, but the page's own scripts can't + replace the functions it calls. A world lasts one document, so each + call makes one (a few per save).""" + + tree = self.cdp.send("Page.getFrameTree", {}) + frame_id = tree["frameTree"]["frame"]["id"] + world = self.cdp.send( + "Page.createIsolatedWorld", + {"frameId": frame_id, "worldName": "openadapt-guard"}, + ) + return int(world["executionContextId"]) diff --git a/packages/flow/openadapt_flow/agent_guard/cli.py b/packages/flow/openadapt_flow/agent_guard/cli.py new file mode 100644 index 000000000..97ea5cc67 --- /dev/null +++ b/packages/flow/openadapt_flow/agent_guard/cli.py @@ -0,0 +1,200 @@ +"""``openadapt-flow guard``: guard an agent you already run. + +Start your agent's Chromium with a DevTools port (the one line you add), then +start the guard for the task before the agent starts it:: + + openadapt-flow guard --cdp http://127.0.0.1:9222 \\ + --config guard.yaml --task task.json --out runs/task-0001 + +The guard attaches to the agent's tab, holds every save until the record on +screen is the one ``task.json`` names, reads the save back, and records the +run. With a ``teach:`` section, a run that reads back correctly becomes a +draft program a person confirms and admits (``compile-agent admit``). + +Exit status: 0 the save was read back VERIFIED; 3 a save was blocked and none +went through; 4 the save went through and was not verified (REFUTED or +RECONCILIATION_REQUIRED: a person checks the record); 5 no save was seen; +2 the config, the task or the browser could not be used. +""" + +from __future__ import annotations + +import argparse +import json +import signal +import sys +from pathlib import Path +from typing import Any + +__all__ = ["add_parser", "cmd_guard"] + + +def add_parser(sub: argparse._SubParsersAction) -> None: + p = sub.add_parser( + "guard", + help=( + "Guard an agent you already run: attach to its Chromium over CDP, " + "hold each save until the right record is on screen, read it back, " + "and record the run for a compiled program" + ), + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument( + "--cdp", + required=True, + help="the agent's Chromium DevTools endpoint on this computer " + "(http://127.0.0.1:9222)", + ) + p.add_argument("--config", required=True, help="guard.yaml") + p.add_argument( + "--task", + required=True, + help="the task file (openadapt.guard-task/v1) from whoever assigned it", + ) + p.add_argument("--out", required=True, help="new folder for this task's run") + p.add_argument( + "--page-url", + help="the tab's exact URL, when more than one tab is open on the app", + ) + p.add_argument( + "--observe-only", + action="store_true", + help="never block: log what the guard would block, still read back", + ) + p.add_argument( + "--approval", + choices=["not_gated", "each_item"], + help="override guard.yaml: each_item holds every save for a person", + ) + p.add_argument( + "--timeout", type=float, default=None, help="seconds before the guard stops" + ) + p.add_argument( + "--attach-wait", + type=float, + default=60.0, + help="seconds to wait for the agent's tab on the app", + ) + p.add_argument( + "--secret", + action="append", + default=[], + metavar="FIELD", + help="a field name or id whose value is never recorded (repeatable)", + ) + p.add_argument( + "--no-teach", + action="store_true", + help="don't hand a verified run to the teach dir", + ) + p.add_argument("--json", action="store_true", help="print the result as JSON") + p.set_defaults(func=cmd_guard) + + +def cmd_guard(args: argparse.Namespace) -> int: + from dataclasses import replace + + from openadapt_flow.agent_guard.config import ConfigError, load_config + from openadapt_flow.agent_guard.handoff import HandoffError, handoff + from openadapt_flow.agent_guard.session import guard_task + from openadapt_flow.guard import GuardRefused, load_task + from openadapt_flow.interactive_recorder import ( + BrowserAttachError, + validate_browser_cdp_endpoint, + ) + + try: + cfg = load_config(args.config) + if args.observe_only: + cfg = replace(cfg, observe_only=True) + if args.approval: + cfg = replace(cfg, approval=args.approval) + task = load_task(args.task) + validate_browser_cdp_endpoint(args.cdp) + except (ConfigError, GuardRefused, BrowserAttachError) as exc: + print(f"guard: {getattr(exc, 'message', exc)}", file=sys.stderr) + return 2 + out = Path(args.out) + if out.exists() and any(out.iterdir()): + print( + f"guard: {out} is not empty; give each task its own --out", file=sys.stderr + ) + return 2 + mode = "observe-only (never blocks)" if cfg.observe_only else "fail-closed" + print( + f"guard: waiting for the agent's tab on {cfg.app_origin} ({mode}, " + f"approval {cfg.approval})", + file=sys.stderr, + flush=True, + ) + # Ctrl-C or SIGTERM ends the session between requests, so a paused save + # is blocked and the recording is finished instead of cut off. + stopping: list[int] = [] + + def _stop(signum: int, _frame: Any) -> None: + stopping.append(signum) + + previous = { + sig: signal.signal(sig, _stop) for sig in (signal.SIGINT, signal.SIGTERM) + } + try: + run = guard_task( + cfg, + task, + cdp_endpoint=args.cdp, + out_dir=out, + page_url=args.page_url, + timeout_s=args.timeout, + attach_wait_s=args.attach_wait, + secret_fields=tuple(args.secret), + on_attached=lambda: print( + "guard: attached; saves are held until the right record is on screen", + file=sys.stderr, + flush=True, + ), + stop=lambda: bool(stopping), + ) + except BrowserAttachError as exc: + print(f"guard: {exc}", file=sys.stderr) + return 2 + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + result: dict[str, Any] = {"run": run.summary, "out": str(run.out_dir)} + if run.verified and cfg.teach is not None and not args.no_teach: + if run.recording is None: + result["handoff"] = {"status": "no_recording"} + else: + try: + result["handoff"] = handoff(run.recording, cfg=cfg, task=task).to_dict() + except HandoffError as exc: + result["handoff"] = {"status": "refused", "detail": str(exc)} + _print(result, as_json=args.json) + if run.verified: + return 0 + if run.verdict is not None: + return 4 + return 3 if run.blocked else 5 + + +def _print(result: dict[str, Any], *, as_json: bool) -> None: + if as_json: + print(json.dumps(result, indent=2, sort_keys=True, default=str)) + return + run = result["run"] + print(f"verdict: {run.get('verdict') or 'no save went through'}") + for item in run.get("blocked") or []: + print(f" blocked a save: {item['code']}") + for item in run.get("would_block") or []: + print(f" would have blocked (observe-only): {item['code']}") + if run.get("hold_ms"): + print(f" held the save {run['hold_ms'][0]} ms for the checks") + if run.get("receipt_path"): + print(f" receipt: {run['receipt_path']}") + print(f" run folder: {result['out']}") + handed = result.get("handoff") + if handed: + print(f"run 2: {handed.get('status')}: {handed.get('detail') or ''}".rstrip()) + for step in handed.get("next_steps") or []: + print(f" next: {step}") diff --git a/packages/flow/openadapt_flow/agent_guard/commit_guard.py b/packages/flow/openadapt_flow/agent_guard/commit_guard.py new file mode 100644 index 000000000..059ec34ea --- /dev/null +++ b/packages/flow/openadapt_flow/agent_guard/commit_guard.py @@ -0,0 +1,485 @@ +"""The commit guard: hold the agent's save until the right record is on screen. + +:class:`CommitGuard` decides what happens to each request the agent's tab +sends. It does no browser I/O itself: a :class:`GuardPage` (the CDP adapter in +:mod:`openadapt_flow.agent_guard.cdp`, or a fake in tests) continues or blocks +the paused request, reads the screen and shows the agent a notice. + +For a save (:func:`~openadapt_flow.agent_guard.requests.classify`) it runs the +guarded-save contract of :mod:`openadapt_flow.guard`, which the agent never +calls itself: + +1. :meth:`GuardedSave.begin ` checks + that every value of the task's ``expected_record`` is in the text of the + open record (:func:`~openadapt_flow.guard.check_identity`), and reads the + system of record for the baseline. A refusal blocks the request. +2. :meth:`Guard.commit ` re-checks the + screen, waits for a person when the approval mode is ``each_item``, and + records that this task's save may happen once. A second save of the same + task is blocked. +3. The request goes on. When it finishes (or ``response_wait_s`` passes), + :meth:`Guard.confirm ` reads the + record back through ``verify-run``. A verdict other than ``VERIFIED`` is + shown on the agent's page; a verified save shows nothing, so the run is + recorded as the app drew it. + +``observe_only`` never blocks. It runs the same checks, logs what the guard +would have blocked, and still reads the save back. Without a baseline (the +store couldn't be read before the write), observe-only lets the save through +and reports that it couldn't be checked. + +The event log holds no record values: methods, resource types, origins, a +digest of each path, decisions and codes. +""" + +from __future__ import annotations + +import hashlib +import html +import json +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional, Protocol, Sequence + +from openadapt_flow.agent_guard.requests import ( + HeldRequest, + NotASave, + classify, + origin_of, +) +from openadapt_flow.guard import ( + Guard, + GuardedSave, + GuardRefused, + GuardResult, + GuardTask, + GuardVerdict, + check_identity, +) + +__all__ = [ + "BLOCK_STATUS", + "CommitGuard", + "GuardPage", + "GuardSummary", +] + +#: The status a blocked save gets. 409 reads as "not now" to most apps; the +#: body and the ``X-OpenAdapt-Guard`` header say why. +BLOCK_STATUS = 409 +GUARD_HEADER = "X-OpenAdapt-Guard" +SUMMARY_SCHEMA = "openadapt.agent-guard-run/v1" + + +class GuardPage(Protocol): + """What the guard needs from the agent's tab.""" + + def continue_request(self, request_id: str) -> None: ... + + def block_request( + self, + request_id: str, + *, + status: int, + body: str, + content_type: str, + headers: dict[str, str], + ) -> None: ... + + def screen_text(self) -> Optional[str]: ... + + def screenshot(self) -> Optional[bytes]: ... + + def notify(self, message: str, *, level: str) -> None: ... + + def clear_notice(self) -> None: ... + + +@dataclass +class GuardSummary: + """One guard session, without record values.""" + + task_id: str + mode: str + approval: str + verdict: Optional[str] = None + receipt_path: Optional[str] = None + blocked: list[dict[str, Any]] = field(default_factory=list) + would_block: list[dict[str, Any]] = field(default_factory=list) + saves_held: int = 0 + hold_ms: list[float] = field(default_factory=list) + readback_ms: Optional[float] = None + requests: dict[str, int] = field(default_factory=dict) + model_calls: int = 0 + + def to_dict(self) -> dict[str, Any]: + return {"schema": SUMMARY_SCHEMA, **self.__dict__} + + +@dataclass +class _Save: + """The save that went through, waiting for its read-back.""" + + guard: Optional[Guard] + network_id: Optional[str] + sent_at: float + done_at: Optional[float] = None + #: observe-only: where its baseline is, or None when it has none. + baseline: Optional[Path] = None + + +def _digest(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16] + + +class CommitGuard: + """Decides each paused request of one task's run (see the module docstring).""" + + def __init__( + self, + guarded: GuardedSave, + task: GuardTask, + page: GuardPage, + *, + app_origin: str, + not_saves: Sequence[NotASave] = (), + observe_only: bool = False, + agent_id: str = "attached-agent", + response_wait_s: float = 30.0, + events_path: Optional[Path] = None, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self.guarded = guarded + self.task = task + self.page = page + self.app_origin = origin_of(app_origin) or app_origin + self.not_saves = tuple(not_saves) + self.observe_only = observe_only + self.agent_id = agent_id + self.response_wait_s = response_wait_s + self.events_path = events_path + self._clock = clock + self._started = clock() + #: A save waiting for a person: (guard, request, paused at). + self._waiting: Optional[tuple[Guard, HeldRequest, float]] = None + self._save: Optional[_Save] = None + self.result: Optional[GuardResult] = None + #: observe-only: the verify-run receipt, when the save was read back. + self.observed: Optional[dict[str, Any]] = None + self.summary = GuardSummary( + task_id=task.task_id, + mode="observe_only" if observe_only else "fail_closed", + approval=guarded.approval, + ) + + # -- state ----------------------------------------------------------------- + + @property + def finished(self) -> bool: + """The task's save was sent and read back.""" + + return self.summary.verdict is not None + + @property + def busy(self) -> bool: + """A save is waiting for a person or for its read-back.""" + + return self._waiting is not None or ( + self._save is not None and not self.finished + ) + + # -- requests ---------------------------------------------------------------- + + def on_request(self, request: HeldRequest) -> None: + """Decide one paused request. Every request is continued or blocked, + except a save that waits for a person (``each_item``).""" + + kind = classify(request, app_origin=self.app_origin, not_saves=self.not_saves) + self.summary.requests[kind] = self.summary.requests.get(kind, 0) + 1 + if kind in ("read", "not_a_save"): + self.page.continue_request(request.request_id) + if kind == "not_a_save": + self._log(request, kind, "continued") + return + held_at = self._clock() + if kind == "other_origin": + self._refuse( + request, + kind, + held_at, + "other_origin", + "the request goes to another site than the app, and the operator " + "didn't list it as not a save", + ) + return + self._on_save(request, held_at) + + def _on_save(self, request: HeldRequest, held_at: float) -> None: + self.summary.saves_held += 1 + if self.observe_only: + self._observe_save(request, held_at) + return + if self._waiting is not None: + self._refuse( + request, + "save", + held_at, + "save_waiting_for_person", + "another save of this task is waiting for a person to approve it", + ) + return + screen = self.page.screen_text() + try: + guard = self.guarded.begin( + self.task, screen_text=screen, agent_id=self.agent_id + ) + self._commit(guard, request, held_at, screen) + except GuardRefused as exc: + self._refuse(request, "save", held_at, exc.code, exc.message) + + def _commit( + self, + guard: Guard, + request: HeldRequest, + held_at: float, + screen: Optional[str], + ) -> None: + shot = self.page.screenshot() if self.guarded.approval == "each_item" else None + commit = guard.commit(screenshot_png=shot, screen_text=screen, wait_s=0.0) + if commit.status == "waiting_for_approval": + self._waiting = (guard, request, held_at) + self._log(request, "save", "held_for_person", guard_id=guard.guard_id) + self.page.notify( + "OpenAdapt is holding this save until a person approves it. " + "Don't save again.", + level="info", + ) + return + self._send(guard, request, held_at) + + def _send( + self, guard: Optional[Guard], request: HeldRequest, held_at: float + ) -> None: + sent = self._clock() + # Known before it goes on, so its completion is never missed. + self._save = _Save(guard=guard, network_id=request.network_id, sent_at=sent) + hold_ms = round((sent - held_at) * 1000.0, 1) + self.summary.hold_ms.append(hold_ms) + if guard is not None and guard.approval is not None: + self.page.clear_notice() # the "waiting for a person" notice + self.page.continue_request(request.request_id) + self._log(request, "save", "continued", hold_ms=hold_ms) + + def _observe_save(self, request: HeldRequest, held_at: float) -> None: + """observe-only: log what the guard would block, never block.""" + + from openadapt_flow.verify_run import capture_baseline + + screen = self.page.screen_text() + status, missing = check_identity(self.task.expected_record, screen) + code = None + if self._save is not None: + code = "already_committed" + elif status != "verified": + code = f"identity_{status}" + if code is not None: + self.summary.would_block.append( + {"code": code, "missing": missing if status != "verified" else []} + ) + if self._save is not None: + # Only the first save is read back; a second one is logged. + self.page.continue_request(request.request_id) + self._log(request, "save", "continued", code=code) + return + baseline: Optional[Path] = None + work = self.guarded.state_dir / f"observe-{self.task.task_id}" + try: + path = work / "baseline.json" + snap = capture_baseline(self.guarded.verify, self.task.params, path) + if snap["state"].get("reachable"): + baseline = path + except Exception: # noqa: BLE001 - observe-only never blocks + baseline = None + self._send(None, request, held_at) + assert self._save is not None + self._save.baseline = baseline + if code is not None: + self._log(request, "save", "would_block", code=code) + + def on_request_done(self, network_id: Optional[str]) -> None: + """The save request finished or failed (the read-back decides which).""" + + if ( + self._save is not None + and self._save.done_at is None + and network_id is not None + and network_id == self._save.network_id + ): + self._save.done_at = self._clock() + + # -- time --------------------------------------------------------------------- + + def poll(self) -> None: + """Ask again for a waiting approval; read a finished save back.""" + + if self._waiting is not None: + guard, request, held_at = self._waiting + try: + commit = guard.commit(wait_s=0.0) + except GuardRefused as exc: + self._waiting = None + self._refuse(request, "save", held_at, exc.code, exc.message) + return + if commit.status == "go": + self._waiting = None + self._send(guard, request, held_at) + return + save = self._save + if save is None or self.finished: + return + now = self._clock() + if save.done_at is None and now - save.sent_at < self.response_wait_s: + return + self._read_back(save) + + def _read_back(self, save: _Save) -> None: + started = self._clock() + if save.guard is not None: + self.result = save.guard.confirm() + verdict = self.result.verdict + self.summary.receipt_path = self.result.receipt_path + message = self.result.what_to_do + else: + verdict, message = self._observe_read_back(save) + self.summary.readback_ms = round((self._clock() - started) * 1000.0, 1) + self.summary.verdict = verdict.value + if verdict is GuardVerdict.VERIFIED: + # Nothing on the page: a verified run 1 is recorded as the app + # drew it, so run 2 never expects the guard's notice. + self.page.clear_notice() + else: + self.page.notify( + "OpenAdapt read the record back: " + message, level="error" + ) + self._event({"event": "read_back", "verdict": verdict.value}) + + def _observe_read_back(self, save: _Save) -> tuple[GuardVerdict, str]: + from openadapt_flow.guard import _verdict + from openadapt_flow.verify_run import verify_run, write_result + + if save.baseline is None: + return ( + GuardVerdict.RECONCILIATION_REQUIRED, + "the system of record couldn't be read before the save, so the " + "save couldn't be checked. A person must check the record.", + ) + try: + doc = verify_run( + self.guarded.verify, + params=self.task.params, + run_record={"executor_kind": "agent", "executor_id": self.agent_id}, + authorization_ref=self.task.authorization, + baseline_path=save.baseline, + private_dir=save.baseline.parent, + write_attempted=True, + ) + self.summary.receipt_path = str(write_result(doc, save.baseline.parent)) + except Exception: # noqa: BLE001 - an unreadable store is not verified + doc = None + self.observed = doc + verdict = _verdict(doc) + text = { + GuardVerdict.VERIFIED: "saved, and the system of record shows it.", + GuardVerdict.REFUTED: ( + "the system of record shows a different write than intended. " + "A person must correct the record." + ), + GuardVerdict.RECONCILIATION_REQUIRED: ( + "the system of record doesn't show this save, or couldn't be " + "read. A person must check the record." + ), + }[verdict] + return verdict, text + + # -- refusing ----------------------------------------------------------------- + + def _refuse( + self, + request: HeldRequest, + kind: str, + held_at: float, + code: str, + message: str, + ) -> None: + if self.observe_only: + self.summary.would_block.append({"code": code}) + self.page.continue_request(request.request_id) + self._log(request, kind, "would_block", code=code) + return + what = ( + "Nothing was sent to the app. Open the record the task names and " + "save again, or stop and tell a person." + if code.startswith("identity_") + else "Nothing was sent to the app. Don't retry this save." + ) + payload = { + "openadapt_guard": { + "blocked": True, + "code": code, + "message": message, + "what_to_do": what, + } + } + if request.resource_type == "Document": + body = ( + "" + "OpenAdapt blocked this save" + '

OpenAdapt blocked this save

' + f"

{html.escape(message)}.

{html.escape(what)}

" + "" + ) + content_type = "text/html; charset=utf-8" + else: + body = json.dumps(payload) + content_type = "application/json" + self.page.block_request( + request.request_id, + status=BLOCK_STATUS, + body=body, + content_type=content_type, + headers={GUARD_HEADER: f"blocked; code={code}"}, + ) + self.summary.blocked.append( + {"code": code, "hold_ms": round((self._clock() - held_at) * 1000.0, 1)} + ) + self._log(request, kind, "blocked", code=code) + self.page.notify( + f"OpenAdapt blocked this save: {message}. {what}", level="error" + ) + + # -- log ------------------------------------------------------------------------- + + def _log( + self, request: HeldRequest, kind: str, decision: str, **extra: Any + ) -> None: + self._event( + { + "event": "request", + "method": request.method, + "resource_type": request.resource_type, + "origin": origin_of(request.url), + "path_sha256": _digest(request.path), + "kind": kind, + "decision": decision, + **{k: v for k, v in extra.items() if v is not None}, + } + ) + + def _event(self, record: dict[str, Any]) -> None: + if self.events_path is None: + return + record = {"t": round(self._clock() - self._started, 3), **record} + self.events_path.parent.mkdir(parents=True, exist_ok=True) + with self.events_path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(record, sort_keys=True) + "\n") diff --git a/packages/flow/openadapt_flow/agent_guard/config.py b/packages/flow/openadapt_flow/agent_guard/config.py new file mode 100644 index 000000000..71cab44f5 --- /dev/null +++ b/packages/flow/openadapt_flow/agent_guard/config.py @@ -0,0 +1,181 @@ +"""``guard.yaml``: what the operator fixes before the agent runs. + +:: + + schema_version: openadapt.agent-guard/v1 + app_url: http://127.0.0.1:8765/#tasks # the guarded origin + verify: verify.yaml # how verify-run reads the save back + screen_selector: "#patient-banner" # optional: where the record is shown + not_saves: # optional: writes that aren't the save + - {method: POST, path: /chart/signin} + approval: not_gated # or each_item: a person approves each save + teach: # optional: compile run 1 for run 2 + dir: .openadapt-teach + name: triage_note + inputs: [note] + +Relative paths are relative to the file. The task (which record, which +values) is a separate ``openadapt.guard-task/v1`` file from whoever assigned +the work (:func:`openadapt_flow.guard.load_task`), never from the agent. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Optional + +from openadapt_flow.agent_guard.requests import NotASave, origin_of +from openadapt_flow.guard import APPROVAL_MODES, ApprovalSetting + +__all__ = [ + "CONFIG_SCHEMA", + "ConfigError", + "GuardConfig", + "TeachSection", + "load_config", + "parse_config", +] + +CONFIG_SCHEMA = "openadapt.agent-guard/v1" +_KEYS = { + "schema_version", + "app_url", + "verify", + "screen_selector", + "not_saves", + "approval", + "observe_only", + "response_wait_s", + "teach", +} +_TEACH_KEYS = { + "dir", + "name", + "inputs", + "instruction", + "key_dir", + "policy", + "db", + "complete_read", +} + + +class ConfigError(ValueError): + """``guard.yaml`` is not usable. Nothing was attached.""" + + +@dataclass(frozen=True) +class TeachSection: + """Where a run that reads back correctly becomes a draft program.""" + + dir: Path + name: str = "guarded_save" + #: The task values that become the program's inputs. Default: the + #: ``intended_fields`` the agent typed. + inputs: Optional[tuple[str, ...]] = None + instruction: Optional[str] = None + key_dir: Optional[Path] = None + policy: str = "clinical-write" + #: A read-only copy of the app's SQLite database for scaffold-verifier + #: (``--db``): every other table gets a "no row added" guard. + db: Optional[Path] = None + #: A person's statement that the reads return everything the save can + #: change (scaffold-verifier ``--complete-read``). Recorded as declared. + complete_read: bool = False + + +@dataclass(frozen=True) +class GuardConfig: + app_url: str + verify: Path + screen_selector: Optional[str] = None + not_saves: tuple[NotASave, ...] = () + approval: ApprovalSetting = "not_gated" + observe_only: bool = False + response_wait_s: float = 30.0 + teach: Optional[TeachSection] = None + + def __post_init__(self) -> None: + if not origin_of(self.app_url).startswith(("http://", "https://")): + raise ConfigError("app_url must be an http(s) URL") + if self.approval not in APPROVAL_MODES: + raise ConfigError(f"approval must be one of {APPROVAL_MODES}") + if not Path(self.verify).is_file(): + raise ConfigError(f"verify file not found: {self.verify}") + + @property + def app_origin(self) -> str: + return origin_of(self.app_url) + + +def _path(base: Path, value: Any, name: str) -> Path: + if not isinstance(value, str) or not value.strip(): + raise ConfigError(f"{name} must be a path") + path = Path(value).expanduser() + return path if path.is_absolute() else (base / path).resolve() + + +def parse_config(raw: Mapping[str, Any], *, base: Path) -> GuardConfig: + if raw.get("schema_version") != CONFIG_SCHEMA: + raise ConfigError(f"schema_version must be {CONFIG_SCHEMA}") + unknown = sorted(set(raw) - _KEYS) + if unknown: + raise ConfigError(f"unknown keys: {', '.join(unknown)}") + rules = [] + for item in raw.get("not_saves") or []: + if not isinstance(item, Mapping) or not {"method", "path"} <= set(item): + raise ConfigError("each not_saves entry needs method and path") + rules.append( + NotASave( + method=str(item["method"]), + path=str(item["path"]), + origin=origin_of(str(item["origin"])) if item.get("origin") else None, + ) + ) + teach = None + section = raw.get("teach") + if section is not None: + if not isinstance(section, Mapping) or "dir" not in section: + raise ConfigError("teach needs dir") + extra = sorted(set(section) - _TEACH_KEYS) + if extra: + raise ConfigError(f"unknown teach keys: {', '.join(extra)}") + inputs = section.get("inputs") + teach = TeachSection( + dir=_path(base, section["dir"], "teach.dir"), + name=str(section.get("name") or "guarded_save"), + inputs=tuple(str(i) for i in inputs) if inputs is not None else None, + instruction=section.get("instruction"), + key_dir=( + _path(base, section["key_dir"], "teach.key_dir") + if section.get("key_dir") + else None + ), + policy=str(section.get("policy") or "clinical-write"), + db=_path(base, section["db"], "teach.db") if section.get("db") else None, + complete_read=bool(section.get("complete_read", False)), + ) + return GuardConfig( + app_url=str(raw.get("app_url") or ""), + verify=_path(base, raw.get("verify"), "verify"), + screen_selector=raw.get("screen_selector") or None, + not_saves=tuple(rules), + approval=raw.get("approval") or "not_gated", + observe_only=bool(raw.get("observe_only", False)), + response_wait_s=float(raw.get("response_wait_s", 30.0)), + teach=teach, + ) + + +def load_config(path: str | Path) -> GuardConfig: + import yaml + + source = Path(path).expanduser().resolve() + try: + raw = yaml.safe_load(source.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError) as exc: + raise ConfigError(f"cannot read {source}: {exc}") from exc + if not isinstance(raw, Mapping): + raise ConfigError(f"{source} is not a mapping") + return parse_config(raw, base=source.parent) diff --git a/packages/flow/openadapt_flow/agent_guard/handoff.py b/packages/flow/openadapt_flow/agent_guard/handoff.py new file mode 100644 index 000000000..ad9466582 --- /dev/null +++ b/packages/flow/openadapt_flow/agent_guard/handoff.py @@ -0,0 +1,408 @@ +"""Run 2: turn a guarded run 1 that read back correctly into a draft program. + +The agent did the task once in its own browser; the guard recorded it with the +network capture and verify-run read the save back as ``VERIFIED``. +:func:`handoff` puts that run where the MCP teach flow (#1335) and +``compile-agent`` keep theirs, so the same people and the same rules take it +from there: + +1. **Compile** the recording (distilled, as ``compile-agent`` does) into the + task's ``program/bundle``. ``compile`` proposes an API step for the save + when the captured form request replays cleanly (``api_proposal.json`` says + which step, or why not); the GUI steps stay as the fallback, and the API + step runs only after a person admits it (``openadapt-flow api admit``). +2. **Check run 1's record** the way ``compile-agent`` does + (:func:`~openadapt_flow.compile_agent.pipeline.check_run_record`): exactly + one new record, nothing else changed, the record the task named, every + input in it. Otherwise the draft is set aside. +3. **Propose the record check** from the app's own reads with + ``scaffold-verifier`` (:mod:`openadapt_flow.record_check_synthesis`): the + reads the app made after the save, a database read path the operator gave + (``teach.db``), or the ``verify.yaml`` read, fault-tested against the + ``effect_e2e`` fault classes. When that check misses a fault it is kept + next to the draft for review and the plain proposal from the before/after + reads is proposed instead. +4. **Write the draft** as a JIT ``DRAFT`` signed as recorded by the attached + agent, with no agent principal: only a person or a registered admitter + key can admit it. + +Nothing here admits anything. A person confirms the record check +(``teach_confirm_checks`` or ``compile-agent admit``), canaries run, and a +person (or an admitter key) admits. ``openadapt-agent serve --teach VERIFY +--teach-dir DIR`` then serves the admitted program as ``taught_``, which +runs with no model calls and reads every save back. +""" + +from __future__ import annotations + +import json +import shutil +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Mapping, Optional + +from openadapt_flow.agent_guard.config import GuardConfig +from openadapt_flow.guard import GuardTask + +__all__ = ["HandoffError", "HandoffResult", "handoff", "typed_inputs"] + +HANDOFF_FILE = "agent_guard.json" +HANDOFF_SCHEMA = "openadapt.agent-guard-handoff/v1" +#: The dialect a recording made by the attached recorder is stamped with. +ATTACHED_DIALECT = "attached-browser" + + +class HandoffError(ValueError): + """Run 1 can't become a draft. Nothing was written to the teach dir.""" + + +@dataclass +class HandoffResult: + status: str + task_dir: Path + task_key: str + detail: Optional[str] = None + inputs: list[str] = field(default_factory=list) + expect: dict[str, str] = field(default_factory=dict) + record_check: Optional[dict[str, Any]] = None + proposed_check: Optional[dict[str, Any]] = None + api_step: Optional[dict[str, Any]] = None + next_steps: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + out = dict(self.__dict__) + out["task_dir"] = str(self.task_dir) + return out + + +def _events(recording: Path) -> list[dict[str, Any]]: + path = recording / "events.jsonl" + if not path.is_file(): + return [] + return [json.loads(line) for line in path.read_text("utf-8").splitlines() if line] + + +def typed_inputs(recording: Path, task: GuardTask) -> list[str]: + """The task's ``intended_fields`` whose whole value the agent typed.""" + + typed = { + str(event.get("text")) + for event in _events(recording) + if event.get("kind") == "type" and event.get("text") + } + return sorted( + name for name, value in task.intended_fields.items() if str(value) in typed + ) + + +def _record_match(cfg: GuardConfig) -> dict[str, Any]: + """The match of the one ``record_written`` effect in ``verify.yaml``.""" + + from openadapt_flow.verify_run import load_verify_config + + config = load_verify_config(cfg.verify) + written = [ + e + for e in config.effects + if getattr(e.kind, "value", e.kind) == "record_written" + and int(e.expected_count) > 0 + ] + if len(written) != 1: + raise HandoffError( + "verify.yaml must claim exactly one new record (a record_written " + "effect) for run 1 to become a program" + ) + return dict(written[0].match) + + +def _expect( + match: Mapping[str, Any], task: GuardTask, inputs: list[str] +) -> dict[str, str]: + """The record's fields every run writes the same: literals in the check, + and task values that aren't inputs.""" + + expect: dict[str, str] = {} + for fld, expr in match.items(): + param = getattr(expr, "param", None) + literal = getattr(expr, "literal", None) + if param: + if param not in inputs and param in task.params: + expect[fld] = str(task.params[param]) + elif literal is not None: + expect[fld] = str(literal) + return expect + + +def _instruction(task: GuardTask, inputs: list[str], given: Optional[str]) -> str: + if given: + return given + record = ", ".join(f"{k} {v}" for k, v in sorted(task.expected_record.items())) + values = ", ".join(f"{name} {{{name}}}" for name in inputs) + return f"On the record {record}, save {values}" + + +def _pin(node: Any, fixed: Mapping[str, str]) -> Any: + """``{param: name}`` -> ``{literal: value}``, and ``{name}`` in a path + template -> the value, for each fixed value in a verifier section (reads + scoped by the record's identity).""" + + if isinstance(node, dict): + if set(node) == {"param"} and node["param"] in fixed: + return {"literal": fixed[node["param"]]} + return {k: _pin(v, fixed) for k, v in node.items()} + if isinstance(node, list): + return [_pin(v, fixed) for v in node] + if isinstance(node, str): + # A read's path template (``/patients/{patient_id}/notes``). + from urllib.parse import quote + + for name, value in fixed.items(): + node = node.replace("{" + name + "}", quote(value, safe="")) + return node + + +def _propose_from_reads( + recording: Path, + cfg: GuardConfig, + values: Mapping[str, str], + fixed: Mapping[str, str], + out_dir: Path, +) -> tuple[Optional[str], dict[str, Any]]: + """scaffold-verifier's check from the app's reads. Returns the rendered + file when it caught every fault, and what it found either way. + + ``fixed`` are the task values every run writes the same (the record's + identity): scaffold-verifier finds the record by them too, and the check + pins them as literals, so the program can only save on that record. + """ + + import yaml + + from openadapt_flow import record_check_synthesis as rcs + + teach = cfg.teach + assert teach is not None + section = (yaml.safe_load(cfg.verify.read_text("utf-8")) or {}).get("verifier") + try: + proposal = rcs.synthesize( + recording, + params={**fixed, **values}, + db_path=teach.db, + verifier=section if isinstance(section, dict) else None, + complete_read=teach.complete_read, + ) + except rcs.SynthesisError as exc: + return None, {"source": "scaffold-verifier", "working": False, "why": str(exc)} + report: dict[str, Any] = { + "source": "scaffold-verifier", + "working": proposal.working, + "reads": [r["source"] for r in proposal.proof()["reads"]], + "faults_caught": sum(1 for r in proposal.faults if r.ok and not r.correct), + "faults": sum(1 for r in proposal.faults if not r.correct), + "misses": proposal.misses, + } + if not proposal.effects: + return None, report + from openadapt_flow.runtime.effects.effect import ValueExpr + + proposal.effects = [ + effect.model_copy( + update={ + "match": { + fld: ( + ValueExpr(literal=fixed[expr.param]) + if expr.param is not None and expr.param in fixed + else expr + ) + for fld, expr in effect.match.items() + } + } + ) + for effect in proposal.effects + ] + proposal.verifier = _pin(proposal.verifier, fixed) + check, proof = rcs.write(proposal, out_dir / "scaffold") + report["file"] = str(check) + report["proof"] = str(proof) + return (rcs.render(proposal) if proposal.working else None), report + + +def handoff( + recording: Path | str, + *, + cfg: GuardConfig, + task: GuardTask, +) -> HandoffResult: + """Make run 1 a draft in ``cfg.teach.dir`` (see the module docstring). + + ``recording`` is the guard's run-1 recording; call this only for a run the + guard read back as ``VERIFIED``. + """ + + from openadapt_flow.compile_agent.pipeline import ( + PROPOSED_FILE, + FirstRunCompiler, + _write_json, + check_run_record, + ) + from openadapt_flow.compiler.compile import compile_recording + from openadapt_flow.compiler.distill import distill_recording + from openadapt_flow.record_sor import SNAPSHOTS_NAME, _write_private + + teach = cfg.teach + if teach is None: + raise HandoffError("guard.yaml has no teach: section") + source = Path(recording) + if not (source / "events.jsonl").is_file(): + raise HandoffError(f"{source} is not a recording") + inputs = ( + sorted(teach.inputs) if teach.inputs is not None else typed_inputs(source, task) + ) + unknown = sorted(set(inputs) - set(task.intended_fields)) + if unknown: + raise HandoffError(f"inputs {unknown} are not intended_fields of the task") + expect = _expect(_record_match(cfg), task, inputs) + if not expect: + raise HandoffError( + "the check names no field every run writes the same (such as the " + "patient id); a program for any record is not made" + ) + values = {name: str(task.intended_fields[name]) for name in inputs} + compiler = FirstRunCompiler( + _instruction(task, inputs, teach.instruction), + params=inputs, + url=cfg.app_url, + verify=cfg.verify, + expect=expect, + on_screen=[str(v) for v in task.expected_record.values()], + task_id=teach.name, + policy=teach.policy, + cache_dir=teach.dir, + key_dir=teach.key_dir, + ) + jit = compiler._jit() + if jit.admitted or jit.candidate() is not None: + raise HandoffError( + f"{compiler.dir} already has a draft or an admitted program; reject " + "the draft to hand a new run over" + ) + compiler.save_spec() + program = jit.program_dir + if program.exists(): + shutil.rmtree(program) + program.mkdir(parents=True) + kept = program / "recording" + shutil.copytree(source, kept) + for stale in (PROPOSED_FILE, "verify.yaml", "record_checks.json", "admission.json"): + (compiler.out_dir / stale).unlink(missing_ok=True) + + # 1. Compile, as compile-agent does: distilled, effects mined from the + # record reads the recording kept. + distilled = program / "recording-distilled" + report = distill_recording(kept, distilled) + # The recorder kept record reads when verify.yaml's read works (as + # compile-agent's record_reader does): mine the effect contract from them. + mine = (kept / SNAPSHOTS_NAME).is_file() + workflow = compile_recording( + distilled, jit.bundle_dir, name=compiler.task.task_id, mine_effects=mine + ) + actions = len(_events(kept)) + # _write_draft reads only these attributes of the capture session and + # computer the JIT's own recording would pass. + shim_session: Any = SimpleNamespace( + recording_dir=kept, compiled_recording_dir=distilled + ) + shim_computer: Any = SimpleNamespace( + actions=actions, + log=[{"source": "agent_guard", "recorded_events": actions}], + ) + compiled = { + "workflow_id": f"wf_{workflow.recording_id or compiler.task.key}", + "distilled": { + "steps_in": report.steps_in, + "steps_out": report.steps_out, + "dropped": [ + {"index": d.index, "kind": d.kind, "rule": d.rule} + for d in report.dropped + ], + }, + } + jit._write_draft( + shim_session, compiled, shim_computer, dict(values), ATTACHED_DIALECT + ) + api_step = None + api_file = jit.bundle_dir / "api_proposal.json" + if api_file.is_file(): + api_step = json.loads(api_file.read_text("utf-8")) + + def finish(status: str, detail: Optional[str], **kw: Any) -> HandoffResult: + result = HandoffResult( + status=status, + task_dir=compiler.dir, + task_key=compiler.task.key, + detail=detail, + inputs=inputs, + expect=expect, + api_step=api_step, + **kw, + ) + _write_json( + compiler.dir / HANDOFF_FILE, + { + "schema": HANDOFF_SCHEMA, + "guard_task_id": task.task_id, + **result.to_dict(), + }, + ) + return result + + # 2. Run 1's record check, from the reads before and after it. + snapshots_path = kept / SNAPSHOTS_NAME + snapshots = ( + json.loads(snapshots_path.read_text("utf-8")) + if snapshots_path.is_file() + else {} + ) + before = (snapshots.get("before") or {}).get("records") + after = (snapshots.get("after") or {}).get("records") + check = check_run_record(before, after, expect=expect, values=values) + if not check.passed: + kept_at = compiler._set_aside(jit, "record_check_failed", check.reasons) + return finish( + "record_check_failed", + "; ".join(check.reasons) + f" (draft set aside at {kept_at})", + record_check=check.to_dict(), + ) + + # 3. The proposed record check: the app's own reads first. + fixed = { + str(expr.param): str(task.params[expr.param]) + for expr in _record_match(cfg).values() + if getattr(expr, "param", None) + and expr.param not in inputs + and expr.param in task.params + } + rendered, found = _propose_from_reads(kept, cfg, values, fixed, compiler.out_dir) + if rendered is not None: + compiler.out_dir.mkdir(parents=True, exist_ok=True) + _write_private(compiler.out_dir / PROPOSED_FILE, rendered) + found["proposed"] = True + else: + compiler._propose(before or [], after or [], values) + found["proposed"] = False + found["fallback"] = "the before/after reads (record --verify's proposer)" + found["proposed_file"] = str(compiler.out_dir / PROPOSED_FILE) + params = " ".join(f"--canary-param {p}=" for p in inputs) + return finish( + "draft", + "run 1 read back correctly; a person confirms the record check, then admits", + record_check=check.to_dict(), + proposed_check=found, + next_steps=[ + f"openadapt-flow compile-agent admit {compiler.dir} --admit-as " + f"'' {params}", + f"openadapt-agent serve --teach {cfg.verify} --teach-dir {teach.dir} " + "(serves the admitted program as a tool)", + ], + ) diff --git a/packages/flow/openadapt_flow/agent_guard/requests.py b/packages/flow/openadapt_flow/agent_guard/requests.py new file mode 100644 index 000000000..0227b1007 --- /dev/null +++ b/packages/flow/openadapt_flow/agent_guard/requests.py @@ -0,0 +1,206 @@ +"""Which of the agent's browser requests is a save. + +The commit guard pauses the page's requests with the CDP ``Fetch`` domain and +asks :func:`classify` what each one is: + +``read`` + ``GET``, ``HEAD`` and ``OPTIONS``, and a GraphQL request whose every + operation is a ``query`` or a ``subscription``. Let through at once. +``save`` + Any other request to the app's origin: ``POST``, ``PUT``, ``PATCH``, + ``DELETE``, an unknown method, or a GraphQL ``mutation``. Held until the + right-record check passes. A GraphQL body Flow can't read (a persisted + query with no text, a multipart upload) counts as a save. +``not_a_save`` + A write the operator listed as not a save (a sign-in form, a search that + posts, an audit beacon). Let through. +``other_origin`` + A write to another origin that the operator didn't list. Blocked when the + guard is closed: Flow can't tell whether it is the save. + +Nothing here reads a value the request carries except a GraphQL operation's +type. +""" + +from __future__ import annotations + +import fnmatch +import json +import re +from dataclasses import dataclass +from typing import Any, Literal, Mapping, Optional, Sequence +from urllib.parse import parse_qs, urlsplit + +__all__ = [ + "HeldRequest", + "NotASave", + "RequestKind", + "classify", + "graphql_operations", + "origin_of", +] + +RequestKind = Literal["read", "save", "not_a_save", "other_origin"] + +READ_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) +_GRAPHQL_KEYS = frozenset({"query", "variables", "operationName", "extensions"}) + +#: GraphQL operation keywords, after comments and fragments are stripped. +_GRAPHQL_COMMENT = re.compile(r"#[^\n\r]*") +_GRAPHQL_STRING = re.compile(r'"""[\s\S]*?"""|"(?:\\.|[^"\\])*"') +_GRAPHQL_OPERATION = re.compile( + r"(?:^|[\s}])(query|mutation|subscription|fragment)\b|(^|})\s*\{" +) + + +@dataclass(frozen=True) +class HeldRequest: + """One request the page paused at, as ``Fetch.requestPaused`` reports it.""" + + request_id: str + method: str + url: str + resource_type: str = "Other" + post_data: Optional[str] = None + content_type: str = "" + network_id: Optional[str] = None + + @classmethod + def from_cdp(cls, params: Mapping[str, Any]) -> "HeldRequest": + request = params.get("request") or {} + headers = { + str(k).lower(): str(v) for k, v in (request.get("headers") or {}).items() + } + return cls( + request_id=str(params.get("requestId") or ""), + method=str(request.get("method") or "GET").upper(), + url=str(request.get("url") or ""), + resource_type=str(params.get("resourceType") or "Other"), + post_data=request.get("postData"), + content_type=headers.get("content-type", ""), + network_id=params.get("networkId"), + ) + + @property + def path(self) -> str: + return urlsplit(self.url).path or "/" + + +@dataclass(frozen=True) +class NotASave: + """A write the operator says is not a save, by method and path pattern. + + ``path`` is an ``fnmatch`` pattern on the URL path (``/chart/signin``, + ``/api/search*``). ``origin`` defaults to the app's origin; give it to + let through a write to another origin (an analytics host). + """ + + method: str + path: str + origin: Optional[str] = None + + def matches(self, request: HeldRequest, app_origin: str) -> bool: + if self.method != "*" and self.method.upper() != request.method: + return False + if origin_of(request.url) != (self.origin or app_origin): + return False + return fnmatch.fnmatchcase(request.path, self.path) + + +def origin_of(url: str) -> str: + """``scheme://host[:port]``, lower-cased; ``""`` for a URL with none.""" + + try: + parts = urlsplit(url) + except ValueError: + return "" + if not parts.scheme or not parts.netloc: + return "" + host = (parts.hostname or "").lower() + port = parts.port + default = {"http": 80, "https": 443}.get(parts.scheme.lower()) + netloc = host if port in (None, default) else f"{host}:{port}" + return f"{parts.scheme.lower()}://{netloc}" + + +def _operation_types(document: str) -> list[str]: + """The operation types in one GraphQL document (``query`` for a bare + selection set). Fragments are not operations.""" + + text = _GRAPHQL_STRING.sub('""', _GRAPHQL_COMMENT.sub("", document)).strip() + found: list[str] = [] + for match in _GRAPHQL_OPERATION.finditer(text): + keyword = match.group(1) + if keyword is None: + found.append("query") # shorthand ``{ ... }`` + elif keyword != "fragment": + found.append(keyword) + return found + + +def graphql_operations(request: HeldRequest) -> Optional[list[str]]: + """The GraphQL operation types the request runs, or None if it isn't one + Flow can read. + + Reads a JSON body (one request or a batch) or a ``query=`` URL parameter. + Returns ``["unknown"]`` for a GraphQL request with no query text (a + persisted query), so it is treated as a save. + """ + + documents: list[Any] = [] + if request.post_data and "json" in request.content_type.lower(): + try: + body = json.loads(request.post_data) + except ValueError: + return None + items = body if isinstance(body, list) else [body] + for item in items: + # Only a body made of GraphQL request keys is GraphQL: a save + # that happens to have a "query" field is still a save. + if not isinstance(item, dict) or not item or set(item) - _GRAPHQL_KEYS: + return None + if "query" in item: + documents.append(item.get("query")) + elif "extensions" in item and "persistedQuery" in ( + item.get("extensions") or {} + ): + documents.append(None) + else: + return None + else: + query = parse_qs(urlsplit(request.url).query).get("query") + if not query: + return None + documents.append(query[0]) + types: list[str] = [] + for document in documents: + if not isinstance(document, str): + types.append("unknown") + continue + found = _operation_types(document) + types.extend(found or ["unknown"]) + return types + + +def classify( + request: HeldRequest, + *, + app_origin: str, + not_saves: Sequence[NotASave] = (), +) -> RequestKind: + """What ``request`` is to the commit guard (see the module docstring).""" + + operations = graphql_operations(request) + if request.method in READ_METHODS: + # A GraphQL mutation sent with GET still changes data. + if operations and any(op in ("mutation", "unknown") for op in operations): + pass + else: + return "read" + elif operations and all(op in ("query", "subscription") for op in operations): + return "read" + if any(rule.matches(request, app_origin) for rule in not_saves): + return "not_a_save" + if origin_of(request.url) != app_origin: + return "other_origin" + return "save" diff --git a/packages/flow/openadapt_flow/agent_guard/session.py b/packages/flow/openadapt_flow/agent_guard/session.py new file mode 100644 index 000000000..c59811b0c --- /dev/null +++ b/packages/flow/openadapt_flow/agent_guard/session.py @@ -0,0 +1,312 @@ +"""One guarded task: attach to the agent's tab, guard its save, record run 1. + +:func:`guard_task` connects to the agent's own Chromium over CDP twice: + +* **The guard**, on this thread, with its own connection. It puts the commit + guard on the tab and runs until the task's save has been read back (or the + tab closes, or the time runs out). +* **The recorder**, on its own thread with its own connection: the browser + recorder (:class:`~openadapt_flow.interactive_recorder.InteractiveRecorder`, + the ``record --browser-cdp-endpoint`` mode) with the network capture on. + It records run 1 for run 2. + +They are kept apart so that the recording can fail without the guard +stopping. The recorder refuses a recording it can't keep exactly (for +example when the agent acts faster than a frame can be kept); then run 1 +can't become a program, but every save is still held and read back. + +The agent keeps driving its browser; nothing in it changes except that it was +started with a DevTools port. The guard covers one task, from when it attaches +until the save is read back. A save the agent sends before the guard attaches +is not seen, so start the guard before the agent starts the task. Only the +attached tab is guarded. + +Output, under ``out_dir``:: + + recording/ run 1, in the format ``compile`` reads (with network.jsonl) + guard/ guarded-save state: baseline, approvals, receipt + guard_events.jsonl each decision, with no record values + summary.json the outcome (``openadapt.agent-guard-run/v1``) +""" + +from __future__ import annotations + +import json +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional + +from openadapt_flow.agent_guard.cdp import CdpGuardPage +from openadapt_flow.agent_guard.commit_guard import CommitGuard +from openadapt_flow.agent_guard.config import GuardConfig +from openadapt_flow.guard import GuardedSave, GuardTask +from openadapt_flow.verify_run import load_verify_config + +__all__ = ["GuardRun", "guard_task"] + +#: How often the guard's loop hands paused requests to the commit guard. +POLL_MS = 25 + + +@dataclass +class GuardRun: + """What one guarded task did.""" + + out_dir: Path + verdict: Optional[str] + summary: dict[str, Any] + recording: Optional[Path] = None + #: Why the session ended: ``read_back``, ``tab_closed``, ``timeout``, + #: ``stopped``. + ended: str = "read_back" + seconds: float = 0.0 + notes: list[str] = field(default_factory=list) + + @property + def verified(self) -> bool: + return self.verdict == "VERIFIED" + + @property + def blocked(self) -> list[dict[str, Any]]: + return list(self.summary.get("blocked") or []) + + +def _connect( + cfg: GuardConfig, + *, + cdp_endpoint: str, + page_url: Optional[str], + attach_wait_s: float, +) -> tuple[Any, Any, Any]: + """The guard's own connection and the agent's tab, waiting for the tab.""" + + from playwright.sync_api import sync_playwright + + from openadapt_flow.interactive_recorder import ( + BrowserAttachError, + select_attached_page, + ) + + pw = sync_playwright().start() + deadline = time.monotonic() + attach_wait_s + try: + while True: + try: + browser = pw.chromium.connect_over_cdp(cdp_endpoint) + except Exception as exc: + if time.monotonic() >= deadline: + raise BrowserAttachError( + "could not connect to the agent's Chromium DevTools endpoint" + ) from exc + time.sleep(0.25) + continue + try: + page = select_attached_page( + browser, app_url=cfg.app_url, page_url=page_url + ) + return pw, browser, page + except BrowserAttachError: + browser.close() + if time.monotonic() >= deadline: + raise + time.sleep(0.25) + except BaseException: + pw.stop() + raise + + +class _RecorderThread: + """The run-1 recorder on its own thread and its own connection.""" + + def __init__(self, **kwargs: Any) -> None: + self._kwargs = kwargs + self._stop = threading.Event() + self.ready = threading.Event() + self.recording: Optional[Path] = None + self.error: Optional[str] = None + self._thread = threading.Thread( + target=self._main, name="openadapt-guard-recorder", daemon=True + ) + + def start(self) -> None: + self._thread.start() + + def _main(self) -> None: + from openadapt_flow.interactive_recorder import InteractiveRecorder + + recorder = None + try: + recorder = InteractiveRecorder(**self._kwargs) + recorder.start() + self.ready.set() + while not self._stop.is_set(): + if not recorder.pump(): + break + self.recording = recorder.finish() + except BaseException as exc: # noqa: BLE001 - reported, never raised + self.error = str(exc) or type(exc).__name__ + if recorder is not None: + try: + recorder.abort() + except Exception: # noqa: BLE001 + pass + finally: + self.ready.set() + + def stop(self, timeout_s: float = 60.0) -> Optional[Path]: + self._stop.set() + self._thread.join(timeout=timeout_s) + if self._thread.is_alive(): + self.error = self.error or "the recorder did not finish in time" + return self.recording + + +def guard_task( + cfg: GuardConfig, + task: GuardTask, + *, + cdp_endpoint: str, + out_dir: Path | str, + page_url: Optional[str] = None, + timeout_s: Optional[float] = None, + attach_wait_s: float = 60.0, + linger_s: float = 1.0, + agent_id: str = "attached-agent", + secret_fields: tuple[str, ...] = (), + record: bool = True, + on_attached: Optional[Callable[[], None]] = None, + stop: Optional[Callable[[], bool]] = None, +) -> GuardRun: + """Guard one task's save in the agent's own browser and record the run. + + ``on_attached`` is called once the guard holds the tab's requests and the + recorder has started (or failed): a harness starts its agent then. ``stop`` + ends the session early when it returns True. After the save is read back + the session keeps recording for ``linger_s`` so the save's result is on + the last frame. + """ + + from openadapt_flow.record_sor import RecordingObservation, RecordSorError + + out = Path(out_dir).expanduser().resolve() + out.mkdir(parents=True, exist_ok=True) + notes: list[str] = [] + observation: Optional[RecordingObservation] = None + reader: Optional[Callable[[], Any]] = None + if record: + try: + # The record reads around run 1 (``record --verify``): run 2's + # record check and compiled effects are proposed from them. + observation = RecordingObservation.start(cfg.verify) + reader = lambda: observation.observer.system_of_record # noqa: E731 + except RecordSorError as exc: + # The save is still read back by verify-run; only the + # recording's record snapshots are missing. + notes.append(f"no record snapshots in the recording: {exc}") + guarded = GuardedSave( + load_verify_config(cfg.verify), out / "guard", approval=cfg.approval + ) + started = time.monotonic() + pw, browser, tab = _connect( + cfg, cdp_endpoint=cdp_endpoint, page_url=page_url, attach_wait_s=attach_wait_s + ) + page: Optional[CdpGuardPage] = None + recorder: Optional[_RecorderThread] = None + ended = "stopped" + try: + page = CdpGuardPage(tab, screen_selector=cfg.screen_selector) + guard = CommitGuard( + guarded, + task, + page, + app_origin=cfg.app_origin, + not_saves=cfg.not_saves, + observe_only=cfg.observe_only, + agent_id=agent_id, + response_wait_s=cfg.response_wait_s, + events_path=out / "guard_events.jsonl", + ) + if record: + recorder = _RecorderThread( + url=cfg.app_url, + out_dir=out / "recording", + cdp_endpoint=cdp_endpoint, + browser_page_url=page_url, + capture_network=True, + system_of_record_reader=reader, + secret_fields=secret_fields, + ) + recorder.start() + # Saves are held from here on; wait (pumping the guard) for the + # recorder so run 1 is recorded from the first action. + deadline = time.monotonic() + attach_wait_s + while not recorder.ready.is_set() and time.monotonic() < deadline: + tab.wait_for_timeout(POLL_MS) + page.drain(guard) + if on_attached is not None: + on_attached() + finished_at: Optional[float] = None + while True: + try: + tab.wait_for_timeout(POLL_MS) + except Exception: # noqa: BLE001 - the tab or the browser is gone + ended = "tab_closed" + break + page.drain(guard) + now = time.monotonic() + if guard.finished and finished_at is None: + finished_at = now + if finished_at is not None and now - finished_at >= linger_s: + ended = "read_back" + break + if timeout_s is not None and now - started >= timeout_s: + ended = "timeout" + break + if stop is not None and stop(): + ended = "stopped" + break + if ended != "tab_closed": + # Hand over whatever the save left queued before letting go. + page.drain(guard) + except KeyboardInterrupt: + ended = "stopped" + except BaseException: + if recorder is not None: + recorder.stop(timeout_s=10.0) + raise + finally: + if page is not None: + page.release() + try: + browser.close() + except Exception: # noqa: BLE001 - only disconnects; the browser stays + pass + pw.stop() + recording: Optional[Path] = None + if recorder is not None: + recording = recorder.stop() + if recorder.error: + notes.append(f"run 1 was not recorded: {recorder.error}") + if recording is not None and observation is not None: + observation.finish(recording, {k: str(v) for k, v in task.params.items()}) + summary = guard.summary.to_dict() + summary.update( + { + "ended": ended, + "seconds": round(time.monotonic() - started, 3), + "recording": str(recording) if recording else None, + "notes": notes, + } + ) + (out / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True)) + return GuardRun( + out_dir=out, + verdict=guard.summary.verdict, + summary=summary, + recording=recording, + ended=ended, + seconds=summary["seconds"], + notes=notes, + ) diff --git a/packages/flow/openadapt_flow/authoring.py b/packages/flow/openadapt_flow/authoring.py index 0db58a5d6..a1c4e0c2a 100644 --- a/packages/flow/openadapt_flow/authoring.py +++ b/packages/flow/openadapt_flow/authoring.py @@ -576,6 +576,12 @@ def remember_node( self._nodes[node_id] = _pixels_from_mapping(backend_pixels) + def node_center(self, node_id: str) -> Optional[tuple[int, int]]: + """The backend-pixel center of an observed node, or None if unknown.""" + + target = self._nodes.get(node_id) + return target.center if target is not None else None + def observe(self) -> dict[str, Any]: """Return a PHI-safe authoring tree of the pinned window. diff --git a/packages/flow/openadapt_flow/compile_agent/cli.py b/packages/flow/openadapt_flow/compile_agent/cli.py index 6e844640d..873d4187a 100644 --- a/packages/flow/openadapt_flow/compile_agent/cli.py +++ b/packages/flow/openadapt_flow/compile_agent/cli.py @@ -15,6 +15,15 @@ openadapt-flow compile-agent run --param note="Recheck in ten days" + openadapt-flow compile-agent admit --admit-as "Dana Reviewer" \\ + --canary-param note="Canary test note" + +``admit`` is for a draft that waits for a person, such as one an agent taught +over MCP (``openadapt-agent serve --teach``): at an interactive terminal the +person confirms the proposed record checks if nobody has yet, reviews the +steps, attends the canary and admits. ``--admit-key`` admits with a +registered key instead, after the checks are confirmed. + The command is re-entrant. Run it again with the same arguments and it picks up where it stopped: the agent runs only when the task has no draft and no admitted program. Confirming the proposed record checks is a person's step at @@ -56,9 +65,14 @@ def add_parser(sub: argparse._SubParsersAction) -> None: ) p.add_argument( "prompt", - help='the task, with {name} for each --param; or "run" to run a compiled task', + help=( + 'the task, with {name} for each --param; "run" to run a compiled ' + 'task; or "admit" to confirm and admit a waiting draft' + ), + ) + p.add_argument( + "task_dir", nargs="?", help="with run or admit: the compiled task directory" ) - p.add_argument("task_dir", nargs="?", help="with run: the compiled task directory") p.add_argument("--param", action="append", metavar="NAME=VALUE") p.add_argument( "--url", @@ -277,6 +291,8 @@ def cmd_compile_agent( ) params = _pairs(args.param, "--param") + if args.prompt == "admit": + return _cmd_admit(args, ask=ask, interactive=interactive) if args.prompt == "run": if not args.task_dir: raise SystemExit("compile-agent run needs the compiled task directory") @@ -349,3 +365,57 @@ def cmd_compile_agent( if result.admitted else (3 if result.status in ("draft", "needs_person") else 1) ) + + +def _cmd_admit( + args: argparse.Namespace, + *, + ask: Callable[[str], str], + interactive: Callable[[], bool], +) -> int: + """A person (or a registered key) admits a draft that waits in TASK_DIR.""" + + from openadapt_flow.compile_agent.pipeline import CONFIRMED_FILE, FirstRunCompiler + + if not args.task_dir: + raise SystemExit("compile-agent admit needs the compiled task directory") + compiler = FirstRunCompiler.load(args.task_dir) + jit = compiler._jit() + if jit.admitted: + _print({"status": "admitted", "detail": "already admitted"}, args.json) + return 0 + if jit.candidate() is None: + raise SystemExit(f"{args.task_dir} has no draft waiting") + tty = interactive() + if not (compiler.out_dir / CONFIRMED_FILE).is_file(): + if not tty: + raise SystemExit( + "a person confirms the proposed record checks at an interactive " + "terminal; there is none" + ) + name = _terminal_confirm(ask)(compiler.proposed()) + if name is None: + print("Not confirmed; nothing changed.") + return 3 + compiler.confirm_checks(principal_id=name, drop_fields=args.drop_field) + admitter = _admitter(args) + if admitter is None and not args.admit_as: + raise SystemExit("pass --admit-as YOUR NAME, or --admit-key") + if admitter is None and not tty: + raise SystemExit("--admit-as needs an interactive terminal") + result = compiler.admit( + admitter=admitter, + person=None if admitter is not None else args.admit_as, + review=_terminal_review(ask) if admitter is None else None, + canary_params=( + [_pairs(args.canary_param, "--canary-param")] if args.canary_param else None + ), + ) + if result.admitted: + result.emitted = compiler.emit() + _print(result.to_dict(), args.json) + return ( + 0 + if result.admitted + else (3 if result.status in ("draft", "needs_person") else 1) + ) diff --git a/packages/flow/openadapt_flow/compile_agent/driven.py b/packages/flow/openadapt_flow/compile_agent/driven.py new file mode 100644 index 000000000..486c614dc --- /dev/null +++ b/packages/flow/openadapt_flow/compile_agent/driven.py @@ -0,0 +1,364 @@ +"""Run 1 driven from outside: an agent that sends one action per call. + +:class:`FirstRunCompiler` records an agent that owns its loop +(``drive(request, harness)``). An agent connected over MCP doesn't: it makes +one tool call per action. :class:`DrivenRun` turns those calls into an agent. +It runs :meth:`FirstRunCompiler.record` on a thread of its own, where the +browser or native backend lives, and hands each call's action to the harness +on that thread. When the caller finishes, the run is compiled and checked +exactly as for any other agent: the same save gate, the same record check, +the same draft. + +The caller decides nothing about the gate. A gated action that a person must +approve is approved by :meth:`DrivenRun.act` only when the caller says a +person did (``held_by_person``); the MCP server sets that only after the +person answered the client's own form. +""" + +from __future__ import annotations + +import json +import queue +import threading +import time +import uuid +from concurrent.futures import Future +from concurrent.futures import TimeoutError as FutureTimeout +from dataclasses import dataclass +from typing import Any, Callable, Mapping, Optional + +from openadapt_flow.compile_agent.agents import AgentLedger +from openadapt_flow.compile_agent.harness import GateRefused, GateRequest, StaleRef +from openadapt_flow.jit.adapters import UnsupportedAction +from openadapt_flow.jit.compile_agent import TaskRequest + +__all__ = ["DrivenAgent", "DrivenRun", "RunEnded", "ACTIONS"] + +#: Actions a caller sends. ``ref`` names a control in the latest observation. +ACTIONS = ("click", "type", "press", "hotkey") + +_STOP = object() + + +class RunEnded(RuntimeError): + """Run 1 is over; no more actions go to it.""" + + +@dataclass +class _Command: + kind: str + payload: Mapping[str, Any] + future: Future + + +class DrivenAgent: + """The agent :class:`FirstRunCompiler` records: actions arrive by queue. + + ``principal_id`` is the identity the recording is signed with. It must be + the registered agent that may later admit the draft, so that admitting it + is self-admission and needs a policy that allows it. + """ + + dialect = "generic" + requires_model_key = False + #: Drives the surface harness too (native windows, RDP, Citrix). + drives_surfaces = True + + def __init__(self, principal_id: Optional[str] = None) -> None: + self.principal_id = principal_id + self.request: Optional[TaskRequest] = None + self.harness: Any = None + self.ready = threading.Event() + self._inbox: "queue.Queue[Any]" = queue.Queue() + self._approved: Optional[str] = None + self.ledger = AgentLedger(agent="driven") + #: This run, and how many actions it has performed: a person's + #: approval is bound to both (see :meth:`binding`). + self.run_id = uuid.uuid4().hex + self.performed = 0 + + # -- called by the gate, on the run thread ----------------------------------- + + def hold(self, request: GateRequest) -> bool: + """True only for the action a person approved, once.""" + + approved, self._approved = self._approved, None + return approved is not None and approved == request.control + + # -- the run thread ------------------------------------------------------------ + + def drive(self, request: TaskRequest, harness: Any) -> AgentLedger: + self.request = request + self.harness = harness + self.ready.set() + while True: + command = self._inbox.get() + if command is _STOP: + raise RunEnded("run 1 was stopped before it finished") + assert isinstance(command, _Command) + if command.kind == "finish": + command.future.set_result({"finished": True}) + break + try: + command.future.set_result(self._handle(command)) + except (StaleRef, UnsupportedAction, LookupError, ValueError) as exc: + # Nothing was performed: report it and keep the run going. + command.future.set_result( + {"error": type(exc).__name__, "detail": str(exc)} + ) + except GateRefused as exc: + command.future.set_result({"refused": str(exc)}) + raise + except BaseException as exc: + command.future.set_exception(exc) + raise + self.ledger.finished = True + self.ledger.wall_seconds = time.monotonic() - self.ledger.started_at + return self.ledger + + def _handle(self, command: _Command) -> dict[str, Any]: + h = self.harness + if command.kind == "observe": + obs = h.observe() + return {"observation": obs.text} + if command.kind == "screenshot": + # The caller decides how to scrub it before it leaves the computer. + h.stats.screenshots_requested += 1 + return {"png": h.computer.screenshot()} + native, ref = self._native(command.payload) + field = ref if native["kind"] == "type" else None + if field is not None: + # type with a ref clicks the field first. That click must commit + # nothing, so it never needs its own approval. + focus_click, _ = h.native_for({"ref": field}) + if h.commit_target(focus_click, field) is not None: + raise ValueError( + f"{field!r} is not a text field; click it with its own teach_act" + ) + commit = h.commit_target(native, ref) + binding = self.binding(native, ref) + if command.kind == "preview": + return { + "commit": commit, + "binding": binding if commit else None, + "action": describe(native), + } + if command.kind == "act": + held = command.payload.get("_held_by_person") + if held and held != binding: + raise ValueError( + "the app or the run changed after the person approved this " + "action, so nothing was done; call teach_act again" + ) + if field is not None: + h.click(field) + self.performed += 1 + # Only what the person was shown is approved, once. + self._approved = commit if held else None + try: + h.act(native, ref) + finally: + self._approved = None + self.performed += 1 + return {"observation": h.observe().text} + raise ValueError(f"unknown command {command.kind!r}") + + def binding(self, native: Mapping[str, Any], ref: Optional[str]) -> str: + """What a person's approval covers: this run, this exact action, and + the app as it was after the last action performed.""" + + return json.dumps( + [self.run_id, self.performed, dict(native), ref], sort_keys=True + ) + + def _native( + self, action: Mapping[str, Any] + ) -> tuple[dict[str, Any], Optional[str]]: + """A caller's action as the harness's generic action, and its ref: the + control a click is on, or the field a ``type`` goes into (clicked + first). Performs nothing.""" + + kind = action.get("action") + if kind not in ACTIONS: + raise ValueError(f"action must be one of {ACTIONS}") + if kind == "click": + if action.get("ref") is not None: + return self.harness.native_for({"ref": action["ref"]}) + if action.get("x") is None or action.get("y") is None: + raise ValueError("click needs ref, or x and y") + return {"kind": "click", "x": int(action["x"]), "y": int(action["y"])}, None + if kind == "type": + if action.get("input") is not None: + inputs = (self.request.inputs if self.request else {}) or {} + name = str(action["input"]) + if name not in inputs: + raise LookupError(f"no task input named {name!r}") + text = str(inputs[name]) + elif action.get("text") is not None: + text = str(action["text"]) + else: + raise ValueError("type needs text or input") + ref = action.get("ref") + return {"kind": "type", "text": text}, ( + str(ref) if ref is not None else None + ) + if kind == "press": + return {"kind": "press", "key": str(action.get("key") or "")}, None + return { + "kind": "hotkey", + "key": str(action.get("key") or ""), + "modifiers": [str(m) for m in action.get("modifiers") or []], + }, None + + # -- called from other threads ------------------------------------------------ + + def submit(self, kind: str, payload: Mapping[str, Any]) -> Future: + future: Future = Future() + self._inbox.put(_Command(kind, dict(payload), future)) + return future + + def stop(self) -> None: + self._inbox.put(_STOP) + + +class DrivenRun: + """One run 1 of ``compiler``'s task, driven call by call. + + ``hold`` says whether every gated action needs a person (on the web the + harness also checks the record's identity itself; on other surfaces it + can't, so a person approves every gated action there whatever ``hold`` + says). + """ + + def __init__( + self, + compiler: Any, + params: Mapping[str, Any], + *, + principal_id: Optional[str] = None, + hold: bool = False, + on_done: Optional[Callable[[], None]] = None, + ) -> None: + self.on_done = on_done + self.compiler = compiler + self.params = dict(params) + self.agent = DrivenAgent(principal_id=principal_id) + self.hold = hold or compiler.backend_kind != "web" + self.result: Any = None + self.error: Optional[BaseException] = None + self.done = threading.Event() + self._thread: Optional[threading.Thread] = None + + @property + def person_approves_saves(self) -> bool: + return self.hold + + def start(self, timeout_s: float = 300.0) -> None: + """Open the app and start recording. Returns once the agent can act. + + Raises: + Exception: whatever stopped the run before the first action + (a draft already exists, the app did not open). + """ + + self._thread = threading.Thread( + target=self._main, name="openadapt-teach-run", daemon=True + ) + self._thread.start() + deadline = time.monotonic() + timeout_s + while not self.agent.ready.wait(0.05): + if self.done.is_set() or time.monotonic() > deadline: + break + if self.error is not None: + raise self.error + if not self.agent.ready.is_set(): + if self.done.is_set(): + raise RunEnded( + "run 1 ended before the agent could act: " + + str(getattr(self.result, "detail", None) or "") + ) + raise TimeoutError("the app did not open in time") + + def _main(self) -> None: + try: + self.result = self.compiler.record( + self.agent, + self.params, + hold=self.agent.hold if self.hold else None, + principal_id=self.agent.principal_id, + ) + except BaseException as exc: # noqa: BLE001 - handed to the caller + self.error = exc + finally: + self.done.set() + if self.on_done is not None: + self.on_done() + + def call( + self, + kind: str, + payload: Optional[Mapping[str, Any]] = None, + timeout_s: float = 120.0, + ) -> dict[str, Any]: + """Send one command to the run thread and wait for its answer. + + Raises: + RunEnded: run 1 is over (finished, refused, or failed). + """ + + if self.done.is_set(): + raise RunEnded("run 1 is over") + future = self.agent.submit(kind, payload or {}) + deadline = time.monotonic() + timeout_s + while True: + try: + return future.result(timeout=0.05) + except FutureTimeout: + pass + except Exception as exc: + raise RunEnded(f"run 1 failed: {type(exc).__name__}: {exc}") from exc + if self.done.is_set() and not future.done(): + raise RunEnded("run 1 is over") + if time.monotonic() > deadline: + # It may still run later; never let it run behind the + # caller's back or twice after a retry. End run 1 instead. + self.abort() + raise RunEnded( + "the action did not finish in time, so run 1 was stopped; " + "check the app, then start again" + ) + + def finish(self, timeout_s: float = 600.0) -> Any: + """End run 1, then wait for the compile and the record check.""" + + if not self.done.is_set(): + self.agent.submit("finish", {}) + if not self.done.wait(timeout_s): + raise TimeoutError("compiling run 1 did not finish in time") + if self.error is not None: + raise self.error + return self.result + + def abort(self) -> None: + """End run 1 now. Nothing more is performed and no draft is made.""" + + self.agent.stop() + + +def describe(native: Mapping[str, Any]) -> str: + """One line a person reads before approving ``native``. No typed text.""" + + kind = native.get("kind") + if kind == "click": + return f"click at x={native.get('x')}, y={native.get('y')}" + if kind == "type": + text = str(native.get("text") or "") + ending = " ending in a line break" if text.endswith(("\n", "\r")) else "" + return f"type {len(text)} characters{ending}" + if kind == "press": + return f"press {native.get('key')}" + if kind == "hotkey": + return "press " + "+".join( + [*native.get("modifiers", []), str(native.get("key"))] + ) + return str(kind) diff --git a/packages/flow/openadapt_flow/compile_agent/harness.py b/packages/flow/openadapt_flow/compile_agent/harness.py index ad604365b..b1a44b0a1 100644 --- a/packages/flow/openadapt_flow/compile_agent/harness.py +++ b/packages/flow/openadapt_flow/compile_agent/harness.py @@ -41,6 +41,9 @@ "HarnessStats", "SaveGate", "StaleRef", + "SurfaceControl", + "SurfaceHarness", + "SurfaceObservation", ] @@ -212,15 +215,51 @@ def type_text(self, text: str, ref: Optional[str] = None) -> Observation: def press(self, key: str) -> Observation: return self.act({"kind": "press", "key": key}) + def native_for( + self, action: Mapping[str, Any] + ) -> tuple[dict[str, Any], Optional[str]]: + """The generic action for a ref-based ``click`` and the ref it named.""" + + ref = action.get("ref") + if ref is None: + return dict(action), None + control = self._control(str(ref)) + return {"kind": "click", "x": control.x, "y": control.y}, control.ref + + def commit_target( + self, native: Mapping[str, Any], ref: Optional[str] = None + ) -> Optional[str]: + """What may commit a write in ``native``, or None. Nothing is performed. + + ``ref`` is the text field a ``type`` will be typed into (it is clicked + first), so the check looks at that field rather than today's focus. + """ + + if ref is not None and native.get("kind") == "type": + control = (self._last.controls if self._last else {}).get(ref) + text = str(native.get("text") or "") + if control is not None and _editable(control): + return "a typed line break" if any(c in text for c in "\r\n") else None + hit = self._commit_target(native) if self.has_page else None + if hit is None: + hit = self._hinted_commit(native, None) + return hit[0] if hit is not None else None + # -- every action ------------------------------------------------------------- def act( - self, native: Mapping[str, Any], *, hint: Optional[CommitHint] = None + self, + native: Mapping[str, Any], + ref: Optional[str] = None, + *, + hint: Optional[CommitHint] = None, ) -> Observation: """Perform one native action in the computer's dialect, behind the gate. - ``hint`` names the control a structure-aware tool resolved (see - :class:`CommitHint`). It can only add a gate. + ``ref`` is accepted for the surface harness's signature; the web + harness reads the page at the point itself. ``hint`` names the control + a structure-aware tool resolved (see :class:`CommitHint`). It can only + add a gate. Raises: GateRefused: the action commits a write and the gate refused it. @@ -316,6 +355,12 @@ def _commit_target( elif op.kind == "type" and any(ch in (op.text or "") for ch in "\r\n"): focused = _observe.focused_element(self.page) or {} return ("a typed line break", focused.get("scope")) + elif op.kind == "type" and " " in (op.text or ""): + # A typed space on a focused button presses it. + focused = _observe.focused_element(self.page) or {} + if not focused.get("editable"): + label = str(focused.get("name") or "the focused element") + return (f"typing a space on {label}", focused.get("scope")) elif op.kind == "hotkey": focused = _observe.focused_element(self.page) or {} shortcut = "+".join([*op.modifiers, op.key or ""]) @@ -398,3 +443,341 @@ def _settle(page: Any, timeout_ms: int = 1500) -> None: page.wait_for_load_state("networkidle", timeout=timeout_ms) except Exception: # noqa: BLE001 - a page that never idles is still usable pass + + +# -- every other surface ---------------------------------------------------------- + + +@dataclass(frozen=True) +class SurfaceControl: + """One control in a surface observation. ``ref`` is the authoring node id.""" + + ref: str + role: str + name: str + x: int + y: int + + +@dataclass +class SurfaceObservation: + """What the agent reads on a native or remote app. + + ``text`` has one line per control the accessibility tree shows (UI + Automation, Accessibility, AT-SPI): ``[n_1a2b3c4d] button "Save"``. A + remote session (RDP, Citrix) has no tree: ``screen_only`` is True and the + agent works from the screen and pixel clicks. + """ + + backend: str + screen_only: bool + text: str + controls: dict[str, SurfaceControl] = field(default_factory=dict) + + def find( + self, name: str, *, near: Optional[str] = None + ) -> Optional[SurfaceControl]: + """The one control named ``name`` (case-insensitive), or None.""" + + wanted = name.strip().lower() + hits = [c for c in self.controls.values() if c.name.lower() == wanted] + return hits[0] if len(hits) == 1 else None + + +def _editable(control: Any) -> bool: + """A web control that takes typed text.""" + + return control.tag in ("input", "textarea") or control.role in ( + "textbox", + "combobox", + ) + + +#: Roles a click only focuses. Typing into them commits nothing. +_EDITABLE_ROLES = frozenset({"text_input", "combobox"}) +#: Roles whose click only changes what the form shows or holds. +_SAFE_CLICK_ROLES = _EDITABLE_ROLES | frozenset({"checkbox", "radio", "tab"}) +#: First words of labels that navigate or open something, and never write. +_NAVIGATION_WORDS = frozenset( + { + "open", + "new", + "next", + "back", + "previous", + "cancel", + "close", + "search", + "find", + "view", + "show", + "edit", + "select", + "start", + "go", + "home", + "menu", + "more", + "expand", + "collapse", + } +) +#: Keys that move focus or dismiss, which never write by themselves. +_FOCUS_KEYS = frozenset( + { + "Tab", + "Escape", + "ArrowUp", + "ArrowDown", + "ArrowLeft", + "ArrowRight", + "Home", + "End", + "PageUp", + "PageDown", + } +) + + +@dataclass +class SurfaceHarness: + """The harness for a native window or a remote session. + + It has the same methods as :class:`Harness`, so an agent drives either. + Observations come from the authoring session's PHI-safe tree (no field + values, no window titles). Every action goes through the same + :class:`CaptureComputer`, so the recording is the one any other capture + makes. + + The save gate is stricter than on the web, because the harness can't read + which form or row a control sits in and so can't do the right-record check + itself. A click is let through only on a control named by its ref whose + role only edits the form (a text field, a check box, a tab) or whose label + plainly navigates (Open, New, Cancel); every other click, including any + click at x,y, is gated. So are Enter, Space and any typing outside a text + field, a typed line break, a single-letter key (a dialog's accelerator), + and every shortcut. A person approves every gated action (the hold). + Without a hold, a gated action is refused. + """ + + computer: CaptureComputer + gate: SaveGate + keep: Sequence[str] = () + evidence_dir: Optional[Path] = None + stats: HarnessStats = field(default_factory=HarnessStats) + gate_log: list[dict[str, Any]] = field(default_factory=list) + refusal: Optional[str] = None + _last: Optional[SurfaceObservation] = None + _focused_role: Optional[str] = None + + page: Any = None + + @property + def dialect(self) -> str: + return self.computer.dialect + + @property + def session(self) -> Any: + return self.computer.session + + # -- observation ------------------------------------------------------------- + + def observe(self) -> SurfaceObservation: + raw = self.session.observe() + controls: dict[str, SurfaceControl] = {} + lines: list[str] = [] + for node in raw.get("tree") or []: + ref = str(node.get("node_id") or "") + center = self.session.node_center(ref) if ref else None + if center is None: + continue + name = str(node.get("name") or "") + role = str(node.get("role") or "unknown") + controls[ref] = SurfaceControl(ref, role, name, center[0], center[1]) + lines.append(f"[{ref}] {role} " + (f'"{name}"' if name else "(no name)")) + screen_only = bool(getattr(self.session, "screen_only", False)) + if screen_only: + lines.append( + "(remote session: no control tree; read the screen and click x,y)" + ) + elif not controls: + lines.append( + "(this app's backend lists no controls; read the screen and click x,y)" + ) + self._last = SurfaceObservation( + backend=str(raw.get("backend") or ""), + screen_only=screen_only, + text="\n".join(lines), + controls=controls, + ) + self.stats.observations += 1 + return self._last + + def screenshot(self, why: str = "requested") -> bytes: + from openadapt_flow.privacy import scrub_image_bytes + + counter = _SCREENSHOT_COUNTERS[why] + png = self.computer.screenshot() + setattr(self.stats, counter, getattr(self.stats, counter) + 1) + return scrub_image_bytes(png) + + # -- actions -------------------------------------------------------------------- + + def _control(self, ref: str) -> SurfaceControl: + last = self._last + if last is None or ref not in last.controls: + raise StaleRef(f"{ref!r} is not in the latest observation; observe again") + control = last.controls[ref] + # The node id hashes the role, automation id and bounds: if the + # control moved or changed, it is gone from a fresh tree. + fresh = { + str(n.get("node_id")): n for n in self.session.observe().get("tree") or [] + } + node = fresh.get(ref) + if node is None or str(node.get("name") or "") != control.name: + raise StaleRef( + f"{ref!r} ({control.name!r}) is no longer at its place; observe again" + ) + return control + + def native_for( + self, action: Mapping[str, Any] + ) -> tuple[dict[str, Any], Optional[str]]: + """The generic action for a ref-based ``click`` and the control's name.""" + + ref = action.get("ref") + if ref is None: + return dict(action), None + control = self._control(str(ref)) + return {"kind": "click", "x": control.x, "y": control.y}, control.ref + + def click(self, ref: str) -> Observation: + control = self._control(ref) + return self.act({"kind": "click", "x": control.x, "y": control.y}, ref=ref) + + def type_text(self, text: str, ref: Optional[str] = None) -> Observation: + if ref is not None: + self.click(ref) + return self.act({"kind": "type", "text": text}, ref=ref) + + def press(self, key: str) -> Observation: + return self.act({"kind": "press", "key": key}) + + def act(self, native: Mapping[str, Any], ref: Optional[str] = None) -> Observation: + """Perform one generic action, behind the gate (see the class doc). + + ``ref`` names the control a click is on, or the text field a ``type`` + goes into. A click with no ref is a click at x,y. + + Raises: + GateRefused: the action may commit a write and no person approved it. + UnsupportedAction: the recording can't hold the action. + """ + + commit = self.commit_target(native, ref) + if commit is not None: + self._check_gate(commit) + obs = self.computer.execute(native) + self.stats.actions += 1 + kind = native.get("kind") + if kind in ("click", "double_click", "right_click"): + control = self._ref_control(ref) + self._focused_role = control.role if control is not None else None + elif kind == "hotkey" or ( + kind == "press" and str(native.get("key")) in _FOCUS_KEYS | {"Enter"} + ): + self._focused_role = None + return obs + + def _ref_control(self, ref: Optional[str]) -> Optional[SurfaceControl]: + if ref is None or self._last is None: + return None + return self._last.controls.get(ref) + + def commit_target( + self, native: Mapping[str, Any], ref: Optional[str] = None + ) -> Optional[str]: + """What may commit a write in ``native``, or None (see the class doc).""" + + control = self._ref_control(ref) + focus = control.role if control is not None else self._focused_role + editable = focus in _EDITABLE_ROLES + for op in translate(self.dialect, native): + if op.kind in ("click", "double_click", "right_click"): + if control is None: + return "a click at a point, not on a named control" + if _observe.commit_label(control.name): + return control.name + words = control.name.lower().split() + if control.role in _SAFE_CLICK_ROLES or ( + words and words[0] in _NAVIGATION_WORDS + ): + continue + return control.name or f"a {control.role} with no name" + elif op.kind == "type": + text = op.text or "" + if any(ch in text for ch in "\r\n"): + return "a typed line break" + if text and not editable: + return "typing outside a text field" + elif op.kind == "hotkey": + return "the shortcut " + "+".join([*op.modifiers, op.key or ""]) + elif op.kind == "press": + key = op.key or "" + if key in ("Enter", "Return"): + return "Enter on the focused control" + if key == "Space" and not editable: + return "Space on the focused control" + if len(key) == 1 and not editable: + return f"the key {key!r} outside a text field" + return None + + def _check_gate(self, control: str) -> None: + self.stats.gate_checks += 1 + shot: Optional[Path] = None + png = self.computer.screenshot() + self.stats.gate_evidence_screenshots += 1 + if self.evidence_dir is not None: + self.evidence_dir.mkdir(parents=True, exist_ok=True) + shot = self.evidence_dir / f"gate-{len(self.gate_log) + 1:02d}.png" + shot.write_bytes(png) + entry: dict[str, Any] = { + "control": control, + "identity": list(self.gate.on_screen), + "missing": [], + "url": "", + "screenshot": str(shot) if shot else None, + "at": time.time(), + "identity_check": "person", + } + hold = self.gate.hold + if hold is None: + entry["decision"] = "refused_no_hold" + self.gate_log.append(entry) + self._refuse( + f"{control!r} may commit a write. On this surface the harness " + "can't read which record it is on, so a person must approve it " + "(a hold), and there is none" + ) + return + approved = bool( + hold( + GateRequest( + control=control, + identity=tuple(self.gate.on_screen), + url="", + screenshot=shot, + ) + ) + ) + if not approved: + entry["decision"] = "refused_by_hold" + self.gate_log.append(entry) + self._refuse(f"the hold did not approve {control!r}") + entry["decision"] = "approved_by_hold" + self.gate_log.append(entry) + + def _refuse(self, reason: str) -> None: + self.stats.gate_refusals += 1 + self.refusal = reason + raise GateRefused(reason) diff --git a/packages/flow/openadapt_flow/compile_agent/pipeline.py b/packages/flow/openadapt_flow/compile_agent/pipeline.py index d087ba033..f5f43e434 100644 --- a/packages/flow/openadapt_flow/compile_agent/pipeline.py +++ b/packages/flow/openadapt_flow/compile_agent/pipeline.py @@ -40,11 +40,17 @@ from pathlib import Path from typing import Any, Callable, Mapping, Optional, Sequence, Union -from openadapt_flow.compile_agent.harness import GateRequest, Harness, SaveGate +from openadapt_flow.compile_agent.harness import ( + GateRequest, + Harness, + SaveGate, + SurfaceHarness, +) from openadapt_flow.jit import lifecycle from openadapt_flow.jit.compile_agent import ( CompiledAgent, JitResult, + SurfaceApp, TaskRequest, WebApp, compile_agent, @@ -132,6 +138,24 @@ def _slug(text: str) -> str: return "compile-agent." + ("-".join(words[:6]) or "task") +#: The ``probe`` prefix that routes an effect to one named read +#: (:func:`openadapt_flow.runtime.effects.surfaces.surface_probe`). +_SURFACE_ROUTE = "surface=" + + +def _claims_new_record( + effect: Mapping[str, Any], *, count_new_only: bool = True +) -> bool: + """Whether a proposed effect says the save adds a record (``record_written`` + with ``expected_count`` above 0); ``expected_count: 0`` says it adds none.""" + + if effect.get("kind") != "record_written": + return False + if count_new_only and not effect.get("count_new_only"): + return False + return int(effect.get("expected_count", 1)) > 0 + + # -- the run-1 record check ------------------------------------------------------ @@ -278,7 +302,7 @@ def __init__( self.keep = list(keep) self.evidence_dir = evidence_dir self.principal_id = principal_id - self.harness: Optional[Harness] = None + self.harness: Optional[Union[Harness, SurfaceHarness]] = None self.ledger: Any = None self.error: Optional[str] = None @@ -287,7 +311,10 @@ def requires_model_key(self) -> bool: return bool(getattr(self.inner, "requires_model_key", True)) def run(self, request: TaskRequest, computer: Any) -> None: - self.harness = Harness( + # A browser page gets the DOM harness; any other surface (a native + # window, an RDP or Citrix client window) gets the surface harness. + harness_type = Harness if computer.page is not None else SurfaceHarness + self.harness = harness_type( computer, self.gate, keep=self.keep, evidence_dir=self.evidence_dir ) try: @@ -314,7 +341,7 @@ def __init__( prompt: str, *, params: Sequence[str], - url: str, + url: Optional[str] = None, verify: Union[str, Path], expect: Mapping[str, Any], on_screen: Sequence[str], @@ -327,10 +354,20 @@ def __init__( on_warning: Optional[Callable[[str], None]] = None, model_key: Optional[CustomerModelKey] = None, backend: str = "web", + surface: Optional[Mapping[str, Any]] = None, + open_backend: Optional[Callable[[Any], Any]] = None, ) -> None: from openadapt_flow.record_sor import load_observer - check_target(url, backend) + self.surface: Optional[Any] = None + if surface is not None: + # A deployment ``backend:`` section for a native or remote app. + # Only an agent that drives the surface harness can record it + # (record() checks); the program runs there like any JIT program. + self.surface = SurfaceApp(surface, open_backend=open_backend).backend + url = None + else: + check_target(url or "", backend) reserved = sorted(set(params) & {"reset", "task_dir"}) if reserved: @@ -356,6 +393,7 @@ def __init__( task_id or _slug(prompt), prompt, {name: str for name in params} ) self.url = url + self.open_backend = open_backend self.verify_path = Path(verify).resolve() self.expect = {str(k): str(v) for k, v in expect.items()} self.on_screen = [str(s) for s in on_screen if str(s).strip()] @@ -381,16 +419,34 @@ def dir(self) -> Path: def out_dir(self) -> Path: return self.dir / OUT_DIR + @property + def backend_kind(self) -> str: + """``web``, or the surface kind (``windows``, ``macos``, ``linux``, + ``rdp``, ``citrix``).""" + + return SurfaceApp(self.surface).kind if self.surface is not None else "web" + + def _app(self) -> Any: + if self.surface is not None: + return SurfaceApp( + self.surface, + record_reader=lambda: self.observer.system_of_record, + reset=self._on_reset, + open_backend=self.open_backend, + ) + assert self.url is not None + return WebApp( + url=self.url, + record_reader=lambda: self.observer.system_of_record, + reset=self._on_reset, + headless=self.headless, + ) + def _jit(self, agent: Any = None) -> CompiledAgent: return compile_agent( agent, task=self.task, - app=WebApp( - url=self.url, - record_reader=lambda: self.observer.system_of_record, - reset=self._on_reset, - headless=self.headless, - ), + app=self._app(), verifier=self.observer.verifier, policy=self.policy, cache_dir=self.cache_dir, @@ -427,6 +483,11 @@ def save_spec(self) -> Path: "key": self.task.key, }, "url": self.url, + "surface": ( + self.surface.model_dump(mode="json", exclude_none=True) + if self.surface is not None + else None + ), "verify": str(self.verify_path), "expect": self.expect, "on_screen": self.on_screen, @@ -439,7 +500,11 @@ def save_spec(self) -> Path: @classmethod def load( - cls, task_dir: Union[str, Path], *, reset: Optional[Callable[[], None]] = None + cls, + task_dir: Union[str, Path], + *, + reset: Optional[Callable[[], None]] = None, + open_backend: Optional[Callable[[Any], Any]] = None, ) -> "FirstRunCompiler": spec = json.loads((Path(task_dir) / SPEC_FILE).read_text()) if spec.get("schema") != SPEC_SCHEMA: @@ -448,7 +513,9 @@ def load( return cls( task["instruction"], params=task["params"], - url=spec["url"], + url=spec.get("url"), + surface=spec.get("surface"), + open_backend=open_backend, verify=spec["verify"], expect=spec["expect"], on_screen=spec["on_screen"], @@ -477,6 +544,12 @@ def record( agent's own. """ + if self.surface is not None and not getattr(agent, "drives_surfaces", False): + raise UnsupportedTarget( + f"this agent reads a web page's DOM; the {self.backend_kind!r} " + "app needs an agent that drives the surface harness (the MCP " + "teach tools do). Nothing ran." + ) self.save_spec() jit = self._jit() values = self.task.validate(params) @@ -650,8 +723,17 @@ def proposed(self) -> dict[str, Any]: doc = yaml.safe_load((self.out_dir / PROPOSED_FILE).read_text()) effects = doc.get("effects") or [] - reads = self.out_dir / AGENT_READS_FILE extra: dict[str, Any] = {} + verifier = doc.get("verifier") + if isinstance(verifier, dict) and verifier != self.verifier_section: + # Built from other reads than the task's verifier: show them. + surfaces = verifier.get("surfaces") + extra["reads"] = ( + sorted(surfaces) + if isinstance(surfaces, dict) + else [str(verifier.get("kind"))] + ) + reads = self.out_dir / AGENT_READS_FILE if reads.is_file(): # What a code-as-action agent read: the identity before the save, # and its inputs read back after it. Evidence for the person. @@ -672,6 +754,11 @@ def proposed(self) -> dict[str, Any]: ) for k, v in (e.get("match") or {}).items() }, + **( + {"expected_count": e["expected_count"]} + if "expected_count" in e + else {} + ), } for e in effects ], @@ -717,8 +804,16 @@ def confirm_checks( for k, v in (effect.get("match") or {}).items() if k not in drop_fields } - if effect.get("kind") == "record_written": - effect["probe"] = ( + if _claims_new_record(effect, count_new_only=False): + # A check routed to one read (scaffold-verifier's + # ``surface=|...``) keeps its route. + route = str(effect.get("probe") or "") + route = ( + route.split("|", 1)[0] + "|" + if route.startswith(_SURFACE_ROUTE) + else "" + ) + effect["probe"] = route + ( "exactly one new record with " + ", ".join( f"{k}=" @@ -732,24 +827,40 @@ def confirm_checks( + f" (confirmed by {name})" ) effects.append(effect) - written = [ - e - for e in effects - if e.get("kind") == "record_written" and e.get("count_new_only") - ] - if len(written) != 1: + # One effect claims the new record. A check built from several reads + # (scaffold-verifier's) claims it once per read, routed to that read; + # then every claim must pin the record and read back every input. + # Other effects may only say that nothing else was added, changed or + # removed (expected_count 0). + written = [e for e in effects if _claims_new_record(e)] + routes = [str(e.get("probe") or "").split("|", 1)[0] for e in written] + if not written or ( + len(written) > 1 + and ( + len(set(routes)) != len(routes) + or not all(r.startswith(_SURFACE_ROUTE) for r in routes) + ) + ): raise ValueError("the checks must claim exactly one new record") - match = written[0]["match"] - for key, wanted in self.expect.items(): - if (match.get(key) or {}).get("literal") != wanted: - raise ValueError(f"the check does not pin {key!r} to {wanted!r}") - bound = {v.get("param") for v in match.values() if isinstance(v, dict)} - unbound = sorted(set(self.task.params) - bound) - if unbound: - raise ValueError(f"no check reads back input(s) {unbound}") + for claim in written: + if int(claim.get("expected_count", 1)) != 1: + raise ValueError("the checks must claim exactly one new record") + match = claim["match"] + for key, wanted in self.expect.items(): + if (match.get(key) or {}).get("literal") != wanted: + raise ValueError(f"the check does not pin {key!r} to {wanted!r}") + bound = {v.get("param") for v in match.values() if isinstance(v, dict)} + unbound = sorted(set(self.task.params) - bound) + if unbound: + raise ValueError(f"no check reads back input(s) {unbound}") + # A check built from other reads than the task's verifier (for + # example scaffold-verifier's) carries them; the person confirms both. + verifier = doc.get("verifier") config = { "schema_version": "openadapt.verify/v1", - "verifier": self.verifier_section, + "verifier": verifier + if isinstance(verifier, dict) + else self.verifier_section, "effects": effects, } parse_verify_config(config, source="confirmed checks") diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 84e0f1e47..752823e16 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -77,6 +77,7 @@ ".github/workflows/release.yml", ".github/workflows/validate-claims.yml", ".pre-commit-config.yaml", + "benchmark/agent_guard/results.json", "benchmark/ap_invoice/results.json", "benchmark/appliance_validation/results.json", "benchmark/appliance_validation/results_identity_full.json", diff --git a/packages/flow/tests/e2e/test_agent_guard_e2e.py b/packages/flow/tests/e2e/test_agent_guard_e2e.py new file mode 100644 index 000000000..caf026820 --- /dev/null +++ b/packages/flow/tests/e2e/test_agent_guard_e2e.py @@ -0,0 +1,107 @@ +"""Guard an agent you already run, end to end in a real Chromium. + +A scripted Playwright agent (``benchmark/agent_guard/agent.py``) drives its +own browser; ``openadapt-flow guard`` runs in another process, attached over +CDP. The guard blocks the agent's save on the wrong patient and tells it +why, lets the right one through and reads it back, and hands that run to the +teach dir. A person confirms the check and admits after a canary; the program +is then a ``taught_*`` tool that saves with no model calls. + +Synthetic MockMed in front of the ``effect_e2e`` record store; localhost only. +""" + +from __future__ import annotations + +import socket +from pathlib import Path + +import pytest + +pytest.importorskip("playwright") +pytest.importorskip("mcp") + +from benchmark.agent_guard import run as bench # noqa: E402 +from benchmark.agent_guard.app import serve # noqa: E402 + +pytestmark = pytest.mark.timeout(900) + + +def _port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _scenario(name: str) -> bench.Scenario: + return next(s for s in bench.SCENARIOS if s.name == name) + + +def _chromium_or_skip() -> None: + from playwright.sync_api import sync_playwright + + try: + with sync_playwright() as pw: + pw.chromium.launch(headless=True).close() + except Exception as exc: # noqa: BLE001 + pytest.skip(f"chromium unavailable: {exc}") + + +def test_guard_blocks_wrong_record_then_run_two_is_a_tool(tmp_path: Path) -> None: + _chromium_or_skip() + app = serve(tmp_path / "app.sqlite") + try: + teach_dir = tmp_path / "teach" + files = bench.write_setup( + tmp_path / "setup", app.base_url, teach_dir=teach_dir, db_path=app.db_path + ) + plain = bench.write_setup(tmp_path / "plain", app.base_url, teach_dir=None) + + wrong = bench.run_one( + app, + _scenario("agent_wrong_patient"), + "guard", + tmp_path / "run-wrong", + _port(), + think_s=1.0, + files=plain, + teach=False, + ) + assert wrong["outcome"] == "blocked", wrong + assert wrong["blocked"] == ["identity_mismatch"] + assert wrong["writes"] == 0 + assert wrong["screen_after"] == "blocked" + assert "patient_id" in (wrong["agent_notice"] or "") + + ok = bench.run_one( + app, + _scenario("ok"), + "guard", + tmp_path / "run-ok", + _port(), + think_s=1.0, + files=files, + teach=True, + ) + assert ok["outcome"] == "clean", ok + assert ok["verdict"] == "VERIFIED" + assert ok["guard_exit"] == 0 + handoff = ok["handoff"] + assert handoff["status"] == "draft", handoff + assert handoff["inputs"] == ["note"] + assert handoff["expect"] == {"patient_id": "p1", "type": "Triage"} + + second = bench.run2( + app, + Path(handoff["task_dir"]), + files["verify"], + teach_dir, + tmp_path / "taught-runs", + ) + assert second["admitted"], second + by_name = {r["scenario"]: r for r in second["runs"]} + assert by_name["ok"]["outcome"] == "clean" + assert by_name["ok"]["model_calls"] == 0 + for fault in ("no_persist", "partial", "duplicate", "wrong_record", "stale"): + assert by_name[fault]["outcome"] == "caught_by_readback", by_name[fault] + finally: + app.stop() diff --git a/packages/flow/tests/e2e/test_mcp_teach_e2e.py b/packages/flow/tests/e2e/test_mcp_teach_e2e.py new file mode 100644 index 000000000..d297498ac --- /dev/null +++ b/packages/flow/tests/e2e/test_mcp_teach_e2e.py @@ -0,0 +1,483 @@ +"""The MCP teach loop on MockMed, driven by a scripted MCP client. + +The client is the real MCP SDK session over its in-memory transport, talking +to the real server. It reads the listings the server returns and picks refs +from them, the way an agent would. No model is called anywhere. + +It asserts: + +* run 1 is recorded through teach_start / teach_act / teach_finish, passes + its record check, and the draft report names the steps, the parameter, the + proposed record check and what still needs a person; +* the agent can't admit before a person confirms the check, and a person + confirms it in the client's form; +* a server with no admission policy refuses to admit and names the terminal + command; a policy that does not allow self-admission refuses the recording + agent before any canary runs; +* under a policy that allows it, the agent admits; the server sends + tools/list_changed; the program is in list_workflows and has its own tool; +* calling that tool saves a new note, ends done, with zero model calls; +* a server started later lists the taught workflow at once; +* a run 1 whose save lands on the wrong record fails its record check and + leaves nothing to admit. +""" + +from __future__ import annotations + +import json +import re +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Iterator, Optional + +import anyio +import pytest + +pytest.importorskip("mcp") + +import mcp.types as types # noqa: E402 +from mcp import ClientSession # noqa: E402 +from mcp.shared.memory import create_client_server_memory_streams # noqa: E402 + +from openadapt_flow.agent.bridge import AgentBridge # noqa: E402 +from openadapt_flow.agent.mcp import build_server # noqa: E402 +from openadapt_flow.agent.runner import RunnerConfig # noqa: E402 +from openadapt_flow.agent.teach import TeachBridge, TeachConfig # noqa: E402 +from openadapt_flow.mockmed.fault_server import serve # noqa: E402 +from openadapt_flow.tutorial import _http_json, _records # noqa: E402 + +TASK = ( + "Open the referral for Jane Sample, start a Triage encounter, and save " + "this note: {note}" +) +NOTE = "Synthetic follow-up in two weeks" +NEW_NOTE = "Recheck knee in ten days" +AGENT_ID = "agent:mcp-teach-e2e" +PERSON = "dana.reviewer" + + +@pytest.fixture(scope="module") +def mockmed() -> Iterator[str]: + base, _db, stop = serve() + yield base.rstrip("/") + stop() + + +def _reset(base: str): + return lambda: _http_json(f"{base}/api/reset", method="POST", body={}) + + +def _verify_yaml(tmp_path: Path, base: str) -> Path: + path = tmp_path / "verify.yaml" + path.write_text( + "verifier:\n" + " kind: rest\n" + f" base_url: {base}\n" + " records_path: /api/db\n" + " records_key: records\n" + " timeout_s: 2.0\n" + " poll_interval_s: 0.05\n" + ) + return path + + +def _agent( + key_dir: Path, base: str, monkeypatch: pytest.MonkeyPatch, *, allow_self: bool +) -> Any: + """The registered agent the operator gives the server, under a policy a + person signed.""" + + from openadapt_flow.jit import authority + + monkeypatch.setenv(authority.KEY_PASSPHRASE_ENV, "e2e passphrase") + files = key_dir.parent / "keyfiles" + signer_pem, agent_pem = files / "signer.pem", files / "agent.pem" + if not agent_pem.exists(): + for role, pid, pem in ( + ("policy_signer", "dana.signer", signer_pem), + ("agent", AGENT_ID, agent_pem), + ): + authority.register_principal( + key_dir, + role=role, + principal_id=pid, + public_key_hex=authority.generate_encrypted_key(pem), + ) + now = datetime.now(timezone.utc) + policy = authority.AdmissionPolicy( + policy_id="mockmed-teach" + ("" if allow_self else "-no-self"), + signer="dana.signer", + signed_at=(now - timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ"), + expires_at=(now + timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ"), + allowed_hosts=(base.split("://", 1)[1],), + allowed_actions=("click", "type"), + allowed_effects=("record_written", "field_equals"), + min_clean_canaries=2, + agent_principals=(AGENT_ID,), + allow_self_admission=allow_self, + ) + signed = authority.sign_policy(policy, authority.load_encrypted_key(signer_pem)) + return authority.principal_from_key( + authority.load_encrypted_key(agent_pem), key_dir, policy=signed + ) + + +def _server( + tmp_path: Path, base: str, *, admitter: Any = None, fault: str = "ok" +) -> tuple[Any, TeachBridge, AgentBridge]: + runs = tmp_path / "runs" + runs.mkdir(exist_ok=True) + workflows = AgentBridge( + None, RunnerConfig(runs_dir=runs), allow_run=True, mode="production" + ) + teach = TeachBridge( + TeachConfig( + verify=_verify_yaml(tmp_path, base), + cache_dir=tmp_path / "teach", + url=f"{base}/?fault={fault}&idempotency=demo#tasks", + key_dir=tmp_path / "keys", + admitter=admitter, + reset=_reset(base), + ), + workflows=workflows, + ) + return build_server(workflows, teach=teach), teach, workflows + + +class Client: + """A scripted MCP client: a person answers forms, an agent picks refs.""" + + def __init__(self, session: ClientSession, notes: list[str]) -> None: + self.session = session + self.notes = notes + self.list_changed = False + + async def call(self, name: str, arguments: Optional[dict] = None) -> dict: + result = await self.session.call_tool(name, arguments or {}) + payload = json.loads(result.content[0].text) + if getattr(result, "is_error", None) or getattr(result, "isError", None): + return {"_error": payload.get("error")} + return payload + + async def tool_names(self) -> set[str]: + return {t.name for t in (await self.session.list_tools()).tools} + + +@asynccontextmanager +async def connect(server: Any, *, person: Optional[str] = PERSON): + notes: list[str] = [] + + async def elicit(context: Any, params: Any) -> types.ElicitResult: + if person is None: + return types.ElicitResult(action="decline", content={}) + return types.ElicitResult( + action="accept", content={"name": person, "confirmed": True} + ) + + async def on_message(message: Any) -> None: + inner = getattr(message, "root", message) + if isinstance(inner, types.ToolListChangedNotification): + notes.append("tools/list_changed") + + async with create_client_server_memory_streams() as (client, transport): + async with anyio.create_task_group() as tasks: + options = server.create_initialization_options() + tasks.start_soon(server.run, *transport, options) + async with ClientSession( + *client, elicitation_callback=elicit, message_handler=on_message + ) as session: + init = await session.initialize() + client_ = Client(session, notes) + tools = init.capabilities.tools + client_.list_changed = bool( + tools + and ( + getattr(tools, "list_changed", None) + or getattr(tools, "listChanged", None) + ) + ) + yield client_ + tasks.cancel_scope.cancel() + + +def _ref(listing: str, name: str, near: Optional[str] = None) -> str: + for line in listing.splitlines(): + if near and near not in line: + continue + found = re.search(r"\[(e\d+)\] \S+ \"" + re.escape(name) + '"', line) + if found: + return found.group(1) + raise AssertionError(f"no {name!r} in listing:\n{listing}") + + +async def _teach_once(client: Client, patient: str = "Jane Sample") -> dict: + """What an agent does: start, act on refs from each listing, finish.""" + + started = await client.call( + "teach_start", + { + "task": TASK, + "inputs": {"note": NOTE}, + "record_fields": {"patient_id": "p1", "type": "Triage"}, + "on_screen": ["Jane Sample", "MRN P1"], + "name": "triage_note", + }, + ) + assert started.get("status") == "recording", started + listing = started["observation"] + for action in ( + {"action": "click", "ref": _ref(listing, "Open", patient)}, + {"action": "click", "name": "New Encounter"}, + {"action": "click", "name": "Triage"}, + {"action": "type", "input": "note", "name": "Note"}, + {"action": "click", "name": "Save Encounter"}, + ): + if "name" in action: + label = action.pop("name") + action = {**action, "ref": _ref(listing, label)} + result = await client.call("teach_act", action) + if result.get("status") == "refused": + break + assert result.get("status") == "ok", result + listing = result["observation"] + return await client.call("teach_finish") + + +def test_teach_compile_admit_then_call_with_no_model_calls( + mockmed: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + async def scenario() -> None: + # Run 1, recorded as the registered agent the operator configured, + # whose policy does not allow self-admission. + no_self = _agent(tmp_path / "keys", mockmed, monkeypatch, allow_self=False) + server, teach, _ = _server(tmp_path, mockmed, admitter=no_self) + async with connect(server) as client: + assert client.list_changed + tools = await client.tool_names() + assert {"teach_start", "teach_act", "teach_finish", "teach_admit"} <= tools + assert "teach_screen" not in tools # the operator didn't turn it on + + draft = await _teach_once(client) + assert draft["status"] == "draft", draft + assert draft["record_check"]["passed"] + assert draft["record_check"]["created"] == 1 + assert [g["decision"] for g in draft["save_gate"]] == ["passed_no_hold"] + assert len(draft["steps"]) >= 5 + assert draft["parameters"]["inputs"] == ["note"] + assert draft["parameters"]["lifted"] == ["note"] + (check,) = draft["proposed_record_check"] + assert check["kind"] == "record_written" + assert check["match"]["patient_id"] == "p1" + assert check["match"]["note"] == "{param: note}" + assert len(draft["needs_person"]) == 1 # confirm the check + task_key = draft["task_key"] + admission = ( + teach.config.cache_dir + / task_key + / "program" + / "bundle" + / "admission.json" + ) + + # The agent can't admit before a person confirms the check. + early = await client.call( + "teach_admit", + {"task_key": task_key, "canary_inputs": [{"note": "Canary"}]}, + ) + assert "confirm the record checks first" in early["_error"], early + + # A person confirms the check in the client's form. + confirmed = await client.call( + "teach_confirm_checks", + {"task_key": task_key, "drop_fields": ["key", "source"]}, + ) + assert confirmed.get("confirmed_by") == PERSON, confirmed + assert confirmed["record_check_confirmed"] + + # The recording agent may not admit its own draft: no canary runs. + before = _records(mockmed) + refused = await client.call( + "teach_admit", + {"task_key": task_key, "canary_inputs": [{"note": "Canary"}]}, + ) + assert refused["status"] == "needs_person", refused + assert "self-admission" in (refused.get("detail") or "") + assert refused["canaries"] == [] + assert _records(mockmed) == before + assert not admission.exists() + + # No policy on the server: refused, with the command a person runs. + server, teach, _ = _server(tmp_path, mockmed) + async with connect(server) as client: + refused = await client.call( + "teach_admit", + {"task_key": task_key, "canary_inputs": [{"note": "Canary"}]}, + ) + assert refused["status"] == "needs_person" + assert "compile-agent admit" in " ".join(refused["needs_person"]) + assert not admission.exists() + + # Under a policy that allows self-admission, the agent admits. + agent = _agent(tmp_path / "keys", mockmed, monkeypatch, allow_self=True) + server, teach, workflows = _server(tmp_path, mockmed, admitter=agent) + async with connect(server) as client: + admitted = await client.call( + "teach_admit", + { + "task_key": task_key, + "canary_inputs": [{"note": "Canary test note"}], + }, + ) + assert admitted["status"] == "admitted", admitted + assert admitted["mode"] == "agent_policy" + assert admitted["admitted_by"] == AGENT_ID + assert [c["status"] for c in admitted["canaries"]] == ["VERIFIED"] * 2 + assert all(c["model_calls"] == 0 for c in admitted["canaries"]) + assert (Path(admitted["skill"]) / "SKILL.md").is_file() + tool = admitted["tool"] + assert client.notes == ["tools/list_changed"] + assert tool in await client.tool_names() + listed = await client.call("list_workflows") + assert [w["name"] for w in listed["workflows"]] == [admitted["workflow"]] + + # Next time: the tool, a new note, no model calls, read back. + ran = await client.call( + tool, {"inputs": {"note": NEW_NOTE}, "request_id": "teach-e2e-0001"} + ) + assert ran["outcome"] == "done", ran + assert ran["model_calls"] == 0 + rows = _records(mockmed) + assert [(r["patient_id"], r["type"], r["note"]) for r in rows] == [ + ("p1", "Triage", NEW_NOTE) + ] + + # A server started later lists it at once. + server, teach, workflows = _server(tmp_path, mockmed, admitter=agent) + async with connect(server) as client: + assert tool in await client.tool_names() + again = await client.call( + "teach_start", + { + "task": TASK, + "inputs": {"note": NOTE}, + "record_fields": {"patient_id": "p1", "type": "Triage"}, + "on_screen": ["Jane Sample", "MRN P1"], + "name": "triage_note", + }, + ) + assert again["started"] is False and again["status"] == "admitted" + + anyio.run(scenario) + + +def test_a_save_on_the_wrong_record_fails_the_read_back( + mockmed: str, tmp_path: Path +) -> None: + async def scenario() -> None: + # MockMed files the note on another patient while the screen shows + # Jane Sample: the gate passes, the read-back does not. + server, teach, _ = _server(tmp_path, mockmed, fault="wrong_record") + async with connect(server) as client: + result = await _teach_once(client) + assert result["status"] == "record_check_failed", result + assert not result["record_check"]["passed"] + assert "wrong record" in result["detail"] + assert result["needs_person"] == [] + status = await client.call("teach_status", {"task_key": result["task_key"]}) + assert status["status"] == "no_draft" + + anyio.run(scenario) + + +def test_the_agent_cannot_confirm_the_check_without_a_person( + mockmed: str, tmp_path: Path +) -> None: + async def scenario() -> None: + server, teach, _ = _server(tmp_path, mockmed) + async with connect(server, person=None) as client: + draft = await _teach_once(client) + assert draft["status"] == "draft", draft + # Extra arguments can't stand in for the person. + forged = await client.call( + "teach_confirm_checks", + {"task_key": draft["task_key"], "person": PERSON}, + ) + assert "_error" in forged + declined = await client.call( + "teach_confirm_checks", {"task_key": draft["task_key"]} + ) + assert "declined" in declined["_error"] + status = await client.call("teach_status", {"task_key": draft["task_key"]}) + assert status["record_check_confirmed"] is False + + anyio.run(scenario) + + +def test_a_person_admits_at_a_terminal_and_the_server_lists_it( + mockmed: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys +) -> None: + """No policy: the agent teaches, a person confirms and admits at a terminal + (``openadapt-flow compile-agent admit``), and the next server lists it.""" + + from openadapt_flow.__main__ import build_parser + from openadapt_flow.compile_agent import FirstRunCompiler + from openadapt_flow.compile_agent.cli import cmd_compile_agent + + async def teach() -> dict: + server, _, _ = _server(tmp_path, mockmed) + async with connect(server) as client: + return await _teach_once(client) + + draft = anyio.run(teach) + assert draft["status"] == "draft", draft + command = next(n for n in draft["needs_person"] if "compile-agent admit" in n) + task_dir = tmp_path / "teach" / draft["task_key"] + assert str(task_dir) in command + + # The CLI has no reset hook (a real app is never reset); MockMed's demo + # idempotency key needs one before the canary. + original = FirstRunCompiler._on_reset + + def reset_first(self: FirstRunCompiler) -> None: + _reset(mockmed)() + original(self) + + monkeypatch.setattr(FirstRunCompiler, "_on_reset", reset_first) + # A person's admission needs a real terminal; pytest has none. + monkeypatch.setattr("openadapt_flow.jit.lifecycle._has_terminal", lambda: True) + answers = iter([PERSON, "ok", "ok", "ok"]) + args = build_parser().parse_args( + [ + "compile-agent", + "admit", + str(task_dir), + "--admit-as", + PERSON, + "--drop-field", + "key", + "--drop-field", + "source", + "--canary-param", + "note=Canary test note", + "--json", + ] + ) + code = cmd_compile_agent( + args, ask=lambda _: next(answers), interactive=lambda: True + ) + out = capsys.readouterr().out + result = json.loads(out[out.index("\n{") + 1 :]) + assert code == 0 and result["status"] == "admitted", result + assert result["admission"]["mode"] == "person" + + async def call() -> None: + server, teach_bridge, _ = _server(tmp_path, mockmed) + async with connect(server) as client: + (tool,) = [t for t in await client.tool_names() if t.startswith("taught_")] + ran = await client.call( + tool, {"inputs": {"note": NEW_NOTE}, "request_id": "teach-e2e-0002"} + ) + assert ran["outcome"] == "done" and ran["model_calls"] == 0, ran + + anyio.run(call) diff --git a/packages/flow/tests/test_agent_guard.py b/packages/flow/tests/test_agent_guard.py new file mode 100644 index 000000000..dc244d513 --- /dev/null +++ b/packages/flow/tests/test_agent_guard.py @@ -0,0 +1,547 @@ +"""The commit guard decides each request of an agent it doesn't control. + +The page here is a fake: when the guard lets a save go on, the fake sends it +to MockMed's fault server the way the agent's browser would, with the patient +that is on its screen. The guard reads the save back from ``GET /api/db``. +No browser, no model calls, nothing beyond localhost. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Optional + +import pytest + +from openadapt_flow.agent_guard.cdp import CdpGuardPage +from openadapt_flow.agent_guard.commit_guard import BLOCK_STATUS, CommitGuard +from openadapt_flow.agent_guard.config import ConfigError, parse_config +from openadapt_flow.agent_guard.requests import ( + HeldRequest, + NotASave, + classify, + origin_of, +) +from openadapt_flow.guard import GuardedSave, GuardTask +from openadapt_flow.guard_demo import NOTE, FakeAgent, person_approves, verify_yaml +from openadapt_flow.mockmed.fault_server import serve as fault_serve +from openadapt_flow.verify_run import parse_verify_config + +APP = "http://127.0.0.1:8765" + + +def _req( + method: str = "POST", + path: str = "/api/encounter", + *, + body: Optional[Any] = None, + content_type: str = "application/json", + origin: str = APP, + resource_type: str = "Fetch", + rid: str = "r1", +) -> HeldRequest: + return HeldRequest( + request_id=rid, + method=method, + url=origin + path, + resource_type=resource_type, + post_data=json.dumps(body) if body is not None else None, + content_type=content_type, + network_id="n-" + rid, + ) + + +# -- which request is a save --------------------------------------------------- + + +@pytest.mark.parametrize( + ("request_", "kind"), + [ + (_req("GET", "/api/db"), "read"), + (_req("HEAD", "/"), "read"), + (_req("OPTIONS", "/api/encounter"), "read"), + (_req("POST", body={"patient_id": "p1", "note": "x"}), "save"), + (_req("PUT"), "save"), + (_req("PATCH"), "save"), + (_req("DELETE", "/api/encounter/4"), "save"), + (_req("PROPFIND"), "save"), + (_req(body={"query": "query Q { patient(id: 1) { name } }"}), "read"), + (_req(body={"query": "{ patient(id: 1) { name } }"}), "read"), + ( + _req(body={"query": "# a comment with mutation\nquery Q { a }"}), + "read", + ), + (_req(body={"query": 'mutation M { addNote(text: "x") { id } }'}), "save"), + ( + _req( + body=[ + {"query": "query A { a }"}, + {"query": "mutation B { b }", "variables": {}}, + ] + ), + "save", + ), + ( + _req(body={"extensions": {"persistedQuery": {"sha256Hash": "ab"}}}), + "save", + ), + # A save that happens to have a "query" field is not GraphQL. + (_req(body={"query": "query about knee pain", "note": "x"}), "save"), + (_req("GET", "/graphql?query=mutation%20M%20%7B%20a%20%7D"), "save"), + (_req("GET", "/graphql?query=query%20Q%20%7B%20a%20%7D"), "read"), + (_req("POST", origin="http://127.0.0.1:9999"), "other_origin"), + ], +) +def test_classify(request_: HeldRequest, kind: str) -> None: + assert classify(request_, app_origin=APP) == kind + + +def test_not_a_save_rules_match_method_path_and_origin() -> None: + rules = ( + NotASave("POST", "/chart/signin"), + NotASave("*", "/collect*", origin="https://stats.example"), + ) + assert classify(_req(path="/chart/signin"), app_origin=APP, not_saves=rules) == ( + "not_a_save" + ) + assert classify(_req("PUT", "/chart/signin"), app_origin=APP, not_saves=rules) == ( + "save" + ) + other = _req(path="/collect/v2", origin="https://stats.example") + assert classify(other, app_origin=APP, not_saves=rules) == "not_a_save" + assert origin_of("HTTPS://Stats.Example:443/x") == "https://stats.example" + + +def test_held_request_from_cdp_event() -> None: + held = HeldRequest.from_cdp( + { + "requestId": "interception-1", + "networkId": "1000.7", + "resourceType": "XHR", + "request": { + "method": "post", + "url": APP + "/api/encounter?fault=ok", + "postData": "{}", + "headers": {"Content-Type": "application/json"}, + }, + } + ) + assert (held.method, held.path, held.content_type) == ( + "POST", + "/api/encounter", + "application/json", + ) + assert held.network_id == "1000.7" + + +# -- the guard against a real record service ---------------------------------------- + + +class FakePage: + """The agent's tab: sends what the guard lets through to MockMed.""" + + def __init__(self, agent: FakeAgent, patient: str, *, fault: str = "") -> None: + self.agent = agent + self.patient = patient + self.fault = fault + self.sent: list[str] = [] + self.blocked: list[dict[str, Any]] = [] + self.notices: list[tuple[str, str]] = [] + self.pending: dict[str, HeldRequest] = {} + self.guard: Optional[CommitGuard] = None + + def screen_text(self) -> Optional[str]: + return self.agent.open_chart(self.patient) + + def screenshot(self) -> Optional[bytes]: + from openadapt_flow.guard_demo import tiny_png + + return tiny_png() + + def continue_request(self, request_id: str) -> None: + self.sent.append(request_id) + held = self.pending.pop(request_id, None) + if held is not None and held.method == "POST": + self.agent.click_save(self.patient, NOTE, fault=self.fault) + assert self.guard is not None + self.guard.on_request_done(held.network_id) + + def block_request(self, request_id: str, **response: Any) -> None: + self.pending.pop(request_id, None) + self.blocked.append({"id": request_id, **response}) + + def notify(self, message: str, *, level: str) -> None: + self.notices.append((level, message)) + + def clear_notice(self) -> None: + self.notices.append(("clear", "")) + + def save(self, rid: str = "r1", **kw: Any) -> HeldRequest: + held = _req(origin=self.agent.base, rid=rid, body={"note": NOTE}, **kw) + self.pending[rid] = held + assert self.guard is not None + self.guard.on_request(held) + return held + + +@pytest.fixture +def sor(): + url, db, stop = fault_serve() + try: + yield url.rstrip("/"), db + finally: + stop() + + +def _task() -> GuardTask: + return GuardTask( + task_id="task-guard-0001", + expected_record={"patient_id": "p1"}, + intended_fields={"note": NOTE}, + authorization_ref="ticket-0001", + ) + + +def _guard( + sor, tmp_path: Path, page: FakePage, *, approval: str = "not_gated", **kw: Any +) -> CommitGuard: + guarded = GuardedSave( + parse_verify_config(verify_yaml(sor[0])), tmp_path / "state", approval=approval + ) + guard = CommitGuard( + guarded, + _task(), + page, + app_origin=sor[0], + events_path=tmp_path / "events.jsonl", + **kw, + ) + page.guard = guard + return guard + + +def _rows(db) -> list[tuple[str, str]]: + return [(r["patient_id"], r["note"]) for r in db.snapshot()["records"]] + + +def test_right_record_save_goes_through_and_reads_back_verified(sor, tmp_path): + page = FakePage(FakeAgent(sor[0]), "p1") + guard = _guard(sor, tmp_path, page) + page.save() + guard.poll() + assert guard.finished + assert guard.summary.verdict == "VERIFIED" + assert _rows(sor[1]) == [("p1", NOTE)] + assert page.sent == ["r1"] and not page.blocked + assert guard.summary.saves_held == 1 and len(guard.summary.hold_ms) == 1 + # A verified save leaves nothing on the agent's page. + assert page.notices == [("clear", "")] + events = [ + json.loads(line) + for line in (tmp_path / "events.jsonl").read_text().splitlines() + ] + # No record values in the log: methods, digests, decisions. + assert NOTE not in json.dumps(events) and "p1" not in json.dumps(events) + assert events[-1] == { + "event": "read_back", + "t": events[-1]["t"], + "verdict": "VERIFIED", + } + + +def test_wrong_record_on_screen_is_blocked_and_the_agent_is_told(sor, tmp_path): + page = FakePage(FakeAgent(sor[0]), "p2") + guard = _guard(sor, tmp_path, page) + page.save() + guard.poll() + assert _rows(sor[1]) == [] + (blocked,) = page.blocked + assert blocked["status"] == BLOCK_STATUS + body = json.loads(blocked["body"])["openadapt_guard"] + assert body["code"] == "identity_mismatch" + assert "patient_id" in body["message"] + assert blocked["headers"]["X-OpenAdapt-Guard"].startswith("blocked") + assert page.notices[-1][0] == "error" + assert "blocked this save" in page.notices[-1][1] + assert not guard.finished + # The agent opens the right record and saves again: that one goes through. + page.patient = "p1" + page.save("r2") + guard.poll() + assert guard.summary.verdict == "VERIFIED" + assert _rows(sor[1]) == [("p1", NOTE)] + + +def test_second_save_of_the_task_is_blocked(sor, tmp_path): + page = FakePage(FakeAgent(sor[0]), "p1") + guard = _guard(sor, tmp_path, page) + # A double-submit: two saves before the first is read back. + guard.on_request(_req(origin=sor[0], rid="a", body={"note": NOTE})) + page.pending["b"] = _req(origin=sor[0], rid="b", body={"note": NOTE}) + guard.on_request(page.pending["b"]) + assert [b["id"] for b in page.blocked] == ["b"] + assert json.loads(page.blocked[0]["body"])["openadapt_guard"]["code"] == ( + "needs_reconciliation" + ) + + +def test_a_save_that_does_not_land_is_not_verified(sor, tmp_path): + page = FakePage(FakeAgent(sor[0]), "p1", fault="no_persist") + guard = _guard(sor, tmp_path, page) + page.save() + guard.poll() + assert guard.summary.verdict == "RECONCILIATION_REQUIRED" + assert page.notices[-1][0] == "error" + + +def test_save_lands_on_another_record_is_not_verified(sor, tmp_path): + page = FakePage(FakeAgent(sor[0]), "p1", fault="wrong_record") + guard = _guard(sor, tmp_path, page) + page.save() + guard.poll() + assert guard.summary.verdict != "VERIFIED" + + +def test_observe_only_never_blocks_and_still_reads_back(sor, tmp_path): + page = FakePage(FakeAgent(sor[0]), "p2") + guard = _guard(sor, tmp_path, page, observe_only=True) + page.save() + guard.poll() + assert page.blocked == [] + assert _rows(sor[1]) == [("p2", NOTE)] + assert guard.summary.would_block[0]["code"] == "identity_mismatch" + assert guard.summary.verdict != "VERIFIED" + # A second save is let through too, and logged. + page.save("r2") + assert page.blocked == [] + assert guard.summary.would_block[-1]["code"] == "already_committed" + + +def test_each_item_holds_the_save_until_a_person_approves(sor, tmp_path): + page = FakePage(FakeAgent(sor[0]), "p1") + guard = _guard(sor, tmp_path, page, approval="each_item") + page.save() + assert page.sent == [] and guard.busy + assert page.notices[-1][0] == "info" + guard.poll() + assert page.sent == [] + # Another save while one waits for a person is blocked. + page.save("r2") + assert [b["id"] for b in page.blocked] == ["r2"] + waiting = guard._waiting + assert waiting is not None + g = waiting[0] + request = g._request + person_approves(str(g.dir), request.request_id, request.digest()) + guard.poll() + assert page.sent == ["r1"] + guard.poll() + assert guard.summary.verdict == "VERIFIED" + + +def test_write_to_another_origin_is_blocked(sor, tmp_path): + page = FakePage(FakeAgent(sor[0]), "p1") + guard = _guard(sor, tmp_path, page) + guard.on_request(_req(origin="http://127.0.0.1:1", rid="x")) + assert page.blocked[0]["id"] == "x" + assert json.loads(page.blocked[0]["body"])["openadapt_guard"]["code"] == ( + "other_origin" + ) + + +def test_blocked_form_post_gets_a_page_that_says_why(sor, tmp_path): + page = FakePage(FakeAgent(sor[0]), "p2") + guard = _guard(sor, tmp_path, page) + held = _req( + origin=sor[0], + path="/encounter/save", + resource_type="Document", + content_type="application/x-www-form-urlencoded", + ) + guard.on_request(held) + (blocked,) = page.blocked + assert blocked["content_type"].startswith("text/html") + assert "OpenAdapt blocked this save" in blocked["body"] + + +def test_no_read_back_waits_past_response_wait(sor, tmp_path): + clock = [0.0] + page = FakePage(FakeAgent(sor[0]), "p1") + guard = _guard(sor, tmp_path, page, response_wait_s=5.0, clock=lambda: clock[0]) + held = _req(origin=sor[0], body={"note": NOTE}) + guard.on_request(held) # sent; the fake never reports it done + guard.poll() + assert not guard.finished + clock[0] = 6.0 + guard.poll() + # Nothing landed (the fake page didn't send it): not verified. + assert guard.summary.verdict == "RECONCILIATION_REQUIRED" + + +# -- the CDP adapter fails closed --------------------------------------------------- + + +class FakeCDP: + def __init__(self) -> None: + self.sent: list[tuple[str, dict]] = [] + self.handlers: dict[str, Any] = {} + self.detached = False + + def on(self, event: str, handler: Any) -> None: + self.handlers[event] = handler + + def send(self, method: str, params: Optional[dict] = None) -> dict: + self.sent.append((method, params or {})) + return {} + + def detach(self) -> None: + self.detached = True + + +class FakeTab: + def __init__(self) -> None: + self.cdp = FakeCDP() + self.context = self + + def new_cdp_session(self, _page: Any) -> FakeCDP: + return self.cdp + + +def _paused(rid: str, method: str = "POST") -> dict: + return { + "requestId": rid, + "networkId": "n" + rid, + "resourceType": "Fetch", + "request": {"method": method, "url": APP + "/api/x", "headers": {}}, + } + + +def test_cdp_page_pauses_writes_and_blocks_on_a_guard_error() -> None: + tab = FakeTab() + page = CdpGuardPage(tab) + enable = dict(tab.cdp.sent)["Fetch.enable"] + kinds = {p["resourceType"] for p in enable["patterns"]} + assert {"Document", "XHR", "Fetch", "Ping"} <= kinds + assert "Image" not in kinds + + class Broken: + def on_request(self, _held: Any) -> None: + raise RuntimeError("boom") + + tab.cdp.handlers["Fetch.requestPaused"](_paused("1")) + with pytest.raises(RuntimeError): + page.drain(Broken()) + method, params = tab.cdp.sent[-1] + assert method == "Fetch.fulfillRequest" and params["requestId"] == "1" + assert params["responseCode"] == BLOCK_STATUS + + +def test_cdp_page_blocks_what_is_still_paused_before_it_detaches() -> None: + tab = FakeTab() + page = CdpGuardPage(tab) + + class Holds: + def on_request(self, _held: Any) -> None: + pass # e.g. waiting for a person + + def on_request_done(self, _nid: Any) -> None: + pass + + def poll(self) -> None: + pass + + tab.cdp.handlers["Fetch.requestPaused"](_paused("7")) + page.drain(Holds()) + tab.cdp.handlers["Fetch.requestPaused"](_paused("8")) # never drained + page.release() + methods = [m for m, _ in tab.cdp.sent] + fulfilled = [p["requestId"] for m, p in tab.cdp.sent if m == "Fetch.fulfillRequest"] + assert fulfilled == ["7", "8"] + assert methods.index("Fetch.disable") > methods.index("Fetch.fulfillRequest") + assert tab.cdp.detached + + +# -- guard.yaml ------------------------------------------------------------------------ + + +def test_config_parses_and_refuses(tmp_path): + verify = tmp_path / "verify.yaml" + verify.write_text("verifier: {kind: rest}\n") + raw = { + "schema_version": "openadapt.agent-guard/v1", + "app_url": "http://127.0.0.1:8765/#tasks", + "verify": "verify.yaml", + "not_saves": [{"method": "POST", "path": "/chart/signin"}], + "teach": {"dir": "teach", "inputs": ["note"], "db": "app.sqlite"}, + } + cfg = parse_config(raw, base=tmp_path) + assert cfg.app_origin == "http://127.0.0.1:8765" + assert cfg.approval == "not_gated" and not cfg.observe_only + assert cfg.teach is not None and cfg.teach.inputs == ("note",) + assert cfg.teach.dir == (tmp_path / "teach").resolve() + for bad in ( + {**raw, "schema_version": "x"}, + {**raw, "surprise": 1}, + {**raw, "approval": "auto"}, + {**raw, "app_url": "file:///etc"}, + {**raw, "not_saves": [{"method": "POST"}]}, + {**raw, "teach": {"name": "x"}}, + {**raw, "verify": "missing.yaml"}, + ): + with pytest.raises(ConfigError): + parse_config(bad, base=tmp_path) + + +# -- run 2's inputs and record fields --------------------------------------------------- + + +def test_handoff_inputs_are_the_typed_values_and_the_rest_pins_the_record(tmp_path): + from openadapt_flow.agent_guard.handoff import _expect, _record_match, typed_inputs + from openadapt_flow.compile_agent.pipeline import _claims_new_record + + recording = tmp_path / "rec" + recording.mkdir() + (recording / "events.jsonl").write_text( + json.dumps({"kind": "click", "i": 0}) + + "\n" + + json.dumps({"kind": "type", "i": 1, "text": NOTE}) + + "\n" + ) + task = GuardTask( + task_id="task-guard-0002", + expected_record={"patient_id": "p1", "name": "Jane Sample"}, + intended_fields={"type": "Triage", "note": NOTE}, + ) + assert typed_inputs(recording, task) == ["note"] + verify = tmp_path / "verify.yaml" + verify.write_text( + "schema_version: openadapt.verify/v1\n" + "verifier: {kind: rest, base_url: 'http://127.0.0.1:1', records_path: /r}\n" + "effects:\n" + " - kind: record_written\n" + " match:\n" + " patient_id: {param: patient_id}\n" + " type: {param: type}\n" + " note: {param: note}\n" + " source: {literal: replay}\n" + " expected_count: 1\n" + " count_new_only: true\n" + ) + cfg = parse_config( + { + "schema_version": "openadapt.agent-guard/v1", + "app_url": APP, + "verify": str(verify), + "teach": {"dir": "t"}, + }, + base=tmp_path, + ) + expect = _expect(_record_match(cfg), task, ["note"]) + # The name is on screen but isn't a field of the record: not expected. + assert expect == {"patient_id": "p1", "type": "Triage", "source": "replay"} + assert _claims_new_record({"kind": "record_written", "count_new_only": True}) + assert not _claims_new_record( + {"kind": "record_written", "count_new_only": True, "expected_count": 0} + ) + assert not _claims_new_record({"kind": "field_equals", "count_new_only": True}) diff --git a/packages/flow/tests/test_compile_agent_surface_harness.py b/packages/flow/tests/test_compile_agent_surface_harness.py new file mode 100644 index 000000000..3307b1f19 --- /dev/null +++ b/packages/flow/tests/test_compile_agent_surface_harness.py @@ -0,0 +1,317 @@ +"""The surface harness and the driven agent, on a fake native window. + +Windows, macOS and Linux apps give the harness an accessibility tree (UI +Automation, Accessibility, AT-SPI) but no way to read which form or row a +control is in. These tests check, with no app and no browser, that every +action that may commit a write waits for a person, that nothing is performed +when it is refused, and that the teach server's configuration keeps an agent +from admitting without a signed policy. +""" + +from __future__ import annotations + +import io +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Optional + +import pytest +from PIL import Image + +from openadapt_flow.compile_agent.driven import DrivenAgent, DrivenRun +from openadapt_flow.compile_agent.harness import ( + GateRefused, + GateRequest, + SaveGate, + StaleRef, + SurfaceHarness, +) +from openadapt_flow.jit.adapters import CaptureComputer + + +def _png() -> bytes: + out = io.BytesIO() + Image.new("RGB", (8, 8), "white").save(out, format="PNG") + return out.getvalue() + + +class FakeSession: + """An authoring session over a native window with three controls.""" + + screen_only = False + + def __init__(self) -> None: + self.nodes = [ + { + "node_id": "n_00000001", + "role": "text_input", + "name": "Note", + "c": (100, 40), + }, + {"node_id": "n_00000002", "role": "button", "name": "Save", "c": (100, 80)}, + { + "node_id": "n_00000003", + "role": "button", + "name": "Open", + "c": (100, 120), + }, + {"node_id": "n_00000004", "role": "button", "name": "Yes", "c": (100, 160)}, + ] + self.performed: list[tuple[Any, ...]] = [] + + def observe(self) -> dict[str, Any]: + return { + "backend": "windows", + "tree": [{k: v for k, v in n.items() if k != "c"} for n in self.nodes], + } + + def node_center(self, node_id: str) -> Optional[tuple[int, int]]: + return next((n["c"] for n in self.nodes if n["node_id"] == node_id), None) + + def click(self, x: int, y: int) -> None: + self.performed.append(("click", x, y)) + + def double_click(self, x: int, y: int) -> None: + self.performed.append(("double_click", x, y)) + + def right_click(self, x: int, y: int) -> None: + self.performed.append(("right_click", x, y)) + + def type_text(self, text: str) -> None: + self.performed.append(("type", text)) + + def press(self, key: str) -> None: + self.performed.append(("press", key)) + + def hotkey(self, modifiers: list[str], key: str) -> None: + self.performed.append(("hotkey", tuple(modifiers), key)) + + def screenshot(self) -> bytes: + return _png() + + +def _harness(hold: Any = None) -> tuple[SurfaceHarness, FakeSession]: + session = FakeSession() + computer = CaptureComputer(session, dialect="generic", sleep=lambda s: None) + harness = SurfaceHarness(computer, SaveGate(["Jane Sample", "MRN P1"], hold)) + harness.observe() + return harness, session + + +def test_the_observation_lists_controls_by_ref_without_values() -> None: + harness, _ = _harness() + obs = harness.observe() + assert obs.text.splitlines() == [ + '[n_00000001] text_input "Note"', + '[n_00000002] button "Save"', + '[n_00000003] button "Open"', + '[n_00000004] button "Yes"', + ] + assert obs.find("save").ref == "n_00000002" + assert not obs.screen_only + + +def test_a_save_without_a_person_is_refused_and_not_performed() -> None: + harness, session = _harness() + with pytest.raises(GateRefused, match="a person must approve it"): + harness.click("n_00000002") + assert session.performed == [] + assert harness.gate_log[-1]["decision"] == "refused_no_hold" + assert harness.refusal is not None + + +def test_a_person_approves_or_refuses_each_save() -> None: + asked: list[GateRequest] = [] + harness, session = _harness(hold=lambda req: asked.append(req) or True) + harness.click("n_00000002") + assert session.performed == [("click", 100, 80)] + assert asked[0].control == "Save" and asked[0].identity == ("Jane Sample", "MRN P1") + assert harness.gate_log[-1]["decision"] == "approved_by_hold" + + harness, session = _harness(hold=lambda req: False) + with pytest.raises(GateRefused, match="did not approve"): + harness.click("n_00000002") + assert session.performed == [] + + +NOTE, SAVE, OPEN, YES = "n_00000001", "n_00000002", "n_00000003", "n_00000004" + + +@pytest.mark.parametrize( + ("setup", "last", "gated"), + [ + ([], ("click", OPEN), None), + ([], ("click", YES), "Yes"), + ([], ("click", SAVE), "Save"), + ([("click", NOTE)], ("type", "a note"), None), + ([], ("type_into", "a note"), None), + ([("click", NOTE)], ("press", "Space"), None), + ( + [("click", NOTE), ("press", "Tab")], + ("press", "Space"), + "Space on the focused control", + ), + ([("click", OPEN)], ("type", "y"), "typing outside a text field"), + ([], ("press", "y"), "the key 'y' outside a text field"), + ([("click", NOTE)], ("type", "a note\n"), "a typed line break"), + ([("click", NOTE)], ("press", "Enter"), "Enter on the focused control"), + ([], ("hotkey", "s"), "the shortcut Control+s"), + # Save's own center, but by x,y: nothing names the control. + ([], ("at", (100, 80)), "a click at a point, not on a named control"), + ], +) +def test_what_is_gated(setup: list, last: tuple, gated: Optional[str]) -> None: + harness, session = _harness() + for kind, arg in setup: + if kind == "click": + harness.click(arg) + else: + harness.press(arg) + kind, arg = last + ref = None + if kind == "click": + native, ref = harness.native_for({"ref": arg}) + elif kind == "type": + native = {"kind": "type", "text": arg} + elif kind == "type_into": + native, ref = {"kind": "type", "text": arg}, NOTE + elif kind == "press": + native = {"kind": "press", "key": arg} + elif kind == "hotkey": + native = {"kind": "hotkey", "key": arg, "modifiers": ["ctrl"]} + else: + native = {"kind": "click", "x": arg[0], "y": arg[1]} + assert harness.commit_target(native, ref) == gated + before = list(session.performed) + if gated is None: + harness.act(native, ref) + assert len(session.performed) == len(before) + 1 + else: + with pytest.raises(GateRefused): + harness.act(native, ref) + assert session.performed == before + + +def test_an_approval_covers_one_action_on_one_run_as_the_app_was() -> None: + harness, session = _harness(hold=None) + agent = DrivenAgent() + harness.gate.hold = agent.hold + agent.harness = harness + agent.request = SimpleNamespace(inputs={}) + + def call(kind: str, payload: dict) -> dict: + from concurrent.futures import Future + + from openadapt_flow.compile_agent.driven import _Command + + return agent._handle(_Command(kind, payload, Future())) + + save = {"action": "click", "ref": SAVE} + preview = call("preview", save) + assert preview["commit"] == "Save" and preview["action"] == "click at x=100, y=80" + # Another action happens while the person reads the form: the approval + # no longer covers what is on screen. + call("act", {"action": "click", "ref": OPEN}) + with pytest.raises(ValueError, match="changed after the person approved"): + call("act", {**save, "_held_by_person": preview["binding"]}) + assert ("click", 100, 80) not in session.performed + # Another run never matches. + other = DrivenAgent() + assert ( + other.binding({"kind": "click", "x": 100, "y": 80}, SAVE) + != call("preview", save)["binding"] + ) + # The fresh approval goes through, once. + fresh = call("preview", save) + call("act", {**save, "_held_by_person": fresh["binding"]}) + assert session.performed[-1] == ("click", 100, 80) + assert harness.gate_log[-1]["decision"] == "approved_by_hold" + + +def test_type_into_a_control_that_is_not_a_field_is_refused() -> None: + harness, session = _harness() + agent = DrivenAgent() + agent.harness, agent.request = harness, SimpleNamespace(inputs={}) + from concurrent.futures import Future + + from openadapt_flow.compile_agent.driven import _Command + + with pytest.raises(ValueError, match="not a text field"): + agent._handle( + _Command("preview", {"action": "type", "text": "x", "ref": SAVE}, Future()) + ) + assert session.performed == [] + + +def test_a_control_that_changed_is_a_stale_ref() -> None: + harness, session = _harness() + session.nodes[1]["name"] = "Delete" + with pytest.raises(StaleRef): + harness.click("n_00000002") + assert session.performed == [] + + +def test_the_driven_agent_approves_only_the_action_a_person_approved_once() -> None: + agent = DrivenAgent() + agent._approved = "Save" + assert not agent.hold(GateRequest("Delete", (), "", None)) + assert not agent.hold(GateRequest("Save", (), "", None)) # used up + agent._approved = "Save" + assert agent.hold(GateRequest("Save", (), "", None)) + + +def test_a_person_approves_every_save_off_the_web() -> None: + for kind, expected in (("web", False), ("macos", True), ("rdp", True)): + run = DrivenRun(SimpleNamespace(backend_kind=kind), {}) + assert run.person_approves_saves is expected + assert DrivenRun(SimpleNamespace(backend_kind="web"), {}, hold=True).hold + + +def _verify(tmp_path: Path) -> Path: + path = tmp_path / "verify.yaml" + path.write_text( + "verifier:\n kind: rest\n base_url: http://127.0.0.1:9\n" + " records_path: /api/db\n records_key: records\n" + ) + return path + + +def test_only_an_agent_that_drives_surfaces_records_one(tmp_path: Path) -> None: + from openadapt_flow.compile_agent import FirstRunCompiler, ScriptedAgent + from openadapt_flow.compile_agent.pipeline import UnsupportedTarget + + compiler = FirstRunCompiler( + "Save this note: {note}", + params=["note"], + surface={"kind": "macos", "macos_app": "TextEdit"}, + verify=_verify(tmp_path), + expect={"patient_id": "p1"}, + on_screen=["Jane Sample"], + cache_dir=tmp_path / "cache", + ) + assert compiler.backend_kind == "macos" and compiler.url is None + with pytest.raises(UnsupportedTarget, match="surface harness"): + compiler.record(ScriptedAgent(steps=[]), {"note": "x"}) + # The spec round-trips with its surface, for runs after a restart. + compiler.save_spec() + again = FirstRunCompiler.load(compiler.dir) + assert again.backend_kind == "macos" + assert again.surface.macos_app == "TextEdit" + + +def test_the_teach_server_admits_only_as_an_agent_under_a_policy( + tmp_path: Path, +) -> None: + from openadapt_flow.agent.teach import TeachConfig + + common = {"verify": _verify(tmp_path), "cache_dir": tmp_path / "teach"} + with pytest.raises(ValueError, match="needs the app"): + TeachConfig(**common) + with pytest.raises(ValueError, match="needs the app"): + TeachConfig(**common, url="http://x", surface={"kind": "macos"}) + key = SimpleNamespace(kind="admitter_key", policy=None, principal_id="ci:x") + with pytest.raises(ValueError, match="registered agent under a signed"): + TeachConfig(**common, url="http://x", admitter=key) + no_policy = SimpleNamespace(kind="agent", policy=None, principal_id="agent:x") + with pytest.raises(ValueError, match="registered agent under a signed"): + TeachConfig(**common, url="http://x", admitter=no_policy) diff --git a/packages/flow/tests/test_mcp_teach_cli.py b/packages/flow/tests/test_mcp_teach_cli.py new file mode 100644 index 000000000..a2f46f9ee --- /dev/null +++ b/packages/flow/tests/test_mcp_teach_cli.py @@ -0,0 +1,169 @@ +"""``openadapt-agent serve --teach``: what the operator fixes at start. + +No app and no browser: ``serve`` is replaced so each test sees the bridges the +command built, or its refusal. +""" + +from __future__ import annotations + +from importlib.util import find_spec +from pathlib import Path +from typing import Any + +import pytest + +from openadapt_flow.agent.cli import main + +# Never import the MCP SDK at collection: other tests in the same process +# check that emitting a server does not import it. +pytestmark = pytest.mark.skipif( + find_spec("mcp") is None, reason="needs the agent extra" +) + + +@pytest.fixture() +def served(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: + calls: list[dict[str, Any]] = [] + + def fake_serve(bridge: Any = None, **kwargs: Any) -> None: + calls.append({"bridge": bridge, **kwargs}) + + monkeypatch.setattr("openadapt_flow.agent.mcp.serve", fake_serve) + return calls + + +@pytest.fixture() +def verify(tmp_path: Path) -> Path: + path = tmp_path / "verify.yaml" + path.write_text( + "verifier:\n kind: rest\n base_url: http://127.0.0.1:9\n" + " records_path: /api/db\n records_key: records\n" + ) + return path + + +def _deploy(tmp_path: Path, backend: str) -> Path: + path = tmp_path / "deploy.yaml" + path.write_text(f"name: notes\nbackend:\n{backend}") + return path + + +def test_a_web_app(served: list, verify: Path, tmp_path: Path) -> None: + code = main( + [ + "serve", + "--teach", + str(verify), + "--url", + "http://127.0.0.1:9/#tasks", + "--runs-dir", + str(tmp_path / "runs"), + ] + ) + assert code == 0 + teach = served[0]["teach"] + assert teach.backend_kind == "web" and teach.config.admitter is None + assert teach.config.cache_dir == (tmp_path / "runs" / "teach").resolve() + bridge = served[0]["bridge"] + assert bridge.allow_run and bridge.catalog == {} + names = {spec.name for spec in teach.list_tool_specs()} + assert {"teach_start", "teach_act", "teach_finish", "teach_admit"} <= names + assert "teach_screen" not in names + + +def test_a_native_app_from_the_deployment( + served: list, verify: Path, tmp_path: Path +) -> None: + deploy = _deploy(tmp_path, " kind: macos\n macos_app: TextEdit\n") + code = main( + [ + "serve", + "--teach", + str(verify), + "--config", + str(deploy), + "--teach-screens", + "scrubbed", + "--runs-dir", + str(tmp_path / "runs"), + ] + ) + assert code == 0 + teach = served[0]["teach"] + assert teach.backend_kind == "macos" + assert "teach_screen" in {spec.name for spec in teach.list_tool_specs()} + + +@pytest.mark.parametrize( + ("extra", "backend", "reason"), + [ + ([], None, "needs the app"), + (["--url", "http://x", "--teach-key", "a.pem"], None, "go together"), + ([], " kind: rdp\n rdp_host: 10.0.0.5\n", "network RDP"), + (["--url", "http://x", "--authoring"], None, "own recording tools"), + (["--url", "http://x", "--bundles", "b"], None, "--mode production"), + ], +) +def test_refusals( + served: list, + verify: Path, + tmp_path: Path, + capsys, + extra: list, + backend: Any, + reason: str, +) -> None: + argv = ["serve", "--teach", str(verify), "--runs-dir", str(tmp_path / "r"), *extra] + if backend: + argv += ["--config", str(_deploy(tmp_path, backend))] + assert main(argv) == 2 + assert reason in capsys.readouterr().err + assert served == [] + + +def test_teach_flags_need_teach(served: list, capsys) -> None: + assert main(["serve", "--teach-hold"]) == 2 + assert "need --teach" in capsys.readouterr().err + + +def test_a_verifier_that_cannot_be_read_is_refused_at_start( + served: list, tmp_path: Path, capsys +) -> None: + assert ( + main(["serve", "--teach", str(tmp_path / "missing.yaml"), "--url", "http://x"]) + == 2 + ) + assert "is not a file" in capsys.readouterr().err + + +def test_a_scrubbed_screen_that_cannot_be_scrubbed_is_not_shared( + verify: Path, tmp_path: Path +) -> None: + import threading + + from openadapt_flow.agent.bridge import BridgeError + from openadapt_flow.agent.teach import TeachBridge, TeachConfig + + def broken() -> Any: + raise RuntimeError("no scrubber") + + teach = TeachBridge( + TeachConfig( + verify=verify, + cache_dir=tmp_path / "teach", + url="http://127.0.0.1:9", + screens="scrubbed", + scrubber_factory=broken, + ) + ) + + class Run: + done = threading.Event() + + def call(self, kind: str, payload: Any = None) -> dict: + assert kind == "screenshot" + return {"png": b"\x89PNG raw frame"} + + teach._run = Run() + with pytest.raises(BridgeError, match="no screen was shared"): + teach.dispatch("teach_screen", {}) diff --git a/packages/flow/tests/test_mcp_teach_surfaces.py b/packages/flow/tests/test_mcp_teach_surfaces.py new file mode 100644 index 000000000..944464fc3 --- /dev/null +++ b/packages/flow/tests/test_mcp_teach_surfaces.py @@ -0,0 +1,290 @@ +"""The MCP teach tools on a remote session (RDP, Citrix), through the real backends. + +MockMed runs in a browser page that Flow never reads: tests/surface_kiosk +shows it to ``RemoteDisplayBackend`` and ``CitrixWorkspaceBackend`` as a +client window, so Flow sees pixels only. A scripted MCP client teaches the +task with pixel clicks, as an agent reading the screen would. + +What it checks: + +* the observation is screen-only and the screen tool works when the operator + turned it on; +* every click a person must approve in the client's form (the harness can't + read which record a remote screen shows), and without a form nothing is + clicked; +* run 1 compiles, passes its record check against the system of record, and + a person confirms the check; +* admission fails closed with its reason: the canary is refused because a + pixel-only click carries no verified record identity (docs/BACKEND_PARITY.md). +""" + +from __future__ import annotations + +import json +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from importlib.util import find_spec +from pathlib import Path +from typing import Any, Iterator, Optional + +import pytest + +from openadapt_flow.mockmed.fault_server import serve +from openadapt_flow.tutorial import _http_json, _records +from tests.surface_kiosk import PageWindowClient + +# The MCP SDK is imported inside the tests, never at collection: other tests +# in the same process check that emitting a server does not import it. +pytestmark = pytest.mark.skipif( + find_spec("mcp") is None or find_spec("playwright") is None, + reason="needs the agent extra and Playwright", +) + +TITLE = "MockMed kiosk" +SECTIONS = { + "rdp": { + "kind": "rdp", + "rdp_window": "mstsc", + "rdp_window_title": TITLE, + "rdp_max_frame_age_s": 30.0, + }, + "citrix": { + "kind": "citrix", + "rdp_window": "Citrix Viewer", + "rdp_window_title": TITLE, + "rdp_readiness_text": "MockMed", + "rdp_max_frame_age_s": 30.0, + }, +} +TASK = "Open the first referral, start a Triage encounter, and save this note: {note}" +NOTE = "Synthetic follow-up in two weeks" +AGENT_ID = "agent:mcp-teach-surface" +#: Where MockMed's controls are at 1280x800, as an agent reading the screen +#: would find them (the same points tests/fixtures/jit/anthropic_add_note.json +#: clicks). +CLICKS = [(775, 186), (114, 159), (85, 214), (344, 350)] +SAVE = (134, 457) + + +@pytest.fixture(scope="module") +def mockmed() -> Iterator[str]: + base, _db, stop = serve() + yield base.rstrip("/") + stop() + + +def _open_backend(base: str): + """Build each backend through the factory, with a fresh kiosk page as the + client window. Each call opens its own browser on the calling thread.""" + + from playwright.sync_api import sync_playwright + + from openadapt_flow._browser_setup import launch_chromium + from openadapt_flow.backends.factory import build_backend + + def open_backend(cfg: Any) -> Any: + pw = sync_playwright().start() + try: + browser = launch_chromium(pw.chromium, headless=True) + except Exception as exc: # noqa: BLE001 - no browser on this host + pw.stop() + pytest.skip(f"chromium unavailable: {exc}") + page = browser.new_page( + viewport={"width": 1280, "height": 800}, device_scale_factor=1 + ) + page.goto(f"{base}/?fault=ok&idempotency=demo#tasks") + page.wait_for_timeout(300) + backend = build_backend( + cfg, window_client=PageWindowClient(page, owner=cfg.rdp_window, title=TITLE) + ) + + def close() -> None: + browser.close() + pw.stop() + + backend.close = close # type: ignore[attr-defined] + return backend + + return open_backend + + +def _admitter(key_dir: Path, origin: str, monkeypatch: pytest.MonkeyPatch) -> Any: + from openadapt_flow.jit import authority + + monkeypatch.setenv(authority.KEY_PASSPHRASE_ENV, "surface passphrase") + files = key_dir.parent / "keyfiles" + for role, pid, name in ( + ("policy_signer", "dana.signer", "signer"), + ("agent", AGENT_ID, "agent"), + ): + authority.register_principal( + key_dir, + role=role, + principal_id=pid, + public_key_hex=authority.generate_encrypted_key(files / f"{name}.pem"), + ) + now = datetime.now(timezone.utc) + policy = authority.AdmissionPolicy( + policy_id="kiosk", + signer="dana.signer", + signed_at=(now - timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ"), + expires_at=(now + timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ"), + allowed_hosts=(origin,), + allowed_actions=("click", "type"), + allowed_effects=("record_written", "field_equals"), + min_clean_canaries=1, + agent_principals=(AGENT_ID,), + allow_self_admission=True, + ) + signed = authority.sign_policy( + policy, authority.load_encrypted_key(files / "signer.pem") + ) + return authority.principal_from_key( + authority.load_encrypted_key(files / "agent.pem"), key_dir, policy=signed + ) + + +@asynccontextmanager +async def connect(server: Any, *, person: Optional[str]): + import anyio + import mcp.types as types + from mcp import ClientSession + from mcp.shared.memory import create_client_server_memory_streams + + async def elicit(context: Any, params: Any) -> types.ElicitResult: + if person is None: + return types.ElicitResult(action="decline", content={}) + return types.ElicitResult( + action="accept", content={"name": person, "confirmed": True} + ) + + async with create_client_server_memory_streams() as (client, transport): + async with anyio.create_task_group() as tasks: + tasks.start_soon( + server.run, *transport, server.create_initialization_options() + ) + kwargs = {"elicitation_callback": elicit} if person else {} + async with ClientSession(*client, **kwargs) as session: + await session.initialize() + yield session + tasks.cancel_scope.cancel() + + +async def _call(session: Any, name: str, args: Optional[dict] = None) -> dict: + result = await session.call_tool(name, args or {}) + payload = json.loads(result.content[0].text) + if getattr(result, "is_error", None) or getattr(result, "isError", None): + return {"_error": payload.get("error")} + return payload + + +@pytest.mark.timeout(600) +@pytest.mark.parametrize("kind", ["rdp", "citrix"]) +def test_teach_on_a_remote_window( + mockmed: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, kind: str +) -> None: + import anyio + + from openadapt_flow.agent.bridge import AgentBridge + from openadapt_flow.agent.mcp import build_server + from openadapt_flow.agent.runner import RunnerConfig + from openadapt_flow.agent.teach import TeachBridge, TeachConfig + from openadapt_flow.jit.compile_agent import SurfaceApp + + section = SECTIONS[kind] + verify = tmp_path / "verify.yaml" + verify.write_text( + "verifier:\n kind: rest\n" + f" base_url: {mockmed}\n records_path: /api/db\n records_key: records\n" + " timeout_s: 2.0\n poll_interval_s: 0.05\n" + ) + runs = tmp_path / "runs" + runs.mkdir() + workflows = AgentBridge( + None, RunnerConfig(runs_dir=runs), allow_run=True, mode="production" + ) + teach = TeachBridge( + TeachConfig( + verify=verify, + cache_dir=tmp_path / "teach", + surface=section, + key_dir=tmp_path / "keys", + admitter=_admitter( + tmp_path / "keys", SurfaceApp(section).origin, monkeypatch + ), + screens="unscrubbed", + reset=lambda: _http_json(f"{mockmed}/api/reset", method="POST", body={}), + open_backend=_open_backend(mockmed), + ), + workflows=workflows, + ) + server = build_server(workflows, teach=teach) + start = { + "task": TASK, + "inputs": {"note": NOTE}, + "record_fields": {"patient_id": "p1", "type": "Triage"}, + "on_screen": ["Jane Sample", "MRN P1"], + } + + async def scenario() -> None: + # A client with no form: the first click needs a person, so nothing + # is clicked and run 1 keeps going. + async with connect(server, person=None) as session: + started = await _call(session, "teach_start", start) + assert started["surface"] == kind and started["person_approves_saves"] + assert "remote session" in started["observation"] + screen = await _call(session, "teach_screen") + assert len(screen["screen_png_base64"]) > 1000 + x, y = CLICKS[0] + refused = await _call( + session, "teach_act", {"action": "click", "x": x, "y": y} + ) + assert "needs a person's approval" in refused["_error"] + assert _records(mockmed) == [] + assert (await _call(session, "teach_abort"))["status"] == "aborted" + + async with connect(server, person="dana.reviewer") as session: + started = await _call(session, "teach_start", start) + assert started["status"] == "recording", started + for x, y in CLICKS: + done = await _call( + session, "teach_act", {"action": "click", "x": x, "y": y} + ) + assert done["status"] == "ok", done + typed = await _call( + session, "teach_act", {"action": "type", "input": "note"} + ) + assert typed["status"] == "ok", typed + x, y = SAVE + saved = await _call( + session, "teach_act", {"action": "click", "x": x, "y": y} + ) + assert saved["status"] == "ok", saved + draft = await _call(session, "teach_finish") + assert draft["status"] == "draft", draft + assert draft["surface"] == kind + assert draft["record_check"]["passed"] + assert {g["decision"] for g in draft["save_gate"]} == {"approved_by_hold"} + assert draft["parameters"]["lifted"] == ["note"] + + confirmed = await _call( + session, + "teach_confirm_checks", + {"task_key": draft["task_key"], "drop_fields": ["key", "source"]}, + ) + assert confirmed["confirmed_by"] == "dana.reviewer", confirmed + + saved_rows = _records(mockmed) + admitted = await _call( + session, + "teach_admit", + {"task_key": draft["task_key"], "canary_inputs": [{"note": "Canary"}]}, + ) + assert admitted["status"] == "canary_failed", admitted + assert "Identity coverage" in admitted["detail"] + assert _records(mockmed) == saved_rows # refused before any input + assert not any( + t.name.startswith("taught_") for t in (await session.list_tools()).tools + ) + + anyio.run(scenario)