diff --git a/README.md b/README.md index d36696f..deac0f5 100644 --- a/README.md +++ b/README.md @@ -229,8 +229,9 @@ The `openadapt-eval-flow` console entry point wraps the same script. Bounded, reproducible performance reports are committed under `eval_results/` and generated by `scripts/report_openadapt_performance.py`, -`scripts/run_current_flow_local_benchmark.py`, and -`scripts/run_flow_transaction_probe.py`, each with an accompanying markdown +`scripts/run_current_flow_local_benchmark.py`, +`scripts/run_flow_transaction_probe.py`, and +`scripts/probe_remote_lease_safety.py`, each with an accompanying markdown summary. These reports are the source for public performance claims and are intended to be regenerable. Deployment-derived thresholds, tuned adversary parameters, per-system-of-record oracle recipes, and real customer datasets are @@ -247,8 +248,8 @@ Re-run the comparison and publish a new evidence set rather than editing an old one: superseded reports stay reproducible against the wheel they were measured on. -See `docs/eval_results/current_flow_v1_27_1_local_20260802/` for the current -comparison, its `COMPARISON_TO_v1_24_0.md` release-over-release delta, and +See `docs/eval_results/current_flow_v1_28_0_local_20260802/` for the current +comparison, its `COMPARISON_TO_v1_27_1.md` release-over-release delta, and `REPRODUCE.md` for the exact commands. ## More workflows diff --git a/docs/eval_results/PUBLISHED_EVIDENCE.json b/docs/eval_results/PUBLISHED_EVIDENCE.json index d64a7ca..46d8bd4 100644 --- a/docs/eval_results/PUBLISHED_EVIDENCE.json +++ b/docs/eval_results/PUBLISHED_EVIDENCE.json @@ -29,11 +29,20 @@ }, { "path": "docs/eval_results/current_flow_v1_27_1_local_20260802", - "status": "current", + "status": "superseded", + "superseded_by": "docs/eval_results/current_flow_v1_28_0_local_20260802", "flow_version": "1.27.1", "wheel_sha256": "99d8f3ef014481356f4bcfc65f694ed2fd47a75e2025c5ddfdae4bfab2194094", "sdist_sha256": "aeabf2b11ae6151fe76c02ee783e8368d10ee984465f858f9d37429b17a15468", "measured_on": "2026-08-02" + }, + { + "path": "docs/eval_results/current_flow_v1_28_0_local_20260802", + "status": "current", + "flow_version": "1.28.0", + "wheel_sha256": "4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202", + "sdist_sha256": "6e108f3469da20226427fee7a11378e067ca08c1fbbfcdbb4018ab9c3d142a8b", + "measured_on": "2026-08-02" } ] } diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/COMPARISON_TO_v1_27_1.md b/docs/eval_results/current_flow_v1_28_0_local_20260802/COMPARISON_TO_v1_27_1.md new file mode 100644 index 0000000..14f539a --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/COMPARISON_TO_v1_27_1.md @@ -0,0 +1,184 @@ +# openadapt-flow 1.27.1 -> 1.28.0, release over release + +Both measurements ran the same two runners on the same host against the same +bundled MockMed application. This is the first release-over-release comparison +in this directory in which the fixture *and* both harness files were +byte-identical, so nothing below can be an artefact of measuring differently. + +## What was held constant, exactly + +| | 1.27.1 (2026-08-02) | 1.28.0 (2026-08-02) | Held? | +|---|---|---|---| +| Comparison runner SHA-256 | `68f9e5a2…8a126` | `68f9e5a2…8a126` | **yes, byte-identical** | +| Probe runner SHA-256 | `097cc900…40ce8` | `097cc900…40ce8` | **yes, byte-identical** | +| MockMed fixture tree | `f0736c7a82ca2aba2beb334feaf461c5f06532a5` | `f0736c7a82ca2aba2beb334feaf461c5f06532a5` | **yes, byte-identical** | +| Host / Python / Playwright | macOS 15.7.3 arm64, 3.12.13, 1.61.0 | macOS 15.7.3 arm64, 3.12.13, 1.61.0 | yes | +| Trials, arms, conditions | 3 x 3 x 3, no retries | 3 x 3 x 3, no retries | yes | +| Evals commit | `8132e607` | `71edc889` | no — docs only, no runner touched | + +The evals commit moved because the 1.27.1 evidence itself landed in between. +That commit changed `README.md` and files under `docs/eval_results/`; it touched +no runner and no script. The two harness digests above are the load-bearing +check, and both are unchanged. + +## What is in 1.28.0 + +Three commits, plus the release commit: + +- `c9618cc` `fix: deliver consequential remote clicks through the frame lease` +- `e054caa` `feat: carry managed Execute authority through BYOC` +- `ccdd155` `fix: simplify the RDP buyer presentation` + +Only `c9618cc` changes replay behaviour, and only on the *remote* actuation +path. The comparison and the transaction probe both drive a local browser +backend, so neither exercises it. It is measured directly instead, in +`remote_lease_safety/`. + +## Delta 1: no outcome classification moved. None. + +Every counted cell reports the same classification as on 1.27.1. + +| Condition | Arm | 1.27.1 primary / replication | 1.28.0 primary / replication | +|---|---|---|---| +| `clean` | compiled replay | 3/3, 0 over-halt / 3/3, 0 over-halt | 3/3, 0 over-halt / 3/3, 0 over-halt | +| `clean` | DOM positional | 3/3 / 3/3 | 3/3 / 3/3 | +| `clean` | DOM name-scoped | 3/3 / 3/3 | 3/3 / 3/3 | +| `theme` | compiled replay | 3/3, 0 over-halt / 3/3, 0 over-halt | 3/3, 0 over-halt / 3/3, 0 over-halt | +| `theme` | DOM positional | 3/3 / 3/3 | 3/3 / 3/3 | +| `theme` | DOM name-scoped | 3/3 / 3/3 | 3/3 / 3/3 | +| `rename` | compiled replay | 3/3 / 3/3 | 3/3 / 3/3 | +| `rename` | DOM positional | 0/3 / 0/3 | 0/3 / 0/3 | +| `rename` | DOM name-scoped | 0/3 / 0/3 | 0/3 / 0/3 | + +Denominators: 3 trials per arm per condition, which is the minimum this +repository's evaluation standard admits for a comparative claim; the runner +fixes it at 3 and the independent replication doubles every cell to 6. One +measurement run is 27 counted trials (3 conditions x 3 arms x 3 trials), so the +primary plus the replication is 54, of which 18 are compiled replay. + +Across those 18 compiled trials: 18/18 task success, 0 over-halts, 0 silent +incorrect successes, 0 wrong-target writes, 0 model calls, $0.00. Identity +coverage is unchanged at 5 of 8 applicable clicks armed. The +`clean_postcondition_over_halt.json` and `theme_postcondition_over_halt.json` +artifacts record 0 observations in the primary run and 0 in the replication, as +they did on 1.27.1. + +## Delta 2: the transaction probe is unchanged, cell for cell + +All 10 cells (5 fault modes x 2 verification configurations) report exactly the +same transaction outcome and the same ground-truth business effect as on +1.27.1, over 3 trials per cell = 30 counted runs. + +| Invariant | Applicable runs | Violations | 1.27.1 | 1.28.0 | +|---|---:|---:|---|---| +| `COMPLETED_UNVERIFIED` is never a production success and never billable | 12 | 0 | holds | holds | +| Only `VERIFIED` may be billable | 30 | 0 | holds | holds | +| An outcome asserting no business effect must not be reported when the record shows the write landed | 0 | 0 | **vacuous** | **vacuous** | +| A consequential step that reached actuation but was never verified must be `RECONCILIATION_REQUIRED` | 21 | 0 | holds | holds | +| A single run must never write the intended record more than once | 24 | 0 | holds | holds | +| `VERIFIED` must never be reported when nothing landed | 0 | 0 | **vacuous** | **vacuous** | +| A healthy compiled run makes no model calls and costs $0 | 30 | 0 | holds | holds | + +Zero violations across 30 counted runs, unchanged. Two of the seven invariants +are `vacuous`: no counted run claimed an absence and no counted run reported +`VERIFIED`, so no run could exercise them. A vacuous invariant proves nothing +and is not a pass. It is reported that way deliberately, in both releases. + +The `timeout` fault mode — commit the row, then hang past the client timeout — +reports `RECONCILIATION_REQUIRED` in all 6 runs, exactly as on 1.27.1. This is +where the 1.24.0 evidence recorded `HALTED_BEFORE_EFFECT` in 6 of 6 runs, and +where the 1.24.0 probe failed `no_false_absence_claim` in 7 of 16 applicable +runs and `unverified_delivered_write_needs_reconciliation` in 14 of 23. Those +were fixed in 1.25.0 and they stay fixed here. + +## Delta 3: one number moved, and it moved the wrong way + +Compiled replay is slower on 1.28.0 in every condition, in both the primary run +and the replication. + +| Condition | 1.27.1 steady median | 1.28.0 steady median | Delta | +|---|---:|---:|---:| +| `clean` primary | 6.828s | 7.033s | +0.205s (+3.0%) | +| `clean` replication | 6.860s | 7.030s | +0.171s (+2.5%) | +| `theme` primary | 7.003s | 7.255s | +0.252s (+3.6%) | +| `theme` replication | 7.025s | 7.190s | +0.165s (+2.3%) | +| `rename` primary | 6.837s | 7.031s | +0.194s (+2.8%) | +| `rename` replication | 6.934s | 7.033s | +0.099s (+1.4%) | + +Six of six cells moved the same way. In the `clean` condition the two samples do +not overlap at all: the six 1.27.1 trials span 6.801-6.947s and the six 1.28.0 +trials span 6.966-7.055s. That is a small, consistent slowdown and it is +reported rather than rounded away. + +**What this design can conclude, and what it cannot.** The harness and the +fixture were byte-identical, so a measurement artefact is excluded. It cannot +separate an engine effect from host drift: the two evidence sets were measured +in separate sessions a few hours apart on one machine, not interleaved, so a +change in machine state between the sessions produces exactly this signature. +Record and compile timings, measured in the same sessions, did not move +(compile 8.151 / 6.407 / 6.374s here against 8.141 / 6.413 / 6.464s on 1.27.1), +which argues against a broad host slowdown but does not settle it. Nothing in +`c9618cc` runs on this path. + +Establishing the cause needs an interleaved A/B of the two wheels in one +session. That is not run here, so the delta is recorded as unattributed rather +than assigned to the engine. It changes no outcome classification. + +## Delta 4: the remote frame-lease change, measured directly + +`c9618cc` lets a consequential remote click be delivered through the backend's +one-shot actuation lease when no typed delivery receipt exists, so that an +opaque pixel-only surface stops over-halting on its write step. Relaxing a +refusal is the kind of change that can buy a green demonstration with a silent +wrong write, so the release note's two safety properties were measured, not +assumed. `remote_lease_safety/` holds the artifacts; 3 cells x 3 trials = 9 +counted runs, each classified under all 3 execution profiles = 27 classifications. + +| Cell | Runs | Input edges delivered | Typed receipts | `transaction_outcome` under demo / standard / regulated | +|---|---:|---:|---:|---| +| `ungoverned_lease` | 3 | 3 | 0 | `COMPLETED_UNVERIFIED` in all 3 profiles | +| `governed_lease` | 3 | **0** | 0 | `HALTED_BEFORE_EFFECT` in all 3 profiles | +| `lease_frame_changed` | 3 | **0** | 0 | `RECONCILIATION_REQUIRED` in all 3 profiles | + +- **A governed run still refuses.** 3 of 3 governed runs stopped before the + first input edge with `safety_halt` set and the error `Step 's1' (click + 'Save') is a consequential remote click, but this backend cannot bind its + exact fresh frame and target to delivery; run aborted`. The backend received + zero input edges. Only the presence of a `GovernedRunAuthorization` + distinguishes this cell from `ungoverned_lease`, which delivered 3 of 3. +- **The lease delivery is never `VERIFIED`.** Over 9 counted runs x 3 profiles + = 27 classifications, `VERIFIED` was returned 0 times, `production_eligible` + was true 0 times, and `transaction_billable` was true 0 times. Every + lease-delivered run carried no delivery receipt and no actuation tier, which + is what makes `COMPLETED_UNVERIFIED` the classifier's only available answer. +- The lease is a real safety property, not a formality: when the remote frame + changed between `acquire_actuation_frame` and the input edge, the backend + received zero input edges in 3 of 3 runs, and the run settled at + `RECONCILIATION_REQUIRED` rather than claiming a proven absence — the runtime + could not prove the click had not landed, so it did not say so. + +Scope: the backend in that probe is a fake exposing only the two-phase lease. +It measures a runtime contract. It is not a real Citrix or RDP session and no +wrong-target-immunity claim on a real remote surface follows from it. + +## What did not change + +- The DOM selector controls still fail 0/3 under `rename` in both releases, and + still fail loudly at the first renamed locator before any mutation. They are + halts, not silent wrong actions. +- Identity coverage remains 5 of 8 applicable clicks armed. The three unarmed + steps carry the same recorded reasons. +- Zero silent incorrect successes, zero wrong actions, zero model calls, $0.00, + in every counted trial of both releases. +- No zero-shot comparison is claimed in either release. It remains not run: it + needs a paid model and a paid WAA environment, and the live replay path still + lacks a wired WAA evaluator. + +## Scope + +One synthetic workflow, one macOS host, headless Chromium, three trials per +cell, doubled by one independent replication of the comparison. No hosted +lifecycle, Windows UIA, RDP, Citrix, or real customer application is +represented. This release-over-release comparison establishes that 1.28.0 +changed no measured outcome on this workflow; it is not a general reliability +claim. diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/README.md b/docs/eval_results/current_flow_v1_28_0_local_20260802/README.md new file mode 100644 index 0000000..9e74e27 --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/README.md @@ -0,0 +1,73 @@ +# Flow 1.28.0 local evidence (2026-08-02) + +Measured against the exact published `openadapt-flow` 1.28.0 wheel, SHA-256 +`4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202`, bound to +release-tagged tracked-clean source `b646276a086c74b65ba850cdef2e475ca53f10c0`. +Zero model calls, $0.00 model cost, loopback only. + +This is the `current` entry in `docs/eval_results/PUBLISHED_EVIDENCE.json`. It +supersedes `current_flow_v1_27_1_local_20260802`, which remains committed and +reproducible against the 1.27.1 wheel it was measured on. + +| File | What it holds | +|---|---| +| `REPORT.md`, `results.json` | Compiled replay versus steelmanned DOM selector controls: 3 conditions x 3 arms x 3 trials, no retries | +| `COMPARISON_TO_v1_27_1.md` | Release-over-release delta, including the one number that moved and the safety properties of the remote frame-lease change | +| `REPRODUCE.md` | Exact commands, digests, and environment capture | +| `clean_postcondition_over_halt.json` | Zero observations — the 1.24.0 `clean` over-halt stays fixed | +| `theme_postcondition_over_halt.json` | Zero observations — the 1.16.1 `theme` over-halt stays fixed | +| `replication/` | An independent re-run of the whole comparison | +| `transaction_probe/` | Transaction outcome taxonomy against MockMed's persistence boundary: 5 fault modes x 2 verification configurations x 3 trials | +| `remote_lease_safety/` | The two safety properties of Flow 1.28.0's remote frame-lease delivery: 3 cells x 3 trials, each classified under all 3 execution profiles | + +There is no `transaction_probe/invariant_violations.json` in this set. The +runner writes that file only when an invariant is violated, and none was. + +## The comparison is controlled + +Both harness files are byte-identical to the 1.27.1 measurement — comparison +runner `68f9e5a2…8a126`, probe runner `097cc900…40ce8` — and the bundled MockMed +tree is byte-identical at `f0736c7a`. This is the first release-over-release +comparison in this directory where neither the fixture nor either harness moved, +so no delta can be a measurement artefact. + +## Headline results + +**Held, and nothing moved.** Compiled replay completed the task 3/3 in every +condition, in the primary run and in its independent replication — 18 counted +compiled trials with zero over-halts, zero silent incorrect successes, zero +wrong-target writes, zero model calls, and $0.00. The DOM selector controls +still fail 0/3 under label drift, loudly, at the first renamed locator and +before any mutation. Identity coverage is unchanged at 5 of 8 applicable clicks +armed. Every one of the 10 transaction-probe cells reports the same outcome as +on 1.27.1, with 0 violations across 30 counted runs. + +**Worse: compiled replay is slower.** The compiled steady-state median rose in +all 6 condition-by-run cells, by +0.099s to +0.252s (+1.4% to +3.6%). In the +`clean` condition the two 6-trial samples do not overlap. The harness and the +fixture were identical, so this is not a measurement artefact; but the two +evidence sets were measured in separate sessions rather than interleaved, so +this design cannot separate an engine effect from host drift, and the delta is +recorded as unattributed. It changes no outcome classification. See +`COMPARISON_TO_v1_27_1.md`. + +**Measured, not trusted: the remote frame-lease change.** `c9618cc` lets a +consequential remote click be delivered through a backend's one-shot actuation +lease when no typed delivery receipt exists. Both stated safety properties hold +under measurement. A governed run still refused in 3 of 3 runs, before the first +input edge, with zero input edges reaching the backend. And across 9 counted +runs classified under all 3 execution profiles — 27 classifications — +`VERIFIED` was returned 0 times, `production_eligible` was true 0 times, and +`transaction_billable` was true 0 times; every lease delivery classified as +`COMPLETED_UNVERIFIED`. A frame that changed between the lease and the input +edge aborted delivery in 3 of 3 runs. + +**Not claimed.** No Flow-versus-zero-shot comparison was run; it needs a paid +model and a paid WAA environment, and the live replay path still lacks a wired +WAA evaluator. Two transaction invariants — "an outcome asserting no business +effect must not be reported when the record shows the write landed" and +"`VERIFIED` must never be reported when nothing landed" — are reported as +`vacuous` rather than as passes, because no counted run could exercise them. A +vacuous invariant proves nothing. The lease-safety probe uses a fake backend +exposing only the two-phase lease; it establishes a runtime contract and says +nothing about a real Citrix or RDP session. diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/REPORT.md b/docs/eval_results/current_flow_v1_28_0_local_20260802/REPORT.md new file mode 100644 index 0000000..b6492d4 --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/REPORT.md @@ -0,0 +1,70 @@ +# Exact-current local Flow performance + +This is a deterministic-runtime overhead and bounded robustness comparison. It is **not** a zero-shot computer-use comparison. + +## Source and environment + +- Flow commit: `b646276a086c74b65ba850cdef2e475ca53f10c0` (version `1.28.0`; tracked-clean source) +- Release tag: `v1.28.0` +- Wheel SHA-256: `4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202` +- Evals base commit: `71edc889035d998cac518ddf69b42860730533d7` +- Runner SHA-256: `68f9e5a27f4f04d831574167ebd6b362bf05184e81708a315d9896969b48a126` +- Platform: `macOS-15.7.3-arm64-arm-64bit` +- Python: `3.12.13` +- Playwright: `1.61.0` +- Chromium: `Playwright-managed headless Chromium` +- Network/provider use: loopback bundled MockMed only; no cloud VM, hosted runner, or model API + +## Counted result + +One synthetic MockMed workflow: sign in, open the intended referral, create a Triage encounter, enter a trial-unique note, and save. Each arm used a fresh browser. The arm-independent screenshot/OCR oracle required the exact saved note, Triage row, and intended patient, and separately flagged wrong-target writes. + +| Condition | Arm | Runs | Task success | Silent incorrect | Wrong action | Over-halt | Halt/error | Steady median | Steady p95 | End-to-end median | End-to-end p95 | Model calls | Cost | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `clean` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 7.033s | 7.046s | 7.613s | 7.622s | 0 | $0.00 | +| `clean` | DOM positional | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.206s | 0.207s | 0.827s | 0.860s | 0 | $0.00 | +| `clean` | DOM name-scoped | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.201s | 0.202s | 0.844s | 0.867s | 0 | $0.00 | +| `theme` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 7.255s | 7.270s | 7.838s | 7.856s | 0 | $0.00 | +| `theme` | DOM positional | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.200s | 0.206s | 0.820s | 0.885s | 0 | $0.00 | +| `theme` | DOM name-scoped | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.199s | 0.203s | 0.852s | 0.918s | 0 | $0.00 | +| `rename` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 7.031s | 7.067s | 7.588s | 7.651s | 0 | $0.00 | +| `rename` | DOM positional | 3 | 0/3 | 0 | 0 | 0 | 3 | 30.061s | 30.063s | 30.685s | 30.704s | 0 | $0.00 | +| `rename` | DOM name-scoped | 3 | 0/3 | 0 | 0 | 0 | 3 | 30.068s | 30.070s | 30.712s | 30.718s | 0 | $0.00 | + +Steady time wraps only the replay/script action loop. End-to-end time additionally includes fresh browser launch, the independent final screenshot/OCR oracle, and browser teardown. Local server startup and one-time record/compile are excluded from both and reported separately below. p95 is nearest-rank; with n=3 it is the slowest counted trial. + +Outcome definitions: **silent incorrect success** means the arm reported completion but the independent oracle did not confirm the intended effect; **wrong action** means the oracle observed a write to the wrong patient or encounter type; **over-halt** means the arm reported halt/incomplete while the independent oracle confirmed the intended effect; **halt/error** means the arm stopped and the intended effect was absent. The same definitions apply to every arm. The selector controls therefore can count as over-halts when alternate final-state evidence contradicts them; their rename failures did not, because the oracle confirmed no write. + +## One-time setup + +- Bundled app server startup: 0.004s (n=1 diagnostic) +- Setup trial 1: record 3.438s; compile 8.151s; bundle `d2c035f3e895e5e0130bd65aa0019fdbce7ddeeaa34ccb641f6a72767d3acb3f` +- Setup trial 2: record 2.500s; compile 6.407s; bundle `7c1e5beed1e271209f5f63ff3a54bc134cf683cc5845432cd116c4dad729567b` +- Setup trial 3: record 2.538s; compile 6.374s; bundle `685e7ba7168ada2571af27eb8d36829b756abf6baa6d52c5718451dcef4e4694` + +## What this establishes + +- The independent final-state oracle confirmed the intended effect for every compiled clean, theme-drift, and label-drift trial. Compiled theme runs reported 0/3 over-halts; these are counted rather than relabelled as clean completions. +- The selector controls are steelmanned Playwright scripts. Their clean/theme speed is the correct reminder that API/structural actuation should remain the preferred tier where available. +- The `rename` surface changes `Open` to `View` and `Save Encounter` to `Submit Encounter`. Both selector controls failed loudly at the first renamed locator before mutation. These are unsupported-drift halts, not silent wrong actions. +- Label drift is an intentionally bounded robustness probe, not evidence for arbitrary drift or arbitrary applications. +- The compiled bundle had 5/8 identity-applicable clicks armed. Task completion is not a universal wrong-target-immunity claim for unarmed steps. +- The exact theme postcondition failures are retained in `theme_postcondition_over_halt.json` as a compact regression artifact. + +## Exact-current Flow versus zero-shot: not run + +No current Flow-versus-zero-shot result is claimed. The Azure WAA VM was not started and no model was called. The existing `scripts/eval_flow_on_waa.py` live replay path currently leaves the WAA evaluator unwired (so it cannot independently score success), while its hybrid live path explicitly returns before execution because the adapter is not wired. A valid future run therefore requires, in order: + +1. Wire `WAALiveAdapter.evaluate` into the Flow replay path and wire the same model/adapter into the zero-shot arm. +2. Prepare one exact compiled bundle per retained task and bind Flow/evals/model/environment revisions in the run manifest. +3. Obtain explicit approval for Azure VM start and a hard model spend cap; then confirm VM snapshot/readiness without changing the task set. +4. Run at least three trials per task per condition for both arms, using WAA's evaluator as the oracle and recording correct, silent incorrect, over-halt, timeout/error, latency, tokens, and cost. +5. Deallocate the VM, verify no orphaned resources, and publish the immutable raw summaries plus a normalized report. + +## Caveats + +- Synthetic MockMed, one workflow, one macOS host, headless Chromium. +- This report runs the exact published wheel named above, extracted locally, and binds it to the release-tagged tracked-clean source. +- Browser startup is fresh per run but OS/browser caches are warm after the first launch; the Latin-rotated arm order reduces but does not eliminate host-order effects. +- Final screenshots were inspected and hashed during the run; only their SHA-256 hashes are retained in this committed report. +- No hosted lifecycle, Windows UIA, RDP, Citrix, or real customer application is represented. diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/REPRODUCE.md b/docs/eval_results/current_flow_v1_28_0_local_20260802/REPRODUCE.md new file mode 100644 index 0000000..5de857a --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/REPRODUCE.md @@ -0,0 +1,150 @@ +# Reproduce this evidence from a clean checkout + +Everything below runs on one machine with no cloud VM, no hosted runner, no +model API, and no paid provider. The only network access is the initial download +of the pinned wheel from PyPI. + +## 1. Pin the exact engine artifact + +```bash +python - <<'PY' +import hashlib, json, pathlib, urllib.request +VERSION = "1.28.0" +EXPECTED = "4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202" +meta = json.load(urllib.request.urlopen( + f"https://pypi.org/pypi/openadapt-flow/{VERSION}/json")) +wheel = next(f for f in meta["urls"] if f["packagetype"] == "bdist_wheel") +path = pathlib.Path(wheel["filename"]) +path.write_bytes(urllib.request.urlopen(wheel["url"]).read()) +digest = hashlib.sha256(path.read_bytes()).hexdigest() +assert digest == EXPECTED, f"wheel digest mismatch: {digest}" +print(path, digest) +PY +``` + +Expected artifacts for `openadapt-flow` 1.28.0: + +| Artifact | SHA-256 | +|---|---| +| `openadapt_flow-1.28.0-py3-none-any.whl` | `4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202` | +| `openadapt_flow-1.28.0.tar.gz` | `6e108f3469da20226427fee7a11378e067ca08c1fbbfcdbb4018ab9c3d142a8b` | + +## 2. Check out the release-tagged, tracked-clean engine source + +All three runners refuse to proceed unless the source checkout is tracked-clean +and its `HEAD` carries the tag matching the wheel's `__version__`, so a report +can never bind a wheel to unrelated source. + +```bash +git clone https://github.com/OpenAdaptAI/openadapt-flow.git flow-1.28.0 +git -C flow-1.28.0 checkout v1.28.0 # commit b646276a086c74b65ba850cdef2e475ca53f10c0 +``` + +## 3. Create the measurement environment + +```bash +uv venv --python 3.12 .venv-evidence +uv pip install --python .venv-evidence/bin/python \ + "openadapt-flow==1.28.0" "playwright==1.61.0" requests +.venv-evidence/bin/python -m playwright install chromium +``` + +Flow is imported from the locally extracted wheel, not from this install; the +install only provides the dependency set. Every runner asserts that the imported +`openadapt_flow` resolves inside the extracted wheel directory and aborts +otherwise. + +Playwright is not a core `openadapt-flow` dependency from 1.25.0 onward: browser +support lives in the `browser` extra (`openadapt-flow[browser]`). The explicit +`playwright==1.61.0` pin above supplies it, and pins the exact driver these +numbers were measured with. + +## 4. Run the comparison, the transaction probe, and the lease-safety probe + +```bash +.venv-evidence/bin/python scripts/run_current_flow_local_benchmark.py \ + --flow-source flow-1.28.0 \ + --flow-wheel openadapt_flow-1.28.0-py3-none-any.whl \ + --out out/comparison + +.venv-evidence/bin/python scripts/run_flow_transaction_probe.py \ + --flow-source flow-1.28.0 \ + --flow-wheel openadapt_flow-1.28.0-py3-none-any.whl \ + --out out/transaction_probe --no-fail-on-violation + +.venv-evidence/bin/python scripts/probe_remote_lease_safety.py \ + --flow-source flow-1.28.0 \ + --flow-wheel openadapt_flow-1.28.0-py3-none-any.whl \ + --out out/remote_lease_safety + +.venv-evidence/bin/python scripts/extract_over_halt_regression.py \ + --results out/comparison/results.json --condition clean \ + --out out/comparison/clean_postcondition_over_halt.json +``` + +Drop `--no-fail-on-violation` to make a violated transaction invariant exit +non-zero. It is passed here for symmetry with the 1.24.0 evidence, which needed +the artifacts written while an invariant failed. On this release no invariant +failed, so the probe exits 0 either way. + +`replication/` is the identical comparison command run a second time into a +separate output directory, with no other change. + +## 5. Environment this evidence was measured on + +| | | +|---|---| +| Platform | `macOS-15.7.3-arm64-arm-64bit` (Apple silicon) | +| Python | 3.12.13 | +| Playwright | 1.61.0 | +| Chromium | Playwright-managed, headless | +| Flow wheel | `openadapt_flow-1.28.0-py3-none-any.whl`, SHA-256 `4d156035…c7202` | +| Flow source | `b646276a086c74b65ba850cdef2e475ca53f10c0`, tag `v1.28.0` | +| Evals commit | `71edc889035d998cac518ddf69b42860730533d7` | +| Comparison runner | SHA-256 `68f9e5a27f4f04d831574167ebd6b362bf05184e81708a315d9896969b48a126` | +| Probe runner | SHA-256 `097cc900b44872712ebd7a5bfbdb53f87c85bf5bd8dcbef88fa893743d240ce8` | +| Lease-safety runner | SHA-256 `2d0cf696df9779446469f0e94eb2f22e3e7634ac924998ebd93735b8d5a6791d` | +| Model calls / cost | 0 / $0.00 | + +## 6. Why this comparison is controlled + +**Both measurement harnesses are byte-identical to the 1.27.1 run.** The +comparison runner is `68f9e5a2…8a126` in both, and the probe runner is +`097cc900…40ce8` in both. Nothing in either file changed between the two +evidence sets, so no delta below can be an artefact of measuring differently. +This is the first release-over-release comparison in this directory where the +comparison runner did not move at all. + +**The bundled application is byte-identical too.** MockMed ships inside the Flow +wheel (`openadapt_flow/mockmed/`), so pinning a different wheel can repin the +application under test. Between `v1.27.1` and `v1.28.0` that tree is unchanged +at `f0736c7a82ca2aba2beb334feaf461c5f06532a5`. Verify it without trusting this +file: + +```bash +for tag in v1.27.1 v1.28.0; do + git -C flow-1.28.0 rev-parse "$tag^{tree}:openadapt_flow/mockmed" +done +``` + +The fixture caveat that governed the 1.16.1 -> 1.24.0 comparison therefore does +not apply, exactly as it did not apply to 1.24.0 -> 1.27.1. + +## 7. The lease-safety probe is a runtime-contract measurement + +`scripts/probe_remote_lease_safety.py` does not drive a browser or a server. It +supplies a fake backend that implements only the two-phase remote actuation +lease — the exact protocol surface a pixel-only no-DOM canvas backend exposes — +and measures what the runtime does with a consequential remote click. It +therefore proves a runtime contract, not the behaviour of a real Citrix or RDP +session. Its scope block says so, and no wider claim is made from it. + +## 8. Expected variation + +Compiled replay was 3/3 correct in every condition in both the primary run and +the independent replication under `replication/` — 18 counted compiled trials +with zero over-halts, zero silent incorrect successes, zero wrong actions, and +zero model calls. Timings are host-specific; see `COMPARISON_TO_v1_27_1.md` for +the one timing figure that moved and for what this design can and cannot +conclude about it. Outcome classifications (task success, silent incorrect, +wrong action, transaction outcome, business effect) should not vary. diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/clean_postcondition_over_halt.json b/docs/eval_results/current_flow_v1_28_0_local_20260802/clean_postcondition_over_halt.json new file mode 100644 index 0000000..ca13672 --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/clean_postcondition_over_halt.json @@ -0,0 +1,31 @@ +{ + "arm": "compiled", + "condition": { + "kind": "clean", + "query": "" + }, + "counted_trials": 3, + "expected": "if the independently verified effect succeeded, the runtime must not leave the run in an unresumable false-incomplete state", + "flow": { + "artifact": { + "filename": "openadapt_flow-1.28.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202" + }, + "commit": "b646276a086c74b65ba850cdef2e475ca53f10c0", + "release_tag": "v1.28.0", + "tags": [ + "v1.28.0" + ], + "tracked_clean": true, + "version": "1.28.0" + }, + "observations": [], + "observed_count": 0, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "reproduce": "python scripts/run_current_flow_local_benchmark.py --flow-source --flow-wheel --out ; python scripts/extract_over_halt_regression.py --results /results.json --condition clean", + "runner_sha256": "68f9e5a27f4f04d831574167ebd6b362bf05184e81708a315d9896969b48a126", + "schema_version": 1, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "title": "clean effect succeeds but the runtime reports halt (compiled arm)" +} diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/remote_lease_safety/REPORT.md b/docs/eval_results/current_flow_v1_28_0_local_20260802/remote_lease_safety/REPORT.md new file mode 100644 index 0000000..7c07ef1 --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/remote_lease_safety/REPORT.md @@ -0,0 +1,53 @@ +# Remote frame-lease delivery: the two safety properties, measured + +openadapt-flow `c9618cc` (1.28.0) lets a consequential remote click be +delivered through the backend's one-shot actuation lease when no typed +delivery receipt exists. That relaxes a refusal, so its two stated +safety properties are measured here rather than trusted. + +## Source and environment + +- Flow commit: `b646276a086c74b65ba850cdef2e475ca53f10c0` (version `1.28.0`; tracked-clean source) +- Release tag: `v1.28.0` +- Wheel SHA-256: `4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202` +- Evals base commit: `71edc889035d998cac518ddf69b42860730533d7` +- Runner SHA-256: `2d0cf696df9779446469f0e94eb2f22e3e7634ac924998ebd93735b8d5a6791d` +- Platform: `macOS-15.7.3-arm64-arm-64bit` +- Python: `3.12.13` +- Network/provider use: none. No server, no browser, no model API. + +## Counted result + +3 trials per cell. `input edges` counts what the backend was actually asked to deliver. + +| Cell | Runs | Delivered | Refused | Receipts | Actuation tier | `transaction_outcome` (demo / standard / regulated) | +|---|---:|---:|---:|---:|---|---| +| `ungoverned_lease` | 3 | 3 | 0 | 0 | None | COMPLETED_UNVERIFIED / COMPLETED_UNVERIFIED / COMPLETED_UNVERIFIED | +| `governed_lease` | 3 | 0 | 3 | 0 | None | HALTED_BEFORE_EFFECT / HALTED_BEFORE_EFFECT / HALTED_BEFORE_EFFECT | +| `lease_frame_changed` | 3 | 0 | 3 | 0 | None | RECONCILIATION_REQUIRED / RECONCILIATION_REQUIRED / RECONCILIATION_REQUIRED | + +## Invariants + +Every invariant states the denominator it was evaluated over. A `vacuous` invariant had no applicable run and proves nothing. + +| Invariant | Applicable runs | Violations | Holds | +|---|---:|---:|---| +| A governed run must refuse a consequential remote click on a backend that cannot bind its exact fresh frame and target to delivery, before the first input edge. | 3 | 0 | yes | +| A remote write delivered through the frame lease alone carries no typed receipt, so no execution profile may classify it VERIFIED or production-eligible. | 9 | 0 | yes | +| A lease-only delivery must leave the result unlabeled: no typed delivery receipt and no recorded actuation tier. | 3 | 0 | yes | +| The lease is the safety property: a remote frame that changed between the lease and the input edge must stop delivery. | 3 | 0 | yes | +| A compiled remote replay makes no model calls and costs $0. | 9 | 0 | yes | + +## Scope + +- The backend is a fake implementing ONLY the two-phase remote + actuation lease, which is the exact protocol surface a pixel-only + no-DOM canvas backend exposes. This measures a runtime contract, not + a real Citrix or RDP session. +- One synthetic single-step workflow whose only step is the + irreversible write. Resolution is scripted to one fixed point so the + delivery decision is the only thing that varies between cells. +- No claim is made here about wrong-target immunity on a real remote + surface, about identity coverage, or about any hosted lifecycle. + +Reproduce: `python scripts/probe_remote_lease_safety.py --flow-source --flow-wheel --out ` diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/remote_lease_safety/results.json b/docs/eval_results/current_flow_v1_28_0_local_20260802/remote_lease_safety/results.json new file mode 100644 index 0000000..e081823 --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/remote_lease_safety/results.json @@ -0,0 +1,584 @@ +{ + "aggregate": { + "governed_lease": { + "actuation_tiers": [ + "None" + ], + "delivered_count": 0, + "delivery_receipt_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "refused_count": 3, + "replayer_success_count": 0, + "transaction_outcomes": { + "demo": [ + "HALTED_BEFORE_EFFECT" + ], + "regulated": [ + "HALTED_BEFORE_EFFECT" + ], + "standard": [ + "HALTED_BEFORE_EFFECT" + ] + } + }, + "lease_frame_changed": { + "actuation_tiers": [ + "None" + ], + "delivered_count": 0, + "delivery_receipt_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "refused_count": 3, + "replayer_success_count": 0, + "transaction_outcomes": { + "demo": [ + "RECONCILIATION_REQUIRED" + ], + "regulated": [ + "RECONCILIATION_REQUIRED" + ], + "standard": [ + "RECONCILIATION_REQUIRED" + ] + } + }, + "ungoverned_lease": { + "actuation_tiers": [ + "None" + ], + "delivered_count": 3, + "delivery_receipt_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "refused_count": 0, + "replayer_success_count": 3, + "transaction_outcomes": { + "demo": [ + "COMPLETED_UNVERIFIED" + ], + "regulated": [ + "COMPLETED_UNVERIFIED" + ], + "standard": [ + "COMPLETED_UNVERIFIED" + ] + } + } + }, + "caveats": [ + "The backend is a fake implementing only the two-phase remote actuation lease; this measures a runtime contract, not a real remote session.", + "One synthetic single-step workflow; resolution is scripted so the delivery decision is the only variable.", + "No wrong-target immunity, identity coverage, or hosted lifecycle claim is made here." + ], + "cells": [ + "ungoverned_lease", + "governed_lease", + "lease_frame_changed" + ], + "environment": { + "platform": "macOS-15.7.3-arm64-arm-64bit", + "python": "3.12.13" + }, + "generated_at": "2026-08-02T14:12:21.951540+00:00", + "invariants": [ + { + "applicable_count": 3, + "holds": true, + "id": "governed_run_refuses_receiptless_remote_write", + "statement": "A governed run must refuse a consequential remote click on a backend that cannot bind its exact fresh frame and target to delivery, before the first input edge.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 9, + "holds": true, + "id": "lease_delivery_is_never_a_production_success", + "statement": "A remote write delivered through the frame lease alone carries no typed receipt, so no execution profile may classify it VERIFIED or production-eligible.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 3, + "holds": true, + "id": "lease_delivery_carries_no_receipt_or_actuation_tier", + "statement": "A lease-only delivery must leave the result unlabeled: no typed delivery receipt and no recorded actuation tier.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 3, + "holds": true, + "id": "changed_frame_aborts_delivery", + "statement": "The lease is the safety property: a remote frame that changed between the lease and the input edge must stop delivery.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 9, + "holds": true, + "id": "zero_model_calls_and_cost", + "statement": "A compiled remote replay makes no model calls and costs $0.", + "vacuous": false, + "violation_count": 0, + "violations": [] + } + ], + "paid_or_remote_mutations": [], + "profiles": [ + "demo", + "standard", + "regulated" + ], + "reproduce": "python scripts/probe_remote_lease_safety.py --flow-source --flow-wheel --out ", + "runs": [ + { + "actuation_tier": null, + "backend_actions": [ + [ + "click", + 110, + 105, + false + ] + ], + "cell": "ungoverned_lease", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 1, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": true, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "regulated": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "standard": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + } + }, + "replayer_success": true, + "safety_halt": false, + "step_error": null, + "trial": 1 + }, + { + "actuation_tier": null, + "backend_actions": [ + [ + "click", + 110, + 105, + false + ] + ], + "cell": "ungoverned_lease", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 1, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": true, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "regulated": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "standard": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + } + }, + "replayer_success": true, + "safety_halt": false, + "step_error": null, + "trial": 2 + }, + { + "actuation_tier": null, + "backend_actions": [ + [ + "click", + 110, + 105, + false + ] + ], + "cell": "ungoverned_lease", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 1, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": true, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "regulated": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "standard": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + } + }, + "replayer_success": true, + "safety_halt": false, + "step_error": null, + "trial": 3 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "governed_lease", + "delivery_receipt_present": false, + "governed": true, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Step 's1' (click 'Save') is a consequential remote click, but this backend cannot bind its exact fresh frame and target to delivery; run aborted", + "trial": 1 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "governed_lease", + "delivery_receipt_present": false, + "governed": true, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Step 's1' (click 'Save') is a consequential remote click, but this backend cannot bind its exact fresh frame and target to delivery; run aborted", + "trial": 2 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "governed_lease", + "delivery_receipt_present": false, + "governed": true, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Step 's1' (click 'Save') is a consequential remote click, but this backend cannot bind its exact fresh frame and target to delivery; run aborted", + "trial": 3 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "lease_frame_changed", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Action delivery was uncertain and was not retried; the complete postcondition and independent effect contract did not confirm the intended outcome", + "trial": 1 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "lease_frame_changed", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Action delivery was uncertain and was not retried; the complete postcondition and independent effect contract did not confirm the intended outcome", + "trial": 2 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "lease_frame_changed", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Action delivery was uncertain and was not retried; the complete postcondition and independent effect contract did not confirm the intended outcome", + "trial": 3 + } + ], + "schema_version": 1, + "scope": "remote frame-lease delivery safety properties (runtime contract)", + "source": { + "evals": { + "commit": "71edc889035d998cac518ddf69b42860730533d7" + }, + "flow": { + "artifact": { + "filename": "openadapt_flow-1.28.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202" + }, + "commit": "b646276a086c74b65ba850cdef2e475ca53f10c0", + "release_tag": "v1.28.0", + "tags": [ + "v1.28.0" + ], + "tracked_clean": true, + "version": "1.28.0" + }, + "runner_sha256": "2d0cf696df9779446469f0e94eb2f22e3e7634ac924998ebd93735b8d5a6791d" + }, + "trials_per_cell": 3 +} diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/REPORT.md b/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/REPORT.md new file mode 100644 index 0000000..7d89402 --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/REPORT.md @@ -0,0 +1,70 @@ +# Exact-current local Flow performance + +This is a deterministic-runtime overhead and bounded robustness comparison. It is **not** a zero-shot computer-use comparison. + +## Source and environment + +- Flow commit: `b646276a086c74b65ba850cdef2e475ca53f10c0` (version `1.28.0`; tracked-clean source) +- Release tag: `v1.28.0` +- Wheel SHA-256: `4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202` +- Evals base commit: `71edc889035d998cac518ddf69b42860730533d7` +- Runner SHA-256: `68f9e5a27f4f04d831574167ebd6b362bf05184e81708a315d9896969b48a126` +- Platform: `macOS-15.7.3-arm64-arm-64bit` +- Python: `3.12.13` +- Playwright: `1.61.0` +- Chromium: `Playwright-managed headless Chromium` +- Network/provider use: loopback bundled MockMed only; no cloud VM, hosted runner, or model API + +## Counted result + +One synthetic MockMed workflow: sign in, open the intended referral, create a Triage encounter, enter a trial-unique note, and save. Each arm used a fresh browser. The arm-independent screenshot/OCR oracle required the exact saved note, Triage row, and intended patient, and separately flagged wrong-target writes. + +| Condition | Arm | Runs | Task success | Silent incorrect | Wrong action | Over-halt | Halt/error | Steady median | Steady p95 | End-to-end median | End-to-end p95 | Model calls | Cost | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `clean` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 7.030s | 7.055s | 7.591s | 7.630s | 0 | $0.00 | +| `clean` | DOM positional | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.203s | 0.207s | 0.801s | 0.851s | 0 | $0.00 | +| `clean` | DOM name-scoped | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.202s | 0.202s | 0.833s | 0.851s | 0 | $0.00 | +| `theme` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 7.190s | 7.409s | 7.772s | 7.977s | 0 | $0.00 | +| `theme` | DOM positional | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.204s | 0.208s | 0.832s | 0.901s | 0 | $0.00 | +| `theme` | DOM name-scoped | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.205s | 0.208s | 0.836s | 0.847s | 0 | $0.00 | +| `rename` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 7.033s | 7.143s | 7.601s | 7.724s | 0 | $0.00 | +| `rename` | DOM positional | 3 | 0/3 | 0 | 0 | 0 | 3 | 30.066s | 30.067s | 30.689s | 30.697s | 0 | $0.00 | +| `rename` | DOM name-scoped | 3 | 0/3 | 0 | 0 | 0 | 3 | 30.064s | 30.066s | 30.688s | 30.699s | 0 | $0.00 | + +Steady time wraps only the replay/script action loop. End-to-end time additionally includes fresh browser launch, the independent final screenshot/OCR oracle, and browser teardown. Local server startup and one-time record/compile are excluded from both and reported separately below. p95 is nearest-rank; with n=3 it is the slowest counted trial. + +Outcome definitions: **silent incorrect success** means the arm reported completion but the independent oracle did not confirm the intended effect; **wrong action** means the oracle observed a write to the wrong patient or encounter type; **over-halt** means the arm reported halt/incomplete while the independent oracle confirmed the intended effect; **halt/error** means the arm stopped and the intended effect was absent. The same definitions apply to every arm. The selector controls therefore can count as over-halts when alternate final-state evidence contradicts them; their rename failures did not, because the oracle confirmed no write. + +## One-time setup + +- Bundled app server startup: 0.002s (n=1 diagnostic) +- Setup trial 1: record 2.824s; compile 6.775s; bundle `68c82fc244cb6cf88269399722b37a223424ebd91ce738a5cf22c8a9aa5a9b42` +- Setup trial 2: record 2.542s; compile 6.457s; bundle `62e553bb1ea30909e06c2eaac18da53167495fcdfc210a4fe7d334eada370d74` +- Setup trial 3: record 2.489s; compile 6.438s; bundle `0cc5adad0f8102825350445376fba3594134dd62b273cf5ce8fb3582c786e472` + +## What this establishes + +- The independent final-state oracle confirmed the intended effect for every compiled clean, theme-drift, and label-drift trial. Compiled theme runs reported 0/3 over-halts; these are counted rather than relabelled as clean completions. +- The selector controls are steelmanned Playwright scripts. Their clean/theme speed is the correct reminder that API/structural actuation should remain the preferred tier where available. +- The `rename` surface changes `Open` to `View` and `Save Encounter` to `Submit Encounter`. Both selector controls failed loudly at the first renamed locator before mutation. These are unsupported-drift halts, not silent wrong actions. +- Label drift is an intentionally bounded robustness probe, not evidence for arbitrary drift or arbitrary applications. +- The compiled bundle had 5/8 identity-applicable clicks armed. Task completion is not a universal wrong-target-immunity claim for unarmed steps. +- The exact theme postcondition failures are retained in `theme_postcondition_over_halt.json` as a compact regression artifact. + +## Exact-current Flow versus zero-shot: not run + +No current Flow-versus-zero-shot result is claimed. The Azure WAA VM was not started and no model was called. The existing `scripts/eval_flow_on_waa.py` live replay path currently leaves the WAA evaluator unwired (so it cannot independently score success), while its hybrid live path explicitly returns before execution because the adapter is not wired. A valid future run therefore requires, in order: + +1. Wire `WAALiveAdapter.evaluate` into the Flow replay path and wire the same model/adapter into the zero-shot arm. +2. Prepare one exact compiled bundle per retained task and bind Flow/evals/model/environment revisions in the run manifest. +3. Obtain explicit approval for Azure VM start and a hard model spend cap; then confirm VM snapshot/readiness without changing the task set. +4. Run at least three trials per task per condition for both arms, using WAA's evaluator as the oracle and recording correct, silent incorrect, over-halt, timeout/error, latency, tokens, and cost. +5. Deallocate the VM, verify no orphaned resources, and publish the immutable raw summaries plus a normalized report. + +## Caveats + +- Synthetic MockMed, one workflow, one macOS host, headless Chromium. +- This report runs the exact published wheel named above, extracted locally, and binds it to the release-tagged tracked-clean source. +- Browser startup is fresh per run but OS/browser caches are warm after the first launch; the Latin-rotated arm order reduces but does not eliminate host-order effects. +- Final screenshots were inspected and hashed during the run; only their SHA-256 hashes are retained in this committed report. +- No hosted lifecycle, Windows UIA, RDP, Citrix, or real customer application is represented. diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/clean_postcondition_over_halt.json b/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/clean_postcondition_over_halt.json new file mode 100644 index 0000000..ca13672 --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/clean_postcondition_over_halt.json @@ -0,0 +1,31 @@ +{ + "arm": "compiled", + "condition": { + "kind": "clean", + "query": "" + }, + "counted_trials": 3, + "expected": "if the independently verified effect succeeded, the runtime must not leave the run in an unresumable false-incomplete state", + "flow": { + "artifact": { + "filename": "openadapt_flow-1.28.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202" + }, + "commit": "b646276a086c74b65ba850cdef2e475ca53f10c0", + "release_tag": "v1.28.0", + "tags": [ + "v1.28.0" + ], + "tracked_clean": true, + "version": "1.28.0" + }, + "observations": [], + "observed_count": 0, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "reproduce": "python scripts/run_current_flow_local_benchmark.py --flow-source --flow-wheel --out ; python scripts/extract_over_halt_regression.py --results /results.json --condition clean", + "runner_sha256": "68f9e5a27f4f04d831574167ebd6b362bf05184e81708a315d9896969b48a126", + "schema_version": 1, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "title": "clean effect succeeds but the runtime reports halt (compiled arm)" +} diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/results.json b/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/results.json new file mode 100644 index 0000000..0290362 --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/results.json @@ -0,0 +1,1110 @@ +{ + "aggregate": { + "compiled": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.5607581249787472, + "end_to_end_wall_s_median": 7.591224709001835, + "end_to_end_wall_s_p95_nearest_rank": 7.630358916998375, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 7.030466584023088, + "steady_wall_s_p95_nearest_rank": 7.055041791987605, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.5685956240049563, + "end_to_end_wall_s_median": 7.601348957978189, + "end_to_end_wall_s_p95_nearest_rank": 7.724001208960544, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 7.032753333973233, + "steady_wall_s_p95_nearest_rank": 7.143259666976519, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.5755896250484511, + "end_to_end_wall_s_median": 7.7722145000007, + "end_to_end_wall_s_p95_nearest_rank": 7.976841999974567, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 7.1895238339784555, + "steady_wall_s_p95_nearest_rank": 7.409202250011731, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + }, + "dom": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.6012434999574907, + "end_to_end_wall_s_median": 0.8005318749928847, + "end_to_end_wall_s_p95_nearest_rank": 0.8514147080131806, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.20274104096461087, + "steady_wall_s_p95_nearest_rank": 0.20654041704256088, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.621758165943902, + "end_to_end_wall_s_median": 30.688583249982912, + "end_to_end_wall_s_p95_nearest_rank": 30.69723583298037, + "failure_taxonomy": { + "halt_or_error": 3 + }, + "halt_or_error_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 0, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 30.065842041978613, + "steady_wall_s_p95_nearest_rank": 30.06682508403901, + "task_success_count": 0, + "task_success_rate": 0.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.624029790982604, + "end_to_end_wall_s_median": 0.8318497079890221, + "end_to_end_wall_s_p95_nearest_rank": 0.9005207500304095, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.2042274170089513, + "steady_wall_s_p95_nearest_rank": 0.2078199170064181, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + }, + "dom_named": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.6306764170294628, + "end_to_end_wall_s_median": 0.8325486670364626, + "end_to_end_wall_s_p95_nearest_rank": 0.8512698329868726, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.20187225000699982, + "steady_wall_s_p95_nearest_rank": 0.20249225001316518, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.6241489579551853, + "end_to_end_wall_s_median": 30.687853790994268, + "end_to_end_wall_s_p95_nearest_rank": 30.69891287497012, + "failure_taxonomy": { + "halt_or_error": 3 + }, + "halt_or_error_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 0, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 30.063704833039083, + "steady_wall_s_p95_nearest_rank": 30.06565954198595, + "task_success_count": 0, + "task_success_rate": 0.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.6353639580192976, + "end_to_end_wall_s_median": 0.835829375020694, + "end_to_end_wall_s_p95_nearest_rank": 0.8471349999890663, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.20520941697759554, + "steady_wall_s_p95_nearest_rank": 0.2075958339846693, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + } + }, + "arms": [ + "compiled", + "dom", + "dom_named" + ], + "caveats": [ + "Synthetic MockMed, one workflow, one macOS host, headless Chromium.", + "Published Flow wheel and release-tagged source are exact-bound.", + "Warm OS/browser caches remain after the first fresh-browser launch.", + "Final screenshot bytes were discarded after their hashes were retained.", + "No hosted, Windows, RDP, Citrix, or customer workflow is represented." + ], + "conditions": [ + "clean", + "theme", + "rename" + ], + "environment": { + "chromium": "Playwright-managed headless Chromium", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "playwright": "1.61.0", + "python": "3.12.13" + }, + "generated_at": "2026-08-02T14:01:59.347933+00:00", + "identity_coverage": { + "applicable_steps": 8, + "armed_steps": 5, + "unarmed": [ + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_000" + }, + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_002" + }, + { + "reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "step_id": "step_004" + } + ] + }, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "outcome_definitions": { + "correct": "arm reported completion and independent oracle confirmed effect", + "halt_or_error": "arm reported halt/incomplete and independent oracle found effect absent", + "over_halt": "arm reported halt/incomplete but independent oracle confirmed effect", + "silent_incorrect_success": "arm reported completion but independent oracle did not confirm effect", + "wrong_action": "independent oracle observed a wrong-patient or wrong-type write" + }, + "paid_or_remote_mutations": [], + "runs": [ + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.591224709001835, + "error": null, + "final_screenshot_sha256": "d173cc62630934f0e9dbc37f51536f46fd65172b6bb47d301dec51e398ca0694", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "abd12a9be34bdba88f72c46bfff764fd89275f3b213bc0d2683cd7f45cbc7bef", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.030466584023088, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8005318749928847, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "2b882a4c17309e8276043d804fce7d72b1744c886d8ef335ac6a95f3914e4beb", + "input_tokens": 0, + "note_found": true, + "note_sha256": "487a06d2518e7cbd44c2bf909f5e1c75efb4a74677014fa6a397eb3199abdba0", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.19928837503539398, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8325486670364626, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "9f7a3c464c84d35c0c730cdb3e42785ce956ee55c4d6ec3db5aad588bb7f3e2d", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0bab187ec8f858a6b7c76c7488cb233a0e63bd4b622c9ab5f0e13cd5e9545164", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20187225000699982, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8514147080131806, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "1774ae9ef0d5e682e45cd963009a5ccd245a1fde0f43a9737a4b569c3af501e6", + "input_tokens": 0, + "note_found": true, + "note_sha256": "fd9a7feca40d5477630dc008704a4cc90c73c8712333125a3fa0c75107e1def9", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20654041704256088, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8116348750190809, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "7525116feaa3ebb69531bb6c0790e63efc37a63d6858b238932af8ae9599b692", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0a79359c51a1562520f7cd213997e0e339a2cd46bba70365624dd0eda5545753", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20249225001316518, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.538542707974557, + "error": null, + "final_screenshot_sha256": "28868357daf60fd146bb494cf7a9840e6507cced77479394cdad1d862f117133", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "ac705b1533fec84a917710f37f46cc3c971668133ac443544636ca5c87730bda", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.981403000012506, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8512698329868726, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "e90e14bd8f07fa67d7a0be4e25e26e3622d0975b363f5e7aab41ae766c4c0799", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0aaceaff9dd0e0f73480518bf40ef1833fff34cc82644c7aca4c7598430b5b0e", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2014904580428265, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.630358916998375, + "error": null, + "final_screenshot_sha256": "0fbf9eb820d2ec10cf4a6db9f5426a12650074df5e676e9118238dd7b048492f", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "d936959a095f73f37e6bb533e6a491a248117661b9428db1436f23a2518f4a9e", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.055041791987605, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.7959354579797946, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "561fa687f35b7da42c7df6cbaea2463f1fbcbeebfad4e4ed207570a58a219b7b", + "input_tokens": 0, + "note_found": true, + "note_sha256": "16bd942cfa1315c6494e526f3c06f4e240f6a41aa434725d3558e69c3c639152", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20274104096461087, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.976841999974567, + "error": null, + "final_screenshot_sha256": "da8d189ea8ab3cf138a39b48a8609a19d129c1e0a856532088bbf30d68c7dd23", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "cd8f603f0405cf4e119d36cb53864e88c769b27b4ac791f97b063f51e5b035c2", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.409202250011731, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.7905578750069253, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "f474b4e55ac374801b5c7c9f434692fed4c0f7b71dbbcfbe7cfb78ba0bd82cf9", + "input_tokens": 0, + "note_found": true, + "note_sha256": "c1612f0bac6a65e866e923f68a636bce28368cb3bcb77d9d6fdb690364511f66", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2042274170089513, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8181798750301823, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "e6df91f58b57fc80a589761c02c3c5c73385bc6832282b398199220ca94a8d09", + "input_tokens": 0, + "note_found": true, + "note_sha256": "96c9f54ba668b83d716997b38595f7eb3ba6003edba85ae47efeb0fe36ba2ec2", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20520941697759554, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9005207500304095, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "ae3fc4f06795f9df0c9f3c7541fb52143bafb2850126c6ee234e3c67fdb22a1a", + "input_tokens": 0, + "note_found": true, + "note_sha256": "3ac4161fc038822d09fc851a31c46ce6933b378df41173685e962b66083afd63", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20326483401004225, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.835829375020694, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "c24e1259a0a8c0e602e56446cca872b3b8cb47f439762a14dd1ed151263c9228", + "input_tokens": 0, + "note_found": true, + "note_sha256": "ba7e43d4c36aece2b8383ce9206a4cb77e63f9803ad80eceacf42d44161fc102", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20046541700139642, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.7722145000007, + "error": null, + "final_screenshot_sha256": "32e3c893937fe74e1c09875b4740047c1088bac7671aadd63b8e1662fd43c2d8", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "853b125d6a17c00943f386979192028cdefc5d9e494e89a85254abf4dc048493", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.1895238339784555, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8471349999890663, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "49e839527a5a933e5b0ecb512366ceef0786dec32f3a1db40020d819f08a510a", + "input_tokens": 0, + "note_found": true, + "note_sha256": "79a353ca71e790724eb96a69724455151e8469bf24405e90565090dd80a3c465", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2075958339846693, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.711514958005864, + "error": null, + "final_screenshot_sha256": "5562db901d104b62f29a46a84d9076c8e19a38e3f2c3dd1350f7931aae1fb7bf", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "859c554a8f0a16f1be8d62fc23b55bef0f1531fcf26544ea26e08aa2eb686b94", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.135925332957413, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8318497079890221, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "5b38789776dcada6b5cc71bb4c91cc38ce21ea72f9e195b4f27ddc2abae48da2", + "input_tokens": 0, + "note_found": true, + "note_sha256": "f3ca35760733ac421075853af9cc0d0e6c44fcaf6bbfbc37f10c63d2e56def1a", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2078199170064181, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.724001208960544, + "error": null, + "final_screenshot_sha256": "69e2242c3c4e20feee7b6fb7eb36a7013606e4cc4f5857dd1a330f14266e5311", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "076b38bf20e6f444fe21df4cf2bf35539439cb1a1e302623fef2398ea623dba6", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.143259666976519, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.69723583298037, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "29c4fef4f1bd398474762a8fe81c754db30e30fe883e16222de95d5c42004aee", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.065842041978613, + "steps_total": 9, + "success": false, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.687853790994268, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "1ab448b77f9ba6bcc9353ab49979cbbf59f3c143ca89b23f0c3acf446089d191", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.063704833039083, + "steps_total": 9, + "success": false, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.66579137503868, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "ac4a87fd1f10c165f46f0bd11ad9ac0566c8c3c5bbf143ee4a38e68fc8581f91", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.05905799998436, + "steps_total": 9, + "success": false, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.69891287497012, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "6b40530e788765ea3d93cbe9bcc5c6b4329da187a82172b6929a0c310206aa40", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.06565954198595, + "steps_total": 9, + "success": false, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.601348957978189, + "error": null, + "final_screenshot_sha256": "2afcfd960cc887cb1f9c1b148f333cd4cadc22e472c2151692fcc36a29337ed8", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "81552308938bce81da752bee7715233079b55b434b433390f22c10ff0e0b847a", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.032753333973233, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.67647249996662, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "db3899469ecf79973b1072c9b3c0acab3bd2c3be46995d2c6f509456b449ea8d", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.05660395801533, + "steps_total": 9, + "success": false, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.491066917020362, + "error": null, + "final_screenshot_sha256": "07af33aea15bff8129be3686b262630407d95809563bb0a73272ca0932ea51c6", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "8763d35a4b13524e96dc508111a49d65d5ff779a044ff936c063cb8993d2ba3f", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.924214833008591, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.688583249982912, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "f68010e8551fe0b53d63da9c2cfa5ec7412e85c700264aedcd369c18b4211856", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.06682508403901, + "steps_total": 9, + "success": false, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + } + ], + "schema_version": 1, + "scope": "exact-current local deterministic runtime overhead and robustness", + "setup": { + "record_compile_trials": [ + { + "bundle_tree_sha256": "68c82fc244cb6cf88269399722b37a223424ebd91ce738a5cf22c8a9aa5a9b42", + "compile_wall_s": 6.774594124988653, + "manifest_sha256": "e975c6c5a2dc9f735aeb7da32dff2925b78b1baac272b7415aaa9e116bc18712", + "record_wall_s": 2.8236827090149745, + "trial": 1, + "workflow_sha256": "c02865bd3b1e2f8e9a4ff09f120409d51baf786e6a4d0f117fcffd1d287af33d" + }, + { + "bundle_tree_sha256": "62e553bb1ea30909e06c2eaac18da53167495fcdfc210a4fe7d334eada370d74", + "compile_wall_s": 6.4574872920056805, + "manifest_sha256": "9a7097326780eb96d5b0263cca7b6532d1d1a1ffa2c0174981ff862b4135aaad", + "record_wall_s": 2.5417830829974264, + "trial": 2, + "workflow_sha256": "d0baacd490a962eddde9fe01aea13f26fe2fa3f529305b7c597e04edd224e773" + }, + { + "bundle_tree_sha256": "0cc5adad0f8102825350445376fba3594134dd62b273cf5ce8fb3582c786e472", + "compile_wall_s": 6.438047292002011, + "manifest_sha256": "8976d439533ce5cc58d6efcd13de4ff2ebe1932c544d284365b7a3444212ac85", + "record_wall_s": 2.48945154203102, + "trial": 3, + "workflow_sha256": "ac24f2ccd4f1eedcd5363ae2a399c1fa7a8c9c456edb05b8a58d73df29e9c3d6" + } + ], + "server_start_wall_s": 0.0024014999507926404 + }, + "source": { + "evals": { + "commit": "71edc889035d998cac518ddf69b42860730533d7" + }, + "flow": { + "artifact": { + "filename": "openadapt_flow-1.28.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202" + }, + "commit": "b646276a086c74b65ba850cdef2e475ca53f10c0", + "release_tag": "v1.28.0", + "tags": [ + "v1.28.0" + ], + "tracked_clean": true, + "version": "1.28.0" + }, + "runner_sha256": "68f9e5a27f4f04d831574167ebd6b362bf05184e81708a315d9896969b48a126" + }, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "trials_per_arm_condition": 3, + "zero_shot_comparison": { + "reason": "requires paid model plus an available paid WAA environment; current live Flow replay also lacks a wired WAA evaluator", + "status": "not_run" + } +} diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/theme_postcondition_over_halt.json b/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/theme_postcondition_over_halt.json new file mode 100644 index 0000000..131f5af --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/replication/theme_postcondition_over_halt.json @@ -0,0 +1,30 @@ +{ + "condition": { + "kind": "cosmetic theme drift", + "query": "?drift=theme" + }, + "expected": "if the independently verified effect succeeded, the runtime must not leave the run in an unresumable false-incomplete state", + "flow": { + "artifact": { + "filename": "openadapt_flow-1.28.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202" + }, + "commit": "b646276a086c74b65ba850cdef2e475ca53f10c0", + "release_tag": "v1.28.0", + "tags": [ + "v1.28.0" + ], + "tracked_clean": true, + "version": "1.28.0" + }, + "observations": [], + "observed_count": 0, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "reproduce": "python scripts/run_current_flow_local_benchmark.py --flow-source --flow-wheel --out ", + "required_trials": 3, + "runner_sha256": "68f9e5a27f4f04d831574167ebd6b362bf05184e81708a315d9896969b48a126", + "schema_version": 1, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "title": "theme effect succeeds but region_stable reports halt" +} diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/results.json b/docs/eval_results/current_flow_v1_28_0_local_20260802/results.json new file mode 100644 index 0000000..1e39003 --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/results.json @@ -0,0 +1,1110 @@ +{ + "aggregate": { + "compiled": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.5751947080716491, + "end_to_end_wall_s_median": 7.613364084041677, + "end_to_end_wall_s_p95_nearest_rank": 7.621691458043642, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 7.032588458969258, + "steady_wall_s_p95_nearest_rank": 7.046496749971993, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.5790180829935707, + "end_to_end_wall_s_median": 7.587831916986033, + "end_to_end_wall_s_p95_nearest_rank": 7.6506177090341225, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 7.030602290993556, + "steady_wall_s_p95_nearest_rank": 7.06677891704021, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.5889935829909518, + "end_to_end_wall_s_median": 7.837825625028927, + "end_to_end_wall_s_p95_nearest_rank": 7.856137208000291, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 7.255227999994531, + "steady_wall_s_p95_nearest_rank": 7.270108749973588, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + }, + "dom": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.6283280000207014, + "end_to_end_wall_s_median": 0.8270078750210814, + "end_to_end_wall_s_p95_nearest_rank": 0.8602663750061765, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.20629208395257592, + "steady_wall_s_p95_nearest_rank": 0.20656483399216086, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.6244353760266677, + "end_to_end_wall_s_median": 30.685251709015574, + "end_to_end_wall_s_p95_nearest_rank": 30.704457291984, + "failure_taxonomy": { + "halt_or_error": 3 + }, + "halt_or_error_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 0, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 30.060816332988907, + "steady_wall_s_p95_nearest_rank": 30.06325312500121, + "task_success_count": 0, + "task_success_rate": 0.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.621672248991672, + "end_to_end_wall_s_median": 0.8200060829985887, + "end_to_end_wall_s_p95_nearest_rank": 0.8851861250004731, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.19954399997368455, + "steady_wall_s_p95_nearest_rank": 0.20598345797043294, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + }, + "dom_named": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.6430105839390308, + "end_to_end_wall_s_median": 0.8438631249591708, + "end_to_end_wall_s_p95_nearest_rank": 0.8666570829809643, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.20085254102014005, + "steady_wall_s_p95_nearest_rank": 0.2015528330230154, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.644110958033707, + "end_to_end_wall_s_median": 30.712198416993488, + "end_to_end_wall_s_p95_nearest_rank": 30.718486084020697, + "failure_taxonomy": { + "halt_or_error": 3 + }, + "halt_or_error_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 0, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 30.06808745895978, + "steady_wall_s_p95_nearest_rank": 30.06989054195583, + "task_success_count": 0, + "task_success_rate": 0.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.6537127509946004, + "end_to_end_wall_s_median": 0.8519075840013102, + "end_to_end_wall_s_p95_nearest_rank": 0.9183884999947622, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.19864449999295175, + "steady_wall_s_p95_nearest_rank": 0.20331287500448525, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + } + }, + "arms": [ + "compiled", + "dom", + "dom_named" + ], + "caveats": [ + "Synthetic MockMed, one workflow, one macOS host, headless Chromium.", + "Published Flow wheel and release-tagged source are exact-bound.", + "Warm OS/browser caches remain after the first fresh-browser launch.", + "Final screenshot bytes were discarded after their hashes were retained.", + "No hosted, Windows, RDP, Citrix, or customer workflow is represented." + ], + "conditions": [ + "clean", + "theme", + "rename" + ], + "environment": { + "chromium": "Playwright-managed headless Chromium", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "playwright": "1.61.0", + "python": "3.12.13" + }, + "generated_at": "2026-08-02T13:56:59.585192+00:00", + "identity_coverage": { + "applicable_steps": 8, + "armed_steps": 5, + "unarmed": [ + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_000" + }, + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_002" + }, + { + "reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "step_id": "step_004" + } + ] + }, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "outcome_definitions": { + "correct": "arm reported completion and independent oracle confirmed effect", + "halt_or_error": "arm reported halt/incomplete and independent oracle found effect absent", + "over_halt": "arm reported halt/incomplete but independent oracle confirmed effect", + "silent_incorrect_success": "arm reported completion but independent oracle did not confirm effect", + "wrong_action": "independent oracle observed a wrong-patient or wrong-type write" + }, + "paid_or_remote_mutations": [], + "runs": [ + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.613364084041677, + "error": null, + "final_screenshot_sha256": "d173cc62630934f0e9dbc37f51536f46fd65172b6bb47d301dec51e398ca0694", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "abd12a9be34bdba88f72c46bfff764fd89275f3b213bc0d2683cd7f45cbc7bef", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.032588458969258, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8037247919710353, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "2b882a4c17309e8276043d804fce7d72b1744c886d8ef335ac6a95f3914e4beb", + "input_tokens": 0, + "note_found": true, + "note_sha256": "487a06d2518e7cbd44c2bf909f5e1c75efb4a74677014fa6a397eb3199abdba0", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20656483399216086, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8438631249591708, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "9f7a3c464c84d35c0c730cdb3e42785ce956ee55c4d6ec3db5aad588bb7f3e2d", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0bab187ec8f858a6b7c76c7488cb233a0e63bd4b622c9ab5f0e13cd5e9545164", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20085254102014005, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8602663750061765, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "1774ae9ef0d5e682e45cd963009a5ccd245a1fde0f43a9737a4b569c3af501e6", + "input_tokens": 0, + "note_found": true, + "note_sha256": "fd9a7feca40d5477630dc008704a4cc90c73c8712333125a3fa0c75107e1def9", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20629208395257592, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8409260839689523, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "7525116feaa3ebb69531bb6c0790e63efc37a63d6858b238932af8ae9599b692", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0a79359c51a1562520f7cd213997e0e339a2cd46bba70365624dd0eda5545753", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.1998167079873383, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.539691416022833, + "error": null, + "final_screenshot_sha256": "28868357daf60fd146bb494cf7a9840e6507cced77479394cdad1d862f117133", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "ac705b1533fec84a917710f37f46cc3c971668133ac443544636ca5c87730bda", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.966205916949548, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8666570829809643, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "e90e14bd8f07fa67d7a0be4e25e26e3622d0975b363f5e7aab41ae766c4c0799", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0aaceaff9dd0e0f73480518bf40ef1833fff34cc82644c7aca4c7598430b5b0e", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2015528330230154, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.621691458043642, + "error": null, + "final_screenshot_sha256": "0fbf9eb820d2ec10cf4a6db9f5426a12650074df5e676e9118238dd7b048492f", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "d936959a095f73f37e6bb533e6a491a248117661b9428db1436f23a2518f4a9e", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.046496749971993, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8270078750210814, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "561fa687f35b7da42c7df6cbaea2463f1fbcbeebfad4e4ed207570a58a219b7b", + "input_tokens": 0, + "note_found": true, + "note_sha256": "16bd942cfa1315c6494e526f3c06f4e240f6a41aa434725d3558e69c3c639152", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.19867987500037998, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.837825625028927, + "error": null, + "final_screenshot_sha256": "da8d189ea8ab3cf138a39b48a8609a19d129c1e0a856532088bbf30d68c7dd23", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "cd8f603f0405cf4e119d36cb53864e88c769b27b4ac791f97b063f51e5b035c2", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.270108749973588, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8096917920047417, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "f474b4e55ac374801b5c7c9f434692fed4c0f7b71dbbcfbe7cfb78ba0bd82cf9", + "input_tokens": 0, + "note_found": true, + "note_sha256": "c1612f0bac6a65e866e923f68a636bce28368cb3bcb77d9d6fdb690364511f66", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20598345797043294, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8519075840013102, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "e6df91f58b57fc80a589761c02c3c5c73385bc6832282b398199220ca94a8d09", + "input_tokens": 0, + "note_found": true, + "note_sha256": "96c9f54ba668b83d716997b38595f7eb3ba6003edba85ae47efeb0fe36ba2ec2", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.19819483300670981, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8851861250004731, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "ae3fc4f06795f9df0c9f3c7541fb52143bafb2850126c6ee234e3c67fdb22a1a", + "input_tokens": 0, + "note_found": true, + "note_sha256": "3ac4161fc038822d09fc851a31c46ce6933b378df41173685e962b66083afd63", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.19954399997368455, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8382444999879226, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "c24e1259a0a8c0e602e56446cca872b3b8cb47f439762a14dd1ed151263c9228", + "input_tokens": 0, + "note_found": true, + "note_sha256": "ba7e43d4c36aece2b8383ce9206a4cb77e63f9803ad80eceacf42d44161fc102", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.19864449999295175, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.747676750004757, + "error": null, + "final_screenshot_sha256": "32e3c893937fe74e1c09875b4740047c1088bac7671aadd63b8e1662fd43c2d8", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "853b125d6a17c00943f386979192028cdefc5d9e494e89a85254abf4dc048493", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.158683167013805, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9183884999947622, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "49e839527a5a933e5b0ecb512366ceef0786dec32f3a1db40020d819f08a510a", + "input_tokens": 0, + "note_found": true, + "note_sha256": "79a353ca71e790724eb96a69724455151e8469bf24405e90565090dd80a3c465", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20331287500448525, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.856137208000291, + "error": null, + "final_screenshot_sha256": "5562db901d104b62f29a46a84d9076c8e19a38e3f2c3dd1350f7931aae1fb7bf", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "859c554a8f0a16f1be8d62fc23b55bef0f1531fcf26544ea26e08aa2eb686b94", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.255227999994531, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8200060829985887, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "5b38789776dcada6b5cc71bb4c91cc38ce21ea72f9e195b4f27ddc2abae48da2", + "input_tokens": 0, + "note_found": true, + "note_sha256": "f3ca35760733ac421075853af9cc0d0e6c44fcaf6bbfbc37f10c63d2e56def1a", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.19833383400691673, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.524622750002891, + "error": null, + "final_screenshot_sha256": "69e2242c3c4e20feee7b6fb7eb36a7013606e4cc4f5857dd1a330f14266e5311", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "076b38bf20e6f444fe21df4cf2bf35539439cb1a1e302623fef2398ea623dba6", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.94560466700932, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.704457291984, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "29c4fef4f1bd398474762a8fe81c754db30e30fe883e16222de95d5c42004aee", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.05950341600692, + "steps_total": 9, + "success": false, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.718486084020697, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "1ab448b77f9ba6bcc9353ab49979cbbf59f3c143ca89b23f0c3acf446089d191", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.06989054195583, + "steps_total": 9, + "success": false, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.685251709015574, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "ac4a87fd1f10c165f46f0bd11ad9ac0566c8c3c5bbf143ee4a38e68fc8581f91", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.060816332988907, + "steps_total": 9, + "success": false, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.712198416993488, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "6b40530e788765ea3d93cbe9bcc5c6b4329da187a82172b6929a0c310206aa40", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.06808745895978, + "steps_total": 9, + "success": false, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.6506177090341225, + "error": null, + "final_screenshot_sha256": "2afcfd960cc887cb1f9c1b148f333cd4cadc22e472c2151692fcc36a29337ed8", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "81552308938bce81da752bee7715233079b55b434b433390f22c10ff0e0b847a", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.06677891704021, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.695095749979373, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "db3899469ecf79973b1072c9b3c0acab3bd2c3be46995d2c6f509456b449ea8d", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.06550145795336, + "steps_total": 9, + "success": false, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.587831916986033, + "error": null, + "final_screenshot_sha256": "07af33aea15bff8129be3686b262630407d95809563bb0a73272ca0932ea51c6", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "8763d35a4b13524e96dc508111a49d65d5ff779a044ff936c063cb8993d2ba3f", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.030602290993556, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.6780553329736, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "f68010e8551fe0b53d63da9c2cfa5ec7412e85c700264aedcd369c18b4211856", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.06325312500121, + "steps_total": 9, + "success": false, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + } + ], + "schema_version": 1, + "scope": "exact-current local deterministic runtime overhead and robustness", + "setup": { + "record_compile_trials": [ + { + "bundle_tree_sha256": "d2c035f3e895e5e0130bd65aa0019fdbce7ddeeaa34ccb641f6a72767d3acb3f", + "compile_wall_s": 8.151417416986078, + "manifest_sha256": "25fb4d9caf819b2b0f70789c265cfc62b9e97f784389b71d00d2b94edcf677f3", + "record_wall_s": 3.438297542044893, + "trial": 1, + "workflow_sha256": "3e849689c1367943c1e237c0173d9533e20f9fe0c9e98dfb8c6455d18d714e99" + }, + { + "bundle_tree_sha256": "7c1e5beed1e271209f5f63ff3a54bc134cf683cc5845432cd116c4dad729567b", + "compile_wall_s": 6.40659108303953, + "manifest_sha256": "fba2379f9bb056a28bce9fafd53f82a6923a1bcd8fd7275b947bbc0717f3f1c7", + "record_wall_s": 2.500314291974064, + "trial": 2, + "workflow_sha256": "c6b2bfca13981ce33fdecae86ae16de9d045f281be13ff4ebb2b1e5e377ffca1" + }, + { + "bundle_tree_sha256": "685e7ba7168ada2571af27eb8d36829b756abf6baa6d52c5718451dcef4e4694", + "compile_wall_s": 6.374280708958395, + "manifest_sha256": "f7787c45ce2b0a81e8e8c3c158a71a75f56c2f09f341c81fac6a2d437ebde509", + "record_wall_s": 2.537935583968647, + "trial": 3, + "workflow_sha256": "f59bea221d266eb265d3fe51144c4ed6253633543276e3844a7a4957fab1011f" + } + ], + "server_start_wall_s": 0.003973458951804787 + }, + "source": { + "evals": { + "commit": "71edc889035d998cac518ddf69b42860730533d7" + }, + "flow": { + "artifact": { + "filename": "openadapt_flow-1.28.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202" + }, + "commit": "b646276a086c74b65ba850cdef2e475ca53f10c0", + "release_tag": "v1.28.0", + "tags": [ + "v1.28.0" + ], + "tracked_clean": true, + "version": "1.28.0" + }, + "runner_sha256": "68f9e5a27f4f04d831574167ebd6b362bf05184e81708a315d9896969b48a126" + }, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "trials_per_arm_condition": 3, + "zero_shot_comparison": { + "reason": "requires paid model plus an available paid WAA environment; current live Flow replay also lacks a wired WAA evaluator", + "status": "not_run" + } +} diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/theme_postcondition_over_halt.json b/docs/eval_results/current_flow_v1_28_0_local_20260802/theme_postcondition_over_halt.json new file mode 100644 index 0000000..131f5af --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/theme_postcondition_over_halt.json @@ -0,0 +1,30 @@ +{ + "condition": { + "kind": "cosmetic theme drift", + "query": "?drift=theme" + }, + "expected": "if the independently verified effect succeeded, the runtime must not leave the run in an unresumable false-incomplete state", + "flow": { + "artifact": { + "filename": "openadapt_flow-1.28.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202" + }, + "commit": "b646276a086c74b65ba850cdef2e475ca53f10c0", + "release_tag": "v1.28.0", + "tags": [ + "v1.28.0" + ], + "tracked_clean": true, + "version": "1.28.0" + }, + "observations": [], + "observed_count": 0, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "reproduce": "python scripts/run_current_flow_local_benchmark.py --flow-source --flow-wheel --out ", + "required_trials": 3, + "runner_sha256": "68f9e5a27f4f04d831574167ebd6b362bf05184e81708a315d9896969b48a126", + "schema_version": 1, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "title": "theme effect succeeds but region_stable reports halt" +} diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/transaction_probe/REPORT.md b/docs/eval_results/current_flow_v1_28_0_local_20260802/transaction_probe/REPORT.md new file mode 100644 index 0000000..aea939a --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/transaction_probe/REPORT.md @@ -0,0 +1,60 @@ +# Flow transaction outcome taxonomy probe + +Does the terminal transaction outcome match what the system of record can actually prove? Judged only against MockMed's independent store, never against the runtime's report or the screen. **Not** a zero-shot computer-use comparison. + +## Source and environment + +- Flow commit: `b646276a086c74b65ba850cdef2e475ca53f10c0` (version `1.28.0`; tracked-clean source) +- Release tag: `v1.28.0` +- Wheel SHA-256: `4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202` +- Evals base commit: `71edc889035d998cac518ddf69b42860730533d7` +- Runner SHA-256: `097cc900b44872712ebd7a5bfbdb53f87c85bf5bd8dcbef88fa893743d240ce8` +- Platform: `macOS-15.7.3-arm64-arm-64bit` +- Python: `3.12.13` +- Playwright: `1.61.0` +- Chromium: `Playwright-managed headless Chromium` +- Network/provider use: loopback bundled MockMed fault server only; no cloud VM, hosted runner, or model API + +## Counted result + +3 trials per cell, no retries. The oracle is the system-of-record snapshot delta at `GET /api/db`: the intended effect is exactly one new `Triage` encounter for the intended patient carrying this run's note, with no pre-existing row destroyed. + +| Fault mode | Verification | Runs | Ground-truth effect | Transaction outcome | Billable | Verification performed | Model calls | +|---|---|---:|---|---|---:|---:|---:| +| `ok` | `unverified` | 3 | intended_once 3 | COMPLETED_UNVERIFIED 3 | 0 | 0 | 0 | +| `ok` | `effect_verified` | 3 | intended_once 3 | COMPLETED_UNVERIFIED 3 | 0 | 3 | 0 | +| `timeout` | `unverified` | 3 | intended_once 3 | RECONCILIATION_REQUIRED 3 | 0 | 0 | 0 | +| `timeout` | `effect_verified` | 3 | intended_once 3 | RECONCILIATION_REQUIRED 3 | 0 | 0 | 0 | +| `optimistic` | `unverified` | 3 | absent 3 | COMPLETED_UNVERIFIED 3 | 0 | 0 | 0 | +| `optimistic` | `effect_verified` | 3 | absent 3 | RECONCILIATION_REQUIRED 3 | 0 | 3 | 0 | +| `session` | `unverified` | 3 | absent 3 | RECONCILIATION_REQUIRED 3 | 0 | 0 | 0 | +| `session` | `effect_verified` | 3 | absent 3 | RECONCILIATION_REQUIRED 3 | 0 | 0 | 0 | +| `duplicate` | `unverified` | 3 | duplicate 3 | COMPLETED_UNVERIFIED 3 | 0 | 0 | 0 | +| `duplicate` | `effect_verified` | 3 | duplicate 3 | RECONCILIATION_REQUIRED 3 | 0 | 3 | 0 | + +`timeout` commits the row and then hangs past the client timeout, so the client sees an error though the write landed. `optimistic` paints a success banner the server then rejects. `session` returns 401 and persists nothing. `duplicate` accepts every arriving write. + +## Invariants + +An invariant no counted run could exercise is reported as `vacuous`; it proves nothing and is not a pass. + +| Invariant | Holds | Applicable runs | Violations | +|---|---|---:|---:| +| COMPLETED_UNVERIFIED must never be reported as a production success and must never be billable. | yes | 12 | 0 | +| Only VERIFIED may be billable. | yes | 30 | 0 | +| An outcome that asserts no business effect occurred (HALTED_BEFORE_EFFECT / FAILED_PLATFORM / CANCELED / REJECTED_POLICY) must not be reported when the system of record shows the write landed. | vacuous | 0 | 0 | +| A consequential step that reached actuation but whose effect was never verified cannot be classified as a proven absence; it must be RECONCILIATION_REQUIRED. | yes | 21 | 0 | +| A single run must never write the intended record more than once. A duplicate row from a run whose backend is NOT the double-accepting 'duplicate' mode would be a blind retry. | yes | 24 | 0 | +| VERIFIED must never be reported when nothing landed. | vacuous | 0 | 0 | +| A healthy compiled run makes no model calls and costs $0. | yes | 30 | 0 | + +## Findings + +- Every invariant held across every counted trial. + +## Caveats + +- Synthetic MockMed fault server, one workflow, one macOS host, headless Chromium. +- The probe runs the exact published wheel named above, extracted locally, bound to the release-tagged tracked-clean source. +- The `effect_verified` configuration is authored by this script, not mined by the compiler; it shows what the taxonomy can prove when a verifier IS configured, and is not a claim that a compiled bundle ships one by default. +- No hosted lifecycle, Windows UIA, RDP, Citrix, or real customer application is represented. diff --git a/docs/eval_results/current_flow_v1_28_0_local_20260802/transaction_probe/results.json b/docs/eval_results/current_flow_v1_28_0_local_20260802/transaction_probe/results.json new file mode 100644 index 0000000..3500aa4 --- /dev/null +++ b/docs/eval_results/current_flow_v1_28_0_local_20260802/transaction_probe/results.json @@ -0,0 +1,1170 @@ +{ + "aggregate": { + "effect_verified": { + "duplicate": { + "billable_count": 0, + "business_effects": { + "duplicate": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 12.407029625028372, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 3 + }, + "ok": { + "billable_count": 0, + "business_effects": { + "intended_once": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 7.055109000008088, + "transaction_outcomes": { + "COMPLETED_UNVERIFIED": 3 + }, + "verification_performed_count": 3 + }, + "optimistic": { + "billable_count": 0, + "business_effects": { + "absent": 3 + }, + "effect_landed_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 12.329612000030465, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 3 + }, + "session": { + "billable_count": 0, + "business_effects": { + "absent": 3 + }, + "effect_landed_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 12.738037167000584, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 0 + }, + "timeout": { + "billable_count": 0, + "business_effects": { + "intended_once": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 12.384449166012928, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 0 + } + }, + "unverified": { + "duplicate": { + "billable_count": 0, + "business_effects": { + "duplicate": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 7.0434110420173965, + "transaction_outcomes": { + "COMPLETED_UNVERIFIED": 3 + }, + "verification_performed_count": 0 + }, + "ok": { + "billable_count": 0, + "business_effects": { + "intended_once": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 6.988184540998191, + "transaction_outcomes": { + "COMPLETED_UNVERIFIED": 3 + }, + "verification_performed_count": 0 + }, + "optimistic": { + "billable_count": 0, + "business_effects": { + "absent": 3 + }, + "effect_landed_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 7.038101040991023, + "transaction_outcomes": { + "COMPLETED_UNVERIFIED": 3 + }, + "verification_performed_count": 0 + }, + "session": { + "billable_count": 0, + "business_effects": { + "absent": 3 + }, + "effect_landed_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 12.79729524999857, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 0 + }, + "timeout": { + "billable_count": 0, + "business_effects": { + "intended_once": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 12.532679125026334, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 0 + } + } + }, + "business_effects": [ + "intended_once", + "absent", + "duplicate", + "partial", + "collateral_loss" + ], + "caveats": [ + "Synthetic MockMed fault server, one workflow, one macOS host, headless Chromium.", + "Published Flow wheel and release-tagged source are exact-bound.", + "The effect contract in the effect_verified cells is authored by this script.", + "No hosted, Windows, RDP, Citrix, or customer workflow is represented." + ], + "consequential_step_id": "step_010", + "environment": { + "chromium": "Playwright-managed headless Chromium", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "playwright": "1.61.0", + "python": "3.12.13" + }, + "fault_modes": [ + "ok", + "timeout", + "optimistic", + "session", + "duplicate" + ], + "generated_at": "2026-08-02T14:07:36.874614+00:00", + "invariants": [ + { + "applicable_count": 12, + "holds": true, + "id": "completed_unverified_is_never_success", + "statement": "COMPLETED_UNVERIFIED must never be reported as a production success and must never be billable.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 30, + "holds": true, + "id": "only_verified_is_billable", + "statement": "Only VERIFIED may be billable.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 0, + "holds": true, + "id": "no_false_absence_claim", + "statement": "An outcome that asserts no business effect occurred (HALTED_BEFORE_EFFECT / FAILED_PLATFORM / CANCELED / REJECTED_POLICY) must not be reported when the system of record shows the write landed.", + "vacuous": true, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 21, + "holds": true, + "id": "unverified_delivered_write_needs_reconciliation", + "statement": "A consequential step that reached actuation but whose effect was never verified cannot be classified as a proven absence; it must be RECONCILIATION_REQUIRED.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 24, + "holds": true, + "id": "no_blind_retry_of_consequential_write", + "statement": "A single run must never write the intended record more than once. A duplicate row from a run whose backend is NOT the double-accepting 'duplicate' mode would be a blind retry.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 0, + "holds": true, + "id": "verified_requires_a_real_effect", + "statement": "VERIFIED must never be reported when nothing landed.", + "vacuous": true, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 30, + "holds": true, + "id": "zero_model_calls_and_cost", + "statement": "A healthy compiled run makes no model calls and costs $0.", + "vacuous": false, + "violation_count": 0, + "violations": [] + } + ], + "oracle": "system-of-record snapshot delta at GET /api/db: exactly one new Triage encounter for the intended patient carrying this run's note, with no pre-existing row destroyed", + "paid_or_remote_mutations": [], + "runs": [ + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "abd12a9be34bdba88f72c46bfff764fd89275f3b213bc0d2683cd7f45cbc7bef", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 6.988184540998191, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "ac705b1533fec84a917710f37f46cc3c971668133ac443544636ca5c87730bda", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.013218875043094, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "d936959a095f73f37e6bb533e6a491a248117661b9428db1436f23a2518f4a9e", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 6.9806581669836305, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "cd8f603f0405cf4e119d36cb53864e88c769b27b4ac791f97b063f51e5b035c2", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.532679125026334, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "853b125d6a17c00943f386979192028cdefc5d9e494e89a85254abf4dc048493", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.536108582979068, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "859c554a8f0a16f1be8d62fc23b55bef0f1531fcf26544ea26e08aa2eb686b94", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.416436375002377, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "076b38bf20e6f444fe21df4cf2bf35539439cb1a1e302623fef2398ea623dba6", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.114110208989587, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "81552308938bce81da752bee7715233079b55b434b433390f22c10ff0e0b847a", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.038101040991023, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "8763d35a4b13524e96dc508111a49d65d5ff779a044ff936c063cb8993d2ba3f", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.0043436249834485, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "b84e97c170e65ff3a7b9712f233507b2a589dec44cf353cd0034de4a31a8d525", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.825688374985475, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "f34561628adb2837d87c87d953e1780f9fee2702b135af1bb096c18a0a4e9cf4", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.79729524999857, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "bbf6c8f549bb48f6303e19264a2f0de4bb587910dcfdf60c49032bff48b869b6", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.709410375042353, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 2, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "5d210517b20451b7a7f52c057e211a2277797520edd3cf6fd76c7055d184b4ac", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.029235833033454, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 2, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "3207eb9c32f1aa33fe88e313eae8cb1a0adf184be9cb2cf35018f8b2cb0dfa88", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.152870999998413, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 2, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "80a6eec7c17331ecb26644d81df16377714db78149d2acd0a497d6768618abf7", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.0434110420173965, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": true, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "29fde12e51ae8bd734dc568e564c038c2bb5dd227c572c64ad3610815a284cfe", + "observed_effect": "present", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.362755666021258, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": true, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "5aa693a87142dcceb4c8db74b5b93165c79313a01579adf3b24fbd1e2336a2c6", + "observed_effect": "present", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.055109000008088, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": true, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "7052069910c3085280370c9aed389f62a16e3d9370f73bb8472a7918acbef17e", + "observed_effect": "present", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.050403666042257, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "929ced0d8f74be8a17242d43e7c61c2e894bfaaf7850937a15450d7e99d95668", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.384449166012928, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "1c6102e34a227a0101775c627e73c880cab193c34848bdda0f43fc4da96cab2e", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.35904837504495, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "f43d194861e88ab46f2cf911b7eb96170709e51cf0adf0296394b6877edef10c", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.417239665985107, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": false, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "1f77084f54803a5f2796ed15d54d1f0198e95883f75b5e2b7e48e72dc74ebc6c", + "observed_effect": "absent", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.329612000030465, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": false, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "609b93e6fc3f4060c19a1dd54aa5b40597cd07d6838e04e8715f569f158c7ac9", + "observed_effect": "absent", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.343746792001184, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": false, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "765ae64e90cf992a6c19c701b6d84450431ec4a23a84507465fa02e3873b20b7", + "observed_effect": "absent", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.267250458011404, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "754e4d054274c16b7c1e99b44172b015ff2a5fd86be82d843ab4b5400cddd85d", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.782000124978367, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "5f565c53fb7c15406a3c21440b9af972cfdfc65bd7ed0f6a3f7573eb7e7c9498", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.738037167000584, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "5b2bcb6c3fa0b9850b340ad9fed810503d86b8de259a87b867064960eaa457ff", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.611504208005499, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": false, + "error": null, + "exact_record_count": 2, + "execution_outcome": "HALTED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "7292d8f70933cee714c91112d2d2908aeeadf60c25665ed89c51414ae829e09d", + "observed_effect": "conflicting", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.378015291993506, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": false, + "error": null, + "exact_record_count": 2, + "execution_outcome": "HALTED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "cac5d1be233ba75af6729a3aa4fcd4f49d764706c7cd6d22b06cb5d68a69dc15", + "observed_effect": "conflicting", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.407029625028372, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": false, + "error": null, + "exact_record_count": 2, + "execution_outcome": "HALTED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "2512a0a87f2c04343e5ea49e761c92aea70efe5ebe2247d73b0db52e6b305d1c", + "observed_effect": "conflicting", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.435963249998167, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "effect_verified", + "verification_performed": true + } + ], + "schema_version": 1, + "scope": "transaction outcome taxonomy against a real persistence boundary", + "setup": { + "bundle_tree_sha256": "4625e5c168651506392d13020c88fcba5aacf502b07246f24b059d4a55ffe9f3", + "compile_wall_s": 6.786902374995407, + "manifest_sha256": "815b72ae509b08d1196c5186df406799556bc1e06b55174c4106d5cd321b84cb", + "record_wall_s": 2.7753359579946846, + "workflow_sha256": "451692d8040c9cf83ff729cc25bdc40dfa27c32892dc6c8df09aba90109a5db4" + }, + "source": { + "evals": { + "commit": "71edc889035d998cac518ddf69b42860730533d7" + }, + "flow": { + "artifact": { + "filename": "openadapt_flow-1.28.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "4d156035ea411e3cbdbc40978d653d50727a7d8664646be62f7f9e95ba0c7202" + }, + "commit": "b646276a086c74b65ba850cdef2e475ca53f10c0", + "release_tag": "v1.28.0", + "tags": [ + "v1.28.0" + ], + "tracked_clean": true, + "version": "1.28.0" + }, + "runner_sha256": "097cc900b44872712ebd7a5bfbdb53f87c85bf5bd8dcbef88fa893743d240ce8" + }, + "task": "MockMed triage against the fault-injection persistence boundary: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "trials_per_cell": 3, + "verification_modes": [ + "unverified", + "effect_verified" + ] +} diff --git a/scripts/probe_remote_lease_safety.py b/scripts/probe_remote_lease_safety.py new file mode 100644 index 0000000..797fe43 --- /dev/null +++ b/scripts/probe_remote_lease_safety.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +"""Probe the two safety properties of Flow 1.28.0's remote frame-lease delivery. + +openadapt-flow ``c9618cc`` (released in 1.28.0) changed how a *consequential* +remote pointer edge is delivered. Before it, such a step required +``GuardedRemotePointerActionBackend`` -- the backend that returns an explicit +typed, target-bound delivery receipt -- so every opaque remote surface that can +offer only the documented one-shot actuation lease (the no-DOM HTML5-canvas +class Citrix Workspace-web presents) halted on its write step. After it, the +runtime delivers through the lease when no typed receipt exists. + +Relaxing a refusal is exactly the kind of change that can buy a green +demonstration with a silent wrong write, so the release note's two claims are +measured here rather than trusted: + +1. **A governed run still refuses.** Under a ``GovernedRunAuthorization`` + (Standard/Regulated), a consequential remote click on a backend that cannot + bind its exact fresh frame and target to delivery must stop *before the first + input edge*. +2. **The lease-delivered result is never a production success.** It carries no + ``actuation`` tier and no ``delivery_receipt``, so Flow's own outcome + classifier must map it to ``COMPLETED_UNVERIFIED`` and never to ``VERIFIED``, + under every execution profile. + +A third cell measures the property the change relies on: the lease itself. A +remote frame that changes between ``acquire_actuation_frame`` and the input edge +must abort delivery, otherwise "the lease is the safety property" is not true. + +Method matches the sibling runners in this directory: the exact published wheel +is extracted and imported from that extraction, the wheel is bound to +release-tagged tracked-clean source, ``>=3`` trials per cell, no retries, no +model calls, no network access, and every invariant is reported with the +denominator it was evaluated over. A violated invariant is counted and (unless +``--no-fail-on-violation``) exits non-zero. It is never silently downgraded. + +The backend here is a fake: it implements only the two-phase remote actuation +lease, which is the exact protocol surface a pixel-only canvas backend exposes. +That is deliberate -- the question is what the *runtime* does when a backend can +offer nothing more, and a fake is the only way to hold everything else constant. +This probe therefore measures a runtime contract, not a real remote session. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import importlib.util +import io +import json +import platform +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +_SIBLING = Path(__file__).resolve().parent / "run_current_flow_local_benchmark.py" +_SPEC = importlib.util.spec_from_file_location("_current_flow_local_benchmark", _SIBLING) +assert _SPEC is not None and _SPEC.loader is not None +_BENCH = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_BENCH) + +# Reuse the sibling's exact source-binding discipline so the reports in one +# evidence set can never drift on how they pin a wheel, a commit, or a tag. +_sha256 = _BENCH._sha256 +_source_binding = _BENCH._source_binding +_extract_wheel = _BENCH._extract_wheel +_git = _BENCH._git + +VIEWPORT = (300, 200) +CLICK_POINT = (110, 105) +REGION = (100, 100, 50, 20) + +#: The only outcome the runtime may treat as a production success. +_PRODUCTION_SUCCESS = "VERIFIED" + +#: Every posture the outcome classifier supports. The claim under test is that +#: none of them can turn a receipt-less lease delivery into a production +#: success, so all three are measured rather than only the governed ones. +PROFILES = ("demo", "standard", "regulated") + +CELLS = ("ungoverned_lease", "governed_lease", "lease_frame_changed") + + +def _make_png(size: tuple[int, int] = VIEWPORT, color: tuple[int, int, int] = (240, 240, 240)) -> bytes: + from PIL import Image # noqa: PLC0415 + + image = Image.new("RGB", size, color) + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + +class _Match: + """The minimal shape the resolver reads back from a vision call.""" + + def __init__(self, point, region, confidence=0.95): + self.point = point + self.region = region + self.confidence = confidence + + +class _ScriptedVision: + """Scripted vision namespace covering everything the Replayer touches. + + Nothing here is measured. It exists so the resolution ladder settles on one + fixed point with no model call and no real screen, leaving the delivery + decision as the only thing that varies between cells. + """ + + def __init__(self) -> None: + self.template_results: list[_Match] = [] + self.settle_count = 0 + + def find_template( + self, + screen_png, + template_png, + *, + search_region=None, + prefer_near=None, + scales=(0.85, 1.0, 1.18), + threshold=0.82, + ): + if self.template_results: + return self.template_results.pop(0) + return None + + def find_structural_template( + self, + screen_png, + template_png, + *, + search_region=None, + prefer_near=None, + scales=(0.85, 1.0, 1.18), + threshold=0.8, + ): + return None + + def find_text( + self, screen_png, text, *, region=None, min_ratio=0.8, raise_on_ambiguity=False + ): + return None + + def text_present(self, screen_png, text, *, region=None, min_ratio=0.8): + return False + + def ocr(self, screen_png, *, region=None): + return [] + + def pixels_changed( + self, before_png, after_png, *, region=None, threshold=20, min_pixels=4 + ): + return True + + def phash_png(self, png, region=None): + return "aa" + + def phash_distance(self, a, b): + return 0 + + def wait_settled(self, backend, *, interval_s=0.1, stable_frames=2, timeout_s=3.0): + self.settle_count += 1 + return backend.screenshot() + + +class PixelOnlyRemoteBackend: + """Opaque remote surface exposing ONLY the two-phase actuation lease. + + This is the protocol surface of a no-DOM HTML5-canvas remote backend: + pixels in, coordinates out, no structural tree, no identity seam, and no + typed delivery receipt. ``acquire_actuation_frame`` takes the one-shot + lease; the next input method consumes it and refuses when the frame content + changed in between. That refusal is the safety property the 1.28.0 change + relies on, so this fake implements it exactly and cell + ``lease_frame_changed`` exercises it. + """ + + def __init__(self, *, viewport=VIEWPORT, frame_after_lease: bytes | None = None): + self._frame = _make_png(viewport) + self._viewport = viewport + self.actions: list[tuple] = [] + self.prepared_pointer_points: list[tuple[int, int]] = [] + self.acquire_count = 0 + self._leased_frame_sha256: str | None = None + self.frame_after_lease = frame_after_lease + + @property + def viewport(self): + return self._viewport + + def screenshot(self) -> bytes: + return self._frame + + def prepare_pointer_actuation(self, x, y) -> None: + self._leased_frame_sha256 = None + self.prepared_pointer_points.append((int(x), int(y))) + + def acquire_actuation_frame(self) -> bytes: + self.acquire_count += 1 + self._leased_frame_sha256 = hashlib.sha256(self._frame).hexdigest() + if self.frame_after_lease is not None: + self._frame = self.frame_after_lease + return self._frame + + def _consume_lease(self) -> None: + leased = self._leased_frame_sha256 + self._leased_frame_sha256 = None + if leased is None: + return + if hashlib.sha256(self._frame).hexdigest() != leased: + raise RuntimeError("remote frame content changed before the input edge") + + def click(self, x, y, *, double=False) -> None: + self._consume_lease() + self.actions.append(("click", int(x), int(y), bool(double))) + + def type_text(self, text) -> None: + self._consume_lease() + self.actions.append(("type", text)) + + def press(self, key) -> None: + self._consume_lease() + self.actions.append(("press", key)) + + def scroll(self, dx, dy) -> None: + self.actions.append(("scroll", dx, dy)) + + +def _run_cell( + cell: str, + trial: int, + root: Path, + flow: dict[str, Callable[..., Any]], +) -> dict[str, Any]: + """Run one trial of one cell and record what the runtime did.""" + + ActionKind = flow["ActionKind"] + Anchor = flow["Anchor"] + Step = flow["Step"] + Workflow = flow["Workflow"] + Replayer = flow["Replayer"] + GovernedRunAuthorization = flow["GovernedRunAuthorization"] + runtime_inputs_digest = flow["runtime_inputs_digest"] + stamp_execution_outcome = flow["stamp_execution_outcome"] + + bundle = root / f"{cell}-{trial}" / "bundle" + (bundle / "templates").mkdir(parents=True) + (bundle / "templates" / "btn.png").write_bytes(_make_png((50, 20))) + run_dir = root / f"{cell}-{trial}" / "run" + + step = Step( + id="s1", + # The step is IRREVERSIBLE: this is the write, the one edge whose + # delivery decision the 1.28.0 change alters. + intent="click 'Save'", + action=ActionKind.CLICK, + anchor=Anchor( + template="templates/btn.png", + region=REGION, + click_point=CLICK_POINT, + ocr_text="Save", + landmarks=[], + ), + expect=[], + risk="irreversible", + ) + workflow = Workflow(name="remote-lease-probe", steps=[step]) + + backend = PixelOnlyRemoteBackend( + frame_after_lease=( + _make_png(color=(10, 20, 30)) if cell == "lease_frame_changed" else None + ) + ) + vision = _ScriptedVision() + vision.template_results = [ + _Match(point=CLICK_POINT, region=REGION), + _Match(point=CLICK_POINT, region=REGION), + ] + + authorization = None + if cell == "governed_lease": + # A governed run is bound to a sealed bundle, so the workflow must be + # saved and reloaded to carry a manifest digest. + workflow.save(bundle) + workflow = Workflow.load(bundle) + assert workflow.manifest is not None + authorization = GovernedRunAuthorization( + bundle_content_digest=workflow.manifest.content_digest, + runtime_inputs_digest=runtime_inputs_digest(workflow, None, None), + admitted_policy_name="remote-lease-safety-probe", + ) + + report = Replayer( + backend, + vision=vision, + governed_authorization=authorization, + ).run(workflow, bundle_dir=bundle, run_dir=run_dir) + + result = report.results[0] if report.results else None + row: dict[str, Any] = { + "cell": cell, + "trial": trial, + "governed": authorization is not None, + "replayer_success": bool(report.success), + "input_edges_delivered": len(backend.actions), + "backend_actions": [list(action) for action in backend.actions], + "prepared_pointer_points": [list(p) for p in backend.prepared_pointer_points], + "lease_acquisitions": backend.acquire_count, + "actuation_tier": getattr(result, "actuation", None) if result else None, + "delivery_receipt_present": bool( + getattr(result, "delivery_receipt", None) is not None if result else False + ), + "step_error": (getattr(result, "error", None) if result else None), + "safety_halt": bool(getattr(result, "safety_halt", False)) if result else False, + "model_calls": int(report.model_calls or 0), + "model_cost_usd": float(getattr(report, "cost_usd", 0.0) or 0.0), + "profiles": {}, + } + + # Stamp the SAME report under every profile on independent copies: the + # classifier mutates the report it is given. + for profile in PROFILES: + stamped = copy.deepcopy(report) + stamp_execution_outcome(stamped, workflow, profile) + row["profiles"][profile] = { + "execution_outcome": stamped.execution_outcome, + "transaction_outcome": stamped.transaction_outcome, + "transaction_billable": stamped.transaction_billable, + "production_eligible": bool(stamped.production_eligible), + "reported_success": bool(stamped.success), + } + return row + + +def _invariant( + identifier: str, + statement: str, + applicable: Callable[[dict[str, Any]], bool], + violated: Callable[[dict[str, Any]], bool], + rows: list[dict[str, Any]], +) -> dict[str, Any]: + applicable_rows = [row for row in rows if applicable(row)] + violations = [row for row in applicable_rows if violated(row)] + return { + "id": identifier, + "statement": statement, + "applicable_count": len(applicable_rows), + "violation_count": len(violations), + "holds": not violations, + "vacuous": not applicable_rows, + "violations": [ + { + "cell": row["cell"], + "trial": row["trial"], + "input_edges_delivered": row["input_edges_delivered"], + "profiles": row["profiles"], + } + for row in violations + ], + } + + +def evaluate_invariants(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """The point of this script. Each invariant states its own denominator.""" + + def any_profile(row: dict[str, Any], predicate: Callable[[dict], bool]) -> bool: + return any(predicate(value) for value in row["profiles"].values()) + + return [ + _invariant( + "governed_run_refuses_receiptless_remote_write", + "A governed run must refuse a consequential remote click on a " + "backend that cannot bind its exact fresh frame and target to " + "delivery, before the first input edge.", + lambda row: row["governed"], + lambda row: row["input_edges_delivered"] != 0 + or row["replayer_success"] + or not row["safety_halt"], + rows, + ), + _invariant( + "lease_delivery_is_never_a_production_success", + "A remote write delivered through the frame lease alone carries no " + "typed receipt, so no execution profile may classify it VERIFIED or " + "production-eligible.", + lambda row: True, + lambda row: any_profile( + row, + lambda value: value["transaction_outcome"] == _PRODUCTION_SUCCESS + or value["execution_outcome"] == _PRODUCTION_SUCCESS + or value["production_eligible"], + ), + rows, + ), + _invariant( + "lease_delivery_carries_no_receipt_or_actuation_tier", + "A lease-only delivery must leave the result unlabeled: no typed " + "delivery receipt and no recorded actuation tier.", + lambda row: row["input_edges_delivered"] > 0, + lambda row: row["delivery_receipt_present"] + or row["actuation_tier"] is not None, + rows, + ), + _invariant( + "changed_frame_aborts_delivery", + "The lease is the safety property: a remote frame that changed " + "between the lease and the input edge must stop delivery.", + lambda row: row["cell"] == "lease_frame_changed", + lambda row: row["input_edges_delivered"] != 0 or row["replayer_success"], + rows, + ), + _invariant( + "zero_model_calls_and_cost", + "A compiled remote replay makes no model calls and costs $0.", + lambda row: True, + lambda row: row["model_calls"] != 0 or row["model_cost_usd"] != 0.0, + rows, + ), + ] + + +def _aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: + summary: dict[str, Any] = {} + for cell in CELLS: + cell_rows = [row for row in rows if row["cell"] == cell] + if not cell_rows: + continue + summary[cell] = { + "n": len(cell_rows), + "delivered_count": sum( + 1 for row in cell_rows if row["input_edges_delivered"] > 0 + ), + "refused_count": sum( + 1 for row in cell_rows if row["input_edges_delivered"] == 0 + ), + "replayer_success_count": sum( + 1 for row in cell_rows if row["replayer_success"] + ), + "delivery_receipt_count": sum( + 1 for row in cell_rows if row["delivery_receipt_present"] + ), + "actuation_tiers": sorted( + {str(row["actuation_tier"]) for row in cell_rows} + ), + "transaction_outcomes": { + profile: sorted( + {str(row["profiles"][profile]["transaction_outcome"]) for row in cell_rows} + ) + for profile in PROFILES + }, + "model_calls_total": sum(row["model_calls"] for row in cell_rows), + "model_cost_usd_total": sum(row["model_cost_usd"] for row in cell_rows), + } + return summary + + +def _render_report(document: dict[str, Any]) -> str: + source = document["source"] + lines = [ + "# Remote frame-lease delivery: the two safety properties, measured", + "", + "openadapt-flow `c9618cc` (1.28.0) lets a consequential remote click be", + "delivered through the backend's one-shot actuation lease when no typed", + "delivery receipt exists. That relaxes a refusal, so its two stated", + "safety properties are measured here rather than trusted.", + "", + "## Source and environment", + "", + f"- Flow commit: `{source['flow']['commit']}` " + f"(version `{source['flow']['version']}`; tracked-clean source)", + f"- Release tag: `{source['flow']['release_tag']}`", + f"- Wheel SHA-256: `{source['flow']['artifact']['sha256']}`", + f"- Evals base commit: `{source['evals']['commit']}`", + f"- Runner SHA-256: `{source['runner_sha256']}`", + f"- Platform: `{document['environment']['platform']}`", + f"- Python: `{document['environment']['python']}`", + "- Network/provider use: none. No server, no browser, no model API.", + "", + "## Counted result", + "", + f"{document['trials_per_cell']} trials per cell. " + "`input edges` counts what the backend was actually asked to deliver.", + "", + "| Cell | Runs | Delivered | Refused | Receipts | Actuation tier | " + "`transaction_outcome` (demo / standard / regulated) |", + "|---|---:|---:|---:|---:|---|---|", + ] + for cell, values in document["aggregate"].items(): + outcomes = " / ".join( + ", ".join(values["transaction_outcomes"][profile]) for profile in PROFILES + ) + lines.append( + f"| `{cell}` | {values['n']} | {values['delivered_count']} | " + f"{values['refused_count']} | {values['delivery_receipt_count']} | " + f"{', '.join(values['actuation_tiers'])} | {outcomes} |" + ) + lines += [ + "", + "## Invariants", + "", + "Every invariant states the denominator it was evaluated over. A " + "`vacuous` invariant had no applicable run and proves nothing.", + "", + "| Invariant | Applicable runs | Violations | Holds |", + "|---|---:|---:|---|", + ] + for invariant in document["invariants"]: + verdict = ( + "vacuous" + if invariant["vacuous"] + else ("yes" if invariant["holds"] else "**NO**") + ) + lines.append( + f"| {invariant['statement']} | {invariant['applicable_count']} | " + f"{invariant['violation_count']} | {verdict} |" + ) + lines += [ + "", + "## Scope", + "", + "- The backend is a fake implementing ONLY the two-phase remote", + " actuation lease, which is the exact protocol surface a pixel-only", + " no-DOM canvas backend exposes. This measures a runtime contract, not", + " a real Citrix or RDP session.", + "- One synthetic single-step workflow whose only step is the", + " irreversible write. Resolution is scripted to one fixed point so the", + " delivery decision is the only thing that varies between cells.", + "- No claim is made here about wrong-target immunity on a real remote", + " surface, about identity coverage, or about any hosted lifecycle.", + "", + f"Reproduce: `{document['reproduce']}`", + "", + ] + return "\n".join(lines) + + +def probe(flow_source: Path, flow_wheel: Path, out_dir: Path, *, trials: int = 3) -> dict[str, Any]: + if trials < 3: + raise ValueError("comparative evaluation requires at least three trials") + flow_source = flow_source.resolve() + flow_wheel = flow_wheel.resolve() + out_dir = out_dir.resolve() + if not (flow_source / "openadapt_flow").is_dir(): + raise ValueError(f"not an openadapt-flow source checkout: {flow_source}") + if not flow_wheel.is_file() or flow_wheel.suffix != ".whl": + raise ValueError(f"not a Flow wheel: {flow_wheel}") + + source = _source_binding(flow_source) + artifact_temp = tempfile.TemporaryDirectory(prefix="oa-flow-wheel-") + artifact_root = Path(artifact_temp.name).resolve() + _extract_wheel(flow_wheel, artifact_root) + sys.path.insert(0, str(artifact_root)) + + from openadapt_flow.execution_profiles import stamp_execution_outcome # noqa: PLC0415 + from openadapt_flow.ir import ( # noqa: PLC0415 + ActionKind, + Anchor, + Step, + Workflow, + ) + from openadapt_flow.runtime.authorization import ( # noqa: PLC0415 + GovernedRunAuthorization, + runtime_inputs_digest, + ) + from openadapt_flow.runtime.replayer import Replayer # noqa: PLC0415 + + imported = Path(sys.modules["openadapt_flow"].__file__).resolve() + if artifact_root not in imported.parents: + raise RuntimeError(f"Flow import escaped requested wheel: {imported}") + flow_version = str(sys.modules["openadapt_flow"].__version__) + release_tag = f"v{flow_version}" + if release_tag not in source["tags"]: + raise RuntimeError(f"source commit is not tagged {release_tag}: {source['tags']}") + + flow = { + "ActionKind": ActionKind, + "Anchor": Anchor, + "Step": Step, + "Workflow": Workflow, + "Replayer": Replayer, + "GovernedRunAuthorization": GovernedRunAuthorization, + "runtime_inputs_digest": runtime_inputs_digest, + "stamp_execution_outcome": stamp_execution_outcome, + } + + evals_root = Path(__file__).resolve().parents[1] + evals_commit = _git(evals_root, "rev-parse", "HEAD") + out_dir.mkdir(parents=True, exist_ok=False) + + rows: list[dict[str, Any]] = [] + with tempfile.TemporaryDirectory(prefix="oa-remote-lease-probe-") as tmp: + root = Path(tmp) + for cell in CELLS: + for trial in range(1, trials + 1): + row = _run_cell(cell, trial, root, flow) + rows.append(row) + print( + f"{cell} trial={trial}: delivered={row['input_edges_delivered']} " + f"success={row['replayer_success']} " + f"actuation={row['actuation_tier']} " + f"receipt={row['delivery_receipt_present']} " + f"standard={row['profiles']['standard']['transaction_outcome']}" + ) + + document = { + "schema_version": 1, + "scope": "remote frame-lease delivery safety properties (runtime contract)", + "generated_at": datetime.now(timezone.utc).isoformat(), + "trials_per_cell": trials, + "cells": list(CELLS), + "profiles": list(PROFILES), + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + }, + "source": { + "evals": {"commit": evals_commit}, + "flow": { + "commit": source["commit"], + "tags": source["tags"], + "tracked_clean": source["tracked_clean"], + "version": flow_version, + "release_tag": release_tag, + "artifact": { + "filename": flow_wheel.name, + "sha256": _sha256(flow_wheel), + "import_mode": "locally extracted published wheel", + }, + }, + "runner_sha256": _sha256(Path(__file__).resolve()), + }, + "paid_or_remote_mutations": [], + "runs": rows, + "aggregate": _aggregate(rows), + "invariants": evaluate_invariants(rows), + "caveats": [ + "The backend is a fake implementing only the two-phase remote " + "actuation lease; this measures a runtime contract, not a real " + "remote session.", + "One synthetic single-step workflow; resolution is scripted so the " + "delivery decision is the only variable.", + "No wrong-target immunity, identity coverage, or hosted lifecycle " + "claim is made here.", + ], + "reproduce": ( + "python scripts/probe_remote_lease_safety.py " + f"--flow-source " + f"--flow-wheel <{flow_wheel.name}> " + "--out " + ), + } + (out_dir / "results.json").write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + (out_dir / "REPORT.md").write_text(_render_report(document), encoding="utf-8") + print(f"Wrote {out_dir / 'results.json'} and {out_dir / 'REPORT.md'}") + artifact_temp.cleanup() + return document + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Measure the governed refusal and the never-VERIFIED property of " + "Flow's remote frame-lease delivery." + ) + ) + parser.add_argument("--flow-source", type=Path, required=True) + parser.add_argument("--flow-wheel", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--trials", type=int, default=3) + parser.add_argument( + "--no-fail-on-violation", + action="store_true", + help="Write the artifacts and exit 0 even when an invariant is violated.", + ) + args = parser.parse_args(argv) + document = probe(args.flow_source, args.flow_wheel, args.out, trials=args.trials) + violated = [item for item in document["invariants"] if not item["holds"]] + for item in violated: + print( + f"VIOLATED: {item['id']}: {item['violation_count']} of " + f"{item['applicable_count']} applicable runs", + file=sys.stderr, + ) + if violated and not args.no_fail_on_violation: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())