From 352af585b04648473e50ed86cd8983bd0da69a08 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 13:20:27 -0400 Subject: [PATCH 1/2] bench(flow): measure the referral's API step under each_item approval + durable benchmark/openemr_api_tier/approval_durable.py runs the OpenEMR referral with API steps off and on, both under ApprovalGate("each_item") and a durable run, interleaved, with the setup's own SQL read-back. The harness answers each approval as a stand-in for the person and its wait is measured and excluded. Every number it writes carries the label "API step under each_item approval + durable, no run gate; harness answers approvals; approval wait excluded": the referral bundle doesn't pass the run gate yet, so this is not the governed production profile. The approval view now lists each record-check selector once: the real bundle's 11 record checks select the same record, which repeated the rows 11 times and pushed the phone view past its row limit. Co-Authored-By: Claude Opus 5.5 --- .../openemr_api_tier/approval_durable.py | 366 ++++++++++++++++++ .../flow/openadapt_flow/runtime/replayer.py | 14 +- packages/flow/tests/test_api_step_approval.py | 27 ++ 3 files changed, 402 insertions(+), 5 deletions(-) create mode 100644 packages/flow/benchmark/openemr_api_tier/approval_durable.py diff --git a/packages/flow/benchmark/openemr_api_tier/approval_durable.py b/packages/flow/benchmark/openemr_api_tier/approval_durable.py new file mode 100644 index 000000000..9094af0d8 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/approval_durable.py @@ -0,0 +1,366 @@ +"""The OpenEMR referral with the API step under each_item approval and a durable run. + +Label every number from this harness exactly as: "API step under each_item +approval + durable, no run gate; harness answers approvals; approval wait +excluded". It is not the governed production profile. The referral bundle +doesn't pass the run gate yet (GUI-path admission; see docs/API_TIER.md and +the admission list in the session notes), so this measures the parts that +do run: the person approving the exact request (docs/APPROVAL_MODES.md) and +the durable checkpoints (docs/API_TIER.md "In a durable run"). + +- Arms: ``browser`` (API steps off) and ``api`` (API steps on), both under + ``ApprovalGate("each_item")`` with ``durable=True``. Interleaved blocks; each + block resets the fixture and seeds the chart r6 must not duplicate. +- Approvals: the harness stands in for the person and answers each request + as soon as it is written (``decided_by="human"``, route ``desktop_run_it``, + approver ``harness-approver``). Time spent waiting for the answer is + measured and subtracted: ``engine_s`` excludes it, ``approval_wait_s`` is it. +- Read-back: the test setup's own direct SQL check (``common.record_check``), + never the arm under test. r6 is right when no chart is created. +- Load: each item waits until the 1-minute load is under GATE_LOAD1 (12). + + PYTHONPATH=packages/flow python -m benchmark.openemr_api_tier.approval_durable \\ + compile --recording --out + PYTHONPATH=packages/flow python -m benchmark.openemr_api_tier.approval_durable \\ + run --bundle --out [--blocks 5 --per-block 4] +""" + +from __future__ import annotations + +import argparse +import csv +import json +import shutil +import statistics +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +from benchmark.openemr_api_tier.common import ( + REFERRALS, + load_demo, + record_check, + reset_and_seed, + spec_for, + sql_patients, +) +from benchmark.openemr_api_tier.feature_e2e import NO_MATCH_TEXT, _load1, wait_quiet +from benchmark.openemr_api_tier.spike_fixture import SpikeFixture + +LABEL = ( + "API step under each_item approval + durable, no run gate; harness answers " + "approvals; approval wait excluded" +) +APPROVER = "harness-approver" +COLS = [ + "arm", + "block", + "pos", + "ref", + "mismatch", + "correct", + "readback", + "new_charts", + "engine_success", + "transaction_outcome", + "actuation", + "api_fallback", + "approvals", + "approval_kinds", + "approval_wait_s", + "engine_s", + "wall_s", + "checkpoints", + "error", + "load_start", + "load_end", + "gate_waited_s", + "engine_sha", +] + + +def cmd_compile(a: argparse.Namespace) -> None: + """The harness compile (feature_e2e), admitted with a right-record parameter.""" + + from openadapt_flow.compiler.api_proposal import ( + admit_api_step, + describe_api_step, + propose_api_step, + ) + from openadapt_flow.ir import Postcondition, PostconditionKind, Workflow + + demo = load_demo() + rec, bundle = Path(a.recording), Path(a.out) + demo.compile_recording(rec, bundle) + wf = Workflow.load(bundle) + dialog = [ + s + for s in wf.steps + if s.anchor is not None + and s.anchor.structural is not None + and s.anchor.structural.selector == "#create" + ] + assert len(dialog) == 1 + dialog[0].expect = list(dialog[0].expect) + [ + Postcondition( + kind=PostconditionKind.TEXT_PRESENT, text=NO_MATCH_TEXT, timeout_s=10.0 + ) + ] + wf, step, reasons = propose_api_step(wf, rec, replace=True) + if step is None: + raise SystemExit(f"no API step: {reasons}") + # The harness admits under the fixture's name so the matrix runs + # unattended; in use a person admits with `openadapt-flow api admit`. + wf, _ = admit_api_step( + wf, admitted_by="benchmark-fixture", identity={"subject_name": "lname"} + ) + wf.save(bundle) + print(describe_api_step(next(s for s in wf.steps if s.id == step.id))) + + +class HarnessPerson: + """Answers every waiting approval request at once, and times the wait.""" + + def __init__(self, store: Any) -> None: + self.store = store + self.waited_s = 0.0 + self.answered: list[str] = [] + + def sleep(self, seconds: float) -> None: + from openadapt_flow.approval_mode.item_approval import make_answer + + t0 = time.monotonic() + for request in self.store.requests(): + if request.request_id in self.answered: + continue + if self.store.status(request) != "waiting": + continue + self.store.write_answer( + make_answer( + request, + decision="approve", + decided_by="human", + route="desktop_run_it", + approver_id=APPROVER, + auth_method="cloud_aal2", + ) + ) + self.answered.append(request.request_id) + time.sleep(min(seconds, 0.02)) + self.waited_s += time.monotonic() - t0 + + +def run_one( + env: dict[str, Any], arm: str, ref_idx: int, bundle: Path, out: Path +) -> dict[str, Any]: + from openadapt_flow.approval_mode.gate import ApprovalGate + from openadapt_flow.approval_mode.item_approval import ItemApprovalStore + from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from openadapt_flow.ir import Workflow + from openadapt_flow.runtime import Replayer + from openadapt_flow.runtime.durable.checkpoint import CheckpointStore + + demo, f, browser, ocr = env["demo"], env["f"], env["browser"], env["ocr"] + ref = REFERRALS[ref_idx - 1] + spec = spec_for(ref) + out.mkdir(parents=True) + engine_dir = out / "engine" + before_ids = {r["id"] for r in sql_patients(f)} + oracle = OpenEMRPatientOracle(f.api_base_url, f.token_session("oracle"), spec) + res: dict[str, Any] = { + "arm": arm, + "ref": ref.key, + "mismatch": ref.mismatch, + "error": None, + } + store = ItemApprovalStore(engine_dir) + person = HarnessPerson(store) + gate = ApprovalGate( + "each_item", + approval_store=store, + approval_wait_s=120.0, + approval_poll_s=0.05, + sleep=person.sleep, + monotonic=time.monotonic, + ) + t0 = time.time() + ctx = browser.new_context( + viewport={"width": 1280, "height": 800}, device_scale_factor=1 + ) + try: + page = ctx.new_page() + page.goto(f.ui_base_url) + backend = demo._BasePlaywrightBackend(page) + demo._preauthenticate_browser(backend, f) + t_setup = time.time() + ocr.clear_ocr_memo() + rep = Replayer( + backend, + effect_verifier=oracle.verifier, + api_steps=(arm == "api"), + approval_gate=gate, + durable=True, + ).run( + Workflow.load(bundle), + params=spec.params(), + bundle_dir=bundle, + run_dir=engine_dir, + ) + t_task = time.time() + raw = rep.model_dump(mode="json") + last = raw["results"][-1] if raw["results"] else {} + res.update( + engine_success=raw.get("success"), + transaction_outcome=raw.get("transaction_outcome"), + actuation=last.get("actuation"), + api_fallback=next( + (r["api_fallback"] for r in raw["results"] if r.get("api_fallback")), + None, + ), + step_errors=[ + (r["step_id"], (r.get("error") or "")[:300]) + for r in raw["results"] + if r.get("error") + ], + ) + except Exception as exc: # noqa: BLE001 + t_setup = locals().get("t_setup", time.time()) + t_task = time.time() + res["error"] = f"{type(exc).__name__}: {exc}"[:500] + time.sleep(demo.ORACLE_SETTLE_S) + res.update(record_check(f, ref, before_ids)) + t_end = time.time() + ctx.close() + requests = store.requests() + res.update( + approvals=len(requests), + approval_kinds="+".join(r.form_state_kind for r in requests), + approval_wait_s=round(person.waited_s, 3), + engine_s=round(t_task - t_setup - person.waited_s, 3), + wall_s=round(t_end - t0, 3), + checkpoints=len(CheckpointStore(engine_dir).checkpoints()), + ) + (out / "result.json").write_text(json.dumps(res, indent=1, default=str)) + shutil.rmtree(engine_dir / "steps", ignore_errors=True) + return res + + +def cmd_run(a: argparse.Namespace) -> None: + from playwright.sync_api import sync_playwright + + import openadapt_flow.vision.ocr # noqa: F401 + + ocr = sys.modules["openadapt_flow.vision.ocr"] + out, bundle = Path(a.out), Path(a.bundle) + out.mkdir(parents=True, exist_ok=True) + sha = subprocess.run( + ["git", "rev-parse", "HEAD"], capture_output=True, text=True + ).stdout.strip()[:9] + demo, f = load_demo(), SpikeFixture() + pw = sync_playwright().start() + browser = pw.chromium.launch(headless=True) + ocr._get_engine() + env = {"demo": demo, "f": f, "browser": browser, "ocr": ocr} + csv_path = out / "runs.csv" + rows: list[dict[str, Any]] = [] + try: + for b in range(a.blocks): + for arm in ["browser", "api"] if b % 2 == 0 else ["api", "browser"]: + reset_and_seed(f) + for pos in range(a.per_block): + idx = ((b * a.per_block + pos) % len(REFERRALS)) + 1 + gate = wait_quiet() + load_start = _load1() + res = run_one( + env, + arm, + idx, + bundle, + out + / "items" + / f"b{b + 1:02d}-{arm}-{pos + 1}-{REFERRALS[idx - 1].key}", + ) + row = {k: res.get(k) for k in COLS} | { + "block": b + 1, + "pos": pos + 1, + "load_start": load_start, + "load_end": _load1(), + "gate_waited_s": gate["gate_waited_s"], + "engine_sha": sha, + } + rows.append(row) + print( + json.dumps( + { + k: row[k] + for k in ( + "arm", + "block", + "ref", + "correct", + "actuation", + "approvals", + "engine_s", + "approval_wait_s", + "error", + ) + } + ), + flush=True, + ) + finally: + browser.close() + pw.stop() + with open(csv_path, "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=COLS) + w.writeheader() + w.writerows(rows) + (out / "summary.json").write_text( + json.dumps(summarize(rows, sha), indent=1) + "\n" + ) + + +def summarize(rows: list[dict[str, Any]], sha: str) -> dict[str, Any]: + """Per arm: correctness by read-back, engine time, approvals.""" + + out: dict[str, Any] = {"label": LABEL, "engine_sha": sha, "arms": {}} + for arm in ("browser", "api"): + mine = [r for r in rows if r["arm"] == arm] + if not mine: + continue + times = [r["engine_s"] for r in mine if r["engine_s"] is not None] + out["arms"][arm] = { + "runs": len(mine), + "correct": sum(bool(r["correct"]) for r in mine), + "errors": sum(bool(r["error"]) for r in mine), + "api_saves": sum(r["actuation"] == "api" for r in mine), + "approvals_per_item": sorted({r["approvals"] for r in mine}), + "approval_kinds": sorted({r["approval_kinds"] for r in mine}), + "median_engine_s": round(statistics.median(times), 2) if times else None, + "median_approval_wait_s": round( + statistics.median(r["approval_wait_s"] for r in mine), 3 + ), + "max_load_start": max(r["load_start"] for r in mine), + } + return out + + +def main() -> None: + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("compile") + c.add_argument("--recording", required=True) + c.add_argument("--out", required=True) + r = sub.add_parser("run") + r.add_argument("--bundle", required=True) + r.add_argument("--out", required=True) + r.add_argument("--blocks", type=int, default=5) + r.add_argument("--per-block", type=int, default=4) + a = ap.parse_args() + {"compile": cmd_compile, "run": cmd_run}[a.cmd](a) + + +if __name__ == "__main__": + main() diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 4f3a94f57..0314f4a2b 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -11778,11 +11778,15 @@ def _web_form_approval_view( if isinstance(prepared, ApiActuationResult): return prepared.reason or "the API step is unavailable" effects = self._resolve_effects(list(binding.effects), params) - record = [ - (key, str(expr)) - for effect in effects - for key, expr in sorted(effect.match.items()) - ] + # Several record checks usually select the same record (one per + # field): the person reads, and the digest binds, each pair once. + record = list( + dict.fromkeys( + (key, str(expr)) + for effect in effects + for key, expr in sorted(effect.match.items()) + ) + ) digest = operation_digest( prepared, record_check=[effect.contract_hash() for effect in effects], diff --git a/packages/flow/tests/test_api_step_approval.py b/packages/flow/tests/test_api_step_approval.py index 98f69e5e7..61a4a1893 100644 --- a/packages/flow/tests/test_api_step_approval.py +++ b/packages/flow/tests/test_api_step_approval.py @@ -315,3 +315,30 @@ def test_operation_digest_binds_the_record_check(): ) assert base != operation_digest(changed, record_check=["h1"], record=[("id", "p1")]) assert up.parse_qs(prepared.encoded()) == {"lname": ["Marsh"]} + + +def test_the_record_rows_are_listed_once_per_field(tmp_path): + from openadapt_flow.runtime.effects import Effect, EffectKind, ValueExpr + + wf = _workflow() + save = wf.steps[-1] + match = {"patient_id": ValueExpr(param="lname")} + effects = [ + Effect(kind=EffectKind.RECORD_WRITTEN, match=match, expected_count=1), + Effect( + kind=EffectKind.FIELD_EQUALS, + match=match, + field="note", + value=ValueExpr(literal="Referred"), + ), + ] + binding = save.api_binding.model_copy(update={"effects": effects}) + save = save.model_copy(update={"effects": effects, "api_binding": binding}) + replayer = Replayer( + EnterWritesBackend("http://127.0.0.1:9"), + vision=FakeVision(), + web_form_actuator=WebFormActuator(SessionTransport("http://127.0.0.1:9")), + ) + view = replayer._web_form_approval_view(save, PARAMS) + record_rows = [row for row in view.rows if row[0].startswith("Record check:")] + assert record_rows == [("Record check: patient_id", "Marsh")] From b983d512c585d9e7588bb35599d1ff7669f0c94d Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 14:50:09 -0400 Subject: [PATCH 2/2] bench(flow): referral API step under each_item approval + durable, 20 runs per arm Label: API step under each_item approval + durable, no run gate; harness answers approvals; approval wait excluded. Not the governed production profile: the referral bundle doesn't pass the run gate yet on its GUI path. On the local OpenEMR test copy (synthetic patients), interleaved blocks: - API step: 20/20 correct by SQL read-back, 18 saved through the API (r6, the possible duplicate, fell back and created nothing, twice), median 1.12 s engine time, 1 approval per item. - API steps off: 19/20 correct plus 1 fail-closed safe stop (nothing saved), median 15.95 s, 2 approvals per new referral. - No wrong or duplicate chart. The run paused for a shared quiet window after 19 runs and resumed into a second directory; approval_durable.py gains --from-block, --skip-done and a summarize command that rebuilds runs.csv and summary.json from the items. Only the CSV, the summary and each item's result.json are committed (the 21 MB of screenshots and engine reports are kept outside the repository). Co-Authored-By: Claude Opus 5.5 --- .../openemr_api_tier/approval_durable.py | 85 ++++++++- .../items/b01-api-1-r1-nadia-brennan.json | 94 ++++++++++ .../items/b01-api-2-r2-marcus-feld.json | 94 ++++++++++ .../items/b01-api-3-r3-ines-carvalho.json | 94 ++++++++++ .../items/b01-api-4-r4-theo-lindqvist.json | 94 ++++++++++ .../items/b01-browser-1-r1-nadia-brennan.json | 94 ++++++++++ .../items/b01-browser-2-r2-marcus-feld.json | 99 +++++++++++ .../items/b01-browser-3-r3-ines-carvalho.json | 94 ++++++++++ .../b01-browser-4-r4-theo-lindqvist.json | 94 ++++++++++ .../items/b02-api-1-r5-grace-adeyemi.json | 94 ++++++++++ .../items/b02-api-2-r6-rosa-delgado.json | 99 +++++++++++ .../items/b02-api-3-r7-arlo-whitcombe.json | 94 ++++++++++ .../items/b02-api-4-r8-priya-raman.json | 94 ++++++++++ .../items/b02-browser-1-r5-grace-adeyemi.json | 94 ++++++++++ .../items/b02-browser-2-r6-rosa-delgado.json | 99 +++++++++++ .../b02-browser-3-r7-arlo-whitcombe.json | 94 ++++++++++ .../items/b02-browser-4-r8-priya-raman.json | 94 ++++++++++ .../items/b03-api-1-r1-nadia-brennan.json | 94 ++++++++++ .../items/b03-api-2-r2-marcus-feld.json | 94 ++++++++++ .../items/b03-api-3-r3-ines-carvalho.json | 94 ++++++++++ .../items/b03-api-4-r4-theo-lindqvist.json | 94 ++++++++++ .../items/b03-browser-1-r1-nadia-brennan.json | 94 ++++++++++ .../items/b03-browser-2-r2-marcus-feld.json | 94 ++++++++++ .../items/b03-browser-3-r3-ines-carvalho.json | 94 ++++++++++ .../b03-browser-4-r4-theo-lindqvist.json | 94 ++++++++++ .../items/b04-api-1-r5-grace-adeyemi.json | 94 ++++++++++ .../items/b04-api-2-r6-rosa-delgado.json | 99 +++++++++++ .../items/b04-api-3-r7-arlo-whitcombe.json | 94 ++++++++++ .../items/b04-api-4-r8-priya-raman.json | 94 ++++++++++ .../items/b04-browser-1-r5-grace-adeyemi.json | 94 ++++++++++ .../items/b04-browser-2-r6-rosa-delgado.json | 99 +++++++++++ .../b04-browser-3-r7-arlo-whitcombe.json | 94 ++++++++++ .../items/b04-browser-4-r8-priya-raman.json | 94 ++++++++++ .../items/b05-api-1-r1-nadia-brennan.json | 94 ++++++++++ .../items/b05-api-2-r2-marcus-feld.json | 94 ++++++++++ .../items/b05-api-3-r3-ines-carvalho.json | 94 ++++++++++ .../items/b05-api-4-r4-theo-lindqvist.json | 94 ++++++++++ .../items/b05-browser-1-r1-nadia-brennan.json | 94 ++++++++++ .../items/b05-browser-2-r2-marcus-feld.json | 94 ++++++++++ .../items/b05-browser-3-r3-ines-carvalho.json | 94 ++++++++++ .../b05-browser-4-r4-theo-lindqvist.json | 94 ++++++++++ .../2026-10-19-combined/runs.csv | 41 +++++ .../2026-10-19-combined/summary.json | 43 +++++ .../results/approval_durable/README.md | 55 ++++++ packages/flow/docs/API_TIER.md | 6 + packages/flow/public-artifacts.json | 168 ++++++++++++++++++ 46 files changed, 4180 insertions(+), 3 deletions(-) create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-1-r5-grace-adeyemi.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-2-r6-rosa-delgado.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-3-r7-arlo-whitcombe.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-4-r8-priya-raman.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-1-r5-grace-adeyemi.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-2-r6-rosa-delgado.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-3-r7-arlo-whitcombe.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-4-r8-priya-raman.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-1-r5-grace-adeyemi.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-2-r6-rosa-delgado.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-3-r7-arlo-whitcombe.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-4-r8-priya-raman.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-1-r5-grace-adeyemi.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-2-r6-rosa-delgado.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-3-r7-arlo-whitcombe.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-4-r8-priya-raman.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/runs.csv create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/summary.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/approval_durable/README.md diff --git a/packages/flow/benchmark/openemr_api_tier/approval_durable.py b/packages/flow/benchmark/openemr_api_tier/approval_durable.py index 9094af0d8..7b3e57f92 100644 --- a/packages/flow/benchmark/openemr_api_tier/approval_durable.py +++ b/packages/flow/benchmark/openemr_api_tier/approval_durable.py @@ -265,12 +265,30 @@ def cmd_run(a: argparse.Namespace) -> None: env = {"demo": demo, "f": f, "browser": browser, "ocr": ocr} csv_path = out / "runs.csv" rows: list[dict[str, Any]] = [] + done = ( + { + d.name + for d in (Path(a.skip_done) / "items").iterdir() + if (d / "result.json").is_file() + } + if a.skip_done + else set() + ) try: - for b in range(a.blocks): + for b in range(a.from_block - 1, a.blocks): for arm in ["browser", "api"] if b % 2 == 0 else ["api", "browser"]: + names = [ + f"b{b + 1:02d}-{arm}-{pos + 1}-" + f"{REFERRALS[(b * a.per_block + pos) % len(REFERRALS)].key}" + for pos in range(a.per_block) + ] + if all(n in done for n in names): + continue reset_and_seed(f) for pos in range(a.per_block): idx = ((b * a.per_block + pos) % len(REFERRALS)) + 1 + if names[pos] in done: + continue gate = wait_quiet() load_start = _load1() res = run_one( @@ -342,14 +360,71 @@ def summarize(rows: list[dict[str, Any]], sha: str) -> dict[str, Any]: "median_approval_wait_s": round( statistics.median(r["approval_wait_s"] for r in mine), 3 ), - "max_load_start": max(r["load_start"] for r in mine), + "max_load_start": max( + (r["load_start"] for r in mine if r["load_start"] is not None), + default=None, + ), + # Stopped before saving, nothing written (a fail-closed halt or a + # refused duplicate): not a wrong save. + "safe_stops": sum( + not r["correct"] and r.get("new_charts") == 0 for r in mine + ), + "wrong_saves": sum( + not r["correct"] and (r.get("new_charts") or 0) > 0 for r in mine + ), } return out +def rows_from_items(out: Path, sha: str) -> list[dict[str, Any]]: + """Rebuild the rows from each item's result.json (an interrupted run).""" + + rows = [] + for item in sorted((out / "items").iterdir()): + result = item / "result.json" + if not result.is_file(): + continue # interrupted before the item finished + res = json.loads(result.read_text()) + block, arm, pos = item.name.split("-")[:3] + rows.append( + {k: res.get(k) for k in COLS} + | { + "block": int(block[1:]), + "pos": int(pos), + "load_start": None, + "load_end": None, + "gate_waited_s": None, + "engine_sha": sha, + } + ) + return rows + + +def cmd_summarize(a: argparse.Namespace) -> None: + """Combine the finished items of one or more --out directories into --to.""" + + out = Path(a.to) + out.mkdir(parents=True, exist_ok=True) + rows = [row for d in a.out for row in rows_from_items(Path(d), a.engine_sha)] + with open(out / "runs.csv", "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=COLS) + w.writeheader() + w.writerows(rows) + (out / "summary.json").write_text( + json.dumps(summarize(rows, a.engine_sha), indent=1) + "\n" + ) + print(json.dumps(summarize(rows, a.engine_sha), indent=1)) + + def main() -> None: ap = argparse.ArgumentParser() sub = ap.add_subparsers(dest="cmd", required=True) + sm = sub.add_parser( + "summarize", help="Rebuild runs.csv and summary.json from items/" + ) + sm.add_argument("--out", required=True, action="append") + sm.add_argument("--to", required=True) + sm.add_argument("--engine-sha", required=True) c = sub.add_parser("compile") c.add_argument("--recording", required=True) c.add_argument("--out", required=True) @@ -358,8 +433,12 @@ def main() -> None: r.add_argument("--out", required=True) r.add_argument("--blocks", type=int, default=5) r.add_argument("--per-block", type=int, default=4) + r.add_argument("--from-block", type=int, default=1) + r.add_argument( + "--skip-done", default=None, help="An earlier --out; skip its finished items" + ) a = ap.parse_args() - {"compile": cmd_compile, "run": cmd_run}[a.cmd](a) + {"compile": cmd_compile, "run": cmd_run, "summarize": cmd_summarize}[a.cmd](a) if __name__ == "__main__": diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-1-r1-nadia-brennan.json new file mode 100644 index 000000000..f716662ab --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.023, + "engine_s": 1.114, + "wall_s": 4.369, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-2-r2-marcus-feld.json new file mode 100644 index 000000000..21282d886 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-2-r2-marcus-feld.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.023, + "engine_s": 1.092, + "wall_s": 3.806, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-3-r3-ines-carvalho.json new file mode 100644 index 000000000..96a6aa09f --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.021, + "engine_s": 1.053, + "wall_s": 3.758, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..b7d10d91c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.026, + "engine_s": 1.1, + "wall_s": 3.783, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-1-r1-nadia-brennan.json new file mode 100644 index 000000000..78095f21c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.043, + "engine_s": 18.324, + "wall_s": 21.29, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-2-r2-marcus-feld.json new file mode 100644 index 000000000..1f8d8bbe9 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-2-r2-marcus-feld.json @@ -0,0 +1,99 @@ +{ + "arm": "browser", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "actuation": null, + "api_fallback": "API steps are off for this run", + "step_errors": [ + [ + "step_040", + "Identity check failed for step 'step_040' (click recorded visual target): a target was found positionally (rung 'structural', confidence 1.00) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Emergency Contact: Confirm Create New Patient Cance" + ] + ], + "new_charts": 0, + "readback": "0/12", + "correct": false, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Marcus", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Feld", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Male", + "got": null, + "match": false + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0124", + "got": null, + "match": false + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.048, + "engine_s": 14.588, + "wall_s": 17.402, + "checkpoints": 40 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-3-r3-ines-carvalho.json new file mode 100644 index 000000000..df48c52c9 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.05, + "engine_s": 16.038, + "wall_s": 18.852, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..43307b2f2 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.049, + "engine_s": 15.285, + "wall_s": 18.056, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..d6f3067dc --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-1-r5-grace-adeyemi.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.025, + "engine_s": 1.188, + "wall_s": 4.307, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-2-r6-rosa-delgado.json new file mode 100644 index 000000000..fa684296b --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-2-r6-rosa-delgado.json @@ -0,0 +1,99 @@ +{ + "arm": "api", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "engine_success": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "actuation": "dom", + "api_fallback": "the check before saving (step_039) did not pass: missing ['No matches were found.']; the save was not sent", + "step_errors": [ + [ + "step_039", + "Postconditions failed for step 'step_039' (click recorded visual target): expected screen state not reached (semantic drift) \u2014 failed: text_present 'No matches were found.' \u2014 run aborted" + ] + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "approvals": 1, + "approval_kinds": "dom", + "approval_wait_s": 0.026, + "engine_s": 22.666, + "wall_s": 25.59, + "checkpoints": 39 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..8c2bd8ca1 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-3-r7-arlo-whitcombe.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.03, + "engine_s": 1.125, + "wall_s": 3.803, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-4-r8-priya-raman.json new file mode 100644 index 000000000..5d47989f4 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-4-r8-priya-raman.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.025, + "engine_s": 1.152, + "wall_s": 3.842, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..b13c98f80 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-1-r5-grace-adeyemi.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.055, + "engine_s": 18.26, + "wall_s": 21.718, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-2-r6-rosa-delgado.json new file mode 100644 index 000000000..2c2218b94 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-2-r6-rosa-delgado.json @@ -0,0 +1,99 @@ +{ + "arm": "browser", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "engine_success": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [ + [ + "step_039", + "Postconditions failed for step 'step_039' (click recorded visual target): expected screen state not reached (semantic drift) \u2014 failed: text_present 'No matches were found.' \u2014 run aborted" + ] + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "approvals": 1, + "approval_kinds": "dom", + "approval_wait_s": 0.031, + "engine_s": 24.871, + "wall_s": 28.107, + "checkpoints": 39 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..ee05a2c9e --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-3-r7-arlo-whitcombe.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.047, + "engine_s": 16.847, + "wall_s": 19.804, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-4-r8-priya-raman.json new file mode 100644 index 000000000..5cfcac1c3 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-4-r8-priya-raman.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.048, + "engine_s": 15.714, + "wall_s": 18.419, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-1-r1-nadia-brennan.json new file mode 100644 index 000000000..da326917a --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.027, + "engine_s": 1.822, + "wall_s": 5.114, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-2-r2-marcus-feld.json new file mode 100644 index 000000000..8dc00a019 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-2-r2-marcus-feld.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.028, + "engine_s": 1.102, + "wall_s": 3.921, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-3-r3-ines-carvalho.json new file mode 100644 index 000000000..e06609ff5 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.031, + "engine_s": 1.081, + "wall_s": 3.904, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..86214e809 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.025, + "engine_s": 1.106, + "wall_s": 3.773, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-1-r1-nadia-brennan.json new file mode 100644 index 000000000..fe0d8e75c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.047, + "engine_s": 15.583, + "wall_s": 18.693, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-2-r2-marcus-feld.json new file mode 100644 index 000000000..59874bcad --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-2-r2-marcus-feld.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.052, + "engine_s": 15.871, + "wall_s": 18.792, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-3-r3-ines-carvalho.json new file mode 100644 index 000000000..6ee11a79c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.047, + "engine_s": 22.049, + "wall_s": 25.165, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..d485d931d --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.053, + "engine_s": 15.59, + "wall_s": 18.753, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..e0ac70c6c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-1-r5-grace-adeyemi.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.028, + "engine_s": 1.082, + "wall_s": 4.212, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-2-r6-rosa-delgado.json new file mode 100644 index 000000000..d7d80d8e3 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-2-r6-rosa-delgado.json @@ -0,0 +1,99 @@ +{ + "arm": "api", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "engine_success": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "actuation": "dom", + "api_fallback": "the check before saving (step_039) did not pass: missing ['No matches were found.']; the save was not sent", + "step_errors": [ + [ + "step_039", + "Postconditions failed for step 'step_039' (click recorded visual target): expected screen state not reached (semantic drift) \u2014 failed: text_present 'No matches were found.' \u2014 run aborted" + ] + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "approvals": 1, + "approval_kinds": "dom", + "approval_wait_s": 0.03, + "engine_s": 22.625, + "wall_s": 25.706, + "checkpoints": 39 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..a7173128c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-3-r7-arlo-whitcombe.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.03, + "engine_s": 1.123, + "wall_s": 3.806, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-4-r8-priya-raman.json new file mode 100644 index 000000000..5d8d09a2f --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-4-r8-priya-raman.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.027, + "engine_s": 1.057, + "wall_s": 3.865, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..986266edf --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-1-r5-grace-adeyemi.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.048, + "engine_s": 15.514, + "wall_s": 18.949, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-2-r6-rosa-delgado.json new file mode 100644 index 000000000..901860315 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-2-r6-rosa-delgado.json @@ -0,0 +1,99 @@ +{ + "arm": "browser", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "engine_success": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [ + [ + "step_039", + "Postconditions failed for step 'step_039' (click recorded visual target): expected screen state not reached (semantic drift) \u2014 failed: text_present 'No matches were found.' \u2014 run aborted" + ] + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "approvals": 1, + "approval_kinds": "dom", + "approval_wait_s": 0.022, + "engine_s": 22.555, + "wall_s": 25.298, + "checkpoints": 39 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..858ce9b65 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-3-r7-arlo-whitcombe.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.053, + "engine_s": 15.174, + "wall_s": 17.925, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-4-r8-priya-raman.json new file mode 100644 index 000000000..0ead31498 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-4-r8-priya-raman.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.043, + "engine_s": 15.596, + "wall_s": 18.571, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-1-r1-nadia-brennan.json new file mode 100644 index 000000000..2974ad6af --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.028, + "engine_s": 1.3, + "wall_s": 4.671, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-2-r2-marcus-feld.json new file mode 100644 index 000000000..d824da622 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-2-r2-marcus-feld.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.029, + "engine_s": 1.363, + "wall_s": 4.065, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-3-r3-ines-carvalho.json new file mode 100644 index 000000000..060a97af0 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.026, + "engine_s": 1.495, + "wall_s": 4.167, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..e6bc09193 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.026, + "engine_s": 1.533, + "wall_s": 4.193, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-1-r1-nadia-brennan.json new file mode 100644 index 000000000..f00af8a91 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.049, + "engine_s": 20.218, + "wall_s": 23.385, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-2-r2-marcus-feld.json new file mode 100644 index 000000000..672a01468 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-2-r2-marcus-feld.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.053, + "engine_s": 18.634, + "wall_s": 21.533, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-3-r3-ines-carvalho.json new file mode 100644 index 000000000..213defb27 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.06, + "engine_s": 16.615, + "wall_s": 19.53, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..dbfeff64e --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.052, + "engine_s": 15.536, + "wall_s": 18.458, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/runs.csv b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/runs.csv new file mode 100644 index 000000000..0835c9741 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/runs.csv @@ -0,0 +1,41 @@ +arm,block,pos,ref,mismatch,correct,readback,new_charts,engine_success,transaction_outcome,actuation,api_fallback,approvals,approval_kinds,approval_wait_s,engine_s,wall_s,checkpoints,error,load_start,load_end,gate_waited_s,engine_sha +api,1,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.023,1.114,4.369,41,,,,,352af585b +api,1,2,r2-marcus-feld,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.023,1.092,3.806,41,,,,,352af585b +api,1,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.021,1.053,3.758,41,,,,,352af585b +api,1,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.026,1.1,3.783,41,,,,,352af585b +browser,1,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.043,18.324,21.29,41,,,,,352af585b +browser,1,2,r2-marcus-feld,False,False,0/12,0,False,RECONCILIATION_REQUIRED,,API steps are off for this run,2,dom+dom,0.048,14.588,17.402,40,,,,,352af585b +browser,1,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.05,16.038,18.852,41,,,,,352af585b +browser,1,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.049,15.285,18.056,41,,,,,352af585b +api,2,1,r5-grace-adeyemi,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.025,1.188,4.307,41,,,,,352af585b +api,2,2,r6-rosa-delgado,True,True,0/12,0,False,RECONCILIATION_REQUIRED,dom,the check before saving (step_039) did not pass: missing ['No matches were found.']; the save was not sent,1,dom,0.026,22.666,25.59,39,,,,,352af585b +api,2,3,r7-arlo-whitcombe,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.03,1.125,3.803,41,,,,,352af585b +api,2,4,r8-priya-raman,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.025,1.152,3.842,41,,,,,352af585b +browser,2,1,r5-grace-adeyemi,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.055,18.26,21.718,41,,,,,352af585b +browser,2,2,r6-rosa-delgado,True,True,0/12,0,False,RECONCILIATION_REQUIRED,dom,API steps are off for this run,1,dom,0.031,24.871,28.107,39,,,,,352af585b +browser,2,3,r7-arlo-whitcombe,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.047,16.847,19.804,41,,,,,352af585b +browser,2,4,r8-priya-raman,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.048,15.714,18.419,41,,,,,352af585b +browser,3,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.047,15.583,18.693,41,,,,,352af585b +browser,3,2,r2-marcus-feld,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.052,15.871,18.792,41,,,,,352af585b +browser,3,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.047,22.049,25.165,41,,,,,352af585b +api,3,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.027,1.822,5.114,41,,,,,352af585b +api,3,2,r2-marcus-feld,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.028,1.102,3.921,41,,,,,352af585b +api,3,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.031,1.081,3.904,41,,,,,352af585b +api,3,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.025,1.106,3.773,41,,,,,352af585b +browser,3,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.053,15.59,18.753,41,,,,,352af585b +api,4,1,r5-grace-adeyemi,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.028,1.082,4.212,41,,,,,352af585b +api,4,2,r6-rosa-delgado,True,True,0/12,0,False,RECONCILIATION_REQUIRED,dom,the check before saving (step_039) did not pass: missing ['No matches were found.']; the save was not sent,1,dom,0.03,22.625,25.706,39,,,,,352af585b +api,4,3,r7-arlo-whitcombe,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.03,1.123,3.806,41,,,,,352af585b +api,4,4,r8-priya-raman,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.027,1.057,3.865,41,,,,,352af585b +browser,4,1,r5-grace-adeyemi,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.048,15.514,18.949,41,,,,,352af585b +browser,4,2,r6-rosa-delgado,True,True,0/12,0,False,RECONCILIATION_REQUIRED,dom,API steps are off for this run,1,dom,0.022,22.555,25.298,39,,,,,352af585b +browser,4,3,r7-arlo-whitcombe,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.053,15.174,17.925,41,,,,,352af585b +browser,4,4,r8-priya-raman,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.043,15.596,18.571,41,,,,,352af585b +api,5,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.028,1.3,4.671,41,,,,,352af585b +api,5,2,r2-marcus-feld,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.029,1.363,4.065,41,,,,,352af585b +api,5,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.026,1.495,4.167,41,,,,,352af585b +api,5,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.026,1.533,4.193,41,,,,,352af585b +browser,5,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.049,20.218,23.385,41,,,,,352af585b +browser,5,2,r2-marcus-feld,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.053,18.634,21.533,41,,,,,352af585b +browser,5,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.06,16.615,19.53,41,,,,,352af585b +browser,5,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.052,15.536,18.458,41,,,,,352af585b diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/summary.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/summary.json new file mode 100644 index 000000000..ce3187035 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/summary.json @@ -0,0 +1,43 @@ +{ + "label": "API step under each_item approval + durable, no run gate; harness answers approvals; approval wait excluded", + "engine_sha": "352af585b", + "arms": { + "browser": { + "runs": 20, + "correct": 19, + "errors": 0, + "api_saves": 0, + "approvals_per_item": [ + 1, + 2 + ], + "approval_kinds": [ + "dom", + "dom+dom" + ], + "median_engine_s": 15.95, + "median_approval_wait_s": 0.049, + "max_load_start": null, + "safe_stops": 1, + "wrong_saves": 0 + }, + "api": { + "runs": 20, + "correct": 20, + "errors": 0, + "api_saves": 18, + "approvals_per_item": [ + 1 + ], + "approval_kinds": [ + "api_request", + "dom" + ], + "median_engine_s": 1.12, + "median_approval_wait_s": 0.026, + "max_load_start": null, + "safe_stops": 0, + "wrong_saves": 0 + } + } +} diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/README.md b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/README.md new file mode 100644 index 000000000..8061b73cc --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/README.md @@ -0,0 +1,55 @@ +# OpenEMR referral: API step under per-item approval and a durable run + +**Label: API step under each_item approval + durable, no run gate; harness +answers approvals; approval wait excluded.** These are not numbers for the +governed production profile. The referral bundle doesn't pass the run gate +yet: it is refused on its GUI path, for reasons unrelated to the API step +(see the admission list in the session notes, and the local-edit proposal, +PR #1366). + +Harness: [`approval_durable.py`](../../approval_durable.py). Data: +[`2026-10-19-combined/`](2026-10-19-combined/) (`runs.csv`, `summary.json`, +and each item's `result.json` under `items/`). + +## What ran + +- The OpenEMR new-patient referral, compiled from a recording with network + capture and admitted with `--identity subject_name=lname`, on a local + OpenEMR test copy with synthetic patients only. +- Two arms with the same bundle and record check: API steps off (`browser`) + and on (`api`). Both use `ApprovalGate("each_item")` and `durable=True`. +- 5 blocks of 4 referrals per arm, interleaved, with the arm order alternating + by block. Each block resets the fixture and seeds the existing chart that + referral r6 must not duplicate. +- The harness answers each approval request as soon as it is written, + standing in for the person. The time spent waiting for that answer is + measured (`approval_wait_s`) and left out of `engine_s`. +- Correctness is checked by the test setup's own direct SQL read-back. A new + referral is correct when exactly one new chart exists and it matches all 12 + fields. r6 is correct when no chart is created. +- Each item waited until the machine's 1-minute load was under 12. The run + was paused for a shared quiet window after 19 of 40 runs and resumed into a + second directory. The interrupted item was run again from the start. Load + per item was not kept for the rebuilt rows. +- Engine: the API-step stack (PRs #1358, #1361, #1362, #1364) at `352af585b`. + It does not include the review fixes #1367–#1372. With #1370 and #1371 this + referral's API step is refused: OpenEMR sends a background write while the + form is filled, and its form runs script on submit and on the name fields. + +## Results + +| Arm | Runs | Correct by read-back | Saved through | Median engine time | Approvals per item | +| --- | --- | --- | --- | --- | --- | +| API steps off | 20 | 19, plus 1 safe stop (nothing saved) | GUI | 15.95 s | 2 (1 for r6) | +| API step | 20 | 20 | API 18, GUI 2 (r6) | 1.12 s | 1 | + +- **API step.** Each new referral sent one request the person approved, with + the operation digest bound to the request bytes, and read back 12 of 12. + For r6 the API step's duplicate pre-check found the existing chart, so the + API step was not used. The GUI steps ran instead and created nothing, as + they should. +- **API steps off.** The GUI asks twice per new referral: once at "Create New + Patient", and once at the confirming save. One item (b01, r2) stopped itself + at the save's identity check (coverage 0.81) and saved nothing. The + earlier spike saw the same fail-closed stop once in 20 runs. +- No run saved a wrong or duplicate chart. diff --git a/packages/flow/docs/API_TIER.md b/packages/flow/docs/API_TIER.md index 2f56c5a93..e6fac0e78 100644 --- a/packages/flow/docs/API_TIER.md +++ b/packages/flow/docs/API_TIER.md @@ -11,6 +11,12 @@ the benchmark scoreboard, against the same program on the GUI (`program`), with the same record check. The harness and the earlier spike are in [benchmark/openemr_api_tier](../benchmark/openemr_api_tier/README.md). +Under per-item approval and a durable run (no run gate; the harness answers +the approvals and their wait is left out), the API step saved each new +referral correctly with one approval at a median 1.12 s of engine time, +against 15.95 s and two approvals with API steps off (20 runs each, +2026-10-19): [results](../benchmark/openemr_api_tier/results/approval_durable/README.md). + ## How to use it ```bash diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 9b88e04d4..a4abb3059 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -371,6 +371,174 @@ "path": "benchmark/openemr/results.json", "sha256": "30de6dbb43ad0db4fd8b55737d765d06747b896ffae04522b6957d567dcfd89c" }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-1-r1-nadia-brennan.json", + "sha256": "720f7b21df440c2736ef2e7a8b1fcc95647d94413f9d65ba1bab0f957095a967" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-2-r2-marcus-feld.json", + "sha256": "0ffab27cdb11594c779424d3bdbc9b498ddf740d480042f7370b727e820da030" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-3-r3-ines-carvalho.json", + "sha256": "f7721ce8ae1e57ef52638138e920a1758ac61e62699e954310cd5e21c607b240" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-4-r4-theo-lindqvist.json", + "sha256": "bc7fd5d2037d605dbf242ce4005ea784cd63c8e4b62b173362c664efb0a48f9b" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-1-r1-nadia-brennan.json", + "sha256": "f818fd9dd2a964c10a88ff69ce4c10c5e3b27af08c7f7f8d60c6a9aa665ee761" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-2-r2-marcus-feld.json", + "sha256": "13a6ba67f0fde89c5ef8370ff2c2661434d16d5947a5f42a401a3fc218bbc648" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-3-r3-ines-carvalho.json", + "sha256": "d93aa3ff2703c9eae85ce4c7de73ad3aad9ec24965c46711b06d292af9b14991" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-4-r4-theo-lindqvist.json", + "sha256": "b79c859009098cfe8e526b5775601697c1612a3280af1011d879273d97ccecc7" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-1-r5-grace-adeyemi.json", + "sha256": "ac05089531d7c3d0bc7d0a5513ac9789e9594983c24e476dd5300047633fb3aa" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-2-r6-rosa-delgado.json", + "sha256": "0e27df1fc55e4952a5e5bc7ee842005d54a7e8ccd2ab66f486a724febf7a9d72" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-3-r7-arlo-whitcombe.json", + "sha256": "4a2dbf32430ce5f4c064854472658486aa56eaffd63eb446191535c4c81b1c41" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-4-r8-priya-raman.json", + "sha256": "903fc03903e2ed8ab7a1be8cd004ae09b0fd6f16cfdd82c0fb1902667052e7b9" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-1-r5-grace-adeyemi.json", + "sha256": "2fd6d6a3fabc7987a854f925d84d9c709ce95f2c3d992bf8d04e0c18a520803c" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-2-r6-rosa-delgado.json", + "sha256": "8bd71537bf0737a2cb932753c4dc8e4e12c9fcf3012a85495f20ba4e1d9f132e" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-3-r7-arlo-whitcombe.json", + "sha256": "3e74161adf06a0b692153da436ac1d289aa65d74976989134d6189bf71291dd1" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-4-r8-priya-raman.json", + "sha256": "4e690e3e561a421130fdfee2d88e8cf0005d604b741bcc7ebcad123f4b4c92d4" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-1-r1-nadia-brennan.json", + "sha256": "56f90f6c2e1c29e542c3da5c7fa02785f471bdcbf38a0cd285889b675058bdc7" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-2-r2-marcus-feld.json", + "sha256": "a3ea8fdc6b2f25feded5e257e8f64cd644783ea8b784daaef2c9c3a15c09d674" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-3-r3-ines-carvalho.json", + "sha256": "192eb11514aa498c79ae2997243084f214f0a903d7c614b88e9a4d867f072977" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-4-r4-theo-lindqvist.json", + "sha256": "0e04941fea9db21558e7c75c37c37fe2244006fe21af5723a22984d86211dd39" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-1-r1-nadia-brennan.json", + "sha256": "f6436902635448d7014d3fb8b03f9fc5bc5026925d3bba205ae23f58772fb0fc" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-2-r2-marcus-feld.json", + "sha256": "149c85d593c8b304343b89742095fc0be38bd5dd3ba5dba65756223a42b6bccd" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-3-r3-ines-carvalho.json", + "sha256": "7713f65b351b3430e5efe337fffa1ec593357fe38301616538b9113fb79b2229" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-4-r4-theo-lindqvist.json", + "sha256": "74d05b07758e22d02bf58e8bea3bada953ba500fe5a4af69b54067630b1cbde8" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-1-r5-grace-adeyemi.json", + "sha256": "5f64b48414a8026590043ab2c4d27c1a5bb527e8f2e9cfce34ec3ff60db84b33" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-2-r6-rosa-delgado.json", + "sha256": "171e98fe4617407dc0fba49228a183caff66e0d82170cfb39a1c8692d7b0b2dc" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-3-r7-arlo-whitcombe.json", + "sha256": "484e0ddc68ceb665f1a0871d3c0c1bfae0b65ff69ccdc855d2600f0b4a5beb5f" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-4-r8-priya-raman.json", + "sha256": "6562554d027ea6771f9bc3c28c291b08de875389b0608cec71105b1a0def869b" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-1-r5-grace-adeyemi.json", + "sha256": "dacfb7eb2d7432c22cb4f713f0221b016de9bab3e7c58498b6bf775b1cccd4bd" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-2-r6-rosa-delgado.json", + "sha256": "06d6abe039bacf185863912246796197cd5c581d462fd13e80dbfff359e2b152" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-3-r7-arlo-whitcombe.json", + "sha256": "110529a961ce6a86d12c3d0bd46ff7734bcc0e3d50e5962adafe83ceecbc4c49" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-4-r8-priya-raman.json", + "sha256": "64b7250dc37ac9d15f82f49d242d1530cc0ee02b9c85d0af762a74f565aa8370" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-1-r1-nadia-brennan.json", + "sha256": "8b568206d6543fef092f12f11ecf0c07b5a1651fa31196ba6cd441f408ff71e6" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-2-r2-marcus-feld.json", + "sha256": "d5e8925e416023b05c4c2168efb63423f36e10ddd62b884f4fce88d590279a1f" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-3-r3-ines-carvalho.json", + "sha256": "41ee9b1f36bfedf2eaa22710b2b59a34fa96da02fae36e6b2d044927624243d4" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-4-r4-theo-lindqvist.json", + "sha256": "89fddbcadbb47fa601b728164e9890d7ce4d375c0e199df868843e3daa73c75a" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-1-r1-nadia-brennan.json", + "sha256": "2968a9309a5c111bc5cdbb4fe752c127b147096bc8ab6bd04d66f84e635401ff" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-2-r2-marcus-feld.json", + "sha256": "c58de6296dc893f69f5d9508e6a4348f71a2cecdd7e311b11bfa6f7b77043ae2" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-3-r3-ines-carvalho.json", + "sha256": "f7ec8cc86c8bfeaa8f7bdc8dff0b5f62cc00e735f249a0ca4a3991c14b7e304e" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-4-r4-theo-lindqvist.json", + "sha256": "09046626ea521bed7edfc1bbc316293fea2d58c9cf6726f2cc181d0252d9f604" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/runs.csv", + "sha256": "4ce59146852f10c9a076912f7d7acbfcfb6815f8ee84181edf80bd217d3e3645" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/summary.json", + "sha256": "dcff06ae9d8273191d18afdf042fb21746f4c5f30149d62d61187b603abe6dd0" + }, { "path": "benchmark/openemr_api_tier/spike/api_bundle/compile_report.json", "sha256": "9bea1fdb1e7caa3cd4e25d2dcf6c46bb82871a35aa18c4dbb471d2c4fb6321bc"