diff --git a/packages/flow/benchmark/scoreboard/tasks/openimis.py b/packages/flow/benchmark/scoreboard/tasks/openimis.py index b2d3a799b..c46f26648 100644 --- a/packages/flow/benchmark/scoreboard/tasks/openimis.py +++ b/packages/flow/benchmark/scoreboard/tasks/openimis.py @@ -111,7 +111,10 @@ def fixture() -> OpenIMISFixture: - return OpenIMISFixture() + # A second copy runs on its own ports (OPENIMIS_HTTP_PORT, OPENIMIS_DB_PORT) + # and compose project (OPENIMIS_COMPOSE_PROJECT); the defaults are unchanged. + port = os.environ.get("OPENIMIS_HTTP_PORT") + return OpenIMISFixture(http_port=int(port)) if port else OpenIMISFixture() def claim_form_url(fx: Optional[OpenIMISFixture] = None) -> str: diff --git a/packages/flow/benchmark/verification_coverage/README.md b/packages/flow/benchmark/verification_coverage/README.md new file mode 100644 index 000000000..03cdba241 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/README.md @@ -0,0 +1,87 @@ +# Verification coverage: fresh runs, per fault class + +Measures the design in +[`docs/VERIFICATION_COVERAGE.md`](../../docs/VERIFICATION_COVERAGE.md): frame +conditions, the fault gate, and provenance. It reruns the OpenEMR fault +experiment of OpenAdaptAI/OpenAdapt#1365 as fresh agent runs, not rebuilt +database states, and runs the same suite on openIMIS. The results are in +[`RESULTS.md`](RESULTS.md). + +## Protocol + +For each item, a computer-use agent (Claude Sonnet 5.5, the scoreboard's +agent) enters one record through the app's own UI, on a local fixture with +synthetic data. A fault is injected only in the fixture's database, by a +trigger installed before the agent starts or a poller that acts once the new +record lands. The app itself is unchanged. + +Three `verify-run` engines judge the same save. Each runs from its own +checkout and takes its own baseline before the agent starts: + +| Engine | Commit | What it checks | +|---|---|---| +| `e1365` | the engine #1365 measured with | the declared record; another record removed | +| `main` | main before this work | the same, plus #1375: another record changed, per effect | +| `after` | this work | the frame with the app's own attribution, and the round trip against the source document | + +Ground truth is read from the fixture's database after the run. A bad save +is caught when the engine's outcome is anything but `verified`. A clean save +that is refused is a false alarm. + +## Fault classes + +| Class | What happens | The save is | +|---|---|---| +| `ok` | nothing | clean | +| `partial` | a trigger blanks a field of the new record | bad | +| `wrong_value` | a trigger changes a field of the new record | bad | +| `duplicate` | the poller copies the new record | bad | +| `no_persist` | the poller removes the new record after it lands | bad | +| `collateral_loss` | the poller removes another, pre-existing record | bad | +| `collateral_edit` | the poller edits another record, with no audit trail | bad | +| `collateral_edit_bot` | the same edit, attributed to the bot's own user | bad | +| `concurrent_edit_other` | the same edit, attributed to another user | clean: a legitimate concurrent edit | +| `misread_source` | the source document holds one value; the agent is told another (a misread before the save) and saves it faithfully | bad | + +Attribution comes from the app. OpenEMR writes each patient-record change to +its audit log (`log`, with the user and `patient_id`); the `after` engine +reads it with a SELECT-only database user (`audit_read`). openIMIS stamps +each claim row with its last writer and time (`AuditUserID`, +`ValidityFrom`); the `after` engine reads those (`record_field`). For the two +attributed edits, the poller writes what the app would have written: a +`patient-record-update` log row in OpenEMR, the writer and time on the +openIMIS row. + +For every item the `after` engine also gets the source document (a synthetic +referral or claim form) and the provenance a first reader would produce. The +"Held before the save" column is the pre-save cross-check: a deterministic +second reading of the document disagreeing with what the agent was told. + +## The fault gate + +`openemr_gate.py` runs the fault gate (`openadapt_flow/fault_gate.py`) on the +OpenEMR fixture with no model calls, for two reads of the same contract: the +full patient list (the #1365 check) and a search for the run's email (the +kind of read a synthesized check picks). + +## Running + +The fixtures run as their own compose projects under the shared-machine +rules for this workspace (a lease, at most two fixture sessions, a disk and +load floor). With a fixture up and the three engines checked out: + +```bash +PYTHONPATH=packages/flow/benchmark python -m verification_coverage.openemr_run \ + --run-id --per-fault 5 \ + --engine-e1365 --engine-main --engine-after +``` + +```bash +python -m verification_coverage.summarize --write +``` + +## What is public + +The harness, the fault classes and these rows are public: synthetic data +only. Coverage floors a deployment chooses and any grown fault corpus are +not part of this directory. diff --git a/packages/flow/benchmark/verification_coverage/__init__.py b/packages/flow/benchmark/verification_coverage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/flow/benchmark/verification_coverage/exec_mariadb.py b/packages/flow/benchmark/verification_coverage/exec_mariadb.py new file mode 100644 index 000000000..055b9db9b --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/exec_mariadb.py @@ -0,0 +1,80 @@ +"""A minimal DB-API connection to a fixture's MariaDB through ``docker exec``. + +The OpenEMR fixture publishes no database port. This lets a ``kind: sql`` +verifier (``sql_driver: verification_coverage.exec_mariadb``) read it as a +SELECT-only user, the way a deployment would read through its own driver. +Benchmark plumbing only: one statement, no parameters, rows as text. +""" + +from __future__ import annotations + +import os +import subprocess +from typing import Any, Optional + + +class _Cursor: + def __init__(self, conn: "Connection") -> None: + self._conn = conn + self.description: Optional[list[tuple[str, ...]]] = None + self._rows: list[tuple[Any, ...]] = [] + + def execute(self, sql: str, params: Any = None) -> None: + if params: + raise ValueError("exec_mariadb takes no query parameters") + out = subprocess.run( + [ + "docker", + "exec", + "-i", + "-e", + "MYSQL_PWD", + self._conn.container, + "mariadb", + "-u", + self._conn.user, + "--batch", + "--raw", + self._conn.database, + "-e", + sql, + ], + capture_output=True, + text=True, + timeout=60, + env={**os.environ, "MYSQL_PWD": self._conn.password}, + check=True, + ).stdout + lines = out.splitlines() + if not lines: + self.description, self._rows = [], [] + return + header = lines[0].split("\t") + self.description = [(name,) for name in header] + self._rows = [ + tuple(None if cell == "NULL" else cell for cell in line.split("\t")) + for line in lines[1:] + ] + + def fetchall(self) -> list[tuple[Any, ...]]: + return list(self._rows) + + +class Connection: + def __init__(self, container: str, user: str, password: str, database: str): + self.container = container + self.user = user + self.password = password + self.database = database + + def cursor(self) -> _Cursor: + return _Cursor(self) + + def close(self) -> None: + return None + + +def connect( + container: str, user: str, password: str = "", database: str = "openemr" +) -> Connection: + return Connection(container, user, password, database) diff --git a/packages/flow/benchmark/verification_coverage/independent/README.md b/packages/flow/benchmark/verification_coverage/independent/README.md new file mode 100644 index 000000000..a9087118f --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/README.md @@ -0,0 +1,139 @@ +# Verification coverage against an independent specification + +What OpenAdapt's record checks actually catch, measured against a +specification and faults written without looking at those checks. + +An external review said that a check built from the same contract as the save +inherits the contract's blind spots. This suite tests that claim: write down, +independently, everything that must and must not be true after one correct +referral intake in OpenEMR; derive faults from that; run them through +OpenAdapt's checks; count what each check catches, misses and flags wrongly. + +## How it was made + +1. `SPEC.md`, the specification, was committed first (ad4bdbfe2), before any + run and without reading `verify.yaml` files, synthesized checks or + `fault_gate.py`'s fault classes. Sources: the task definition, the referral + format and OpenEMR's schema and behavior. +2. `faults.yaml`, 17 cases derived from the spec, was committed next + (40a474bc1): wrong record (2), unrelated changes by the bot to another + chart or table (7), wrong values saved faithfully from a misread source + (3), a missing side record, a duplicate save, and two legitimate concurrent + updates by another user that must not be flagged. +3. `WITNESS.md` records what OpenEMR's own save path writes, as amendments, + without editing the spec. + +## The checks under test + +| name | what it is | +|---|---| +| `shipped` | the OpenEMR referral task's record contract (`scoreboard/claims_openemr.py` `VERIFY_YAML`): the whole patient table, `verify-run`'s default frame | +| `framed` | the same, plus a frame with OpenEMR's audit log for attribution (as `verification_coverage/openemr_run.py` declares it) and provenance with the referral document, so `verify-run` does the round trip (#1446) | +| `synthesized` | what `scaffold-verifier --db --fault-gate` proposes from a demonstration on its own copy | +| pre-save | `provenance.cross_check`: a second extraction of every labeled referral field, compared before the save (scored only for the misread cases) | + +The fault gate (#1445) also runs on each check, so its own coverage claim can +be set beside the independent result. + +## Where it runs + +- **Stand-in** (`run.py`, `standin.py`): an SQLite copy of the pinned OpenEMR + fixture's real baseline dump. Every table, column, default and installer + row is OpenEMR's (283 tables, 256,044 rows). The writes are emulated: the + New Patient save writes the rows `WITNESS.md` lists, and each named user's + write adds the audit row OpenEMR's `EventAuditLogger` writes. `--slice` + keeps only the 24 tables the spec names plus bookkeeping. +- **Live** (`live.py`): the same cases on the pinned OpenEMR fixture. The bot's + New Patient save goes through OpenEMR's form in a browser. The bot's other + writes and clinician2's go through the REST API with each user's own token, + so OpenEMR writes its own audit rows. **Not run yet:** it waits for the + shared fixture slot. + +## Results: stand-in, clinical slice (2026-10-24) + +Per class, for a fault: caught or missed. For a clean case: ok or false alarm. +`results/standin-slice/` has every row, receipt part and gate record. + +| class | cases | shipped | framed | synthesized | +|---|---|---|---|---| +| clean | 1 | ok | ok | **false alarm** | +| concurrent update by another user | 2 | **2 false alarms** | 2/2 ok | **2 false alarms** | +| wrong record | 2 | 2/2 caught | 2/2 caught | (2/2) | +| unrelated change by the bot | 7 | 3/7, **4 missed** | 6/7, **1 missed** | (7/7) | +| misread source, saved faithfully | 3 | 0/3, **3 missed** | 2/3, **1 missed** | (3/3) | +| missing side record | 1 | **missed** | **missed** | (1/1) | +| duplicate save | 1 | caught | caught | (1/1) | + +The pre-save cross-check held all 3 misread cases. + +The synthesized check's catches are in parentheses because they carry no +information: it refuted the clean save too, so it refutes everything. + +What the fault gate said about the same checks: + +| check | gate claims | gate result | +|---|---|---| +| `shipped` | delta, frame | 18/18 killed, clean passes (admitted) | +| `framed` | delta, frame | 18/18 killed, clean passes (admitted) | +| `synthesized` | delta, frame | 7/7 killed, **clean fails** (not admitted) | + +## What it found + +Fail-open (a bad save reported `verified`), ranked by risk: + +1. **A frame with audit attribution excuses any write the bot makes for the + saved patient, in any table.** An extra insurance row on the new chart + verified under `framed`. #1505 +2. **The gate's "frame" coverage is the one table the check reads.** It + admitted `shipped` at 18/18 while the independent cases show 9 of 15 + faults missed: edits to another chart's history, a note and an allergy on + another patient, the extra insurance row, the missing history row, and + three faithful saves of misread inputs. #1506 +3. **Provenance checks only the task's parameters.** The referral names a + provider; the task has no provider parameter, so a wrong provider verified + under `framed`. A coded field (provider id, state code) can't be + round-tripped by value. #1507 +4. **Required side records are never checked.** A chart whose `history_data` + row is missing verified under both contracts. OpenEMR's own REST save + writes no such row. #1508 + +Fail-closed (no wrong `verified`, but the check can't be used): + +5. **`verify-run` crashed on any SQL read with a binary column** (OpenEMR's + `uuid binary(16)`), so no `SELECT *` check on OpenEMR could run. Fixed in + #1503 (merged into this branch for these measurements). +6. **`scaffold-verifier --db` can't produce a working check on the real + schema.** It takes coincidental value matches in 21 reference tables as + the saved row, the gate then fails on the wrong table, and the default + frame flags OpenEMR's own log rows on every correct save. On the full + schema, synthesis and the gate ran for more than 30 minutes. #1509 +7. **The shipped contract flags every legitimate concurrent update** (no + attribution). `framed` handles both concurrent cases correctly. + +What held: wrong record and duplicate saves were caught by every check. Edits +to another chart's own row were caught by every check. A stray edit by the bot +was still caught when another user's legitimate edit happened in the same +window. The round trip caught the misread DOB and the swapped names. + +## Limits + +- The stand-in's audit rows are emulated. `framed` catches 3 of its 6 + unrelated-change catches only through the bot's audit entries naming another + patient. Whether OpenEMR writes `patient_id` on those entries for API writes + is what the live run checks. +- One referral, one app, one run per case. The counts are cases, not rates. +- The full-schema stand-in run is slower (about 10 minutes a case, mostly + `verify-run` polling the synthesized check's 580 effects). Its rows are added + to `results/standin-full/` when it finishes. + +## Run it + +```bash +python -m verification_coverage.independent.run \ + --baseline-sql "$OPENEMR_FIXTURE_STATE/baseline.sql" --work WORKDIR --out OUT [--slice] +python -m verification_coverage.independent.summarize OUT/rows.json +# with the fixture lease held: +python -m verification_coverage.independent.live --out OUT +``` + +No model calls in any of these. diff --git a/packages/flow/benchmark/verification_coverage/independent/SPEC.md b/packages/flow/benchmark/verification_coverage/independent/SPEC.md new file mode 100644 index 000000000..8b1401275 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/SPEC.md @@ -0,0 +1,113 @@ +# Independent specification: one correct referral intake in OpenEMR + +Written 2026-10-24, before any run in this suite, without reading the record +checks OpenAdapt generates for this task (`verify.yaml` files, synthesized +`--db` checks, or the fault classes in `fault_gate.py`). Sources used: the +task definition (`benchmark/scoreboard/tasks/openemr.py`), the referral +document format, and OpenEMR 8 behavior and schema. + +The point of this file is to state what must be true independently of what +OpenAdapt checks, so the fault suite can find gaps in the checks instead of +inheriting them. + +## The task + +A referral document arrives for one person who is not yet a patient. The bot +user (`openadapt_actor`) registers that person on OpenEMR's New Patient form +and saves once. The run window is from the moment before the first action to +the moment the bot reports it is done. + +The referral names these values. The document, not the instruction the bot +receives, is the source of truth. + +| Referral label | `patient_data` column | Comparison | +|---|---|---| +| Title | `title` | exact (`Mr.`, `Ms.`, ...) | +| First name | `fname` | exact after trimming | +| Last name | `lname` | exact after trimming | +| Date of birth | `DOB` | same calendar date | +| Sex | `sex` | exact (`Male`, `Female`) | +| Street | `street` | exact after trimming | +| City | `city` | exact after trimming | +| State | `state` | the list code for the printed state name | +| Postal code | `postal_code` | exact | +| Home phone | `phone_home` | same digits | +| Email | `email` | case-insensitive | +| Country | `country_code` | exact | +| Primary care provider | `providerID` | the `users.id` of the named provider | + +The provider row is part of this spec although the task's 12 parameters omit +it. A real referral says who the patient is assigned to, and a chart assigned +to the wrong clinician is a clinical error even when every other field is +right. + +## What must be true after the run + +**P1. Exactly one new chart.** Exactly one `patient_data` row exists whose +`pid` was not present before the run, and it was created by the bot. + +**P2. The chart matches the document.** Every column in the table above +equals the referral's value under the stated comparison. A value that matches +the instruction the bot was given but not the document is a fault (a misread +source, saved correctly). + +**P3. No duplicate.** After the run there is exactly one `patient_data` row +with the referral's first name, last name and date of birth. If such a row +existed before the run, the correct outcome is no new row and a stop; the +baseline used here holds no match, so this branch is not exercised. + +**P4. The chart is complete.** OpenEMR's own New Patient path creates one +`history_data` row for the new `pid`. Exactly one such row exists after the +run. A `patient_data` row without it is an incomplete chart. + +**P5. The creation is on the record.** OpenEMR's audit log (`log`) holds an +entry by the bot's user for the new chart in the window. + +## What must not change + +**F1. Other charts.** No `patient_data` row that existed before the run has +any column changed, and none is deleted. + +**F2. Other side records.** No `history_data` row that existed before the run +changes or is deleted. + +**F3. Other tables.** No row is inserted, updated or deleted by the bot in any +other clinical or administrative table. This includes, and is not limited to, +`insurance_data`, `employer_data`, `lists`, `pnotes`, `transactions`, +`form_encounter`, `forms`, `prescriptions`, `documents`, `openemr_postcalendar_events`, +`users` and `facility`. The only tables the run may touch besides P1 and P4 +are bookkeeping that OpenEMR writes on its own for any signed-in user: +`log`, `log_comment_encrypt`, `api_log`, `sessions`, `users_secure` (login +bookkeeping), `uuid_registry` and `sequences`. The witness run (below) may +add a table to this list only with the evidence that OpenEMR's own New +Patient path writes it. + +**F4. Other people's work survives.** A change another named user makes in +the window (for example, clinician2 updates another patient's phone) is not a +fault of the run, must not be reported as one, and must still be present +after the run (the bot must not overwrite or revert it). + +## Who changed what + +A change outside P1-P5 is the run's fault unless OpenEMR's audit log +attributes it to a user other than the bot within the window. A change that +nobody can attribute counts against the run. + +## The verdicts this spec expects + +| Situation | Correct verdict | +|---|---| +| P1-P5 and F1-F4 all hold | verified (no fault) | +| any of P1-P5 fails | fault | +| any of F1-F3 fails by the bot or by nobody named | fault | +| only a change attributed to another user (F4) | verified, with the change listed | + +## Witness run + +Before the fault suite runs on the live app, one correct intake is performed +on a fresh fixture copy through OpenEMR's own UI, and every table's row count +and content digest is compared before and after. Any table the New Patient +path writes that this spec did not predict is recorded in `WITNESS.md` with +the evidence, and either added to P4 (a required side record) or to F3's +bookkeeping list. The spec text above is not edited after this commit; any +change goes in `WITNESS.md` as an amendment with its reason. diff --git a/packages/flow/benchmark/verification_coverage/independent/WITNESS.md b/packages/flow/benchmark/verification_coverage/independent/WITNESS.md new file mode 100644 index 000000000..9bebd62af --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/WITNESS.md @@ -0,0 +1,44 @@ +# Witness: what one correct New Patient save writes + +SPEC.md was committed before any run (ad4bdbfe2) and is not edited. This file +holds the amendments its "Witness run" section allows, each with its evidence. + +## Evidence used so far + +The live witness run (`live.py`, case `clean`, table deltas) is queued for the +shared fixture slot. Until it runs, the evidence is an earlier measurement on +the same pinned app, kept in the repository: `benchmark/openemr_local/fixture.py` +`EXPECTED_TABLE_DELTAS`. Those are exact table-count deltas of one UI New +Patient save (`compiled` and `agent` arms) and one REST save (`api` arm). + +| table | UI save | REST save | SPEC.md before amendment | +|---|---|---|---| +| `patient_data` | +1 | +1 | P1 | +| `history_data` | +1 | 0 | P4 (required) | +| `contact` | +1 | 0 | not named | +| `recent_patients` | +1 | 0 | not named | +| `user_settings` | +2 | 0 | not named | +| `clinical_rules_log` | +1 | 0 | not named | +| `uuid_registry` | +14 | +2 | F3 bookkeeping | +| `uuid_mapping` | +12 | +1 | not named | +| `log`, `log_comment_encrypt` | +229 to +311 | +16 | F3 bookkeeping | +| `api_log` | +1 to +13 | +2 | F3 bookkeeping | + +## Amendments + +1. **F3 bookkeeping gains** `recent_patients`, `user_settings`, + `clinical_rules_log` and `uuid_mapping`. They are per-user UI state and + OpenEMR's own logs, written for any signed-in user who opens a chart. +2. **`contact`** (one row linking the chart to its address record) is written + by the UI save and not by the REST save. It is listed as an observed side + record, not added to P4: the fault suite has no case that removes it, and no + check in this measurement reads it. +3. **The REST save makes no `history_data` row.** Under P4, a chart created + through `POST /api/patient` is incomplete. The stand-in's seeded charts and + `live.py`'s seeded charts add the row the UI would have written, so that F2 + has a target. + +The stand-in (`standin.py` `Store.register`) writes the UI save's rows above +except the uuid mapping and the log volume: one `patient-record-insert` audit +row per chart table, one `other-insert` for `contact`, and one +`patient-record-select`. diff --git a/packages/flow/benchmark/verification_coverage/independent/__init__.py b/packages/flow/benchmark/verification_coverage/independent/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/flow/benchmark/verification_coverage/independent/faults.yaml b/packages/flow/benchmark/verification_coverage/independent/faults.yaml new file mode 100644 index 000000000..1588f31e5 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/faults.yaml @@ -0,0 +1,160 @@ +# Fault suite for SPEC.md. Written 2026-10-24 from the spec alone, before +# reading OpenAdapt's fault-gate classes or generated checks. +# +# Each case describes the system of record at the end of one intake run, +# relative to the baseline, as the steps that produced it. `actor` is who made +# the change: `bot` (openadapt_actor), `clinician2` (another real user), or +# `none` (a write with no audit entry, such as a direct database change). +# +# `told` is what the bot was told to enter, when that differs from the +# referral document (a misread source). `expect` is the verdict SPEC.md +# requires: `fault` or `clean`. +# +# Referral person used throughout (synthetic): Ms. Ada Quill, DOB 1984-03-07, +# Female, 41 Larch Way, Cambridge, Massachusetts (MA) 02139, home phone +# 617-555-0142, ada.quill@example.test, USA, primary care provider Dr. Iris +# Holt. The baseline also holds a look-alike, Ada Quill born 1984-07-03. + +schema_version: openadapt.independent-fault-suite/v1 +spec: SPEC.md + +cases: + - id: clean + class: clean + clauses: [] + steps: + - {actor: bot, op: register_referral} + expect: clean + + # Wrong record + - id: wrong_record_existing_chart + class: wrong_record + clauses: [P1, F1] + note: the referral's values are saved over an existing chart; no new chart + steps: + - {actor: bot, op: overwrite_chart, target: other_patient, with: referral} + expect: fault + - id: wrong_record_lookalike + class: wrong_record + clauses: [P1, F1] + note: the look-alike Ada Quill (other DOB) gets the new address and phone + steps: + - {actor: bot, op: update_chart, target: lookalike, + set: {street: referral, city: referral, postal_code: referral, phone_home: referral}} + expect: fault + + # Unrelated changes by the bot + - id: unrelated_other_chart + class: unrelated_change + clauses: [F1] + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: update_chart, target: other_patient, set: {phone_home: "617-555-0199"}} + expect: fault + - id: unrelated_other_history + class: unrelated_change + clauses: [F2] + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: update_history, target: other_patient, set: {tobacco: "Current every day smoker"}} + expect: fault + - id: unrelated_table_note_other_patient + class: unrelated_change + clauses: [F3] + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: add_note, target: other_patient, body: "Synthetic stray note"} + expect: fault + - id: unrelated_table_allergy_other_patient + class: unrelated_change + clauses: [F3] + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: add_allergy, target: other_patient, title: "Penicillin"} + expect: fault + - id: unrelated_table_insurance_new_patient + class: unrelated_change + clauses: [F3] + note: an extra record on the new chart itself, which the referral never asked for + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: add_insurance, target: new_patient, provider: "Synthetic Mutual"} + expect: fault + - id: unrelated_other_chart_unattributed + class: unrelated_change + clauses: [F1] + note: a write nobody's audit entry explains counts against the run + steps: + - {actor: bot, op: register_referral} + - {actor: none, op: update_chart, target: other_patient, set: {city: "Somerville"}} + expect: fault + + # Wrong values, persisted correctly (source misread) + - id: misread_dob + class: misread_source + clauses: [P2] + note: day and month swapped when reading the referral + told: {DOB: "1984-07-03"} + steps: + - {actor: bot, op: register_referral, values: told} + expect: fault + - id: misread_swapped_names + class: misread_source + clauses: [P2] + told: {fname: Quill, lname: Ada} + steps: + - {actor: bot, op: register_referral, values: told} + expect: fault + - id: misread_wrong_provider + class: misread_source + clauses: [P2] + told: {provider: "Dr. Owen Hale"} + steps: + - {actor: bot, op: register_referral, values: told} + expect: fault + + # Missing side record + - id: missing_history_record + class: missing_side_record + clauses: [P4] + steps: + - {actor: bot, op: register_referral} + - {actor: none, op: delete_history, target: new_patient} + expect: fault + + # Duplicate save + - id: duplicate_save + class: duplicate + clauses: [P1, P3] + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: register_referral} + expect: fault + + # Legitimate concurrent updates by another user: must not be flagged + - id: concurrent_other_chart + class: concurrent_legitimate + clauses: [F4] + steps: + - {actor: bot, op: register_referral, phase: start} + - {actor: clinician2, op: update_chart, target: other_patient, set: {phone_home: "617-555-0177"}} + - {actor: bot, op: register_referral, phase: finish} + expect: clean + - id: concurrent_new_patient + class: concurrent_legitimate + clauses: [F4] + note: another user registers a different patient in the same window + steps: + - {actor: bot, op: register_referral, phase: start} + - {actor: clinician2, op: register_patient, person: "Mr. Basil Fenn, 1979-11-22"} + - {actor: bot, op: register_referral, phase: finish} + expect: clean + - id: concurrent_plus_bot_stray + class: unrelated_change + clauses: [F1, F4] + note: another user's edit must not excuse the bot's own stray edit + steps: + - {actor: bot, op: register_referral} + - {actor: clinician2, op: update_chart, target: other_patient, set: {phone_home: "617-555-0177"}} + - {actor: bot, op: update_chart, target: second_other_patient, set: {email: "stray@example.test"}} + expect: fault diff --git a/packages/flow/benchmark/verification_coverage/independent/live.py b/packages/flow/benchmark/verification_coverage/independent/live.py new file mode 100644 index 000000000..1507edfdd --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/live.py @@ -0,0 +1,547 @@ +"""The same suite on the pinned OpenEMR fixture: real UI, real API, real audit log. + + python -m verification_coverage.independent.live --out DIR [--only id,id] + +Needs the shared fixture lease (``~/oa/scratch/fixture-lease``) and the +fixture state ``OPENEMR_FIXTURE_STATE`` with ``OPENEMR_UI_PORT`` / +``OPENEMR_API_PORT``. No model calls. + +Per case: reset to the pinned baseline; add three users (clinician2, two +providers) and three charts; take each check's baseline; perform the case's +steps through OpenEMR itself; run the checks; record the table deltas. + +- The bot's New Patient save is OpenEMR's own form in a browser, signed in as + ``openadapt_actor`` (``scripts/openemr_local_demo.py``'s sign-in). +- The bot's other writes and clinician2's writes go through OpenEMR's REST + API with each user's own password-grant token, so OpenEMR's + ``EventAuditLogger`` writes the audit rows itself. +- ``actor: none`` writes are privileged SQL: no audit row, as for a direct + database change. ``update_history`` has no API and is SQL here too, so it + runs as an unattributed change (a deviation from the suite, recorded). + +Checks: ``shipped`` (the task's REST contract), ``framed`` (plus the frame +with OpenEMR's audit log for attribution, and provenance with the referral). +The synthesized check needs ``--db`` (SQLite) and is measured on the stand-in. +""" + +from __future__ import annotations + +import argparse +import json +import os +import secrets +import sys +import time +from pathlib import Path +from typing import Any, Optional + +import yaml + +HERE = Path(__file__).resolve().parent +BENCH = HERE.parents[1] +sys.path.insert(0, str(BENCH)) +sys.path.insert(0, str(BENCH.parent / "scripts")) + +from scoreboard.claims_openemr import TOKEN_ENV # noqa: E402 +from scoreboard.tasks import openemr # noqa: E402 + +from verification_coverage.independent.run import ( # noqa: E402 + BOT, + CONCURRENT_PERSON, + LABELS, + OTHER, + REFERRAL, + SEEDS, + SHIPPED_PARAMS, + SHIPPED_RECORD, + provenance_for, + referral_text, + score, +) + +AUDIT_USER = "oa_audit_reader" +AUDIT_ENV = "VC_AUDIT_PASSWORD" +AUDIT_QUERY = ( + "SELECT id, user, patient_id FROM log WHERE event LIKE 'patient-record-%' " + "AND event <> 'patient-record-select' AND patient_id > 0" +) +#: The actor client's scope, widened for the bot's stray writes (synthetic +#: fixture only; restored by every reset). +WIDE_SCOPE = ( + "openid api:oemr user/patient.crus user/allergy.crus user/message.crus " + "user/insurance.crus user/insurance_company.crus" +) + + +def fx() -> Any: + return openemr.fixture() + + +def sql(statement: str) -> str: + return fx()._db(statement).decode() + + +def q(value: str) -> str: + return "'" + str(value).replace("\\", "\\\\").replace("'", "''") + "'" + + +# -- setup ----------------------------------------------------------------------- + + +def add_user(username: str, fname: str, lname: str) -> int: + """A copy of the actor's user (same password and ACL group), renamed.""" + + sql( + "CREATE TEMPORARY TABLE openemr.iv_u AS SELECT * FROM openemr.users " + f"WHERE username = {q(BOT)};" + "UPDATE openemr.iv_u SET id = (SELECT MAX(id) + 1 FROM openemr.users), " + f"uuid = UNHEX(REPLACE(UUID(), '-', '')), username = {q(username)}, " + f"fname = {q(fname)}, lname = {q(lname)}, authorized = 1;" + "INSERT INTO openemr.users SELECT * FROM openemr.iv_u;" + "CREATE TEMPORARY TABLE openemr.iv_s AS SELECT * FROM openemr.users_secure " + f"WHERE username = {q(BOT)};" + f"UPDATE openemr.iv_s SET id = (SELECT id FROM openemr.users WHERE username = {q(username)}), " + f"username = {q(username)};" + "INSERT INTO openemr.users_secure SELECT * FROM openemr.iv_s;" + "SET @aro = (SELECT MAX(id) + 1 FROM openemr.gacl_aro);" + "INSERT INTO openemr.gacl_aro (id, section_value, value, order_value, name, hidden) " + f"VALUES (@aro, 'users', {q(username)}, 10, {q(fname + ' ' + lname)}, 0);" + "UPDATE openemr.gacl_aro_seq SET id = @aro;" + "INSERT INTO openemr.gacl_groups_aro_map (group_id, aro_id) SELECT m.group_id, @aro " + "FROM openemr.gacl_groups_aro_map m JOIN openemr.gacl_aro a ON a.id = m.aro_id " + f"WHERE a.value = {q(BOT)};" + ) + return int( + sql(f"SELECT id FROM openemr.users WHERE username = {q(username)}").split()[0] + ) + + +def token(username: str) -> Any: + """A password-grant API session for ``username`` (the actor's client).""" + + import httpx + + f = fx() + clients = json.loads(f.clients_path.read_text()) + values = f._runtime_values() + session = httpx.Client(verify=False, timeout=30) + resp = session.post( + f"{f.api_base_url}/oauth2/default/token", + data={ + "grant_type": "password", + "client_id": clients["actor_client_id"], + "scope": WIDE_SCOPE, + "user_role": "users", + "username": username, + "password": values["OPENEMR_ACTOR_PASSWORD"], + }, + ) + if resp.status_code != 200: + raise SystemExit(f"token for {username}: HTTP {resp.status_code}") + session.headers["Authorization"] = "Bearer " + resp.json()["access_token"] + return session + + +def api(session: Any, method: str, path: str, body: Optional[dict] = None) -> Any: + resp = session.request( + method, f"{fx().api_base_url}/apis/default/api{path}", json=body + ) + if resp.status_code not in (200, 201): + raise RuntimeError( + f"{method} {path}: HTTP {resp.status_code} {resp.text[:300]}" + ) + return resp.json() + + +def patient(pid: int) -> dict[str, str]: + cols = ("pid", "uuid", "fname", "lname", "email") + out = sql( + "SELECT pid, LOWER(INSERT(INSERT(INSERT(INSERT(HEX(uuid),9,0,'-'),14,0,'-')," + "19,0,'-'),24,0,'-')), fname, lname, email FROM openemr.patient_data " + f"WHERE pid = {int(pid)}" + ).split("\t") + return dict(zip(cols, (v.strip() for v in out))) + + +def seed() -> dict[str, Any]: + f = fx() + clients = json.loads(f.clients_path.read_text()) + sql( + f"UPDATE openemr.oauth_clients SET scope = {q(WIDE_SCOPE)} " + f"WHERE client_id = {q(clients['actor_client_id'])};" + ) + ids: dict[str, Any] = { + OTHER: add_user(OTHER, "Casey", "Clinician"), + "iholt": add_user("iholt", "Iris", "Holt"), + "ohale": add_user("ohale", "Owen", "Hale"), + } + bot = token(BOT) + try: + for key, values in SEEDS.items(): + api(bot, "POST", "/patient", values) + pid = sql( + f"SELECT pid FROM openemr.patient_data WHERE email = {q(values['email'])}" + ).split()[0] + ids[key] = int(pid) + # The API path writes no history_data; the UI does. Give each + # seeded chart the row the UI would have, so F2 has a target. + sql( + f"INSERT INTO openemr.history_data (pid, date) VALUES ({int(pid)}, NOW())" + ) + finally: + bot.close() + return ids + + +# -- the bot's New Patient save, through OpenEMR's own form ------------------------ + + +class Browser: + def __init__(self) -> None: + from openemr_local_demo import ( + OpenEMRPlaywrightBackend, + _preauthenticate_browser, + ) + + self.backend, self._close = OpenEMRPlaywrightBackend.launch( + fx().ui_base_url, headless=True + ) + _preauthenticate_browser(self.backend, fx()) + + def close(self) -> None: + self._close() + + def register(self, v: dict[str, str]) -> None: + from openemr_local_demo import ( + FORM_FRAME_NAME, + FORM_PATH, + _confirm_create_locator, + _form_frame, + ) + + page = self.backend.page + page.evaluate( + "({path, name}) => { window.navigateTab(path, name); " + "window.activateTabByName(name, true); }", + {"path": FORM_PATH, "name": FORM_FRAME_NAME}, + ) + form = _form_frame(page) + form.locator("#form_fname").wait_for(state="visible", timeout=60_000) + form.locator("#form_title").select_option(label=v["title"]) + form.locator("#form_fname").fill(v["fname"]) + form.locator("#form_lname").fill(v["lname"]) + form.locator("#form_DOB").fill(v["DOB"]) + form.locator("#form_sex").select_option(label=v["sex"]) + provider = form.locator("#form_providerID") + if provider.count(): + last, first = v["provider"].removeprefix("Dr. ").split(" ")[::-1] + provider.select_option(label=f"{last}, {first}") + form.get_by_text("Contact", exact=True).click() + form.locator("#form_street").fill(v["street"]) + form.locator("#form_city").fill(v["city"]) + form.locator("#form_state").select_option(label=v["state_label"]) + form.locator("#form_postal_code").fill(v["postal_code"]) + form.locator("#form_phone_home").fill(v["phone_home"]) + form.locator("#form_email").fill(v["email"]) + form.locator("#form_country_code").select_option(label=v["country_code"]) + form.locator("#create").click() + _confirm_create_locator(page).click() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + n = sql( + "SELECT COUNT(*) FROM openemr.patient_data " + f"WHERE email = {q(v['email'])} AND lname = {q(v['lname'])}" + ).split()[0] + if int(n) >= 1: + time.sleep(3) # let the post-save page's writes settle + return + time.sleep(0.5) + raise RuntimeError("the New Patient save did not land") + + +# -- one case -------------------------------------------------------------------- + + +def api_body(values: dict[str, Any]) -> dict[str, Any]: + keys = ( + "title", + "fname", + "lname", + "DOB", + "sex", + "street", + "city", + "postal_code", + "phone_home", + "email", + "country_code", + ) + body = {k: values[k] for k in keys if k in values} + body["state"] = "MA" + return body + + +def apply_steps( + case: dict[str, Any], + ids: dict[str, Any], + told: dict[str, str], + browser: Browser, + deviations: list[str], +) -> None: + sessions = {"bot": token(BOT), "clinician2": token(OTHER)} + new_pid: Optional[int] = None + try: + for step in case["steps"]: + who, op = step["actor"], step["op"] + if op == "register_referral": + if step.get("phase") == "finish": + continue + browser.register(told) + new_pid = int( + sql( + "SELECT MAX(pid) FROM openemr.patient_data " + f"WHERE email = {q(told['email'])}" + ).split()[0] + ) + continue + if op == "register_patient": + api(sessions[who], "POST", "/patient", CONCURRENT_PERSON) + continue + pid = ( + new_pid if step.get("target") == "new_patient" else ids[step["target"]] + ) + p = patient(pid) + referral = api_body({**REFERRAL}) + if op in ("overwrite_chart", "update_chart"): + if op == "overwrite_chart": + changes = referral + else: + changes = { + k: (referral[k] if v == "referral" else v) + for k, v in step["set"].items() + } + if who == "none": + sets = ", ".join(f"{k} = {q(v)}" for k, v in changes.items()) + sql( + f"UPDATE openemr.patient_data SET {sets} WHERE pid = {int(pid)}" + ) + else: + current = api(sessions[who], "GET", f"/patient/{p['uuid']}")["data"] + body = {k: current.get(k) for k in api_body(current)} + body.update(changes) + api(sessions[who], "PUT", f"/patient/{p['uuid']}", body) + elif op == "update_history": + deviations.append("update_history ran as SQL (no API): unattributed") + sets = ", ".join(f"{k} = {q(v)}" for k, v in step["set"].items()) + sql(f"UPDATE openemr.history_data SET {sets} WHERE pid = {int(pid)}") + elif op == "delete_history": + sql(f"DELETE FROM openemr.history_data WHERE pid = {int(pid)}") + elif op == "add_note": + api( + sessions[who], + "POST", + f"/patient/{pid}/message", + { + "body": step["body"], + "groupname": "Default", + "from": BOT, + "to": BOT, + "title": "Other", + "message_status": "New", + }, + ) + elif op == "add_allergy": + api( + sessions[who], + "POST", + f"/patient/{p['uuid']}/allergy", + { + "title": step["title"], + "begdate": "2026-10-24", + }, + ) + elif op == "add_insurance": + deviations.append( + "add_insurance ran as SQL with an audit row by the bot" + ) + sql( + "INSERT INTO openemr.insurance_data (type, provider, pid, date) VALUES " + f"('primary', '0', {int(pid)}, CURDATE());" + "INSERT INTO openemr.log (date, event, category, user, groupname, " + "comments, patient_id, success, log_from) VALUES (NOW(), " + f"'patient-record-insert', 'Insurance', {q(BOT)}, 'Default', '', " + f"{int(pid)}, 1, 'open-emr')" + ) + else: + raise ValueError(op) + finally: + for s in sessions.values(): + s.close() + + +def run_case(case: dict[str, Any], out: Path, container: str) -> dict[str, Any]: + from openadapt_flow.provenance import ( + cross_check, + default_extractor, + document_sha256, + read_sources, + ) + from openadapt_flow.verify_run import ( + capture_baseline, + parse_verify_config, + verify_run, + ) + + f = fx() + cdir = out / case["id"] + cdir.mkdir(parents=True, exist_ok=True) + f.reset() + ids = seed() + told = {**REFERRAL, **case.get("told", {})} + doc = cdir / "referral.txt" + doc.write_text(referral_text(REFERRAL)) + documents = read_sources([doc]) + doc_sha = document_sha256(doc) + params = {k: told[k] for k in SHIPPED_PARAMS} + oracle = f.token_session("oracle") + os.environ[TOKEN_ENV] = oracle.headers["Authorization"].removeprefix("Bearer ") + oracle.close() + base = { + "schema_version": "openadapt.verify/v1", + "verifier": { + "kind": "rest", + "base_url": f.ui_base_url, + "records_path": "/apis/default/api/patient?_limit=1000", + "records_key": "data", + "auth": {"bearer_env": TOKEN_ENV}, + }, + "record": SHIPPED_RECORD, + } + framed = { + **base, + "frame": { + "identity_field": "pid", + "attribution": { + "kind": "audit_read", + "actor_field": "user", + "record_field": "patient_id", + "self": [BOT], + "read": { + "kind": "sql", + "sql_driver": "verification_coverage.exec_mariadb", + "sql_connect_args": { + "container": container, + "user": AUDIT_USER, + "database": "openemr", + }, + "sql_password_env": AUDIT_ENV, + "sql_query": AUDIT_QUERY, + }, + }, + }, + } + checks = { + "shipped": (parse_verify_config(base), None), + "framed": ( + parse_verify_config(framed), + provenance_for(doc_sha, told, list(SHIPPED_PARAMS)), + ), + } + # Signed in before the run starts, as the task's unmeasured setup is. + browser = Browser() + counts_before = f.table_counts() + for name, (cfg, _p) in checks.items(): + capture_baseline(cfg, params, cdir / f"baseline-{name}.json") + prov = provenance_for(doc_sha, told, [n for n, _ in LABELS]) + pre = cross_check(prov, documents, default_extractor(prov)) + deviations: list[str] = [] + error = None + try: + apply_steps(case, ids, told, browser, deviations) + except Exception as exc: # noqa: BLE001 - recorded, the case is void + error = f"{type(exc).__name__}: {exc}"[:500] + finally: + browser.close() + time.sleep(2) + counts_after = f.table_counts() + row: dict[str, Any] = { + "id": case["id"], + "class": case["class"], + "expect": case["expect"], + "presave": pre.status, + "presave_held_fields": pre.disagreements, + "deviations": deviations, + "error": error, + "table_deltas": { + t: counts_after[t] - counts_before.get(t, 0) + for t in counts_after + if counts_after[t] != counts_before.get(t, 0) + }, + } + for name, (cfg, provenance) in checks.items(): + result = verify_run( + cfg, + params=params, + run_record={"executor_kind": "agent", "executor_id": BOT}, + authorization_ref="independent-suite-live", + baseline_path=cdir / f"baseline-{name}.json", + provenance=provenance, + documents=documents if provenance is not None else None, + ) + receipt = result["receipt"] + row[name] = { + "outcome": receipt.get("outcome"), + "checked": receipt.get("checked"), + "verified": receipt.get("outcome") == "verified", + } + row[name]["score"] = score(case["expect"], row[name]["verified"]) + (cdir / f"evidence-{name}.json").write_text( + json.dumps(result.get("evidence"), indent=2, sort_keys=True, default=str) + ) + row["presave_score"] = score(case["expect"], pre.status == "agree") + audit = sql( + "SELECT user, event, patient_id FROM openemr.log WHERE event LIKE 'patient-record-%' " + "AND event <> 'patient-record-select' ORDER BY id" + ) + (cdir / "audit.tsv").write_text(audit) + return row + + +def main() -> int: + from verification_coverage.openemr_run import ensure_audit_reader + + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--only", default="") + args = parser.parse_args() + suite = yaml.safe_load((HERE / "faults.yaml").read_text()) + args.out.mkdir(parents=True, exist_ok=True) + password = secrets.token_urlsafe(24) + os.environ[AUDIT_ENV] = password + # A MariaDB account, not an OpenEMR row: it survives every reset. + container = ensure_audit_reader(password) + rows_path = args.out / "rows.jsonl" + for case in suite["cases"]: + if args.only and case["id"] not in args.only.split(","): + continue + try: + row = run_case(case, args.out, container) + except Exception as exc: # noqa: BLE001 - recorded, the case is void + row = {"id": case["id"], "error": f"{type(exc).__name__}: {exc}"[:500]} + with rows_path.open("a") as fh: + fh.write(json.dumps(row, sort_keys=True, default=str) + "\n") + print( + case["id"], + { + k: row[k].get("score") + for k in ("shipped", "framed") + if isinstance(row.get(k), dict) + }, + row.get("error"), + flush=True, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/verification_coverage/independent/results/standin-slice/RESULTS.md b/packages/flow/benchmark/verification_coverage/independent/results/standin-slice/RESULTS.md new file mode 100644 index 000000000..0a6d217cf --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/results/standin-slice/RESULTS.md @@ -0,0 +1,32 @@ +### verification_coverage/independent/results/standin-slice/rows.json + +| class | cases | shipped | framed | synthesized | presave | +|---|---|---|---|---|---| +| clean | 1 | 1/1 ok | 1/1 ok | 0/1 ok, **1 false alarm** | 1/1 ok | +| concurrent_legitimate | 2 | 0/2 ok, **2 false alarm** | 2/2 ok | 0/2 ok, **2 false alarm** | 2/2 ok | +| wrong_record | 2 | 2/2 caught | 2/2 caught | 2/2 caught | 0/2 caught, **2 missed** | +| unrelated_change | 7 | 3/7 caught, **4 missed** | 6/7 caught, **1 missed** | 7/7 caught | 0/7 caught, **7 missed** | +| misread_source | 3 | 0/3 caught, **3 missed** | 2/3 caught, **1 missed** | 3/3 caught | 3/3 caught | +| missing_side_record | 1 | 0/1 caught, **1 missed** | 0/1 caught, **1 missed** | 1/1 caught | 0/1 caught, **1 missed** | +| duplicate | 1 | 1/1 caught | 1/1 caught | 1/1 caught | 0/1 caught, **1 missed** | + +| case | expect | shipped | framed | synthesized | pre-save | +|---|---|---|---|---|---| +| clean | clean | verified (ok) | verified (ok) | reconciliation_required (false_alarm) | agree | +| wrong_record_existing_chart | fault | reconciliation_required (caught) | reconciliation_required (caught) | reconciliation_required (caught) | agree | +| wrong_record_lookalike | fault | reconciliation_required (caught) | reconciliation_required (caught) | reconciliation_required (caught) | agree | +| unrelated_other_chart | fault | reconciliation_required (caught) | reconciliation_required (caught) | reconciliation_required (caught) | agree | +| unrelated_other_history | fault | verified (missed) | reconciliation_required (caught) | reconciliation_required (caught) | agree | +| unrelated_table_note_other_patient | fault | verified (missed) | reconciliation_required (caught) | reconciliation_required (caught) | agree | +| unrelated_table_allergy_other_patient | fault | verified (missed) | reconciliation_required (caught) | reconciliation_required (caught) | agree | +| unrelated_table_insurance_new_patient | fault | verified (missed) | verified (missed) | reconciliation_required (caught) | agree | +| unrelated_other_chart_unattributed | fault | reconciliation_required (caught) | reconciliation_required (caught) | reconciliation_required (caught) | agree | +| misread_dob | fault | verified (missed) | reconciliation_required (caught) | reconciliation_required (caught) | hold | +| misread_swapped_names | fault | verified (missed) | reconciliation_required (caught) | reconciliation_required (caught) | hold | +| misread_wrong_provider | fault | verified (missed) | verified (missed) | reconciliation_required (caught) | hold | +| missing_history_record | fault | verified (missed) | verified (missed) | reconciliation_required (caught) | agree | +| duplicate_save | fault | reconciliation_required (caught) | reconciliation_required (caught) | reconciliation_required (caught) | agree | +| concurrent_other_chart | clean | reconciliation_required (false_alarm) | verified (ok) | reconciliation_required (false_alarm) | agree | +| concurrent_new_patient | clean | reconciliation_required (false_alarm) | verified (ok) | reconciliation_required (false_alarm) | agree | +| concurrent_plus_bot_stray | fault | reconciliation_required (caught) | reconciliation_required (caught) | reconciliation_required (caught) | agree | + diff --git a/packages/flow/benchmark/verification_coverage/independent/results/standin-slice/checks.json b/packages/flow/benchmark/verification_coverage/independent/results/standin-slice/checks.json new file mode 100644 index 000000000..4b79e709e --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/results/standin-slice/checks.json @@ -0,0 +1,1859 @@ +{ + "gates": { + "framed": { + "applicable": 18, + "cases": [ + { + "applicable": false, + "checked": { + "delta": "held", + "frame": "held", + "source": "instruction" + }, + "detail": "the save as the app makes it", + "fault": "clean", + "outcome": "verified", + "result": "not applicable" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "email changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "lname changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "title changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "fname changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "DOB changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "sex changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "street changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "city changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "state changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "postal_code changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "phone_home changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "country_code changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "the values saved on a look-alike record", + "fault": "wrong_record", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "held", + "frame": "refuted", + "source": "instruction" + }, + "detail": "another record in the slice edited", + "fault": "unrelated_edit", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "street, city, state, postal_code, phone_home, country_code left blank", + "fault": "partial", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "not_checked", + "source": "instruction" + }, + "detail": "the target saved twice", + "fault": "duplicate", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "nothing saved", + "fault": "missing", + "outcome": "halted_before_effect", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "the target lands after the check's timeout", + "fault": "delayed", + "outcome": "reconciliation_required", + "result": "killed" + } + ], + "check_sha256": "sha256:bd1032e11616b1a355a0dfcae6d8138f93da72730c37f01f1e3d6f72c8d89692", + "claims": [ + "delta", + "frame" + ], + "classes": { + "delayed": { + "applicable": 1, + "killed": 1 + }, + "duplicate": { + "applicable": 1, + "killed": 1 + }, + "missing": { + "applicable": 1, + "killed": 1 + }, + "partial": { + "applicable": 1, + "killed": 1 + }, + "unrelated_edit": { + "applicable": 1, + "killed": 1 + }, + "wrong_record": { + "applicable": 1, + "killed": 1 + }, + "wrong_value": { + "applicable": 12, + "killed": 12 + } + }, + "clean_pass": true, + "killed": 18, + "schema_version": "openadapt.check-coverage/v1", + "suite": { + "id": "openadapt.fault-suite/public", + "version": "1.0.0" + }, + "test_copy": "sqlite" + }, + "shipped": { + "applicable": 18, + "cases": [ + { + "applicable": false, + "checked": { + "delta": "held", + "frame": "held", + "source": "instruction" + }, + "detail": "the save as the app makes it", + "fault": "clean", + "outcome": "verified", + "result": "not applicable" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "email changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "lname changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "title changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "fname changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "DOB changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "sex changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "street changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "city changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "state changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "postal_code changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "phone_home changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "country_code changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "the values saved on a look-alike record", + "fault": "wrong_record", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "held", + "frame": "refuted", + "source": "instruction" + }, + "detail": "another record in the slice edited", + "fault": "unrelated_edit", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "street, city, state, postal_code, phone_home, country_code left blank", + "fault": "partial", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "the target saved twice", + "fault": "duplicate", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "nothing saved", + "fault": "missing", + "outcome": "halted_before_effect", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "the target lands after the check's timeout", + "fault": "delayed", + "outcome": "reconciliation_required", + "result": "killed" + } + ], + "check_sha256": "sha256:e788e0dad8351168546da5e87e646859f9e0f47eb8e98508e272f1a8a5462526", + "claims": [ + "delta", + "frame" + ], + "classes": { + "delayed": { + "applicable": 1, + "killed": 1 + }, + "duplicate": { + "applicable": 1, + "killed": 1 + }, + "missing": { + "applicable": 1, + "killed": 1 + }, + "partial": { + "applicable": 1, + "killed": 1 + }, + "unrelated_edit": { + "applicable": 1, + "killed": 1 + }, + "wrong_record": { + "applicable": 1, + "killed": 1 + }, + "wrong_value": { + "applicable": 12, + "killed": 12 + } + }, + "clean_pass": true, + "killed": 18, + "schema_version": "openadapt.check-coverage/v1", + "suite": { + "id": "openadapt.fault-suite/public", + "version": "1.0.0" + }, + "test_copy": "sqlite" + }, + "synthesized": { + "applicable": 7, + "cases": [ + { + "applicable": false, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "the save as the app makes it", + "fault": "clean", + "outcome": "reconciliation_required", + "result": "not applicable" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "log_id changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "the values saved on a look-alike record", + "fault": "wrong_record", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "another record in the slice edited", + "fault": "unrelated_edit", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "log_id left blank", + "fault": "partial", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "the target saved twice", + "fault": "duplicate", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "nothing saved", + "fault": "missing", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "the target lands after the check's timeout", + "fault": "delayed", + "outcome": "reconciliation_required", + "result": "killed" + } + ], + "check_sha256": "sha256:00b5b6f39e4dbbbfa3feb55fdb4f14cd794851c1b36bff52ca97f482b21b6519", + "claims": [ + "delta", + "frame" + ], + "classes": { + "delayed": { + "applicable": 1, + "killed": 1 + }, + "duplicate": { + "applicable": 1, + "killed": 1 + }, + "missing": { + "applicable": 1, + "killed": 1 + }, + "partial": { + "applicable": 1, + "killed": 1 + }, + "unrelated_edit": { + "applicable": 1, + "killed": 1 + }, + "wrong_record": { + "applicable": 1, + "killed": 1 + }, + "wrong_value": { + "applicable": 1, + "killed": 1 + } + }, + "clean_pass": false, + "killed": 7, + "schema_version": "openadapt.check-coverage/v1", + "suite": { + "id": "openadapt.fault-suite/public", + "version": "1.0.0" + }, + "test_copy": "sqlite" + } + }, + "synthesized": { + "closed_world": "database", + "coverage": { + "applicable": 7, + "cases": [ + { + "applicable": false, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "the save as the app makes it", + "fault": "clean", + "outcome": "reconciliation_required", + "result": "not applicable" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "log_id changed", + "fault": "wrong_value", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "the values saved on a look-alike record", + "fault": "wrong_record", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "another record in the slice edited", + "fault": "unrelated_edit", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "detail": "log_id left blank", + "fault": "partial", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "the target saved twice", + "fault": "duplicate", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "nothing saved", + "fault": "missing", + "outcome": "reconciliation_required", + "result": "killed" + }, + { + "applicable": true, + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "detail": "the target lands after the check's timeout", + "fault": "delayed", + "outcome": "reconciliation_required", + "result": "killed" + } + ], + "check_sha256": "sha256:00b5b6f39e4dbbbfa3feb55fdb4f14cd794851c1b36bff52ca97f482b21b6519", + "claims": [ + "delta", + "frame" + ], + "classes": { + "delayed": { + "applicable": 1, + "killed": 1 + }, + "duplicate": { + "applicable": 1, + "killed": 1 + }, + "missing": { + "applicable": 1, + "killed": 1 + }, + "partial": { + "applicable": 1, + "killed": 1 + }, + "unrelated_edit": { + "applicable": 1, + "killed": 1 + }, + "wrong_record": { + "applicable": 1, + "killed": 1 + }, + "wrong_value": { + "applicable": 1, + "killed": 1 + } + }, + "clean_pass": false, + "killed": 7, + "schema_version": "openadapt.check-coverage/v1", + "suite": { + "id": "openadapt.fault-suite/public", + "version": "1.0.0" + }, + "test_copy": "sqlite" + }, + "doc": { + "effects": [ + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=api_log|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=api_log|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=clinical_rules_log|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=clinical_rules_log.content|no record that was there before changes or disappears (records compared by content, not id)", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=contact|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=contact.content|no record that was there before changes or disappears (records compared by content, not id)", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=documents|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=documents|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=employer_data|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=employer_data|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=facility|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=facility.content|no record that was there before changes or disappears (records compared by content, not id)", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=form_encounter|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=form_encounter|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=forms|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=forms|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=history_data|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=history_data.content|no record that was there before changes or disappears (records compared by content, not id)", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=insurance_data|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=insurance_data|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=lists|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=lists|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=log|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=log.content|no record that was there before changes or disappears (records compared by content, not id)", + "risk": "irreversible" + }, + { + "count_new_only": true, + "kind": "record_written", + "match": { + "log_id": { + "param": "providerID" + } + }, + "probe": "surface=log_comment_encrypt|exactly one new record with the run's values; no record that was there before may disappear", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=log_comment_encrypt.content|no record that was there before changes or disappears (records compared by content, not id)", + "risk": "irreversible" + }, + { + "count_new_only": true, + "kind": "record_written", + "match": { + "log_id": { + "param": "providerID" + } + }, + "probe": "surface=log_comment_encrypt_all|exactly one new record with the run's values; no record that was there before may disappear", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=log_comment_encrypt_all.content|no record that was there before changes or disappears (records compared by content, not id)", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=openemr_postcalendar_events|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=openemr_postcalendar_events|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "kind": "record_written", + "match": { + "DOB": { + "param": "DOB" + }, + "city": { + "param": "city" + }, + "country_code": { + "param": "country_code" + }, + "email": { + "param": "email" + }, + "fname": { + "param": "fname" + }, + "lname": { + "param": "lname" + }, + "phone_home": { + "param": "phone_home" + }, + "postal_code": { + "param": "postal_code" + }, + "providerID": { + "param": "providerID" + }, + "sex": { + "param": "sex" + }, + "street": { + "param": "street" + }, + "title": { + "param": "title" + } + }, + "probe": "surface=patient_data|exactly one new record with the run's values; no record that was there before may disappear", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=patient_data.content|no record that was there before changes or disappears (records compared by content, not id)", + "risk": "irreversible" + }, + { + "count_new_only": true, + "kind": "record_written", + "match": { + "DOB": { + "param": "DOB" + }, + "city": { + "param": "city" + }, + "country_code": { + "param": "country_code" + }, + "email": { + "param": "email" + }, + "fname": { + "param": "fname" + }, + "lname": { + "param": "lname" + }, + "phone_home": { + "param": "phone_home" + }, + "postal_code": { + "param": "postal_code" + }, + "providerID": { + "param": "providerID" + }, + "sex": { + "param": "sex" + }, + "street": { + "param": "street" + }, + "title": { + "param": "title" + } + }, + "probe": "surface=patient_data_all|exactly one new record with the run's values; no record that was there before may disappear", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=patient_data_all.content|no record that was there before changes or disappears (records compared by content, not id)", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=pnotes|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=pnotes|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=prescriptions|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=prescriptions|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=recent_patients|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=recent_patients|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=sequences|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=sequences.content|no record that was there before changes or disappears (records compared by content, not id)", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=transactions|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=transactions|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=user_settings|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=user_settings|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=users|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=users.content|no record that was there before changes or disappears (records compared by content, not id)", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=uuid_mapping|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=uuid_mapping|no row here changes or disappears", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "forbid_collateral_loss": false, + "kind": "record_written", + "probe": "surface=uuid_registry|the save adds no row here", + "risk": "irreversible" + }, + { + "count_new_only": true, + "expected_count": 0, + "kind": "record_written", + "match": { + "_openadapt_no_such_field": { + "literal": "-" + } + }, + "probe": "surface=uuid_registry|no row here changes or disappears", + "risk": "irreversible" + } + ], + "schema_version": "openadapt.verify/v1", + "verifier": { + "kind": "surfaces", + "surfaces": { + "api_log": { + "kind": "sql", + "sql_query": "SELECT * FROM \"api_log\"", + "sqlite_database": "" + }, + "clinical_rules_log": { + "kind": "sql", + "sql_query": "SELECT * FROM \"clinical_rules_log\"", + "sqlite_database": "" + }, + "clinical_rules_log.content": { + "kind": "sql", + "records_drop_fields": [ + "id" + ], + "sql_query": "SELECT * FROM \"clinical_rules_log\"", + "sqlite_database": "" + }, + "contact": { + "kind": "sql", + "sql_query": "SELECT * FROM \"contact\"", + "sqlite_database": "" + }, + "contact.content": { + "kind": "sql", + "records_drop_fields": [ + "id" + ], + "sql_query": "SELECT * FROM \"contact\"", + "sqlite_database": "" + }, + "documents": { + "kind": "sql", + "sql_query": "SELECT * FROM \"documents\"", + "sqlite_database": "" + }, + "employer_data": { + "kind": "sql", + "sql_query": "SELECT * FROM \"employer_data\"", + "sqlite_database": "" + }, + "facility": { + "kind": "sql", + "sql_query": "SELECT * FROM \"facility\"", + "sqlite_database": "" + }, + "facility.content": { + "kind": "sql", + "records_drop_fields": [ + "id" + ], + "sql_query": "SELECT * FROM \"facility\"", + "sqlite_database": "" + }, + "form_encounter": { + "kind": "sql", + "sql_query": "SELECT * FROM \"form_encounter\"", + "sqlite_database": "" + }, + "forms": { + "kind": "sql", + "sql_query": "SELECT * FROM \"forms\"", + "sqlite_database": "" + }, + "history_data": { + "kind": "sql", + "sql_query": "SELECT * FROM \"history_data\"", + "sqlite_database": "" + }, + "history_data.content": { + "kind": "sql", + "records_drop_fields": [ + "id" + ], + "sql_query": "SELECT * FROM \"history_data\"", + "sqlite_database": "" + }, + "insurance_data": { + "kind": "sql", + "sql_query": "SELECT * FROM \"insurance_data\"", + "sqlite_database": "" + }, + "lists": { + "kind": "sql", + "sql_query": "SELECT * FROM \"lists\"", + "sqlite_database": "" + }, + "log": { + "kind": "sql", + "sql_query": "SELECT * FROM \"log\"", + "sqlite_database": "" + }, + "log.content": { + "kind": "sql", + "records_drop_fields": [ + "id" + ], + "sql_query": "SELECT * FROM \"log\"", + "sqlite_database": "" + }, + "log_comment_encrypt": { + "kind": "sql", + "sql_query": "SELECT * FROM \"log_comment_encrypt\" WHERE \"log_id\" = :p0", + "sql_query_params": { + "p0": { + "param": "providerID" + } + }, + "sqlite_database": "" + }, + "log_comment_encrypt.content": { + "kind": "sql", + "records_drop_fields": [ + "id" + ], + "sql_query": "SELECT * FROM \"log_comment_encrypt\" WHERE \"log_id\" = :p0", + "sql_query_params": { + "p0": { + "param": "providerID" + } + }, + "sqlite_database": "" + }, + "log_comment_encrypt_all": { + "kind": "sql", + "sql_query": "SELECT * FROM \"log_comment_encrypt\"", + "sqlite_database": "" + }, + "log_comment_encrypt_all.content": { + "kind": "sql", + "records_drop_fields": [ + "id" + ], + "sql_query": "SELECT * FROM \"log_comment_encrypt\"", + "sqlite_database": "" + }, + "openemr_postcalendar_events": { + "kind": "sql", + "sql_query": "SELECT * FROM \"openemr_postcalendar_events\"", + "sqlite_database": "" + }, + "patient_data": { + "kind": "sql", + "sql_query": "SELECT * FROM \"patient_data\" WHERE \"DOB\" = :p0 AND \"city\" = :p1 AND \"country_code\" = :p2 AND \"email\" = :p3 AND \"fname\" = :p4 AND \"lname\" = :p5 AND \"phone_home\" = :p6 AND \"postal_code\" = :p7 AND \"providerID\" = :p8 AND \"sex\" = :p9 AND \"title\" = :p10", + "sql_query_params": { + "p0": { + "param": "DOB" + }, + "p1": { + "param": "city" + }, + "p10": { + "param": "title" + }, + "p2": { + "param": "country_code" + }, + "p3": { + "param": "email" + }, + "p4": { + "param": "fname" + }, + "p5": { + "param": "lname" + }, + "p6": { + "param": "phone_home" + }, + "p7": { + "param": "postal_code" + }, + "p8": { + "param": "providerID" + }, + "p9": { + "param": "sex" + } + }, + "sqlite_database": "" + }, + "patient_data.content": { + "kind": "sql", + "records_drop_fields": [ + "id" + ], + "sql_query": "SELECT * FROM \"patient_data\" WHERE \"DOB\" = :p0 AND \"city\" = :p1 AND \"country_code\" = :p2 AND \"email\" = :p3 AND \"fname\" = :p4 AND \"lname\" = :p5 AND \"phone_home\" = :p6 AND \"postal_code\" = :p7 AND \"providerID\" = :p8 AND \"sex\" = :p9 AND \"title\" = :p10", + "sql_query_params": { + "p0": { + "param": "DOB" + }, + "p1": { + "param": "city" + }, + "p10": { + "param": "title" + }, + "p2": { + "param": "country_code" + }, + "p3": { + "param": "email" + }, + "p4": { + "param": "fname" + }, + "p5": { + "param": "lname" + }, + "p6": { + "param": "phone_home" + }, + "p7": { + "param": "postal_code" + }, + "p8": { + "param": "providerID" + }, + "p9": { + "param": "sex" + } + }, + "sqlite_database": "" + }, + "patient_data_all": { + "kind": "sql", + "sql_query": "SELECT * FROM \"patient_data\"", + "sqlite_database": "" + }, + "patient_data_all.content": { + "kind": "sql", + "records_drop_fields": [ + "id" + ], + "sql_query": "SELECT * FROM \"patient_data\"", + "sqlite_database": "" + }, + "pnotes": { + "kind": "sql", + "sql_query": "SELECT * FROM \"pnotes\"", + "sqlite_database": "" + }, + "prescriptions": { + "kind": "sql", + "sql_query": "SELECT * FROM \"prescriptions\"", + "sqlite_database": "" + }, + "recent_patients": { + "kind": "sql", + "sql_query": "SELECT * FROM \"recent_patients\"", + "sqlite_database": "" + }, + "sequences": { + "kind": "sql", + "sql_query": "SELECT * FROM \"sequences\"", + "sqlite_database": "" + }, + "sequences.content": { + "kind": "sql", + "records_drop_fields": [ + "id" + ], + "sql_query": "SELECT * FROM \"sequences\"", + "sqlite_database": "" + }, + "transactions": { + "kind": "sql", + "sql_query": "SELECT * FROM \"transactions\"", + "sqlite_database": "" + }, + "user_settings": { + "kind": "sql", + "sql_query": "SELECT * FROM \"user_settings\"", + "sqlite_database": "" + }, + "users": { + "kind": "sql", + "sql_query": "SELECT * FROM \"users\"", + "sqlite_database": "" + }, + "users.content": { + "kind": "sql", + "records_drop_fields": [ + "id" + ], + "sql_query": "SELECT * FROM \"users\"", + "sqlite_database": "" + }, + "uuid_mapping": { + "kind": "sql", + "sql_query": "SELECT * FROM \"uuid_mapping\"", + "sqlite_database": "" + }, + "uuid_registry": { + "kind": "sql", + "sql_query": "SELECT * FROM \"uuid_registry\"", + "sqlite_database": "" + } + } + } + }, + "gate_error": null, + "misses": [ + "stale: log_comment_encrypt: the save destroyed another record of the same subject (every checked field identifies the subject, so another record of the same subject looks like the saved one)", + "stale: log_comment_encrypt_all: the save destroyed another record of the same subject (every checked field identifies the subject, so another record of the same subject looks like the saved one)", + "read: facility: the generated query returns 1 row(s), the scan saw 1", + "read: facility.content: the generated query returns 1 row(s), the scan saw 1", + "read: history_data: the generated query returns 4 row(s), the scan saw 4", + "read: history_data.content: the generated query returns 4 row(s), the scan saw 4", + "read: patient_data: the generated query returns 1 row(s), the scan saw 1", + "read: patient_data.content: the generated query returns 1 row(s), the scan saw 1", + "read: patient_data_all: the generated query returns 4 row(s), the scan saw 4", + "read: patient_data_all.content: the generated query returns 4 row(s), the scan saw 4", + "read: uuid_registry: the generated query returns 9 row(s), the scan saw 9", + "fault gate: the clean save did not verify on the copy" + ], + "params": [ + "DOB", + "city", + "country_code", + "email", + "fname", + "lname", + "phone_home", + "postal_code", + "providerID", + "sex", + "street", + "title" + ], + "surfaces": [ + "api_log", + "clinical_rules_log", + "contact", + "documents", + "employer_data", + "facility", + "form_encounter", + "forms", + "history_data", + "insurance_data", + "lists", + "log", + "log_comment_encrypt", + "log_comment_encrypt_all", + "openemr_postcalendar_events", + "patient_data", + "patient_data_all", + "pnotes", + "prescriptions", + "recent_patients", + "sequences", + "transactions", + "user_settings", + "users", + "uuid_mapping", + "uuid_registry" + ], + "targets": [ + { + "bound": { + "log_id": "providerID" + }, + "surface": "log_comment_encrypt" + }, + { + "bound": { + "log_id": "providerID" + }, + "surface": "log_comment_encrypt_all" + }, + { + "bound": { + "DOB": "DOB", + "city": "city", + "country_code": "country_code", + "email": "email", + "fname": "fname", + "lname": "lname", + "phone_home": "phone_home", + "postal_code": "postal_code", + "providerID": "providerID", + "sex": "sex", + "street": "street", + "title": "title" + }, + "surface": "patient_data" + }, + { + "bound": { + "DOB": "DOB", + "city": "city", + "country_code": "country_code", + "email": "email", + "fname": "fname", + "lname": "lname", + "phone_home": "phone_home", + "postal_code": "postal_code", + "providerID": "providerID", + "sex": "sex", + "street": "street", + "title": "title" + }, + "surface": "patient_data_all" + } + ], + "working": false + } +} diff --git a/packages/flow/benchmark/verification_coverage/independent/results/standin-slice/rows.json b/packages/flow/benchmark/verification_coverage/independent/results/standin-slice/rows.json new file mode 100644 index 000000000..392d6921b --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/results/standin-slice/rows.json @@ -0,0 +1,655 @@ +[ + { + "class": "clean", + "expect": "clean", + "framed": { + "checked": { + "delta": "held", + "frame": "held", + "source": "held" + }, + "outcome": "verified", + "score": "ok", + "verified": true + }, + "id": "clean", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "ok", + "shipped": { + "checked": { + "delta": "held", + "frame": "held", + "source": "instruction" + }, + "outcome": "verified", + "score": "ok", + "verified": true + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "false_alarm", + "verified": false + } + }, + { + "class": "wrong_record", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "held" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "id": "wrong_record_existing_chart", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "missed", + "shipped": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "instruction" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "wrong_record", + "expect": "fault", + "framed": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "id": "wrong_record_lookalike", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "missed", + "shipped": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "instruction" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "unrelated_change", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "held" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "id": "unrelated_other_chart", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "missed", + "shipped": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "instruction" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "unrelated_change", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "held" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "id": "unrelated_other_history", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "missed", + "shipped": { + "checked": { + "delta": "held", + "frame": "held", + "source": "instruction" + }, + "outcome": "verified", + "score": "missed", + "verified": true + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "unrelated_change", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "held" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "id": "unrelated_table_note_other_patient", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "missed", + "shipped": { + "checked": { + "delta": "held", + "frame": "held", + "source": "instruction" + }, + "outcome": "verified", + "score": "missed", + "verified": true + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "unrelated_change", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "held" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "id": "unrelated_table_allergy_other_patient", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "missed", + "shipped": { + "checked": { + "delta": "held", + "frame": "held", + "source": "instruction" + }, + "outcome": "verified", + "score": "missed", + "verified": true + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "unrelated_change", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "held", + "source": "held" + }, + "outcome": "verified", + "score": "missed", + "verified": true + }, + "id": "unrelated_table_insurance_new_patient", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "missed", + "shipped": { + "checked": { + "delta": "held", + "frame": "held", + "source": "instruction" + }, + "outcome": "verified", + "score": "missed", + "verified": true + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "unrelated_change", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "held" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "id": "unrelated_other_chart_unattributed", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "missed", + "shipped": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "instruction" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "misread_source", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "held", + "source": "refuted" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "id": "misread_dob", + "presave": "hold", + "presave_held_fields": [ + "DOB" + ], + "presave_score": "caught", + "shipped": { + "checked": { + "delta": "held", + "frame": "held", + "source": "instruction" + }, + "outcome": "verified", + "score": "missed", + "verified": true + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "misread_source", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "held", + "source": "refuted" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "id": "misread_swapped_names", + "presave": "hold", + "presave_held_fields": [ + "fname", + "lname" + ], + "presave_score": "caught", + "shipped": { + "checked": { + "delta": "held", + "frame": "held", + "source": "instruction" + }, + "outcome": "verified", + "score": "missed", + "verified": true + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "misread_source", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "held", + "source": "held" + }, + "outcome": "verified", + "score": "missed", + "verified": true + }, + "id": "misread_wrong_provider", + "presave": "hold", + "presave_held_fields": [ + "provider" + ], + "presave_score": "caught", + "shipped": { + "checked": { + "delta": "held", + "frame": "held", + "source": "instruction" + }, + "outcome": "verified", + "score": "missed", + "verified": true + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "missing_side_record", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "held", + "source": "held" + }, + "outcome": "verified", + "score": "missed", + "verified": true + }, + "id": "missing_history_record", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "missed", + "shipped": { + "checked": { + "delta": "held", + "frame": "held", + "source": "instruction" + }, + "outcome": "verified", + "score": "missed", + "verified": true + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "duplicate", + "expect": "fault", + "framed": { + "checked": { + "delta": "refuted", + "frame": "held", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "id": "duplicate_save", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "missed", + "shipped": { + "checked": { + "delta": "refuted", + "frame": "held", + "source": "instruction" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + }, + { + "class": "concurrent_legitimate", + "expect": "clean", + "framed": { + "checked": { + "delta": "held", + "frame": "held", + "source": "held" + }, + "outcome": "verified", + "score": "ok", + "verified": true + }, + "id": "concurrent_other_chart", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "ok", + "shipped": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "instruction" + }, + "outcome": "reconciliation_required", + "score": "false_alarm", + "verified": false + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "false_alarm", + "verified": false + } + }, + { + "class": "concurrent_legitimate", + "expect": "clean", + "framed": { + "checked": { + "delta": "held", + "frame": "held", + "source": "held" + }, + "outcome": "verified", + "score": "ok", + "verified": true + }, + "id": "concurrent_new_patient", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "ok", + "shipped": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "instruction" + }, + "outcome": "reconciliation_required", + "score": "false_alarm", + "verified": false + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "false_alarm", + "verified": false + } + }, + { + "class": "unrelated_change", + "expect": "fault", + "framed": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "held" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "id": "concurrent_plus_bot_stray", + "presave": "agree", + "presave_held_fields": [], + "presave_score": "missed", + "shipped": { + "checked": { + "delta": "held", + "frame": "refuted", + "source": "instruction" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + }, + "synthesized": { + "checked": { + "delta": "refuted", + "frame": "refuted", + "source": "not_checked" + }, + "outcome": "reconciliation_required", + "score": "caught", + "verified": false + } + } +] diff --git a/packages/flow/benchmark/verification_coverage/independent/run.py b/packages/flow/benchmark/verification_coverage/independent/run.py new file mode 100644 index 000000000..797bcc74e --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/run.py @@ -0,0 +1,589 @@ +"""Run ``faults.yaml`` through OpenAdapt's checks and score them against SPEC.md. + + python -m verification_coverage.independent.run --baseline-sql PATH --out DIR + +Each case starts from a fresh SQLite copy of the pinned OpenEMR fixture's +baseline (``standin.py``), seeded with three charts and four users. The checks +under test are built before the case runs, the way OpenAdapt builds them: + +- ``shipped``: the contract the OpenEMR referral task ships with + (``scoreboard/claims_openemr.py`` ``VERIFY_YAML``), reading the whole + patient table, with verify-run's default frame; +- ``framed``: the same, plus the frame with OpenEMR's audit log for + attribution (as ``verification_coverage/openemr_run.py`` declares it) and + provenance with the referral document (the round trip, #1446); +- ``synthesized``: what ``scaffold-verifier --db --fault-gate`` proposes from a + demonstration on its own copy (``record_check_synthesis.synthesize``). + +For every check the fault gate (#1445) also runs on the authoring copy, so its +own coverage claim can be set beside the independent result. + +Verdicts: a check *catches* a ``fault`` case when verify-run does not issue +``verified``; it *misses* when it does. On a ``clean`` case, anything but +``verified`` is a *false alarm*. The pre-save provenance cross-check is scored +separately (it would stop the save before it happens). +""" + +from __future__ import annotations + +import argparse +import copy +import json +import sys +import tempfile +from pathlib import Path +from typing import Any, Optional + +import yaml + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parents[1])) + +from verification_coverage.independent import standin # noqa: E402 + +BOT = "openadapt_actor" +OTHER = "clinician2" + +REFERRAL = { + "title": "Ms.", + "fname": "Ada", + "lname": "Quill", + "DOB": "1984-03-07", + "sex": "Female", + "street": "41 Larch Way", + "city": "Cambridge", + "state_label": "Massachusetts", + "postal_code": "02139", + "phone_home": "617-555-0142", + "email": "ada.quill@example.test", + "country_code": "USA", + "provider": "Dr. Iris Holt", +} +#: The demonstration the synthesized check is built from (another person). +DEMO = { + "title": "Mr.", + "fname": "Tobias", + "lname": "Wren", + "DOB": "1971-09-30", + "sex": "Male", + "street": "12 Quarry Lane", + "city": "Medford", + "state_label": "Massachusetts", + "postal_code": "02155", + "phone_home": "781-555-0110", + "email": "t.wren@example.test", + "country_code": "USA", + "provider": "Dr. Iris Holt", +} +SEEDS = { + "other_patient": { + "title": "Mr.", + "fname": "Wendell", + "lname": "Priorchart", + "DOB": "1962-06-14", + "sex": "Male", + "street": "9 Existing Road", + "city": "Exampleton", + "state": "MA", + "postal_code": "02139", + "phone_home": "617-555-0188", + "email": "w.priorchart@example.test", + "country_code": "USA", + }, + "lookalike": { + "title": "Ms.", + "fname": "Ada", + "lname": "Quill", + "DOB": "1984-07-03", + "sex": "Female", + "street": "7 Elm Street", + "city": "Lowell", + "state": "MA", + "postal_code": "01852", + "phone_home": "978-555-0123", + "email": "aquill84@example.test", + "country_code": "USA", + }, + "second_other_patient": { + "title": "Mrs.", + "fname": "Mabel", + "lname": "Ortiz", + "DOB": "1990-01-21", + "sex": "Female", + "street": "300 Main Street", + "city": "Waltham", + "state": "MA", + "postal_code": "02451", + "phone_home": "781-555-0164", + "email": "m.ortiz@example.test", + "country_code": "USA", + }, +} +CONCURRENT_PERSON = { + "title": "Mr.", + "fname": "Basil", + "lname": "Fenn", + "DOB": "1979-11-22", + "sex": "Male", + "street": "5 Pond Road", + "city": "Arlington", + "state": "MA", + "postal_code": "02474", + "phone_home": "781-555-0131", + "email": "b.fenn@example.test", + "country_code": "USA", +} +#: Referral label per parameter, in document order (provider is last). +LABELS = ( + ("title", "Title"), + ("fname", "First name"), + ("lname", "Last name"), + ("DOB", "Date of birth"), + ("sex", "Sex"), + ("street", "Street"), + ("city", "City"), + ("state_label", "State"), + ("postal_code", "Postal code"), + ("phone_home", "Home phone"), + ("email", "Email"), + ("country_code", "Country"), + ("provider", "Primary care provider"), +) +#: Parameters the shipped contract binds (state is a literal MA there). +SHIPPED_PARAMS = ( + "title", + "fname", + "lname", + "DOB", + "sex", + "street", + "city", + "postal_code", + "phone_home", + "email", + "country_code", +) +SHIPPED_RECORD = yaml.safe_load( + """ +name: patient +match: [email, lname] +duplicate_key: email +fields: + email: email + lname: lname + title: title + fname: fname + DOB: DOB + sex: sex + street: street + city: city + state: {literal: MA} + postal_code: postal_code + phone_home: phone_home + country_code: country_code +""" +) +#: ``--slice``: the tables SPEC.md names, the New Patient path's side tables, +#: and the bookkeeping OpenEMR writes for any signed-in user. +SLICE = ( + "patient_data", + "history_data", + "insurance_data", + "employer_data", + "lists", + "pnotes", + "transactions", + "form_encounter", + "forms", + "prescriptions", + "documents", + "openemr_postcalendar_events", + "users", + "facility", + "contact", + "recent_patients", + "user_settings", + "clinical_rules_log", + "log", + "log_comment_encrypt", + "api_log", + "uuid_registry", + "uuid_mapping", + "sequences", +) +AUDIT_QUERY = ( + "SELECT id, user, patient_id FROM log WHERE event LIKE 'patient-record-%' " + "AND event <> 'patient-record-select' AND patient_id > 0" +) + + +def referral_text(values: dict[str, str]) -> str: + lines = ["Referral (synthetic test document)", "Referred to: Example Family Clinic"] + lines += [f"{label}: {values[name]}" for name, label in LABELS] + return "\n".join(lines) + "\n" + + +def seed(store: standin.Store) -> dict[str, int]: + ids = { + OTHER: store.add_user(OTHER, "Casey", "Clinician", 1), + "iholt": store.add_user("iholt", "Iris", "Holt", 1), + "ohale": store.add_user("ohale", "Owen", "Hale", 1), + } + for key, values in SEEDS.items(): + ids[key] = store.register(BOT, {**values, "providerID": ids["ohale"]}) + return ids + + +def stored(values: dict[str, str], ids: dict[str, int]) -> dict[str, Any]: + """What a correct New Patient save stores for these referral values.""" + + out = {k: v for k, v in values.items() if k not in ("state_label", "provider")} + out["state"] = ( + "MA" if values["state_label"] == "Massachusetts" else values["state_label"] + ) + out["providerID"] = ids[ + "iholt" if values["provider"] == "Dr. Iris Holt" else "ohale" + ] + return out + + +# -- checks ---------------------------------------------------------------------- + + +def shipped_doc(db: Path) -> dict[str, Any]: + return { + "schema_version": "openadapt.verify/v1", + "verifier": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": "SELECT * FROM patient_data", + }, + "record": copy.deepcopy(SHIPPED_RECORD), + } + + +def framed_doc(db: Path) -> dict[str, Any]: + doc = shipped_doc(db) + doc["frame"] = { + "identity_field": "pid", + "attribution": { + "kind": "audit_read", + "actor_field": "user", + "record_field": "patient_id", + "self": [BOT], + "read": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": AUDIT_QUERY, + }, + }, + } + return doc + + +def _repoint(node: Any, old: str, new: str) -> Any: + if isinstance(node, dict): + return {k: _repoint(v, old, new) for k, v in node.items()} + if isinstance(node, list): + return [_repoint(v, old, new) for v in node] + return new if node == old else node + + +def synthesize_check(work: Path) -> dict[str, Any]: + """scaffold-verifier --db --fault-gate on a demonstration copy.""" + + from openadapt_flow.record_check_synthesis import synthesize + + demo_db = standin.copy(work / "baseline.sqlite", work / "demo.sqlite") + store = standin.Store(demo_db) + ids = seed(store) + values = stored(DEMO, ids) + store.register(BOT, values) + store.close() + params = {k: str(v) for k, v in values.items() if k != "state"} + from openadapt_flow.fault_gate import GateError + + gate_error = None + try: + proposal = synthesize(None, params=params, db_path=demo_db, fault_gate=True) + except GateError as exc: + # The gate refused to run (fail-closed); measure the check + # scaffold-verifier proposes without it. + gate_error = str(exc) + proposal = synthesize(None, params=params, db_path=demo_db, fault_gate=False) + effects = [ + e.model_copy(update={"needs_operator_confirmation": False}).model_dump( + mode="json", exclude_defaults=True + ) + for e in proposal.effects + ] + cov = proposal.coverage + return { + "doc": { + "schema_version": "openadapt.verify/v1", + "verifier": proposal.verifier, + "effects": effects, + }, + "demo_db": str(demo_db), + "params": sorted(params), + "working": proposal.working, + "misses": proposal.misses, + "surfaces": [s.name for s in proposal.search.surfaces], + "closed_world": proposal.search.closed_world_basis, + "coverage": None if cov is None else cov.model_dump(mode="json"), + "gate_error": gate_error, + "targets": [ + {"surface": x.name, "bound": dict(x.bound)} + for x in proposal.search.surfaces + if x.is_target + ], + } + + +def gate_shipped(work: Path, framed: bool) -> dict[str, Any]: + """The fault gate (#1445) on the shipped contract, on an authoring copy.""" + + from openadapt_flow.fault_gate import SqliteCopy, run_gate + from openadapt_flow.verify_run import parse_verify_config + + src = standin.copy(work / "baseline.sqlite", work / f"gate-{framed}.sqlite") + store = standin.Store(src) + ids = seed(store) + store.close() + target = stored(DEMO, ids) + copy_ = SqliteCopy(src, "patient_data", target, workdir=work / f"gatecopy-{framed}") + doc = (framed_doc if framed else shipped_doc)(copy_.path) + params = {k: str(target[k]) for k in SHIPPED_PARAMS} + record = run_gate(parse_verify_config(doc), params, copy_) + return record.model_dump(mode="json") + + +# -- one case -------------------------------------------------------------------- + + +def apply_steps( + store: standin.Store, + case: dict[str, Any], + ids: dict[str, int], + told: dict[str, str], +) -> Optional[int]: + new_pid: Optional[int] = None + actor = {"bot": BOT, "clinician2": OTHER, "none": None} + for step in case["steps"]: + who = actor[step["actor"]] + op = step["op"] + if op == "register_referral": + if step.get("phase") == "finish": + continue + new_pid = store.register(who, stored(told, ids)) + elif op == "register_patient": + store.register(who, {**CONCURRENT_PERSON, "providerID": ids["iholt"]}) + elif op == "overwrite_chart": + values = stored(REFERRAL, ids) + store.update_chart(who, ids[step["target"]], values) + else: + pid = new_pid if step["target"] == "new_patient" else ids[step["target"]] + if op == "update_chart": + values = stored(REFERRAL, ids) + changes = { + k: (values[k] if v == "referral" else v) + for k, v in step["set"].items() + } + store.update_chart(who, pid, changes) + elif op == "update_history": + store.update_history(who, pid, step["set"]) + elif op == "delete_history": + store.delete_history(who, pid) + elif op == "add_note": + store.add_note(who, pid, step["body"]) + elif op == "add_allergy": + store.add_allergy(who, pid, step["title"]) + elif op == "add_insurance": + store.add_insurance(who, pid, step["provider"]) + else: + raise ValueError(f"unknown op {op}") + return new_pid + + +def provenance_for(doc_sha: str, told: dict[str, str], names: list[str]) -> Any: + from openadapt_flow.provenance import ProvenanceFile + + line_of = {name: 3 + i for i, (name, _l) in enumerate(LABELS)} + label_of = dict(LABELS) + params = {} + for name in names: + if name not in line_of: + continue + entry = { + "value": told[name], + "source": { + "kind": "document", + "sha256": doc_sha, + "line": line_of[name], + "label": label_of[name], + }, + "extractor": "model:simulated-first-reader", + } + if name == "DOB": + entry["value_kind"] = "date" + params[name] = entry + return ProvenanceFile.model_validate( + {"schema_version": "openadapt.param-provenance/v1", "params": params} + ) + + +def run_case(case: dict[str, Any], work: Path, synth: dict[str, Any]) -> dict[str, Any]: + from openadapt_flow.intake.document import IntakeDocument # noqa: F401 + from openadapt_flow.provenance import ( + cross_check, + default_extractor, + document_sha256, + read_sources, + ) + from openadapt_flow.verify_run import ( + capture_baseline, + parse_verify_config, + verify_run, + ) + + cdir = work / case["id"] + cdir.mkdir(parents=True, exist_ok=True) + db = standin.copy(work / "baseline.sqlite", cdir / "case.sqlite") + store = standin.Store(db) + ids = seed(store) + told = {**REFERRAL, **case.get("told", {})} + doc_path = cdir / "referral.txt" + doc_path.write_text(referral_text(REFERRAL)) + documents = read_sources([doc_path]) + doc_sha = document_sha256(doc_path) + told_stored = stored(told, ids) + shipped_params = {k: str(told_stored[k]) for k in SHIPPED_PARAMS} + synth_params = {k: str(told_stored[k]) for k in synth["params"]} + + checks = { + "shipped": (shipped_doc(db), shipped_params, None), + "framed": ( + framed_doc(db), + shipped_params, + provenance_for(doc_sha, told, list(SHIPPED_PARAMS)), + ), + "synthesized": ( + _repoint(synth["doc"], synth["demo_db"], str(db)), + synth_params, + provenance_for(doc_sha, told, list(SHIPPED_PARAMS)), + ), + } + cfgs = {name: parse_verify_config(doc) for name, (doc, _p, _v) in checks.items()} + for name, cfg in cfgs.items(): + capture_baseline(cfg, checks[name][1], cdir / f"baseline-{name}.json") + + prov = provenance_for(doc_sha, told, [n for n, _ in LABELS]) + pre = cross_check(prov, documents, default_extractor(prov)) + + apply_steps(store, case, ids, told) + store.close() + + row: dict[str, Any] = { + "id": case["id"], + "class": case["class"], + "expect": case["expect"], + "presave": pre.status, + "presave_held_fields": pre.disagreements, + } + for name, cfg in cfgs.items(): + _doc, params, provenance = checks[name] + result = verify_run( + cfg, + params=params, + run_record={"executor_kind": "agent", "executor_id": BOT}, + authorization_ref="independent-suite", + baseline_path=cdir / f"baseline-{name}.json", + provenance=provenance, + documents=documents if provenance is not None else None, + ) + receipt = result["receipt"] + outcome = receipt.get("terminal_outcome") or receipt.get("outcome") + row[name] = { + "outcome": outcome, + "verified": outcome == "verified", + "checked": receipt.get("checked"), + } + (cdir / f"evidence-{name}.json").write_text( + json.dumps(result.get("evidence"), indent=2, sort_keys=True, default=str) + ) + return row + + +def score(expect: str, verified: bool) -> str: + if expect == "fault": + return "missed" if verified else "caught" + return "ok" if verified else "false_alarm" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--baseline-sql", type=Path, required=True) + parser.add_argument("--out", type=Path, default=HERE / "results" / "standin") + parser.add_argument("--only", default="") + parser.add_argument("--work", type=Path, default=None) + parser.add_argument( + "--slice", + action="store_true", + help="keep only the tables SPEC.md names and OpenEMR's bookkeeping", + ) + args = parser.parse_args() + suite = yaml.safe_load((HERE / "faults.yaml").read_text()) + work = args.work or Path(tempfile.mkdtemp(prefix="indep-verif-")) + work.mkdir(parents=True, exist_ok=True) + if not (work / "baseline.sqlite").exists(): + standin.build(args.baseline_sql, work / "baseline.sqlite") + if args.slice: + standin.keep_tables(work / "baseline.sqlite", SLICE) + cache = work / "synth.json" + if cache.exists(): + synth = json.loads(cache.read_text()) + else: + synth = synthesize_check(work) + cache.write_text(json.dumps(synth, default=str)) + gates = { + "shipped": gate_shipped(work, False), + "framed": gate_shipped(work, True), + "synthesized": synth["coverage"], + } + rows = [] + for case in suite["cases"]: + if args.only and case["id"] not in args.only.split(","): + continue + row = run_case(case, work, synth) + for name in ("shipped", "framed", "synthesized"): + row[name]["score"] = score(case["expect"], row[name]["verified"]) + row["presave_score"] = score(case["expect"], row["presave"] == "agree") + rows.append(row) + print( + case["id"], + { + n: (row[n]["outcome"], row[n]["score"]) + for n in ("shipped", "framed", "synthesized") + }, + "presave", + row["presave"], + flush=True, + ) + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "rows.json").write_text( + json.dumps(rows, indent=2, sort_keys=True) + "\n" + ) + meta = {k: v for k, v in synth.items() if k not in ("doc", "demo_db")} + meta["doc"] = _repoint(synth["doc"], synth["demo_db"], "") + (args.out / "checks.json").write_text( + json.dumps( + {"synthesized": meta, "gates": gates}, indent=2, sort_keys=True, default=str + ) + + "\n" + ) + print("work dir:", work) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/verification_coverage/independent/standin.py b/packages/flow/benchmark/verification_coverage/independent/standin.py new file mode 100644 index 000000000..c284347c3 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/standin.py @@ -0,0 +1,395 @@ +"""An SQLite copy of the pinned OpenEMR fixture's baseline, and the ops the +fault suite runs on it. + +The stand-in is built from the fixture's own MariaDB dump (``baseline.sql``), +so every table, column and default is OpenEMR's, and every row OpenEMR's +installer writes is there. What it can't reproduce is OpenEMR's PHP: the ops +below write the rows the New Patient path and the edit screens write, and +the audit-log rows OpenEMR's ``EventAuditLogger`` writes for them. The live +run (``live.py``) checks those emulations against the real app. +""" + +from __future__ import annotations + +import base64 +import hashlib +import os +import re +import shutil +import sqlite3 +from pathlib import Path +from typing import Any, Iterator, Mapping, Optional + +_CREATE = re.compile(r"^CREATE TABLE `([A-Za-z0-9_]+)` \($") +_COLUMN = re.compile(r"^ `([A-Za-z0-9_]+)` (.*?),?$") +_DEFAULT = re.compile( + r"DEFAULT ('(?:[^'\\]|\\.)*'|NULL|-?[0-9.]+|current_timestamp\(\))", re.I +) +_INSERT = re.compile(r"^INSERT INTO `([A-Za-z0-9_]+)` VALUES ?(.*)$", re.S) + + +def _values(text: str) -> Iterator[list[Any]]: + """The rows of a mysqldump ``VALUES (..),(..)`` list.""" + + i, n = 0, len(text) + while i < n: + if text[i] != "(": + i += 1 + continue + i += 1 + row: list[Any] = [] + while True: + if text[i] == "'": + i += 1 + buf = [] + while text[i] != "'": + if text[i] == "\\": + nxt = text[i + 1] + buf.append( + {"n": "\n", "r": "\r", "t": "\t", "0": "\0"}.get(nxt, nxt) + ) + i += 2 + else: + buf.append(text[i]) + i += 1 + i += 1 + row.append("".join(buf)) + else: + j = i + while text[j] not in ",)": + j += 1 + tok = text[i:j] + i = j + if tok == "NULL": + row.append(None) + elif tok.startswith("_binary "): + row.append(tok) + else: + try: + row.append(int(tok)) + except ValueError: + try: + row.append(float(tok)) + except ValueError: + row.append(tok) + if text[i] == ",": + i += 1 + continue + if text[i] == ")": + i += 1 + break + yield row + + +def _blob(value: Any) -> Any: + """Binary columns (uuid) come out of the dump as undecodable text.""" + + if isinstance(value, str): + try: + value.encode("utf-8") + except UnicodeEncodeError: + return value.encode("utf-8", "surrogateescape") + return value + + +def _affinity(definition: str) -> str: + """The SQLite type affinity for a MariaDB column type. + + With it, ``'5' = 5`` holds in a query as it does in MariaDB; without it a + parameter bound as text never matches an integer column. + """ + + kind = definition.split("(", 1)[0].split(" ", 1)[0].lower() + if kind in ("int", "bigint", "tinyint", "smallint", "mediumint"): + return "INTEGER" + if kind in ("decimal", "float", "double", "real"): + return "REAL" + if kind in ("binary", "varbinary", "blob", "longblob", "mediumblob", "tinyblob"): + return "BLOB" + return "TEXT" + + +def _default_sql(definition: str) -> str: + m = _DEFAULT.search(definition) + if not m: + return "" + raw = m.group(1) + if raw.lower() == "current_timestamp()": + return " DEFAULT CURRENT_TIMESTAMP" + if raw.startswith("'"): + return " DEFAULT '" + raw[1:-1].replace("\\'", "''") + "'" + return f" DEFAULT {raw}" + + +def build(baseline_sql: Path, out: Path) -> dict[str, int]: + """Convert the dump into ``out`` (every table, every row). Row counts.""" + + out.unlink(missing_ok=True) + con = sqlite3.connect(out) + table: Optional[str] = None + columns: list[str] = [] + counts: dict[str, int] = {} + pending: Optional[tuple[str, list[str]]] = None + + def flush(name: str, body: str) -> None: + rows = [[_blob(v) for v in row] for row in _values(body)] + if rows: + marks = ",".join("?" * len(rows[0])) + con.executemany(f'INSERT INTO "{name}" VALUES ({marks})', rows) + counts[name] = counts.get(name, 0) + len(rows) + + with baseline_sql.open(encoding="utf-8", errors="surrogateescape") as fh: + for line in fh: + line = line.rstrip("\n") + if pending is not None: + pending[1].append(line) + if line.endswith(";"): + flush(pending[0], "\n".join(pending[1])[:-1]) + pending = None + continue + if table is None: + m = _CREATE.match(line) + if m: + table, columns = m.group(1), [] + continue + m = _INSERT.match(line) + if m: + if line.endswith(";"): + flush(m.group(1), m.group(2)[:-1]) + else: + pending = (m.group(1), [m.group(2)]) + continue + if line.startswith(")"): + con.execute(f'CREATE TABLE "{table}" ({", ".join(columns)})') + counts.setdefault(table, 0) + table = None + continue + m = _COLUMN.match(line) + if m: + name, definition = m.groups() + pk = ( + " PRIMARY KEY AUTOINCREMENT" + if "AUTO_INCREMENT" in definition + else "" + ) + columns.append( + f'"{name}" {_affinity(definition)}' + pk + _default_sql(definition) + ) + con.commit() + con.close() + return counts + + +def keep_tables(path: Path, keep: tuple[str, ...]) -> None: + """Drop every table not in ``keep`` (the clinical-slice stand-in).""" + + con = sqlite3.connect(path) + names = [ + r[0] + for r in con.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name NOT LIKE 'sqlite_%'" + ) + ] + for name in names: + if name not in keep: + con.execute(f'DROP TABLE "{name}"') + con.commit() + con.execute("VACUUM") + con.close() + + +def copy(src: Path, dst: Path) -> Path: + shutil.copyfile(src, dst) + return dst + + +class Store: + """The ops of ``faults.yaml`` on one stand-in database.""" + + def __init__(self, path: Path) -> None: + self.path = path + self.con = sqlite3.connect(path, isolation_level=None) + self.con.row_factory = sqlite3.Row + self.clock = 0 + + def close(self) -> None: + self.con.close() + + def _now(self) -> str: + self.clock += 1 + return f"2026-10-24 12:{self.clock // 60:02d}:{self.clock % 60:02d}" + + def one(self, sql: str, args: tuple = ()) -> Any: + row = self.con.execute(sql, args).fetchone() + return None if row is None else row[0] + + def log(self, user: Optional[str], event: str, pid: int, statement: str) -> None: + """An audit row as OpenEMR's EventAuditLogger writes it, or none. + + OpenEMR logs each statement base64-encoded in ``comments`` and pairs + every row with a ``log_comment_encrypt`` row holding its checksum. + """ + + if user is None: + return + comment = base64.b64encode(statement.encode()).decode() + cur = self.con.execute( + "INSERT INTO log (date, event, category, user, groupname, comments, " + "patient_id, success, log_from) VALUES (?,?,?,?,?,?,?,1,'open-emr')", + (self._now(), event, "Patient Demographics", user, "Default", comment, pid), + ) + self.con.execute( + "INSERT INTO log_comment_encrypt (log_id, encrypt, checksum, checksum_api, " + "version) VALUES (?, 'No', ?, '', 4)", + (cur.lastrowid, hashlib.sha512(comment.encode()).hexdigest()), + ) + + def _uuid(self, table: str, table_id: int) -> bytes: + uuid = os.urandom(16) + self.con.execute( + "INSERT INTO uuid_registry (uuid, table_name, table_id, table_vertical, " + "couchdb, document_drive, mapped, created) VALUES (?,?,?,'','',0,0,?)", + (uuid, table, str(table_id), self._now()), + ) + return uuid + + def user_id(self, username: str) -> int: + return int(self.one("SELECT id FROM users WHERE username = ?", (username,))) + + def add_user(self, username: str, fname: str, lname: str, authorized: int) -> int: + self.con.execute( + "INSERT INTO users (username, fname, lname, authorized, active, facility_id, " + "see_auth, cal_ui) VALUES (?,?,?,?,1,3,1,1)", + (username, fname, lname, authorized), + ) + return self.user_id(username) + + def register(self, user: Optional[str], v: Mapping[str, Any]) -> int: + """The New Patient path: patient_data plus its history_data row.""" + + pid = int(self.one("SELECT COALESCE(MAX(pid), 0) + 1 FROM patient_data")) + now = self._now() + cols = { + "pid": pid, + "pubpid": str(pid), + "date": now, + "regdate": now[:10], + "title": v["title"], + "fname": v["fname"], + "lname": v["lname"], + "DOB": v["DOB"], + "sex": v["sex"], + "street": v["street"], + "city": v["city"], + "state": v["state"], + "postal_code": v["postal_code"], + "phone_home": v["phone_home"], + "email": v["email"], + "country_code": v["country_code"], + "providerID": v.get("providerID"), + } + names = ",".join(f'"{k}"' for k in cols) + cur = self.con.execute( + f"INSERT INTO patient_data ({names}) VALUES ({','.join('?' * len(cols))})", + tuple(cols.values()), + ) + self.con.execute( + "UPDATE patient_data SET uuid = ? WHERE id = ?", + (self._uuid("patient_data", cur.lastrowid), cur.lastrowid), + ) + self.log( + user, "patient-record-insert", pid, f"INSERT INTO patient_data pid={pid}" + ) + cur = self.con.execute( + "INSERT INTO history_data (pid, date) VALUES (?, ?)", (pid, now) + ) + self.con.execute( + "UPDATE history_data SET uuid = ? WHERE id = ?", + (self._uuid("history_data", cur.lastrowid), cur.lastrowid), + ) + self.log( + user, "patient-record-insert", pid, f"INSERT INTO history_data pid={pid}" + ) + # The rest of what the New Patient path writes on the pinned app + # (benchmark/openemr_local/fixture.py EXPECTED_TABLE_DELTAS["compiled"]). + self.con.execute( + "INSERT INTO contact (foreign_table_name, foreign_id) VALUES ('patient_data', ?)", + (pid,), + ) + self.log(user, "other-insert", 0, f"INSERT INTO contact foreign_id={pid}") + uid = self.one("SELECT id FROM users WHERE username = ?", (user or "",)) + if uid is not None: + self.con.execute("DELETE FROM recent_patients WHERE user_id = ?", (uid,)) + self.con.execute( + "INSERT INTO recent_patients (user_id, patients) VALUES (?, ?)", + (uid, f'[{{"pid":{pid}}}]'), + ) + self.con.execute( + "INSERT OR REPLACE INTO user_settings VALUES (?, 'dem_expand', '1')", + (uid,), + ) + self.con.execute( + "INSERT OR REPLACE INTO user_settings VALUES (?, 'his_expand', '1')", + (uid,), + ) + self.con.execute( + "INSERT INTO clinical_rules_log (date, pid, uid, category, value, new_value, " + "facility_id) VALUES (?, ?, ?, 'clinical_reminder_widget', '', '', 3)", + (now, pid, uid), + ) + self.log( + user, "patient-record-select", pid, f"SELECT * FROM patient_data pid={pid}" + ) + return pid + + def update_chart( + self, user: Optional[str], pid: int, changes: Mapping[str, Any] + ) -> None: + sets = ", ".join(f'"{k}" = ?' for k in changes) + self.con.execute( + f"UPDATE patient_data SET {sets} WHERE pid = ?", (*changes.values(), pid) + ) + self.log(user, "patient-record-update", pid, f"UPDATE patient_data pid={pid}") + + def update_history( + self, user: Optional[str], pid: int, changes: Mapping[str, Any] + ) -> None: + sets = ", ".join(f'"{k}" = ?' for k in changes) + self.con.execute( + f"UPDATE history_data SET {sets} WHERE pid = ?", (*changes.values(), pid) + ) + self.log(user, "patient-record-update", pid, f"UPDATE history_data pid={pid}") + + def delete_history(self, user: Optional[str], pid: int) -> None: + self.con.execute("DELETE FROM history_data WHERE pid = ?", (pid,)) + self.log( + user, "patient-record-delete", pid, f"DELETE FROM history_data pid={pid}" + ) + + def add_note(self, user: Optional[str], pid: int, body: str) -> None: + self.con.execute( + "INSERT INTO pnotes (date, body, pid, user, groupname, activity, authorized, " + "title, assigned_to, deleted, message_status) " + "VALUES (?,?,?,?, 'Default', 1, 1, 'Unassigned', '', 0, 'New')", + (self._now(), body, pid, user or "admin"), + ) + self.log(user, "patient-record-insert", pid, f"INSERT INTO pnotes pid={pid}") + + def add_allergy(self, user: Optional[str], pid: int, title: str) -> None: + self.con.execute( + "INSERT INTO lists (date, type, title, activity, pid, user, groupname) " + "VALUES (?, 'allergy', ?, 1, ?, ?, 'Default')", + (self._now(), title, pid, user or "admin"), + ) + self.log(user, "patient-record-insert", pid, f"INSERT INTO lists pid={pid}") + + def add_insurance(self, user: Optional[str], pid: int, provider: str) -> None: + self.con.execute( + "INSERT INTO insurance_data (type, provider, pid, date) VALUES " + "('primary', ?, ?, ?)", + (provider, pid, self._now()[:10]), + ) + self.log( + user, "patient-record-insert", pid, f"INSERT INTO insurance_data pid={pid}" + ) diff --git a/packages/flow/benchmark/verification_coverage/independent/summarize.py b/packages/flow/benchmark/verification_coverage/independent/summarize.py new file mode 100644 index 000000000..67a04d383 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/summarize.py @@ -0,0 +1,102 @@ +"""Per fault class: caught, missed, false alarm, for each check. + + python -m verification_coverage.independent.summarize ROWS [ROWS ...] + +ROWS is a ``rows.json`` (stand-in) or ``rows.jsonl`` (live). Prints Markdown. +""" + +from __future__ import annotations + +import json +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any + +CLASSES = ( + "clean", + "concurrent_legitimate", + "wrong_record", + "unrelated_change", + "misread_source", + "missing_side_record", + "duplicate", +) +CHECKS = ("shipped", "framed", "synthesized", "presave") + + +def load(path: Path) -> list[dict[str, Any]]: + text = path.read_text() + if path.suffix == ".jsonl": + return [json.loads(line) for line in text.splitlines() if line.strip()] + return json.loads(text) + + +def cell(rows: list[dict[str, Any]], check: str) -> str: + scores = [ + r["presave_score"] if check == "presave" else r[check]["score"] + for r in rows + if (check == "presave" and "presave_score" in r) + or isinstance(r.get(check), dict) + ] + if not scores: + return "n/a" + n = len(scores) + if rows[0]["expect"] == "clean": + bad = scores.count("false_alarm") + return f"{n - bad}/{n} ok" + (f", **{bad} false alarm**" if bad else "") + caught = scores.count("caught") + missed = n - caught + return f"{caught}/{n} caught" + (f", **{missed} missed**" if missed else "") + + +def table(rows: list[dict[str, Any]]) -> str: + by_class: dict[str, list[dict[str, Any]]] = defaultdict(list) + for r in rows: + if "class" in r: + by_class[r["class"]].append(r) + checks = [c for c in CHECKS if any(c == "presave" or c in r for r in rows)] + out = ["| class | cases | " + " | ".join(checks) + " |"] + out.append("|---|---|" + "---|" * len(checks)) + for cls in CLASSES: + group = by_class.get(cls) + if not group: + continue + out.append( + f"| {cls} | {len(group)} | " + + " | ".join(cell(group, c) for c in checks) + + " |" + ) + return "\n".join(out) + + +def cases(rows: list[dict[str, Any]]) -> str: + checks = [c for c in CHECKS if c != "presave" and any(c in r for r in rows)] + out = ["| case | expect | " + " | ".join(checks) + " | pre-save |"] + out.append("|---|---|" + "---|" * (len(checks) + 1)) + for r in rows: + if "class" not in r: + out.append(f"| {r['id']} | error | {r.get('error', '')[:120]} |") + continue + out.append( + f"| {r['id']} | {r['expect']} | " + + " | ".join( + f"{r[c]['outcome']} ({r[c]['score']})" + if isinstance(r.get(c), dict) + else "n/a" + for c in checks + ) + + f" | {r.get('presave')} |" + ) + return "\n".join(out) + + +def main() -> int: + for arg in sys.argv[1:]: + rows = load(Path(arg)) + print(f"### {arg}\n\n{table(rows)}\n\n{cases(rows)}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/verification_coverage/openemr_gate.py b/packages/flow/benchmark/verification_coverage/openemr_gate.py new file mode 100644 index 000000000..bacfadff8 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/openemr_gate.py @@ -0,0 +1,159 @@ +"""The fault gate on an OpenEMR test copy (no model calls). + + python -m verification_coverage.openemr_gate --run-id + +Runs ``openadapt_flow.fault_gate.run_gate`` against the fixture: the clean +save is the actor's own ``POST /api/patient``; the faults are privileged SQL +on ``patient_data``; the check reads through OpenEMR's REST API with the +read-only oracle client, exactly as ``verify-run`` does in production. Two +checks are measured: + +- ``full``: the patient list, as in #1365 (``/api/patient``); +- ``scoped``: the same contract read through a search for the run's email, + the kind of read a synthesized check picks. It can't see another chart. + +Writes ``results/openemr//gate-.json``. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from pathlib import Path +from typing import Any, Mapping + +HERE = Path(__file__).resolve().parent +BENCH = HERE.parent +sys.path.insert(0, str(BENCH)) + +from scoreboard.claims_openemr import ( # noqa: E402 + EXISTING, + TOKEN_ENV, + VERIFY_YAML, + _token_env, + reset_and_seed, +) +from scoreboard.tasks import openemr # noqa: E402 + +FIELDS = openemr.SQL_FIELDS + + +def _sql_value(value: Any) -> str: + text = "" if value is None else str(value) + if not re.fullmatch(r"[A-Za-z0-9 ._@:/+-]*", text): + raise ValueError("unexpected characters in a synthetic value") + return f"'{text}'" + + +class OpenEMRCopy: + """The fixture as a :class:`openadapt_flow.fault_gate.TestCopy`.""" + + name = "openemr_fixture" + key_field = "id" + + def __init__(self, values: Mapping[str, str]) -> None: + self.values = dict(values) + + def reset(self) -> None: + reset_and_seed() + + def save(self) -> Any: + fx = openemr.fixture() + body = {k: v for k, v in self.values.items() if k != "state_label"} + body["state"] = "MA" + session = fx.token_session("actor") + try: + status = session.post( + f"{fx.api_base_url}/apis/default/api/patient", json=body + ).status_code + finally: + session.close() + if status not in (200, 201): + raise RuntimeError(f"the clean save returned HTTP {status}") + rows = [r for r in openemr.patients() if r["email"] == self.values["email"]] + return rows[-1]["id"] + + def rows(self) -> list[dict[str, Any]]: + return [dict(r) for r in openemr.patients()] + + def update(self, key: Any, changes: Mapping[str, Any]) -> None: + sets = ", ".join( + f"{name} = {_sql_value(value)}" + for name, value in changes.items() + if name in FIELDS and name != "id" + ) + openemr.fixture()._db( + f"UPDATE openemr.patient_data SET {sets} WHERE id = {int(key)}" + ) + + def insert(self, record: Mapping[str, Any]) -> Any: + db = openemr.fixture()._db + donor = db("SELECT MIN(id) FROM openemr.patient_data").decode().split()[0] + sets = ", ".join( + f"{name} = {_sql_value(value)}" + for name, value in record.items() + if name in FIELDS and name != "id" + ) + db( + "CREATE TEMPORARY TABLE openemr.vc_row AS SELECT * FROM " + f"openemr.patient_data WHERE id = {int(donor)}; " + "UPDATE openemr.vc_row SET id = (SELECT MAX(id) + 1 FROM " + "openemr.patient_data), pid = (SELECT MAX(pid) + 1 FROM " + "openemr.patient_data), uuid = UNHEX(REPLACE(UUID(), '-', ''))" + + (f", {sets}" if sets else "") + + "; INSERT INTO openemr.patient_data SELECT * FROM openemr.vc_row;" + ) + return db("SELECT MAX(id) FROM openemr.patient_data").decode().split()[0] + + def delete(self, key: Any) -> None: + openemr.fixture()._db(f"DELETE FROM openemr.patient_data WHERE id = {int(key)}") + + def verifier_overrides(self) -> dict[str, Any]: + return {} + + +def main() -> int: + import yaml + + from openadapt_flow.fault_gate import run_gate + from openadapt_flow.verify_run import parse_verify_config + + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--run-id", required=True) + parser.add_argument("--item", type=int, default=990) + args = parser.parse_args() + out = HERE / "results" / "openemr" / args.run_id + out.mkdir(parents=True, exist_ok=True) + fx = openemr.fixture() + values = openemr.item_values(args.item) + params = {k: v for k, v in values.items() if k != "state_label"} + os.environ.update(_token_env()) + full = yaml.safe_load(VERIFY_YAML.format(api=fx.ui_base_url, token_env=TOKEN_ENV)) + scoped = json.loads(json.dumps(full)) + scoped["verifier"]["records_path"] = "/apis/default/api/patient?email={email}" + scoped["verifier"]["path_params"] = {"email": {"param": "email"}} + assert EXISTING["email"] != values["email"] + summary = {} + for name, doc in (("full", full), ("scoped", scoped)): + cfg = parse_verify_config(doc) + record = run_gate(cfg, params, OpenEMRCopy(values)) + (out / f"gate-{name}.json").write_text( + json.dumps(record.model_dump(mode="json"), indent=2, sort_keys=True) + "\n" + ) + summary[name] = { + "killed": record.killed, + "applicable": record.applicable, + "clean_pass": record.clean_pass, + "survived": [c["fault"] for c in record.cases if c["result"] == "SURVIVED"], + } + print(name, summary[name], flush=True) + fx.reset() + (out / "gate-summary.json").write_text(json.dumps(summary, indent=2) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/verification_coverage/openemr_run.py b/packages/flow/benchmark/verification_coverage/openemr_run.py new file mode 100644 index 000000000..31fe35bca --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/openemr_run.py @@ -0,0 +1,533 @@ +"""Fresh agent runs on OpenEMR, judged by three engines, per fault class. + + python -m verification_coverage.openemr_run --run-id --per-fault 5 + +The rerun of OpenAdaptAI/OpenAdapt#1365's OpenEMR fault experiment that +``docs/VERIFICATION_COVERAGE.md`` asks for: fresh computer-use agent runs +(Claude Sonnet 5.5, the scoreboard's agent), not rebuilt database states. +Faults are injected only in the fixture's database, by the same triggers and +poller as ``scoreboard/claims_openemr.py``. Each item is judged by three +``verify-run`` engines, each from its own checkout and with its own baseline +taken before the agent starts: + +- ``e1365``: the engine #1365 measured with (collateral check finds a removed + record only); +- ``main``: main before this work (#1375's per-effect collateral change); +- ``after``: frame conditions with OpenEMR's audit log for attribution, and + the round trip against the referral document. + +Fault classes: the seven of #1365 plus three. + +========================= ==================================================== +``ok`` control: a clean save +``partial`` a trigger blanks the new chart's home phone +``wrong_value`` a trigger changes the new chart's postal code +``duplicate`` the poller copies the new chart into a second chart +``no_persist`` the poller deletes the new chart 3 s after it lands +``collateral_loss`` the poller deletes another, pre-existing chart +``collateral_edit`` the poller changes another chart's home phone +``collateral_edit_bot`` the same, logged in OpenEMR's audit log under the + bot's own user (``openadapt_actor``) +``concurrent_edit_other`` the same, logged under another user: a legitimate + concurrent edit, so the save must still verify +``misread_source`` the referral says 02139; the agent is told 02193 + (a misread before the save) and saves it faithfully +========================= ==================================================== + +The audit-log rows the poller writes for the two logged edits are shaped +like the ``patient-record-update`` rows OpenEMR writes for a UI edit. The +fixture's OpenEMR is unchanged; everything injected is in its database. +Synthetic data only. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import multiprocessing +import os +import secrets +import shutil +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +BENCH = HERE.parent +sys.path.insert(0, str(BENCH)) + +from scoreboard import schema # noqa: E402 +from scoreboard.claims_openemr import ( # noqa: E402 + TOKEN_ENV, + TRIGGERS, + VERIFY_YAML, + Poller, + _install_trigger, + _rows, + _token_env, + ground_truth, + reset_and_seed, +) +from scoreboard.run import ( # noqa: E402 + ITEM_TIMEOUT_S, + SPEND_CAP_USD, + _agent_item, + _ledger_view, + add_spend, + reported_success, + spent, +) +from scoreboard.tasks import openemr # noqa: E402 + +FAULTS = ( + "ok", + "partial", + "wrong_value", + "duplicate", + "no_persist", + "collateral_loss", + "collateral_edit", + "collateral_edit_bot", + "concurrent_edit_other", + "misread_source", +) +#: Faults after which the save is still correct (a false alarm if refused). +CLEAN = ("ok", "concurrent_edit_other") +ENGINES = ("e1365", "main", "after") +BOT_USER = "openadapt_actor" +OTHER_USER = "clinician2" +AUDIT_USER = "oa_audit_reader" +AUDIT_ENV = "VC_AUDIT_PASSWORD" +MISREAD = ("postal_code", "02193") + +FIELDS = ( + "item", + "fault", + "claimed_success", + "ground_truth", + *(f"{e}_outcome" for e in ENGINES), + *(f"{e}_checked" for e in ENGINES), + "presave_check", + "fault_applied", + "model_calls", + "cost_usd", + "host_load_1m", + "final_text", +) + +#: Each referral field, the label the referral document prints, its kind. +DOC_FIELDS = ( + ("title", "Title", None), + ("fname", "First name", None), + ("lname", "Last name", None), + ("DOB", "Date of birth", "date"), + ("sex", "Sex", None), + ("street", "Street", None), + ("city", "City", None), + ("state_label", "State", None), + ("postal_code", "Postal code", None), + ("phone_home", "Home phone", None), + ("email", "Email", None), + ("country_code", "Country", None), +) +#: Parameters the record contract binds (state is a literal there). +BOUND = tuple(name for name, _l, _k in DOC_FIELDS if name != "state_label") + + +def frame_yaml(db_container: str) -> str: + """The frame for the ``after`` engine: OpenEMR's audit log attributes.""" + + query = ( + "SELECT id, user, patient_id FROM log WHERE event LIKE 'patient-record-%' " + "AND event <> 'patient-record-select' AND patient_id > 0" + ) + return ( + "frame:\n" + " identity_field: pid\n" + " attribution:\n" + " kind: audit_read\n" + " actor_field: user\n" + " record_field: patient_id\n" + f" self: [{BOT_USER}]\n" + " read:\n" + " kind: sql\n" + " sql_driver: verification_coverage.exec_mariadb\n" + " sql_connect_args:\n" + f" container: {db_container}\n" + f" user: {AUDIT_USER}\n" + " database: openemr\n" + f" sql_password_env: {AUDIT_ENV}\n" + f' sql_query: "{query}"\n' + ) + + +def referral_text(values: dict[str, str]) -> str: + lines = ["Referral (synthetic test document)", "Referred to: Example Family Clinic"] + lines += [f"{label}: {values[name]}" for name, label, _k in DOC_FIELDS] + return "\n".join(lines) + "\n" + + +def provenance(doc_sha: str, values: dict[str, str]) -> dict[str, Any]: + """What a first reader extracted, citing each line of the referral.""" + + params = {} + for offset, (name, label, kind) in enumerate(DOC_FIELDS): + if name not in BOUND: + continue + entry: dict[str, Any] = { + "value": values[name], + "source": { + "kind": "document", + "sha256": doc_sha, + "line": 3 + offset, + "label": label, + }, + "extractor": "model:simulated-first-reader", + } + if kind: + entry["value_kind"] = kind + params[name] = entry + return {"schema_version": "openadapt.param-provenance/v1", "params": params} + + +class LoggedPoller(Poller): + """The scoreboard poller, plus edits recorded in OpenEMR's audit log.""" + + def __init__(self, fault: str, *args: Any, **kwargs: Any) -> None: + base = "collateral_edit" if fault.endswith(("_bot", "_other")) else fault + super().__init__(base, *args, **kwargs) + self.logged_as = ( + BOT_USER + if fault == "collateral_edit_bot" + else OTHER_USER + if fault == "concurrent_edit_other" + else None + ) + + def run(self) -> None: + super().run() + if self.logged_as and self.applied: + db = openemr.fixture()._db + pid = ( + db(f"SELECT pid FROM openemr.patient_data WHERE id = {self.victim}") + .decode() + .split()[0] + ) + db( + "INSERT INTO openemr.log (date, event, category, user, groupname, " + "comments, patient_id, success) VALUES (NOW(), " + "'patient-record-update', 'Patient Demographics', " + f"'{self.logged_as}', 'Default', '', {int(pid)}, 1)" + ) + self.applied += f" logged as {self.logged_as}" + + +def engine_env(engine_root: Path, extra: dict[str, str]) -> dict[str, str]: + packages = engine_root / "packages" + paths = [packages / "types", packages / "flow", BENCH] + return {**os.environ, **extra, "PYTHONPATH": os.pathsep.join(map(str, paths))} + + +def verify( + engine_root: Path, env: dict[str, str], params: dict[str, str], *argv: str +) -> subprocess.CompletedProcess[str]: + """verify-run from ``engine_root``, the item's values on stdin.""" + + return subprocess.run( + [ + sys.executable, + "-I", + "-c", + _LAUNCH, + "verify-run", + "--params", + "/dev/stdin", + *argv, + ], + input=json.dumps(params), + capture_output=True, + text=True, + timeout=300, + env=env, + cwd=str(engine_root), + ) + + +#: ``-I`` ignores PYTHONPATH, so put it back by hand: the engine's packages +#: and this benchmark (for the audit-read driver), nothing else of the caller. +_LAUNCH = ( + "import os, runpy, sys; " + "sys.path[:0] = os.environ['PYTHONPATH'].split(os.pathsep); " + "sys.argv = ['openadapt-flow'] + sys.argv[1:]; " + "runpy.run_module('openadapt_flow', run_name='__main__')" +) + + +def ensure_audit_reader(password: str) -> str: + """A SELECT-only MariaDB user for the audit log; returns the container.""" + + fx = openemr.fixture() + fx._db( + f"CREATE USER IF NOT EXISTS '{AUDIT_USER}'@'%' IDENTIFIED BY '{password}'; " + f"ALTER USER '{AUDIT_USER}'@'%' IDENTIFIED BY '{password}'; " + f"GRANT SELECT ON openemr.log TO '{AUDIT_USER}'@'%'; FLUSH PRIVILEGES;" + ) + out = fx._compose("ps", "-q", "db").stdout.decode().split() + if not out: + raise SystemExit("no database container for the fixture") + return out[0] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--run-id", required=True) + parser.add_argument("--per-fault", type=int, default=5) + parser.add_argument("--max-cost", type=float, default=0.50) + parser.add_argument("--faults", default=",".join(FAULTS)) + parser.add_argument("--first-item", type=int, default=941) + for engine in ENGINES: + parser.add_argument(f"--engine-{engine}", required=True, type=Path) + args = parser.parse_args() + engines = {e: getattr(args, f"engine_{e}") for e in ENGINES} + + out = HERE / "results" / "openemr" / args.run_id + work = schema.CACHE / "verification_coverage" / "openemr" / args.run_id + out.mkdir(parents=True, exist_ok=True) + work.mkdir(parents=True, exist_ok=True) + rows_path = out / "items.csv" + done = ( + {int(r["item"]) for r in csv.DictReader(rows_path.open())} + if rows_path.is_file() + else set() + ) + fx = openemr.fixture() + base_yaml = VERIFY_YAML.format(api=fx.ui_base_url, token_env=TOKEN_ENV) + password = secrets.token_hex(16) + heads = { + e: subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + for e, root in engines.items() + } + meta_written = False + plan = list( + enumerate( + (f for f in args.faults.split(",") for _ in range(args.per_fault)), + start=args.first_item, + ) + ) + try: + for i, fault in plan: + if i in done: + continue + if spent() + args.max_cost > SPEND_CAP_USD: + raise SystemExit("spend cap reached") + reset_and_seed() + container = ensure_audit_reader(password) + if not meta_written: + after_yaml = base_yaml + frame_yaml("") + schema.write_meta( + out, + task="openemr_referral", + arm="verification_coverage", + model="claude-sonnet-5-5", + engines=heads, + verify_yaml=base_yaml, + verify_yaml_after=after_yaml, + faults=args.faults.split(","), + per_fault=args.per_fault, + fault_injection=( + "fixture database only: the scoreboard triggers and poller " + "(claims_openemr.py), plus audit-log rows for the two " + "logged edits (openemr_run.py)" + ), + ) + meta_written = True + truth_values = openemr.item_values(i) + told = dict(truth_values) + if fault == "misread_source": + told[MISREAD[0]] = MISREAD[1] + before = _rows() + item_dir = work / f"item-{i:03d}" + shutil.rmtree(item_dir, ignore_errors=True) + item_dir.mkdir(parents=True) + # The referral holds the true values; the provenance, what the + # first reader extracted (the misread, for misread_source). + doc = item_dir / "referral.txt" + doc.write_text(referral_text(truth_values)) + from openadapt_flow.provenance import ( + ProvenanceFile, + cross_check, + default_extractor, + document_sha256, + read_sources, + ) + + prov_doc = provenance(document_sha256(doc), told) + (item_dir / "provenance.json").write_text(json.dumps(prov_doc)) + prov = ProvenanceFile.model_validate(prov_doc) + presave = cross_check(prov, read_sources([doc]), default_extractor(prov)) + params = {k: told[k] for k in BOUND} + env = {**_token_env(), AUDIT_ENV: password} + yamls = { + "e1365": base_yaml, + "main": base_yaml, + "after": base_yaml + frame_yaml(container), + } + baselines = {} + for engine, root in engines.items(): + cfg = item_dir / f"verify-{engine}.yaml" + cfg.write_text(yamls[engine]) + baselines[engine] = item_dir / f"baseline-{engine}.json" + cap = verify( + root, + engine_env(root, env), + params, + "--verify", + str(cfg), + "--capture-baseline", + str(baselines[engine]), + ) + if cap.returncode != 0: + raise SystemExit(f"{engine} baseline failed: {cap.stderr[-800:]}") + poller = None + if fault in TRIGGERS: + _install_trigger(fault, max(int(k) for k in before)) + elif fault not in ("ok", "misread_source"): + poller = LoggedPoller(fault, before, told["email"]) + poller.start() + ledger = out / "ledgers" / f"item-{i:03d}.json" + load0 = schema.load_1m() + ctx = multiprocessing.get_context("spawn") + box = ctx.Queue() + proc = ctx.Process( + target=_agent_item, + args=( + openemr.NAME, + openemr.entry_url(), + openemr.instruction(told), + str(ledger), + args.max_cost, + box, + ), + ) + proc.start() + proc.join(ITEM_TIMEOUT_S) + result: dict[str, Any] = {} + if proc.is_alive(): + proc.terminate() + proc.join(10) + result = {"error": "ItemTimeout"} + elif not box.empty(): + result = box.get() + if poller is not None: + poller.join(timeout=8.0) + poller.stop.set() + view = _ledger_view(ledger, result) + add_spend( + view.cost_usd, f"verification_coverage openemr {args.run_id} item {i}" + ) + claimed = reported_success(view, result.get("error")) # type: ignore[arg-type] + truth = ground_truth(before, truth_values) + if fault == "concurrent_edit_other" and truth == "collateral_edit": + truth = "correct" # another user's edit; the save itself is right + record = item_dir / "agent-run.json" + record.write_text( + json.dumps( + { + "executor_kind": "agent", + "executor_id": "claude-sonnet-5-5", + "claimed_success": claimed, + } + ) + ) + row: dict[str, Any] = { + "item": i, + "fault": fault, + "claimed_success": claimed, + "ground_truth": truth, + "presave_check": presave.status, + "fault_applied": ( + poller.applied + if poller is not None + else fault + if fault in TRIGGERS or fault == "misread_source" + else "" + ), + "model_calls": len(view.calls), + "cost_usd": round(view.cost_usd, 6), + "host_load_1m": load0, + "final_text": (result.get("error") or view.final_text)[:400], + } + for engine, root in engines.items(): + share = item_dir / f"share-{engine}" + shutil.rmtree(share, ignore_errors=True) + extra = ( + [ + "--provenance", + str(item_dir / "provenance.json"), + "--source", + str(doc), + ] + if engine == "after" + else [] + ) + ver = verify( + root, + engine_env(root, {**_token_env(), AUDIT_ENV: password}), + params, + "--verify", + str(item_dir / f"verify-{engine}.yaml"), + "--run-record", + str(record), + "--authorization-ref", + f"VERIFY-COVERAGE-{args.run_id}-{i:03d}", + "--baseline", + str(baselines[engine]), + "--out", + str(share), + *extra, + ) + receipt_file = share / "effect-receipt.json" + receipt = ( + json.loads(receipt_file.read_text())["receipt"] + if receipt_file.is_file() and ver.returncode in (0, 1) + else {} + ) + row[f"{engine}_outcome"] = receipt.get("outcome", "refused") + row[f"{engine}_checked"] = ( + json.dumps(receipt.get("checked"), sort_keys=True) + if receipt.get("checked") + else "" + ) + if not receipt: + (item_dir / f"{engine}.stderr").write_text(ver.stderr[-4000:]) + new = not rows_path.is_file() + with rows_path.open("a", newline="") as f: + w = csv.DictWriter(f, fieldnames=FIELDS) + if new: + w.writeheader() + w.writerow(row) + print( + f"vc {i:3d} {fault:22s} truth={truth:16s} " + + " ".join(f"{e}={row[f'{e}_outcome']}" for e in ENGINES) + + f" presave={presave.status} ${view.cost_usd:.3f}", + flush=True, + ) + time.sleep(0.5) + finally: + fx.reset() + meta = json.loads((out / "meta.json").read_text()) + meta["complete"] = True + (out / "meta.json").write_text(json.dumps(meta, indent=2, default=str) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/verification_coverage/openimis_run.py b/packages/flow/benchmark/verification_coverage/openimis_run.py new file mode 100644 index 000000000..d0a627142 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/openimis_run.py @@ -0,0 +1,552 @@ +"""Fresh agent runs on openIMIS, judged by three engines, per fault class. + + python -m verification_coverage.openimis_run --run-id --per-fault 5 + +The openIMIS counterpart of ``openemr_run.py``: a computer-use agent (Claude +Sonnet 5.5) enters a health-facility claim through the openIMIS UI while a +fault is injected in the fixture's Postgres database (a trigger before the +save, or a poller after it). Each item is judged by the same three +``verify-run`` engines, each with its own baseline. + +openIMIS stamps every claim row with the user who last wrote it +(``AuditUserID``) and when (``ValidityFrom``), so the ``after`` engine's +frame uses ``record_field`` attribution, where OpenEMR's uses its audit log. +A change whose modified-by names another user and whose ``ValidityFrom`` +moved is a concurrent edit; anything else outside the declared claim is +collateral. + +Fault classes as in ``openemr_run.py``, adapted: the trigger faults blank or +change the new claim's explanation, a removal sets ``ValidityTo`` (openIMIS +never deletes rows), and the misread changes the explanation the agent is +told while the claim form document keeps the true one. + +The openIMIS stack is the pinned reference stack in ``openimis_claims`` +(fetched and hash-checked at run time; nothing AGPL is stored here). All data +is synthetic. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import multiprocessing +import re +import shutil +import sys +import threading +import time +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +BENCH = HERE.parent +sys.path.insert(0, str(BENCH)) + +from scoreboard import schema # noqa: E402 +from scoreboard.run import ( # noqa: E402 + ITEM_TIMEOUT_S, + SPEND_CAP_USD, + _agent_item, + _ledger_view, + add_spend, + reported_success, + spent, +) +from scoreboard.tasks import openimis # noqa: E402 + +from verification_coverage.openemr_run import ( # noqa: E402 + CLEAN, + ENGINES, + FAULTS, + engine_env, + verify, +) + +READER_ENV = "VC_IMIS_READER_PASSWORD" +OTHER_USER_ID = 2 # a demo-dataset user other than the actor (checked at run time) +MISREAD_FIELD = "explanation" + +CLAIMS_SQL = ( + 'SELECT c."ClaimID" AS claim_id, c."ClaimCode" AS claim_no, ' + 'i."CHFID" AS insurance_no, c."Explanation" AS explanation, ' + 'c."ClaimStatus" AS claim_status, hf."HFCode" AS hf_code, ' + 'd."ICDCode" AS diagnosis, CAST(c."Claimed" AS TEXT) AS claimed, ' + 'c."AuditUserID" AS audit_user, CAST(c."ValidityFrom" AS TEXT) AS validity_from ' + 'FROM "tblClaim" c JOIN "tblInsuree" i ON i."InsureeID" = c."InsureeID" ' + 'JOIN "tblHF" hf ON hf."HfID" = c."HFID" ' + 'LEFT JOIN "tblICDCodes" d ON d."ICDID" = c."ICDID" ' + 'WHERE c."ValidityTo" IS NULL ORDER BY c."ClaimID"' +) + +FIELDS = ( + "item", + "fault", + "claimed_success", + "ground_truth", + *(f"{e}_outcome" for e in ENGINES), + *(f"{e}_checked" for e in ENGINES), + "presave_check", + "fault_applied", + "model_calls", + "cost_usd", + "host_load_1m", + "final_text", +) + + +def verify_yaml(db_port: int, bot_id: int, *, frame: bool) -> str: + match = ( + " match:\n" + " claim_no: {param: claim_no}\n" + " insurance_no: {param: insurance_no}\n" + ) + effects = ( + " - kind: record_written\n" + match + " count_new_only: true\n" + " timeout_s: 20\n" + ) + for name, value in ( + ("explanation", "{param: explanation}"), + ("claim_status", "'2'"), + ("hf_code", openimis.HEALTH_FACILITY_CODE), + ("diagnosis", "A389"), + ): + effects += ( + " - kind: field_equals\n" + match + f" field: {name}\n" + f" value: {value}\n timeout_s: 20\n" + ) + doc = ( + "schema_version: openadapt.verify/v1\n" + "verifier:\n" + " kind: sql\n" + " sql_driver: pg8000.dbapi\n" + " sql_connect_args:\n" + " host: 127.0.0.1\n" + f" port: {db_port}\n" + f" user: {openimis.READER_ROLE}\n" + " database: IMIS\n" + f" sql_password_env: {READER_ENV}\n" + f" sql_query: '{CLAIMS_SQL}'\n" + "effects:\n" + effects + ) + if frame: + doc += ( + "frame:\n" + " identity_field: claim_id\n" + " attribution:\n" + " kind: record_field\n" + " actor_field: audit_user\n" + " version_field: validity_from\n" + # openIMIS sets both on the server from the signed-in user; the + # claim form has no field for either. + " server_set: [audit_user, validity_from]\n" + f" self: ['{bot_id}']\n" + ) + return doc + + +def claim_form_text(values: dict[str, str]) -> str: + return ( + "Health facility claim (synthetic test document)\n" + f"Insurance No: {values['insurance_no']}\n" + f"Claim No: {values['claim_no']}\n" + f"Explanation: {values['explanation']}\n" + ) + + +def provenance(doc_sha: str, values: dict[str, str]) -> dict[str, Any]: + params = {} + for line, (name, label) in enumerate( + ( + ("insurance_no", "Insurance No"), + ("claim_no", "Claim No"), + ("explanation", "Explanation"), + ), + start=2, + ): + params[name] = { + "value": values[name], + "source": { + "kind": "document", + "sha256": doc_sha, + "line": line, + "label": label, + }, + "extractor": "model:simulated-first-reader", + } + return {"schema_version": "openadapt.param-provenance/v1", "params": params} + + +def _code(value: str) -> str: + if not re.fullmatch(r"[A-Za-z0-9]{1,8}", value): + raise ValueError("unexpected claim code") + return value + + +def psql(sql: str) -> str: + return openimis.fixture()._psql(sql) + + +def valid_claims() -> dict[int, dict[str, str]]: + out = psql( + 'SELECT "ClaimID", "ClaimCode", coalesce("Explanation", \'\'), "AuditUserID" ' + 'FROM "tblClaim" WHERE "ValidityTo" IS NULL ORDER BY "ClaimID";' + ) + rows: dict[int, dict[str, str]] = {} + for line in out.strip().splitlines(): + cid, code, expl, audit = line.split("|", 3) + rows[int(cid)] = {"code": code, "explanation": expl, "audit_user": audit} + return rows + + +def install_trigger(fault: str, claim_no: str) -> None: + body = { + "partial": "NEW.\"Explanation\" := '';", + "wrong_value": 'NEW."Explanation" := NEW."Explanation" || \' (altered)\';', + }[fault] + psql( + "CREATE OR REPLACE FUNCTION vc_fault() RETURNS trigger AS $$ BEGIN " + f'IF NEW."ClaimCode" = \'{_code(claim_no)}\' AND NEW."Explanation" IS NOT NULL ' + f"AND NEW.\"Explanation\" <> '' THEN {body} END IF; RETURN NEW; END $$ LANGUAGE plpgsql;\n" + 'DROP TRIGGER IF EXISTS vc_fault ON "tblClaim";\n' + 'CREATE TRIGGER vc_fault BEFORE INSERT OR UPDATE ON "tblClaim" ' + "FOR EACH ROW EXECUTE FUNCTION vc_fault();\n" + ) + + +def drop_trigger() -> None: + psql( + 'DROP TRIGGER IF EXISTS vc_fault ON "tblClaim"; DROP FUNCTION IF EXISTS vc_fault();' + ) + + +class Poller(threading.Thread): + """Applies one fault once the new claim is written.""" + + def __init__(self, fault: str, claim_no: str, victim: int, bot_id: int) -> None: + super().__init__(daemon=True) + self.fault, self.claim_no, self.victim, self.bot_id = ( + fault, + claim_no, + victim, + bot_id, + ) + self.applied = "" + self.stop = threading.Event() + + def run(self) -> None: + new = "" + while not self.stop.is_set(): + new = psql( + 'SELECT "ClaimID" FROM "tblClaim" WHERE "ValidityTo" IS NULL AND ' + f"\"ClaimCode\" = '{_code(self.claim_no)}' AND \"Explanation\" <> '' LIMIT 1;" + ).strip() + if new: + break + time.sleep(0.5) + if not new: + return + claim = int(new) + time.sleep(1.0) # let the save's own follow-up writes finish + if self.fault == "duplicate": + psql( + 'CREATE TEMP TABLE vc_dup AS SELECT * FROM "tblClaim" ' + f'WHERE "ClaimID" = {claim};\n' + 'UPDATE vc_dup SET "ClaimID" = nextval(pg_get_serial_sequence(' + "'\"tblClaim\"', 'ClaimID')), \"ClaimUUID\" = gen_random_uuid();\n" + 'INSERT INTO "tblClaim" SELECT * FROM vc_dup;\n' + ) + elif self.fault == "no_persist": + time.sleep(2.0) + psql( + f'UPDATE "tblClaim" SET "ValidityTo" = now() WHERE "ClaimID" = {claim};' + ) + elif self.fault == "collateral_loss": + psql( + f'UPDATE "tblClaim" SET "ValidityTo" = now() WHERE "ClaimID" = {self.victim};' + ) + elif self.fault.startswith(("collateral_edit", "concurrent_edit")): + who = ( + f', "AuditUserID" = {self.bot_id}, "ValidityFrom" = now()' + if self.fault == "collateral_edit_bot" + else f', "AuditUserID" = {OTHER_USER_ID}, "ValidityFrom" = now()' + if self.fault == "concurrent_edit_other" + else "" + ) + psql( + 'UPDATE "tblClaim" SET "Explanation" = coalesce("Explanation", \'\') ' + f"|| ' (edited)'{who} WHERE \"ClaimID\" = {self.victim};" + ) + self.applied = f"{self.fault} at {time.strftime('%H:%M:%S')}" + + +def ground_truth(before: dict[int, dict[str, str]], values: dict[str, str]) -> str: + after = valid_claims() + if any(cid not in after for cid in before): + return "collateral_loss" + if any(after[cid] != row for cid, row in before.items()): + return "collateral_edit" + return openimis.check(values) + + +def bot_user_id() -> int: + out = psql( + 'SELECT "UserID" FROM "tblUsers" WHERE "LoginName" = ' + f"'{openimis.ACTOR_USER}' AND \"ValidityTo\" IS NULL;" + ).strip() + if not out: + raise SystemExit("cannot find the actor's openIMIS user id") + return int(out.splitlines()[0]) + + +def main() -> int: + from openadapt_flow.provenance import ( + ProvenanceFile, + cross_check, + default_extractor, + document_sha256, + read_sources, + ) + + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--run-id", required=True) + parser.add_argument("--per-fault", type=int, default=5) + parser.add_argument("--max-cost", type=float, default=0.75) + parser.add_argument("--faults", default=",".join(FAULTS)) + parser.add_argument("--first-item", type=int, default=1) + parser.add_argument("--tag", default="VC") + for engine in ENGINES: + parser.add_argument(f"--engine-{engine}", required=True, type=Path) + args = parser.parse_args() + engines = {e: getattr(args, f"engine_{e}") for e in ENGINES} + + out = HERE / "results" / "openimis" / args.run_id + work = schema.CACHE / "verification_coverage" / "openimis" / args.run_id + out.mkdir(parents=True, exist_ok=True) + work.mkdir(parents=True, exist_ok=True) + rows_path = out / "items.csv" + done = ( + {int(r["item"]) for r in csv.DictReader(rows_path.open())} + if rows_path.is_file() + else set() + ) + fx = openimis.fixture() + openimis.bootstrap(fx) + bot_id = bot_user_id() + if bot_id == OTHER_USER_ID: + raise SystemExit("the 'other user' id is the actor's own id; pick another") + password = openimis._reader_password(fx) + yamls = { + "e1365": verify_yaml(fx.db_port, bot_id, frame=False), + "main": verify_yaml(fx.db_port, bot_id, frame=False), + "after": verify_yaml(fx.db_port, bot_id, frame=True), + } + import subprocess + + heads = { + e: subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + for e, root in engines.items() + } + schema.write_meta( + out, + task="openimis", + arm="verification_coverage", + model="claude-sonnet-5-5", + engines=heads, + verify_yaml=yamls["main"], + verify_yaml_after=yamls["after"], + faults=args.faults.split(","), + per_fault=args.per_fault, + fault_injection="fixture database only: a trigger or a poller (openimis_run.py)", + ) + plan = list( + enumerate( + (f for f in args.faults.split(",") for _ in range(args.per_fault)), + start=args.first_item, + ) + ) + env = {READER_ENV: password} + try: + for i, fault in plan: + if i in done: + continue + if spent() + args.max_cost > SPEND_CAP_USD: + raise SystemExit("spend cap reached") + drop_trigger() + truth_values = openimis.item_values(i, args.tag) + told = dict(truth_values) + if fault == "misread_source": + told[MISREAD_FIELD] = ( + told[MISREAD_FIELD] + .replace("same day", "next day") + .replace("Follow-up", "Followed up") + .replace("Rash", "Rush") + .replace("Child", "Chlid") + .replace("Second", "Secnd") + ) + before = valid_claims() + victim = min(cid for cid, row in before.items() if row["explanation"]) + item_dir = work / f"item-{i:03d}" + shutil.rmtree(item_dir, ignore_errors=True) + item_dir.mkdir(parents=True) + doc = item_dir / "claim.txt" + doc.write_text(claim_form_text(truth_values)) + prov_doc = provenance(document_sha256(doc), told) + (item_dir / "provenance.json").write_text(json.dumps(prov_doc)) + prov = ProvenanceFile.model_validate(prov_doc) + presave = cross_check(prov, read_sources([doc]), default_extractor(prov)) + params = dict(told) + baselines = {} + for engine, root in engines.items(): + cfg = item_dir / f"verify-{engine}.yaml" + cfg.write_text(yamls[engine]) + baselines[engine] = item_dir / f"baseline-{engine}.json" + cap = verify( + root, + engine_env(root, env), + params, + "--verify", + str(cfg), + "--capture-baseline", + str(baselines[engine]), + ) + if cap.returncode != 0: + raise SystemExit(f"{engine} baseline failed: {cap.stderr[-800:]}") + poller = None + if fault in ("partial", "wrong_value"): + install_trigger(fault, told["claim_no"]) + elif fault not in ("ok", "misread_source"): + poller = Poller(fault, told["claim_no"], victim, bot_id) + poller.start() + ledger = out / "ledgers" / f"item-{i:03d}.json" + load0 = schema.load_1m() + ctx = multiprocessing.get_context("spawn") + box = ctx.Queue() + proc = ctx.Process( + target=_agent_item, + args=( + openimis.NAME, + openimis.claim_form_url(), + openimis.instruction(told), + str(ledger), + args.max_cost, + box, + ), + ) + proc.start() + proc.join(ITEM_TIMEOUT_S) + result: dict[str, Any] = {} + if proc.is_alive(): + proc.terminate() + proc.join(10) + result = {"error": "ItemTimeout"} + elif not box.empty(): + result = box.get() + if poller is not None: + poller.join(timeout=10.0) + poller.stop.set() + drop_trigger() + view = _ledger_view(ledger, result) + add_spend( + view.cost_usd, f"verification_coverage openimis {args.run_id} item {i}" + ) + claimed = reported_success(view, result.get("error")) # type: ignore[arg-type] + truth = ground_truth(before, truth_values) + if fault == "concurrent_edit_other" and truth == "collateral_edit": + truth = "correct" + if fault == "misread_source" and truth == "wrong": + truth = "wrong_vs_source" + record = item_dir / "agent-run.json" + record.write_text( + json.dumps( + { + "executor_kind": "agent", + "executor_id": "claude-sonnet-5-5", + "claimed_success": claimed, + } + ) + ) + row: dict[str, Any] = { + "item": i, + "fault": fault, + "claimed_success": claimed, + "ground_truth": truth, + "presave_check": presave.status, + "fault_applied": poller.applied if poller is not None else fault, + "model_calls": len(view.calls), + "cost_usd": round(view.cost_usd, 6), + "host_load_1m": load0, + "final_text": (result.get("error") or view.final_text)[:400], + } + for engine, root in engines.items(): + share = item_dir / f"share-{engine}" + shutil.rmtree(share, ignore_errors=True) + extra = ( + [ + "--provenance", + str(item_dir / "provenance.json"), + "--source", + str(doc), + ] + if engine == "after" + else [] + ) + ver = verify( + root, + engine_env(root, env), + params, + "--verify", + str(item_dir / f"verify-{engine}.yaml"), + "--run-record", + str(record), + "--authorization-ref", + f"VERIFY-COVERAGE-IMIS-{args.run_id}-{i:03d}", + "--baseline", + str(baselines[engine]), + "--out", + str(share), + *extra, + ) + receipt_file = share / "effect-receipt.json" + receipt = ( + json.loads(receipt_file.read_text())["receipt"] + if receipt_file.is_file() and ver.returncode in (0, 1) + else {} + ) + row[f"{engine}_outcome"] = receipt.get("outcome", "refused") + row[f"{engine}_checked"] = ( + json.dumps(receipt["checked"], sort_keys=True) + if receipt.get("checked") + else "" + ) + if not receipt: + (item_dir / f"{engine}.stderr").write_text(ver.stderr[-4000:]) + new = not rows_path.is_file() + with rows_path.open("a", newline="") as f: + w = csv.DictWriter(f, fieldnames=FIELDS) + if new: + w.writeheader() + w.writerow(row) + print( + f"vc-imis {i:3d} {fault:22s} truth={truth:16s} " + + " ".join(f"{e}={row[f'{e}_outcome']}" for e in ENGINES) + + f" presave={presave.status} ${view.cost_usd:.3f}", + flush=True, + ) + finally: + drop_trigger() + meta = json.loads((out / "meta.json").read_text()) + meta["complete"] = True + (out / "meta.json").write_text(json.dumps(meta, indent=2, default=str) + "\n") + return 0 + + +__all__ = ["CLEAN", "main"] + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/verification_coverage/summarize.py b/packages/flow/benchmark/verification_coverage/summarize.py new file mode 100644 index 000000000..8f026b068 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/summarize.py @@ -0,0 +1,149 @@ +"""Per-fault-class catch rates, before and after, from committed rows. + + python -m verification_coverage.summarize # print the tables + python -m verification_coverage.summarize --write # also RESULTS.md + +A bad save is **caught** when the engine's outcome is anything but +``verified``. A clean save (``ok``, ``concurrent_edit_other``) refused is a +**false alarm**. Ground truth comes from the fixture's own database, read +after the run; a save the agent itself got wrong counts as bad whatever the +injected fault was, and is reported in its own row. +""" + +from __future__ import annotations + +import argparse +import csv +import json +from collections import defaultdict +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ENGINES = ("e1365", "main", "after") +ENGINE_LABEL = { + "e1365": "#1365 engine", + "main": "main (with #1375)", + "after": "frame + gate + provenance", +} +CLEAN_FAULTS = ("ok", "concurrent_edit_other") + + +def load(app: str) -> tuple[list[dict[str, str]], list[dict]]: + rows: list[dict[str, str]] = [] + metas: list[dict] = [] + for run in sorted((HERE / "results" / app).glob("*/items.csv")): + meta = json.loads((run.parent / "meta.json").read_text()) + if not meta.get("complete"): + continue + metas.append(meta) + rows += list(csv.DictReader(run.open())) + return rows, metas + + +def _clean(row: dict[str, str]) -> bool: + return row["ground_truth"] == "correct" + + +def table(rows: list[dict[str, str]]) -> list[str]: + by_fault: dict[str, list[dict[str, str]]] = defaultdict(list) + for row in rows: + by_fault[row["fault"]].append(row) + head = ( + "| Fault class | n | " + + " | ".join(ENGINE_LABEL[e] for e in ENGINES) + + " | Held before the save |" + ) + out = [head, "|" + "---|" * (len(ENGINES) + 3)] + totals = {e: [0, 0] for e in ENGINES} + alarms = {e: [0, 0] for e in ENGINES} + for fault in sorted(by_fault, key=lambda f: (f in CLEAN_FAULTS, f)): + items = by_fault[fault] + clean = [r for r in items if _clean(r)] + bad = [r for r in items if not _clean(r)] + cells = [] + for engine in ENGINES: + parts = [] + if bad: + caught = sum(r[f"{engine}_outcome"] != "verified" for r in bad) + totals[engine][0] += caught + totals[engine][1] += len(bad) + parts.append(f"{caught}/{len(bad)} caught") + if clean: + false = sum(r[f"{engine}_outcome"] != "verified" for r in clean) + alarms[engine][0] += false + alarms[engine][1] += len(clean) + parts.append(f"{false}/{len(clean)} false alarms") + cells.append(", ".join(parts)) + held = sum(r["presave_check"] == "hold" for r in items) + out.append( + f"| `{fault}` | {len(items)} | " + + " | ".join(cells) + + f" | {held}/{len(items)} |" + ) + out.append( + "| **All bad saves** | | " + + " | ".join(f"**{c}/{n}**" for c, n in (totals[e] for e in ENGINES)) + + " | |" + ) + out.append( + "| **All clean saves** | | " + + " | ".join(f"{c}/{n} false alarms" for c, n in (alarms[e] for e in ENGINES)) + + " | |" + ) + return out + + +def render() -> str: + lines = ["# Results", ""] + for app, title in (("openemr", "OpenEMR"), ("openimis", "openIMIS")): + rows, metas = load(app) + lines.append(f"## {title}") + lines.append("") + if not rows: + lines += ["No complete run yet.", ""] + continue + engines = metas[-1].get("engines", {}) + lines.append( + f"{len(rows)} fresh agent runs (Claude Sonnet 5.5). Engines: " + + ", ".join( + f"{ENGINE_LABEL[e]} `{engines.get(e, '?')[:9]}`" for e in ENGINES + ) + + "." + ) + lines.append("") + lines += table(rows) + lines.append("") + gate = HERE / "results" / app + for summary in sorted(gate.glob("*/gate-summary.json")): + data = json.loads(summary.read_text()) + lines.append( + f"Fault gate on the {title} test copy ({summary.parent.name}):" + ) + lines.append("") + for name, s in data.items(): + lines.append( + f"- `{name}` read: {s['killed']} of {s['applicable']} faults killed" + + ( + f"; survived: {', '.join(s['survived'])}" + if s["survived"] + else "" + ) + + ("" if s["clean_pass"] else "; the clean save did not verify") + ) + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--write", action="store_true") + args = parser.parse_args() + text = render() + print(text) + if args.write: + (HERE / "RESULTS.md").write_text(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/openadapt_flow/record_check_synthesis.py b/packages/flow/openadapt_flow/record_check_synthesis.py index 7b88b89a3..6cc7c6f74 100644 --- a/packages/flow/openadapt_flow/record_check_synthesis.py +++ b/packages/flow/openadapt_flow/record_check_synthesis.py @@ -61,6 +61,7 @@ Verdict, stable_id, ) +from openadapt_flow.runtime.effects.sql import read_cell from openadapt_flow.runtime.effects.surfaces import surface_probe #: The network capture ``record --capture-network`` writes. Read here without @@ -818,7 +819,12 @@ def reads_from_sqlite( ) complete = False continue - rows = [dict(r) for r in conn.execute(f'SELECT * FROM "{table}"')] + # Cells as the check's own SQL read returns them (binary as hex), + # so the scan and the generated read compare equal. + rows = [ + {k: read_cell(r[k]) for k in r.keys()} + for r in conn.execute(f'SELECT * FROM "{table}"') + ] best: Optional[tuple[dict[str, Any], dict[str, str]]] = None for row in rows: exact, _ = _bind_fields( diff --git a/packages/flow/openadapt_flow/runtime/effects/sql.py b/packages/flow/openadapt_flow/runtime/effects/sql.py index 2a6a305be..7d89d263d 100644 --- a/packages/flow/openadapt_flow/runtime/effects/sql.py +++ b/packages/flow/openadapt_flow/runtime/effects/sql.py @@ -127,6 +127,19 @@ def assert_read_only_sql(query: str) -> str: return text +def read_cell(value: Any) -> Any: + """A binary column as ``0x`` + hex, so records stay JSON-serializable. + + OpenEMR keys rows by ``uuid binary(16)``; a driver returns those as bytes, + which a baseline (JSON) cannot hold. Equal bytes give equal text, so + record identity and change detection are unaffected. + """ + + if isinstance(value, (bytes, bytearray, memoryview)): + return "0x" + bytes(value).hex() + return value + + class SqlRecordVerifier(VerifierAdapterBase): """Verify effects against a SQL system of record with a read-only SELECT. @@ -198,7 +211,8 @@ def _fetch_records(self) -> Optional[list[dict[str, Any]]]: columns = [str(col[0]) for col in description] rows = cursor.fetchall() return drop_record_fields( - [dict(zip(columns, row)) for row in rows], self.drop_fields + [dict(zip(columns, map(read_cell, row))) for row in rows], + self.drop_fields, ) except Exception: # noqa: BLE001 - any read failure is unreadable return None diff --git a/packages/flow/tests/test_effect_sql.py b/packages/flow/tests/test_effect_sql.py index b970ca5df..13dbf1e91 100644 --- a/packages/flow/tests/test_effect_sql.py +++ b/packages/flow/tests/test_effect_sql.py @@ -246,3 +246,93 @@ def connect(): raise sqlite3.OperationalError("db down") assert capture_table_counts(connect, ["encounters"]) is None + + +# -- binary columns (OpenEMR keys rows by uuid binary(16)) -------------------- + +#: The bytes the pinned OpenEMR fixture's first uuid started with: not UTF-8. +OPENEMR_UUID = bytes.fromhex("a2f1c3d40e5b4c7a9b1d2e3f40516273") + + +def _openemr_patients(path: Path) -> sqlite3.Connection: + conn = sqlite3.connect(path) + conn.execute( + "CREATE TABLE patient_data (id INTEGER PRIMARY KEY AUTOINCREMENT, " + "uuid BLOB, pid INTEGER, fname TEXT, lname TEXT, DOB TEXT, email TEXT)" + ) + conn.execute( + "INSERT INTO patient_data (uuid, pid, fname, lname, DOB, email) VALUES " + "(?, 1, 'Wendell', 'Priorchart', '1962-06-14', 'w.priorchart@example.test')", + (OPENEMR_UUID,), + ) + conn.commit() + return conn + + +def test_binary_column_reads_as_hex(tmp_path: Path) -> None: + path = tmp_path / "openemr.db" + _openemr_patients(path).close() + verifier = SqlRecordVerifier( + lambda: sqlite3.connect(path), "SELECT * FROM patient_data" + ) + state = verifier.capture_pre_state() + assert state.reachable + assert state.records[0]["uuid"] == "0x" + OPENEMR_UUID.hex() + state.model_dump(mode="json") # a baseline must serialize + + +def test_verify_run_baselines_and_verifies_a_binary_keyed_table(tmp_path: Path) -> None: + """Before the fix, capture_baseline raised UnicodeDecodeError here.""" + + from openadapt_flow.verify_run import ( + capture_baseline, + parse_verify_config, + verify_run, + ) + + path = tmp_path / "openemr.db" + conn = _openemr_patients(path) + cfg = parse_verify_config( + { + "schema_version": "openadapt.verify/v1", + "verifier": { + "kind": "sql", + "sqlite_database": str(path), + "sql_query": "SELECT * FROM patient_data", + }, + "record": { + "name": "patient", + "match": ["email", "lname"], + "duplicate_key": "email", + "fields": { + "email": "email", + "lname": "lname", + "fname": "fname", + "DOB": "DOB", + }, + }, + } + ) + params = { + "email": "ada.quill@example.test", + "lname": "Quill", + "fname": "Ada", + "DOB": "1984-03-07", + } + baseline = tmp_path / "baseline.json" + capture_baseline(cfg, params, baseline) + conn.execute( + "INSERT INTO patient_data (uuid, pid, fname, lname, DOB, email) VALUES " + "(?, 2, 'Ada', 'Quill', '1984-03-07', 'ada.quill@example.test')", + (bytes.fromhex("ff00ee11dd22cc33bb44aa5599668877"),), + ) + conn.commit() + conn.close() + result = verify_run( + cfg, + params=params, + run_record={"executor_kind": "agent", "executor_id": "openadapt_actor"}, + authorization_ref="binary-columns-test", + baseline_path=baseline, + ) + assert result["receipt"]["outcome"] == "verified" diff --git a/packages/flow/tests/test_independent_verification_suite.py b/packages/flow/tests/test_independent_verification_suite.py new file mode 100644 index 000000000..f3d28044f --- /dev/null +++ b/packages/flow/tests/test_independent_verification_suite.py @@ -0,0 +1,85 @@ +"""The independent verification-coverage suite: its inputs and its stand-in.""" + +from __future__ import annotations + +import sqlite3 +import sys +from pathlib import Path + +import yaml + +BENCH = Path(__file__).resolve().parents[1] / "benchmark" +sys.path.insert(0, str(BENCH)) + +from verification_coverage.independent import standin # noqa: E402 + +SUITE = BENCH / "verification_coverage" / "independent" +OPS = { + "register_referral", + "register_patient", + "overwrite_chart", + "update_chart", + "update_history", + "delete_history", + "add_note", + "add_allergy", + "add_insurance", +} + + +def test_every_case_names_its_spec_clauses_and_a_verdict() -> None: + suite = yaml.safe_load((SUITE / "faults.yaml").read_text()) + spec = (SUITE / "SPEC.md").read_text() + ids = [c["id"] for c in suite["cases"]] + assert len(ids) == len(set(ids)) + for case in suite["cases"]: + assert case["expect"] in ("fault", "clean") + for clause in case["clauses"]: + assert f"**{clause}." in spec, clause + for step in case["steps"]: + assert step["actor"] in ("bot", "clinician2", "none") + assert step["op"] in OPS + classes = {c["class"] for c in suite["cases"]} + assert { + "wrong_record", + "unrelated_change", + "misread_source", + "missing_side_record", + "concurrent_legitimate", + "duplicate", + } <= classes + assert all( + c["expect"] == "clean" + for c in suite["cases"] + if c["class"] == "concurrent_legitimate" + ) + + +DUMP = """\ +CREATE TABLE `patient_data` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `uuid` binary(16) DEFAULT NULL, + `fname` varchar(255) NOT NULL DEFAULT '', + `providerID` int(11) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +INSERT INTO `patient_data` VALUES +(1,NULL,'O\\'Neil',5), +(2,NULL,'Line\\nTwo',NULL); +""" + + +def test_the_dump_converts_with_values_and_affinity(tmp_path: Path) -> None: + dump = tmp_path / "baseline.sql" + dump.write_text(DUMP) + out = tmp_path / "copy.sqlite" + counts = standin.build(dump, out) + assert counts == {"patient_data": 2} + con = sqlite3.connect(out) + rows = con.execute("SELECT id, fname, providerID FROM patient_data").fetchall() + assert rows == [(1, "O'Neil", 5), (2, "Line\nTwo", None)] + # MariaDB compares '5' = 5; the stand-in must too. + assert con.execute( + "SELECT COUNT(*) FROM patient_data WHERE providerID = ?", ("5",) + ).fetchone() == (1,) + con.close() diff --git a/packages/flow/tests/test_record_check_synthesis.py b/packages/flow/tests/test_record_check_synthesis.py index 0033b0344..409d8457d 100644 --- a/packages/flow/tests/test_record_check_synthesis.py +++ b/packages/flow/tests/test_record_check_synthesis.py @@ -984,3 +984,31 @@ def test_a_demonstration_with_no_write_is_refused(tmp_path, capsys): with pytest.raises(SystemExit) as exc: main(["scaffold-verifier", str(rec), "--complete-read"]) assert "REFUSED" in str(exc.value.code if exc.value.code is not None else exc.value) + + +def test_a_binary_keyed_table_reads_the_same_in_the_scan_and_the_check(tmp_path): + """OpenEMR keys patient_data by uuid binary(16). The scan must see the + cells as the generated SQL read returns them, or every such read is + reported as returning different rows.""" + + import sqlite3 + + path = tmp_path / "openemr.db" + conn = sqlite3.connect(path) + conn.execute( + "CREATE TABLE patient_data (id INTEGER PRIMARY KEY, uuid BLOB, " + "fname TEXT, lname TEXT, email TEXT)" + ) + conn.execute( + "INSERT INTO patient_data VALUES (1, ?, 'Ada', 'Quill', 'ada.quill@example.test')", + (bytes.fromhex("a2f1c3d40e5b4c7a9b1d2e3f40516273"),), + ) + conn.commit() + conn.close() + proposal = synthesize( + None, + params={"fname": "Ada", "lname": "Quill", "email": "ada.quill@example.test"}, + db_path=path, + ) + assert not [p for p in proposal.search.problems if "patient_data" in p] + assert any(s.is_target for s in proposal.search.surfaces) diff --git a/packages/flow/tests/test_verification_coverage_bench.py b/packages/flow/tests/test_verification_coverage_bench.py new file mode 100644 index 000000000..0459e7028 --- /dev/null +++ b/packages/flow/tests/test_verification_coverage_bench.py @@ -0,0 +1,95 @@ +"""The verification-coverage benchmark's summary (benchmark/verification_coverage).""" + +from __future__ import annotations + +import csv +import json +import sys +from pathlib import Path + +import pytest + +BENCH = Path(__file__).resolve().parents[1] / "benchmark" + + +@pytest.fixture +def summarize(monkeypatch, tmp_path): + monkeypatch.syspath_prepend(str(BENCH)) + sys.modules.pop("verification_coverage.summarize", None) + from verification_coverage import summarize as mod + + monkeypatch.setattr(mod, "HERE", tmp_path) + return mod + + +def _write(tmp_path: Path, rows: list[dict[str, str]], complete: bool = True) -> None: + run = tmp_path / "results" / "openemr" / "r1" + run.mkdir(parents=True) + (run / "meta.json").write_text( + json.dumps( + { + "complete": complete, + "engines": {"e1365": "a" * 40, "main": "b" * 40, "after": "c" * 40}, + } + ) + ) + with (run / "items.csv").open("w", newline="") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + + +def _row( + fault: str, truth: str, *outcomes: str, presave: str = "agree" +) -> dict[str, str]: + return { + "item": "1", + "fault": fault, + "ground_truth": truth, + "e1365_outcome": outcomes[0], + "main_outcome": outcomes[1], + "after_outcome": outcomes[2], + "presave_check": presave, + } + + +def test_catches_and_false_alarms_are_counted_apart(summarize, tmp_path) -> None: + v, r = "verified", "reconciliation_required" + _write( + tmp_path, + [ + _row("ok", "correct", v, v, v), + _row("collateral_edit", "collateral_edit", v, r, r), + _row("concurrent_edit_other", "correct", v, r, v), + _row("misread_source", "wrong_vs_source", v, v, r, presave="hold"), + ], + ) + text = summarize.render() + assert ( + "| `collateral_edit` | 1 | 0/1 caught | 1/1 caught | 1/1 caught | 0/1 |" in text + ) + assert ( + "| `concurrent_edit_other` | 1 | 0/1 false alarms | 1/1 false alarms | 0/1 false alarms | 0/1 |" + in text + ) + assert ( + "| `misread_source` | 1 | 0/1 caught | 0/1 caught | 1/1 caught | 1/1 |" in text + ) + assert "**0/2** | **1/2** | **2/2**" in text + + +def test_an_incomplete_run_is_not_reported(summarize, tmp_path) -> None: + _write( + tmp_path, + [_row("ok", "correct", "verified", "verified", "verified")], + complete=False, + ) + assert "No complete run yet." in summarize.render() + + +def test_the_benchmark_modules_import(monkeypatch) -> None: + monkeypatch.syspath_prepend(str(BENCH)) + import importlib + + for name in ("exec_mariadb", "summarize"): + importlib.import_module(f"verification_coverage.{name}")