From 3e26ada87eedc7d959b3816ac65eba59e2d83693 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 12:16:18 +0200 Subject: [PATCH 01/19] test: add real-RDP multi-window vision campaign --- .../workflows/docker-rdp-multiapp-vision.yml | 68 ++ benchmark/rdp_multiapp/README.md | 49 ++ benchmark/rdp_multiapp/campaign.json | 72 ++ benchmark/rdp_multiapp/fixture/Dockerfile | 16 + benchmark/rdp_multiapp/fixture/run_fixture.sh | 28 + benchmark/rdp_multiapp/fixture/suite_app.py | 488 ++++++++++++++ benchmark/rdp_multiapp/policy.yaml | 17 + benchmark/rdp_multiapp/run_qualification.py | 635 ++++++++++++++++++ .../test_docker_rdp_multiapp_vision_e2e.py | 32 + tests/test_rdp_multiapp_campaign_contract.py | 32 + 10 files changed, 1437 insertions(+) create mode 100644 .github/workflows/docker-rdp-multiapp-vision.yml create mode 100644 benchmark/rdp_multiapp/README.md create mode 100644 benchmark/rdp_multiapp/campaign.json create mode 100644 benchmark/rdp_multiapp/fixture/Dockerfile create mode 100644 benchmark/rdp_multiapp/fixture/run_fixture.sh create mode 100644 benchmark/rdp_multiapp/fixture/suite_app.py create mode 100644 benchmark/rdp_multiapp/policy.yaml create mode 100644 benchmark/rdp_multiapp/run_qualification.py create mode 100644 tests/e2e/test_docker_rdp_multiapp_vision_e2e.py create mode 100644 tests/test_rdp_multiapp_campaign_contract.py diff --git a/.github/workflows/docker-rdp-multiapp-vision.yml b/.github/workflows/docker-rdp-multiapp-vision.yml new file mode 100644 index 00000000..5fd0997b --- /dev/null +++ b/.github/workflows/docker-rdp-multiapp-vision.yml @@ -0,0 +1,68 @@ +name: docker-rdp-multiapp-vision + +# This expensive real-protocol test runs only when its own campaign changes or +# when a maintainer starts it. Core changes use the smaller RDP ladder during +# pull-request feedback; release qualification can start this campaign once. +on: + pull_request: + paths: + - ".github/workflows/docker-rdp-multiapp-vision.yml" + - "benchmark/rdp_multiapp/**" + - "tests/e2e/test_docker_rdp_multiapp_vision_e2e.py" + - "tests/test_rdp_multiapp_campaign_contract.py" + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: docker-rdp-multiapp-vision-${{ github.ref }} + cancel-in-progress: true + +jobs: + qualify: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install the vision and RDP stack + run: | + python -m pip install --upgrade pip + pip install -e ".[rdp]" + pip check + + - name: Build and start the RDP fixture + run: | + docker build -t oaflow-rdp-multiapp:latest benchmark/rdp_multiapp/fixture + mkdir -p "${RUNNER_TEMP}/rdp-multiapp-oracle" + docker run -d --name oaflow-rdp-multiapp --shm-size=1g \ + -e RDP_MULTIAPP_ORACLE_ROOT=/oracle \ + -v "${RUNNER_TEMP}/rdp-multiapp-oracle:/oracle" \ + oaflow-rdp-multiapp:latest + sleep 22 + + - name: Run the real-RDP visual subset + run: | + mkdir -p "${RUNNER_TEMP}/rdp-multiapp-result" + python benchmark/rdp_multiapp/run_qualification.py \ + --container oaflow-rdp-multiapp \ + --oracle-root "${RUNNER_TEMP}/rdp-multiapp-oracle" \ + --work-dir "${RUNNER_TEMP}/rdp-multiapp-work" \ + --output "${RUNNER_TEMP}/rdp-multiapp-result/results.json" + + - name: Upload exact result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rdp-multiapp-vision-subset + path: ${{ runner.temp }}/rdp-multiapp-result/results.json + if-no-files-found: error + + - name: Tear down + if: always() + run: docker rm -f oaflow-rdp-multiapp || true diff --git a/benchmark/rdp_multiapp/README.md b/benchmark/rdp_multiapp/README.md new file mode 100644 index 00000000..75236ecd --- /dev/null +++ b/benchmark/rdp_multiapp/README.md @@ -0,0 +1,49 @@ +# Real-RDP multi-window vision campaign + +This campaign tests the main pixel-automation risk that the existing complex +API benchmarks do not test. It records, compiles, and replays one back-office +workflow across three separate task windows. Observation is limited to pixels +decoded by a real FreeRDP client. Input returns through the same RDP session. + +The workflow reads a referral in **Inbox**, finds the request in a scrollable +**Worklist**, enters the appointment in **Scheduler**, reconciles the worklist, +and sends a confirmation from Inbox. The result contract reads three persisted +surfaces without trusting the UI: + +- SQLite: exactly one appointment for the authorized request and record. +- CSV: the exact worklist row has status `Scheduled` and no adjacent row changed. +- Maildir: exactly one confirmation has the expected request correlation key. + +`campaign.json` defines the fault campaign. Every condition requires three +trials. The report must count silent incorrect success and over-halt, not only +task completion. + +## Scope + +This fixture uses deterministic synthetic data. It tests the production visual +resolver and RDP input path without a DOM or accessibility tree. It does not +replace qualification of a named Windows or Citrix application. The separate +task windows exercise window switching and focus behavior, but they are hosted +by one synthetic fixture process. + +## Fixture + +```bash +docker build -t oaflow-rdp-multiapp:latest benchmark/rdp_multiapp/fixture +docker run --rm --name oaflow-rdp-multiapp \ + -v "$PWD/.tmp/rdp-multiapp-oracle:/opt/rdp_multiapp/oracle" \ + oaflow-rdp-multiapp:latest +``` + +Run the implemented subset after the container is ready: + +```bash +python benchmark/rdp_multiapp/run_qualification.py \ + --oracle-root "$PWD/.tmp/rdp-multiapp-oracle" \ + --output benchmark/rdp_multiapp/results.json +``` + +The runner uses the same `DockerX11RdpTransport` and `FreeRDPBackend` contract +as `benchmark/rdp_ladder`. The first subset runs healthy, row-reordered, +wrong-record, and focus-theft conditions. The result cannot describe the full +campaign as complete until every condition in `campaign.json` has run. diff --git a/benchmark/rdp_multiapp/campaign.json b/benchmark/rdp_multiapp/campaign.json new file mode 100644 index 00000000..14c309aa --- /dev/null +++ b/benchmark/rdp_multiapp/campaign.json @@ -0,0 +1,72 @@ +{ + "schema_version": "openadapt.rdp-multiapp-campaign.v1", + "task": "Process one referral across inbox, worklist, and scheduler windows through RDP pixels, then prove the appointment, worklist status, and confirmation email on independent persisted surfaces.", + "trials_per_condition": 3, + "required_metrics": [ + "verified_outcomes", + "safe_halts", + "silent_incorrect_successes", + "over_halts", + "wrong_record_writes", + "duplicate_effects", + "model_calls", + "p50_runtime_s", + "p95_runtime_s" + ], + "conditions": [ + { + "id": "healthy", + "expect": "verified", + "fault": "none", + "oracle": ["sqlite", "csv", "maildir"] + }, + { + "id": "row_reordered", + "expect": "verified_or_safe_halt", + "fault": "The target worklist row moves to a different visual position.", + "oracle": ["sqlite", "csv", "maildir"] + }, + { + "id": "focus_theft_before_write", + "expect": "safe_halt", + "fault": "Another task window takes focus after resolution and before the scheduler write.", + "oracle": ["sqlite"] + }, + { + "id": "wrong_record_before_write", + "expect": "safe_halt", + "fault": "The active scheduler record changes before the consequential click.", + "oracle": ["sqlite"] + }, + { + "id": "duplicate_save_control", + "expect": "safe_halt", + "fault": "A competing Save appointment control appears in the same visual state.", + "oracle": ["sqlite"] + }, + { + "id": "partial_render", + "expect": "safe_halt", + "fault": "The target window is incomplete when the next action is due.", + "oracle": ["sqlite", "csv", "maildir"] + }, + { + "id": "moderate_display_drift", + "expect": "verified_or_safe_halt", + "fault": "Theme, scaling, and compression change while text remains readable.", + "oracle": ["sqlite", "csv", "maildir"] + }, + { + "id": "severe_display_drift", + "expect": "safe_halt", + "fault": "The frame no longer supports unambiguous target and record identity.", + "oracle": ["sqlite", "csv", "maildir"] + }, + { + "id": "commit_then_timeout", + "expect": "verified_or_reconciliation_required", + "fault": "The scheduler write can persist before the input channel reports uncertainty.", + "oracle": ["sqlite"] + } + ] +} diff --git a/benchmark/rdp_multiapp/fixture/Dockerfile b/benchmark/rdp_multiapp/fixture/Dockerfile new file mode 100644 index 00000000..9791f2e0 --- /dev/null +++ b/benchmark/rdp_multiapp/fixture/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + xvfb x11-utils python3-tk fonts-dejavu-core \ + freerdp3-shadow-x11 freerdp3-x11 \ + imagemagick xdotool openbox procps \ + && rm -rf /var/lib/apt/lists/* + +COPY suite_app.py /opt/rdp_multiapp/suite_app.py +COPY run_fixture.sh /opt/rdp_multiapp/run_fixture.sh +RUN chmod +x /opt/rdp_multiapp/run_fixture.sh \ + && mkdir -p /opt/rdp_multiapp/oracle + +EXPOSE 3389 +CMD ["/opt/rdp_multiapp/run_fixture.sh"] diff --git a/benchmark/rdp_multiapp/fixture/run_fixture.sh b/benchmark/rdp_multiapp/fixture/run_fixture.sh new file mode 100644 index 00000000..6b3e1747 --- /dev/null +++ b/benchmark/rdp_multiapp/fixture/run_fixture.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -euo pipefail + +export HOME=/root +export RDP_MULTIAPP_ORACLE_ROOT="${RDP_MULTIAPP_ORACLE_ROOT:-/opt/rdp_multiapp/oracle}" +mkdir -p "${RDP_MULTIAPP_ORACLE_ROOT}" + +Xvfb :0 -screen 0 1280x800x24 -ac +extension DAMAGE +extension RANDR +extension XFIXES \ + >/tmp/xvfb0.log 2>&1 & +sleep 2 +DISPLAY=:0 openbox >/tmp/openbox0.log 2>&1 & +sleep 1 +DISPLAY=:0 python3 /opt/rdp_multiapp/suite_app.py >/tmp/suite.log 2>&1 & +sleep 3 +DISPLAY=:0 freerdp-shadow-cli3 /port:3389 /bind-address:0.0.0.0 -auth \ + >/tmp/shadow.log 2>&1 & +sleep 3 + +Xvfb :1 -screen 0 1280x800x24 -ac >/tmp/xvfb1.log 2>&1 & +sleep 2 +DISPLAY=:1 openbox >/tmp/openbox1.log 2>&1 & +sleep 1 +DISPLAY=:1 xfreerdp3 /v:127.0.0.1:3389 /u:ubuntu /p:ubuntu /size:1280x800 /f \ + -gfx -rfx -nsc /cert:ignore +auto-reconnect /log-level:ERROR \ + >/tmp/client.log 2>&1 & +sleep 4 + +tail -f /dev/null diff --git a/benchmark/rdp_multiapp/fixture/suite_app.py b/benchmark/rdp_multiapp/fixture/suite_app.py new file mode 100644 index 00000000..870f86ce --- /dev/null +++ b/benchmark/rdp_multiapp/fixture/suite_app.py @@ -0,0 +1,488 @@ +#!/usr/bin/env python3 +"""Deterministic, pixel-only back-office suite for the RDP vision campaign. + +The fixture presents three separate task windows and a small launcher. Flow +can observe only the RDP-decoded pixels and can act only through the RDP input +channel. The independent result surfaces are SQLite, CSV, and Maildir files. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import signal +import sqlite3 +import tkinter as tk +from email.message import EmailMessage +from pathlib import Path + +ROOT = Path(os.environ.get("RDP_MULTIAPP_ORACLE_ROOT", "/opt/rdp_multiapp/oracle")) +DB_PATH = ROOT / "appointments.sqlite3" +CSV_PATH = ROOT / "worklist.csv" +MAILDIR = ROOT / "outbox" +CONTROL_PATH = ROOT / "control.json" +ACK_PATH = ROOT / "reset_ack.txt" + +BG = "#eef2f6" +PANEL = "#ffffff" +FG = "#142033" +BLUE = "#2457d6" +GREEN = "#16803c" +RED = "#b42318" +ROW_HEIGHT = 52 +TARGET_REQUEST = "REQ-LIVE-2048" +TARGET_NAME = "Jordan Lee" +TARGET_RECORD = "REC-2048" + + +def _connect() -> sqlite3.Connection: + ROOT.mkdir(parents=True, exist_ok=True) + connection = sqlite3.connect(DB_PATH) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS appointments ( + appointment_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL UNIQUE, + record_id TEXT NOT NULL, + entity_name TEXT NOT NULL, + appointment_slot TEXT NOT NULL, + appointment_type TEXT NOT NULL, + status TEXT NOT NULL + ) + """ + ) + connection.commit() + return connection + + +def _scenario() -> str: + try: + return str(json.loads(CONTROL_PATH.read_text()).get("scenario", "healthy")) + except (OSError, ValueError, TypeError): + return "healthy" + + +def _rows() -> list[dict[str, str]]: + rows = [ + { + "request_id": f"REQ-{1000 + i}", + "record_id": f"REC-{1000 + i}", + "name": f"Sample Person {i:02d}", + "status": "New", + } + for i in range(18) + ] + target = { + "request_id": TARGET_REQUEST, + "record_id": TARGET_RECORD, + "name": TARGET_NAME, + "status": "New", + } + insert_at = 4 if _scenario() == "row_reordered" else 15 + rows.insert(insert_at, target) + return rows + + +def _write_rows(rows: list[dict[str, str]]) -> None: + with CSV_PATH.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter( + stream, fieldnames=["request_id", "record_id", "name", "status"] + ) + writer.writeheader() + writer.writerows(rows) + + +def _read_rows() -> list[dict[str, str]]: + with CSV_PATH.open(newline="", encoding="utf-8") as stream: + return list(csv.DictReader(stream)) + + +def _reset_persisted_state() -> None: + ROOT.mkdir(parents=True, exist_ok=True) + with _connect() as connection: + connection.execute("DELETE FROM appointments") + connection.commit() + _write_rows(_rows()) + for subdir in ("cur", "new", "tmp"): + path = MAILDIR / subdir + path.mkdir(parents=True, exist_ok=True) + for child in path.iterdir(): + if child.is_file(): + child.unlink() + + +def _label(parent: tk.Misc, text: str, x: int, y: int, **kwargs) -> tk.Label: + label = tk.Label(parent, text=text, bg=BG, fg=FG, **kwargs) + label.place(x=x, y=y) + return label + + +class Suite: + def __init__(self) -> None: + _reset_persisted_state() + self.root = tk.Tk() + self.root.withdraw() + self.windows: dict[str, tk.Toplevel] = {} + self.selected_request: str | None = None + self.active_record: tuple[str, str] | None = None + self.reset_counter = 0 + self._build_inbox() + self._build_worklist() + self._build_scheduler() + self._build_launcher() + self.reset() + self.root.after(250, lambda: self.show("Inbox")) + signal.signal(signal.SIGUSR1, self._signal_reset) + + def _window(self, title: str) -> tk.Toplevel: + window = tk.Toplevel(self.root) + window.title(title) + window.geometry("1280x740+0+0") + window.configure(bg=BG) + window.overrideredirect(True) + window.resizable(False, False) + self.windows[title] = window + return window + + def show(self, title: str) -> None: + window = self.windows[title] + window.deiconify() + window.lift() + window.focus_force() + self.launcher.lift() + + def _build_launcher(self) -> None: + launcher = tk.Toplevel(self.root) + launcher.title("OpenAdapt Fixture Launcher") + launcher.geometry("1280x60+0+740") + launcher.configure(bg="#172033") + launcher.overrideredirect(True) + launcher.attributes("-topmost", True) + self.launcher = launcher + tk.Label( + launcher, + text="Remote workspace", + bg="#172033", + fg="white", + font=("DejaVu Sans", 14, "bold"), + ).place(x=28, y=17) + for index, title in enumerate(("Inbox", "Worklist", "Scheduler")): + tk.Button( + launcher, + text=title, + command=lambda value=title: self.show(value), + bg="#2d3d5e", + fg="white", + activebackground=BLUE, + activeforeground="white", + font=("DejaVu Sans", 13, "bold"), + relief="flat", + ).place(x=285 + index * 190, y=8, width=165, height=44) + + def _build_inbox(self) -> None: + window = self._window("Inbox") + _label(window, "Referral inbox", 42, 30, font=("DejaVu Sans", 26, "bold")) + _label( + window, + "Incoming requests that need a scheduled appointment", + 42, + 76, + font=("DejaVu Sans", 14), + ) + self.inbox_row = tk.Button( + window, + text=( + f"{TARGET_NAME} {TARGET_RECORD} {TARGET_REQUEST}\n" + "Cardiology referral · requested this week" + ), + command=self._select_inbox, + anchor="w", + justify="left", + bg=PANEL, + fg=FG, + activebackground="#dce7ff", + font=("DejaVu Sans", 16), + relief="solid", + bd=1, + ) + self.inbox_row.place(x=42, y=130, width=870, height=86) + self.inbox_detail = _label( + window, + "Select the request to review its structured details.", + 42, + 250, + font=("DejaVu Sans", 15), + ) + self.send_button = tk.Button( + window, + text="Send confirmation", + command=self._send_confirmation, + state="disabled", + bg=BLUE, + fg="white", + disabledforeground="#7b879a", + font=("DejaVu Sans", 16, "bold"), + ) + self.send_button.place(x=42, y=420, width=250, height=54) + self.inbox_status = _label( + window, "", 42, 500, font=("DejaVu Sans", 16, "bold") + ) + + def _select_inbox(self) -> None: + self.selected_request = TARGET_REQUEST + self.inbox_detail.config( + text=( + f"Request: {TARGET_REQUEST}\n" + f"Record: {TARGET_RECORD} · Name: {TARGET_NAME}\n" + "Requested type: Cardiology follow-up" + ) + ) + self.send_button.config(state="normal") + + def _send_confirmation(self) -> None: + rows = _read_rows() + status = next( + (row["status"] for row in rows if row["request_id"] == TARGET_REQUEST), + None, + ) + with _connect() as connection: + appointment = connection.execute( + "SELECT appointment_id, appointment_slot FROM appointments " + "WHERE request_id = ?", + (TARGET_REQUEST,), + ).fetchone() + if ( + self.selected_request != TARGET_REQUEST + or status != "Scheduled" + or not appointment + ): + self.inbox_status.config( + text="Refused: schedule and reconcile this request first", fg=RED + ) + return + message = EmailMessage() + message["From"] = "scheduling@example.test" + message["To"] = "referrals@example.test" + message["Subject"] = f"Scheduled {TARGET_REQUEST}" + message["X-Request-ID"] = TARGET_REQUEST + message.set_content( + f"{TARGET_NAME} is scheduled for {appointment[1]}. " + f"Appointment {appointment[0]}." + ) + name = hashlib.sha256(TARGET_REQUEST.encode()).hexdigest()[:16] + ".eml" + path = MAILDIR / "new" / name + if not path.exists(): + path.write_bytes(message.as_bytes()) + self.inbox_status.config(text="Confirmation queued", fg=GREEN) + + def _build_worklist(self) -> None: + window = self._window("Worklist") + _label(window, "Scheduling worklist", 42, 30, font=("DejaVu Sans", 26, "bold")) + _label( + window, + "Scroll to the request, select it, then update its persisted status.", + 42, + 76, + font=("DejaVu Sans", 14), + ) + canvas = tk.Canvas( + window, + bg=PANEL, + highlightthickness=1, + highlightbackground="#bdc7d6", + yscrollincrement=ROW_HEIGHT, + ) + canvas.place(x=42, y=125, width=900, height=460) + scrollbar = tk.Scrollbar(window, orient="vertical", command=canvas.yview) + scrollbar.place(x=942, y=125, width=22, height=460) + canvas.configure(yscrollcommand=scrollbar.set) + self.work_canvas = canvas + self.work_inner = tk.Frame(canvas, bg=PANEL) + canvas.create_window((0, 0), window=self.work_inner, anchor="nw") + self.work_inner.bind( + "", + lambda _event: canvas.configure(scrollregion=canvas.bbox("all")), + ) + for widget in (canvas, self.work_inner): + widget.bind("", lambda _event: canvas.yview_scroll(-1, "units")) + widget.bind("", lambda _event: canvas.yview_scroll(1, "units")) + self.work_status = _label( + window, "No request selected", 42, 610, font=("DejaVu Sans", 15, "bold") + ) + self.mark_button = tk.Button( + window, + text="Mark scheduled", + command=self._mark_scheduled, + bg=BLUE, + fg="white", + font=("DejaVu Sans", 16, "bold"), + state="disabled", + ) + self.mark_button.place(x=990, y=190, width=240, height=54) + + def _render_worklist(self) -> None: + for child in self.work_inner.winfo_children(): + child.destroy() + self.work_canvas.yview_moveto(0) + self.work_selected: str | None = None + for index, row in enumerate(_read_rows()): + text = ( + f"{row['request_id']} {row['record_id']} " + f"{row['name']} {row['status']}" + ) + button = tk.Button( + self.work_inner, + text=text, + command=lambda value=row["request_id"]: self._select_work(value), + anchor="w", + bg=PANEL, + fg=FG, + activebackground="#dce7ff", + font=("DejaVu Sans", 13), + relief="solid", + bd=1, + ) + button.grid(row=index, column=0, sticky="ew") + button.configure(width=88, height=2) + button.bind( + "", lambda _event: self.work_canvas.yview_scroll(-1, "units") + ) + button.bind( + "", lambda _event: self.work_canvas.yview_scroll(1, "units") + ) + + def _select_work(self, request_id: str) -> None: + self.work_selected = request_id + self.work_status.config(text=f"Selected request: {request_id}", fg=FG) + self.mark_button.config(state="normal") + + def _mark_scheduled(self) -> None: + if self.work_selected != TARGET_REQUEST: + self.work_status.config(text="Refused: wrong request selected", fg=RED) + return + with _connect() as connection: + appointment = connection.execute( + "SELECT 1 FROM appointments WHERE request_id = ?", (TARGET_REQUEST,) + ).fetchone() + if not appointment: + self.work_status.config( + text="Refused: appointment is not persisted", fg=RED + ) + return + rows = _read_rows() + for row in rows: + if row["request_id"] == TARGET_REQUEST: + row["status"] = "Scheduled" + _write_rows(rows) + self.work_status.config(text=f"Reconciled {TARGET_REQUEST}", fg=GREEN) + self._render_worklist() + + def _build_scheduler(self) -> None: + window = self._window("Scheduler") + _label( + window, "Appointment scheduler", 42, 30, font=("DejaVu Sans", 26, "bold") + ) + _label(window, "Record list", 42, 92, font=("DejaVu Sans", 15, "bold")) + self.record_buttons: list[tk.Button] = [] + for index, (name, record_id) in enumerate( + ((TARGET_NAME, TARGET_RECORD), ("Morgan Reed", "REC-3099")) + ): + button = tk.Button( + window, + text=f"{name} {record_id}", + command=lambda n=name, r=record_id: self._select_record(n, r), + anchor="w", + bg=PANEL, + fg=FG, + font=("DejaVu Sans", 15), + ) + button.place(x=42, y=130 + index * 68, width=410, height=54) + self.record_buttons.append(button) + self.active_label = _label( + window, "Active record: (none)", 520, 94, font=("DejaVu Sans", 16, "bold") + ) + self.slot = self._entry(window, "Appointment date and time", 520, 165) + self.kind = self._entry(window, "Appointment type", 520, 285) + self.request = self._entry(window, "Request ID", 520, 405) + self.save_button = tk.Button( + window, + text="Save appointment", + command=self._save_appointment, + bg=BLUE, + fg="white", + font=("DejaVu Sans", 16, "bold"), + ) + self.save_button.place(x=520, y=540, width=260, height=56) + self.scheduler_status = _label( + window, "", 520, 625, font=("DejaVu Sans", 16, "bold") + ) + + def _entry(self, parent: tk.Misc, title: str, x: int, y: int) -> tk.Entry: + _label(parent, title, x, y, font=("DejaVu Sans", 14, "bold")) + entry = tk.Entry(parent, font=("DejaVu Sans", 17), bg=PANEL, fg=FG) + entry.place(x=x, y=y + 36, width=590, height=42) + return entry + + def _select_record(self, name: str, record_id: str) -> None: + self.active_record = (name, record_id) + self.active_label.config(text=f"Active record: {name} {record_id}") + + def _save_appointment(self) -> None: + slot = self.slot.get().strip() + kind = self.kind.get().strip() + request_id = self.request.get().strip() + if self.active_record != (TARGET_NAME, TARGET_RECORD): + self.scheduler_status.config(text="Refused: wrong active record", fg=RED) + return + if not slot or not kind or request_id != TARGET_REQUEST: + self.scheduler_status.config(text="Refused: incomplete request", fg=RED) + return + appointment_id = ( + "APT-" + hashlib.sha256(request_id.encode()).hexdigest()[:8].upper() + ) + with _connect() as connection: + connection.execute( + """ + INSERT OR IGNORE INTO appointments ( + appointment_id, request_id, record_id, entity_name, + appointment_slot, appointment_type, status + ) VALUES (?, ?, ?, ?, ?, ?, 'scheduled') + """, + (appointment_id, request_id, TARGET_RECORD, TARGET_NAME, slot, kind), + ) + connection.commit() + self.scheduler_status.config( + text=f"Appointment saved · {appointment_id}", fg=GREEN + ) + + def _signal_reset(self, _signum, _frame) -> None: + self.root.after(0, self.reset) + + def reset(self) -> None: + _reset_persisted_state() + self.selected_request = None + self.active_record = None + self.inbox_detail.config( + text="Select the request to review its structured details." + ) + self.inbox_status.config(text="") + self.send_button.config(state="disabled") + self._render_worklist() + self.work_status.config(text="No request selected", fg=FG) + self.mark_button.config(state="disabled") + self.active_label.config(text="Active record: (none)") + for entry in (self.slot, self.kind, self.request): + entry.delete(0, "end") + self.scheduler_status.config(text="") + self.reset_counter += 1 + ACK_PATH.write_text(str(self.reset_counter), encoding="utf-8") + self.show("Inbox") + + def run(self) -> None: + self.root.mainloop() + + +if __name__ == "__main__": + Suite().run() diff --git a/benchmark/rdp_multiapp/policy.yaml b/benchmark/rdp_multiapp/policy.yaml new file mode 100644 index 00000000..eb92fc34 --- /dev/null +++ b/benchmark/rdp_multiapp/policy.yaml @@ -0,0 +1,17 @@ +name: rdp-multiapp-qualified +description: >- + Fail-closed policy for the synthetic real-RDP multi-window campaign. Every + consequential pointer action carries pixel identity, a screen postcondition, + an independent persisted-effect contract, and an idempotency key. + +prohibit_unarmed_clicks: true +require_identity_for: + - entity_navigation + - write +require_system_effects_for: + - write +require_idempotency_key_for: + - write +prohibit_unconfirmed_effect_bindings: true +require_screen_postconditions_for: + - write diff --git a/benchmark/rdp_multiapp/run_qualification.py b/benchmark/rdp_multiapp/run_qualification.py new file mode 100644 index 00000000..fec55da6 --- /dev/null +++ b/benchmark/rdp_multiapp/run_qualification.py @@ -0,0 +1,635 @@ +#!/usr/bin/env python3 +"""Run the first real-RDP multi-window vision qualification campaign. + +The harness uses the production Recorder, compiler, FreeRDPBackend, resolver, +run gate, authorization, Replayer, and effect-verifier adapters. The fixture +is synthetic. Pixels and input still cross a real FreeRDP round trip. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import secrets +import sqlite3 +import subprocess +import tempfile +import time +from email import policy as email_policy +from email.parser import BytesParser +from pathlib import Path +from typing import Any, Optional + +from benchmark.multiapp_common import CsvRecordVerifier, SurfaceRoutedVerifier +from benchmark.rdp_ladder.run_rdp_ladder_qualification import ( + DockerX11RdpTransport, +) + +VIEWPORT = (1280, 800) +TARGET_REQUEST = "REQ-LIVE-2048" +TARGET_RECORD = "REC-2048" +TARGET_NAME = "Jordan Lee" +SLOT_PARAM = "appointment_slot" +TYPE_PARAM = "appointment_type" +REQUEST_PARAM = "request_id" +DEMO_PARAMS = { + SLOT_PARAM: "2026-08-12 09:30", + TYPE_PARAM: "Cardiology follow-up", + REQUEST_PARAM: TARGET_REQUEST, +} +REPLAY_PARAMS = { + SLOT_PARAM: "2026-08-14 10:45", + TYPE_PARAM: "Intake consultation", + REQUEST_PARAM: TARGET_REQUEST, +} +TRIALS = 3 + +# Stable synthetic-fixture geometry. Runtime replay resolves visual anchors +# from the recording; these points are used only to make the demonstration. +INBOX_REQUEST = (380, 170) +WORKLIST_TAB = (555, 770) +SCHEDULER_TAB = (745, 770) +INBOX_TAB = (365, 770) +WORKLIST_VIEW = (400, 350) +WORKLIST_TARGET_AFTER_SCROLL = (430, 490) +MARK_SCHEDULED = (1110, 217) +TARGET_RECORD_ROW = (250, 155) +WRONG_RECORD_ROW = (250, 223) +SLOT_FIELD = (720, 222) +TYPE_FIELD = (720, 342) +REQUEST_FIELD = (720, 462) +SAVE_APPOINTMENT = (650, 568) +SEND_CONFIRMATION = (165, 447) + +# Identity bands are recorded pixels, not live values sent out of the runner. +ACTIVE_RECORD_REGION = (510, 86, 740, 60) +WORKLIST_SELECTION_REGION = (35, 596, 920, 58) +INBOX_DETAIL_REGION = (35, 238, 915, 150) + +POLICY_PATH = Path(__file__).with_name("policy.yaml") + + +def _read_ack(root: Path) -> Optional[int]: + try: + return int((root / "reset_ack.txt").read_text().strip()) + except (OSError, ValueError): + return None + + +def _read_database(root: Path) -> Optional[list[dict[str, str]]]: + path = root / "appointments.sqlite3" + if not path.exists(): + return None + try: + connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + connection.row_factory = sqlite3.Row + rows = connection.execute( + "SELECT appointment_id, request_id, record_id, entity_name, " + "appointment_slot, appointment_type, status FROM appointments" + ).fetchall() + return [dict(row) for row in rows] + except sqlite3.Error: + return None + finally: + if "connection" in locals(): + connection.close() + + +def _read_worklist(root: Path) -> Optional[list[dict[str, str]]]: + try: + with (root / "worklist.csv").open(newline="", encoding="utf-8") as stream: + return list(csv.DictReader(stream)) + except (OSError, csv.Error, UnicodeDecodeError): + return None + + +def _read_mail(root: Path) -> Optional[list[dict[str, str]]]: + mail_root = root / "outbox" + if not mail_root.is_dir(): + return None + records: list[dict[str, str]] = [] + try: + for folder in (mail_root / "new", mail_root / "cur"): + if not folder.is_dir(): + continue + for path in sorted(folder.iterdir()): + if not path.is_file(): + continue + message = BytesParser(policy=email_policy.default).parsebytes( + path.read_bytes() + ) + records.append( + { + "file": path.name, + "to": str(message.get("To", "")), + "subject": str(message.get("Subject", "")), + "request_id": str(message.get("X-Request-ID", "")), + } + ) + except (OSError, ValueError): + return None + return records + + +def _reset(container: str, root: Path, scenario: str = "healthy") -> None: + before = _read_ack(root) + root.mkdir(parents=True, exist_ok=True) + (root / "control.json").write_text( + json.dumps({"scenario": scenario}) + "\n", encoding="utf-8" + ) + result = subprocess.run( + ["docker", "exec", container, "pkill", "-USR1", "-f", "suite_app.py"], + capture_output=True, + timeout=30, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr.decode(errors="replace")[:300]) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + after = _read_ack(root) + advanced = after is not None and (before is None or after > before) + rows = _read_database(root) + worklist = _read_worklist(root) + mail = _read_mail(root) + if advanced and rows == [] and worklist and mail == []: + return + time.sleep(0.1) + raise RuntimeError("fixture reset did not produce clean persisted state") + + +def _record(backend: Any, recording_dir: Path) -> None: + from openadapt_flow.recorder import Recorder + + recorder = Recorder( + backend, + recording_dir, + settle_interval_s=0.3, + settle_stable_frames=2, + settle_timeout_s=6.0, + ) + recorder.click(*INBOX_REQUEST) + recorder.click(*WORKLIST_TAB) + recorder.click(*WORKLIST_VIEW) + recorder.scroll(0, 800) + recorder.click(*WORKLIST_TARGET_AFTER_SCROLL) + recorder.click(*SCHEDULER_TAB) + recorder.click(*TARGET_RECORD_ROW) + recorder.click(*SLOT_FIELD) + recorder.type_text(DEMO_PARAMS[SLOT_PARAM], param=SLOT_PARAM) + recorder.click(*TYPE_FIELD) + recorder.type_text(DEMO_PARAMS[TYPE_PARAM], param=TYPE_PARAM) + recorder.click(*REQUEST_FIELD) + recorder.type_text(DEMO_PARAMS[REQUEST_PARAM], param=REQUEST_PARAM) + recorder.click(*SAVE_APPOINTMENT) + recorder.click(*WORKLIST_TAB) + recorder.click(*WORKLIST_VIEW) + recorder.scroll(0, 800) + recorder.click(*WORKLIST_TARGET_AFTER_SCROLL) + recorder.click(*MARK_SCHEDULED) + recorder.click(*INBOX_TAB) + recorder.click(*INBOX_REQUEST) + recorder.click(*SEND_CONFIRMATION) + recorder.finish() + + +def _arm_recording(recording_dir: Path) -> None: + path = recording_dir / "events.jsonl" + events = [json.loads(line) for line in path.read_text().splitlines() if line] + expected = { + 13: ACTIVE_RECORD_REGION, + 18: WORKLIST_SELECTION_REGION, + 21: INBOX_DETAIL_REGION, + } + for index, region in expected.items(): + if index >= len(events) or events[index].get("kind") != "click": + raise RuntimeError(f"recorded event {index} is not the expected click") + events[index]["identifier_region"] = list(region) + path.write_text( + "".join(json.dumps(event, sort_keys=True) + "\n" for event in events), + encoding="utf-8", + ) + + +def _build_verifier(root: Path) -> SurfaceRoutedVerifier: + from openadapt_flow.runtime.effects import ( + MaildirDeliveryVerifier, + SqlRecordVerifier, + ) + + database = root / "appointments.sqlite3" + return SurfaceRoutedVerifier( + { + "appointments": SqlRecordVerifier( + lambda: sqlite3.connect(f"file:{database}?mode=ro", uri=True), + "SELECT appointment_id, request_id, record_id, entity_name, " + "appointment_slot, appointment_type, status FROM appointments", + ), + "worklist": CsvRecordVerifier(root / "worklist.csv"), + "outbox": MaildirDeliveryVerifier( + str(root / "outbox"), + content_probe=r"Jordan Lee is scheduled", + poll_interval_s=0.05, + ), + }, + default_surface="appointments", + ) + + +def _qualify(workflow: Any, bundle_dir: Path, root: Path): + from openadapt_flow import __version__ + from openadapt_flow.deployment import DeploymentConfig, PolicySection + from openadapt_flow.ir import Postcondition, PostconditionKind, Workflow + from openadapt_flow.qualification import ( + ActionRiskClass, + ActionRiskClassification, + EnvironmentBoundary, + init_project, + set_action_classification, + ) + from openadapt_flow.run_gate import evaluate_run_gate + from openadapt_flow.runtime.effects import Effect, EffectKind, ValueExpr + + if len(workflow.steps) != 22: + raise RuntimeError( + f"expected 22 compiled actions from the demonstration, got {len(workflow.steps)}" + ) + save, reconcile, send = workflow.steps[13], workflow.steps[18], workflow.steps[21] + for step, label in ((save, "save"), (reconcile, "reconcile"), (send, "send")): + step.risk = "irreversible" + step.risk_explanation = f"qualified {label} write" + step.risk_review_required = False + save.expect = [ + Postcondition(kind=PostconditionKind.TEXT_PRESENT, text="Appointment saved") + ] + save.effects = [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={ + "request_id": ValueExpr(param=REQUEST_PARAM), + "record_id": ValueExpr(literal=TARGET_RECORD), + "appointment_slot": ValueExpr(param=SLOT_PARAM), + "appointment_type": ValueExpr(param=TYPE_PARAM), + "status": ValueExpr(literal="scheduled"), + }, + expected_count=1, + count_new_only=True, + key_field="request_id", + idempotency_key=ValueExpr(param=REQUEST_PARAM), + risk="irreversible", + probe="surface=appointments|read-only exact appointment lookup", + ) + ] + reconcile.expect = [ + Postcondition( + kind=PostconditionKind.TEXT_PRESENT, text=f"Reconciled {TARGET_REQUEST}" + ) + ] + reconcile.effects = [ + Effect( + kind=EffectKind.FIELD_EQUALS, + match={"request_id": ValueExpr(param=REQUEST_PARAM)}, + field="status", + value=ValueExpr(literal="Scheduled"), + idempotency_key=ValueExpr(param=REQUEST_PARAM), + key_field="request_id", + risk="irreversible", + probe="surface=worklist|CSV row re-read", + ) + ] + send.expect = [ + Postcondition(kind=PostconditionKind.TEXT_PRESENT, text="Confirmation queued") + ] + send.effects = [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={ + "to": ValueExpr(literal="referrals@example.test"), + "subject": ValueExpr(literal=f"Scheduled {TARGET_REQUEST}"), + "content_match": ValueExpr(literal="True"), + }, + expected_count=1, + count_new_only=True, + idempotency_key=ValueExpr(literal=f"Scheduled {TARGET_REQUEST}"), + key_field="subject", + risk="irreversible", + probe="surface=outbox|Maildir delivery capture", + ) + ] + + environment_payload = json.dumps( + { + "application": "rdp-multiapp-suite", + "policy_sha256": hashlib.sha256(POLICY_PATH.read_bytes()).hexdigest(), + "surface": "freerdp3-roundtrip", + "viewport": VIEWPORT, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + init_project( + workflow, + environment=EnvironmentBoundary( + target_kind="rdp", + application="RDP multi-window synthetic suite", + application_version="v1", + environment_digest=hashlib.sha256(environment_payload).hexdigest(), + runtime_version=__version__, + required_capabilities=[ + "vision-only-resolution", + "window-switching", + "read-only-sql-effect", + "csv-effect", + "maildir-effect", + ], + ), + ) + for step in workflow.steps: + classification = ( + ActionRiskClass.IRREVERSIBLE + if step.id in {save.id, reconcile.id, send.id} + else ActionRiskClass.READ_ONLY + ) + set_action_classification( + workflow, + ActionRiskClassification( + step_id=step.id, + classification=classification, + explanation=( + "Qualified persisted business write" + if classification is ActionRiskClass.IRREVERSIBLE + else "Navigation, focus, data entry, or scrolling before a qualified write" + ), + operator_confirmed=True, + ), + ) + + key = secrets.token_urlsafe(32) + workflow.save(bundle_dir, encrypt=True, key=key) + workflow = Workflow.load(bundle_dir, key=key) + verifier = _build_verifier(root) + gate = evaluate_run_gate( + workflow, + bundle_dir=bundle_dir, + deployment=DeploymentConfig(policy=PolicySection(policy=str(POLICY_PATH))), + effect_verifier=verifier, + policy_source=str(POLICY_PATH), + strict_templates=True, + require_encryption=True, + ) + if not gate.passed: + raise RuntimeError(gate.render()) + return ( + workflow, + verifier, + gate, + {"save": save.id, "reconcile": reconcile.id, "send": send.id}, + ) + + +def _oracle_result(root: Path, params: dict[str, str]) -> dict[str, Any]: + database = _read_database(root) + worklist = _read_worklist(root) + mail = _read_mail(root) + expected_db = { + "request_id": TARGET_REQUEST, + "record_id": TARGET_RECORD, + "entity_name": TARGET_NAME, + "appointment_slot": params[SLOT_PARAM], + "appointment_type": params[TYPE_PARAM], + "status": "scheduled", + } + db_ok = bool( + database is not None + and len(database) == 1 + and all(database[0].get(key) == value for key, value in expected_db.items()) + ) + target_rows = [ + row for row in worklist or [] if row.get("request_id") == TARGET_REQUEST + ] + adjacent_unchanged = bool( + worklist is not None + and all( + row.get("status") == "New" + for row in worklist + if row.get("request_id") != TARGET_REQUEST + ) + ) + csv_ok = ( + len(target_rows) == 1 + and target_rows[0].get("status") == "Scheduled" + and adjacent_unchanged + ) + mail_ok = bool( + mail is not None + and len(mail) == 1 + and mail[0].get("to") == "referrals@example.test" + and mail[0].get("subject") == f"Scheduled {TARGET_REQUEST}" + and mail[0].get("request_id") == TARGET_REQUEST + ) + return { + "database_ok": db_ok, + "worklist_ok": csv_ok, + "mail_ok": mail_ok, + "all_effects_ok": db_ok and csv_ok and mail_ok, + "database": database, + "worklist_target": target_rows, + "mail": mail, + "adjacent_rows_unchanged": adjacent_unchanged, + } + + +def _run_once( + *, + container: str, + root: Path, + workflow: Any, + verifier: Any, + gate: Any, + bundle_dir: Path, + run_dir: Path, + condition: str, + save_pointer_acquisition: int, +) -> dict[str, Any]: + from openadapt_flow.backends.rdp_backend import FreeRDPBackend + from openadapt_flow.run_gate import build_runtime_authorization + from openadapt_flow.runtime.replayer import Replayer + + _reset( + container, root, "row_reordered" if condition == "row_reordered" else "healthy" + ) + params = dict(REPLAY_PARAMS) + authorization = build_runtime_authorization( + workflow, + gate, + approval_source="rdp-multiapp-qualification", + params=params, + ) + transport = DockerX11RdpTransport(container) + backend = FreeRDPBackend(transport, connect=True) + original_acquire = backend.acquire_actuation_frame + acquisitions = 0 + injected = False + + def acquire_with_fault() -> bytes: + nonlocal acquisitions, injected + acquisitions += 1 + if acquisitions == save_pointer_acquisition: + if condition == "wrong_record_before_write": + transport.pointer(*WRONG_RECORD_ROW, "left", True) + transport.pointer(*WRONG_RECORD_ROW, "left", False) + time.sleep(0.45) + injected = True + elif condition == "focus_theft_before_write": + transport.pointer(*INBOX_TAB, "left", True) + transport.pointer(*INBOX_TAB, "left", False) + time.sleep(0.45) + injected = True + return original_acquire() + + backend.acquire_actuation_frame = acquire_with_fault # type: ignore[method-assign] + started = time.monotonic() + report = Replayer( + backend, + poll_interval_s=0.3, + effect_verifier=verifier, + governed_authorization=authorization, + pixel_verify_enabled=True, + durable=True, + require_settled=True, + ).run( + workflow, + params=params, + bundle_dir=bundle_dir, + run_dir=run_dir, + idempotency_key=f"{condition}-{run_dir.name}", + ) + oracle = _oracle_result(root, params) + expected_halt = condition in { + "wrong_record_before_write", + "focus_theft_before_write", + } + safe_halt = expected_halt and not report.success and not oracle["database"] + if condition == "row_reordered": + passed = bool( + (report.success and oracle["all_effects_ok"]) + or (not report.success and not oracle["database"]) + ) + elif expected_halt: + passed = bool(injected and safe_halt) + else: + passed = bool(report.success and oracle["all_effects_ok"]) + return { + "condition": condition, + "passed": passed, + "runtime_s": round(time.monotonic() - started, 3), + "runtime_success": bool(report.success), + "model_calls": int(report.model_calls), + "fault_injected": injected, + "safe_halt": safe_halt, + "silent_incorrect_success": bool( + report.success and not oracle["all_effects_ok"] + ), + "over_halt": bool(condition == "healthy" and not report.success), + "oracle": oracle, + "rung_counts": dict(report.rung_counts), + "errors": [result.error for result in report.results if result.error], + } + + +def run(container: str, root: Path, out: Path, work: Path) -> dict[str, Any]: + from openadapt_flow.backends.rdp_backend import FreeRDPBackend + from openadapt_flow.compiler import compile_recording + from openadapt_flow.ir import ActionKind + + root.mkdir(parents=True, exist_ok=True) + out.parent.mkdir(parents=True, exist_ok=True) + work.mkdir(parents=True, exist_ok=True) + _reset(container, root) + record_backend = FreeRDPBackend(DockerX11RdpTransport(container), connect=True) + recording_dir = work / "recording" + bundle_dir = work / "bundle" + _record(record_backend, recording_dir) + _arm_recording(recording_dir) + compiled = compile_recording(recording_dir, bundle_dir, name="rdp-multiapp-vision") + workflow, verifier, gate, step_ids = _qualify(compiled, bundle_dir, root) + pointer_actions = { + ActionKind.CLICK, + ActionKind.DOUBLE_CLICK, + ActionKind.RIGHT_CLICK, + ActionKind.DRAG, + } + save_index = next( + i for i, step in enumerate(workflow.steps) if step.id == step_ids["save"] + ) + save_pointer_acquisition = sum( + step.action in pointer_actions for step in workflow.steps[: save_index + 1] + ) + conditions = ( + "healthy", + "row_reordered", + "wrong_record_before_write", + "focus_theft_before_write", + ) + trials: list[dict[str, Any]] = [] + for condition in conditions: + for index in range(1, TRIALS + 1): + trials.append( + _run_once( + container=container, + root=root, + workflow=workflow, + verifier=verifier, + gate=gate, + bundle_dir=bundle_dir, + run_dir=work / f"run-{condition}-{index}", + condition=condition, + save_pointer_acquisition=save_pointer_acquisition, + ) + ) + result = { + "schema_version": "openadapt.rdp-multiapp-results.v1", + "campaign_contract": "benchmark/rdp_multiapp/campaign.json", + "implemented_conditions": list(conditions), + "full_campaign_complete": False, + "full_campaign_pending_conditions": [ + "duplicate_save_control", + "partial_render", + "moderate_display_drift", + "severe_display_drift", + "commit_then_timeout", + ], + "run_count": len(trials), + "accepted_subset": all(trial["passed"] for trial in trials), + "silent_incorrect_successes": sum( + bool(trial["silent_incorrect_success"]) for trial in trials + ), + "over_halts": sum(bool(trial["over_halt"]) for trial in trials), + "model_calls": sum(int(trial["model_calls"]) for trial in trials), + "trials": trials, + } + out.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--container", default="oaflow-rdp-multiapp") + parser.add_argument("--oracle-root", type=Path, required=True) + parser.add_argument( + "--output", type=Path, default=Path("benchmark/rdp_multiapp/results.json") + ) + parser.add_argument("--work-dir", type=Path) + args = parser.parse_args() + work = args.work_dir or Path(tempfile.mkdtemp(prefix="oaflow-rdp-multiapp-")) + result = run(args.container, args.oracle_root.resolve(), args.output, work) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 if result["accepted_subset"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py b/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py new file mode 100644 index 00000000..5233ec8c --- /dev/null +++ b/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py @@ -0,0 +1,32 @@ +"""Opt-in real-RDP multi-window visual workflow qualification.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from benchmark.rdp_multiapp.run_qualification import run + + +def test_real_rdp_multiapp_visual_subset(tmp_path: Path) -> None: + container = os.getenv("OPENADAPT_RDP_MULTIAPP_CONTAINER") + oracle = os.getenv("OPENADAPT_RDP_MULTIAPP_ORACLE_ROOT") + if not container or not oracle: + pytest.skip( + "set OPENADAPT_RDP_MULTIAPP_CONTAINER and " + "OPENADAPT_RDP_MULTIAPP_ORACLE_ROOT for the real-RDP fixture" + ) + + output = tmp_path / "results.json" + result = run(container, Path(oracle), output, tmp_path / "work") + + assert result["accepted_subset"] is True + assert result["run_count"] == 12 + assert result["silent_incorrect_successes"] == 0 + assert result["over_halts"] == 0 + assert result["model_calls"] == 0 + assert result["full_campaign_complete"] is False + assert json.loads(output.read_text()) == result diff --git a/tests/test_rdp_multiapp_campaign_contract.py b/tests/test_rdp_multiapp_campaign_contract.py new file mode 100644 index 00000000..292a51c2 --- /dev/null +++ b/tests/test_rdp_multiapp_campaign_contract.py @@ -0,0 +1,32 @@ +"""Behavioral contract for the real-RDP visual fault campaign.""" + +from __future__ import annotations + +import json +from pathlib import Path + +ROOT = Path(__file__).parents[1] + + +def test_visual_campaign_has_repeated_trials_and_business_oracles() -> None: + campaign = json.loads( + (ROOT / "benchmark/rdp_multiapp/campaign.json").read_text(encoding="utf-8") + ) + conditions = campaign["conditions"] + + assert campaign["trials_per_condition"] >= 3 + assert len({condition["id"] for condition in conditions}) == len(conditions) + assert {"sqlite", "csv", "maildir"} == { + oracle for condition in conditions for oracle in condition["oracle"] + } + assert { + "silent_incorrect_successes", + "over_halts", + "wrong_record_writes", + "duplicate_effects", + } <= set(campaign["required_metrics"]) + + by_id = {condition["id"]: condition for condition in conditions} + assert by_id["wrong_record_before_write"]["expect"] == "safe_halt" + assert by_id["focus_theft_before_write"]["expect"] == "safe_halt" + assert by_id["commit_then_timeout"]["oracle"] == ["sqlite"] From ef409ef6d3505c6d18bf4c1105d614131fe2b437 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 12:18:26 +0200 Subject: [PATCH 02/19] chore: register RDP campaign public artifacts --- public-artifacts.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/public-artifacts.json b/public-artifacts.json index 65bbe4af..b36187db 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -95,6 +95,10 @@ "path": ".github/workflows/citrix-workspace-standin.yml", "sha256": "0182320bfda16d62b36f2a71642f11fcaa7747d16b2a6be87db714c0669de253" }, + { + "path": ".github/workflows/docker-rdp-multiapp-vision.yml", + "sha256": "b069c5114066c10a2f12a53839aed682307ed60a4eea75a3cfef99d2a696d284" + }, { "path": ".github/workflows/docker-rdp-vision-ladder.yml", "sha256": "dbcb47f6ba33b0932f9aa0d49a6b5f66170e20701d77581713d0dc5ec20e5dd1" @@ -423,6 +427,14 @@ "path": "benchmark/rdp_ladder/results.json", "sha256": "3b946dd098c4a4c2ca56c1ab531a2eaeeb37804aa10c440e576f7196a03ae37c" }, + { + "path": "benchmark/rdp_multiapp/campaign.json", + "sha256": "b63def3cfacf17ee438e9bee9fd3a6057ee387f40956ee1adc38175ddad0e0a6" + }, + { + "path": "benchmark/rdp_multiapp/policy.yaml", + "sha256": "ff978d7755b24731c22f311b1791b4f9b70c8dd515c17e8beefb4d0e22f78c23" + }, { "path": "benchmark/reliability/summary.json", "sha256": "54ca722fd49a9eb4e0cb1fda7de6781e2a71f45aa872b849045dd106cce4ad1a" From 127bd9c6e785fc4c6cfb4bf825aff032488ea7a5 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 12:23:07 +0200 Subject: [PATCH 03/19] ci: expose bounded RDP fixture diagnostics --- .../workflows/docker-rdp-multiapp-vision.yml | 10 +++++++++- benchmark/rdp_multiapp/run_qualification.py | 19 ++++++++++++++++++- public-artifacts.json | 2 +- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-rdp-multiapp-vision.yml b/.github/workflows/docker-rdp-multiapp-vision.yml index 5fd0997b..23ec52e0 100644 --- a/.github/workflows/docker-rdp-multiapp-vision.yml +++ b/.github/workflows/docker-rdp-multiapp-vision.yml @@ -61,7 +61,15 @@ jobs: with: name: rdp-multiapp-vision-subset path: ${{ runner.temp }}/rdp-multiapp-result/results.json - if-no-files-found: error + if-no-files-found: warn + + - name: Print bounded fixture diagnostics + if: failure() + run: | + docker logs --tail 80 oaflow-rdp-multiapp || true + docker exec oaflow-rdp-multiapp sh -c \ + 'for path in /tmp/suite.log /tmp/client.log /tmp/shadow.log; do echo "== ${path} =="; tail -n 80 "${path}" 2>/dev/null || true; done' + ls -la "${RUNNER_TEMP}/rdp-multiapp-oracle" || true - name: Tear down if: always() diff --git a/benchmark/rdp_multiapp/run_qualification.py b/benchmark/rdp_multiapp/run_qualification.py index fec55da6..eed1437b 100644 --- a/benchmark/rdp_multiapp/run_qualification.py +++ b/benchmark/rdp_multiapp/run_qualification.py @@ -148,16 +148,33 @@ def _reset(container: str, root: Path, scenario: str = "healthy") -> None: if result.returncode != 0: raise RuntimeError(result.stderr.decode(errors="replace")[:300]) deadline = time.monotonic() + 10 + diagnostics: dict[str, Any] = {} while time.monotonic() < deadline: after = _read_ack(root) advanced = after is not None and (before is None or after > before) rows = _read_database(root) worklist = _read_worklist(root) mail = _read_mail(root) + diagnostics = { + "ack_before": before, + "ack_after": after, + "ack_advanced": advanced, + "database": rows, + "worklist_rows": None if worklist is None else len(worklist), + "mail": mail, + "root_entries": ( + sorted(path.name for path in root.iterdir()) + if root.is_dir() + else None + ), + } if advanced and rows == [] and worklist and mail == []: return time.sleep(0.1) - raise RuntimeError("fixture reset did not produce clean persisted state") + raise RuntimeError( + "fixture reset did not produce clean persisted state: " + + json.dumps(diagnostics, sort_keys=True, default=str) + ) def _record(backend: Any, recording_dir: Path) -> None: diff --git a/public-artifacts.json b/public-artifacts.json index b36187db..2b131361 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -97,7 +97,7 @@ }, { "path": ".github/workflows/docker-rdp-multiapp-vision.yml", - "sha256": "b069c5114066c10a2f12a53839aed682307ed60a4eea75a3cfef99d2a696d284" + "sha256": "9959bd76d8d7590377c49113805990f48ee84be3364506870e641b395152d835" }, { "path": ".github/workflows/docker-rdp-vision-ladder.yml", From 67faac97afecb3fad30472aaff08169ca79d4fca Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 12:27:00 +0200 Subject: [PATCH 04/19] fix: use deterministic RDP fixture reset handshake --- benchmark/rdp_multiapp/fixture/suite_app.py | 24 +++++++++++----- benchmark/rdp_multiapp/run_qualification.py | 31 ++++++++------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/benchmark/rdp_multiapp/fixture/suite_app.py b/benchmark/rdp_multiapp/fixture/suite_app.py index 870f86ce..d77f2637 100644 --- a/benchmark/rdp_multiapp/fixture/suite_app.py +++ b/benchmark/rdp_multiapp/fixture/suite_app.py @@ -12,7 +12,6 @@ import hashlib import json import os -import signal import sqlite3 import tkinter as tk from email.message import EmailMessage @@ -64,6 +63,14 @@ def _scenario() -> str: return "healthy" +def _reset_token() -> str | None: + try: + value = json.loads(CONTROL_PATH.read_text()).get("reset_token") + except (OSError, ValueError, TypeError): + return None + return value if isinstance(value, str) and value else None + + def _rows() -> list[dict[str, str]]: rows = [ { @@ -127,14 +134,14 @@ def __init__(self) -> None: self.windows: dict[str, tk.Toplevel] = {} self.selected_request: str | None = None self.active_record: tuple[str, str] | None = None - self.reset_counter = 0 + self.last_reset_token = _reset_token() self._build_inbox() self._build_worklist() self._build_scheduler() self._build_launcher() self.reset() self.root.after(250, lambda: self.show("Inbox")) - signal.signal(signal.SIGUSR1, self._signal_reset) + self.root.after(100, self._poll_control) def _window(self, title: str) -> tk.Toplevel: window = tk.Toplevel(self.root) @@ -457,8 +464,12 @@ def _save_appointment(self) -> None: text=f"Appointment saved · {appointment_id}", fg=GREEN ) - def _signal_reset(self, _signum, _frame) -> None: - self.root.after(0, self.reset) + def _poll_control(self) -> None: + token = _reset_token() + if token is not None and token != self.last_reset_token: + self.last_reset_token = token + self.reset() + self.root.after(100, self._poll_control) def reset(self) -> None: _reset_persisted_state() @@ -476,8 +487,7 @@ def reset(self) -> None: for entry in (self.slot, self.kind, self.request): entry.delete(0, "end") self.scheduler_status.config(text="") - self.reset_counter += 1 - ACK_PATH.write_text(str(self.reset_counter), encoding="utf-8") + ACK_PATH.write_text(self.last_reset_token or "startup", encoding="utf-8") self.show("Inbox") def run(self) -> None: diff --git a/benchmark/rdp_multiapp/run_qualification.py b/benchmark/rdp_multiapp/run_qualification.py index eed1437b..2b569e9b 100644 --- a/benchmark/rdp_multiapp/run_qualification.py +++ b/benchmark/rdp_multiapp/run_qualification.py @@ -14,7 +14,6 @@ import json import secrets import sqlite3 -import subprocess import tempfile import time from email import policy as email_policy @@ -71,10 +70,11 @@ POLICY_PATH = Path(__file__).with_name("policy.yaml") -def _read_ack(root: Path) -> Optional[int]: +def _read_ack(root: Path) -> Optional[str]: try: - return int((root / "reset_ack.txt").read_text().strip()) - except (OSError, ValueError): + value = (root / "reset_ack.txt").read_text().strip() + return value or None + except OSError: return None @@ -133,32 +133,25 @@ def _read_mail(root: Path) -> Optional[list[dict[str, str]]]: return records -def _reset(container: str, root: Path, scenario: str = "healthy") -> None: - before = _read_ack(root) +def _reset(_container: str, root: Path, scenario: str = "healthy") -> None: + token = secrets.token_hex(16) root.mkdir(parents=True, exist_ok=True) (root / "control.json").write_text( - json.dumps({"scenario": scenario}) + "\n", encoding="utf-8" - ) - result = subprocess.run( - ["docker", "exec", container, "pkill", "-USR1", "-f", "suite_app.py"], - capture_output=True, - timeout=30, - check=False, + json.dumps({"reset_token": token, "scenario": scenario}) + "\n", + encoding="utf-8", ) - if result.returncode != 0: - raise RuntimeError(result.stderr.decode(errors="replace")[:300]) deadline = time.monotonic() + 10 diagnostics: dict[str, Any] = {} while time.monotonic() < deadline: after = _read_ack(root) - advanced = after is not None and (before is None or after > before) + acknowledged = after == token rows = _read_database(root) worklist = _read_worklist(root) mail = _read_mail(root) diagnostics = { - "ack_before": before, + "ack_expected": token, "ack_after": after, - "ack_advanced": advanced, + "acknowledged": acknowledged, "database": rows, "worklist_rows": None if worklist is None else len(worklist), "mail": mail, @@ -168,7 +161,7 @@ def _reset(container: str, root: Path, scenario: str = "healthy") -> None: else None ), } - if advanced and rows == [] and worklist and mail == []: + if acknowledged and rows == [] and worklist and mail == []: return time.sleep(0.1) raise RuntimeError( From b7796c21f8a6d3c3141000a4b64be0d899e3ffef Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 12:29:30 +0200 Subject: [PATCH 05/19] test: strengthen RDP vision acceptance oracles --- benchmark/rdp_multiapp/campaign.json | 2 +- benchmark/rdp_multiapp/fixture/suite_app.py | 20 ++++ benchmark/rdp_multiapp/run_qualification.py | 109 ++++++++++++++++--- public-artifacts.json | 2 +- tests/test_rdp_multiapp_campaign_contract.py | 1 + 5 files changed, 119 insertions(+), 15 deletions(-) diff --git a/benchmark/rdp_multiapp/campaign.json b/benchmark/rdp_multiapp/campaign.json index 14c309aa..2133e206 100644 --- a/benchmark/rdp_multiapp/campaign.json +++ b/benchmark/rdp_multiapp/campaign.json @@ -22,7 +22,7 @@ }, { "id": "row_reordered", - "expect": "verified_or_safe_halt", + "expect": "verified", "fault": "The target worklist row moves to a different visual position.", "oracle": ["sqlite", "csv", "maildir"] }, diff --git a/benchmark/rdp_multiapp/fixture/suite_app.py b/benchmark/rdp_multiapp/fixture/suite_app.py index d77f2637..5ed41d4b 100644 --- a/benchmark/rdp_multiapp/fixture/suite_app.py +++ b/benchmark/rdp_multiapp/fixture/suite_app.py @@ -23,6 +23,7 @@ MAILDIR = ROOT / "outbox" CONTROL_PATH = ROOT / "control.json" ACK_PATH = ROOT / "reset_ack.txt" +INPUT_LEDGER_PATH = ROOT / "input-ledger.jsonl" BG = "#eef2f6" PANEL = "#ffffff" @@ -112,6 +113,7 @@ def _reset_persisted_state() -> None: connection.execute("DELETE FROM appointments") connection.commit() _write_rows(_rows()) + INPUT_LEDGER_PATH.write_text("", encoding="utf-8") for subdir in ("cur", "new", "tmp"): path = MAILDIR / subdir path.mkdir(parents=True, exist_ok=True) @@ -120,6 +122,12 @@ def _reset_persisted_state() -> None: child.unlink() +def _record_input(action: str, **details: str | None) -> None: + entry = {"action": action, **details} + with INPUT_LEDGER_PATH.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(entry, sort_keys=True) + "\n") + + def _label(parent: tk.Misc, text: str, x: int, y: int, **kwargs) -> tk.Label: label = tk.Label(parent, text=text, bg=BG, fg=FG, **kwargs) label.place(x=x, y=y) @@ -249,6 +257,10 @@ def _select_inbox(self) -> None: self.send_button.config(state="normal") def _send_confirmation(self) -> None: + _record_input( + "send_confirmation", + selected_request=self.selected_request, + ) rows = _read_rows() status = next( (row["status"] for row in rows if row["request_id"] == TARGET_REQUEST), @@ -366,6 +378,7 @@ def _select_work(self, request_id: str) -> None: self.mark_button.config(state="normal") def _mark_scheduled(self) -> None: + _record_input("mark_scheduled", selected_request=self.work_selected) if self.work_selected != TARGET_REQUEST: self.work_status.config(text="Refused: wrong request selected", fg=RED) return @@ -440,6 +453,13 @@ def _save_appointment(self) -> None: slot = self.slot.get().strip() kind = self.kind.get().strip() request_id = self.request.get().strip() + _record_input( + "save_appointment", + active_record_id=(self.active_record or (None, None))[1], + appointment_slot=slot, + appointment_type=kind, + request_id=request_id, + ) if self.active_record != (TARGET_NAME, TARGET_RECORD): self.scheduler_status.config(text="Refused: wrong active record", fg=RED) return diff --git a/benchmark/rdp_multiapp/run_qualification.py b/benchmark/rdp_multiapp/run_qualification.py index 2b569e9b..308a86dc 100644 --- a/benchmark/rdp_multiapp/run_qualification.py +++ b/benchmark/rdp_multiapp/run_qualification.py @@ -14,6 +14,7 @@ import json import secrets import sqlite3 +import statistics import tempfile import time from email import policy as email_policy @@ -133,6 +134,17 @@ def _read_mail(root: Path) -> Optional[list[dict[str, str]]]: return records +def _read_input_ledger(root: Path) -> Optional[list[dict[str, Any]]]: + try: + return [ + json.loads(line) + for line in (root / "input-ledger.jsonl").read_text().splitlines() + if line + ] + except (OSError, ValueError, TypeError): + return None + + def _reset(_container: str, root: Path, scenario: str = "healthy") -> None: token = secrets.token_hex(16) root.mkdir(parents=True, exist_ok=True) @@ -148,6 +160,7 @@ def _reset(_container: str, root: Path, scenario: str = "healthy") -> None: rows = _read_database(root) worklist = _read_worklist(root) mail = _read_mail(root) + input_ledger = _read_input_ledger(root) diagnostics = { "ack_expected": token, "ack_after": after, @@ -155,13 +168,18 @@ def _reset(_container: str, root: Path, scenario: str = "healthy") -> None: "database": rows, "worklist_rows": None if worklist is None else len(worklist), "mail": mail, + "input_ledger": input_ledger, "root_entries": ( - sorted(path.name for path in root.iterdir()) - if root.is_dir() - else None + sorted(path.name for path in root.iterdir()) if root.is_dir() else None ), } - if acknowledged and rows == [] and worklist and mail == []: + if ( + acknowledged + and rows == [] + and worklist + and mail == [] + and input_ledger == [] + ): return time.sleep(0.1) raise RuntimeError( @@ -403,6 +421,7 @@ def _oracle_result(root: Path, params: dict[str, str]) -> dict[str, Any]: database = _read_database(root) worklist = _read_worklist(root) mail = _read_mail(root) + input_ledger = _read_input_ledger(root) expected_db = { "request_id": TARGET_REQUEST, "record_id": TARGET_RECORD, @@ -439,15 +458,56 @@ def _oracle_result(root: Path, params: dict[str, str]) -> dict[str, Any]: and mail[0].get("subject") == f"Scheduled {TARGET_REQUEST}" and mail[0].get("request_id") == TARGET_REQUEST ) + expected_input_counts = { + "save_appointment": 1, + "mark_scheduled": 1, + "send_confirmation": 1, + } + input_counts = { + action: sum(1 for entry in input_ledger or [] if entry.get("action") == action) + for action in expected_input_counts + } + input_cardinality_ok = bool( + input_ledger is not None and input_counts == expected_input_counts + ) + wrong_record_input = bool( + input_ledger is not None + and any( + entry.get("action") == "save_appointment" + and entry.get("active_record_id") != TARGET_RECORD + for entry in input_ledger + ) + ) return { "database_ok": db_ok, "worklist_ok": csv_ok, "mail_ok": mail_ok, - "all_effects_ok": db_ok and csv_ok and mail_ok, + "all_effects_ok": db_ok and csv_ok and mail_ok and input_cardinality_ok, "database": database, "worklist_target": target_rows, "mail": mail, "adjacent_rows_unchanged": adjacent_unchanged, + "input_ledger": input_ledger, + "input_counts": input_counts, + "input_cardinality_ok": input_cardinality_ok, + "no_consequential_input": not any(input_counts.values()), + "wrong_record_write": bool( + wrong_record_input + or ( + database is not None + and any( + row.get("request_id") != TARGET_REQUEST + or row.get("record_id") != TARGET_RECORD + for row in database + ) + ) + ), + "duplicate_effect": bool( + (database is not None and len(database) > 1) + or len(target_rows) > 1 + or (mail is not None and len(mail) > 1) + or any(count > 1 for count in input_counts.values()) + ), } @@ -521,13 +581,13 @@ def acquire_with_fault() -> bytes: "wrong_record_before_write", "focus_theft_before_write", } - safe_halt = expected_halt and not report.success and not oracle["database"] - if condition == "row_reordered": - passed = bool( - (report.success and oracle["all_effects_ok"]) - or (not report.success and not oracle["database"]) - ) - elif expected_halt: + safe_halt = bool( + expected_halt + and not report.success + and not oracle["database"] + and oracle["no_consequential_input"] + ) + if expected_halt: passed = bool(injected and safe_halt) else: passed = bool(report.success and oracle["all_effects_ok"]) @@ -542,7 +602,9 @@ def acquire_with_fault() -> bytes: "silent_incorrect_success": bool( report.success and not oracle["all_effects_ok"] ), - "over_halt": bool(condition == "healthy" and not report.success), + "over_halt": bool( + condition in {"healthy", "row_reordered"} and not report.success + ), "oracle": oracle, "rung_counts": dict(report.rung_counts), "errors": [result.error for result in report.results if result.error], @@ -599,6 +661,14 @@ def run(container: str, root: Path, out: Path, work: Path) -> dict[str, Any]: save_pointer_acquisition=save_pointer_acquisition, ) ) + runtimes = sorted(float(trial["runtime_s"]) for trial in trials) + + def nearest_rank(percentile: float) -> float: + index = max( + 0, min(len(runtimes) - 1, int(len(runtimes) * percentile + 0.999999) - 1) + ) + return runtimes[index] + result = { "schema_version": "openadapt.rdp-multiapp-results.v1", "campaign_contract": "benchmark/rdp_multiapp/campaign.json", @@ -613,11 +683,24 @@ def run(container: str, root: Path, out: Path, work: Path) -> dict[str, Any]: ], "run_count": len(trials), "accepted_subset": all(trial["passed"] for trial in trials), + "verified_outcomes": sum( + bool(trial["runtime_success"] and trial["oracle"]["all_effects_ok"]) + for trial in trials + ), + "safe_halts": sum(bool(trial["safe_halt"]) for trial in trials), "silent_incorrect_successes": sum( bool(trial["silent_incorrect_success"]) for trial in trials ), "over_halts": sum(bool(trial["over_halt"]) for trial in trials), + "wrong_record_writes": sum( + bool(trial["oracle"]["wrong_record_write"]) for trial in trials + ), + "duplicate_effects": sum( + bool(trial["oracle"]["duplicate_effect"]) for trial in trials + ), "model_calls": sum(int(trial["model_calls"]) for trial in trials), + "p50_runtime_s": round(statistics.median(runtimes), 3), + "p95_runtime_s": round(nearest_rank(0.95), 3), "trials": trials, } out.write_text( diff --git a/public-artifacts.json b/public-artifacts.json index 2b131361..8b6dfa21 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -429,7 +429,7 @@ }, { "path": "benchmark/rdp_multiapp/campaign.json", - "sha256": "b63def3cfacf17ee438e9bee9fd3a6057ee387f40956ee1adc38175ddad0e0a6" + "sha256": "b00bc748a8a2d54d8e117aa267fa1669d52d2ecd5a9b11a080c7d43e032f728a" }, { "path": "benchmark/rdp_multiapp/policy.yaml", diff --git a/tests/test_rdp_multiapp_campaign_contract.py b/tests/test_rdp_multiapp_campaign_contract.py index 292a51c2..1e5fb3a6 100644 --- a/tests/test_rdp_multiapp_campaign_contract.py +++ b/tests/test_rdp_multiapp_campaign_contract.py @@ -27,6 +27,7 @@ def test_visual_campaign_has_repeated_trials_and_business_oracles() -> None: } <= set(campaign["required_metrics"]) by_id = {condition["id"]: condition for condition in conditions} + assert by_id["row_reordered"]["expect"] == "verified" assert by_id["wrong_record_before_write"]["expect"] == "safe_halt" assert by_id["focus_theft_before_write"]["expect"] == "safe_halt" assert by_id["commit_then_timeout"]["oracle"] == ["sqlite"] From 4a8da5bfdc3990edefa3f8b38360b63ab58d28fd Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 12:34:46 +0200 Subject: [PATCH 06/19] fix: qualify every RDP visual pointer action --- benchmark/rdp_multiapp/run_qualification.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/benchmark/rdp_multiapp/run_qualification.py b/benchmark/rdp_multiapp/run_qualification.py index 308a86dc..9332e29d 100644 --- a/benchmark/rdp_multiapp/run_qualification.py +++ b/benchmark/rdp_multiapp/run_qualification.py @@ -67,6 +67,11 @@ ACTIVE_RECORD_REGION = (510, 86, 740, 60) WORKLIST_SELECTION_REGION = (35, 596, 920, 58) INBOX_DETAIL_REGION = (35, 238, 915, 150) +INBOX_HEADER_REGION = (35, 20, 930, 82) +INBOX_REQUEST_REGION = (35, 123, 890, 100) +WORKLIST_HEADER_REGION = (35, 20, 930, 82) +WORKLIST_TARGET_REGION = (35, 455, 920, 70) +TARGET_RECORD_REGION = (35, 123, 430, 68) POLICY_PATH = Path(__file__).with_name("policy.yaml") @@ -227,8 +232,22 @@ def _arm_recording(recording_dir: Path) -> None: path = recording_dir / "events.jsonl" events = [json.loads(line) for line in path.read_text().splitlines() if line] expected = { + 0: INBOX_REQUEST_REGION, + 1: INBOX_HEADER_REGION, + 2: WORKLIST_HEADER_REGION, + 4: WORKLIST_TARGET_REGION, + 5: WORKLIST_SELECTION_REGION, + 6: TARGET_RECORD_REGION, + 7: ACTIVE_RECORD_REGION, + 9: ACTIVE_RECORD_REGION, + 11: ACTIVE_RECORD_REGION, 13: ACTIVE_RECORD_REGION, + 14: ACTIVE_RECORD_REGION, + 15: WORKLIST_HEADER_REGION, + 17: WORKLIST_TARGET_REGION, 18: WORKLIST_SELECTION_REGION, + 19: WORKLIST_SELECTION_REGION, + 20: INBOX_REQUEST_REGION, 21: INBOX_DETAIL_REGION, } for index, region in expected.items(): From 2ac226ce68059ecfd53b7d2974d259211b20ca8f Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 12:57:44 +0200 Subject: [PATCH 07/19] fix: continue bounded scroll after OCR ambiguity --- benchmark/rdp_multiapp/fixture/suite_app.py | 15 +++++++ benchmark/rdp_multiapp/run_qualification.py | 2 +- openadapt_flow/runtime/replayer.py | 27 ++++++----- tests/test_replayer.py | 50 +++++++++++++++++++++ 4 files changed, 83 insertions(+), 11 deletions(-) diff --git a/benchmark/rdp_multiapp/fixture/suite_app.py b/benchmark/rdp_multiapp/fixture/suite_app.py index 5ed41d4b..6c1742dd 100644 --- a/benchmark/rdp_multiapp/fixture/suite_app.py +++ b/benchmark/rdp_multiapp/fixture/suite_app.py @@ -327,6 +327,21 @@ def _build_worklist(self) -> None: for widget in (canvas, self.work_inner): widget.bind("", lambda _event: canvas.yview_scroll(-1, "units")) widget.bind("", lambda _event: canvas.yview_scroll(1, "units")) + self.work_focus_button = tk.Button( + window, + text="Scroll request list", + command=canvas.focus_set, + bg="#dce7ff", + fg=FG, + font=("DejaVu Sans", 13, "bold"), + ) + self.work_focus_button.place(x=990, y=125, width=240, height=44) + self.work_focus_button.bind( + "", lambda _event: canvas.yview_scroll(-1, "units") + ) + self.work_focus_button.bind( + "", lambda _event: canvas.yview_scroll(1, "units") + ) self.work_status = _label( window, "No request selected", 42, 610, font=("DejaVu Sans", 15, "bold") ) diff --git a/benchmark/rdp_multiapp/run_qualification.py b/benchmark/rdp_multiapp/run_qualification.py index 9332e29d..e1fdfd04 100644 --- a/benchmark/rdp_multiapp/run_qualification.py +++ b/benchmark/rdp_multiapp/run_qualification.py @@ -52,7 +52,7 @@ WORKLIST_TAB = (555, 770) SCHEDULER_TAB = (745, 770) INBOX_TAB = (365, 770) -WORKLIST_VIEW = (400, 350) +WORKLIST_VIEW = (1110, 147) WORKLIST_TARGET_AFTER_SCROLL = (430, 490) MARK_SCHEDULED = (1110, 217) TARGET_RECORD_ROW = (250, 155) diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py index 513e64dd..2ad5cd4d 100644 --- a/openadapt_flow/runtime/replayer.py +++ b/openadapt_flow/runtime/replayer.py @@ -11597,16 +11597,23 @@ def _implicit_scroll_target_ready( if self.use_structural and hasattr(self.backend, "locate_structural") else None ) - resolved = resolve( - step.anchor, - frame_png, - self.vision, - None, # scroll readiness must remain deterministic and model-free - step.intent, - template_png=template_png, - viewport=self.backend.viewport, - structural=structural, - ) + try: + resolved = resolve( + step.anchor, + frame_png, + self.vision, + None, # scroll readiness must remain deterministic and model-free + step.intent, + template_png=template_png, + viewport=self.backend.viewport, + structural=structural, + ) + except OcrResolutionRefused: + # Readiness is a bounded, non-actuating probe. An ambiguous OCR + # candidate means that the target is not ready yet, so the scroll + # loop must continue. The later target action runs the full + # resolver again and still halts on the same ambiguity. + return False if resolved is None: return False resolution, _matched_region = resolved diff --git a/tests/test_replayer.py b/tests/test_replayer.py index 0b6771be..8cb453ff 100644 --- a/tests/test_replayer.py +++ b/tests/test_replayer.py @@ -21,6 +21,7 @@ Landmark, Postcondition, PostconditionKind, + Resolution, RunReport, Step, Workflow, @@ -1846,6 +1847,55 @@ def test_closed_loop_scroll_noops_when_anchor_already_in_view(bundle, run_dir): assert backend.actions == [("click", 110, 105, False)] +def test_closed_loop_scroll_ocr_ambiguity_continues_but_click_halts( + bundle, run_dir, monkeypatch +): + """Only the non-actuating readiness probe can treat ambiguity as not ready.""" + + target = ( + Resolution( + rung="ocr", + point=(110, 105), + confidence=0.95, + elapsed_ms=1.0, + ), + (100, 100, 50, 20), + ) + outcomes = iter( + [ + AmbiguousOcrMatchError("two off-screen candidates"), + target, + AmbiguousOcrMatchError("two live click candidates"), + ] + ) + + def scripted_resolve(*args, **kwargs): + del args, kwargs + outcome = next(outcomes) + if isinstance(outcome, Exception): + raise outcome + return outcome + + monkeypatch.setattr( + "openadapt_flow.runtime.replayer.resolve", + scripted_resolve, + ) + backend = FakeBackend() + workflow = Workflow(name="wf", steps=[scroll_step(), click_step()]) + + report = Replayer(backend, vision=FakeVision(), poll_interval_s=0.01).run( + workflow, + bundle_dir=bundle, + run_dir=run_dir, + ) + + assert report.success is False + assert backend.actions == [("scroll", 0, 400)] + assert any( + "two live click candidates" in (result.error or "") for result in report.results + ) + + def test_closed_loop_scroll_requires_armed_target_identity( bundle, run_dir, monkeypatch ): From 2b07b96fe1c9e0056724056d74ec6c74738bcf62 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 12:54:44 +0200 Subject: [PATCH 08/19] test: exercise uncertain RDP save delivery --- benchmark/rdp_multiapp/run_qualification.py | 97 ++++++++++++++++++- .../test_docker_rdp_multiapp_vision_e2e.py | 17 +++- tests/test_rdp_multiapp_campaign_contract.py | 43 ++++++++ 3 files changed, 154 insertions(+), 3 deletions(-) diff --git a/benchmark/rdp_multiapp/run_qualification.py b/benchmark/rdp_multiapp/run_qualification.py index e1fdfd04..37083dff 100644 --- a/benchmark/rdp_multiapp/run_qualification.py +++ b/benchmark/rdp_multiapp/run_qualification.py @@ -17,6 +17,7 @@ import statistics import tempfile import time +from collections.abc import Callable from email import policy as email_policy from email.parser import BytesParser from pathlib import Path @@ -76,6 +77,46 @@ POLICY_PATH = Path(__file__).with_name("policy.yaml") +def _install_commit_then_timeout_fault( + backend: Any, + *, + condition: str, + save_pointer_acquisition: int, + acquisition_count: Callable[[], int], +) -> dict[str, Any]: + """Lose the receipt only after the real qualified Save click returns. + + The wrapper does not simulate the write. It lets the RDP backend send the + guarded pointer action to the fixture first. It then reports typed delivery + uncertainty to the runtime. The runtime must not send the click again. + """ + + state: dict[str, Any] = {"injected": False, "save_delivery_calls": 0} + original_click_guarded = backend.click_guarded + + def click_guarded(*args: Any, **kwargs: Any): + from openadapt_flow.backend import ActionDeliveryUncertain + + receipt = original_click_guarded(*args, **kwargs) + if ( + condition == "commit_then_timeout" + and acquisition_count() == save_pointer_acquisition + ): + state["save_delivery_calls"] += 1 + if not state["injected"]: + state["injected"] = True + raise ActionDeliveryUncertain( + operation="rdp_click", + native=False, + target_fingerprint=receipt.target_fingerprint, + cause_type="TimeoutError", + ) + return receipt + + backend.click_guarded = click_guarded + return state + + def _read_ack(root: Path) -> Optional[str]: try: value = (root / "reset_ack.txt").read_text().strip() @@ -579,6 +620,12 @@ def acquire_with_fault() -> bytes: return original_acquire() backend.acquire_actuation_frame = acquire_with_fault # type: ignore[method-assign] + commit_timeout = _install_commit_then_timeout_fault( + backend, + condition=condition, + save_pointer_acquisition=save_pointer_acquisition, + acquisition_count=lambda: acquisitions, + ) started = time.monotonic() report = Replayer( backend, @@ -608,6 +655,38 @@ def acquire_with_fault() -> bytes: ) if expected_halt: passed = bool(injected and safe_halt) + elif condition == "commit_then_timeout": + save_result = next( + result + for result in report.results + if result.step_id == workflow.steps[13].id + ) + uncertainty = save_result.delivery_uncertainty + sqlite_proved_one_write = bool( + oracle["database_ok"] + and oracle["input_counts"].get("save_appointment") == 1 + and not oracle["duplicate_effect"] + ) + verified_after_uncertainty = bool( + report.success + and report.transaction_outcome == "VERIFIED" + and sqlite_proved_one_write + and uncertainty is not None + and uncertainty.retried is False + and uncertainty.effects_confirmed is True + and uncertainty.resolved_by_contract is True + ) + reconciliation_required = bool( + not report.success + and not sqlite_proved_one_write + and report.transaction_outcome == "RECONCILIATION_REQUIRED" + ) + passed = bool( + commit_timeout["injected"] + and commit_timeout["save_delivery_calls"] == 1 + and oracle["input_counts"].get("save_appointment") == 1 + and (verified_after_uncertainty or reconciliation_required) + ) else: passed = bool(report.success and oracle["all_effects_ok"]) return { @@ -616,7 +695,21 @@ def acquire_with_fault() -> bytes: "runtime_s": round(time.monotonic() - started, 3), "runtime_success": bool(report.success), "model_calls": int(report.model_calls), - "fault_injected": injected, + "fault_injected": bool(injected or commit_timeout["injected"]), + "commit_timeout_injected": bool(commit_timeout["injected"]), + "save_delivery_calls": int(commit_timeout["save_delivery_calls"]), + "uncertain_delivery_outcome": ( + "verified" + if condition == "commit_then_timeout" + and report.success + and report.transaction_outcome == "VERIFIED" + and oracle["database_ok"] + else ( + "reconciliation_required" + if condition == "commit_then_timeout" + else None + ) + ), "safe_halt": safe_halt, "silent_incorrect_success": bool( report.success and not oracle["all_effects_ok"] @@ -663,6 +756,7 @@ def run(container: str, root: Path, out: Path, work: Path) -> dict[str, Any]: "row_reordered", "wrong_record_before_write", "focus_theft_before_write", + "commit_then_timeout", ) trials: list[dict[str, Any]] = [] for condition in conditions: @@ -698,7 +792,6 @@ def nearest_rank(percentile: float) -> float: "partial_render", "moderate_display_drift", "severe_display_drift", - "commit_then_timeout", ], "run_count": len(trials), "accepted_subset": all(trial["passed"] for trial in trials), diff --git a/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py b/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py index 5233ec8c..b35439c8 100644 --- a/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py +++ b/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py @@ -24,9 +24,24 @@ def test_real_rdp_multiapp_visual_subset(tmp_path: Path) -> None: result = run(container, Path(oracle), output, tmp_path / "work") assert result["accepted_subset"] is True - assert result["run_count"] == 12 + assert result["run_count"] == 15 assert result["silent_incorrect_successes"] == 0 assert result["over_halts"] == 0 assert result["model_calls"] == 0 assert result["full_campaign_complete"] is False + uncertain = [ + trial + for trial in result["trials"] + if trial["condition"] == "commit_then_timeout" + ] + assert len(uncertain) == 3 + assert all(trial["commit_timeout_injected"] for trial in uncertain) + assert all(trial["save_delivery_calls"] == 1 for trial in uncertain) + assert all( + trial["oracle"]["input_counts"]["save_appointment"] == 1 for trial in uncertain + ) + assert all( + trial["uncertain_delivery_outcome"] in {"verified", "reconciliation_required"} + for trial in uncertain + ) assert json.loads(output.read_text()) == result diff --git a/tests/test_rdp_multiapp_campaign_contract.py b/tests/test_rdp_multiapp_campaign_contract.py index 1e5fb3a6..df6b7d51 100644 --- a/tests/test_rdp_multiapp_campaign_contract.py +++ b/tests/test_rdp_multiapp_campaign_contract.py @@ -31,3 +31,46 @@ def test_visual_campaign_has_repeated_trials_and_business_oracles() -> None: assert by_id["wrong_record_before_write"]["expect"] == "safe_halt" assert by_id["focus_theft_before_write"]["expect"] == "safe_halt" assert by_id["commit_then_timeout"]["oracle"] == ["sqlite"] + + +def test_commit_then_timeout_fault_raises_only_after_one_real_save_delivery() -> None: + from benchmark.rdp_multiapp.run_qualification import ( + _install_commit_then_timeout_fault, + ) + from openadapt_flow.backend import ActionDeliveryUncertain + from openadapt_flow.ir import ActionDeliveryReceipt + + class Backend: + def __init__(self) -> None: + self.calls = 0 + + def click_guarded(self, *args, **kwargs): + del args, kwargs + self.calls += 1 + return ActionDeliveryReceipt( + receipt_id="real-save-delivery", + operation="rdp_click", + native=False, + target_fingerprint="a" * 64, + delivered_at="2026-08-02T00:00:00+00:00", + ) + + backend = Backend() + state = _install_commit_then_timeout_fault( + backend, + condition="commit_then_timeout", + save_pointer_acquisition=7, + acquisition_count=lambda: 7, + ) + + try: + backend.click_guarded(10, 20, expected_frame_sha256="0" * 64) + except ActionDeliveryUncertain as exc: + assert exc.operation == "rdp_click" + assert exc.cause_type == "TimeoutError" + assert exc.target_fingerprint == "a" * 64 + else: # pragma: no cover - the assertion describes the fault contract + raise AssertionError("the post-delivery timeout was not injected") + + assert backend.calls == 1 + assert state == {"injected": True, "save_delivery_calls": 1} From fc3f9bd6137af22d41349c50209378cce5b75abd Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 13:01:01 +0200 Subject: [PATCH 09/19] test: make uncertain RDP delivery evidence exact --- benchmark/rdp_multiapp/run_qualification.py | 41 ++++++++----------- .../test_docker_rdp_multiapp_vision_e2e.py | 2 +- tests/test_rdp_multiapp_campaign_contract.py | 7 +++- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/benchmark/rdp_multiapp/run_qualification.py b/benchmark/rdp_multiapp/run_qualification.py index 37083dff..9e930e7d 100644 --- a/benchmark/rdp_multiapp/run_qualification.py +++ b/benchmark/rdp_multiapp/run_qualification.py @@ -17,7 +17,6 @@ import statistics import tempfile import time -from collections.abc import Callable from email import policy as email_policy from email.parser import BytesParser from pathlib import Path @@ -62,6 +61,7 @@ TYPE_FIELD = (720, 342) REQUEST_FIELD = (720, 462) SAVE_APPOINTMENT = (650, 568) +SAVE_APPOINTMENT_REGION = (520, 540, 260, 56) SEND_CONFIRMATION = (165, 447) # Identity bands are recorded pixels, not live values sent out of the runner. @@ -81,8 +81,7 @@ def _install_commit_then_timeout_fault( backend: Any, *, condition: str, - save_pointer_acquisition: int, - acquisition_count: Callable[[], int], + save_region: tuple[int, int, int, int], ) -> dict[str, Any]: """Lose the receipt only after the real qualified Save click returns. @@ -98,10 +97,11 @@ def click_guarded(*args: Any, **kwargs: Any): from openadapt_flow.backend import ActionDeliveryUncertain receipt = original_click_guarded(*args, **kwargs) - if ( - condition == "commit_then_timeout" - and acquisition_count() == save_pointer_acquisition - ): + x = int(args[0] if args else kwargs["x"]) + y = int(args[1] if len(args) > 1 else kwargs["y"]) + left, top, width, height = save_region + is_save_attempt = left <= x < left + width and top <= y < top + height + if condition == "commit_then_timeout" and is_save_attempt: state["save_delivery_calls"] += 1 if not state["injected"]: state["injected"] = True @@ -582,6 +582,7 @@ def _run_once( run_dir: Path, condition: str, save_pointer_acquisition: int, + save_step_id: str, ) -> dict[str, Any]: from openadapt_flow.backends.rdp_backend import FreeRDPBackend from openadapt_flow.run_gate import build_runtime_authorization @@ -623,8 +624,7 @@ def acquire_with_fault() -> bytes: commit_timeout = _install_commit_then_timeout_fault( backend, condition=condition, - save_pointer_acquisition=save_pointer_acquisition, - acquisition_count=lambda: acquisitions, + save_region=SAVE_APPOINTMENT_REGION, ) started = time.monotonic() report = Replayer( @@ -657,11 +657,12 @@ def acquire_with_fault() -> bytes: passed = bool(injected and safe_halt) elif condition == "commit_then_timeout": save_result = next( - result - for result in report.results - if result.step_id == workflow.steps[13].id + (result for result in report.results if result.step_id == save_step_id), + None, + ) + uncertainty = ( + save_result.delivery_uncertainty if save_result is not None else None ) - uncertainty = save_result.delivery_uncertainty sqlite_proved_one_write = bool( oracle["database_ok"] and oracle["input_counts"].get("save_appointment") == 1 @@ -698,17 +699,8 @@ def acquire_with_fault() -> bytes: "fault_injected": bool(injected or commit_timeout["injected"]), "commit_timeout_injected": bool(commit_timeout["injected"]), "save_delivery_calls": int(commit_timeout["save_delivery_calls"]), - "uncertain_delivery_outcome": ( - "verified" - if condition == "commit_then_timeout" - and report.success - and report.transaction_outcome == "VERIFIED" - and oracle["database_ok"] - else ( - "reconciliation_required" - if condition == "commit_then_timeout" - else None - ) + "transaction_outcome": ( + report.transaction_outcome if condition == "commit_then_timeout" else None ), "safe_halt": safe_halt, "silent_incorrect_success": bool( @@ -772,6 +764,7 @@ def run(container: str, root: Path, out: Path, work: Path) -> dict[str, Any]: run_dir=work / f"run-{condition}-{index}", condition=condition, save_pointer_acquisition=save_pointer_acquisition, + save_step_id=step_ids["save"], ) ) runtimes = sorted(float(trial["runtime_s"]) for trial in trials) diff --git a/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py b/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py index b35439c8..0a1a1d26 100644 --- a/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py +++ b/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py @@ -41,7 +41,7 @@ def test_real_rdp_multiapp_visual_subset(tmp_path: Path) -> None: trial["oracle"]["input_counts"]["save_appointment"] == 1 for trial in uncertain ) assert all( - trial["uncertain_delivery_outcome"] in {"verified", "reconciliation_required"} + trial["transaction_outcome"] in {"VERIFIED", "RECONCILIATION_REQUIRED"} for trial in uncertain ) assert json.loads(output.read_text()) == result diff --git a/tests/test_rdp_multiapp_campaign_contract.py b/tests/test_rdp_multiapp_campaign_contract.py index df6b7d51..1e4d160a 100644 --- a/tests/test_rdp_multiapp_campaign_contract.py +++ b/tests/test_rdp_multiapp_campaign_contract.py @@ -59,8 +59,7 @@ def click_guarded(self, *args, **kwargs): state = _install_commit_then_timeout_fault( backend, condition="commit_then_timeout", - save_pointer_acquisition=7, - acquisition_count=lambda: 7, + save_region=(0, 0, 100, 100), ) try: @@ -74,3 +73,7 @@ def click_guarded(self, *args, **kwargs): assert backend.calls == 1 assert state == {"injected": True, "save_delivery_calls": 1} + + backend.click_guarded(10, 20, expected_frame_sha256="0" * 64) + assert backend.calls == 2 + assert state == {"injected": True, "save_delivery_calls": 2} From b0e1a10aeea43c9e17c61eaa015a9180ef35a491 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 13:09:18 +0200 Subject: [PATCH 10/19] test: add RDP visual fault campaign cells --- benchmark/rdp_multiapp/fixture/suite_app.py | 122 ++++++++++++-- benchmark/rdp_multiapp/run_qualification.py | 151 ++++++++++++++++-- openadapt_flow/runtime/replayer.py | 11 ++ .../test_docker_rdp_multiapp_vision_e2e.py | 38 ++++- tests/test_rdp_multiapp_campaign_contract.py | 12 ++ tests/test_replayer.py | 33 ++++ 6 files changed, 347 insertions(+), 20 deletions(-) diff --git a/benchmark/rdp_multiapp/fixture/suite_app.py b/benchmark/rdp_multiapp/fixture/suite_app.py index 6c1742dd..d02ab7cf 100644 --- a/benchmark/rdp_multiapp/fixture/suite_app.py +++ b/benchmark/rdp_multiapp/fixture/suite_app.py @@ -15,6 +15,7 @@ import sqlite3 import tkinter as tk from email.message import EmailMessage +from functools import partial from pathlib import Path ROOT = Path(os.environ.get("RDP_MULTIAPP_ORACLE_ROOT", "/opt/rdp_multiapp/oracle")) @@ -23,6 +24,7 @@ MAILDIR = ROOT / "outbox" CONTROL_PATH = ROOT / "control.json" ACK_PATH = ROOT / "reset_ack.txt" +FAULT_ACK_PATH = ROOT / "fault_ack.json" INPUT_LEDGER_PATH = ROOT / "input-ledger.jsonl" BG = "#eef2f6" @@ -72,6 +74,36 @@ def _reset_token() -> str | None: return value if isinstance(value, str) and value else None +def _fault_control() -> tuple[str | None, str | None]: + try: + control = json.loads(CONTROL_PATH.read_text()) + except (OSError, ValueError, TypeError): + return None, None + fault = control.get("fault") + token = control.get("fault_token") + return ( + fault if isinstance(fault, str) and fault else None, + token if isinstance(token, str) and token else None, + ) + + +def _fault_ack_payload() -> dict[str, object]: + """Describe the exact synthetic visual fault armed by the control token.""" + + scenario = _scenario() + fault, token = _fault_control() + partial = fault == "partial_render" + return { + "reset_token": _reset_token(), + "scenario": scenario, + "fault": fault, + "fault_token": token, + "scheduler_ready_visible": not partial, + "active_identity_visible": not partial, + "save_control_count": 2 if scenario == "duplicate_save_control" else 1, + } + + def _rows() -> list[dict[str, str]]: rows = [ { @@ -143,6 +175,8 @@ def __init__(self) -> None: self.selected_request: str | None = None self.active_record: tuple[str, str] | None = None self.last_reset_token = _reset_token() + self.last_fault_token: str | None = None + self.partial_render_latched = False self._build_inbox() self._build_worklist() self._build_scheduler() @@ -187,7 +221,7 @@ def _build_launcher(self) -> None: tk.Button( launcher, text=title, - command=lambda value=title: self.show(value), + command=partial(self.show, title), bg="#2d3d5e", fg="white", activebackground=BLUE, @@ -369,7 +403,7 @@ def _render_worklist(self) -> None: button = tk.Button( self.work_inner, text=text, - command=lambda value=row["request_id"]: self._select_work(value), + command=partial(self._select_work, row["request_id"]), anchor="w", bg=PANEL, fg=FG, @@ -427,7 +461,7 @@ def _build_scheduler(self) -> None: button = tk.Button( window, text=f"{name} {record_id}", - command=lambda n=name, r=record_id: self._select_record(n, r), + command=partial(self._select_record, name, record_id), anchor="w", bg=PANEL, fg=FG, @@ -438,9 +472,19 @@ def _build_scheduler(self) -> None: self.active_label = _label( window, "Active record: (none)", 520, 94, font=("DejaVu Sans", 16, "bold") ) - self.slot = self._entry(window, "Appointment date and time", 520, 165) - self.kind = self._entry(window, "Appointment type", 520, 285) - self.request = self._entry(window, "Request ID", 520, 405) + self.identity_loading = tk.Label( + window, + text="Loading record identity…", + bg="#d8dee8", + fg="#667085", + font=("DejaVu Sans", 16, "bold"), + anchor="w", + ) + self.slot, self.slot_label = self._entry( + window, "Appointment date and time", 520, 165 + ) + self.kind, self.kind_label = self._entry(window, "Appointment type", 520, 285) + self.request, self.request_label = self._entry(window, "Request ID", 520, 405) self.save_button = tk.Button( window, text="Save appointment", @@ -450,15 +494,25 @@ def _build_scheduler(self) -> None: font=("DejaVu Sans", 16, "bold"), ) self.save_button.place(x=520, y=540, width=260, height=56) + self.duplicate_save_button = tk.Button( + window, + text="Save appointment", + command=self._save_appointment, + bg=BLUE, + fg="white", + font=("DejaVu Sans", 16, "bold"), + ) self.scheduler_status = _label( window, "", 520, 625, font=("DejaVu Sans", 16, "bold") ) - def _entry(self, parent: tk.Misc, title: str, x: int, y: int) -> tk.Entry: - _label(parent, title, x, y, font=("DejaVu Sans", 14, "bold")) + def _entry( + self, parent: tk.Misc, title: str, x: int, y: int + ) -> tuple[tk.Entry, tk.Label]: + label = _label(parent, title, x, y, font=("DejaVu Sans", 14, "bold")) entry = tk.Entry(parent, font=("DejaVu Sans", 17), bg=PANEL, fg=FG) entry.place(x=x, y=y + 36, width=590, height=42) - return entry + return entry, label def _select_record(self, name: str, record_id: str) -> None: self.active_record = (name, record_id) @@ -504,10 +558,40 @@ def _poll_control(self) -> None: if token is not None and token != self.last_reset_token: self.last_reset_token = token self.reset() + fault, fault_token = _fault_control() + if ( + fault == "partial_render" + and fault_token is not None + and fault_token != self.last_fault_token + ): + self.last_fault_token = fault_token + self.partial_render_latched = True + self._apply_partial_render() + self._write_fault_ack() self.root.after(100, self._poll_control) + def _apply_partial_render(self) -> None: + """Latch the incomplete scheduler frame without changing saved state.""" + + self.active_label.place_forget() + self.identity_loading.place(x=520, y=94, width=740, height=44) + + def _write_fault_ack(self) -> None: + payload = _fault_ack_payload() + payload["scheduler_ready_visible"] = not self.partial_render_latched + payload["active_identity_visible"] = not self.partial_render_latched + payload["identity_surface"] = ( + "loading_skeleton" if self.partial_render_latched else "active_record" + ) + FAULT_ACK_PATH.write_text( + json.dumps(payload, sort_keys=True) + "\n", + encoding="utf-8", + ) + def reset(self) -> None: _reset_persisted_state() + self.partial_render_latched = False + self.last_fault_token = None self.selected_request = None self.active_record = None self.inbox_detail.config( @@ -519,10 +603,30 @@ def reset(self) -> None: self.work_status.config(text="No request selected", fg=FG) self.mark_button.config(state="disabled") self.active_label.config(text="Active record: (none)") + self.identity_loading.place_forget() + self.active_label.place(x=520, y=94) + self.slot_label.place(x=520, y=165) + self.slot.place(x=520, y=201, width=590, height=42) + self.kind_label.place(x=520, y=285) + self.kind.place(x=520, y=321, width=590, height=42) + self.request.place(x=520, y=441, width=590, height=42) + self.request_label.place(x=520, y=405) + if _scenario() == "duplicate_save_control": + # Neither same-label candidate remains at the recorded target + # origin. The pixel resolver must therefore refuse ambiguity + # instead of preferring the original coordinate. + self.save_button.place(x=260, y=540, width=260, height=56) + else: + self.save_button.place(x=520, y=540, width=260, height=56) + if _scenario() == "duplicate_save_control": + self.duplicate_save_button.place(x=840, y=540, width=260, height=56) + else: + self.duplicate_save_button.place_forget() for entry in (self.slot, self.kind, self.request): entry.delete(0, "end") self.scheduler_status.config(text="") ACK_PATH.write_text(self.last_reset_token or "startup", encoding="utf-8") + self._write_fault_ack() self.show("Inbox") def run(self) -> None: diff --git a/benchmark/rdp_multiapp/run_qualification.py b/benchmark/rdp_multiapp/run_qualification.py index 9e930e7d..2c216158 100644 --- a/benchmark/rdp_multiapp/run_qualification.py +++ b/benchmark/rdp_multiapp/run_qualification.py @@ -125,6 +125,14 @@ def _read_ack(root: Path) -> Optional[str]: return None +def _read_fault_ack(root: Path) -> Optional[dict[str, Any]]: + try: + value = json.loads((root / "fault_ack.json").read_text(encoding="utf-8")) + except (OSError, ValueError, TypeError): + return None + return value if isinstance(value, dict) else None + + def _read_database(root: Path) -> Optional[list[dict[str, str]]]: path = root / "appointments.sqlite3" if not path.exists(): @@ -191,7 +199,7 @@ def _read_input_ledger(root: Path) -> Optional[list[dict[str, Any]]]: return None -def _reset(_container: str, root: Path, scenario: str = "healthy") -> None: +def _reset(_container: str, root: Path, scenario: str = "healthy") -> dict[str, Any]: token = secrets.token_hex(16) root.mkdir(parents=True, exist_ok=True) (root / "control.json").write_text( @@ -207,6 +215,7 @@ def _reset(_container: str, root: Path, scenario: str = "healthy") -> None: worklist = _read_worklist(root) mail = _read_mail(root) input_ledger = _read_input_ledger(root) + fault_ack = _read_fault_ack(root) diagnostics = { "ack_expected": token, "ack_after": after, @@ -215,6 +224,7 @@ def _reset(_container: str, root: Path, scenario: str = "healthy") -> None: "worklist_rows": None if worklist is None else len(worklist), "mail": mail, "input_ledger": input_ledger, + "fault_ack": fault_ack, "root_entries": ( sorted(path.name for path in root.iterdir()) if root.is_dir() else None ), @@ -225,8 +235,11 @@ def _reset(_container: str, root: Path, scenario: str = "healthy") -> None: and worklist and mail == [] and input_ledger == [] + and fault_ack is not None + and fault_ack.get("reset_token") == token + and fault_ack.get("scenario") == scenario ): - return + return fault_ack time.sleep(0.1) raise RuntimeError( "fixture reset did not produce clean persisted state: " @@ -234,6 +247,45 @@ def _reset(_container: str, root: Path, scenario: str = "healthy") -> None: ) +def _arm_partial_render(root: Path) -> dict[str, Any]: + """Arm the latched incomplete frame without changing the reset token.""" + + token = secrets.token_hex(16) + try: + control = json.loads((root / "control.json").read_text(encoding="utf-8")) + except (OSError, ValueError, TypeError) as exc: + raise RuntimeError( + "fixture control is unavailable before partial render" + ) from exc + if not isinstance(control, dict) or not isinstance(control.get("reset_token"), str): + raise RuntimeError("fixture control has no reset token before partial render") + control.update({"fault": "partial_render", "fault_token": token}) + (root / "control.json").write_text( + json.dumps(control, sort_keys=True) + "\n", encoding="utf-8" + ) + deadline = time.monotonic() + 10 + latest: Optional[dict[str, Any]] = None + while time.monotonic() < deadline: + latest = _read_fault_ack(root) + if ( + latest is not None + and latest.get("reset_token") == control["reset_token"] + and latest.get("scenario") == "healthy" + and latest.get("fault") == "partial_render" + and latest.get("fault_token") == token + and latest.get("scheduler_ready_visible") is False + and latest.get("active_identity_visible") is False + and latest.get("save_control_count") == 1 + and latest.get("identity_surface") == "loading_skeleton" + ): + return latest + time.sleep(0.05) + raise RuntimeError( + "fixture did not acknowledge latched partial render: " + + json.dumps(latest, sort_keys=True, default=str) + ) + + def _record(backend: Any, recording_dir: Path) -> None: from openadapt_flow.recorder import Recorder @@ -511,6 +563,12 @@ def _oracle_result(root: Path, params: dict[str, str]) -> dict[str, Any]: and target_rows[0].get("status") == "Scheduled" and adjacent_unchanged ) + worklist_unchanged = bool( + worklist is not None + and len(worklist) == 19 + and len(target_rows) == 1 + and all(row.get("status") == "New" for row in worklist) + ) mail_ok = bool( mail is not None and len(mail) == 1 @@ -547,6 +605,7 @@ def _oracle_result(root: Path, params: dict[str, str]) -> dict[str, Any]: "worklist_target": target_rows, "mail": mail, "adjacent_rows_unchanged": adjacent_unchanged, + "worklist_unchanged": worklist_unchanged, "input_ledger": input_ledger, "input_counts": input_counts, "input_cardinality_ok": input_cardinality_ok, @@ -588,9 +647,13 @@ def _run_once( from openadapt_flow.run_gate import build_runtime_authorization from openadapt_flow.runtime.replayer import Replayer - _reset( - container, root, "row_reordered" if condition == "row_reordered" else "healthy" + fixture_scenarios = {"row_reordered", "duplicate_save_control"} + reset_ack = _reset( + container, + root, + condition if condition in fixture_scenarios else "healthy", ) + fault_ack = reset_ack params = dict(REPLAY_PARAMS) authorization = build_runtime_authorization( workflow, @@ -605,7 +668,7 @@ def _run_once( injected = False def acquire_with_fault() -> bytes: - nonlocal acquisitions, injected + nonlocal acquisitions, fault_ack, injected acquisitions += 1 if acquisitions == save_pointer_acquisition: if condition == "wrong_record_before_write": @@ -618,6 +681,12 @@ def acquire_with_fault() -> bytes: transport.pointer(*INBOX_TAB, "left", False) time.sleep(0.45) injected = True + elif condition == "partial_render": + # This runs after the form fields are populated and immediately + # before the qualified Save fresh-frame acquisition. The fixture + # polling loop only removes visual regions; it never resets data. + fault_ack = _arm_partial_render(root) + injected = True return original_acquire() backend.acquire_actuation_frame = acquire_with_fault # type: ignore[method-assign] @@ -646,15 +715,62 @@ def acquire_with_fault() -> bytes: expected_halt = condition in { "wrong_record_before_write", "focus_theft_before_write", + "duplicate_save_control", + "partial_render", } + errors = [result.error for result in report.results if result.error] + refusal_evidence = [ + result.safety_refusal_evidence + for result in report.results + if result.safety_refusal_evidence is not None + ] + typed_target_refusal = any( + evidence.code == "target_ambiguous" + and evidence.stage in {"target_resolution", "actuation_revalidation"} + for evidence in refusal_evidence + ) + relevant_partial_refusal = any( + result.safety_halt + and result.failure_category in {"governed_refusal", "safety_halt"} + and result.safety_refusal_evidence is not None + and result.safety_refusal_evidence.stage == "identity_verification" + and result.safety_refusal_evidence.code + in {"identity_conflict", "identity_unverifiable"} + and result.delivery_attempted is False + for result in report.results + ) + exact_fault_evidence = { + "duplicate_save_control": bool( + fault_ack.get("scenario") == condition + and fault_ack.get("save_control_count") == 2 + and typed_target_refusal + ), + "partial_render": bool( + fault_ack.get("scenario") == "healthy" + and fault_ack.get("fault") == "partial_render" + and isinstance(fault_ack.get("fault_token"), str) + and fault_ack.get("scheduler_ready_visible") is False + and fault_ack.get("active_identity_visible") is False + and fault_ack.get("save_control_count") == 1 + and fault_ack.get("identity_surface") == "loading_skeleton" + and relevant_partial_refusal + ), + }.get(condition, True) safe_halt = bool( expected_halt and not report.success and not oracle["database"] + and oracle["worklist_unchanged"] + and oracle["mail"] == [] and oracle["no_consequential_input"] ) if expected_halt: - passed = bool(injected and safe_halt) + injected_fault = ( + injected + if condition in {"wrong_record_before_write", "focus_theft_before_write"} + else exact_fault_evidence + ) + passed = bool(injected_fault and safe_halt) elif condition == "commit_then_timeout": save_result = next( (result for result in report.results if result.step_id == save_step_id), @@ -696,7 +812,22 @@ def acquire_with_fault() -> bytes: "runtime_s": round(time.monotonic() - started, 3), "runtime_success": bool(report.success), "model_calls": int(report.model_calls), - "fault_injected": bool(injected or commit_timeout["injected"]), + "fault_injected": bool( + injected + or commit_timeout["injected"] + or ( + condition in {"duplicate_save_control", "partial_render"} + and exact_fault_evidence + ) + ), + "fault_ack": fault_ack, + "reset_ack": reset_ack, + "exact_fault_evidence": exact_fault_evidence, + "typed_target_refusal": typed_target_refusal, + "relevant_partial_refusal": relevant_partial_refusal, + "safety_refusal_evidence": [ + evidence.model_dump(mode="json") for evidence in refusal_evidence + ], "commit_timeout_injected": bool(commit_timeout["injected"]), "save_delivery_calls": int(commit_timeout["save_delivery_calls"]), "transaction_outcome": ( @@ -711,7 +842,7 @@ def acquire_with_fault() -> bytes: ), "oracle": oracle, "rung_counts": dict(report.rung_counts), - "errors": [result.error for result in report.results if result.error], + "errors": errors, } @@ -748,6 +879,8 @@ def run(container: str, root: Path, out: Path, work: Path) -> dict[str, Any]: "row_reordered", "wrong_record_before_write", "focus_theft_before_write", + "duplicate_save_control", + "partial_render", "commit_then_timeout", ) trials: list[dict[str, Any]] = [] @@ -781,8 +914,6 @@ def nearest_rank(percentile: float) -> float: "implemented_conditions": list(conditions), "full_campaign_complete": False, "full_campaign_pending_conditions": [ - "duplicate_save_control", - "partial_render", "moderate_display_drift", "severe_display_drift", ], diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py index 2ad5cd4d..66a85657 100644 --- a/openadapt_flow/runtime/replayer.py +++ b/openadapt_flow/runtime/replayer.py @@ -7907,6 +7907,17 @@ def _revalidate_consequential_actuation( result, evidence_run_dir=run_dir, ) + if error is not None and result.identity is not None: + code: Literal["identity_conflict", "identity_unverifiable"] = ( + "identity_conflict" + if result.identity.status == "mismatch" + else "identity_unverifiable" + ) + result.safety_refusal_evidence = SafetyRefusalEvidence( + stage="identity_verification", + code=code, + detector_input_sha256=sha256_bytes(fresh_png), + ) except Exception: if guarded_coordinate: self._cancel_guarded_coordinate() diff --git a/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py b/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py index 0a1a1d26..faddd464 100644 --- a/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py +++ b/tests/e2e/test_docker_rdp_multiapp_vision_e2e.py @@ -24,11 +24,47 @@ def test_real_rdp_multiapp_visual_subset(tmp_path: Path) -> None: result = run(container, Path(oracle), output, tmp_path / "work") assert result["accepted_subset"] is True - assert result["run_count"] == 15 + assert result["run_count"] == 21 assert result["silent_incorrect_successes"] == 0 assert result["over_halts"] == 0 assert result["model_calls"] == 0 assert result["full_campaign_complete"] is False + fault_trials = { + condition: [ + trial for trial in result["trials"] if trial["condition"] == condition + ] + for condition in ("duplicate_save_control", "partial_render") + } + assert all(len(trials) == 3 for trials in fault_trials.values()) + assert all( + trial["safe_halt"] + and trial["exact_fault_evidence"] + and trial["oracle"]["database"] == [] + and trial["oracle"]["worklist_unchanged"] + and trial["oracle"]["mail"] == [] + and trial["oracle"]["no_consequential_input"] + and trial["model_calls"] == 0 + for trials in fault_trials.values() + for trial in trials + ) + assert all( + trial["typed_target_refusal"] + for trial in fault_trials["duplicate_save_control"] + ) + assert all( + trial["relevant_partial_refusal"] + and trial["fault_ack"]["fault"] == "partial_render" + and trial["fault_ack"]["scenario"] == "healthy" + and trial["fault_ack"]["fault_token"] + and trial["fault_ack"]["save_control_count"] == 1 + and trial["fault_ack"]["identity_surface"] == "loading_skeleton" + and any( + evidence["stage"] == "identity_verification" + and evidence["code"] in {"identity_unverifiable", "identity_conflict"} + for evidence in trial["safety_refusal_evidence"] + ) + for trial in fault_trials["partial_render"] + ) uncertain = [ trial for trial in result["trials"] diff --git a/tests/test_rdp_multiapp_campaign_contract.py b/tests/test_rdp_multiapp_campaign_contract.py index 1e4d160a..91edce45 100644 --- a/tests/test_rdp_multiapp_campaign_contract.py +++ b/tests/test_rdp_multiapp_campaign_contract.py @@ -30,6 +30,18 @@ def test_visual_campaign_has_repeated_trials_and_business_oracles() -> None: assert by_id["row_reordered"]["expect"] == "verified" assert by_id["wrong_record_before_write"]["expect"] == "safe_halt" assert by_id["focus_theft_before_write"]["expect"] == "safe_halt" + assert by_id["duplicate_save_control"] == { + "id": "duplicate_save_control", + "expect": "safe_halt", + "fault": "A competing Save appointment control appears in the same visual state.", + "oracle": ["sqlite"], + } + assert by_id["partial_render"] == { + "id": "partial_render", + "expect": "safe_halt", + "fault": "The target window is incomplete when the next action is due.", + "oracle": ["sqlite", "csv", "maildir"], + } assert by_id["commit_then_timeout"]["oracle"] == ["sqlite"] diff --git a/tests/test_replayer.py b/tests/test_replayer.py index 8cb453ff..176588ef 100644 --- a/tests/test_replayer.py +++ b/tests/test_replayer.py @@ -2192,6 +2192,39 @@ def test_consequential_remote_reruns_identity_on_fresh_frame(bundle, run_dir): assert "Identity check failed" in report.results[0].error +def test_fresh_pre_actuation_identity_loss_has_typed_refusal_and_zero_input( + bundle, run_dir +): + frame = make_png() + backend = RemoteLeaseBackend(initial_frame=frame, fresh_frame=frame) + vision = FakeVision() + vision.template_results = [ + Match(point=(110, 105), region=(100, 100, 50, 20), confidence=0.99), + Match(point=(110, 105), region=(100, 100, 50, 20), confidence=0.99), + ] + vision.ocr_results = [ + [OcrLine("Jane Sample Knee pain referral High")], + [OcrLine("Taylor Duplicate Knee pain referral High")], + ] + step = context_click_step( + "Jane Sample Knee pain referral High", risk="irreversible" + ) + + report = Replayer(backend, vision=vision).run( + Workflow(name="wf", steps=[step]), + bundle_dir=bundle, + run_dir=run_dir, + ) + + result = report.results[0] + assert report.success is False + assert backend.actions == [] + assert result.delivery_attempted is False + assert result.safety_refusal_evidence is not None + assert result.safety_refusal_evidence.stage == "identity_verification" + assert result.safety_refusal_evidence.code == "identity_conflict" + + def test_identity_verified_clicks_normally(bundle, run_dir): vision = resolving_vision() vision.ocr_lines = [OcrLine("Jane Sample Knee pain referral High")] From 6a8156f7dea4ddb7dad114af1d29779511c78213 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 13:26:12 +0200 Subject: [PATCH 11/19] test: export bounded RDP failure diagnostics --- benchmark/rdp_multiapp/run_qualification.py | 69 +++++++++++++++++---- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/benchmark/rdp_multiapp/run_qualification.py b/benchmark/rdp_multiapp/run_qualification.py index 2c216158..56f22921 100644 --- a/benchmark/rdp_multiapp/run_qualification.py +++ b/benchmark/rdp_multiapp/run_qualification.py @@ -806,6 +806,39 @@ def acquire_with_fault() -> bytes: ) else: passed = bool(report.success and oracle["all_effects_ok"]) + step_diagnostics = [ + { + "step_id": result.step_id, + "ok": result.ok, + "failure_category": result.failure_category, + "safety_halt": result.safety_halt, + "delivery_attempted": result.delivery_attempted, + "actuation": result.actuation, + "resolution_rung": ( + result.resolution.rung if result.resolution is not None else None + ), + "error": result.error, + "delivery_uncertainty": ( + result.delivery_uncertainty.model_dump( + mode="json", + exclude={"observed_at"}, + ) + if result.delivery_uncertainty is not None + else None + ), + "safety_refusal_evidence": ( + result.safety_refusal_evidence.model_dump(mode="json") + if result.safety_refusal_evidence is not None + else None + ), + } + for result in report.results + if ( + result.error is not None + or result.delivery_uncertainty is not None + or result.safety_refusal_evidence is not None + ) + ] return { "condition": condition, "passed": passed, @@ -843,6 +876,10 @@ def acquire_with_fault() -> bytes: "oracle": oracle, "rung_counts": dict(report.rung_counts), "errors": errors, + "failed_step_ids": [ + result.step_id for result in report.results if not result.ok + ], + "step_diagnostics": step_diagnostics, } @@ -884,22 +921,27 @@ def run(container: str, root: Path, out: Path, work: Path) -> dict[str, Any]: "commit_then_timeout", ) trials: list[dict[str, Any]] = [] + stopped_early = False for condition in conditions: for index in range(1, TRIALS + 1): - trials.append( - _run_once( - container=container, - root=root, - workflow=workflow, - verifier=verifier, - gate=gate, - bundle_dir=bundle_dir, - run_dir=work / f"run-{condition}-{index}", - condition=condition, - save_pointer_acquisition=save_pointer_acquisition, - save_step_id=step_ids["save"], - ) + trial = _run_once( + container=container, + root=root, + workflow=workflow, + verifier=verifier, + gate=gate, + bundle_dir=bundle_dir, + run_dir=work / f"run-{condition}-{index}", + condition=condition, + save_pointer_acquisition=save_pointer_acquisition, + save_step_id=step_ids["save"], ) + trials.append(trial) + if not trial["passed"]: + stopped_early = True + break + if stopped_early: + break runtimes = sorted(float(trial["runtime_s"]) for trial in trials) def nearest_rank(percentile: float) -> float: @@ -918,6 +960,7 @@ def nearest_rank(percentile: float) -> float: "severe_display_drift", ], "run_count": len(trials), + "stopped_early": stopped_early, "accepted_subset": all(trial["passed"] for trial in trials), "verified_outcomes": sum( bool(trial["runtime_success"] and trial["oracle"]["all_effects_ok"]) From 4fc20c245b5bda232ebf46b430750b5bc6141b2f Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 13:41:44 +0200 Subject: [PATCH 12/19] fix: refresh remote scroll frame lease --- openadapt_flow/runtime/replayer.py | 39 ++++++++++++++++++++++++++++++ tests/test_replayer.py | 35 +++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py index 66a85657..8e9c30be 100644 --- a/openadapt_flow/runtime/replayer.py +++ b/openadapt_flow/runtime/replayer.py @@ -11716,6 +11716,9 @@ def _act_scroll( if refusal is not None: return refusal self._require_qualification_environment_current() + remote_preflight_error = self._prepare_remote_scroll_input(step, result) + if remote_preflight_error is not None: + return remote_preflight_error self._deliver_backend_call( result, lambda: self.backend.scroll(dx, dy), @@ -11754,6 +11757,7 @@ def readiness_holds(frame_png: bytes) -> bool: # every wheel edge to a new exact remote frame and repeat the # context/identity checks after those callbacks. This also avoids # reusing the one-shot lease consumed by the prior wheel edge. + remote_scroll_revalidated = False if self._step_needs_consequential_revalidation(step, workflow): ( _scroll_resolution, @@ -11775,12 +11779,19 @@ def readiness_holds(frame_png: bytes) -> bool: return scroll_error if readiness_holds(fresh_scroll_frame): return None + remote_scroll_revalidated = isinstance( + self.backend, RemoteActuationBackend + ) refusal = self._delivery_authorization_refusal( workflow, params, step, result ) if refusal is not None: return refusal self._require_qualification_environment_current() + if not remote_scroll_revalidated: + remote_preflight_error = self._prepare_remote_scroll_input(step, result) + if remote_preflight_error is not None: + return remote_preflight_error self._deliver_backend_call( result, lambda: self.backend.scroll(dx, dy), @@ -11818,6 +11829,34 @@ def readiness_holds(frame_png: bytes) -> bool: f"{target_desc} resolving — target never came into view; run aborted" ) + def _prepare_remote_scroll_input( + self, + step: Step, + result: StepResult, + ) -> Optional[str]: + """Bind one remote wheel edge to a fresh, one-use frame lease. + + Target-readiness probes can take longer than a remote backend's frame + lease. A wheel gesture has no coordinate to re-resolve, but it is + still real input into an opaque session. Acquire the remote backend's + exact-content lease immediately before delivery instead of extending + the allowed frame age or reusing the observation from the probe. + """ + + if not isinstance(self.backend, RemoteActuationBackend): + return None + try: + self.backend.acquire_actuation_frame() + except Exception as exc: # noqa: BLE001 - backend boundary must halt + if self.governed_authorization is not None: + result.safety_halt = True + detail = _scrub_phi(str(exc)) or type(exc).__name__ + return ( + "Remote scroll preflight HALTED before input for step " + f"'{step.id}' ({step.intent}): {detail}" + ) + return None + @staticmethod def _next_anchored_step(workflow: Workflow, step_index: int) -> Optional[Step]: """The first step after ``step_index`` that carries an anchor.""" diff --git a/tests/test_replayer.py b/tests/test_replayer.py index 176588ef..39be177d 100644 --- a/tests/test_replayer.py +++ b/tests/test_replayer.py @@ -1798,6 +1798,41 @@ def test_scroll_step_scrolls_backend(bundle, run_dir): assert report.heal_count == 0 +def test_remote_scroll_acquires_fresh_one_use_frame_before_input(bundle, run_dir): + frame = make_png() + backend = RemoteLeaseBackend(initial_frame=frame, fresh_frame=frame) + workflow = Workflow(name="wf", steps=[scroll_step()]) + + report = Replayer(backend, vision=FakeVision()).run( + workflow, bundle_dir=bundle, run_dir=run_dir + ) + + assert report.success is True + assert backend.acquire_count == 1 + assert backend.actions == [("scroll", 0, 400)] + + +def test_remote_scroll_preflight_refusal_sends_no_input(bundle, run_dir): + class RefusingRemoteScrollBackend(RemoteLeaseBackend): + def acquire_actuation_frame(self): + self.acquire_count += 1 + raise RuntimeError("remote session changed") + + frame = make_png() + backend = RefusingRemoteScrollBackend(initial_frame=frame, fresh_frame=frame) + workflow = Workflow(name="wf", steps=[scroll_step()]) + + report = Replayer(backend, vision=FakeVision()).run( + workflow, bundle_dir=bundle, run_dir=run_dir + ) + + assert report.success is False + assert backend.acquire_count == 1 + assert backend.actions == [] + assert report.results[0].delivery_attempted is False + assert report.results[0].error is not None + + def scroll_step(step_id="sc1", dx=0, dy=400) -> Step: return Step( id=step_id, From db45aabd15109a7968e0c4fa594a00762ab85f8a Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 13:46:57 +0200 Subject: [PATCH 13/19] test: enforce remote scroll preflight order --- openadapt_flow/runtime/replayer.py | 22 +++++------ tests/test_replayer.py | 59 +++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py index 8e9c30be..4ad4c0d3 100644 --- a/openadapt_flow/runtime/replayer.py +++ b/openadapt_flow/runtime/replayer.py @@ -11710,15 +11710,15 @@ def _act_scroll( intent=next_step.intent, ) if stop_pred is None or (dx == 0 and dy == 0): + remote_preflight_error = self._prepare_remote_scroll_input(step, result) + if remote_preflight_error is not None: + return remote_preflight_error refusal = self._delivery_authorization_refusal( workflow, params, step, result ) if refusal is not None: return refusal self._require_qualification_environment_current() - remote_preflight_error = self._prepare_remote_scroll_input(step, result) - if remote_preflight_error is not None: - return remote_preflight_error self._deliver_backend_call( result, lambda: self.backend.scroll(dx, dy), @@ -11757,7 +11757,6 @@ def readiness_holds(frame_png: bytes) -> bool: # every wheel edge to a new exact remote frame and repeat the # context/identity checks after those callbacks. This also avoids # reusing the one-shot lease consumed by the prior wheel edge. - remote_scroll_revalidated = False if self._step_needs_consequential_revalidation(step, workflow): ( _scroll_resolution, @@ -11779,19 +11778,15 @@ def readiness_holds(frame_png: bytes) -> bool: return scroll_error if readiness_holds(fresh_scroll_frame): return None - remote_scroll_revalidated = isinstance( - self.backend, RemoteActuationBackend - ) + remote_preflight_error = self._prepare_remote_scroll_input(step, result) + if remote_preflight_error is not None: + return remote_preflight_error refusal = self._delivery_authorization_refusal( workflow, params, step, result ) if refusal is not None: return refusal self._require_qualification_environment_current() - if not remote_scroll_revalidated: - remote_preflight_error = self._prepare_remote_scroll_input(step, result) - if remote_preflight_error is not None: - return remote_preflight_error self._deliver_backend_call( result, lambda: self.backend.scroll(dx, dy), @@ -11839,8 +11834,9 @@ def _prepare_remote_scroll_input( Target-readiness probes can take longer than a remote backend's frame lease. A wheel gesture has no coordinate to re-resolve, but it is still real input into an opaque session. Acquire the remote backend's - exact-content lease immediately before delivery instead of extending - the allowed frame age or reusing the observation from the probe. + exact-content lease after readiness checks and before the final + authorization, environment, and delivery checks. Do not extend the + allowed frame age or reuse the observation from the probe. """ if not isinstance(self.backend, RemoteActuationBackend): diff --git a/tests/test_replayer.py b/tests/test_replayer.py index 39be177d..246af529 100644 --- a/tests/test_replayer.py +++ b/tests/test_replayer.py @@ -1833,6 +1833,62 @@ def acquire_actuation_frame(self): assert report.results[0].error is not None +def test_closed_loop_remote_scroll_orders_fresh_lease_before_final_gates( + bundle, run_dir, monkeypatch +): + events = [] + + class OrderedRemoteScrollBackend(RemoteLeaseBackend): + def acquire_actuation_frame(self): + events.append("acquire") + return super().acquire_actuation_frame() + + def scroll(self, dx, dy): + events.append("scroll") + return super().scroll(dx, dy) + + backend = OrderedRemoteScrollBackend( + initial_frame=make_png(), fresh_frame=make_png() + ) + replayer = Replayer(backend, vision=FakeVision()) + monkeypatch.setattr( + replayer, + "_implicit_scroll_target_ready", + lambda *args, **kwargs: events.append("readiness") or False, + ) + monkeypatch.setattr( + replayer, + "_delivery_authorization_refusal", + lambda *args, **kwargs: events.append("authorization") or None, + ) + monkeypatch.setattr( + replayer, + "_require_qualification_environment_current", + lambda: events.append("environment"), + ) + + report = replayer.run( + Workflow(name="wf", steps=[scroll_step(), click_step()]), + bundle_dir=bundle, + run_dir=run_dir, + ) + + assert report.success is False + assert events == [ + "readiness", + "acquire", + "authorization", + "environment", + "scroll", + "readiness", + "acquire", + "authorization", + "environment", + "scroll", + "readiness", + ] + + def scroll_step(step_id="sc1", dx=0, dy=400) -> Step: return Step( id=step_id, @@ -2040,7 +2096,8 @@ def test_consequential_remote_scroll_reacquires_each_wheel_edge(bundle, run_dir) assert report.success is False assert backend.actions == [("scroll", 0, -400), ("scroll", 0, -400)] - assert backend.acquire_count == 3 # outer preflight plus one per wheel edge + # One outer step preflight, then identity and final wheel leases per edge. + assert backend.acquire_count == 5 def test_consecutive_scroll_steps_share_the_loop(bundle, run_dir): From 8bd79b216eadb41693b5669b87b8e4a7f0e1482a Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 13:57:48 +0200 Subject: [PATCH 14/19] fix: keep RDP fixture text delivery atomic --- .../run_rdp_ladder_qualification.py | 34 +++++++++++++++++++ tests/test_rdp_ladder_qualification.py | 22 ++++++++++++ 2 files changed, 56 insertions(+) diff --git a/benchmark/rdp_ladder/run_rdp_ladder_qualification.py b/benchmark/rdp_ladder/run_rdp_ladder_qualification.py index 37fc5323..c8a405e8 100644 --- a/benchmark/rdp_ladder/run_rdp_ladder_qualification.py +++ b/benchmark/rdp_ladder/run_rdp_ladder_qualification.py @@ -270,6 +270,15 @@ def _focus_client(self) -> None: ] ) + def focus_input_surface(self) -> None: + """Restore the outer FreeRDP window before keyboard delivery.""" + + self._focus_client() + # Let the window manager and FreeRDP restore the keyboard grab before + # the next XTest key event. This delay is outside the production RDP + # transport. It makes the two-Xvfb qualification fixture deterministic. + time.sleep(0.1) + def _remote_pointer(self) -> Optional[tuple[int, int]]: """Return the fixture server's cursor as a delivery acknowledgement. @@ -352,6 +361,31 @@ def key(self, keysym_or_char: str, down: bool) -> None: verb = "keydown" if down else "keyup" self._exec(["xdotool", verb, "--clearmodifiers", keysym]) + @staticmethod + def supports_bulk_text(text: str) -> bool: + """Use one X11 client for printable ASCII fixture parameters.""" + + return ( + bool(text) and text.isascii() and all(" " <= char <= "~" for char in text) + ) + + def bulk_type_text(self, text: str) -> None: + """Type one value without losing the FreeRDP grab between characters.""" + + if not self.supports_bulk_text(text): + raise ValueError("RDP fixture bulk text must be printable ASCII") + self._exec( + [ + "xdotool", + "type", + "--clearmodifiers", + "--delay", + "35", + "--", + text, + ] + ) + def wheel(self, dx: int, dy: int) -> None: if not dy: return diff --git a/tests/test_rdp_ladder_qualification.py b/tests/test_rdp_ladder_qualification.py index a969d52c..16abd4fa 100644 --- a/tests/test_rdp_ladder_qualification.py +++ b/tests/test_rdp_ladder_qualification.py @@ -315,6 +315,28 @@ def test_rdp_fixture_transport_uses_unambiguous_punctuation_keysyms( ] +def test_rdp_fixture_transport_types_parameter_in_one_input_operation() -> None: + transport = qualification.DockerX11RdpTransport("synthetic-fixture") + commands: list[list[str]] = [] + transport._exec = lambda args, **_kwargs: commands.append(args) # type: ignore[method-assign] + + value = "2026-08-14 10:45" + assert transport.supports_bulk_text(value) + transport.bulk_type_text(value) + + assert commands == [ + [ + "xdotool", + "type", + "--clearmodifiers", + "--delay", + "35", + "--", + value, + ] + ] + + def test_recorded_identity_regions_cover_every_pointer_action(tmp_path: Path) -> None: recording = tmp_path / "recording" recording.mkdir() From 37ab60fd83251faaaa252f128e8fda3949c82030 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 14:14:33 +0200 Subject: [PATCH 15/19] fix: verify remote text across the live field --- openadapt_flow/runtime/replayer.py | 21 ++++++++++- tests/test_replayer.py | 56 ++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py index 4ad4c0d3..0f1e9a5c 100644 --- a/openadapt_flow/runtime/replayer.py +++ b/openadapt_flow/runtime/replayer.py @@ -9167,7 +9167,26 @@ def _act( resolution = refreshed result.resolution = refreshed field_point = refreshed.point - field_region = refreshed_region + # A visual match region is the retained template crop. It + # is not the editable control's bounds. Treating that crop + # as the typed-value readback region can exclude text that + # renders at the left edge of a wide field. Keep exact + # structural bounds when the backend supplies them; + # otherwise use the point-centred visual readback window. + if step.action is ActionKind.SELECT_OPTION: + # Composite selection binds its type-ahead and commit + # to the exact live resolved selection region. + field_region = refreshed_region + else: + field_region = ( + refreshed.structural_handle.region + if ( + refreshed is not None + and refreshed.rung == "structural" + and refreshed.structural_handle is not None + ) + else None + ) else: before_png = self.backend.screenshot() elif self._prev_was_click(workflow, step_index, graph_ctx): diff --git a/tests/test_replayer.py b/tests/test_replayer.py index 246af529..151b2cf9 100644 --- a/tests/test_replayer.py +++ b/tests/test_replayer.py @@ -3268,6 +3268,62 @@ def test_visual_click_crop_does_not_narrow_following_type_verification(bundle, r assert vision.pixels_changed_calls == [(0, 0, 300, 200)] +def test_remote_visual_type_revalidation_does_not_use_template_crop_as_field_bounds( + bundle, run_dir +): + """The final remote resolve retains target evidence, not field bounds. + + A compact template can correctly locate a wide remote field while the + typed value renders outside that crop. The visual verifier must use the + point-centred readback window after the post-focus fresh-frame resolve. + """ + + class RegionAwareVision(FakeVision): + def ocr(self, screen_png, *, region=None): + del screen_png + if region == (0, 0, 300, 200): + return [OcrLine("Massachusetts")] + return [] + + vision = RegionAwareVision() + vision.template_results = [ + Match(point=(110, 105), region=(100, 100, 50, 20), confidence=0.95) + for _ in range(3) + ] + vision.pixels_changed_results = [True] + frame = make_png() + backend = RemoteLeaseBackend(initial_frame=frame, fresh_frame=frame) + backend._text_value_supported = False + workflow = Workflow( + name="remote-visual-type", + surface="rdp", + execution_mode="external", + steps=[ + Step( + id="t1", + intent="type the governed value", + action=ActionKind.TYPE, + text="Massachusetts", + anchor=click_step().anchor, + risk="irreversible", + ) + ], + ) + + report = Replayer(backend, vision=vision).run( + workflow, + bundle_dir=bundle, + run_dir=run_dir, + ) + + assert report.success is True + assert report.results[0].input_verified is True + assert backend.actions == [ + ("click", 110, 105, False), + ("type", "Massachusetts"), + ] + + def test_type_verification_prefers_exact_structural_field_region() -> None: """Wide native text fields must be observed across their full UIA bounds. From 1142fdd1cc4004b75d1aec930286af8c45f286d3 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 14:28:21 +0200 Subject: [PATCH 16/19] test: retain bounded RDP failure frames --- .../workflows/docker-rdp-multiapp-vision.yml | 2 +- benchmark/rdp_multiapp/run_qualification.py | 52 ++++++++++++++++++- tests/test_rdp_multiapp_campaign_contract.py | 37 +++++++++++++ 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-rdp-multiapp-vision.yml b/.github/workflows/docker-rdp-multiapp-vision.yml index 23ec52e0..b3211423 100644 --- a/.github/workflows/docker-rdp-multiapp-vision.yml +++ b/.github/workflows/docker-rdp-multiapp-vision.yml @@ -60,7 +60,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: rdp-multiapp-vision-subset - path: ${{ runner.temp }}/rdp-multiapp-result/results.json + path: ${{ runner.temp }}/rdp-multiapp-result/ if-no-files-found: warn - name: Print bounded fixture diagnostics diff --git a/benchmark/rdp_multiapp/run_qualification.py b/benchmark/rdp_multiapp/run_qualification.py index 56f22921..39a95225 100644 --- a/benchmark/rdp_multiapp/run_qualification.py +++ b/benchmark/rdp_multiapp/run_qualification.py @@ -13,6 +13,7 @@ import hashlib import json import secrets +import shutil import sqlite3 import statistics import tempfile @@ -77,6 +78,48 @@ POLICY_PATH = Path(__file__).with_name("policy.yaml") +def _export_failed_step_frames( + *, + artifact_root: Path, + run_dir: Path, + failed_step_ids: list[str], +) -> list[dict[str, str]]: + """Export only the first failed step's exact before and after frames.""" + + if not failed_step_ids: + return [] + step_id = failed_step_ids[0] + if Path(step_id).name != step_id or step_id in {"", ".", ".."}: + raise ValueError(f"unsafe failed step id: {step_id!r}") + + artifact_root = artifact_root.resolve() + run_root = run_dir.resolve() + destination = artifact_root / "failure" / run_dir.name + destination.mkdir(parents=True, exist_ok=True) + destination.resolve().relative_to(artifact_root) + + exported: list[dict[str, str]] = [] + for phase in ("before", "after"): + source = run_dir / "steps" / f"{step_id}_{phase}.png" + if source.is_symlink(): + raise ValueError(f"refusing linked failure frame: {source}") + resolved_source = source.resolve(strict=True) + resolved_source.relative_to(run_root) + if not resolved_source.is_file(): + raise ValueError(f"failure frame is not a file: {source}") + + target = destination / source.name + shutil.copyfile(resolved_source, target) + exported.append( + { + "kind": f"failed_step_{phase}_frame", + "path": target.relative_to(artifact_root).as_posix(), + "sha256": hashlib.sha256(target.read_bytes()).hexdigest(), + } + ) + return exported + + def _install_commit_then_timeout_fault( backend: Any, *, @@ -924,6 +967,7 @@ def run(container: str, root: Path, out: Path, work: Path) -> dict[str, Any]: stopped_early = False for condition in conditions: for index in range(1, TRIALS + 1): + trial_run_dir = work / f"run-{condition}-{index}" trial = _run_once( container=container, root=root, @@ -931,11 +975,17 @@ def run(container: str, root: Path, out: Path, work: Path) -> dict[str, Any]: verifier=verifier, gate=gate, bundle_dir=bundle_dir, - run_dir=work / f"run-{condition}-{index}", + run_dir=trial_run_dir, condition=condition, save_pointer_acquisition=save_pointer_acquisition, save_step_id=step_ids["save"], ) + if not trial["passed"]: + trial["failure_artifacts"] = _export_failed_step_frames( + artifact_root=out.parent, + run_dir=trial_run_dir, + failed_step_ids=trial["failed_step_ids"], + ) trials.append(trial) if not trial["passed"]: stopped_early = True diff --git a/tests/test_rdp_multiapp_campaign_contract.py b/tests/test_rdp_multiapp_campaign_contract.py index 91edce45..3f28dc70 100644 --- a/tests/test_rdp_multiapp_campaign_contract.py +++ b/tests/test_rdp_multiapp_campaign_contract.py @@ -8,6 +8,43 @@ ROOT = Path(__file__).parents[1] +def test_failure_artifact_exports_only_exact_failed_step_frames(tmp_path: Path) -> None: + from benchmark.rdp_multiapp.run_qualification import _export_failed_step_frames + + run_dir = tmp_path / "run-healthy-1" + steps = run_dir / "steps" + steps.mkdir(parents=True) + (steps / "step_008_before.png").write_bytes(b"before") + (steps / "step_008_after.png").write_bytes(b"after") + (steps / "step_007_before.png").write_bytes(b"unrelated") + (run_dir / "report.json").write_text('{"not": "exported"}', encoding="utf-8") + artifact_root = tmp_path / "artifacts" + artifact_root.mkdir() + + exported = _export_failed_step_frames( + artifact_root=artifact_root, + run_dir=run_dir, + failed_step_ids=["step_008"], + ) + + assert [item["kind"] for item in exported] == [ + "failed_step_before_frame", + "failed_step_after_frame", + ] + assert [item["path"] for item in exported] == [ + "failure/run-healthy-1/step_008_before.png", + "failure/run-healthy-1/step_008_after.png", + ] + assert sorted( + path.relative_to(artifact_root).as_posix() + for path in artifact_root.rglob("*") + if path.is_file() + ) == [ + "failure/run-healthy-1/step_008_after.png", + "failure/run-healthy-1/step_008_before.png", + ] + + def test_visual_campaign_has_repeated_trials_and_business_oracles() -> None: campaign = json.loads( (ROOT / "benchmark/rdp_multiapp/campaign.json").read_text(encoding="utf-8") From 2ad2bdf07f6e372900ad3e8cc37e4ba98be8592a Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 14:30:57 +0200 Subject: [PATCH 17/19] build: refresh public workflow inventory --- public-artifacts.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public-artifacts.json b/public-artifacts.json index 8b6dfa21..ab58a241 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -97,7 +97,7 @@ }, { "path": ".github/workflows/docker-rdp-multiapp-vision.yml", - "sha256": "9959bd76d8d7590377c49113805990f48ee84be3364506870e641b395152d835" + "sha256": "b3ab41f3eb6735a3fbcc6a34fa5b672970a3e049a6f05c3f78337db683681add" }, { "path": ".github/workflows/docker-rdp-vision-ladder.yml", From 6293e2d1b78f12ce2c32e7845746a4bceea31601 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 14:43:36 +0200 Subject: [PATCH 18/19] fix: preserve active RDP keyboard focus --- .../run_rdp_ladder_qualification.py | 35 +++++++++++++------ tests/test_rdp_ladder_qualification.py | 19 ++++++++++ 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/benchmark/rdp_ladder/run_rdp_ladder_qualification.py b/benchmark/rdp_ladder/run_rdp_ladder_qualification.py index c8a405e8..4ae48978 100644 --- a/benchmark/rdp_ladder/run_rdp_ladder_qualification.py +++ b/benchmark/rdp_ladder/run_rdp_ladder_qualification.py @@ -251,29 +251,42 @@ def framebuffer(self): img = self._grab() return img, img.width, img.height - def _focus_client(self) -> None: - """Focus the isolated FreeRDP window before injecting XTest input. - - The fixture runs a minimal Openbox session. Focusing its only visible - FreeRDP window once is deterministic and remains entirely inside - display ``:1`` in the container. - """ - self._exec( + def _client_window_id(self) -> int: + raw = self._exec( [ "xdotool", "search", "--onlyvisible", "--name", "^FreeRDP:", - "windowfocus", - "%@", ] ) + try: + return int(raw.decode().splitlines()[0]) + except (IndexError, ValueError) as exc: + raise RuntimeError("isolated FreeRDP window is unavailable") from exc + + def _focus_client(self, window_id: Optional[int] = None) -> None: + """Focus the isolated FreeRDP window before injecting XTest input. + + The fixture runs a minimal Openbox session. Focusing its only visible + FreeRDP window once is deterministic and remains entirely inside + display ``:1`` in the container. + """ + client_window = window_id if window_id is not None else self._client_window_id() + self._exec(["xdotool", "windowfocus", str(client_window)]) def focus_input_surface(self) -> None: """Restore the outer FreeRDP window before keyboard delivery.""" - self._focus_client() + client_window = self._client_window_id() + try: + active_window = int(self._exec(["xdotool", "getactivewindow"]).decode()) + except ValueError as exc: + raise RuntimeError("active X11 input window is unavailable") from exc + if active_window == client_window: + return + self._focus_client(client_window) # Let the window manager and FreeRDP restore the keyboard grab before # the next XTest key event. This delay is outside the production RDP # transport. It makes the two-Xvfb qualification fixture deterministic. diff --git a/tests/test_rdp_ladder_qualification.py b/tests/test_rdp_ladder_qualification.py index 16abd4fa..b6186c3d 100644 --- a/tests/test_rdp_ladder_qualification.py +++ b/tests/test_rdp_ladder_qualification.py @@ -337,6 +337,25 @@ def test_rdp_fixture_transport_types_parameter_in_one_input_operation() -> None: ] +def test_rdp_fixture_does_not_refocus_an_already_active_client() -> None: + transport = qualification.DockerX11RdpTransport("synthetic-fixture") + commands: list[list[str]] = [] + + def execute(args, **_kwargs): + commands.append(args) + if args[1] == "search": + return b"123\n" + if args[1] == "getactivewindow": + return b"123\n" + raise AssertionError(f"unexpected input-changing command: {args!r}") + + transport._exec = execute # type: ignore[method-assign] + + transport.focus_input_surface() + + assert [command[1] for command in commands] == ["search", "getactivewindow"] + + def test_recorded_identity_regions_cover_every_pointer_action(tmp_path: Path) -> None: recording = tmp_path / "recording" recording.mkdir() From b26608266b2f3e0040d9d4b8eeb33f25237ba66b Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 2 Aug 2026 20:45:46 +0200 Subject: [PATCH 19/19] fix: resolve moved RDP targets safely --- benchmark/rdp_multiapp/fixture/run_fixture.sh | 4 +- benchmark/rdp_multiapp/fixture/suite_app.py | 20 +++- openadapt_flow/compiler/compile.py | 102 +++++++++++++++++- openadapt_flow/runtime/heal.py | 37 ++++++- openadapt_flow/runtime/healing/governance.py | 51 ++++++--- openadapt_flow/runtime/replayer.py | 61 ++++++++--- tests/test_compile_identifier_crop.py | 44 ++++++++ tests/test_governed_authorization.py | 34 ++++++ tests/test_governed_healing.py | 16 +++ tests/test_heal.py | 54 ++++++++++ tests/test_replayer.py | 22 ++++ 11 files changed, 409 insertions(+), 36 deletions(-) diff --git a/benchmark/rdp_multiapp/fixture/run_fixture.sh b/benchmark/rdp_multiapp/fixture/run_fixture.sh index 6b3e1747..6e0a59c0 100644 --- a/benchmark/rdp_multiapp/fixture/run_fixture.sh +++ b/benchmark/rdp_multiapp/fixture/run_fixture.sh @@ -8,8 +8,8 @@ mkdir -p "${RDP_MULTIAPP_ORACLE_ROOT}" Xvfb :0 -screen 0 1280x800x24 -ac +extension DAMAGE +extension RANDR +extension XFIXES \ >/tmp/xvfb0.log 2>&1 & sleep 2 -DISPLAY=:0 openbox >/tmp/openbox0.log 2>&1 & -sleep 1 +# Keep the remote display free of a second window manager. The fixture owns +# the X keyboard focus while the client-side Openbox session manages FreeRDP. DISPLAY=:0 python3 /opt/rdp_multiapp/suite_app.py >/tmp/suite.log 2>&1 & sleep 3 DISPLAY=:0 freerdp-shadow-cli3 /port:3389 /bind-address:0.0.0.0 -auth \ diff --git a/benchmark/rdp_multiapp/fixture/suite_app.py b/benchmark/rdp_multiapp/fixture/suite_app.py index d02ab7cf..fa792e3b 100644 --- a/benchmark/rdp_multiapp/fixture/suite_app.py +++ b/benchmark/rdp_multiapp/fixture/suite_app.py @@ -120,7 +120,9 @@ def _rows() -> list[dict[str, str]]: "name": TARGET_NAME, "status": "New", } - insert_at = 4 if _scenario() == "row_reordered" else 15 + # One scroll keeps the target inside the viewport in both layouts. The + # reordered case moves it to a different row so replay must re-resolve it. + insert_at = 11 if _scenario() == "row_reordered" else 13 rows.insert(insert_at, target) return rows @@ -170,7 +172,10 @@ class Suite: def __init__(self) -> None: _reset_persisted_state() self.root = tk.Tk() - self.root.withdraw() + # The remote display has no window manager. Keep the root mapped so + # its toplevels can own the X focus used by RDP keyboard delivery. + self.root.geometry("1x1+0+0") + self.root.overrideredirect(True) self.windows: dict[str, tk.Toplevel] = {} self.selected_request: str | None = None self.active_record: tuple[str, str] | None = None @@ -197,6 +202,8 @@ def _window(self, title: str) -> tk.Toplevel: def show(self, title: str) -> None: window = self.windows[title] + if title == "Worklist": + self.work_canvas.yview_moveto(0) window.deiconify() window.lift() window.focus_force() @@ -510,7 +517,14 @@ def _entry( self, parent: tk.Misc, title: str, x: int, y: int ) -> tuple[tk.Entry, tk.Label]: label = _label(parent, title, x, y, font=("DejaVu Sans", 14, "bold")) - entry = tk.Entry(parent, font=("DejaVu Sans", 17), bg=PANEL, fg=FG) + entry = tk.Entry( + parent, + font=("DejaVu Sans", 17), + bg=PANEL, + fg=FG, + insertbackground=FG, + insertofftime=0, + ) entry.place(x=x, y=y + 36, width=590, height=42) return entry, label diff --git a/openadapt_flow/compiler/compile.py b/openadapt_flow/compiler/compile.py index 34f94a75..6987ddac 100644 --- a/openadapt_flow/compiler/compile.py +++ b/openadapt_flow/compiler/compile.py @@ -332,6 +332,70 @@ def _best_target_text( return crop_text +def _action_scoped_target_line( + frame_lines: list[OcrLine], + marked_region: Region, + *, + frame_size: tuple[int, int], + reference_date: Optional[date], +) -> Optional[tuple[Region, str]]: + """Select one stable visual locator from an operator-marked action row. + + A complete marked row is strong identity evidence, but it is often too + wide for strict template matching after the row moves. Prefer one unique, + stable OCR line inside the row as the visual template. Identifier-shaped + text wins when available. The recorded click keeps its offset from this + locator, so replay can find the row by its stable key and still actuate the + demonstrated control anywhere else inside that row. + """ + + rx, ry, rw, rh = marked_region + normalized_counts: Counter[str] = Counter( + normalize_text(line.text.strip()) + for line in frame_lines + if line.confidence >= MIN_OCR_CONFIDENCE and line.text.strip() + ) + candidates: list[tuple[tuple[int, int, int, float], OcrLine, str]] = [] + for line in frame_lines: + text = line.text.strip() + if line.confidence < MIN_OCR_CONFIDENCE or not text: + continue + x, y, w, h = line.region + center_x = x + w // 2 + center_y = y + h // 2 + if not (rx <= center_x < rx + rw and ry <= center_y < ry + rh): + continue + if volatility.is_volatile_line(text, reference_date=reference_date): + continue + normalized = normalize_text(text) + if normalized_counts[normalized] != 1: + continue + compact = re.sub(r"\W+", "", text) + if len(compact) < MIN_TEXT_PRESENT_LEN: + continue + has_alpha = any(char.isalpha() for char in compact) + has_digit = any(char.isdigit() for char in compact) + score = ( + int(has_alpha and has_digit), + int(has_digit), + len(compact), + float(line.confidence), + ) + candidates.append((score, line, text)) + if not candidates: + return None + + _score, selected, text = max(candidates, key=lambda candidate: candidate[0]) + x, y, w, h = selected.region + frame_w, frame_h = frame_size + pad = 4 + x0 = max(0, rx, x - pad) + y0 = max(0, ry, y - pad) + x1 = min(frame_w, rx + rw, x + w + pad) + y1 = min(frame_h, ry + rh, y + h + pad) + return (x0, y0, x1 - x0, y1 - y0), text + + def _landmarks_for( frame_lines: list[OcrLine], crop_region: Region, @@ -1756,6 +1820,7 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: # can segment or recognize identical pixels differently (notably # l/1 and O/0), while replacing the original row context can drop # names/DOBs on some OCR builds. + cropped_context: Optional[str] = None if identifier_region is not None: cropped_context = identifier_text_from_lines( frame_lines, @@ -1765,15 +1830,44 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: ) if cropped_context is not None: context_text = cropped_context + anchor_template = template_rel + anchor_region = crop_region + anchor_ocr_text = ocr_text + anchor_landmarks = landmarks + if ( + event_marked is not None + and identifier_crop_rel is not None + and identifier_region is not None + ): + rx, ry, rw, rh = identifier_region + if rx <= click[0] < rx + rw and ry <= click[1] < ry + rh: + selected_target = _action_scoped_target_line( + frame_lines, + identifier_region, + frame_size=(frame.shape[1], frame.shape[0]), + reference_date=reference_date, + ) + if selected_target is not None: + anchor_region, anchor_ocr_text = selected_target + (bundle / template_rel).write_bytes( + _crop_png(before_png, anchor_region) + ) + else: + # A marked image-only row still provides a complete + # visual target when no stable unique text is present. + anchor_template = identifier_crop_rel + anchor_region = identifier_region + anchor_ocr_text = cropped_context or ocr_text + anchor_landmarks = [] anchor = Anchor( - template=template_rel, - region=crop_region, + template=anchor_template, + region=anchor_region, click_point=click, - ocr_text=ocr_text, + ocr_text=anchor_ocr_text, context_text=context_text, structured_identity=structured_identity, structural=structural, - landmarks=landmarks, + landmarks=anchor_landmarks, identifier_crop=identifier_crop_rel, identifier_region=identifier_region, ) diff --git a/openadapt_flow/runtime/heal.py b/openadapt_flow/runtime/heal.py index 29d9e53c..e83d10dd 100644 --- a/openadapt_flow/runtime/heal.py +++ b/openadapt_flow/runtime/heal.py @@ -107,6 +107,8 @@ def _recontext( region: Region, click_point: Point, frame: tuple[int, int], + *, + identifier_region: Region | None = None, ) -> str | None: """Re-derive the anchor's identity context band from the live frame. @@ -128,6 +130,12 @@ def _recontext( lines = vision.ocr(frame_png) except Exception: return None + if identifier_region is not None: + return identity_mod.identifier_text_from_lines( + lines, + region=identifier_region, + reference_date=date.today(), + ) return identity_mod.context_from_lines( lines, exclude_region=region, @@ -186,13 +194,40 @@ def build_heal_event( crop_png = _crop_png(frame_png, new_region) new_text = _reocr_text(vision, frame_png, new_region, click_y=resolution.point[1]) - new_context = _recontext(vision, frame_png, new_region, resolution.point, frame) + translated_identifier_region = old_anchor.identifier_region + if translated_identifier_region is not None: + ix, iy, iw, ih = translated_identifier_region + translated_identifier_region = ( + ix + resolution.point[0] - old_anchor.click_point[0], + iy + resolution.point[1] - old_anchor.click_point[1], + iw, + ih, + ) + if ( + old_anchor.context_text is None + and old_anchor.identity_template is not None + and old_anchor.identifier_crop is not None + and translated_identifier_region is not None + ): + # The PHI-free template and the unchanged pixel crop remain the + # identity evidence. Do not add live plaintext to the healed anchor. + new_context = None + else: + new_context = _recontext( + vision, + frame_png, + new_region, + resolution.point, + frame, + identifier_region=translated_identifier_region, + ) new_anchor = old_anchor.model_copy( update={ "region": new_region, "click_point": resolution.point, "ocr_text": new_text if new_text is not None else old_anchor.ocr_text, "context_text": new_context, + "identifier_region": translated_identifier_region, } ) event = HealEvent( diff --git a/openadapt_flow/runtime/healing/governance.py b/openadapt_flow/runtime/healing/governance.py index 6231cd9b..13f36ce7 100644 --- a/openadapt_flow/runtime/healing/governance.py +++ b/openadapt_flow/runtime/healing/governance.py @@ -42,6 +42,26 @@ def _default_band_verifier(recorded: str, observed: str) -> str: return identity_mod.verify_target_identity(recorded, observed).status +def _identifier_region_geometry_preserved( + old_anchor: Anchor, new_anchor: Anchor +) -> bool: + """Accept only a rigid locator translation of pixel identity evidence.""" + + old = old_anchor.identifier_region + new = new_anchor.identifier_region + if old == new: + return True + if old is None or new is None: + return False + ox, oy, ow, oh = old + nx, ny, nw, nh = new + return bool( + (ow, oh) == (nw, nh) + and ox - old_anchor.click_point[0] == nx - new_anchor.click_point[0] + and oy - old_anchor.click_point[1] == ny - new_anchor.click_point[1] + ) + + class PreservationVerdict(BaseModel): """Whether a heal preserved the step's identity band.""" @@ -86,14 +106,17 @@ def identity_preserved( # A program-level revision checks every surviving step, including anchors # the revision did not touch. An unchanged PHI-free identity template has # no readable band to feed through the live-verification path below, but it - # also cannot have been weakened. Admit that narrow no-op case only when - # *all* identity-bearing anchor fields are value-identical. Locator fields - # are deliberately excluded: moving the region/click point or refreshing - # locator text is the legitimate work of a heal. + # also cannot have been weakened. Admit that narrow case only when all + # identity evidence stays identical and an identifier region, if present, + # keeps its exact size and offset from the translated click point. Locator + # fields remain free to move or refresh. + unchanged_identity_fields = tuple( + field for field in IDENTITY_FIELDS if field != "identifier_region" + ) if all( getattr(old_anchor, field) == getattr(new_anchor, field) - for field in IDENTITY_FIELDS - ): + for field in unchanged_identity_fields + ) and _identifier_region_geometry_preserved(old_anchor, new_anchor): return PreservationVerdict( preserved=True, reason="identity evidence unchanged", @@ -128,12 +151,16 @@ def identity_preserved( reason="heal changed identity_template evidence", ) - for field in ("identifier_crop", "identifier_region"): - if getattr(old_anchor, field) != getattr(new_anchor, field): - return PreservationVerdict( - preserved=False, - reason=f"heal changed {field} identity evidence", - ) + if old_anchor.identifier_crop != new_anchor.identifier_crop: + return PreservationVerdict( + preserved=False, + reason="heal changed identifier_crop identity evidence", + ) + if not _identifier_region_geometry_preserved(old_anchor, new_anchor): + return PreservationVerdict( + preserved=False, + reason="heal changed identifier_region identity evidence", + ) # (2) structured identity may never be added, dropped, or changed by a # heal. Adding a new identity tier is still an identity revision and needs diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py index 0f1e9a5c..90a7402c 100644 --- a/openadapt_flow/runtime/replayer.py +++ b/openadapt_flow/runtime/replayer.py @@ -25,10 +25,12 @@ postconditions; direct API writes always require an independent verifier. This path makes ZERO model calls — effect verification reads the system of record. -Steps that succeed via any rung other than ``template`` are healed: the -anchor is refreshed from the live frame, the heal is recorded under -``run_dir/heals//``, and — when ``save_healed_to`` is set — a full -healed bundle is written. +Steps that succeed via any rung other than ``template`` produce a governed +repair. An ungoverned local run can apply the refreshed anchor in memory. An +exactly authorized run records a promotable candidate without changing its +admitted program. Repair evidence is written under +``run_dir/heals//``; an explicit later lifecycle can qualify and +promote a new bundle. """ from __future__ import annotations @@ -11811,7 +11813,7 @@ def readiness_holds(frame_png: bytes) -> bool: lambda: self.backend.scroll(dx, dy), ) scrolled += increment - frame = self.vision.wait_settled(self.backend) + frame = self._wait_for_scroll_transition_and_settle(before_png) before_png = frame holds = readiness_holds(frame) if self._governed_asset_mutation is not None: @@ -11843,6 +11845,27 @@ def readiness_holds(frame_png: bytes) -> bool: f"{target_desc} resolving — target never came into view; run aborted" ) + def _wait_for_scroll_transition_and_settle(self, baseline_png: bytes) -> bytes: + """Wait for a delivered scroll to change the frame, then settle it. + + A remote client can return two identical pre-action frames before the + wheel packet reaches the remote application. A generic settle poll can + therefore accept the old screen as stable. Require one visual + transition within the configured readiness window before applying the + normal settled-state check. At a real scroll boundary, return the last + unchanged frame after the bounded wait so the closed loop can continue + or refuse when its action budget is exhausted. + """ + + deadline = time.monotonic() + self.settle_readiness_timeout_s + frame = self.backend.screenshot() + while not self.vision.pixels_changed(baseline_png, frame): + if time.monotonic() >= deadline: + return frame + time.sleep(self.poll_interval_s) + frame = self.backend.screenshot() + return self.vision.wait_settled(self.backend) + def _prepare_remote_scroll_input( self, step: Step, @@ -12192,7 +12215,7 @@ def _heal_step( run_dir: Path, new_crops: dict[str, bytes], ): - """Build, govern, and (if promoted) apply/persist a heal. + """Build, govern, and persist a repair candidate. A heal is a governed PATCH, not a silent bundle swap: the raw event is wrapped in a reviewable :class:`~openadapt_flow.runtime.healing.HealPatch` @@ -12200,21 +12223,31 @@ def _heal_step( it may touch the workflow. A patch that would weaken the step's identity band -- the reviewed context-drop bug -- is QUARANTINED (persisted under ``run_dir/heals//patch.json`` for review) - and NOT applied; the returned outcome's ``promoted`` is False and the - caller halts the run (refuse-rather-than-guess). + and NOT applied; the caller halts the run. A gate-passing local repair + can update the in-memory workflow. A gate-passing repair discovered + under exact authorization remains a promotable candidate, because the + running program cannot change after admission. """ event, crop_png = heal_mod.build_heal_event( step, resolution, matched_region, frame_png, self.vision ) outcome = healing_mod.govern_heal(step, event, run_dir=run_dir) if outcome.promoted: - heal_mod.apply_heal(workflow, event) - source = self._execution_workflow_source - if source is not None and source is not workflow: - heal_mod.apply_heal(source, event) - self._accept_healed_anchor_in_workflow_snapshot(workflow, event) + if self.governed_authorization is None: + heal_mod.apply_heal(workflow, event) + source = self._execution_workflow_source + if source is not None and source is not workflow: + heal_mod.apply_heal(source, event) + self._accept_healed_anchor_in_workflow_snapshot(workflow, event) + new_crops[step.id] = crop_png + else: + # Exact authorization binds the admitted workflow semantics. + # Keep a gate-passing repair as a reviewable candidate for a + # later qualified bundle. Never change the program that is + # currently executing under that authorization. + outcome.patch.status = "promotable" + healing_mod.persist_patch(outcome.patch, run_dir) heal_mod.persist_heal(event, crop_png, frame_png, run_dir) - new_crops[step.id] = crop_png return outcome def _accept_healed_anchor_in_workflow_snapshot( diff --git a/tests/test_compile_identifier_crop.py b/tests/test_compile_identifier_crop.py index 2fa9f74c..42cdcb85 100644 --- a/tests/test_compile_identifier_crop.py +++ b/tests/test_compile_identifier_crop.py @@ -48,6 +48,7 @@ from openadapt_flow.runtime import identity as identity_mod from openadapt_flow.runtime.identity_template import verify_template_identity from openadapt_flow.runtime.replayer import Replayer +from openadapt_flow.runtime.resolver import resolve from openadapt_flow.vision.ocr import OcrLine VIEWPORT = (1280, 800) @@ -218,6 +219,49 @@ def test_event_marked_region_wins_and_forces_crop_despite_structured( assert (bundle / anchor.identifier_crop).is_file() +def test_action_scoped_identifier_region_produces_movable_visual_target( + tmp_path: Path, +) -> None: + """A marked row resolves by a stable key after it moves on the screen.""" + + marked = [60, ROW_Y - 24, 1060, 40] + recording, bundle = _build_recording( + tmp_path, with_structured=False, event_identifier_region=marked + ) + + anchor = _click_step( + compile_recording(recording, bundle, name="complete-visual-target") + ).anchor + + assert anchor is not None + assert anchor.identifier_crop is not None + assert anchor.template != anchor.identifier_crop + assert anchor.identifier_region == tuple(marked) + assert anchor.landmarks == [] + + rx, ry, rw, rh = anchor.region + mx, my, mw, mh = marked + assert mx <= rx < rx + rw <= mx + mw + assert my <= ry < ry + rh <= my + mh + + shifted = _blank() + _draw_text(shifted, 80, ROW_Y + 88, IDENTITY_TEXT) + _draw_button(shifted, 980, ROW_Y + 58, 120, 44, "Open") + assert anchor.template is not None + resolved = resolve( + anchor, + _png(shifted), + vision, + template_png=(bundle / anchor.template).read_bytes(), + ) + + assert resolved is not None + resolution, _matched_region = resolved + assert resolution.rung in {"template", "template_global"} + assert abs(resolution.point[0] - 1040) <= 2 + assert abs(resolution.point[1] - (ROW_Y + 80)) <= 2 + + def test_meta_marked_region_applies_to_pixel_recording(tmp_path: Path) -> None: """Desktop marking (``record --identifier X,Y,W,H`` -> meta.json) scopes the crop to the operator-designated region instead of the band box.""" diff --git a/tests/test_governed_authorization.py b/tests/test_governed_authorization.py index 5395b2c3..e6879534 100644 --- a/tests/test_governed_authorization.py +++ b/tests/test_governed_authorization.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json + import pytest from openadapt_flow.ir import ( @@ -136,6 +138,38 @@ def test_in_memory_semantic_mutation_halts_before_action(tmp_path): assert "in-memory workflow semantics" in (report.results[0].error or "") +def test_governed_drift_stages_repair_without_mutating_authorized_workflow(tmp_path): + first = click_step("first") + second = click_step("second") + workflow, bundle = _seal( + tmp_path, Workflow(name="governed-repair-candidate", steps=[first, second]) + ) + authorization = _authorization(workflow) + vision = FakeVision() + vision.template_results = [ + None, + Match(point=(150, 125), region=(140, 120, 50, 20), confidence=0.99), + Match(point=(110, 105), region=(100, 100, 50, 20), confidence=0.99), + ] + backend = FakeBackend() + + report = Replayer( + backend, + vision=vision, + governed_authorization=authorization, + ).run(workflow, bundle_dir=bundle, run_dir=tmp_path / "run") + + assert report.success is True + assert workflow.steps[0].anchor is not None + assert workflow.steps[0].anchor.region == (100, 100, 50, 20) + assert report.results[0].heal is not None + assert report.results[0].heal.applied is False + persisted = json.loads( + (tmp_path / "run" / "heals" / "first" / "patch.json").read_text() + ) + assert persisted["status"] == "promotable" + + @pytest.mark.parametrize( ("field", "replacement"), [ diff --git a/tests/test_governed_healing.py b/tests/test_governed_healing.py index 9a7c0370..4d0d9dca 100644 --- a/tests/test_governed_healing.py +++ b/tests/test_governed_healing.py @@ -383,6 +383,22 @@ def test_invariant_blocks_changed_pixel_identity_evidence(field, value): assert field in verdict.reason +def test_invariant_allows_identifier_region_to_move_with_locator(): + old = _template_armed_anchor() + new = old.model_copy( + update={ + "region": (140, 180, 50, 20), + "click_point": (150, 185), + "identifier_region": (60, 180, 60, 20), + }, + deep=True, + ) + + verdict = identity_preserved(old, new) + + assert verdict.preserved is True + + # --------------------------------------------------------------------------- # # 2. HealPatch -- reviewable diff # --------------------------------------------------------------------------- # diff --git a/tests/test_heal.py b/tests/test_heal.py index eaa411d9..2edd9cbe 100644 --- a/tests/test_heal.py +++ b/tests/test_heal.py @@ -22,6 +22,7 @@ Workflow, ) from openadapt_flow.runtime.heal import build_heal_event, write_healed_bundle +from openadapt_flow.runtime.identity_template import build_identity_template from openadapt_flow.runtime.replayer import Replayer VIEWPORT = (300, 200) @@ -360,6 +361,59 @@ def test_build_heal_event_clamps_region_at_frame_edge(): assert event.new_anchor.context_text is None +def test_heal_translates_and_refreshes_marked_identifier_region(): + step = ocr_anchored_step() + assert step.anchor is not None + step.anchor.identifier_crop = "templates/identifiers/s1.png" + step.anchor.identifier_region = (10, 90, 240, 40) + vision = FakeVision() + vision.ocr_lines = [ + OcrLine("REQ-LIVE-2048", region=(20, 132, 90, 18), confidence=0.99), + OcrLine("REC-2048", region=(120, 132, 70, 18), confidence=0.99), + OcrLine("Unrelated", region=(20, 20, 70, 18), confidence=0.99), + ] + resolution = Resolution( + rung="template_global", point=(110, 145), confidence=0.99, elapsed_ms=1.0 + ) + + event, _crop = build_heal_event( + step, + resolution, + (100, 140, 50, 20), + make_png(VIEWPORT), + vision, + ) + + assert event.new_anchor.identifier_region == (10, 130, 240, 40) + assert event.new_anchor.context_text == "REQ-LIVE-2048 REC-2048" + + +def test_heal_keeps_phi_free_pixel_identity_without_plaintext_context(): + step = ocr_anchored_step() + assert step.anchor is not None + step.anchor.context_text = None + step.anchor.identity_template = build_identity_template( + "REQ-LIVE-2048 REC-2048", salt_hex="11" * 16 + ) + step.anchor.identifier_crop = "templates/identifiers/s1.png" + step.anchor.identifier_region = (10, 90, 240, 40) + resolution = Resolution( + rung="template_global", point=(110, 145), confidence=0.99, elapsed_ms=1.0 + ) + + event, _crop = build_heal_event( + step, + resolution, + (100, 140, 50, 20), + make_png(VIEWPORT), + FakeVision(), + ) + + assert event.new_anchor.identifier_region == (10, 130, 240, 40) + assert event.new_anchor.context_text is None + assert event.new_anchor.identity_template == step.anchor.identity_template + + def test_write_healed_bundle_direct(tmp_path): src = tmp_path / "src" (src / "templates").mkdir(parents=True) diff --git a/tests/test_replayer.py b/tests/test_replayer.py index 151b2cf9..15c727df 100644 --- a/tests/test_replayer.py +++ b/tests/test_replayer.py @@ -1923,6 +1923,28 @@ def test_closed_loop_scroll_stops_when_next_anchor_resolves(bundle, run_dir): assert report.rung_counts == {"template": 1} +def test_closed_loop_scroll_waits_for_delayed_visual_transition(bundle, run_dir): + """A delayed remote wheel packet cannot make the old frame look settled.""" + + vision = FakeVision() + target = Match(point=(110, 105), region=(100, 100, 50, 20), confidence=0.95) + vision.template_results = [None, None, target, target] + vision.pixels_changed_results = [False, False, True] + backend = FakeBackend() + workflow = Workflow(name="wf", steps=[scroll_step(), click_step()]) + + report = Replayer(backend, vision=vision, poll_interval_s=0.001).run( + workflow, bundle_dir=bundle, run_dir=run_dir + ) + + assert report.success is True + assert backend.actions == [ + ("scroll", 0, 400), + ("click", 110, 105, False), + ] + assert len(vision.pixels_changed_calls) >= 3 + + def test_closed_loop_scroll_noops_when_anchor_already_in_view(bundle, run_dir): """The pre-scroll probe resolving means the target is already on screen: the SCROLL step must not scroll at all."""