test: add real-RDP multi-window vision campaign

.github/workflows/docker-rdp-multiapp-vision.yml

@@ -0,0 +1,76 @@
+name: docker-rdp-multiapp-vision
+
+# This expensive real-protocol test runs only when its own campaign changes or
+# when a maintainer starts it. Core changes use the smaller RDP ladder during
+# pull-request feedback; release qualification can start this campaign once.
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/docker-rdp-multiapp-vision.yml"
+      - "benchmark/rdp_multiapp/**"
+      - "tests/e2e/test_docker_rdp_multiapp_vision_e2e.py"
+      - "tests/test_rdp_multiapp_campaign_contract.py"
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docker-rdp-multiapp-vision-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  qualify:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install the vision and RDP stack
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[rdp]"
+          pip check
+
+      - name: Build and start the RDP fixture
+        run: |
+          docker build -t oaflow-rdp-multiapp:latest benchmark/rdp_multiapp/fixture
+          mkdir -p "${RUNNER_TEMP}/rdp-multiapp-oracle"
+          docker run -d --name oaflow-rdp-multiapp --shm-size=1g \
+            -e RDP_MULTIAPP_ORACLE_ROOT=/oracle \
+            -v "${RUNNER_TEMP}/rdp-multiapp-oracle:/oracle" \
+            oaflow-rdp-multiapp:latest
+          sleep 22
+
+      - name: Run the real-RDP visual subset
+        run: |
+          mkdir -p "${RUNNER_TEMP}/rdp-multiapp-result"
+          python benchmark/rdp_multiapp/run_qualification.py \
+            --container oaflow-rdp-multiapp \
+            --oracle-root "${RUNNER_TEMP}/rdp-multiapp-oracle" \
+            --work-dir "${RUNNER_TEMP}/rdp-multiapp-work" \
+            --output "${RUNNER_TEMP}/rdp-multiapp-result/results.json"
+
+      - name: Upload exact result
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rdp-multiapp-vision-subset
+          path: ${{ runner.temp }}/rdp-multiapp-result/
+          if-no-files-found: warn
+
+      - name: Print bounded fixture diagnostics
+        if: failure()
+        run: |
+          docker logs --tail 80 oaflow-rdp-multiapp || true
+          docker exec oaflow-rdp-multiapp sh -c \
+            'for path in /tmp/suite.log /tmp/client.log /tmp/shadow.log; do echo "== ${path} =="; tail -n 80 "${path}" 2>/dev/null || true; done'
+          ls -la "${RUNNER_TEMP}/rdp-multiapp-oracle" || true
+
+      - name: Tear down
+        if: always()
+        run: docker rm -f oaflow-rdp-multiapp || true

benchmark/rdp_ladder/run_rdp_ladder_qualification.py

@@ -251,24 +251,46 @@ def framebuffer(self):
         img = self._grab()
         return img, img.width, img.height
 
-    def _focus_client(self) -> None:
-        """Focus the isolated FreeRDP window before injecting XTest input.
-
-        The fixture runs a minimal Openbox session. Focusing its only visible
-        FreeRDP window once is deterministic and remains entirely inside
-        display ``:1`` in the container.
-        """
-        self._exec(
+    def _client_window_id(self) -> int:
+        raw = self._exec(
             [
                 "xdotool",
                 "search",
                 "--onlyvisible",
                 "--name",
                 "^FreeRDP:",
-                "windowfocus",
-                "%@",
             ]
         )
+        try:
+            return int(raw.decode().splitlines()[0])
+        except (IndexError, ValueError) as exc:
+            raise RuntimeError("isolated FreeRDP window is unavailable") from exc
+
+    def _focus_client(self, window_id: Optional[int] = None) -> None:
+        """Focus the isolated FreeRDP window before injecting XTest input.
+
+        The fixture runs a minimal Openbox session. Focusing its only visible
+        FreeRDP window once is deterministic and remains entirely inside
+        display ``:1`` in the container.
+        """
+        client_window = window_id if window_id is not None else self._client_window_id()
+        self._exec(["xdotool", "windowfocus", str(client_window)])
+
+    def focus_input_surface(self) -> None:
+        """Restore the outer FreeRDP window before keyboard delivery."""
+
+        client_window = self._client_window_id()
+        try:
+            active_window = int(self._exec(["xdotool", "getactivewindow"]).decode())
+        except ValueError as exc:
+            raise RuntimeError("active X11 input window is unavailable") from exc
+        if active_window == client_window:
+            return
+        self._focus_client(client_window)
+        # Let the window manager and FreeRDP restore the keyboard grab before
+        # the next XTest key event. This delay is outside the production RDP
+        # transport. It makes the two-Xvfb qualification fixture deterministic.
+        time.sleep(0.1)
 
     def _remote_pointer(self) -> Optional[tuple[int, int]]:
         """Return the fixture server's cursor as a delivery acknowledgement.
@@ -352,6 +374,31 @@ def key(self, keysym_or_char: str, down: bool) -> None:
         verb = "keydown" if down else "keyup"
         self._exec(["xdotool", verb, "--clearmodifiers", keysym])
 
+    @staticmethod
+    def supports_bulk_text(text: str) -> bool:
+        """Use one X11 client for printable ASCII fixture parameters."""
+
+        return (
+            bool(text) and text.isascii() and all(" " <= char <= "~" for char in text)
+        )
+
+    def bulk_type_text(self, text: str) -> None:
+        """Type one value without losing the FreeRDP grab between characters."""
+
+        if not self.supports_bulk_text(text):
+            raise ValueError("RDP fixture bulk text must be printable ASCII")
+        self._exec(
+            [
+                "xdotool",
+                "type",
+                "--clearmodifiers",
+                "--delay",
+                "35",
+                "--",
+                text,
+            ]
+        )
+
     def wheel(self, dx: int, dy: int) -> None:
         if not dy:
             return

benchmark/rdp_multiapp/README.md

@@ -0,0 +1,49 @@
+# Real-RDP multi-window vision campaign
+
+This campaign tests the main pixel-automation risk that the existing complex
+API benchmarks do not test. It records, compiles, and replays one back-office
+workflow across three separate task windows. Observation is limited to pixels
+decoded by a real FreeRDP client. Input returns through the same RDP session.
+
+The workflow reads a referral in **Inbox**, finds the request in a scrollable
+**Worklist**, enters the appointment in **Scheduler**, reconciles the worklist,
+and sends a confirmation from Inbox. The result contract reads three persisted
+surfaces without trusting the UI:
+
+- SQLite: exactly one appointment for the authorized request and record.
+- CSV: the exact worklist row has status `Scheduled` and no adjacent row changed.
+- Maildir: exactly one confirmation has the expected request correlation key.
+
+`campaign.json` defines the fault campaign. Every condition requires three
+trials. The report must count silent incorrect success and over-halt, not only
+task completion.
+
+## Scope
+
+This fixture uses deterministic synthetic data. It tests the production visual
+resolver and RDP input path without a DOM or accessibility tree. It does not
+replace qualification of a named Windows or Citrix application. The separate
+task windows exercise window switching and focus behavior, but they are hosted
+by one synthetic fixture process.
+
+## Fixture
+
+```bash
+docker build -t oaflow-rdp-multiapp:latest benchmark/rdp_multiapp/fixture
+docker run --rm --name oaflow-rdp-multiapp \
+  -v "$PWD/.tmp/rdp-multiapp-oracle:/opt/rdp_multiapp/oracle" \
+  oaflow-rdp-multiapp:latest
+```
+
+Run the implemented subset after the container is ready:
+
+```bash
+python benchmark/rdp_multiapp/run_qualification.py \
+  --oracle-root "$PWD/.tmp/rdp-multiapp-oracle" \
+  --output benchmark/rdp_multiapp/results.json
+```
+
+The runner uses the same `DockerX11RdpTransport` and `FreeRDPBackend` contract
+as `benchmark/rdp_ladder`. The first subset runs healthy, row-reordered,
+wrong-record, and focus-theft conditions. The result cannot describe the full
+campaign as complete until every condition in `campaign.json` has run.

benchmark/rdp_multiapp/campaign.json

@@ -0,0 +1,72 @@
+{
+  "schema_version": "openadapt.rdp-multiapp-campaign.v1",
+  "task": "Process one referral across inbox, worklist, and scheduler windows through RDP pixels, then prove the appointment, worklist status, and confirmation email on independent persisted surfaces.",
+  "trials_per_condition": 3,
+  "required_metrics": [
+    "verified_outcomes",
+    "safe_halts",
+    "silent_incorrect_successes",
+    "over_halts",
+    "wrong_record_writes",
+    "duplicate_effects",
+    "model_calls",
+    "p50_runtime_s",
+    "p95_runtime_s"
+  ],
+  "conditions": [
+    {
+      "id": "healthy",
+      "expect": "verified",
+      "fault": "none",
+      "oracle": ["sqlite", "csv", "maildir"]
+    },
+    {
+      "id": "row_reordered",
+      "expect": "verified",
+      "fault": "The target worklist row moves to a different visual position.",
+      "oracle": ["sqlite", "csv", "maildir"]
+    },
+    {
+      "id": "focus_theft_before_write",
+      "expect": "safe_halt",
+      "fault": "Another task window takes focus after resolution and before the scheduler write.",
+      "oracle": ["sqlite"]
+    },
+    {
+      "id": "wrong_record_before_write",
+      "expect": "safe_halt",
+      "fault": "The active scheduler record changes before the consequential click.",
+      "oracle": ["sqlite"]
+    },
+    {
+      "id": "duplicate_save_control",
+      "expect": "safe_halt",
+      "fault": "A competing Save appointment control appears in the same visual state.",
+      "oracle": ["sqlite"]
+    },
+    {
+      "id": "partial_render",
+      "expect": "safe_halt",
+      "fault": "The target window is incomplete when the next action is due.",
+      "oracle": ["sqlite", "csv", "maildir"]
+    },
+    {
+      "id": "moderate_display_drift",
+      "expect": "verified_or_safe_halt",
+      "fault": "Theme, scaling, and compression change while text remains readable.",
+      "oracle": ["sqlite", "csv", "maildir"]
+    },
+    {
+      "id": "severe_display_drift",
+      "expect": "safe_halt",
+      "fault": "The frame no longer supports unambiguous target and record identity.",
+      "oracle": ["sqlite", "csv", "maildir"]
+    },
+    {
+      "id": "commit_then_timeout",
+      "expect": "verified_or_reconciliation_required",
+      "fault": "The scheduler write can persist before the input channel reports uncertainty.",
+      "oracle": ["sqlite"]
+    }
+  ]
+}

benchmark/rdp_multiapp/fixture/Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90
+
+RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      xvfb x11-utils python3-tk fonts-dejavu-core \
+      freerdp3-shadow-x11 freerdp3-x11 \
+      imagemagick xdotool openbox procps \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY suite_app.py /opt/rdp_multiapp/suite_app.py
+COPY run_fixture.sh /opt/rdp_multiapp/run_fixture.sh
+RUN chmod +x /opt/rdp_multiapp/run_fixture.sh \
+ && mkdir -p /opt/rdp_multiapp/oracle
+
+EXPOSE 3389
+CMD ["/opt/rdp_multiapp/run_fixture.sh"]

benchmark/rdp_multiapp/fixture/run_fixture.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -euo pipefail
+
+export HOME=/root
+export RDP_MULTIAPP_ORACLE_ROOT="${RDP_MULTIAPP_ORACLE_ROOT:-/opt/rdp_multiapp/oracle}"
+mkdir -p "${RDP_MULTIAPP_ORACLE_ROOT}"
+
+Xvfb :0 -screen 0 1280x800x24 -ac +extension DAMAGE +extension RANDR +extension XFIXES \
+    >/tmp/xvfb0.log 2>&1 &
+sleep 2
+# Keep the remote display free of a second window manager. The fixture owns
+# the X keyboard focus while the client-side Openbox session manages FreeRDP.
+DISPLAY=:0 python3 /opt/rdp_multiapp/suite_app.py >/tmp/suite.log 2>&1 &
+sleep 3
+DISPLAY=:0 freerdp-shadow-cli3 /port:3389 /bind-address:0.0.0.0 -auth \
+    >/tmp/shadow.log 2>&1 &
+sleep 3
+
+Xvfb :1 -screen 0 1280x800x24 -ac >/tmp/xvfb1.log 2>&1 &
+sleep 2
+DISPLAY=:1 openbox >/tmp/openbox1.log 2>&1 &
+sleep 1
+DISPLAY=:1 xfreerdp3 /v:127.0.0.1:3389 /u:ubuntu /p:ubuntu /size:1280x800 /f \
+    -gfx -rfx -nsc /cert:ignore +auto-reconnect /log-level:ERROR \
+    >/tmp/client.log 2>&1 &
+sleep 4
+
+tail -f /dev/null