flow: take extra looks before stopping on a look-alike record ID

packages/flow/docs/LIMITS.md

@@ -188,6 +188,30 @@ halts with a named reason when the confirmation is missing
 before and after numbers are in
 [`benchmark/known_ids/KNOWN_IDS.md`](../benchmark/known_ids/KNOWN_IDS.md).
 
+Before a step stops on a look-alike identifier, the replayer takes extra looks
+that change nothing
+([`openadapt_flow/runtime/identity_looks.py`](../openadapt_flow/runtime/identity_looks.py)):
+the same read on two fresh screenshots, the 2x re-read, the known-ID list when
+one is configured, and, where the surface allows it, a select-and-copy of the
+identifier's text box (`runtime.identity_copy_look`). OCR looks can't settle
+the record on their own, because a look-alike twin can read the same on every
+frame and at every size. Only two looks can: the known-ID list, and a copied
+identifier that equals a run parameter in the identity band. A look that names
+another record stops the run. Each look is recorded on the step's identity
+check.
+
+The copy look has limits. It needs a surface where selecting the identifier
+and copying it changes nothing, and a clipboard that reaches the machine Flow
+runs on (RDP or Citrix clipboard redirection). It drags across only the one
+text box that holds the identifier, and the run stops if the screen changes
+anywhere else during the look. A canvas with no selectable text has no copy
+look. A recording's own identifier is OCR text and can be a misread (the RDP
+kiosk recorded `A1001` as `Alo01`), so in a workflow that always opens the
+recorded record, the copy can stop a run but never confirm one. It confirms
+only a record chosen by a run parameter. Governed and qualification runs never
+take it. The check just before the click takes no input; it reuses the copy
+only when the identifier's pixels are unchanged since the look.
+
 Permissive `replay` may proceed on an unreadable reversible target and reports
 that condition. Governed `run` requires an affirmative live verdict for every
 identity-required admitted step; mismatch, unreadable, or abstain halts before

packages/flow/openadapt_flow/deployment.py

@@ -712,6 +712,24 @@ class GroundingModelConfig(BaseModel):
     api_key_env: str = ""
 
 
+class IdentityCopyLookSection(BaseModel):
+    """Select-and-copy look for a look-alike identifier on a pixel surface.
+
+    Turn on only where selecting the recorded identifier cell and copying it
+    changes nothing (a read-only text cell) and the session's clipboard reaches
+    the machine Flow runs on (RDP/Citrix clipboard redirection). See
+    :mod:`openadapt_flow.runtime.identity_looks`.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: bool = False
+    #: ``drag`` across the cell at mid height, or ``double_click`` its centre.
+    gesture: Literal["drag", "double_click"] = "drag"
+    #: Keys that copy the selection in the driven session.
+    copy_keys: list[str] = Field(default_factory=lambda: ["Control+c"], min_length=1)
+
+
 class RuntimeSection(BaseModel):
     """Runtime posture: durability and model-egress opt-in."""
 
@@ -761,6 +779,13 @@ class RuntimeSection(BaseModel):
     known_ids_confirmation_path: Optional[str] = None
     #: Oldest export (by its export time) replay will use.
     known_ids_max_age_hours: float = Field(default=24.0, gt=0)
+    #: Before stopping on a look-alike identifier, select the identifier cell,
+    #: copy it and read the clipboard. Settles the record only when the copy
+    #: equals a run parameter in the identity band; halts when the screen
+    #: changes outside the cell. Off by default. Never used by governed runs.
+    identity_copy_look: IdentityCopyLookSection = Field(
+        default_factory=IdentityCopyLookSection
+    )
     #: Font/layout drift check before each recorded click step
     #: (:mod:`openadapt_flow.runtime.layout_drift`). None (default) turns it on
     #: for remote-desktop replays (``rdp``/``citrix``) and off elsewhere;
@@ -1562,8 +1587,21 @@ def build_replayer(
             confirmation_path=runtime_config.known_ids_confirmation_path,
             max_age_s=runtime_config.known_ids_max_age_hours * 3600.0,
         )
+    identity_copy = None
+    read_clipboard = None
+    copy_cfg = runtime_config.identity_copy_look if runtime_config is not None else None
+    if copy_cfg is not None and copy_cfg.enabled:
+        from openadapt_flow.records.read import SystemClipboard
+        from openadapt_flow.runtime.identity_looks import IdentityCopyConfig
+
+        identity_copy = IdentityCopyConfig(
+            gesture=copy_cfg.gesture, copy_keys=tuple(copy_cfg.copy_keys)
+        )
+        read_clipboard = SystemClipboard()
     return Replayer(
         backend,
+        identity_copy=identity_copy,
+        read_clipboard=read_clipboard,
         layout_drift=(
             runtime_config.layout_drift_check if runtime_config is not None else None
         ),

packages/flow/openadapt_flow/ir.py

@@ -3293,6 +3293,26 @@ class KnownIdEvidence(BaseModel):
     export_sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$")
 
 
+class IdentityLook(BaseModel):
+    """One extra look taken before stopping on a look-alike identifier.
+
+    See :mod:`openadapt_flow.runtime.identity_looks`. Holds no identifier
+    values: only what kind of look it was, what it concluded, and the
+    SHA-256 of the frame it read when it read one.
+    """
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    kind: Literal["frame", "magnified", "known_ids", "clipboard"]
+    #: ``agrees``: consistent with the first read, settles nothing alone.
+    #: ``resolves``: names exactly one record, the expected one.
+    #: ``contradicts``: names another record; the step stops.
+    #: ``unavailable``: the look could not be taken or read nothing usable.
+    outcome: Literal["agrees", "resolves", "contradicts", "unavailable"]
+    reason: Optional[str] = Field(default=None, max_length=80)
+    frame_sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$")
+
+
 class IdentityCheck(BaseModel):
     """Outcome of the pre-click target-identity check (runtime.identity).
 
@@ -3377,6 +3397,20 @@ class IdentityCheck(BaseModel):
             "confirmed, or it is older than the configured age. The step halts."
         ),
     )
+    looks: list[IdentityLook] = Field(
+        default_factory=list,
+        description=(
+            "Extra looks taken because the first read abstained only on a "
+            "look-alike identifier (runtime.identity_looks), in order."
+        ),
+    )
+    look_halt: Optional[Literal["identity_look_changed_screen"]] = Field(
+        default=None,
+        description=(
+            "Set when an extra look changed the screen outside the "
+            "identifier it read. The step halts."
+        ),
+    )
 
 
 class HealEvent(BaseModel):