From 7fb80513b81c8bd494be99c21aaf0b82a832614f Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Mon, 2 Nov 2026 15:07:00 +0530 Subject: [PATCH 01/15] feat(harness): refuse browser, computer-use and crawler agents Only voice and chat agents are supported. An agent whose main job is driving a browser, operating a computer or crawling sites used to spend a whole build and end as a crash or an error blaming us. It now stops at the reading step with agent_type_unsupported in failure.json, and exits 78. No code can say for certain what an agent's main job is, so a check of the agent's code and the model's reading work together. Where they disagree the model looks once more, and its answer stands. Refs: TH-8806 Signed-off-by: khushalsonawat --- .github/workflows/sdk-smoke.yml | 1 + src/fi/alk/harness/HOW-IT-WORKS.md | 4 + src/fi/alk/harness/agent_type_check.py | 287 ++++ src/fi/alk/harness/amend.py | 3 +- src/fi/alk/harness/authoring_entrypoint.py | 6 + src/fi/alk/harness/chat.py | 4 + src/fi/alk/harness/cli.py | 19 +- src/fi/alk/harness/failure_file.py | 32 + src/fi/alk/harness/hosted_chat_entrypoint.py | 21 + .../harness/skills/understand-agent/SKILL.md | 16 +- src/fi/alk/harness/tools.py | 69 +- src/fi/alk/harness/understand.py | 1 + src/fi/alk/harness/world/tools.py | 2 +- tests/harness/test_agent_type_refusal.py | 1172 +++++++++++++++++ 14 files changed, 1626 insertions(+), 11 deletions(-) create mode 100644 src/fi/alk/harness/agent_type_check.py create mode 100644 src/fi/alk/harness/failure_file.py create mode 100644 tests/harness/test_agent_type_refusal.py diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index c2256018b..7ed0cf674 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -66,6 +66,7 @@ jobs: tests/harness/test_scenario_source_provision_agent.py tests/harness/test_hosted_entrypoint_provision_retry.py tests/harness/test_instrumentation_evidence.py + tests/harness/test_agent_type_refusal.py tests/test_bump_deployment_values.py tests/test_release_workflow.py tests/test_release_pr_step.py diff --git a/src/fi/alk/harness/HOW-IT-WORKS.md b/src/fi/alk/harness/HOW-IT-WORKS.md index d5a788b67..1768fec95 100644 --- a/src/fi/alk/harness/HOW-IT-WORKS.md +++ b/src/fi/alk/harness/HOW-IT-WORKS.md @@ -78,6 +78,10 @@ not exist, or — the one that mattered most in practice — *every* tool having means the arguments were read and then not recorded. Problems are returned **into the conversation**, so the model corrects and resubmits rather than a bad contract landing. +A browser reading with a browser, crawling or computer-use tool found in the source is refused to +`failure.json`, and the build exits 78. Where the model and that code check disagree, the model gets +one second look, and its answer stands. + **How it is changed later** (`amend.py`). The contract is not frozen, but every change is recorded with a reason in `amendments[]`, so months later you can still tell what came from the agent and what came from us: diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py new file mode 100644 index 000000000..f6c6e2af0 --- /dev/null +++ b/src/fi/alk/harness/agent_type_check.py @@ -0,0 +1,287 @@ +"""Does an agent use a browser, crawling or computer-use tool, and what follows from that. + +No code can say for certain what an agent's main job is, so this is one of two opinions: what the +source uses, set against what the model read. It calls no model and runs none of the agent's code, +and it needs no job or session, so a reading can be checked without a build. +""" + +from __future__ import annotations + +import ast +import fnmatch +import io +import json +import os +import re +import tokenize +from collections.abc import Iterator, Sequence +from pathlib import Path +from typing import Any, Literal + +from .failure_file import FAILURE_FILE +from .instrumentation_evidence import _SKIPPED_DIRECTORIES, _imports +from .job import FailureDomain, HarnessFailure, HarnessStage + +AGENT_TYPE_UNSUPPORTED = "agent_type_unsupported" +# EX_CONFIG, which the platform doesn't retry even if it can't read the file. +EXIT_REFUSED = 78 +BROWSER_IMPORTS = frozenset( + { + "playwright", + "patchright", + "camoufox", + "playwright_stealth", + "selenium", + "seleniumbase", + "seleniumwire", + "undetected_chromedriver", + "splinter", + "pyppeteer", + "pydoll", + "DrissionPage", + "zendriver", + "nodriver", + "botasaurus", + "browser_use", + "stagehand", + "browserbase", + "steel", + "hyperbrowser", + "skyvern", + "anchorbrowser", + "agentql", + "scrapy", + "crawl4ai", + "firecrawl", + "crawlee", + "scrapegraphai", + "apify_client", + "pyautogui", + "pywinauto", + "e2b_desktop", + "nova_act", + } +) +BROWSER_PACKAGES = frozenset( + { + "playwright", + "playwright-core", + "@playwright/mcp", + "patchright", + "puppeteer", + "puppeteer-core", + "selenium-webdriver", + "browser-use", + "stagehand", + "@browserbasehq/stagehand", + "@browserbasehq/sdk", + "crawlee", + "firecrawl", + "@mendable/firecrawl-js", + "apify-client", + } +) +TOOL_MARKER = re.compile( + r"\bcomputer[_-]use(?:[_-]preview)?\b|\bComputerUse\b|\bcomputer_\d{8}\b" + r"|@playwright/mcp|mcp-server-playwright" +) +# Hidden folders and anything named for tests or e2e describe the developer's setup too. +NOT_THE_AGENT = frozenset( + {"docs", "doc", "examples", "example", "samples", "vendor", "third_party", "evals"} +) +# What the world registry already treats as a browser. +BROWSER_MODALITIES = frozenset({"browser", "computer_use", "cua"}) + +_TOOLKIT = "PlayWrightBrowserToolkit" +_TEST_FILES = ("test_*.py", "*_test.py", "conftest.py", "*.test.*", "*.spec.*") +_SCRIPT_SUFFIXES = frozenset({".js", ".ts", ".mjs", ".cjs", ".tsx", ".jsx"}) +_LARGEST_CODE_FILE = 1024 * 1024 +# Strings are matched first and kept, so "https://..." or "src/**/*.ts" isn't read as a comment. +# An unclosed one ends at its line or the file: a start that could fail makes a customer's odd file +# cost the square of its size, which holds up the build's conversation until it's done. +_SCRIPT_COMMENT_OR_STRING = re.compile( + r"""("(?:\\.|[^"\\\n])*(?:"|(?=\n)|\\?\Z)|'(?:\\.|[^'\\\n])*(?:'|(?=\n)|\\?\Z)|`(?:\\.|[^`\\])*(?:`|\\?\Z))""" + r"""|//[^\n]*|/\*.*?(?:\*/|\Z)""", + re.DOTALL, +) + + +# A whole word only: "latest" and "contest" hold real code. +_TEST_FOLDER = re.compile(r"(?:^|[^a-z0-9])(?:tests?|testing|testdata|unittests?|integrationtests?|e2e(?:tests?)?)(?:$|[^a-z0-9])") + + +def _not_the_agent_dir(name: str) -> bool: + lowered = name.lower() + return ( + name in _SKIPPED_DIRECTORIES + or lowered in NOT_THE_AGENT + or name.startswith(".") + or _TEST_FOLDER.search(lowered) is not None + ) + + +def _not_the_agent_file(name: str) -> bool: + lowered = name.lower() + return any(fnmatch.fnmatchcase(lowered, pattern) for pattern in _TEST_FILES) + + +def _agent_files(root: Path) -> Iterator[Path]: + for directory, subdirectories, files in os.walk(root): + # A folder of unpacked distributions holds other people's code, whose SDKs name these + # tools whether or not this agent uses them. + if Path(directory) != root and any( + name.endswith(".dist-info") for name in subdirectories + ): + subdirectories[:] = [] + continue + subdirectories[:] = [ + name for name in subdirectories if not _not_the_agent_dir(name) + ] + for name in files: + if not _not_the_agent_file(name): + yield Path(directory) / name + + +def _packages_used(path: Path) -> set[str]: + try: + if not path.is_file(): + return set() + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError, RecursionError): + return set() + dependencies = manifest.get("dependencies") if isinstance(manifest, dict) else None + if not isinstance(dependencies, dict): + return set() + return BROWSER_PACKAGES.intersection(dependencies) + + +def _imports_in(path: Path) -> set[str]: + # One file at a time: a customer's odd file can run the parser out of memory, and that mustn't + # take the rest of the scan, or the reading it serves, down with it. + try: + if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: + return set() + return _imports(ast.parse(path.read_text(encoding="utf-8"))) + except (OSError, ValueError, SyntaxError, RecursionError, MemoryError): + return set() + + +def _modules_at_top(root: Path) -> set[str]: + """What sits ahead of an installed package on the import path, so isn't that package.""" + try: + return { + entry.stem if entry.is_file() else entry.name + for entry in root.iterdir() + if entry.suffix == ".py" or (entry / "__init__.py").is_file() + } + except OSError: + return set() + + +def _python_code(text: str) -> str: + return " ".join( + token.string + for token in tokenize.generate_tokens(io.StringIO(text).readline) + if token.type != tokenize.COMMENT + ) + + +def _markers_in(path: Path) -> set[str]: + try: + if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: + return set() + text = path.read_text(encoding="utf-8") + code = ( + _python_code(text) + if path.suffix == ".py" + else _SCRIPT_COMMENT_OR_STRING.sub(lambda hit: hit.group(1) or "", text) + ) + except (OSError, ValueError, SyntaxError, tokenize.TokenError): + return set() + return { + "Playwright MCP" if "playwright" in hit.group() else "computer-use tool" + for hit in TOOL_MARKER.finditer(code) + } + + +def browser_tools_found(source_root: Path | None) -> list[str]: + """The browser, crawling and computer-use tools the agent's own code uses, by display name. + + Python dependency files aren't read: an agent that uses a tool imports it, and they're where + dev and test lines live. Config files aren't either: they describe the developer's editor. + """ + if source_root is None: + return [] + root = Path(source_root).resolve() + found: set[str] = set() + imported: set[str] = set() + for path in _agent_files(root): + if path.name == "package.json": + found |= _packages_used(path) + elif path.suffix == ".py" or path.suffix in _SCRIPT_SUFFIXES: + found |= _markers_in(path) + if path.suffix == ".py": + imported |= _imports_in(path) + local = _modules_at_top(root) + for name in imported: + top = name.split(".")[0] + if top in local: + continue + if top in BROWSER_IMPORTS: + found.add(top) + elif name.endswith(f".{_TOOLKIT}"): + found.add(_TOOLKIT) + return sorted({name.replace("_", "-") for name in found}) + + +def modality_text(value: Any) -> str: + """The modality as text to compare, with a list's items one per line.""" + if isinstance(value, list): + value = "\n".join(str(item) for item in value) + return "" if value is None else str(value).strip().lower() + + +def is_browser(text: str) -> bool: + # Whole list items only: free text such as "not a browser" isn't a reading of browser. + return text in BROWSER_MODALITIES or "browser" in text.split("\n") + + +def decide( + claimed: Any, found: Sequence[str], *, looked_again: bool +) -> Literal["refuse", "second_look", "keep"]: + """What the model read against what the code check found. Where they disagree the model gets + one more look, and its answer then stands.""" + if is_browser(modality_text(claimed)): + return "refuse" if found or looked_again else "second_look" + return "second_look" if found and not looked_again else "keep" + + +def agent_type_failure(found: Sequence[str]) -> HarnessFailure: + if found: + message = ( + "This agent's main job is driving a browser, operating a computer or crawling sites " + f"(found: {', '.join(found)}). Only voice and chat agents are supported for now." + ) + else: + message = ( + "This agent's main job was read as driving a browser, operating a computer or " + "crawling sites. Only voice and chat agents are supported for now." + ) + return HarnessFailure( + domain=FailureDomain.AGENT, + stage=HarnessStage.UNDERSTANDING_AGENT, + code=AGENT_TYPE_UNSUPPORTED, + message=message, + retryable=False, + details={"agent_type": "browser", "tools": list(found)}, + ) + + +def failure_message(output: Path) -> str: + """The message of the failure a build left in ``output``, or nothing if it can't be read.""" + try: + failure = json.loads((Path(output) / FAILURE_FILE).read_text(encoding="utf-8")) + except (OSError, ValueError, RecursionError): + return "" + message = failure.get("message") if isinstance(failure, dict) else None + return message if isinstance(message, str) else "" diff --git a/src/fi/alk/harness/amend.py b/src/fi/alk/harness/amend.py index ed63edc04..841568394 100644 --- a/src/fi/alk/harness/amend.py +++ b/src/fi/alk/harness/amend.py @@ -150,6 +150,8 @@ def set_modality( False, f"{named!r} is not a modality. It is one of: {', '.join(known_modalities())}", ) + if named == "browser": + return False, "browser agents aren't supported, so the type stays " + contract.modality + "." if not why.strip(): return False, "say why: modality decides how every scenario is run" if named == contract.modality: @@ -174,7 +176,6 @@ def set_modality( "voice": "a call placed to the agent where it is hosted, its own tools answered over a " "webhook", "chat": "a typed conversation, the agent reconstructed here from its contract", - "browser": "a browser the agent drives", }[named] return True, f"modality is now {named}. Every scenario will be run as {reached}." diff --git a/src/fi/alk/harness/authoring_entrypoint.py b/src/fi/alk/harness/authoring_entrypoint.py index ed23fe9a1..402e82ce1 100644 --- a/src/fi/alk/harness/authoring_entrypoint.py +++ b/src/fi/alk/harness/authoring_entrypoint.py @@ -22,7 +22,9 @@ import sys import tempfile +from .agent_type_check import EXIT_REFUSED, failure_message from .cli import _auto +from .failure_file import FAILURE_FILE from .job import HarnessJob, ProviderExecutionMode, SourceKind from .hosted_compose_bootstrap import initialize_hosted_compose from .provider_import import inspect_provider_target @@ -552,6 +554,10 @@ def _await_harness(output: Path) -> int: finished = output / ".futureagi-build-finished.json" deadline = time.monotonic() + int(os.environ.get("ALK_HOSTED_AUTHORING_MAX_DURATION_SECONDS") or 3300) while time.monotonic() < deadline: + # The failure wins: a build that was refused and also finished is still refused. + if (output / FAILURE_FILE).is_file(): + print(failure_message(output), file=sys.stderr) + return EXIT_REFUSED if finished.is_file(): return 0 time.sleep(5) diff --git a/src/fi/alk/harness/chat.py b/src/fi/alk/harness/chat.py index 61cd61480..06424a9db 100644 --- a/src/fi/alk/harness/chat.py +++ b/src/fi/alk/harness/chat.py @@ -24,6 +24,7 @@ from .backends import SessionSpec from .config import artifact_dir from .contract import AgentContract +from .failure_file import FAILURE_FILE from .job import HarnessJob from .run import stage as run_stage from .session import Stage @@ -152,6 +153,9 @@ async def _open(self, stage_name: str) -> str: "cannot re-read the agent without knowing where it lives" ) if stage_name == UNDERSTAND: + # A person's own new reading starts clean. + if self.out: + (self.out / FAILURE_FILE).unlink(missing_ok=True) self.stage, _ = understand_stage.open_stage( self.source, out=self.out, ask=self.ask ) diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index aae907583..f117cef1e 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -23,6 +23,7 @@ from .build import open_stage as build_stage from .build import opening as build_opening from .build import require_buildable +from .agent_type_check import EXIT_REFUSED, failure_message from .chat import open_conversation from .config import ( artifact_dir, @@ -30,6 +31,7 @@ credentials_hint, permission_gate, ) +from .failure_file import FAILURE_FILE from .run.targets import supported as target_kinds from .scenarios import load as load_written from .scenarios import open_stage as scenario_stage @@ -265,6 +267,10 @@ def after_world(source_root: Any, destination: Path, contract: Any) -> Any: return export_session_bundle(source_root, destination, name=f"{contract.agent}-environment") +def _report_failure_file(output: Path) -> None: + print(f"\n{failure_message(output)}\n{output / FAILURE_FILE}", file=sys.stderr) + + async def _understand(args: argparse.Namespace) -> int: if args.kind == "provider": source = resolve( @@ -299,17 +305,25 @@ async def _understand(args: argparse.Namespace) -> int: print(f"model: {chosen_model()}") print(f"out: {destination}\n") + # A person's own new reading starts clean; a job's failure stands for the platform to read. + if job is None: + (destination / FAILURE_FILE).unlink(missing_ok=True) + await _converse( stage, opening(source) + _guidance(args), interactive=args.interactive, - until=lambda: load(destination) is not None, + until=lambda: load(destination) is not None + or (destination / FAILURE_FILE).is_file(), nudge=( "Nothing was saved: you finished without calling submit_contract. Call it now " "with the contract you worked out." ), ) + if (destination / FAILURE_FILE).is_file(): + _report_failure_file(destination) + return EXIT_REFUSED contract = load(destination) if contract is None: print("\nNo contract was submitted.", file=sys.stderr) @@ -1481,6 +1495,9 @@ async def _chat(args: argparse.Namespace) -> int: await conversation.say(said, on_event=_render_observable) await conversation.close() print(f"\nspent: ${conversation.spent_usd:.4f}") + if conversation.out and (conversation.out / FAILURE_FILE).is_file(): + _report_failure_file(conversation.out) + return EXIT_REFUSED return 0 diff --git a/src/fi/alk/harness/failure_file.py b/src/fi/alk/harness/failure_file.py new file mode 100644 index 000000000..ff7ae789e --- /dev/null +++ b/src/fi/alk/harness/failure_file.py @@ -0,0 +1,32 @@ +"""The failure a build leaves for the platform before it exits non-zero.""" + +from __future__ import annotations + +import json +import os +import tempfile +from pathlib import Path + +from .job import HarnessFailure + +FAILURE_FILE = "failure.json" + + +def write_failure_file(failure: HarnessFailure, output: Path) -> Path: + """Write ``failure`` whole, in one step. The first failure of a build stands.""" + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + path = output / FAILURE_FILE + with tempfile.NamedTemporaryFile( + "w", dir=output, prefix=".failure-", suffix=".tmp", encoding="utf-8", delete=False + ) as temporary: + json.dump(failure.model_dump(mode="json"), temporary, indent=2, sort_keys=True) + temporary.write("\n") + try: + os.chmod(temporary.name, 0o644) + os.link(temporary.name, path) + except FileExistsError: + pass + finally: + os.unlink(temporary.name) + return path diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 509b6ed5e..4e5ee5e34 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -24,6 +24,7 @@ tool_server, ) from .authoring_entrypoint import _load_provider_import_profile, agent_source_kind +from .failure_file import FAILURE_FILE from .understand import PROVIDER_IMPORT_PROFILE_PATH_ENV from .chat_policy import EffectiveChatPolicy, apply_chat_policy, load_chat_policy from .hosted_conversation import ( @@ -1030,6 +1031,14 @@ def _control_only(self) -> bool: capabilities = self.client.turn_context.get("capabilities") or {} return bool(capabilities.get("control_only")) + def _build_is_over(self) -> bool: + """A control-only build conversation has nothing left to build: it was refused, or its job ended.""" + if not self._control_only(): + return False + # Not "completed": the post-build check is a platform command on a completed job. + status = self.client.turn_context.get("status") or {} + return status.get("state") in {"failed", "canceled"} or (self.workspace / FAILURE_FILE).is_file() + def _side_chat(self) -> bool: capabilities = self.client.turn_context.get("capabilities") or {} return self._control_only() and capabilities.get("builder_session") is False @@ -1193,6 +1202,8 @@ async def _carry_on(self) -> bool: finish_build: when it has saved something that opens more tools, it is reopened with them and resumes; otherwise it is told the build is not finished. Waits on the person while a question is open, and gives up after a few turns that change nothing.""" + if self._build_is_over(): + return False if not self._control_only() or self._side_chat() or (self.workspace / _BUILD_FINISHED).is_file(): return False if (self.workspace / _PENDING_QUESTION).is_file(): @@ -1368,6 +1379,16 @@ async def _handle_command(self, command: dict[str, Any]) -> None: acknowledge_through=sequence, ) return + # A restart can redeliver the build command to a fresh sandbox after its job has ended. + if self._from_platform and self._build_is_over(): + await self.client.emit( + "turn_completed", + stage=self.conversation.stage_name, + payload={"command_message_id": message_id, "ignored": "the build is over"}, + acknowledge_through=sequence, + ) + self._journal_unsuccessful(journal, message_id) + return resume_invocation_id: str | None = None if state == "started": resume_invocation_id = ( diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index 27a955e1b..7ddbf297a 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -114,8 +114,9 @@ Find, in roughly this order: `system_prompt_from`, every part it is assembled from: it is copied into the contract as written, and every later stage decides what the agent should do from it. -7. **The modality.** How a person reaches this agent: a voice session, a text interface, or a - browser it drives. This decides how it is later run, so getting it wrong reroutes every test. +7. **The modality.** How this agent is run: a voice session, a text conversation, or work it + carries out on websites or a computer. This decides how it is later run, so getting it wrong + reroutes every test. **Decide it from the source and always record it.** A run is unattended, so there is nobody to ask, and a modality left unsaid is read as chat: a voice agent then never places a call and the whole run tests nothing. Read it off what the agent depends on: @@ -124,7 +125,12 @@ Find, in roughly this order: name, an STT or TTS provider, an audio session it enters on connect. - **chat** if a person reaches it as text: an HTTP endpoint taking messages, a completions-shaped API, a socket carrying turns. - - **browser** if it drives a page rather than being talked to. + - **browser** if a person uses it to carry out work on websites or a computer rather than for + a conversation: driving a browser, operating a computer or crawling sites is its main job. + Ask: does a person come to this agent for a conversation, or to have it carry out that work? + An agent that looks things up on the web to answer a person (research, comparison, support) + is chat, or voice if it takes calls, even if it can't work without that tool. So is one with a + web page in front of it. A coding agent is chat. When an agent genuinely ships more than one runtime, take the one its entrypoint starts, and say in `notes` which others exist and that you chose by entrypoint. Never leave the field out. @@ -307,8 +313,8 @@ directory. **List what you found and ask which one is being tested.** Do not pic contract for the wrong one wastes every stage after it, and the person who pointed you here knows which they meant. -**One agent with several runtimes.** The same tools reachable over voice, over chat, or through a -browser. That is one agent, and what to ask about is the modality, not which agent. +**One agent with several runtimes.** The same tools reachable over voice or over chat. That is one +agent, and what to ask about is the modality, not which agent. How to tell them apart: look for repeated structure. Several directories that each define their own set of tools, their own instructions and their own data are several agents. Several entry points diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index a29f9f60e..5482f32b4 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -7,6 +7,7 @@ from __future__ import annotations +import asyncio import json from pathlib import Path from typing import Any @@ -16,6 +17,14 @@ CompletedToolRows, ) +from .agent_type_check import ( + agent_type_failure, + browser_tools_found, + decide, + failure_message, + is_browser, + modality_text, +) from .backends import qualified as qualified # noqa: F401 (re-export; callers import it here) from .backends import tool, tool_server @@ -34,6 +43,7 @@ generated_chat_interface_problem, validate_contract, ) +from .failure_file import FAILURE_FILE, write_failure_file from .instrumentation_evidence import detect_instrumentation CONTRACT_SERVER = "contract" @@ -237,11 +247,35 @@ def _without_nulls(value: Any) -> Any: MOST_CHOSEN_EVALS = 6 +def _stop_message(destination: Path) -> str: + said = failure_message(destination) or "This build can't go on." + return f"{said} Don't submit again or do more work; tell the person." + + +def _second_look(claimed: str, found: list[str]) -> str: + if is_browser(claimed): + return ( + "second-look: you recorded modality browser, which stops this build as not " + "supported, and no browser, crawling or computer-use tool was found in the source. " + "Does a person come to this agent for a conversation, or to have it carry out work " + "on websites or a computer? For a conversation, submit again as chat or voice. If " + "you're sure, submit again as browser. Decide this from the source; don't ask anyone." + ) + return ( + f"second-look: you recorded modality {claimed or 'chat'}, and the source uses " + f"{', '.join(found)}. If that's a helper and a person comes to this agent for a " + "conversation, submit again as is. If its main job is carrying out work on websites or " + "a computer, submit again as browser, which stops this build as not supported. Decide " + "this from the source; don't ask anyone." + ) + + def contract_tools( destination: Path, available_evals: list[dict[str, Any]] | None = None, *, source_root: Path | None = None, + scan_root: Path | None = None, require_setup_contract: bool = False, ) -> Any: """A server exposing ``submit_contract``, writing to ``destination`` on acceptance. @@ -272,6 +306,9 @@ def contract_tools( # A gate with no way through would permanently block the rare agent that genuinely lacks it, # and this stage cannot tell those two apart from the outside. nudged: set[str] = set() + refused = False + looked_again = False + found: list[str] | None = None @tool( "submit_contract", @@ -305,9 +342,10 @@ def contract_tools( "modality": { "type": "string", "enum": list(known_modalities()), - "description": "How a person reaches it, read from its runtime. A voice " - "session (LiveKit, telephony, TTS/STT) is voice; a text interface is chat; " - "a browser-driving agent is browser. This decides how it is later run.", + "description": "How this agent is run, read from its runtime. A voice " + "session (LiveKit, telephony, TTS/STT) is voice; a text conversation is chat; " + "an agent a person uses to carry out work on websites or a computer is " + "browser. This decides how it is later run.", }, "call_direction": { "type": "string", @@ -1049,6 +1087,31 @@ def contract_tools( ) async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: payload = unwrapped(args) + nonlocal refused, looked_again, found + if refused or (destination / FAILURE_FILE).is_file(): + return _ok(_stop_message(destination)) + if found is None: + found = await asyncio.to_thread(browser_tools_found, scan_root) + # A call running alongside this one may have refused while this one waited. + if refused or (destination / FAILURE_FILE).is_file(): + return _ok(_stop_message(destination)) + claimed = modality_text(payload.get("modality")) + if claimed and not is_browser(claimed) and claimed not in known_modalities(): + return _problems( + [ + f"{claimed!r} is not a modality. Use voice or chat, or browser if its main " + "job is work on websites or a computer." + ] + ) + verdict = decide(claimed, found, looked_again=looked_again) + if verdict == "refuse": + # Set first: if the write fails, the model must not get a resubmission through. + refused = True + write_failure_file(agent_type_failure(found), destination) + return _ok(_stop_message(destination)) + if verdict == "second_look": + looked_again = True + return _problems([_second_look(claimed, found)]) data_store = payload.get("data_store") if not isinstance(data_store, dict): data_store = {} diff --git a/src/fi/alk/harness/understand.py b/src/fi/alk/harness/understand.py index 4bd405532..f131e532b 100644 --- a/src/fi/alk/harness/understand.py +++ b/src/fi/alk/harness/understand.py @@ -136,6 +136,7 @@ def open_stage( destination, available_evals, source_root=source.workdir() if source.kind == "repo" else None, + scan_root=source.workdir() if source.kind in {"repo", "github"} else None, require_setup_contract=require_setup_contract and source.kind == "repo", ), diff --git a/src/fi/alk/harness/world/tools.py b/src/fi/alk/harness/world/tools.py index 6cc167e29..50357914b 100644 --- a/src/fi/alk/harness/world/tools.py +++ b/src/fi/alk/harness/world/tools.py @@ -1234,7 +1234,7 @@ async def add_rule_tool(args: dict[str, Any]) -> dict[str, Any]: @tool( "set_modality", - "Correct how a person actually reaches this agent: voice, chat or browser. Modality " + "Correct how a person actually reaches this agent: voice or chat. Modality " "picks the world, the simulated person and the transport, so a wrong one does not weaken " "a run, it runs a different test. Use it when the operator says where the agent is " "deployed and the contract disagrees: an agent's code reads the same answering a chat " diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py new file mode 100644 index 000000000..b4045b9ae --- /dev/null +++ b/tests/harness/test_agent_type_refusal.py @@ -0,0 +1,1172 @@ +"""An agent whose main job isn't a conversation stops its build with a failure that says so.""" + +from __future__ import annotations + +import asyncio +import json +import os +import stat +import threading +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from fi.alk.harness import ( + amend, + authoring_entrypoint, + chat, + cli, + tools, +) +from fi.alk.harness import hosted_authoring_entrypoint as hosted_authoring +from fi.alk.harness.agent_type_check import ( + agent_type_failure, + browser_tools_found, + decide, + failure_message, + is_browser, + modality_text, +) +from fi.alk.harness.contract import AgentContract, known_modalities +from fi.alk.harness.failure_file import FAILURE_FILE, write_failure_file +from fi.alk.harness.hosted_chat_entrypoint import HostedChatRuntime +from fi.alk.harness.job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage +from fi.alk.harness.sources import GitHubSource, RepoSource +from fi.alk.harness.tools import CONTRACT_SERVER, contract_tools +from fi.alk.harness.understand import load, open_stage + +_FOUND_MESSAGE = ( + "This agent's main job is driving a browser, operating a computer or crawling sites " + "(found: playwright). Only voice and chat agents are supported for now." +) +_READ_MESSAGE = ( + "This agent's main job was read as driving a browser, operating a computer or crawling " + "sites. Only voice and chat agents are supported for now." +) +_FINISHED = ".futureagi-build-finished.json" + + +@pytest.fixture(autouse=True) +def _nothing_reaches_a_network(monkeypatch): + for name in ("FI_BASE_URL", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"): + monkeypatch.setenv(name, "http://127.0.0.1:9") + monkeypatch.delenv("FI_API_KEY", raising=False) + monkeypatch.delenv("FI_SECRET_KEY", raising=False) + + +def _tree(root: Path, files: dict[str, str]) -> Path: + for name, body in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + return root + + +@pytest.fixture +def browser_repo(tmp_path) -> Path: + return _tree(tmp_path / "source", {"agent.py": "from playwright.sync_api import sync_playwright\n"}) + + +@pytest.fixture +def plain_repo(tmp_path) -> Path: + return _tree(tmp_path / "source", {"agent.py": "import openai\n"}) + + +# Complete enough that no nudge fires, so the first answer is about the modality alone. +def _reading(modality="chat", **fields): + payload = { + "agent": "fixture", + "one_liner": "Answers questions.", + "real_use_cases": ["Answer a question"], + "tools": [{"name": "lookup", "args": ["query"]}], + "hard_constraints": ["Stay polite"], + "system_prompt_excerpt": "You answer questions.", + **fields, + } + if modality is not None: + payload["modality"] = modality + return payload + + +def _submit_tool(server): + return next(one for one in server.tools if one.name == "submit_contract") + + +def _submitter(destination: Path, scan_root: Path | None = None): + tool = _submit_tool(contract_tools(destination, scan_root=scan_root)) + return lambda payload: asyncio.run(tool.handler(payload)) + + +def _text(answer) -> str: + return answer["content"][0]["text"] + + +def _failure(destination: Path) -> dict: + return json.loads((destination / FAILURE_FILE).read_text(encoding="utf-8")) + + +def _a_failure() -> HarnessFailure: + return HarnessFailure( + domain=FailureDomain.AGENT, + stage=HarnessStage.UNDERSTANDING_AGENT, + code="first", + message="the first failure", + retryable=False, + details={"key": "value"}, + ) + + +# The writer + + +def test_writer_creates_folder(tmp_path): + output = tmp_path / "gone" / "authoring" + assert write_failure_file(_a_failure(), output) == output / FAILURE_FILE + assert (output / FAILURE_FILE).is_file() + + +def test_writer_writes_full_shape(tmp_path): + write_failure_file(_a_failure(), tmp_path) + assert _failure(tmp_path) == { + "domain": "agent", + "stage": "understanding_agent", + "code": "first", + "message": "the first failure", + "retryable": False, + "details": {"key": "value"}, + } + + +def test_first_failure_stands(tmp_path): + write_failure_file(_a_failure(), tmp_path) + later = _a_failure().model_copy(update={"code": "later"}) + write_failure_file(later, tmp_path) + assert _failure(tmp_path)["code"] == "first" + + +def test_file_is_world_readable(tmp_path): + path = write_failure_file(_a_failure(), tmp_path) + assert stat.S_IMODE(path.stat().st_mode) == 0o644 + + +def test_writer_leaves_no_temp_file(tmp_path): + write_failure_file(_a_failure(), tmp_path) + write_failure_file(_a_failure(), tmp_path) + assert [path.name for path in tmp_path.iterdir()] == [FAILURE_FILE] + + +def test_failure_message_is_empty_when_the_file_cannot_be_read(tmp_path): + assert failure_message(tmp_path) == "" + for body in ("{", "[1]", '{"message": 3}'): + (tmp_path / FAILURE_FILE).write_text(body, encoding="utf-8") + assert failure_message(tmp_path) == "" + + +# The code check + + +@pytest.mark.parametrize( + "files, expected", + [ + ({"agent.py": "from playwright.sync_api import sync_playwright\n"}, ["playwright"]), + ({"agent.py": "import browser_use\n"}, ["browser-use"]), + ( + {"package.json": json.dumps({"dependencies": {"@playwright/mcp": "1"}})}, + ["@playwright/mcp"], + ), + ({"agent.py": 'TOOL = {"type": "computer_20241022"}\n'}, ["computer-use tool"]), + ({"agent.ts": 'const tool = {type: "computer_use_preview"};\n'}, ["computer-use tool"]), + ( + {"agent.py": 'from google.genai import types\nTOOL = types.ComputerUse(environment="x")\n'}, + ["computer-use tool"], + ), + ({"agent.py": 'ARGS = ["@playwright/mcp@latest"]\n'}, ["Playwright MCP"]), + ( + {"agent.py": "from langchain_community.agent_toolkits import PlayWrightBrowserToolkit\n"}, + ["PlayWrightBrowserToolkit"], + ), + # A string holding "//" isn't a comment, so the code after it still counts. + ( + {"agent.js": 'const home = "https://example.test"; const t = "computer_20241022";\n'}, + ["computer-use tool"], + ), + ], +) +def test_browser_tools_found_finds_what_the_agent_uses(tmp_path, files, expected): + assert browser_tools_found(_tree(tmp_path, files)) == expected + + +def test_browser_tools_found_is_sorted_and_names_each_tool_once(tmp_path): + _tree( + tmp_path, + { + "agent.py": "import browser_use\nimport selenium\n", + "web/package.json": json.dumps({"dependencies": {"browser-use": "1", "crawlee": "1"}}), + }, + ) + assert browser_tools_found(tmp_path) == ["browser-use", "crawlee", "selenium"] + + +@pytest.mark.parametrize( + "folder", ["e2eTests", "testdata", "unittests", "integrationtests", "Test_A"] +) +def test_tools_found_skips_these_test_folders(tmp_path, folder): + (tmp_path / folder).mkdir() + (tmp_path / folder / "a.py").write_text("import playwright\n", encoding="utf-8") + assert browser_tools_found(tmp_path) == [] + + +def test_tools_found_skips_an_upper_case_test_file(tmp_path): + (tmp_path / "Test_A.py").write_text("import playwright\n", encoding="utf-8") + assert browser_tools_found(tmp_path) == [] + + +def test_tools_found_skips_a_vendored_package_json(tmp_path): + folder = tmp_path / "node_modules" / "lib" + folder.mkdir(parents=True) + (folder / "package.json").write_text( + json.dumps({"dependencies": {"playwright": "1"}}), encoding="utf-8" + ) + assert browser_tools_found(tmp_path) == [] + + +@pytest.mark.parametrize( + "body", + [ + "`" + "a\\`" * 40_000, + "'" + "a\\'" * 40_000, + "`" + "a\\`" * 40_000 + "\\", + '"' + 'a\\"' * 40_000, + "'" + "a\\'" * 40_000 + "\\", + "'" + "a\\'" * 40_000 + "\n", + ], + ids=["template", "string", "ends-in-a-backslash", "double", "single-backslash", "single-newline"], +) +def test_an_odd_script_file_stays_quick(tmp_path, body): + (tmp_path / "a.js").write_text(body, encoding="utf-8") + started = time.monotonic() + browser_tools_found(tmp_path) + assert time.monotonic() - started < 5 + + +@pytest.mark.parametrize("folder", ["latest", "contest"]) +def test_tools_found_reads_a_folder_with_test_inside_a_word(tmp_path, folder): + (tmp_path / folder).mkdir() + (tmp_path / folder / "a.py").write_text("import playwright\n", encoding="utf-8") + assert browser_tools_found(tmp_path) == ["playwright"] + + +@pytest.mark.parametrize( + "name, body", + [ + ("node_modules/lib/a.py", "import playwright\n"), + ("venv/lib/a.py", "import playwright\n"), + ("tests/a.py", "import playwright\n"), + ("e2e_tests/a.py", "import playwright\n"), + ("Tests/a.py", "import playwright\n"), + ("docs/a.py", "import playwright\n"), + ("examples/a.py", "import playwright\n"), + ("vendor/a.py", "import playwright\n"), + ("evals/a.ts", 'const tool = "computer_20241022";\n'), + (".cursor/a.py", 'ARGS = ["@playwright/mcp"]\n'), + (".cursor/mcp.json", '{"args": ["@playwright/mcp"]}\n'), + (".mcp.json", '{"args": ["@playwright/mcp"]}\n'), + ("conftest.py", "import playwright\n"), + ("test_e2e.py", "import playwright\n"), + ("app_test.py", "import playwright\n"), + ("app.spec.ts", 'const tool = "computer_20241022";\n'), + ("app.test.js", 'const tool = "computer_20241022";\n'), + ("pnpm-lock.yaml", "playwright: 1\n@playwright/mcp: 1\n"), + ("requirements.txt", "playwright\n"), + ("package.json", json.dumps({"devDependencies": {"playwright": "1"}})), + ("tests/package.json", json.dumps({"dependencies": {"playwright": "1"}})), + ("a.py", "# unlike computer_20241022, this only talks\n"), + ("a.ts", '// see @playwright/mcp\n/* computer_use\n and ComputerUse */\nconst a = 1;\n'), + ], +) +def test_browser_tools_found_ignores_what_is_not_the_agent(tmp_path, name, body): + _tree(tmp_path, {"agent.py": "import openai\n", name: body}) + assert browser_tools_found(tmp_path) == [] + + +def test_unpacked_distributions_are_not_the_agent(tmp_path): + _tree( + tmp_path, + { + "lib/anthropic-1.0.dist-info/METADATA": "x\n", + "lib/anthropic/types.py": 'NAME = "computer_20241022"\n', + }, + ) + assert browser_tools_found(tmp_path) == [] + + +def test_a_customers_other_package_names_never_appear(tmp_path): + _tree( + tmp_path, + {"package.json": json.dumps({"dependencies": {"acme-private-sdk": "1", "playwright": "1"}})}, + ) + assert browser_tools_found(tmp_path) == ["playwright"] + + +@pytest.mark.parametrize( + "name, body", + [ + ("package.json", "{"), + ("package.json", '["playwright"]'), + ("package.json", json.dumps({"dependencies": ["playwright"]})), + ("broken.py", 'TOOL = ("computer_20241022"\n'), + ], +) +def test_malformed_files_are_skipped(tmp_path, name, body): + _tree(tmp_path, {name: body}) + assert browser_tools_found(tmp_path) == [] + + +def test_undecodable_and_oversized_files_are_skipped(tmp_path): + (tmp_path / "binary.py").write_bytes(b'NAME = "computer_20241022"\xff\n') + (tmp_path / "huge.js").write_text( + 'const tool = "computer_20241022";\n' + "x" * (1024 * 1024), encoding="utf-8" + ) + assert browser_tools_found(tmp_path) == [] + + +def test_no_source_finds_nothing(): + assert browser_tools_found(None) == [] + + +@pytest.mark.parametrize( + "body", + [ + "x = " + "-" * 200_000 + "1\n", + "x = " + "not " * 100_000 + "1\n", + "x = " + "lambda: " * 50_000 + "1\n", + "x = 1" + "+1" * 100_000 + "\n", + ], + ids=["unary", "not", "lambda", "sum"], +) +def test_a_file_the_parser_cannot_hold_costs_only_that_file(tmp_path, body): + (tmp_path / "odd.py").write_text(body, encoding="utf-8") + (tmp_path / "agent.py").write_text("import playwright\n", encoding="utf-8") + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_a_package_of_the_same_name_at_the_top_is_the_agents_own(tmp_path): + (tmp_path / "steel").mkdir() + (tmp_path / "steel" / "__init__.py").write_text("x = 1\n", encoding="utf-8") + (tmp_path / "agent.py").write_text("import steel\n", encoding="utf-8") + assert browser_tools_found(tmp_path) == [] + + +def test_a_pipe_named_like_code_is_left_alone(tmp_path): + os.mkfifo(tmp_path / "x.py") + (tmp_path / "agent.py").write_text("import playwright\n", encoding="utf-8") + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_a_module_of_the_same_name_at_the_top_is_the_agents_own(tmp_path): + (tmp_path / "scrapy.py").write_text("x = 1\n", encoding="utf-8") + (tmp_path / "agent.py").write_text("import scrapy\n", encoding="utf-8") + assert browser_tools_found(tmp_path) == [] + + +def test_an_oversized_python_file_is_left_out(tmp_path): + (tmp_path / "big.py").write_text("import playwright\n" + "x = 1\n" * 200_000, encoding="utf-8") + assert browser_tools_found(tmp_path) == [] + + +# The decision + + +@pytest.mark.parametrize( + "claimed, found, looked_again, expected", + [ + ("browser", ["playwright"], False, "refuse"), + ("browser", ["playwright"], True, "refuse"), + ("browser", [], False, "second_look"), + ("browser", [], True, "refuse"), + ("chat", ["playwright"], False, "second_look"), + ("chat", ["playwright"], True, "keep"), + ("chat", [], False, "keep"), + ("chat", [], True, "keep"), + ("voice", ["playwright"], False, "second_look"), + ("voice", ["playwright"], True, "keep"), + (None, ["playwright"], False, "second_look"), + (None, [], False, "keep"), + ], +) +def test_decide_rows(claimed, found, looked_again, expected): + assert decide(claimed, found, looked_again=looked_again) == expected + + +@pytest.mark.parametrize( + "claimed", + [" Browser ", ["browser"], ["browser", "chat"], "computer_use", "cua", "COMPUTER_USE"], +) +def test_decide_normalises_every_form_of_browser(claimed): + assert decide(claimed, ["playwright"], looked_again=False) == "refuse" + assert decide(claimed, [], looked_again=False) == "second_look" + + +def test_modality_text_and_is_browser(): + assert modality_text(None) == "" + assert modality_text(["Browser", "chat"]) == "browser\nchat" + assert is_browser("browser\nchat") + assert not is_browser("chat") + assert not is_browser("browsers") + + +def test_unknown_modality_gets_a_problem_not_a_save(tmp_path, plain_repo): + submit = _submitter(tmp_path / "out", plain_repo) + answer = submit(_reading("web")) + assert answer["is_error"] + assert "'web' is not a modality" in _text(answer) + assert load(tmp_path / "out") is None + assert not (tmp_path / "out" / FAILURE_FILE).exists() + + +# Browser reading, tool found + + +def test_the_scan_runs_off_the_event_loop(tmp_path, monkeypatch): + threads = [] + + def scan(_root): + threads.append(threading.current_thread()) + return [] + + monkeypatch.setattr(tools, "browser_tools_found", scan) + _submitter(tmp_path / "out", tmp_path)(_reading("chat")) + assert threads and threads[0] is not threading.main_thread() + + +def test_a_refusal_during_the_scan_stops_this_call_too(tmp_path, monkeypatch): + output = tmp_path / "out" + + def scan(_root): + write_failure_file(agent_type_failure(["playwright"]), output) + return [] + + monkeypatch.setattr(tools, "browser_tools_found", scan) + _submitter(output, tmp_path)(_reading("chat")) + assert not (output / "contract.json").exists() + + +def test_a_build_that_times_out_is_still_a_crash(tmp_path, monkeypatch): + monkeypatch.setenv("ALK_HOSTED_AUTHORING_MAX_DURATION_SECONDS", "0") + # `or 3300` keeps a zero from meaning "no time", so step the clock past the deadline instead. + clock = iter([0.0, 10_000.0]) + monkeypatch.setattr(authoring_entrypoint.time, "monotonic", lambda: next(clock)) + assert authoring_entrypoint._await_harness(tmp_path) == 1 + + +def test_ci_runs_these_tests(): + workflows = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "sdk-smoke.yml" + assert "tests/harness/test_agent_type_refusal.py" in workflows.read_text(encoding="utf-8") + + +def test_the_file_is_what_the_platform_reads(): + assert FAILURE_FILE == "failure.json" + + +@pytest.mark.parametrize("modality", ["browser", ["browser"]]) +def test_a_wrapped_browser_reading_is_refused(tmp_path, browser_repo, modality): + output = tmp_path / "out" + _submitter(output, browser_repo)({"contract": _reading(modality)}) + assert _failure(output)["code"] == "agent_type_unsupported" + assert not (output / "contract.json").exists() + + +def test_free_text_mentioning_browser_gets_a_problem_not_a_block(tmp_path, browser_repo): + output = tmp_path / "out" + answer = _submitter(output, browser_repo)({"contract": _reading("chat via browser")}) + assert answer.get("is_error") + assert "is not a modality" in _text(answer) + assert not (output / FAILURE_FILE).exists() + + +def test_browser_with_tool_is_refused_on_the_first_submission(tmp_path, browser_repo): + output = tmp_path / "out" + answer = _submitter(output, browser_repo)(_reading("browser")) + assert not answer.get("is_error") + assert _FOUND_MESSAGE in _text(answer) + assert "Don't submit again or do more work; tell the person." in _text(answer) + assert _failure(output) == { + "domain": "agent", + "stage": "understanding_agent", + "code": "agent_type_unsupported", + "message": _FOUND_MESSAGE, + "retryable": False, + "details": {"agent_type": "browser", "tools": ["playwright"]}, + } + assert not (output / "contract.json").exists() + + +# After a refusal + + +def test_chat_resubmission_after_a_refusal_saves_nothing(tmp_path, browser_repo): + output = tmp_path / "out" + submit = _submitter(output, browser_repo) + submit(_reading("browser")) + answer = submit(_reading("chat")) + assert not answer.get("is_error") + assert _FOUND_MESSAGE in _text(answer) + assert not (output / "contract.json").exists() + + +def test_stop_from_a_fresh_conversation_with_the_file(tmp_path, plain_repo): + output = tmp_path / "out" + write_failure_file(_a_failure(), output) + answer = _submitter(output, plain_repo)(_reading("chat")) + assert not answer.get("is_error") + assert "the first failure" in _text(answer) + assert "Don't submit again" in _text(answer) + assert not (output / "contract.json").exists() + assert _failure(output)["code"] == "first" + + +def test_stop_message_copes_with_an_unreadable_file(tmp_path, plain_repo): + output = tmp_path / "out" + output.mkdir() + (output / FAILURE_FILE).write_text("{", encoding="utf-8") + answer = _submitter(output, plain_repo)(_reading("chat")) + assert _text(answer).startswith("This build can't go on.") + + +# The write fails + + +def _failing_write(monkeypatch): + def refuse_to_write(failure, output): + raise OSError("disk full") + + monkeypatch.setattr(tools, "write_failure_file", refuse_to_write) + + +def test_writer_failure_blocks_resubmission(tmp_path, browser_repo, monkeypatch): + output = tmp_path / "out" + submit = _submitter(output, browser_repo) + _failing_write(monkeypatch) + with pytest.raises(OSError, match="disk full"): + submit(_reading("browser")) + assert not (output / FAILURE_FILE).exists() + answer = submit(_reading("chat")) + assert not answer.get("is_error") + assert "This build can't go on." in _text(answer) + assert not (output / "contract.json").exists() + + +def test_writer_failure_reaches_the_model_as_a_tool_error(tmp_path, browser_repo, monkeypatch): + from mcp import types + + from fi.alk.harness.backends.claude import _sdk_server + + output = tmp_path / "out" + instance = _sdk_server(contract_tools(output, scan_root=browser_repo))["instance"] + handler = instance.request_handlers[types.CallToolRequest] + _failing_write(monkeypatch) + + async def call(): + return await handler( + types.CallToolRequest( + method="tools/call", + params=types.CallToolRequestParams( + name="submit_contract", arguments=_reading("browser") + ), + ) + ) + + result = asyncio.run(call()).root + assert result.isError + assert "disk full" in result.content[0].text + assert not (output / FAILURE_FILE).exists() + + +# Browser reading, no tool found + + +def test_browser_with_no_tool_gets_a_second_look(tmp_path, plain_repo): + output = tmp_path / "out" + answer = _submitter(output, plain_repo)(_reading("browser")) + assert answer["is_error"] + assert "second-look: you recorded modality browser" in _text(answer) + assert "Does a person come to this agent for a conversation" in _text(answer) + assert "don't ask anyone" in _text(answer) + assert not (output / FAILURE_FILE).exists() + assert not (output / "contract.json").exists() + + +def test_second_look_then_chat_saves_as_chat(tmp_path, plain_repo): + output = tmp_path / "out" + submit = _submitter(output, plain_repo) + submit(_reading("browser")) + answer = submit(_reading("chat")) + assert not answer.get("is_error") + assert load(output).modality == "chat" + assert not (output / FAILURE_FILE).exists() + + +def test_second_look_then_browser_is_refused_with_no_tools(tmp_path, plain_repo): + output = tmp_path / "out" + submit = _submitter(output, plain_repo) + submit(_reading("browser")) + answer = submit(_reading("browser")) + assert not answer.get("is_error") + assert _READ_MESSAGE in _text(answer) + assert _failure(output)["details"] == {"agent_type": "browser", "tools": []} + assert not (output / "contract.json").exists() + + +def test_no_scan_root_behaves_as_no_tool(tmp_path): + output = tmp_path / "out" + submit = _submitter(output, None) + assert "second-look" in _text(submit(_reading("browser"))) + assert not (output / FAILURE_FILE).exists() + submit(_reading("browser")) + assert _failure(output)["details"]["tools"] == [] + + +# Chat or voice reading, tool found + + +@pytest.mark.parametrize("modality", ["chat", "voice"]) +def test_conversation_with_tool_gets_a_second_look_naming_it(tmp_path, browser_repo, modality): + output = tmp_path / "out" + answer = _submitter(output, browser_repo)(_reading(modality)) + assert answer["is_error"] + assert f"second-look: you recorded modality {modality}, and the source uses playwright." in _text(answer) + assert not (output / "contract.json").exists() + assert not (output / FAILURE_FILE).exists() + + +def test_empty_modality_reads_as_chat_in_the_second_look(tmp_path, browser_repo): + answer = _submitter(tmp_path / "out", browser_repo)(_reading(None)) + assert "you recorded modality chat, and the source uses playwright" in _text(answer) + + +@pytest.mark.parametrize("modality", ["chat", "voice"]) +def test_conversation_with_tool_then_same_answer_saves(tmp_path, browser_repo, modality): + output = tmp_path / "out" + submit = _submitter(output, browser_repo) + submit(_reading(modality)) + assert not submit(_reading(modality)).get("is_error") + assert load(output).modality == modality + assert not (output / FAILURE_FILE).exists() + + +@pytest.mark.parametrize("modality", ["chat", "voice"]) +def test_conversation_with_tool_then_browser_is_refused_naming_it(tmp_path, browser_repo, modality): + output = tmp_path / "out" + submit = _submitter(output, browser_repo) + submit(_reading(modality)) + submit(_reading("browser")) + assert _failure(output)["message"] == _FOUND_MESSAGE + assert _failure(output)["details"]["tools"] == ["playwright"] + assert not (output / "contract.json").exists() + + +# Everything else + + +@pytest.mark.parametrize("modality", ["chat", "voice"]) +def test_conversation_with_no_tool_saves_with_no_second_look(tmp_path, plain_repo, modality): + output = tmp_path / "out" + assert not _submitter(output, plain_repo)(_reading(modality)).get("is_error") + assert load(output).modality == modality + + +def test_only_one_second_look_and_one_scan(tmp_path, browser_repo, monkeypatch): + output = tmp_path / "out" + scans = [] + real_scan = tools.browser_tools_found + + def counted(root): + scans.append(root) + return real_scan(root) + + monkeypatch.setattr(tools, "browser_tools_found", counted) + submit = _submitter(output, browser_repo) + assert "second-look" in _text(submit(_reading("chat"))) + # A rejected submission after the look doesn't earn another one. + rejected = submit(_reading("chat", real_use_cases=[])) + assert rejected["is_error"] and "second-look" not in _text(rejected) + assert not submit(_reading("chat")).get("is_error") + assert load(output).modality == "chat" + assert scans == [browser_repo] + + +# Every way into the stage reaches the scan + + +def _server_of(stage): + return stage.spec.servers[CONTRACT_SERVER] + + +def test_both_build_modes_reach_the_scan(tmp_path, browser_repo): + from fi.alk.harness.builder import open_for_building + + source = RepoSource(name="fixture", root=browser_repo) + staged, _ = open_stage(source, out=tmp_path / "staged") + conversational = open_for_building(None, out=tmp_path / "conversational", source=source) + for stage, output in ((staged, "staged"), (conversational, "conversational")): + tool = _submit_tool(_server_of(stage)) + asyncio.run(tool.handler(_reading("browser"))) + assert _failure(tmp_path / output)["details"]["tools"] == ["playwright"] + + +def test_github_source_reaches_the_scan(tmp_path, browser_repo): + source = GitHubSource(name="fixture", root=browser_repo, url="https://github.test/a/b") + stage, _ = open_stage(source, out=tmp_path / "out") + asyncio.run(_submit_tool(_server_of(stage)).handler(_reading("browser"))) + assert _failure(tmp_path / "out")["details"]["tools"] == ["playwright"] + + +# The process waiting on the conversation's build + + +@pytest.fixture +def no_waiting(monkeypatch): + def fail(_): + raise AssertionError("the wait should have ended already") + + monkeypatch.setattr(authoring_entrypoint.time, "sleep", fail) + + +def test_await_harness_stops_on_the_file(tmp_path, no_waiting, capsys): + write_failure_file(agent_type_failure(["playwright"]), tmp_path) + assert authoring_entrypoint._await_harness(tmp_path) == 78 + assert _FOUND_MESSAGE in capsys.readouterr().err + + +def test_await_harness_failure_wins_over_finished(tmp_path, no_waiting): + write_failure_file(agent_type_failure(["playwright"]), tmp_path) + (tmp_path / _FINISHED).write_text("{}", encoding="utf-8") + assert authoring_entrypoint._await_harness(tmp_path) == 78 + + +def test_await_harness_finished_alone_returns_0(tmp_path, no_waiting): + (tmp_path / _FINISHED).write_text("{}", encoding="utf-8") + assert authoring_entrypoint._await_harness(tmp_path) == 0 + + +def test_main_await_harness_with_the_file_never_validates(tmp_path, no_waiting, monkeypatch): + source = tmp_path / "source" + source.mkdir() + output = tmp_path / "output" + write_failure_file(agent_type_failure(["playwright"]), output) + job = HarnessJob( + job_id="refused", + run_id="refused", + execution="local", + source={"kind": "local_repository", "local_path": str(source)}, + agent={"connector": "auto"}, + scenario_count=2, + metadata={"generic_harness_v1": True}, + ) + job_path = tmp_path / "job.json" + job_path.write_text(job.model_dump_json(), encoding="utf-8") + validated = [] + + async def validate(*args, **kwargs): + validated.append(args) + + monkeypatch.setattr(authoring_entrypoint, "_review_while_writing", lambda *args: None) + monkeypatch.setattr( + "fi.alk.harness.authoring_runtime_validation.validate_and_repair", validate + ) + status = authoring_entrypoint.main( + [str(job_path), "--source", str(source), "--output", str(output), "--await-harness"], + validate_runtime=True, + ) + assert status == 78 + assert validated == [] + + +# The unattended and local reading + + +class _Stage: + def __init__(self, on_say=None): + self.said: list[str] = [] + self.on_say = on_say + self.spent_usd = 0.0 + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + return None + + async def say(self, message, on_event=None): + self.said.append(message) + if self.on_say: + self.on_say() + + +def _understand_args(output: Path, **fields) -> SimpleNamespace: + return SimpleNamespace( + kind="repo", name="fixture", path=output, out=output, interactive=False, **fields + ) + + +def _patch_understand(monkeypatch, output: Path, stage) -> None: + monkeypatch.setattr( + cli, "resolve", lambda *args, **kwargs: SimpleNamespace(name="fixture", kind="repo") + ) + monkeypatch.setattr(cli, "open_stage", lambda source, **kwargs: (stage, output)) + monkeypatch.setattr(cli, "opening", lambda _: "") + monkeypatch.setattr(cli, "_guidance", lambda _: "") + + +def test_understand_returns_1_with_the_file_and_no_nudge(tmp_path, monkeypatch, capsys): + stage = _Stage(lambda: write_failure_file(agent_type_failure(["playwright"]), tmp_path)) + _patch_understand(monkeypatch, tmp_path, stage) + args = _understand_args(tmp_path, job=SimpleNamespace(metadata={})) + assert asyncio.run(cli._understand(args)) == 78 + assert len(stage.said) == 1, "a refused reading must not be told nothing was saved" + err = capsys.readouterr().err + assert _FOUND_MESSAGE in err + assert str(tmp_path / FAILURE_FILE) in err + assert "No contract was submitted" not in err + + +def test_understand_still_nudges_when_nothing_happened(tmp_path, monkeypatch, capsys): + stage = _Stage() + _patch_understand(monkeypatch, tmp_path, stage) + args = _understand_args(tmp_path, job=SimpleNamespace(metadata={})) + assert asyncio.run(cli._understand(args)) == 1 + assert len(stage.said) == 2 + assert "No contract was submitted" in capsys.readouterr().err + + +def test_understand_with_no_job_clears_an_old_file_before_reading(tmp_path, monkeypatch): + write_failure_file(_a_failure(), tmp_path) + seen = [] + stage = _Stage(lambda: seen.append((tmp_path / FAILURE_FILE).exists())) + _patch_understand(monkeypatch, tmp_path, stage) + asyncio.run(cli._understand(_understand_args(tmp_path))) + assert seen and seen[0] is False + + +def test_understand_with_a_job_keeps_an_earlier_failure(tmp_path, monkeypatch, capsys): + write_failure_file(_a_failure(), tmp_path) + stage = _Stage() + _patch_understand(monkeypatch, tmp_path, stage) + args = _understand_args(tmp_path, job=SimpleNamespace(metadata={})) + assert asyncio.run(cli._understand(args)) == 78 + assert _failure(tmp_path)["code"] == "first" + assert "the first failure" in capsys.readouterr().err + + +# A person's own chat + + +class _Chat: + def __init__(self, out): + self.out = out + self.stage_name = "understand" + self.spent_usd = 0.0 + + async def start(self, on_event=None): + return None + + async def close(self): + return None + + +def _patch_chat(monkeypatch, output: Path) -> None: + async def nobody_there(_): + raise EOFError + + monkeypatch.setattr(cli, "open_conversation", lambda **kwargs: _Chat(output)) + monkeypatch.setattr(cli, "_prompt", nobody_there) + + +def test_chat_returns_1_and_prints_with_the_file(tmp_path, monkeypatch, capsys): + write_failure_file(agent_type_failure(["playwright"]), tmp_path) + _patch_chat(monkeypatch, tmp_path) + args = SimpleNamespace(name="", path="", kind="repo", out=str(tmp_path)) + assert asyncio.run(cli._chat(args)) == 78 + err = capsys.readouterr().err + assert _FOUND_MESSAGE in err + assert str(tmp_path / FAILURE_FILE) in err + + +def test_chat_returns_0_without_the_file(tmp_path, monkeypatch): + _patch_chat(monkeypatch, tmp_path) + args = SimpleNamespace(name="", path="", kind="repo", out=str(tmp_path)) + assert asyncio.run(cli._chat(args)) == 0 + + +def test_chat_understand_clears_an_old_file(tmp_path, monkeypatch): + write_failure_file(_a_failure(), tmp_path) + seen = [] + + class Opened(Exception): + pass + + def opening_stage(source, **kwargs): + seen.append((tmp_path / FAILURE_FILE).exists()) + raise Opened + + monkeypatch.setattr(chat.understand_stage, "open_stage", opening_stage) + conversation = chat.Conversation( + source=SimpleNamespace(name="fixture"), out=tmp_path, stage_name=chat.UNDERSTAND + ) + with pytest.raises(Opened): + asyncio.run(conversation._open(chat.UNDERSTAND)) + assert seen == [False] + + +# The exit code passes straight up + + +def test_auto_returns_1_when_understand_does_and_later_stages_do_not_run(tmp_path, monkeypatch): + source = tmp_path / "source" + source.mkdir() + output = tmp_path / "output" + later = [] + + async def refused(args): + return 78 + + async def stage_after(args): + later.append("ran") + return 0 + + monkeypatch.setattr(cli, "_understand", refused) + monkeypatch.setattr(cli, "_build", stage_after) + monkeypatch.setattr(cli, "_scenarios", stage_after) + monkeypatch.setattr("fi.alk.harness.provision.stop", lambda _: None) + args = SimpleNamespace( + path=str(source), + out=str(output), + name="fixture", + kind="repo", + count=2, + model=None, + authoring_only=True, + job=HarnessJob( + job_id="refused", + run_id="refused", + execution="local", + source={"kind": "local_repository", "local_path": str(source)}, + agent={"connector": "auto"}, + scenario_count=2, + metadata={"generic_harness_v1": True}, + ), + runtime_readiness=None, + ) + assert asyncio.run(cli._auto(args)) == 78 + assert later == [] + + +def test_hosted_authoring_main_passes_the_refusal_through(tmp_path, monkeypatch): + secrets = tmp_path / "secrets.json" + gateway = tmp_path / "simulator-secrets.json" + secrets.write_text("{}", encoding="utf-8") + gateway.write_text("{}", encoding="utf-8") + monkeypatch.setattr(hosted_authoring, "_SECRETS_PATH", secrets) + monkeypatch.setattr(hosted_authoring, "_SIMULATOR_SECRETS_PATH", gateway) + monkeypatch.setattr(hosted_authoring, "_ADC_PATH", tmp_path / "adc.json") + monkeypatch.setattr(hosted_authoring, "authoring_main", lambda argv, *, validate_runtime: 78) + assert hosted_authoring.main([]) == 78 + + +# The conversational builder + + +def _runtime(workspace: Path, *, state=None, control_only=True) -> HostedChatRuntime: + runtime = object.__new__(HostedChatRuntime) + runtime.workspace = workspace + runtime.job = {"scenario_count": 2} + emitted: list[tuple[str, dict]] = [] + + async def emit(kind, **fields): + emitted.append((kind, fields)) + + context = {"capabilities": {"control_only": control_only}} + if state is not None: + context["status"] = {"state": state} + runtime.client = SimpleNamespace(turn_context=context, command_watermark=3, emit=emit) + runtime.emitted = emitted + runtime.conversation = SimpleNamespace(stage_name="understand", stage=None) + runtime._nudge = None + runtime._from_platform = False + runtime._last_progress, runtime._idle_turns, runtime._opened_with = None, 0, None + runtime._what_exists = lambda: (False, False) + runtime._after_saved = lambda: None + runtime.handled = [] + + async def handle(command): + runtime.handled.append(command) + + runtime._handle = handle + return runtime + + +@pytest.mark.parametrize("state", ["failed", "canceled"]) +def test_carry_on_stops_when_the_job_ended(tmp_path, state): + runtime = _runtime(tmp_path, state=state) + assert asyncio.run(runtime._carry_on()) is False + assert runtime.handled == [] + + +def test_carry_on_stops_with_the_file(tmp_path): + write_failure_file(agent_type_failure(["playwright"]), tmp_path) + runtime = _runtime(tmp_path, state="running") + assert asyncio.run(runtime._carry_on()) is False + assert runtime.handled == [] + + +@pytest.mark.parametrize("state", [None, "running", "completed"]) +def test_carry_on_keeps_going_otherwise(tmp_path, state): + runtime = _runtime(tmp_path, state=state) + assert asyncio.run(runtime._carry_on()) is True + assert len(runtime.handled) == 1 + + +class _TurnRan(Exception): + """Reached the point where a turn starts, which is as far as a test without a model goes.""" + + +def _command(*, from_platform=True) -> dict: + return { + "sequence": 7, + "message_id": "command-7", + "kind": "message", + "payload": {"content": "Build it.", "from_platform": from_platform}, + } + + +def _handle_command(runtime, command) -> None: + async def checkpoint(): + raise _TurnRan + + runtime._checkpoint = checkpoint + asyncio.run(runtime._handle_command(command)) + + +def _assert_answered_with_no_turn(runtime) -> None: + assert [kind for kind, _ in runtime.emitted] == ["turn_completed"] + fields = runtime.emitted[0][1] + assert fields["acknowledge_through"] == 7 + assert fields["payload"] == {"command_message_id": "command-7", "ignored": "the build is over"} + assert runtime._read_journal() == {"command-7": "completed"} + + +@pytest.mark.parametrize("state", ["failed", "canceled"]) +def test_platform_command_on_a_finished_job_runs_no_turn(tmp_path, state): + runtime = _runtime(tmp_path, state=state) + _handle_command(runtime, _command()) + _assert_answered_with_no_turn(runtime) + + +def test_platform_command_with_the_file_runs_no_turn(tmp_path): + write_failure_file(agent_type_failure(["playwright"]), tmp_path) + runtime = _runtime(tmp_path, state="running") + _handle_command(runtime, _command()) + _assert_answered_with_no_turn(runtime) + + +def test_a_finished_command_replays_on_a_finished_job(tmp_path): + runtime = _runtime(tmp_path, state="failed") + runtime._write_journal({"command-7": "completed"}) + _handle_command(runtime, _command()) + assert runtime.emitted[0][1]["payload"] == {"command_message_id": "command-7", "replayed": True} + + +def test_a_redelivered_command_already_started_runs_no_turn(tmp_path): + runtime = _runtime(tmp_path, state="failed") + runtime._write_journal({"command-7": "started"}) + _handle_command(runtime, _command()) + _assert_answered_with_no_turn(runtime) + + +def test_a_failed_turn_still_fails_again_on_a_finished_job(tmp_path): + runtime = _runtime(tmp_path, state="failed") + runtime._write_journal({"command-7": "ended"}) + _handle_command(runtime, _command()) + assert runtime.emitted[0][1]["payload"] == {"command_message_id": "command-7", "outcome": "failed"} + + +def test_a_persons_message_on_a_failed_job_still_runs_a_turn(tmp_path): + runtime = _runtime(tmp_path, state="failed") + with pytest.raises(_TurnRan): + _handle_command(runtime, _command(from_platform=False)) + assert runtime.emitted == [] + + +def test_post_build_check_still_runs_on_a_completed_job(tmp_path): + runtime = _runtime(tmp_path, state="completed") + with pytest.raises(_TurnRan): + _handle_command(runtime, _command()) + assert runtime.emitted == [] + + +def test_failed_job_outside_control_only_still_runs(tmp_path): + runtime = _runtime(tmp_path, state="failed", control_only=False) + with pytest.raises(_TurnRan): + _handle_command(runtime, _command()) + assert runtime.emitted == [] + + +# Correcting the type mid-build + + +@pytest.mark.parametrize("why", ["the operator said so", ""]) +def test_set_modality_to_browser_changes_nothing(tmp_path, why): + contract = AgentContract.model_validate(_reading("chat")) + saved = json.dumps(contract.model_dump()) + path = tmp_path / amend.CONTRACT + path.write_text(saved, encoding="utf-8") + done, said = amend.set_modality(contract, tmp_path, modality=" Browser ", why=why) + assert done is False + assert said == "browser agents aren't supported, so the type stays chat." + assert contract.modality == "chat" + assert contract.amendments == [] + assert path.read_text(encoding="utf-8") == saved + assert not (tmp_path / FAILURE_FILE).exists() + + +def test_set_modality_to_voice_still_works(tmp_path): + contract = AgentContract.model_validate(_reading("chat")) + done, said = amend.set_modality(contract, tmp_path, modality="voice", why="it takes calls") + assert done is True + assert "a call placed to the agent" in said + assert load(tmp_path).modality == "voice" + + +# What the model is told + + +def test_skill_text_asks_the_new_question(): + from fi.alk.harness.config import load_skill + + text = " ".join(load_skill("understand-agent").split()) + assert "does a person come to this agent for a conversation, or to have it carry out that work?" in text + assert "A coding agent is chat." in text + assert "a browser it drives" not in text + assert "through a browser" not in text + + +def test_schema_description_carries_the_new_question(tmp_path): + schema = _submit_tool(contract_tools(tmp_path)).input_schema + description = schema["properties"]["modality"]["description"] + assert "an agent a person uses to carry out work on websites or a computer is browser" in description + assert "browser-driving" not in description + + +def test_set_modality_tool_no_longer_offers_browser(): + from fi.alk.harness.world import tools as world_tools + + source = Path(world_tools.__file__).read_text(encoding="utf-8") + assert "actually reaches this agent: voice or chat." in source + assert "voice, chat or browser" not in source + + +def test_browser_stays_a_known_modality(): + assert "browser" in known_modalities() + assert "browser" in _submit_tool(contract_tools(Path("unused"))).input_schema["properties"]["modality"]["enum"] From 15f74f31e9aa270ba82f3e5e38a127278c17aabd Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Mon, 2 Nov 2026 15:21:31 +0530 Subject: [PATCH 02/15] test(harness): stop depending on MCP internals in the write-failure test Refs: TH-8806 Signed-off-by: khushalsonawat --- tests/harness/test_agent_type_refusal.py | 27 ++++++------------------ 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index b4045b9ae..3eff86031 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -558,29 +558,14 @@ def test_writer_failure_blocks_resubmission(tmp_path, browser_repo, monkeypatch) assert not (output / "contract.json").exists() -def test_writer_failure_reaches_the_model_as_a_tool_error(tmp_path, browser_repo, monkeypatch): - from mcp import types - - from fi.alk.harness.backends.claude import _sdk_server - +def test_writer_failure_leaves_the_handler(tmp_path, browser_repo, monkeypatch): + # The MCP server turns what leaves the handler into a tool error the model reads, so it has to + # leave: swallowed here, the model would be told nothing. output = tmp_path / "out" - instance = _sdk_server(contract_tools(output, scan_root=browser_repo))["instance"] - handler = instance.request_handlers[types.CallToolRequest] + submit = _submitter(output, browser_repo) _failing_write(monkeypatch) - - async def call(): - return await handler( - types.CallToolRequest( - method="tools/call", - params=types.CallToolRequestParams( - name="submit_contract", arguments=_reading("browser") - ), - ) - ) - - result = asyncio.run(call()).root - assert result.isError - assert "disk full" in result.content[0].text + with pytest.raises(OSError, match="disk full"): + submit(_reading("browser")) assert not (output / FAILURE_FILE).exists() From b874c149cea51864a6962748d403b5d322c069d0 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Mon, 2 Nov 2026 17:28:30 +0530 Subject: [PATCH 03/15] fix(harness): address review of the browser agent refusal - Two submissions at once no longer skip the second look. - A disk with no hard links still gets failure.json, and no temporary file is left behind on a failed write. - Only folders named wholly for tests are skipped, so e2e_agent/ is read. - Only the tools' API names count as computer-use, not the bare words. - Packages unpacked beside the agent's code aren't scanned, by each distribution's own record; a browser package imported from there counts. - A lone apostrophe no longer protects a comment in script files. - The scan has an overall size limit and is reused per checkout. - A refusal removes an earlier contract.json from the folder. - The sandbox-server route reports the failure a build left. Refs: TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_type_check.py | 91 ++++++++-- src/fi/alk/harness/executor.py | 17 +- src/fi/alk/harness/failure_file.py | 34 +++- src/fi/alk/harness/tools.py | 7 +- tests/harness/test_agent_type_refusal.py | 156 +++++++++++++++++- .../harness/test_executor_completed_counts.py | 40 +++++ 6 files changed, 318 insertions(+), 27 deletions(-) diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index f6c6e2af0..19755e091 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -15,6 +15,7 @@ import re import tokenize from collections.abc import Iterator, Sequence +from functools import lru_cache from pathlib import Path from typing import Any, Literal @@ -81,11 +82,12 @@ "apify-client", } ) +# The tools' own API names only: the bare words turn up in prompts and tracing attributes. TOOL_MARKER = re.compile( - r"\bcomputer[_-]use(?:[_-]preview)?\b|\bComputerUse\b|\bcomputer_\d{8}\b" - r"|@playwright/mcp|mcp-server-playwright" + r"\bcomputer[_-]use[_-]preview\b|\bcomputer-use-20\d\d-\d\d-\d\d\b|\bComputerUse\b" + r"|\bcomputer_\d{8}\b|@playwright/mcp|mcp-server-playwright" ) -# Hidden folders and anything named for tests or e2e describe the developer's setup too. +# Hidden folders and folders named for tests or e2e describe the developer's setup too. NOT_THE_AGENT = frozenset( {"docs", "doc", "examples", "example", "samples", "vendor", "third_party", "evals"} ) @@ -96,6 +98,9 @@ _TEST_FILES = ("test_*.py", "*_test.py", "conftest.py", "*.test.*", "*.spec.*") _SCRIPT_SUFFIXES = frozenset({".js", ".ts", ".mjs", ".cjs", ".tsx", ".jsx"}) _LARGEST_CODE_FILE = 1024 * 1024 +# Past this a repository holds the conversation up for longer than the rest of the scan is worth. +_LARGEST_SCAN = 128 * 1024 * 1024 +_LARGEST_RECORD = 1024 * 1024 # Strings are matched first and kept, so "https://..." or "src/**/*.ts" isn't read as a comment. # An unclosed one ends at its line or the file: a start that could fail makes a customer's odd file # cost the square of its size, which holds up the build's conversation until it's done. @@ -106,8 +111,11 @@ ) -# A whole word only: "latest" and "contest" hold real code. -_TEST_FOLDER = re.compile(r"(?:^|[^a-z0-9])(?:tests?|testing|testdata|unittests?|integrationtests?|e2e(?:tests?)?)(?:$|[^a-z0-9])") +# The whole name: "e2e_agent" and "testing_agent" hold agents, and test automation is a main kind +# of browser agent. +_TEST_FOLDER = re.compile( + r"_*(?:(?:unit|integration|e2e)[_-]?)?(?:tests?|testing|test[_-]?data)_*|e2e" +) def _not_the_agent_dir(name: str) -> bool: @@ -116,7 +124,7 @@ def _not_the_agent_dir(name: str) -> bool: name in _SKIPPED_DIRECTORIES or lowered in NOT_THE_AGENT or name.startswith(".") - or _TEST_FOLDER.search(lowered) is not None + or _TEST_FOLDER.fullmatch(lowered) is not None ) @@ -125,13 +133,34 @@ def _not_the_agent_file(name: str) -> bool: return any(fnmatch.fnmatchcase(lowered, pattern) for pattern in _TEST_FILES) +def _unpacked_at_top(root: Path) -> set[str]: + """What ``pip install -t .`` put beside the agent's code, by each distribution's own record.""" + names: set[str] = set() + try: + records = [entry for entry in root.iterdir() if entry.name.endswith(".dist-info")] + except OSError: + return names + for entry in records: + try: + record = entry / "RECORD" + if record.stat().st_size > _LARGEST_RECORD: + continue + text = record.read_text(encoding="utf-8") + except (OSError, ValueError): + continue + names.update(line.split(",", 1)[0].split("/", 1)[0] for line in text.splitlines()) + return names + + def _agent_files(root: Path) -> Iterator[Path]: + unpacked = _unpacked_at_top(root) for directory, subdirectories, files in os.walk(root): + if Path(directory) == root: + subdirectories[:] = [name for name in subdirectories if name not in unpacked] + files = [name for name in files if name not in unpacked] # A folder of unpacked distributions holds other people's code, whose SDKs name these # tools whether or not this agent uses them. - if Path(directory) != root and any( - name.endswith(".dist-info") for name in subdirectories - ): + elif any(name.endswith(".dist-info") for name in subdirectories): subdirectories[:] = [] continue subdirectories[:] = [ @@ -168,11 +197,13 @@ def _imports_in(path: Path) -> set[str]: def _modules_at_top(root: Path) -> set[str]: """What sits ahead of an installed package on the import path, so isn't that package.""" + unpacked = _unpacked_at_top(root) try: return { entry.stem if entry.is_file() else entry.name for entry in root.iterdir() - if entry.suffix == ".py" or (entry / "__init__.py").is_file() + if entry.name not in unpacked + and (entry.suffix == ".py" or (entry / "__init__.py").is_file()) } except OSError: return set() @@ -186,6 +217,14 @@ def _python_code(text: str) -> str: ) +def _script_kept(hit: re.Match[str]) -> str: + kept = hit.group(1) or "" + # A lone apostrophe, as in JSX text, isn't a string, so it can't protect a comment after it. + if kept.startswith("'") and (len(kept) == 1 or not kept.endswith("'")): + return kept.split("//", 1)[0] + return kept + + def _markers_in(path: Path) -> set[str]: try: if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: @@ -194,7 +233,7 @@ def _markers_in(path: Path) -> set[str]: code = ( _python_code(text) if path.suffix == ".py" - else _SCRIPT_COMMENT_OR_STRING.sub(lambda hit: hit.group(1) or "", text) + else _SCRIPT_COMMENT_OR_STRING.sub(_script_kept, text) ) except (OSError, ValueError, SyntaxError, tokenize.TokenError): return set() @@ -204,6 +243,13 @@ def _markers_in(path: Path) -> set[str]: } +def _file_size(path: Path) -> int: + try: + return path.stat().st_size + except OSError: + return 0 + + def browser_tools_found(source_root: Path | None) -> list[str]: """The browser, crawling and computer-use tools the agent's own code uses, by display name. @@ -213,12 +259,31 @@ def browser_tools_found(source_root: Path | None) -> list[str]: if source_root is None: return [] root = Path(source_root).resolve() + try: + # A new checkout at the same path has a new time, so it doesn't get the old answer. + checked_out = root.stat().st_mtime_ns + except OSError: + return [] + return list(_scan(root, checked_out)) + + +@lru_cache(maxsize=8) +def _scan(root: Path, _checked_out: int) -> tuple[str, ...]: + """One scan per checkout: every builder reopen in a process asks again.""" found: set[str] = set() imported: set[str] = set() + scanned = 0 for path in _agent_files(root): + if path.name != "package.json" and not ( + path.suffix == ".py" or path.suffix in _SCRIPT_SUFFIXES + ): + continue + scanned += _file_size(path) + if scanned > _LARGEST_SCAN: + break if path.name == "package.json": found |= _packages_used(path) - elif path.suffix == ".py" or path.suffix in _SCRIPT_SUFFIXES: + else: found |= _markers_in(path) if path.suffix == ".py": imported |= _imports_in(path) @@ -231,7 +296,7 @@ def browser_tools_found(source_root: Path | None) -> list[str]: found.add(top) elif name.endswith(f".{_TOOLKIT}"): found.add(_TOOLKIT) - return sorted({name.replace("_", "-") for name in found}) + return tuple(sorted({name.replace("_", "-") for name in found})) def modality_text(value: Any) -> str: diff --git a/src/fi/alk/harness/executor.py b/src/fi/alk/harness/executor.py index 900a5763d..9dbedb42c 100644 --- a/src/fi/alk/harness/executor.py +++ b/src/fi/alk/harness/executor.py @@ -12,6 +12,7 @@ import subprocess from typing import Callable, Protocol +from .failure_file import FAILURE_FILE from .job import ( FailureDomain, HarnessFailure, @@ -269,7 +270,11 @@ async def run( retryable=False, ) status = 1 - failure = _failure_from_events(output) if status not in (0, 2) else None + failure = ( + _failure_from_file(output) or _failure_from_events(output) + if status not in (0, 2) + else None + ) if generic_failure is not None: failure = generic_failure return HarnessJobStatus( @@ -337,6 +342,16 @@ def _scenario_count(output: Path) -> int: return len(value) if isinstance(value, list) else 0 +def _failure_from_file(output: Path) -> HarnessFailure | None: + """The failure a build left for the platform, which says more than its stage event can.""" + try: + return HarnessFailure.model_validate_json( + (output / FAILURE_FILE).read_text(encoding="utf-8") + ) + except (OSError, ValueError): + return None + + def _failure_from_events(output: Path) -> HarnessFailure: failed: dict = {} path = output / "harness-events.jsonl" diff --git a/src/fi/alk/harness/failure_file.py b/src/fi/alk/harness/failure_file.py index ff7ae789e..93b27e57e 100644 --- a/src/fi/alk/harness/failure_file.py +++ b/src/fi/alk/harness/failure_file.py @@ -12,21 +12,41 @@ FAILURE_FILE = "failure.json" +def _write_in_place(path: Path, body: str) -> None: + """For a disk with no hard links: still first-wins, but a reader could catch it part-written.""" + try: + descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + except FileExistsError: + return + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as file: + file.write(body) + os.chmod(path, 0o644) + except OSError: + path.unlink(missing_ok=True) + raise + + def write_failure_file(failure: HarnessFailure, output: Path) -> Path: """Write ``failure`` whole, in one step. The first failure of a build stands.""" output = Path(output) output.mkdir(parents=True, exist_ok=True) path = output / FAILURE_FILE - with tempfile.NamedTemporaryFile( + body = json.dumps(failure.model_dump(mode="json"), indent=2, sort_keys=True) + "\n" + temporary = tempfile.NamedTemporaryFile( "w", dir=output, prefix=".failure-", suffix=".tmp", encoding="utf-8", delete=False - ) as temporary: - json.dump(failure.model_dump(mode="json"), temporary, indent=2, sort_keys=True) - temporary.write("\n") + ) try: + with temporary: + temporary.write(body) os.chmod(temporary.name, 0o644) - os.link(temporary.name, path) - except FileExistsError: - pass + try: + os.link(temporary.name, path) + except FileExistsError: + pass + except OSError: + # No file reads as a crash, which pays for another full run to reach this same failure. + _write_in_place(path, body) finally: os.unlink(temporary.name) return path diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index 5482f32b4..0f326bf39 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -1090,6 +1090,9 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: nonlocal refused, looked_again, found if refused or (destination / FAILURE_FILE).is_file(): return _ok(_stop_message(destination)) + # Read before the wait: a call running alongside this one mustn't count as the model + # having read the second look. + had_second_look = looked_again if found is None: found = await asyncio.to_thread(browser_tools_found, scan_root) # A call running alongside this one may have refused while this one waited. @@ -1103,11 +1106,13 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: "job is work on websites or a computer." ] ) - verdict = decide(claimed, found, looked_again=looked_again) + verdict = decide(claimed, found, looked_again=had_second_look) if verdict == "refuse": # Set first: if the write fails, the model must not get a resubmission through. refused = True write_failure_file(agent_type_failure(found), destination) + # An earlier reading in this folder would otherwise build the agent just refused. + (destination / "contract.json").unlink(missing_ok=True) return _ok(_stop_message(destination)) if verdict == "second_look": looked_again = True diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index 3eff86031..15c6281df 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -3,9 +3,11 @@ from __future__ import annotations import asyncio +import errno import json import os import stat +import tempfile import threading import time from pathlib import Path @@ -14,10 +16,12 @@ import pytest from fi.alk.harness import ( + agent_type_check, amend, authoring_entrypoint, chat, cli, + failure_file, tools, ) from fi.alk.harness import hosted_authoring_entrypoint as hosted_authoring @@ -157,6 +161,58 @@ def test_writer_leaves_no_temp_file(tmp_path): assert [path.name for path in tmp_path.iterdir()] == [FAILURE_FILE] +def _no_hard_links(monkeypatch): + def refuse(*_args, **_kwargs): + raise OSError(errno.EXDEV, "cross-device link") + + monkeypatch.setattr(failure_file.os, "link", refuse) + + +def test_a_disk_with_no_hard_links_still_gets_the_whole_file(tmp_path, monkeypatch): + _no_hard_links(monkeypatch) + path = write_failure_file(_a_failure(), tmp_path) + assert HarnessFailure.model_validate_json(path.read_text()) == _a_failure() + assert stat.S_IMODE(path.stat().st_mode) == 0o644 + assert [item.name for item in tmp_path.iterdir()] == [FAILURE_FILE] + + +def test_first_failure_stands_with_no_hard_links(tmp_path, monkeypatch): + _no_hard_links(monkeypatch) + write_failure_file(_a_failure(), tmp_path) + write_failure_file(agent_type_failure([]), tmp_path) + assert HarnessFailure.model_validate_json((tmp_path / FAILURE_FILE).read_text()) == _a_failure() + + +def test_a_write_that_fails_on_a_disk_with_no_hard_links_leaves_nothing(tmp_path, monkeypatch): + _no_hard_links(monkeypatch) + + def disk_full(*_args, **_kwargs): + raise OSError(errno.ENOSPC, "no space") + + monkeypatch.setattr(failure_file.os, "fdopen", disk_full) + with pytest.raises(OSError): + write_failure_file(_a_failure(), tmp_path) + assert list(tmp_path.iterdir()) == [] + + +def test_a_full_disk_leaves_no_temporary_file(tmp_path, monkeypatch): + real = tempfile.NamedTemporaryFile + + def filling(*args, **kwargs): + opened = real(*args, **kwargs) + + def disk_full(_text): + raise OSError(errno.ENOSPC, "no space") + + opened.write = disk_full + return opened + + monkeypatch.setattr(failure_file.tempfile, "NamedTemporaryFile", filling) + with pytest.raises(OSError): + write_failure_file(_a_failure(), tmp_path) + assert list(tmp_path.iterdir()) == [] + + def test_failure_message_is_empty_when_the_file_cannot_be_read(tmp_path): assert failure_message(tmp_path) == "" for body in ("{", "[1]", '{"message": 3}'): @@ -178,6 +234,8 @@ def test_failure_message_is_empty_when_the_file_cannot_be_read(tmp_path): ), ({"agent.py": 'TOOL = {"type": "computer_20241022"}\n'}, ["computer-use tool"]), ({"agent.ts": 'const tool = {type: "computer_use_preview"};\n'}, ["computer-use tool"]), + ({"agent.py": 'BETA = "computer-use-2025-01-24"\n'}, ["computer-use tool"]), + ({"agent.py": 'MODEL = "gemini-2.5-computer-use-preview-10-2025"\n'}, ["computer-use tool"]), ( {"agent.py": 'from google.genai import types\nTOOL = types.ComputerUse(environment="x")\n'}, ["computer-use tool"], @@ -210,7 +268,8 @@ def test_browser_tools_found_is_sorted_and_names_each_tool_once(tmp_path): @pytest.mark.parametrize( - "folder", ["e2eTests", "testdata", "unittests", "integrationtests", "Test_A"] + "folder", + ["e2e", "e2eTests", "e2e_tests", "testdata", "unittests", "integration_tests", "Test", "__tests__"], ) def test_tools_found_skips_these_test_folders(tmp_path, folder): (tmp_path / folder).mkdir() @@ -251,9 +310,11 @@ def test_an_odd_script_file_stays_quick(tmp_path, body): assert time.monotonic() - started < 5 -@pytest.mark.parametrize("folder", ["latest", "contest"]) +@pytest.mark.parametrize( + "folder", ["latest", "contest", "e2e_agent", "testing_agent", "src/testing_agent", "test_runner"] +) def test_tools_found_reads_a_folder_with_test_inside_a_word(tmp_path, folder): - (tmp_path / folder).mkdir() + (tmp_path / folder).mkdir(parents=True) (tmp_path / folder / "a.py").write_text("import playwright\n", encoding="utf-8") assert browser_tools_found(tmp_path) == ["playwright"] @@ -284,6 +345,9 @@ def test_tools_found_reads_a_folder_with_test_inside_a_word(tmp_path, folder): ("tests/package.json", json.dumps({"dependencies": {"playwright": "1"}})), ("a.py", "# unlike computer_20241022, this only talks\n"), ("a.ts", '// see @playwright/mcp\n/* computer_use\n and ComputerUse */\nconst a = 1;\n'), + ("a.py", 'PROMPT = "never do computer-use"\n'), + ("a.py", 'SPAN = "gen_ai.computer_use.action"\n'), + ("a.jsx", "export const A = () =>

Don't

// ComputerUse\n"), ], ) def test_browser_tools_found_ignores_what_is_not_the_agent(tmp_path, name, body): @@ -302,6 +366,64 @@ def test_unpacked_distributions_are_not_the_agent(tmp_path): assert browser_tools_found(tmp_path) == [] +def _unpacked_at_the_top(root: Path) -> None: + _tree( + root, + { + "openai-1.0.dist-info/RECORD": "openai/x.py,sha256=a,1\nsix.py,sha256=b,2\n", + "openai/__init__.py": "", + "openai/x.py": 'NAME = "computer_use_preview"\n', + "six.py": 'NAME = "computer_20241022"\n', + }, + ) + + +def test_packages_unpacked_beside_the_agent_are_not_the_agent(tmp_path): + _unpacked_at_the_top(tmp_path) + _tree(tmp_path, {"handler.py": "import openai\n"}) + assert browser_tools_found(tmp_path) == [] + + +def test_the_agents_own_folder_is_read_beside_unpacked_packages(tmp_path): + _unpacked_at_the_top(tmp_path) + _tree(tmp_path, {"app/run.py": "import playwright\n"}) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_a_browser_package_unpacked_at_the_top_still_counts_when_imported(tmp_path): + _tree( + tmp_path, + { + "playwright-1.0.dist-info/RECORD": "playwright/__init__.py,sha256=a,1\n", + "playwright/__init__.py": "", + "handler.py": "import playwright\n", + }, + ) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_the_scan_stops_at_its_limit_with_what_it_found(tmp_path, monkeypatch): + _tree(tmp_path, {f"m{n}.py": f"import {name}\n" for n, name in enumerate(["playwright", "selenium", "scrapy"])}) + monkeypatch.setattr(agent_type_check, "_LARGEST_SCAN", 20) + assert len(browser_tools_found(tmp_path)) == 1 + + +def test_a_second_scan_of_a_checkout_reuses_the_first(tmp_path, monkeypatch): + _tree(tmp_path, {"a.py": "import playwright\n"}) + assert browser_tools_found(tmp_path) == ["playwright"] + monkeypatch.setattr(agent_type_check, "_agent_files", lambda root: pytest.fail("scanned again")) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_a_new_checkout_at_the_same_path_is_scanned_again(tmp_path): + _tree(tmp_path, {"a.py": "import playwright\n"}) + assert browser_tools_found(tmp_path) == ["playwright"] + (tmp_path / "a.py").unlink() + (tmp_path / "b.py").write_text("import selenium\n", encoding="utf-8") + os.utime(tmp_path, ns=(1, 1)) + assert browser_tools_found(tmp_path) == ["selenium"] + + def test_a_customers_other_package_names_never_appear(tmp_path): _tree( tmp_path, @@ -503,6 +625,30 @@ def test_browser_with_tool_is_refused_on_the_first_submission(tmp_path, browser_ assert not (output / "contract.json").exists() +def test_two_submissions_at_once_both_get_the_second_look(tmp_path, plain_repo): + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) + + async def both(): + return await asyncio.gather( + tool.handler(_reading("browser")), tool.handler(_reading("browser")) + ) + + for answer in asyncio.run(both()): + assert answer.get("is_error") + assert not (output / FAILURE_FILE).exists() + + +def test_a_refused_new_reading_removes_the_earlier_one(tmp_path, browser_repo): + output = tmp_path / "out" + output.mkdir() + (output / "contract.json").write_text("{}", encoding="utf-8") + _submitter(output, browser_repo)(_reading("browser")) + assert (output / FAILURE_FILE).exists() + assert not (output / "contract.json").exists() + assert load(output) is None + + # After a refusal @@ -559,8 +705,8 @@ def test_writer_failure_blocks_resubmission(tmp_path, browser_repo, monkeypatch) def test_writer_failure_leaves_the_handler(tmp_path, browser_repo, monkeypatch): - # The MCP server turns what leaves the handler into a tool error the model reads, so it has to - # leave: swallowed here, the model would be told nothing. + # Swallowed here, nothing would say the refusal was lost: the model would be told nothing, and + # with no file the build would end as a crash. output = tmp_path / "out" submit = _submitter(output, browser_repo) _failing_write(monkeypatch) diff --git a/tests/harness/test_executor_completed_counts.py b/tests/harness/test_executor_completed_counts.py index 21f5ae5c2..e6de4d915 100644 --- a/tests/harness/test_executor_completed_counts.py +++ b/tests/harness/test_executor_completed_counts.py @@ -4,7 +4,9 @@ import pytest +from fi.alk.harness.agent_type_check import agent_type_failure from fi.alk.harness.executor import HarnessExecutor +from fi.alk.harness.failure_file import FAILURE_FILE, write_failure_file from fi.alk.harness.job import HarnessJob, HarnessStage from fi.alk.harness.local_certified_runtime import LocalCertifiedRunError from fi.alk.harness.process_runtime import ProcessRuntimeError @@ -46,3 +48,41 @@ async def failed_runtime(*args, **kwargs): assert status.total_scenarios == 1 assert status.completed_scenarios == (1 if partial_run else 0) assert status.failure.code == ("runtime_stopped" if partial_run else "resource_capacity_unknown") + + +def _job(source): + return HarnessJob( + job_id="refused", run_id="refused-run", execution="local", + source={"kind": "local_repository", "local_path": str(source)}, + agent={"connector": "auto"}, scenario_count=1, + ) + + +def test_the_failure_a_build_left_is_the_jobs_failure(tmp_path, monkeypatch): + source = tmp_path / "source" + source.mkdir() + output = tmp_path / "output" + + async def refused(args): + write_failure_file(agent_type_failure(["playwright"]), Path(args.out)) + return 78 + + monkeypatch.setattr("fi.alk.harness.cli._auto", refused) + status = asyncio.run(HarnessExecutor().run(_job(source), source=source, output=output)) + assert status.stage is HarnessStage.FAILED + assert status.failure == agent_type_failure(["playwright"]) + + +def test_an_unreadable_failure_file_falls_back_to_the_events(tmp_path, monkeypatch): + source = tmp_path / "source" + source.mkdir() + output = tmp_path / "output" + + async def crashed(args): + Path(args.out).mkdir() + (Path(args.out) / FAILURE_FILE).write_text("{") + return 1 + + monkeypatch.setattr("fi.alk.harness.cli._auto", crashed) + status = asyncio.run(HarnessExecutor().run(_job(source), source=source, output=output)) + assert status.failure.code == "running_failed" From 7e977c655f6aac06a472357911f34e35b9498205 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Mon, 2 Nov 2026 18:09:37 +0530 Subject: [PATCH 04/15] fix(harness): keep test folders out of the scan and its reuse honest - Folders whose last part is a test word, such as functional_tests/, are skipped again; e2e_agent/ and testing_agent/ are still read. - A scan is reused only while the files it reads are unchanged, so a nested edit or a removed helper is scanned again. - Files too big to read no longer use up the scan's size limit. Refs: TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_type_check.py | 45 +++++++++++------ tests/harness/test_agent_type_refusal.py | 61 ++++++++++++++++++++---- 2 files changed, 83 insertions(+), 23 deletions(-) diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index 19755e091..bd7ca5a6c 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -111,10 +111,11 @@ ) -# The whole name: "e2e_agent" and "testing_agent" hold agents, and test automation is a main kind -# of browser agent. +# The whole name or its last part: "functional_tests" holds tests, while "e2e_agent" and +# "testing_agent" hold agents, and test automation is a main kind of browser agent. _TEST_FOLDER = re.compile( r"_*(?:(?:unit|integration|e2e)[_-]?)?(?:tests?|testing|test[_-]?data)_*|e2e" + r"|.*[_-](?:tests?|testing|test[_-]?data|e2e)_*" ) @@ -259,26 +260,42 @@ def browser_tools_found(source_root: Path | None) -> list[str]: if source_root is None: return [] root = Path(source_root).resolve() - try: - # A new checkout at the same path has a new time, so it doesn't get the old answer. - checked_out = root.stat().st_mtime_ns - except OSError: - return [] - return list(_scan(root, checked_out)) + return list(_scan(root, _signature(root))) + + +def _is_read(path: Path) -> bool: + return path.name == "package.json" or path.suffix == ".py" or path.suffix in _SCRIPT_SUFFIXES + + +def _signature(root: Path) -> tuple[int, int, int]: + """What changes when the files a scan reads change: a nested edit or a new checkout.""" + latest = count = size = 0 + for path in _agent_files(root): + if _is_read(path): + try: + info = path.stat() + except OSError: + continue + latest = max(latest, info.st_mtime_ns) + count += 1 + size += info.st_size + return latest, count, size @lru_cache(maxsize=8) -def _scan(root: Path, _checked_out: int) -> tuple[str, ...]: - """One scan per checkout: every builder reopen in a process asks again.""" +def _scan(root: Path, _files: tuple[int, int, int]) -> tuple[str, ...]: + """One scan per state of the files: every builder reopen in a process asks again.""" found: set[str] = set() imported: set[str] = set() scanned = 0 for path in _agent_files(root): - if path.name != "package.json" and not ( - path.suffix == ".py" or path.suffix in _SCRIPT_SUFFIXES - ): + if not _is_read(path): + continue + size = _file_size(path) + # Never read, so it mustn't use up the limit before the code that is. + if size > _LARGEST_CODE_FILE: continue - scanned += _file_size(path) + scanned += size if scanned > _LARGEST_SCAN: break if path.name == "package.json": diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index 15c6281df..d67b97a68 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -269,7 +269,20 @@ def test_browser_tools_found_is_sorted_and_names_each_tool_once(tmp_path): @pytest.mark.parametrize( "folder", - ["e2e", "e2eTests", "e2e_tests", "testdata", "unittests", "integration_tests", "Test", "__tests__"], + [ + "e2e", + "e2eTests", + "e2e_tests", + "testdata", + "unittests", + "integration_tests", + "Test", + "__tests__", + "functional_tests", + "ui-tests", + "api_tests", + "playwright-tests", + ], ) def test_tools_found_skips_these_test_folders(tmp_path, folder): (tmp_path / folder).mkdir() @@ -408,20 +421,50 @@ def test_the_scan_stops_at_its_limit_with_what_it_found(tmp_path, monkeypatch): assert len(browser_tools_found(tmp_path)) == 1 -def test_a_second_scan_of_a_checkout_reuses_the_first(tmp_path, monkeypatch): +def test_files_too_big_to_read_do_not_use_up_the_limit(tmp_path, monkeypatch): + _tree(tmp_path, {"big.js": "x" * 80, "agent/run.py": "import playwright\n"}) + monkeypatch.setattr(agent_type_check, "_LARGEST_CODE_FILE", 50) + monkeypatch.setattr(agent_type_check, "_LARGEST_SCAN", 60) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_a_second_scan_of_unchanged_files_reuses_the_first(tmp_path, monkeypatch): _tree(tmp_path, {"a.py": "import playwright\n"}) assert browser_tools_found(tmp_path) == ["playwright"] - monkeypatch.setattr(agent_type_check, "_agent_files", lambda root: pytest.fail("scanned again")) + + def read_again(_path): + pytest.fail("scanned again") + + monkeypatch.setattr(agent_type_check, "_markers_in", read_again) assert browser_tools_found(tmp_path) == ["playwright"] -def test_a_new_checkout_at_the_same_path_is_scanned_again(tmp_path): - _tree(tmp_path, {"a.py": "import playwright\n"}) +def test_a_nested_edit_is_scanned_again(tmp_path): + # The same size, so only the time says it changed. + _tree(tmp_path, {"src/agent.py": "import openai\n"}) + assert browser_tools_found(tmp_path) == [] + (tmp_path / "src/agent.py").write_text("import scrapy\n", encoding="utf-8") + os.utime(tmp_path / "src/agent.py", ns=(2**62, 2**62)) + assert browser_tools_found(tmp_path) == ["scrapy"] + + +def test_one_file_fewer_is_scanned_again(tmp_path): + # The same total size and latest time, so only the count says it changed. + _tree(tmp_path, {"a.py": "import openai\n", "b.py": "import scrapy\n"}) + for name in ("a.py", "b.py"): + os.utime(tmp_path / name, ns=(1, 1)) + assert browser_tools_found(tmp_path) == ["scrapy"] + (tmp_path / "b.py").unlink() + (tmp_path / "a.py").write_text("import openai\n" * 2, encoding="utf-8") + os.utime(tmp_path / "a.py", ns=(1, 1)) + assert browser_tools_found(tmp_path) == [] + + +def test_removing_a_nested_helper_is_scanned_again(tmp_path): + _tree(tmp_path, {"src/agent.py": "import openai\n", "src/helper.py": "import playwright\n"}) assert browser_tools_found(tmp_path) == ["playwright"] - (tmp_path / "a.py").unlink() - (tmp_path / "b.py").write_text("import selenium\n", encoding="utf-8") - os.utime(tmp_path, ns=(1, 1)) - assert browser_tools_found(tmp_path) == ["selenium"] + (tmp_path / "src/helper.py").unlink() + assert browser_tools_found(tmp_path) == [] def test_a_customers_other_package_names_never_appear(tmp_path): From a89d71c2984b2564453603afa843efb8949c658a Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Wed, 4 Nov 2026 11:06:54 +0530 Subject: [PATCH 05/15] fix(harness): make the second look hold on both loops, and harden the scan - A second look now gives the model a value to send back, and an answer counts only if it holds one. Calls in one model message run before the model reads any result, on either loop, so none of them can know it. This replaces the entry-time read and the one-second rule. - RECORD files are matched by exact path, read only when they are regular files, and capped at 64 MiB in total. - Python and package.json are read first, in path order. - The computer-use object key needs an environment, so flags and labels don't count. - More test-helper folder names are skipped. - The earlier contract.json is removed before the failure is written, and the failure is written even if the removal fails. - A disk that refuses chmod still gets the whole failure file. - The executor tests run in CI. Refs: TH-8806 Signed-off-by: khushalsonawat --- .github/workflows/sdk-smoke.yml | 1 + src/fi/alk/harness/HOW-IT-WORKS.md | 4 +- src/fi/alk/harness/agent_type_check.py | 57 ++- src/fi/alk/harness/failure_file.py | 12 +- src/fi/alk/harness/tools.py | 53 ++- tests/harness/test_agent_type_refusal.py | 364 ++++++++++++++++-- .../harness/test_executor_completed_counts.py | 31 +- 7 files changed, 460 insertions(+), 62 deletions(-) diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 7ed0cf674..2853b6220 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -67,6 +67,7 @@ jobs: tests/harness/test_hosted_entrypoint_provision_retry.py tests/harness/test_instrumentation_evidence.py tests/harness/test_agent_type_refusal.py + tests/harness/test_executor_completed_counts.py tests/test_bump_deployment_values.py tests/test_release_workflow.py tests/test_release_pr_step.py diff --git a/src/fi/alk/harness/HOW-IT-WORKS.md b/src/fi/alk/harness/HOW-IT-WORKS.md index 1768fec95..f4f8a8db1 100644 --- a/src/fi/alk/harness/HOW-IT-WORKS.md +++ b/src/fi/alk/harness/HOW-IT-WORKS.md @@ -80,7 +80,9 @@ conversation**, so the model corrects and resubmits rather than a bad contract l A browser reading with a browser, crawling or computer-use tool found in the source is refused to `failure.json`, and the build exits 78. Where the model and that code check disagree, the model gets -one second look, and its answer stands. +one second look, and its answer stands. That answer has to send back a value the second look gave, +because calls in one model message run before the model reads any result. A model that never +sends it back keeps getting second looks until its turn limit, and the build ends as a crash. **How it is changed later** (`amend.py`). The contract is not frozen, but every change is recorded with a reason in `amendments[]`, so months later you can still tell what came from the diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index bd7ca5a6c..b6ab6e30c 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -8,6 +8,7 @@ from __future__ import annotations import ast +import csv import fnmatch import io import json @@ -85,6 +86,7 @@ # The tools' own API names only: the bare words turn up in prompts and tracing attributes. TOOL_MARKER = re.compile( r"\bcomputer[_-]use[_-]preview\b|\bcomputer-use-20\d\d-\d\d-\d\d\b|\bComputerUse\b" + r"""|\bcomputer_?[uU]se\b["']?\s*[:=]\s*\{\s*["']?environment\b""" r"|\bcomputer_\d{8}\b|@playwright/mcp|mcp-server-playwright" ) # Hidden folders and folders named for tests or e2e describe the developer's setup too. @@ -101,6 +103,7 @@ # Past this a repository holds the conversation up for longer than the rest of the scan is worth. _LARGEST_SCAN = 128 * 1024 * 1024 _LARGEST_RECORD = 1024 * 1024 +_LARGEST_RECORDS = 64 * 1024 * 1024 # Strings are matched first and kept, so "https://..." or "src/**/*.ts" isn't read as a comment. # An unclosed one ends at its line or the file: a start that could fail makes a customer's odd file # cost the square of its size, which holds up the build's conversation until it's done. @@ -111,11 +114,14 @@ ) -# The whole name or its last part: "functional_tests" holds tests, while "e2e_agent" and -# "testing_agent" hold agents, and test automation is a main kind of browser agent. +# The whole name, its last part, or a test word and a helper word: "functional_tests" and +# "test_utils" hold tests, while "e2e_agent" and "testing_agent" hold agents, and test automation +# is a main kind of browser agent. _TEST_FOLDER = re.compile( r"_*(?:(?:unit|integration|e2e)[_-]?)?(?:tests?|testing|test[_-]?data)_*|e2e" r"|.*[_-](?:tests?|testing|test[_-]?data|e2e)_*" + r"|(?:tests?|testing)[_.-]?(?:utils?|helpers?|integration|ui|unit|e2e|old|fixtures?|support" + r"|common|lib|mocks?|selenium|browser|functional|acceptance|scripts?)_*" ) @@ -135,41 +141,51 @@ def _not_the_agent_file(name: str) -> bool: def _unpacked_at_top(root: Path) -> set[str]: - """What ``pip install -t .`` put beside the agent's code, by each distribution's own record.""" + """The files ``pip install -t .`` put beside the agent's code, by each distribution's record. + + Exact paths: one distribution shipping ``tools/x.py`` says nothing about the customer's own + ``tools/``. + """ names: set[str] = set() try: - records = [entry for entry in root.iterdir() if entry.name.endswith(".dist-info")] + records = sorted(entry for entry in root.iterdir() if entry.name.endswith(".dist-info")) except OSError: return names + read = 0 for entry in records: try: record = entry / "RECORD" - if record.stat().st_size > _LARGEST_RECORD: + # A link to a device or a named pipe reports no size, and reading it never ends. + if not record.is_file() or record.stat().st_size > _LARGEST_RECORD: continue - text = record.read_text(encoding="utf-8") - except (OSError, ValueError): + read += record.stat().st_size + if read > _LARGEST_RECORDS: + break + names.update( + row[0] for row in csv.reader(io.StringIO(record.read_text(encoding="utf-8"))) if row + ) + except (OSError, ValueError, csv.Error): continue - names.update(line.split(",", 1)[0].split("/", 1)[0] for line in text.splitlines()) return names def _agent_files(root: Path) -> Iterator[Path]: unpacked = _unpacked_at_top(root) for directory, subdirectories, files in os.walk(root): - if Path(directory) == root: - subdirectories[:] = [name for name in subdirectories if name not in unpacked] - files = [name for name in files if name not in unpacked] # A folder of unpacked distributions holds other people's code, whose SDKs name these # tools whether or not this agent uses them. - elif any(name.endswith(".dist-info") for name in subdirectories): + if Path(directory) != root and any( + name.endswith(".dist-info") for name in subdirectories + ): subdirectories[:] = [] continue subdirectories[:] = [ name for name in subdirectories if not _not_the_agent_dir(name) ] for name in files: - if not _not_the_agent_file(name): - yield Path(directory) / name + path = Path(directory) / name + if not _not_the_agent_file(name) and path.relative_to(root).as_posix() not in unpacked: + yield path def _packages_used(path: Path) -> set[str]: @@ -204,6 +220,7 @@ def _modules_at_top(root: Path) -> set[str]: entry.stem if entry.is_file() else entry.name for entry in root.iterdir() if entry.name not in unpacked + and f"{entry.name}/__init__.py" not in unpacked and (entry.suffix == ".py" or (entry / "__init__.py").is_file()) } except OSError: @@ -221,7 +238,7 @@ def _python_code(text: str) -> str: def _script_kept(hit: re.Match[str]) -> str: kept = hit.group(1) or "" # A lone apostrophe, as in JSX text, isn't a string, so it can't protect a comment after it. - if kept.startswith("'") and (len(kept) == 1 or not kept.endswith("'")): + if kept.startswith("'") and not kept.endswith("'"): return kept.split("//", 1)[0] return kept @@ -267,6 +284,12 @@ def _is_read(path: Path) -> bool: return path.name == "package.json" or path.suffix == ".py" or path.suffix in _SCRIPT_SUFFIXES +def _reading_order(path: Path) -> tuple[bool, str]: + # Python and package.json first: script bundles shouldn't use up the limit before them, and the + # answer shouldn't depend on the order the disk lists files in. + return path.suffix in _SCRIPT_SUFFIXES, str(path) + + def _signature(root: Path) -> tuple[int, int, int]: """What changes when the files a scan reads change: a nested edit or a new checkout.""" latest = count = size = 0 @@ -288,9 +311,7 @@ def _scan(root: Path, _files: tuple[int, int, int]) -> tuple[str, ...]: found: set[str] = set() imported: set[str] = set() scanned = 0 - for path in _agent_files(root): - if not _is_read(path): - continue + for path in sorted(filter(_is_read, _agent_files(root)), key=_reading_order): size = _file_size(path) # Never read, so it mustn't use up the limit before the code that is. if size > _LARGEST_CODE_FILE: diff --git a/src/fi/alk/harness/failure_file.py b/src/fi/alk/harness/failure_file.py index 93b27e57e..2def5ce79 100644 --- a/src/fi/alk/harness/failure_file.py +++ b/src/fi/alk/harness/failure_file.py @@ -13,7 +13,10 @@ def _write_in_place(path: Path, body: str) -> None: - """For a disk with no hard links: still first-wins, but a reader could catch it part-written.""" + """For a disk with no hard links: still first-wins, but a reader could catch it part-written. + + A disk that refuses ``chmod`` keeps the umask's mode: a whole file beats none. + """ try: descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) except FileExistsError: @@ -21,10 +24,13 @@ def _write_in_place(path: Path, body: str) -> None: try: with os.fdopen(descriptor, "w", encoding="utf-8") as file: file.write(body) - os.chmod(path, 0o644) except OSError: path.unlink(missing_ok=True) raise + try: + os.chmod(path, 0o644) + except OSError: + pass def write_failure_file(failure: HarnessFailure, output: Path) -> Path: @@ -39,8 +45,8 @@ def write_failure_file(failure: HarnessFailure, output: Path) -> Path: try: with temporary: temporary.write(body) - os.chmod(temporary.name, 0o644) try: + os.chmod(temporary.name, 0o644) os.link(temporary.name, path) except FileExistsError: pass diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index 0f326bf39..b818c8659 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -9,6 +9,7 @@ import asyncio import json +import secrets from pathlib import Path from typing import Any @@ -247,6 +248,13 @@ def _without_nulls(value: Any) -> Any: MOST_CHOSEN_EVALS = 6 +SECOND_LOOK_READ = "second_look_read" + + +def _new_receipt() -> str: + return secrets.token_hex(4) + + def _stop_message(destination: Path) -> str: said = failure_message(destination) or "This build can't go on." return f"{said} Don't submit again or do more work; tell the person." @@ -307,7 +315,10 @@ def contract_tools( # and this stage cannot tell those two apart from the outside. nudged: set[str] = set() refused = False - looked_again = False + # What a call must hold to count as having read a second look. Calls in one model message run + # before the model reads any answer, together or one after another, so none of them can know it. + second_look_receipts: set[str] = set() + second_look_read = False found: list[str] | None = None @tool( @@ -330,6 +341,11 @@ def contract_tools( # call and a sequence of rejected guesses. schema( { + SECOND_LOOK_READ: { + "type": "string", + "description": "Only when a second look asked for it: the value it gave. " + "It is not part of the contract.", + }, "agent": { "type": "string", "description": "Short lower-case identifier, no spaces. Only a label for " @@ -1086,13 +1102,13 @@ def contract_tools( ), ) async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: - payload = unwrapped(args) - nonlocal refused, looked_again, found + payload = dict(unwrapped(args)) + receipt = payload.pop(SECOND_LOOK_READ, None) or ( + args.get(SECOND_LOOK_READ) if isinstance(args, dict) else None + ) + nonlocal refused, found, second_look_read if refused or (destination / FAILURE_FILE).is_file(): return _ok(_stop_message(destination)) - # Read before the wait: a call running alongside this one mustn't count as the model - # having read the second look. - had_second_look = looked_again if found is None: found = await asyncio.to_thread(browser_tools_found, scan_root) # A call running alongside this one may have refused while this one waited. @@ -1106,17 +1122,30 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: "job is work on websites or a computer." ] ) - verdict = decide(claimed, found, looked_again=had_second_look) + # Once shown, it stays shown: a later rejected submission doesn't earn another look. + second_look_read = second_look_read or ( + isinstance(receipt, str) and receipt in second_look_receipts + ) + verdict = decide(claimed, found, looked_again=second_look_read) if verdict == "refuse": # Set first: if the write fails, the model must not get a resubmission through. refused = True - write_failure_file(agent_type_failure(found), destination) - # An earlier reading in this folder would otherwise build the agent just refused. - (destination / "contract.json").unlink(missing_ok=True) + # First: an earlier reading in this folder would otherwise build the agent just + # refused, even if the write below fails. + try: + (destination / "contract.json").unlink(missing_ok=True) + finally: + write_failure_file(agent_type_failure(found), destination) return _ok(_stop_message(destination)) if verdict == "second_look": - looked_again = True - return _problems([_second_look(claimed, found)]) + issued = _new_receipt() + second_look_receipts.add(issued) + return _problems( + [ + f"{_second_look(claimed, found)} When you submit again, set " + f'{SECOND_LOOK_READ} to "{issued}", which shows you read this.' + ] + ) data_store = payload.get("data_store") if not isinstance(data_store, dict): data_store = {} diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index d67b97a68..4a81ccc52 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -50,6 +50,9 @@ "sites. Only voice and chat agents are supported for now." ) _FINISHED = ".futureagi-build-finished.json" +_STOP = "Don't submit again or do more work; tell the person." +_READ = "read-it" +_REAL_RECEIPT = tools._new_receipt @pytest.fixture(autouse=True) @@ -60,6 +63,11 @@ def _nothing_reaches_a_network(monkeypatch): monkeypatch.delenv("FI_SECRET_KEY", raising=False) +@pytest.fixture(autouse=True) +def _a_known_receipt(monkeypatch): + monkeypatch.setattr(tools, "_new_receipt", lambda: _READ) + + def _tree(root: Path, files: dict[str, str]) -> Path: for name, body in files.items(): path = root / name @@ -150,8 +158,13 @@ def test_first_failure_stands(tmp_path): assert _failure(tmp_path)["code"] == "first" -def test_file_is_world_readable(tmp_path): - path = write_failure_file(_a_failure(), tmp_path) +@pytest.mark.parametrize("umask", [0o022, 0o077]) +def test_file_is_world_readable(tmp_path, umask): + before = os.umask(umask) + try: + path = write_failure_file(_a_failure(), tmp_path) + finally: + os.umask(before) assert stat.S_IMODE(path.stat().st_mode) == 0o644 @@ -161,21 +174,42 @@ def test_writer_leaves_no_temp_file(tmp_path): assert [path.name for path in tmp_path.iterdir()] == [FAILURE_FILE] -def _no_hard_links(monkeypatch): +def _no_hard_links(monkeypatch, number=errno.EXDEV): def refuse(*_args, **_kwargs): - raise OSError(errno.EXDEV, "cross-device link") + raise OSError(number, "no hard links here") monkeypatch.setattr(failure_file.os, "link", refuse) -def test_a_disk_with_no_hard_links_still_gets_the_whole_file(tmp_path, monkeypatch): - _no_hard_links(monkeypatch) +@pytest.mark.parametrize("number", [errno.EXDEV, errno.EPERM, errno.ENOTSUP, errno.EMLINK]) +def test_a_disk_with_no_hard_links_still_gets_the_whole_file(tmp_path, monkeypatch, number): + _no_hard_links(monkeypatch, number) path = write_failure_file(_a_failure(), tmp_path) assert HarnessFailure.model_validate_json(path.read_text()) == _a_failure() assert stat.S_IMODE(path.stat().st_mode) == 0o644 assert [item.name for item in tmp_path.iterdir()] == [FAILURE_FILE] +def test_the_file_is_readable_under_a_strict_umask_with_no_hard_links(tmp_path, monkeypatch): + _no_hard_links(monkeypatch) + before = os.umask(0o077) + try: + path = write_failure_file(_a_failure(), tmp_path) + finally: + os.umask(before) + assert stat.S_IMODE(path.stat().st_mode) == 0o644 + + +def test_a_disk_with_no_permissions_still_gets_the_whole_file(tmp_path, monkeypatch): + def refuse(*_args, **_kwargs): + raise OSError(errno.EPERM, "no permissions here") + + monkeypatch.setattr(failure_file.os, "chmod", refuse) + path = write_failure_file(_a_failure(), tmp_path) + assert HarnessFailure.model_validate_json(path.read_text()) == _a_failure() + assert [item.name for item in tmp_path.iterdir()] == [FAILURE_FILE] + + def test_first_failure_stands_with_no_hard_links(tmp_path, monkeypatch): _no_hard_links(monkeypatch) write_failure_file(_a_failure(), tmp_path) @@ -235,6 +269,12 @@ def test_failure_message_is_empty_when_the_file_cannot_be_read(tmp_path): ({"agent.py": 'TOOL = {"type": "computer_20241022"}\n'}, ["computer-use tool"]), ({"agent.ts": 'const tool = {type: "computer_use_preview"};\n'}, ["computer-use tool"]), ({"agent.py": 'BETA = "computer-use-2025-01-24"\n'}, ["computer-use tool"]), + ({"agent.py": 'TOOLS = [{"computer_use": {"environment": "browser"}}]\n'}, ["computer-use tool"]), + ({"agent.py": 'TOOL = dict(computer_use={"environment": "browser"})\n'}, ["computer-use tool"]), + ({"agent.js": "const u = 'https://x/computer_20241022';\n"}, ["computer-use tool"]), + ({"agent.ts": "const tools = [{computerUse: {environment}}];\n"}, ["computer-use tool"]), + ({"agent.js": "const t = 'computer_20241022'; // x\n"}, ["computer-use tool"]), + ({"agent.jsx": "export const A = () =>

Don't use computer_20241022

\n"}, ["computer-use tool"]), ({"agent.py": 'MODEL = "gemini-2.5-computer-use-preview-10-2025"\n'}, ["computer-use tool"]), ( {"agent.py": 'from google.genai import types\nTOOL = types.ComputerUse(environment="x")\n'}, @@ -282,6 +322,23 @@ def test_browser_tools_found_is_sorted_and_names_each_tool_once(tmp_path): "ui-tests", "api_tests", "playwright-tests", + "web_test", + "qa-testing", + "agent-e2e", + "test_utils", + "test_helpers", + "tests_integration", + "tests_ui", + "tests.old", + "test_selenium", + "tests_selenium", + "test_browser", + "tests_functional", + "tests_acceptance", + "test_scripts", + "testutils", + "testing_utils", + "test_fixtures", ], ) def test_tools_found_skips_these_test_folders(tmp_path, folder): @@ -361,6 +418,13 @@ def test_tools_found_reads_a_folder_with_test_inside_a_word(tmp_path, folder): ("a.py", 'PROMPT = "never do computer-use"\n'), ("a.py", 'SPAN = "gen_ai.computer_use.action"\n'), ("a.jsx", "export const A = () =>

Don't

// ComputerUse\n"), + ("a.js", "const t = 'x'; // computer_20241022\n"), + ("a.py", "computer_use = False\n"), + ("a.ts", "const a = {computer_use: true};\n"), + ("a.py", 'LABEL = f"computer_use: {x}"\n'), + ("a.tsx", "export const A = () => ;\n"), + ("a.py", "computer_use = {}\n"), + ("a.py", 'FLAGS = {"computer_use": {"on": True}}\n'), ], ) def test_browser_tools_found_ignores_what_is_not_the_agent(tmp_path, name, body): @@ -383,7 +447,9 @@ def _unpacked_at_the_top(root: Path) -> None: _tree( root, { - "openai-1.0.dist-info/RECORD": "openai/x.py,sha256=a,1\nsix.py,sha256=b,2\n", + "openai-1.0.dist-info/RECORD": ( + "openai/__init__.py,sha256=a,0\nopenai/x.py,sha256=a,1\nsix.py,sha256=b,2\n" + ), "openai/__init__.py": "", "openai/x.py": 'NAME = "computer_use_preview"\n', "six.py": 'NAME = "computer_20241022"\n', @@ -403,6 +469,81 @@ def test_the_agents_own_folder_is_read_beside_unpacked_packages(tmp_path): assert browser_tools_found(tmp_path) == ["playwright"] +def test_a_distribution_shipping_a_folder_name_does_not_hide_the_customers_folder(tmp_path): + _tree( + tmp_path, + { + "cron_descriptor-2.0.dist-info/RECORD": "tools/compilepos.py,sha256=a,1\n", + "tools/compilepos.py": 'NAME = "computer_20241022"\n', + "tools/browser.py": "import playwright\n", + }, + ) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def _vendored_marker_with_record(root: Path, record: bytes) -> None: + _tree(root, {"vendored.py": 'NAME = "computer_20241022"\n', "x-1.dist-info/METADATA": ""}) + (root / "x-1.dist-info/RECORD").write_bytes(record) + + +def test_a_record_past_its_limit_is_not_believed(tmp_path, monkeypatch): + _vendored_marker_with_record(tmp_path, b"vendored.py,sha256=a,1\n") + monkeypatch.setattr(agent_type_check, "_LARGEST_RECORD", 5) + assert browser_tools_found(tmp_path) == ["computer-use tool"] + + +@pytest.mark.parametrize("record", [b"\xff\xfe\x00binary", b"a\x00b,c\n", b"x" * 200_000 + b",y\n"]) +def test_a_record_that_cannot_be_read_costs_only_itself(tmp_path, record): + _vendored_marker_with_record(tmp_path, record) + assert browser_tools_found(tmp_path) == ["computer-use tool"] + + +def test_a_blank_record_row_is_harmless(tmp_path): + _vendored_marker_with_record(tmp_path, b"\nvendored.py,sha256=a,1\n\n") + assert browser_tools_found(tmp_path) == [] + + +def test_records_past_their_total_limit_are_not_believed(tmp_path, monkeypatch): + _tree( + tmp_path, + { + "a-1.dist-info/RECORD": "a.py,sha256=a,1\n", + "b-1.dist-info/RECORD": "b.py,sha256=a,1\n", + "a.py": "x = 1\n", + "b.py": 'NAME = "computer_20241022"\n', + }, + ) + assert browser_tools_found(tmp_path) == [] + monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", 20) + agent_type_check._scan.cache_clear() + assert browser_tools_found(tmp_path) == ["computer-use tool"] + + +def test_a_record_linked_to_a_device_is_not_read(tmp_path, monkeypatch): + _tree(tmp_path, {"x-1.dist-info/METADATA": "", "a.py": "import playwright\n"}) + (tmp_path / "x-1.dist-info/RECORD").symlink_to("/dev/zero") + real = Path.read_text + + def careful(self, *args, **kwargs): + assert self.name != "RECORD", "read a device to the end of memory" + return real(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", careful) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_a_record_that_is_a_named_pipe_does_not_hang_the_scan(tmp_path): + _tree(tmp_path, {"x-1.dist-info/METADATA": "", "a.py": "import playwright\n"}) + os.mkfifo(tmp_path / "x-1.dist-info/RECORD") + answers = [] + scan = threading.Thread( + target=lambda: answers.append(browser_tools_found(tmp_path)), daemon=True + ) + scan.start() + scan.join(10) + assert answers == [["playwright"]] + + def test_a_browser_package_unpacked_at_the_top_still_counts_when_imported(tmp_path): _tree( tmp_path, @@ -422,12 +563,36 @@ def test_the_scan_stops_at_its_limit_with_what_it_found(tmp_path, monkeypatch): def test_files_too_big_to_read_do_not_use_up_the_limit(tmp_path, monkeypatch): - _tree(tmp_path, {"big.js": "x" * 80, "agent/run.py": "import playwright\n"}) + # Read first by name, so it's the one that would use up the limit. + _tree(tmp_path, {"a_big.py": "x" * 80, "agent/run.py": "import playwright\n"}) monkeypatch.setattr(agent_type_check, "_LARGEST_CODE_FILE", 50) monkeypatch.setattr(agent_type_check, "_LARGEST_SCAN", 60) assert browser_tools_found(tmp_path) == ["playwright"] +def test_python_is_read_before_script_bundles_use_up_the_limit(tmp_path, monkeypatch): + _tree(tmp_path, {"a.js": "x" * 40, "b.js": "x" * 40, "z/run.py": "import playwright\n"}) + monkeypatch.setattr(agent_type_check, "_LARGEST_SCAN", 60) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_package_json_is_read_before_script_bundles(tmp_path, monkeypatch): + _tree( + tmp_path, + {"a.js": "x" * 40, "package.json": json.dumps({"dependencies": {"playwright": "1"}})}, + ) + monkeypatch.setattr(agent_type_check, "_LARGEST_SCAN", 60) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_files_are_read_by_path_whatever_order_the_disk_lists_them(tmp_path, monkeypatch): + _tree(tmp_path, {"a.py": "import selenium\n", "b.py": "import playwright\n"}) + listed = [tmp_path.resolve() / "b.py", tmp_path.resolve() / "a.py"] + monkeypatch.setattr(agent_type_check, "_agent_files", lambda _root: iter(listed)) + monkeypatch.setattr(agent_type_check, "_LARGEST_SCAN", 20) + assert browser_tools_found(tmp_path) == ["selenium"] + + def test_a_second_scan_of_unchanged_files_reuses_the_first(tmp_path, monkeypatch): _tree(tmp_path, {"a.py": "import playwright\n"}) assert browser_tools_found(tmp_path) == ["playwright"] @@ -439,15 +604,34 @@ def read_again(_path): assert browser_tools_found(tmp_path) == ["playwright"] -def test_a_nested_edit_is_scanned_again(tmp_path): - # The same size, so only the time says it changed. - _tree(tmp_path, {"src/agent.py": "import openai\n"}) +@pytest.mark.parametrize("edited", ["src/a.py", "src/b.py"]) +def test_a_nested_edit_is_scanned_again(tmp_path, edited): + # The same size, so only the time says it changed; either file, whichever the walk ends on. + _tree(tmp_path, {"src/a.py": "import openai\n", "src/b.py": "import openai\n"}) assert browser_tools_found(tmp_path) == [] - (tmp_path / "src/agent.py").write_text("import scrapy\n", encoding="utf-8") - os.utime(tmp_path / "src/agent.py", ns=(2**62, 2**62)) + (tmp_path / edited).write_text("import scrapy\n", encoding="utf-8") + os.utime(tmp_path / edited, ns=(2**62, 2**62)) assert browser_tools_found(tmp_path) == ["scrapy"] +@pytest.mark.parametrize( + "name, before, after", + [ + ("agent.ts", "const a = 1;\n", 'const t = "computer_20241022";\n'), + ( + "package.json", + json.dumps({"dependencies": {"zod": "1"}}), + json.dumps({"dependencies": {"playwright": "1"}}), + ), + ], +) +def test_an_edit_outside_python_is_scanned_again(tmp_path, name, before, after): + _tree(tmp_path, {name: before}) + assert browser_tools_found(tmp_path) == [] + (tmp_path / name).write_text(after, encoding="utf-8") + assert browser_tools_found(tmp_path) != [] + + def test_one_file_fewer_is_scanned_again(tmp_path): # The same total size and latest time, so only the count says it changed. _tree(tmp_path, {"a.py": "import openai\n", "b.py": "import scrapy\n"}) @@ -628,7 +812,9 @@ def test_a_build_that_times_out_is_still_a_crash(tmp_path, monkeypatch): def test_ci_runs_these_tests(): workflows = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "sdk-smoke.yml" - assert "tests/harness/test_agent_type_refusal.py" in workflows.read_text(encoding="utf-8") + text = workflows.read_text(encoding="utf-8") + assert "tests/harness/test_agent_type_refusal.py" in text + assert "tests/harness/test_executor_completed_counts.py" in text def test_the_file_is_what_the_platform_reads(): @@ -668,20 +854,150 @@ def test_browser_with_tool_is_refused_on_the_first_submission(tmp_path, browser_ assert not (output / "contract.json").exists() -def test_two_submissions_at_once_both_get_the_second_look(tmp_path, plain_repo): +def _in_one_model_message(tool, modality: str, *, together: bool): + """How the two loops run parallel calls: Gemini's together, Claude Code's one after another.""" + + async def calls(): + if together: + return await asyncio.gather(*(tool.handler(_reading(modality)) for _ in range(2))) + return [await tool.handler(_reading(modality)) for _ in range(2)] + + return asyncio.run(calls()) + + +@pytest.mark.parametrize("together", [True, False]) +@pytest.mark.parametrize("scanned_before", [False, True]) +def test_two_browser_readings_in_one_message_both_get_the_second_look( + tmp_path, plain_repo, together, scanned_before +): output = tmp_path / "out" tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) + if scanned_before: + asyncio.run(tool.handler(_reading("voice", one_liner=""))) + for answer in _in_one_model_message(tool, "browser", together=together): + assert answer.get("is_error") + assert "second-look" in _text(answer) + assert not (output / FAILURE_FILE).exists() + assert _STOP in _text(asyncio.run(tool.handler(_reading("browser", second_look_read=_READ)))) + assert (output / FAILURE_FILE).exists() - async def both(): - return await asyncio.gather( - tool.handler(_reading("browser")), tool.handler(_reading("browser")) - ) - for answer in asyncio.run(both()): +@pytest.mark.parametrize("together", [True, False]) +def test_two_chat_readings_of_a_browser_repo_in_one_message_save_nothing( + tmp_path, browser_repo, together +): + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=browser_repo)) + for answer in _in_one_model_message(tool, "chat", together=together): assert answer.get("is_error") + assert "second-look" in _text(answer) + assert not (output / "contract.json").exists() + + +def _looked_at(tmp_path, repo): + """A handler that has given one second look, and that look's answer.""" + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=repo)) + return tool, asyncio.run(tool.handler(_reading("browser"))) + + +def test_a_second_look_says_what_to_send_back(tmp_path, plain_repo): + _tool, answer = _looked_at(tmp_path, plain_repo) + assert f'second_look_read to "{_READ}"' in _text(answer) + + +def test_each_second_look_gets_its_own_value(tmp_path, plain_repo, monkeypatch): + monkeypatch.setattr(tools, "_new_receipt", _REAL_RECEIPT) + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) + answers = [_text(asyncio.run(tool.handler(_reading("browser")))) for _ in range(2)] + first, second = [answer.split('second_look_read to "')[1].split('"')[0] for answer in answers] + assert first != second + + +def test_an_earlier_value_still_counts(tmp_path, plain_repo, monkeypatch): + # Two calls in one message each got a value; the model may send back either. + values = iter(["first", "second"]) + monkeypatch.setattr(tools, "_new_receipt", lambda: next(values)) + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) + _in_one_model_message(tool, "browser", together=False) + assert _STOP in _text(asyncio.run(tool.handler(_reading("browser", second_look_read="first")))) + + +def test_a_second_look_answered_with_its_value_stands(tmp_path, browser_repo): + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=browser_repo)) + assert asyncio.run(tool.handler(_reading("chat"))).get("is_error") + assert not asyncio.run(tool.handler(_reading("chat", second_look_read=_READ))).get("is_error") + assert (output / "contract.json").exists() + assert "second_look_read" not in (output / "contract.json").read_text(encoding="utf-8") + + +@pytest.mark.parametrize("made_up", ["guess", "", 7, ["read-it"], {"a": 1}, None]) +def test_a_value_it_never_gave_is_not_a_second_look(tmp_path, plain_repo, made_up): + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) + asyncio.run(tool.handler(_reading("browser"))) + answer = asyncio.run(tool.handler(_reading("browser", second_look_read=made_up))) + assert "second-look" in _text(answer) + assert not (output / FAILURE_FILE).exists() + + +def test_a_value_before_any_second_look_is_not_a_second_look(tmp_path, plain_repo): + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) + answer = asyncio.run(tool.handler(_reading("browser", second_look_read=_READ))) + assert "second-look" in _text(answer) assert not (output / FAILURE_FILE).exists() +@pytest.mark.parametrize( + "packed", + [ + {"contract": _reading("browser"), "second_look_read": _READ}, + {"contract": _reading("browser", second_look_read=_READ)}, + {"contract": json.dumps(_reading("browser", second_look_read=_READ))}, + ], +) +def test_the_value_counts_however_the_contract_was_packaged(tmp_path, plain_repo, packed): + tool, _answer = _looked_at(tmp_path, plain_repo) + assert _STOP in _text(asyncio.run(tool.handler(packed))) + + +def test_the_tool_declares_the_value_for_every_model(tmp_path): + # Required, or of another type, it would be refused before the handler on every build. + declared = _submit_tool(contract_tools(tmp_path)).input_schema + assert "string" in declared["properties"]["second_look_read"]["type"] + assert "integer" not in declared["properties"]["second_look_read"]["type"] + assert "second_look_read" not in declared.get("required", []) + + +def test_a_failed_removal_still_writes_the_failure(tmp_path, browser_repo, monkeypatch): + output = tmp_path / "out" + output.mkdir() + (output / "contract.json").write_text("{}", encoding="utf-8") + real = Path.unlink + + def refuse(self, *args, **kwargs): + if self.name == "contract.json": + raise PermissionError("read-only") + return real(self, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", refuse) + with pytest.raises(PermissionError): + _submitter(output, browser_repo)(_reading("browser")) + assert (output / FAILURE_FILE).exists() + + +def test_a_failed_write_still_removes_the_earlier_reading(tmp_path, browser_repo, monkeypatch): + output = tmp_path / "out" + output.mkdir() + (output / "contract.json").write_text("{}", encoding="utf-8") + _failing_write(monkeypatch) + with pytest.raises(OSError, match="disk full"): + _submitter(output, browser_repo)(_reading("browser")) + assert load(output) is None + + def test_a_refused_new_reading_removes_the_earlier_one(tmp_path, browser_repo): output = tmp_path / "out" output.mkdir() @@ -786,7 +1102,7 @@ def test_second_look_then_browser_is_refused_with_no_tools(tmp_path, plain_repo) output = tmp_path / "out" submit = _submitter(output, plain_repo) submit(_reading("browser")) - answer = submit(_reading("browser")) + answer = submit(_reading("browser", second_look_read=_READ)) assert not answer.get("is_error") assert _READ_MESSAGE in _text(answer) assert _failure(output)["details"] == {"agent_type": "browser", "tools": []} @@ -798,7 +1114,7 @@ def test_no_scan_root_behaves_as_no_tool(tmp_path): submit = _submitter(output, None) assert "second-look" in _text(submit(_reading("browser"))) assert not (output / FAILURE_FILE).exists() - submit(_reading("browser")) + submit(_reading("browser", second_look_read=_READ)) assert _failure(output)["details"]["tools"] == [] @@ -825,7 +1141,7 @@ def test_conversation_with_tool_then_same_answer_saves(tmp_path, browser_repo, m output = tmp_path / "out" submit = _submitter(output, browser_repo) submit(_reading(modality)) - assert not submit(_reading(modality)).get("is_error") + assert not submit(_reading(modality, second_look_read=_READ)).get("is_error") assert load(output).modality == modality assert not (output / FAILURE_FILE).exists() @@ -864,7 +1180,7 @@ def counted(root): submit = _submitter(output, browser_repo) assert "second-look" in _text(submit(_reading("chat"))) # A rejected submission after the look doesn't earn another one. - rejected = submit(_reading("chat", real_use_cases=[])) + rejected = submit(_reading("chat", real_use_cases=[], second_look_read=_READ)) assert rejected["is_error"] and "second-look" not in _text(rejected) assert not submit(_reading("chat")).get("is_error") assert load(output).modality == "chat" diff --git a/tests/harness/test_executor_completed_counts.py b/tests/harness/test_executor_completed_counts.py index e6de4d915..a00ca6131 100644 --- a/tests/harness/test_executor_completed_counts.py +++ b/tests/harness/test_executor_completed_counts.py @@ -58,14 +58,15 @@ def _job(source): ) -def test_the_failure_a_build_left_is_the_jobs_failure(tmp_path, monkeypatch): +@pytest.mark.parametrize("exit_code", [1, 78]) +def test_the_failure_a_build_left_is_the_jobs_failure(tmp_path, monkeypatch, exit_code): source = tmp_path / "source" source.mkdir() output = tmp_path / "output" async def refused(args): write_failure_file(agent_type_failure(["playwright"]), Path(args.out)) - return 78 + return exit_code monkeypatch.setattr("fi.alk.harness.cli._auto", refused) status = asyncio.run(HarnessExecutor().run(_job(source), source=source, output=output)) @@ -73,14 +74,36 @@ async def refused(args): assert status.failure == agent_type_failure(["playwright"]) -def test_an_unreadable_failure_file_falls_back_to_the_events(tmp_path, monkeypatch): +@pytest.mark.parametrize("exit_code", [0, 2]) +def test_a_build_that_ran_reports_no_failure_whatever_the_folder_holds( + tmp_path, monkeypatch, exit_code +): + source = tmp_path / "source" + source.mkdir() + output = tmp_path / "output" + + async def ran(args): + write_failure_file(agent_type_failure(["playwright"]), Path(args.out)) + return exit_code + + monkeypatch.setattr("fi.alk.harness.cli._auto", ran) + status = asyncio.run(HarnessExecutor().run(_job(source), source=source, output=output)) + assert status.stage is HarnessStage.COMPLETED + assert status.failure is None + + +@pytest.mark.parametrize("unreadable", ["malformed", "a folder"]) +def test_an_unreadable_failure_file_falls_back_to_the_events(tmp_path, monkeypatch, unreadable): source = tmp_path / "source" source.mkdir() output = tmp_path / "output" async def crashed(args): Path(args.out).mkdir() - (Path(args.out) / FAILURE_FILE).write_text("{") + if unreadable == "malformed": + (Path(args.out) / FAILURE_FILE).write_text("{") + else: + (Path(args.out) / FAILURE_FILE).mkdir() return 1 monkeypatch.setattr("fi.alk.harness.cli._auto", crashed) From 880f9a22bd88acd7e512196b2410717104892377 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Thu, 5 Nov 2026 12:19:38 +0530 Subject: [PATCH 06/15] fix(harness): end the second look, and read what the scan missed - Take the call after an unanswered second look as the model's answer, so a model that never sends the look's value back can still be saved or refused instead of looking again until its turns run out. - Decide each call on the value it carries, so a value-less call in the same message can't undo an answer that was just saved. - Skip a file the tokenizer can't read (a NUL byte on Python 3.12) instead of failing every scan. - Keep a JS regex literal like a string, so a `/*` inside one no longer hides the rest of the file. - Apply the RECORD rule to every folder holding a dist-info, so agent code beside bundled packages in a subfolder is still read. - Read Python and package.json saved with a BOM. - Read devDependencies, and add puppeteer-extra, playwright-extra, webdriverio and @crawlee/ packages. TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_type_check.py | 69 ++++++--- src/fi/alk/harness/tools.py | 21 ++- tests/harness/test_agent_type_refusal.py | 187 ++++++++++++++++++++--- 3 files changed, 229 insertions(+), 48 deletions(-) diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index b6ab6e30c..97bad88bc 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -81,8 +81,13 @@ "firecrawl", "@mendable/firecrawl-js", "apify-client", + "puppeteer-extra", + "playwright-extra", + "webdriverio", } ) +# Every package in these scopes drives a browser. +BROWSER_SCOPES = ("@crawlee/",) # The tools' own API names only: the bare words turn up in prompts and tracing attributes. TOOL_MARKER = re.compile( r"\bcomputer[_-]use[_-]preview\b|\bcomputer-use-20\d\d-\d\d-\d\d\b|\bComputerUse\b" @@ -107,8 +112,12 @@ # Strings are matched first and kept, so "https://..." or "src/**/*.ts" isn't read as a comment. # An unclosed one ends at its line or the file: a start that could fail makes a customer's odd file # cost the square of its size, which holds up the build's conversation until it's done. +# A regex literal is kept the same way, so the "/*" in /\/*$/ doesn't hide the rest of the file. A +# slash where a value is expected starts one: after punctuation or at a line's start, never after a +# name or a number, where it divides. _SCRIPT_COMMENT_OR_STRING = re.compile( - r"""("(?:\\.|[^"\\\n])*(?:"|(?=\n)|\\?\Z)|'(?:\\.|[^'\\\n])*(?:'|(?=\n)|\\?\Z)|`(?:\\.|[^`\\])*(?:`|\\?\Z))""" + r"""("(?:\\.|[^"\\\n])*(?:"|(?=\n)|\\?\Z)|'(?:\\.|[^'\\\n])*(?:'|(?=\n)|\\?\Z)|`(?:\\.|[^`\\])*(?:`|\\?\Z)""" + r"""|(?])[ \t]*/(?![/*])(?:\\.|\[(?:\\.|[^\]\\\n])*\]?|[^/\\\n\[])*(?:/|(?=\n)|\\?\Z))""" r"""|//[^\n]*|/\*.*?(?:\*/|\Z)""", re.DOTALL, ) @@ -170,21 +179,20 @@ def _unpacked_at_top(root: Path) -> set[str]: def _agent_files(root: Path) -> Iterator[Path]: - unpacked = _unpacked_at_top(root) + # Unpacked distributions hold other people's code, whose SDKs name these tools whether or not + # this agent uses them. Their records name those files wherever they were unpacked, so the + # agent's own code beside them, as in a Lambda folder, is still read. + unpacked: set[Path] = set() for directory, subdirectories, files in os.walk(root): - # A folder of unpacked distributions holds other people's code, whose SDKs name these - # tools whether or not this agent uses them. - if Path(directory) != root and any( - name.endswith(".dist-info") for name in subdirectories - ): - subdirectories[:] = [] - continue + here = Path(directory) + if any(name.endswith(".dist-info") for name in subdirectories): + unpacked.update(here / name for name in _unpacked_at_top(here)) subdirectories[:] = [ name for name in subdirectories if not _not_the_agent_dir(name) ] for name in files: - path = Path(directory) / name - if not _not_the_agent_file(name) and path.relative_to(root).as_posix() not in unpacked: + path = here / name + if not _not_the_agent_file(name) and path not in unpacked: yield path @@ -192,13 +200,23 @@ def _packages_used(path: Path) -> set[str]: try: if not path.is_file(): return set() - manifest = json.loads(path.read_text(encoding="utf-8")) + # Windows editors often save a BOM, which json refuses. + manifest = json.loads(path.read_text(encoding="utf-8").removeprefix("\ufeff")) except (OSError, ValueError, RecursionError): return set() - dependencies = manifest.get("dependencies") if isinstance(manifest, dict) else None - if not isinstance(dependencies, dict): + if not isinstance(manifest, dict): return set() - return BROWSER_PACKAGES.intersection(dependencies) + names: set[str] = set() + # A test-automation agent usually lists its browser library as a dev dependency. + for field in ("dependencies", "devDependencies"): + listed = manifest.get(field) + if isinstance(listed, dict): + names.update(listed) + return { + name + for name in names + if name in BROWSER_PACKAGES or name.startswith(BROWSER_SCOPES) + } def _imports_in(path: Path) -> set[str]: @@ -207,7 +225,8 @@ def _imports_in(path: Path) -> set[str]: try: if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: return set() - return _imports(ast.parse(path.read_text(encoding="utf-8"))) + # From bytes, so a BOM or a coding line is honoured as Python itself would. + return _imports(ast.parse(path.read_bytes())) except (OSError, ValueError, SyntaxError, RecursionError, MemoryError): return set() @@ -319,12 +338,18 @@ def _scan(root: Path, _files: tuple[int, int, int]) -> tuple[str, ...]: scanned += size if scanned > _LARGEST_SCAN: break - if path.name == "package.json": - found |= _packages_used(path) - else: - found |= _markers_in(path) - if path.suffix == ".py": - imported |= _imports_in(path) + # One file at a time: the tokenizer raises more than it documents (SystemError on a NUL + # byte in 3.12), and one odd file mustn't stop every reading from being checked. + try: + if path.name == "package.json": + found |= _packages_used(path) + else: + markers = _markers_in(path) + names = _imports_in(path) if path.suffix == ".py" else set() + found |= markers + imported |= names + except Exception: + continue local = _modules_at_top(root) for name in imported: top = name.split(".")[0] diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index b818c8659..f77a3d73c 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -318,7 +318,9 @@ def contract_tools( # What a call must hold to count as having read a second look. Calls in one model message run # before the model reads any answer, together or one after another, so none of them can know it. second_look_receipts: set[str] = set() - second_look_read = False + # Whether the latest second look has had no value back yet. A model that never sends one back + # must still be able to answer, or every submission earns another look until the turns run out. + look_unanswered = False found: list[str] | None = None @tool( @@ -1106,7 +1108,9 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: receipt = payload.pop(SECOND_LOOK_READ, None) or ( args.get(SECOND_LOOK_READ) if isinstance(args, dict) else None ) - nonlocal refused, found, second_look_read + nonlocal refused, found, look_unanswered + # Taken before the scan's wait: a call already running when a look went out hasn't read it. + looked_before = bool(second_look_receipts) if refused or (destination / FAILURE_FILE).is_file(): return _ok(_stop_message(destination)) if found is None: @@ -1122,11 +1126,15 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: "job is work on websites or a computer." ] ) - # Once shown, it stays shown: a later rejected submission doesn't earn another look. - second_look_read = second_look_read or ( - isinstance(receipt, str) and receipt in second_look_receipts + # A call holding a value read a look, so its answer stands. Without one, the call after a + # look that no value came back for is taken as the answer. Once a value came back, a call + # without one may have run beside it unread, so it gets a look of its own. + read_it = isinstance(receipt, str) and receipt in second_look_receipts + if read_it: + look_unanswered = False + verdict = decide( + claimed, found, looked_again=read_it or (looked_before and look_unanswered) ) - verdict = decide(claimed, found, looked_again=second_look_read) if verdict == "refuse": # Set first: if the write fails, the model must not get a resubmission through. refused = True @@ -1140,6 +1148,7 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: if verdict == "second_look": issued = _new_receipt() second_look_receipts.add(issued) + look_unanswered = True return _problems( [ f"{_second_look(claimed, found)} When you submit again, set " diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index 4a81ccc52..2e9de3a0d 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -380,6 +380,88 @@ def test_an_odd_script_file_stays_quick(tmp_path, body): assert time.monotonic() - started < 5 +@pytest.mark.parametrize( + "line", + [ + 'const base = url.replace(/\\/*$/, "");', + "/src\\/*/.test(f);", + "if (/src\\/*/.test(f)) {}", + "const kept = files.filter((f) => /src\\/*/.test(f));", + "const ratio = a / b / c;", + ], + ids=["replace", "line-start", "condition", "arrow", "division"], +) +def test_a_slash_in_script_code_hides_nothing_after_it(tmp_path, line): + _tree(tmp_path, {"agent.js": f'{line}\nconst tool = {{ type: "computer_20250124" }};\n'}) + assert browser_tools_found(tmp_path) == ["computer-use tool"] + + +@pytest.mark.parametrize( + "body", + [ + '/* const tool = { type: "computer_20250124" }; */\n', + 'const a = 1; /* { type: "computer_20250124" } */\n', + 'const ratio = a / b; // { type: "computer_20250124" }\n', + ], + ids=["line-start", "after-code", "after-division"], +) +def test_a_script_comment_still_hides_what_it_holds(tmp_path, body): + _tree(tmp_path, {"agent.js": body}) + assert browser_tools_found(tmp_path) == [] + + +@pytest.mark.parametrize( + "line", + ["const r = /[/*]/;", "const r = " + "(/[" * 20_000], + ids=["slash-in-a-class", "unclosed-classes"], +) +def test_an_odd_regex_literal_stays_quick_and_hides_nothing(tmp_path, line): + _tree(tmp_path, {"agent.js": f'{line}\nconst tool = {{ type: "computer_20250124" }};\n'}) + started = time.monotonic() + assert browser_tools_found(tmp_path) == ["computer-use tool"] + assert time.monotonic() - started < 5 + + +def test_a_file_the_tokenizer_breaks_on_costs_only_that_file(tmp_path): + (tmp_path / "blob.py").write_bytes(b" x = 1\n\x00") + (tmp_path / "agent.py").write_text( + "from playwright.sync_api import sync_playwright\n", encoding="utf-8" + ) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_a_python_file_saved_with_a_bom_is_read(tmp_path): + (tmp_path / "agent.py").write_bytes( + b"\xef\xbb\xbffrom playwright.sync_api import sync_playwright\n" + ) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_a_package_json_saved_with_a_bom_is_read(tmp_path): + (tmp_path / "package.json").write_bytes( + b"\xef\xbb\xbf" + json.dumps({"dependencies": {"puppeteer": "22.0.0"}}).encode() + ) + assert browser_tools_found(tmp_path) == ["puppeteer"] + + +@pytest.mark.parametrize( + "manifest, expected", + [ + ({"devDependencies": {"puppeteer": "22.0.0"}}, ["puppeteer"]), + ( + {"dependencies": {"puppeteer-extra": "1", "webdriverio": "8"}}, + ["puppeteer-extra", "webdriverio"], + ), + ({"dependencies": {"playwright-extra": "4"}}, ["playwright-extra"]), + ({"dependencies": {"@crawlee/playwright": "3"}}, ["@crawlee/playwright"]), + ], + ids=["dev", "extra-and-webdriverio", "playwright-extra", "crawlee-scope"], +) +def test_package_json_names_these_browser_packages(tmp_path, manifest, expected): + _tree(tmp_path, {"package.json": json.dumps(manifest)}) + assert browser_tools_found(tmp_path) == expected + + @pytest.mark.parametrize( "folder", ["latest", "contest", "e2e_agent", "testing_agent", "src/testing_agent", "test_runner"] ) @@ -411,7 +493,6 @@ def test_tools_found_reads_a_folder_with_test_inside_a_word(tmp_path, folder): ("app.test.js", 'const tool = "computer_20241022";\n'), ("pnpm-lock.yaml", "playwright: 1\n@playwright/mcp: 1\n"), ("requirements.txt", "playwright\n"), - ("package.json", json.dumps({"devDependencies": {"playwright": "1"}})), ("tests/package.json", json.dumps({"dependencies": {"playwright": "1"}})), ("a.py", "# unlike computer_20241022, this only talks\n"), ("a.ts", '// see @playwright/mcp\n/* computer_use\n and ComputerUse */\nconst a = 1;\n'), @@ -437,12 +518,25 @@ def test_unpacked_distributions_are_not_the_agent(tmp_path): tmp_path, { "lib/anthropic-1.0.dist-info/METADATA": "x\n", + "lib/anthropic-1.0.dist-info/RECORD": "anthropic/types.py,sha256=a,1\n", "lib/anthropic/types.py": 'NAME = "computer_20241022"\n', }, ) assert browser_tools_found(tmp_path) == [] +def test_the_agents_code_beside_packages_in_a_subfolder_is_read(tmp_path): + _tree( + tmp_path, + { + "lambda/handler.py": "from playwright.sync_api import sync_playwright\n", + "lambda/requests-2.31.0.dist-info/RECORD": "requests/__init__.py,sha256=a,1\n", + "lambda/requests/__init__.py": 'NAME = "computer_20241022"\n', + }, + ) + assert browser_tools_found(tmp_path) == ["playwright"] + + def _unpacked_at_the_top(root: Path) -> None: _tree( root, @@ -865,16 +959,12 @@ async def calls(): return asyncio.run(calls()) -@pytest.mark.parametrize("together", [True, False]) -@pytest.mark.parametrize("scanned_before", [False, True]) -def test_two_browser_readings_in_one_message_both_get_the_second_look( - tmp_path, plain_repo, together, scanned_before -): +# Calls waiting on the first scan together are the only siblings the handler can tell apart from a +# later message's call; one after another, the second is taken as the model's answer. +def test_two_browser_readings_waiting_on_the_scan_both_get_the_second_look(tmp_path, plain_repo): output = tmp_path / "out" tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) - if scanned_before: - asyncio.run(tool.handler(_reading("voice", one_liner=""))) - for answer in _in_one_model_message(tool, "browser", together=together): + for answer in _in_one_model_message(tool, "browser", together=True): assert answer.get("is_error") assert "second-look" in _text(answer) assert not (output / FAILURE_FILE).exists() @@ -882,18 +972,65 @@ def test_two_browser_readings_in_one_message_both_get_the_second_look( assert (output / FAILURE_FILE).exists() -@pytest.mark.parametrize("together", [True, False]) -def test_two_chat_readings_of_a_browser_repo_in_one_message_save_nothing( - tmp_path, browser_repo, together +def test_two_chat_readings_of_a_browser_repo_waiting_on_the_scan_save_nothing( + tmp_path, browser_repo ): output = tmp_path / "out" tool = _submit_tool(contract_tools(output, scan_root=browser_repo)) - for answer in _in_one_model_message(tool, "chat", together=together): + for answer in _in_one_model_message(tool, "chat", together=True): assert answer.get("is_error") assert "second-look" in _text(answer) assert not (output / "contract.json").exists() +def _looks(answers) -> list[bool]: + return ["second-look" in _text(answer) for answer in answers] + + +def test_a_chat_answer_without_the_value_stands_after_one_look(tmp_path, browser_repo): + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=browser_repo)) + answers = [asyncio.run(tool.handler(_reading("chat"))) for _ in range(5)] + assert _looks(answers) == [True, False, False, False, False] + assert not answers[1].get("is_error") + assert load(output).modality == "chat" + assert not (output / FAILURE_FILE).exists() + + +def test_a_browser_answer_without_the_value_stands_after_one_look(tmp_path, plain_repo): + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) + answers = [asyncio.run(tool.handler(_reading("browser"))) for _ in range(5)] + assert _looks(answers) == [True, False, False, False, False] + assert _STOP in _text(answers[1]) + assert _failure(output)["code"] == "agent_type_unsupported" + assert not (output / "contract.json").exists() + + +@pytest.mark.parametrize("together", [True, False]) +def test_a_call_without_the_value_beside_an_answer_does_not_undo_it( + tmp_path, plain_repo, together +): + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) + assert "second-look" in _text(asyncio.run(tool.handler(_reading("browser")))) + + async def next_message(): + calls = [ + tool.handler(_reading("chat", second_look_read=_READ)), + tool.handler(_reading("browser")), + ] + if together: + return await asyncio.gather(*calls) + return [await one for one in calls] + + chat_answer, browser_answer = asyncio.run(next_message()) + assert not chat_answer.get("is_error") + assert "second-look" in _text(browser_answer) + assert load(output).modality == "chat" + assert not (output / FAILURE_FILE).exists() + + def _looked_at(tmp_path, repo): """A handler that has given one second look, and that look's answer.""" tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=repo)) @@ -908,7 +1045,7 @@ def test_a_second_look_says_what_to_send_back(tmp_path, plain_repo): def test_each_second_look_gets_its_own_value(tmp_path, plain_repo, monkeypatch): monkeypatch.setattr(tools, "_new_receipt", _REAL_RECEIPT) tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) - answers = [_text(asyncio.run(tool.handler(_reading("browser")))) for _ in range(2)] + answers = [_text(one) for one in _in_one_model_message(tool, "browser", together=True)] first, second = [answer.split('second_look_read to "')[1].split('"')[0] for answer in answers] assert first != second @@ -919,7 +1056,7 @@ def test_an_earlier_value_still_counts(tmp_path, plain_repo, monkeypatch): monkeypatch.setattr(tools, "_new_receipt", lambda: next(values)) output = tmp_path / "out" tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) - _in_one_model_message(tool, "browser", together=False) + assert all(_looks(_in_one_model_message(tool, "browser", together=True))) assert _STOP in _text(asyncio.run(tool.handler(_reading("browser", second_look_read="first")))) @@ -932,14 +1069,22 @@ def test_a_second_look_answered_with_its_value_stands(tmp_path, browser_repo): assert "second_look_read" not in (output / "contract.json").read_text(encoding="utf-8") +def _answered(tmp_path, repo): + """A handler whose second look came back with its value, as a chat answer that was saved.""" + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=repo)) + assert "second-look" in _text(asyncio.run(tool.handler(_reading("browser")))) + assert not asyncio.run(tool.handler(_reading("chat", second_look_read=_READ))).get("is_error") + return tool + + @pytest.mark.parametrize("made_up", ["guess", "", 7, ["read-it"], {"a": 1}, None]) def test_a_value_it_never_gave_is_not_a_second_look(tmp_path, plain_repo, made_up): output = tmp_path / "out" - tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) - asyncio.run(tool.handler(_reading("browser"))) + tool = _answered(tmp_path, plain_repo) answer = asyncio.run(tool.handler(_reading("browser", second_look_read=made_up))) assert "second-look" in _text(answer) assert not (output / FAILURE_FILE).exists() + assert load(output).modality == "chat" def test_a_value_before_any_second_look_is_not_a_second_look(tmp_path, plain_repo): @@ -959,7 +1104,7 @@ def test_a_value_before_any_second_look_is_not_a_second_look(tmp_path, plain_rep ], ) def test_the_value_counts_however_the_contract_was_packaged(tmp_path, plain_repo, packed): - tool, _answer = _looked_at(tmp_path, plain_repo) + tool = _answered(tmp_path, plain_repo) assert _STOP in _text(asyncio.run(tool.handler(packed))) @@ -1167,7 +1312,7 @@ def test_conversation_with_no_tool_saves_with_no_second_look(tmp_path, plain_rep assert load(output).modality == modality -def test_only_one_second_look_and_one_scan(tmp_path, browser_repo, monkeypatch): +def test_one_scan_and_at_most_one_more_look_after_an_answer(tmp_path, browser_repo, monkeypatch): output = tmp_path / "out" scans = [] real_scan = tools.browser_tools_found @@ -1179,9 +1324,11 @@ def counted(root): monkeypatch.setattr(tools, "browser_tools_found", counted) submit = _submitter(output, browser_repo) assert "second-look" in _text(submit(_reading("chat"))) - # A rejected submission after the look doesn't earn another one. + # An answer with the value stands, even when the contract is then sent back for another fix. rejected = submit(_reading("chat", real_use_cases=[], second_look_read=_READ)) assert rejected["is_error"] and "second-look" not in _text(rejected) + # Without the value it could have run beside that answer, so it gets one look of its own. + assert "second-look" in _text(submit(_reading("chat"))) assert not submit(_reading("chat")).get("is_error") assert load(output).modality == "chat" assert scans == [browser_repo] From fd726861184cf5964bce32af8ce079e149ae7f45 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Thu, 5 Nov 2026 15:24:55 +0530 Subject: [PATCH 07/15] fix(harness): close a regex literal's bracket at its line end An unclosed character class could end either at a "]" or nowhere, so a run of "[" could be split many ways. Make its end a "]", the line's end or the file's end, so each input has one reading. TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_type_check.py | 2 +- tests/harness/test_agent_type_refusal.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index 97bad88bc..6ef0e1251 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -117,7 +117,7 @@ # name or a number, where it divides. _SCRIPT_COMMENT_OR_STRING = re.compile( r"""("(?:\\.|[^"\\\n])*(?:"|(?=\n)|\\?\Z)|'(?:\\.|[^'\\\n])*(?:'|(?=\n)|\\?\Z)|`(?:\\.|[^`\\])*(?:`|\\?\Z)""" - r"""|(?])[ \t]*/(?![/*])(?:\\.|\[(?:\\.|[^\]\\\n])*\]?|[^/\\\n\[])*(?:/|(?=\n)|\\?\Z))""" + r"""|(?])[ \t]*/(?![/*])(?:\\.|\[(?:\\.|[^\]\\\n])*(?:\]|(?=\n)|\\?\Z)|[^/\\\n\[])*(?:/|(?=\n)|\\?\Z))""" r"""|//[^\n]*|/\*.*?(?:\*/|\Z)""", re.DOTALL, ) diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index 2e9de3a0d..bfeff729e 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -412,8 +412,13 @@ def test_a_script_comment_still_hides_what_it_holds(tmp_path, body): @pytest.mark.parametrize( "line", - ["const r = /[/*]/;", "const r = " + "(/[" * 20_000], - ids=["slash-in-a-class", "unclosed-classes"], + [ + "const r = /[/*]/;", + "const r = " + "(/[" * 20_000, + "const r = /[" + "[" * 20_000, + "const r = /[" + "[" * 20_000 + "\\", + ], + ids=["slash-in-a-class", "unclosed-classes", "repeated-brackets", "repeated-then-escape"], ) def test_an_odd_regex_literal_stays_quick_and_hides_nothing(tmp_path, line): _tree(tmp_path, {"agent.js": f'{line}\nconst tool = {{ type: "computer_20250124" }};\n'}) From 4435f0e6823b5f5feb3cf4077b26472f8c29e82c Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Fri, 6 Nov 2026 11:34:46 +0530 Subject: [PATCH 08/15] fix(harness): bound record reads across folders, and keep one look per answer - One record budget for the whole walk; linked dist-info folders and records are skipped, and rows stay text. - The same answer re-sent without the value stands; a made-up value no longer ends the unanswered look. - Calls in one message share one scan. - A regex literal's closing slash must read as one, so a division that starts a line or a slash in JSX text isn't taken for a literal. - Python markers are read from bytes, as imports are. - Tests pin each of these, the accepted residual, and three scan paths that were unpinned. TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_type_check.py | 50 +++-- src/fi/alk/harness/tools.py | 22 +- tests/harness/test_agent_type_refusal.py | 268 ++++++++++++++++++++--- 3 files changed, 291 insertions(+), 49 deletions(-) diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index 6ef0e1251..32bf740ad 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -114,10 +114,15 @@ # cost the square of its size, which holds up the build's conversation until it's done. # A regex literal is kept the same way, so the "/*" in /\/*$/ doesn't hide the rest of the file. A # slash where a value is expected starts one: after punctuation or at a line's start, never after a -# name or a number, where it divides. +# name or a number, where it divides. Its closing slash must be followed as a regex's is, by flags +# and then no name, quote, "/" or "*": a division that starts a line, or a "/" in JSX text, then +# reads as division, not as a literal that eats a string's opening quote. +# Such a start can fail, so a class holds no "[" and the rest no "]": a failed literal then can't +# run over the classes of the slashes after it, which would make the line cost its square. _SCRIPT_COMMENT_OR_STRING = re.compile( r"""("(?:\\.|[^"\\\n])*(?:"|(?=\n)|\\?\Z)|'(?:\\.|[^'\\\n])*(?:'|(?=\n)|\\?\Z)|`(?:\\.|[^`\\])*(?:`|\\?\Z)""" - r"""|(?])[ \t]*/(?![/*])(?:\\.|\[(?:\\.|[^\]\\\n])*(?:\]|(?=\n)|\\?\Z)|[^/\\\n\[])*(?:/|(?=\n)|\\?\Z))""" + r"""|(?])[ \t]*/(?![/*])(?:\\.|\[(?:\\.|[^\]\[\\\n])*(?:\]|(?=\n)|\\?\Z)|[^/\\\n\[\]])*""" + r"""(?:/(?=[dgimsuvy]*(?![\w$/*"'`]))|(?=\n)|\\?\Z))""" r"""|//[^\n]*|/\*.*?(?:\*/|\Z)""", re.DOTALL, ) @@ -149,26 +154,30 @@ def _not_the_agent_file(name: str) -> bool: return any(fnmatch.fnmatchcase(lowered, pattern) for pattern in _TEST_FILES) -def _unpacked_at_top(root: Path) -> set[str]: +def _unpacked_at_top(root: Path, left: list[int] | None = None) -> set[str]: """The files ``pip install -t .`` put beside the agent's code, by each distribution's record. Exact paths: one distribution shipping ``tools/x.py`` says nothing about the customer's own - ``tools/``. + ``tools/``. ``left`` is the record bytes still to read, shared across one walk's folders. """ names: set[str] = set() try: records = sorted(entry for entry in root.iterdir() if entry.name.endswith(".dist-info")) except OSError: return names - read = 0 + if left is None: + left = [_LARGEST_RECORDS] for entry in records: try: record = entry / "RECORD" + # pip writes neither as a link, and following them reads one record once per link. + if entry.is_symlink() or record.is_symlink(): + continue # A link to a device or a named pipe reports no size, and reading it never ends. if not record.is_file() or record.stat().st_size > _LARGEST_RECORD: continue - read += record.stat().st_size - if read > _LARGEST_RECORDS: + left[0] -= record.stat().st_size + if left[0] < 0: break names.update( row[0] for row in csv.reader(io.StringIO(record.read_text(encoding="utf-8"))) if row @@ -181,19 +190,22 @@ def _unpacked_at_top(root: Path) -> set[str]: def _agent_files(root: Path) -> Iterator[Path]: # Unpacked distributions hold other people's code, whose SDKs name these tools whether or not # this agent uses them. Their records name those files wherever they were unpacked, so the - # agent's own code beside them, as in a Lambda folder, is still read. - unpacked: set[Path] = set() + # agent's own code beside them, as in a Lambda folder, is still read. One budget for the whole + # walk: a repository can hold any number of folders. Rows stay text, a fraction of a Path's size. + unpacked: set[str] = set() + left = [_LARGEST_RECORDS] for directory, subdirectories, files in os.walk(root): here = Path(directory) if any(name.endswith(".dist-info") for name in subdirectories): - unpacked.update(here / name for name in _unpacked_at_top(here)) + unpacked.update( + os.path.join(directory, name) for name in _unpacked_at_top(here, left) + ) subdirectories[:] = [ name for name in subdirectories if not _not_the_agent_dir(name) ] for name in files: - path = here / name - if not _not_the_agent_file(name) and path not in unpacked: - yield path + if not _not_the_agent_file(name) and os.path.join(directory, name) not in unpacked: + yield here / name def _packages_used(path: Path) -> set[str]: @@ -246,11 +258,12 @@ def _modules_at_top(root: Path) -> set[str]: return set() -def _python_code(text: str) -> str: +def _python_code(source: bytes) -> str: + # From bytes, as imports are read, so a coding line or a BOM is honoured as Python itself would. return " ".join( token.string - for token in tokenize.generate_tokens(io.StringIO(text).readline) - if token.type != tokenize.COMMENT + for token in tokenize.tokenize(io.BytesIO(source).readline) + if token.type not in (tokenize.COMMENT, tokenize.ENCODING) ) @@ -266,11 +279,10 @@ def _markers_in(path: Path) -> set[str]: try: if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: return set() - text = path.read_text(encoding="utf-8") code = ( - _python_code(text) + _python_code(path.read_bytes()) if path.suffix == ".py" - else _SCRIPT_COMMENT_OR_STRING.sub(_script_kept, text) + else _SCRIPT_COMMENT_OR_STRING.sub(_script_kept, path.read_text(encoding="utf-8")) ) except (OSError, ValueError, SyntaxError, tokenize.TokenError): return set() diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index f77a3d73c..fd7653952 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -321,7 +321,12 @@ def contract_tools( # Whether the latest second look has had no value back yet. A model that never sends one back # must still be able to answer, or every submission earns another look until the turns run out. look_unanswered = False + # Whether the last answer that carried a value read browser. The same answer again, without the + # value, is still that answer: a fix after the contract was sent back mustn't earn a new look. + answered_browser: bool | None = None found: list[str] | None = None + # Calls in one message share one scan: each running its own takes that many times longer. + scan: asyncio.Future[list[str]] | None = None @tool( "submit_contract", @@ -1108,13 +1113,17 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: receipt = payload.pop(SECOND_LOOK_READ, None) or ( args.get(SECOND_LOOK_READ) if isinstance(args, dict) else None ) - nonlocal refused, found, look_unanswered + nonlocal refused, found, look_unanswered, answered_browser, scan # Taken before the scan's wait: a call already running when a look went out hasn't read it. looked_before = bool(second_look_receipts) if refused or (destination / FAILURE_FILE).is_file(): return _ok(_stop_message(destination)) if found is None: - found = await asyncio.to_thread(browser_tools_found, scan_root) + # A scan that ended while nothing is set raised, so this call runs it again. + if scan is None or scan.done(): + scan = asyncio.ensure_future(asyncio.to_thread(browser_tools_found, scan_root)) + # Shielded: one call cancelled mustn't cancel the scan the others wait on. + found = await asyncio.shield(scan) # A call running alongside this one may have refused while this one waited. if refused or (destination / FAILURE_FILE).is_file(): return _ok(_stop_message(destination)) @@ -1128,12 +1137,17 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: ) # A call holding a value read a look, so its answer stands. Without one, the call after a # look that no value came back for is taken as the answer. Once a value came back, a call - # without one may have run beside it unread, so it gets a look of its own. + # without one that reads the same stands too; one that reads otherwise may have run beside + # that answer unread, so it gets a look of its own. read_it = isinstance(receipt, str) and receipt in second_look_receipts if read_it: look_unanswered = False + answered_browser = is_browser(claimed) + same_answer = answered_browser is not None and answered_browser == is_browser(claimed) verdict = decide( - claimed, found, looked_again=read_it or (looked_before and look_unanswered) + claimed, + found, + looked_again=read_it or (looked_before and (look_unanswered or same_answer)), ) if verdict == "refuse": # Set first: if the write fails, the model must not get a resubmission through. diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index bfeff729e..a53270a24 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -384,12 +384,24 @@ def test_an_odd_script_file_stays_quick(tmp_path, body): "line", [ 'const base = url.replace(/\\/*$/, "");', - "/src\\/*/.test(f);", + "const ok = 1;\n/src\\/*/.test(f);", "if (/src\\/*/.test(f)) {}", "const kept = files.filter((f) => /src\\/*/.test(f));", - "const ratio = a / b / c;", + 'const ratio = a / b; const label = "1 / 2 /*";', + 'const r = a\n / b; const p = "src/**/*.ts";', + 'x = a\n/ b; s = "http://h/*"; const t = { type: "computer_20250124" };', + "const C = () => /src/**/*.ts;", + ], + ids=[ + "replace", + "line-start", + "condition", + "arrow", + "division", + "division-at-a-line-start", + "division-at-a-line-start-before-a-url", + "jsx-text", ], - ids=["replace", "line-start", "condition", "arrow", "division"], ) def test_a_slash_in_script_code_hides_nothing_after_it(tmp_path, line): _tree(tmp_path, {"agent.js": f'{line}\nconst tool = {{ type: "computer_20250124" }};\n'}) @@ -402,8 +414,9 @@ def test_a_slash_in_script_code_hides_nothing_after_it(tmp_path, line): '/* const tool = { type: "computer_20250124" }; */\n', 'const a = 1; /* { type: "computer_20250124" } */\n', 'const ratio = a / b; // { type: "computer_20250124" }\n', + 'const r = a\n / 2; // { type: "computer_20250124" }\n', ], - ids=["line-start", "after-code", "after-division"], + ids=["line-start", "after-code", "after-division", "after-a-division-at-a-line-start"], ) def test_a_script_comment_still_hides_what_it_holds(tmp_path, body): _tree(tmp_path, {"agent.js": body}) @@ -417,8 +430,19 @@ def test_a_script_comment_still_hides_what_it_holds(tmp_path, body): "const r = " + "(/[" * 20_000, "const r = /[" + "[" * 20_000, "const r = /[" + "[" * 20_000 + "\\", + "const r = /[/*", + "x = /" + "[=/]" * 20_000 + "/b", + "x = /" + "[=/" * 20_000 + "]/b", + ], + ids=[ + "slash-in-a-class", + "unclosed-classes", + "repeated-brackets", + "repeated-then-escape", + "class-open-at-the-line-end", + "classes-of-slashes", + "nested-classes-of-slashes", ], - ids=["slash-in-a-class", "unclosed-classes", "repeated-brackets", "repeated-then-escape"], ) def test_an_odd_regex_literal_stays_quick_and_hides_nothing(tmp_path, line): _tree(tmp_path, {"agent.js": f'{line}\nconst tool = {{ type: "computer_20250124" }};\n'}) @@ -428,13 +452,36 @@ def test_an_odd_regex_literal_stays_quick_and_hides_nothing(tmp_path, line): def test_a_file_the_tokenizer_breaks_on_costs_only_that_file(tmp_path): - (tmp_path / "blob.py").write_bytes(b" x = 1\n\x00") + # Read first by name, so a scan that stopped at it would miss the agent. 3.11's tokenizer + # doesn't raise on a NUL byte, so there the file is just read. + (tmp_path / "a_blob.py").write_bytes(b" x = 1\n\x00") (tmp_path / "agent.py").write_text( "from playwright.sync_api import sync_playwright\n", encoding="utf-8" ) assert browser_tools_found(tmp_path) == ["playwright"] +def test_a_file_that_raises_something_unexpected_costs_only_that_file(tmp_path, monkeypatch): + _tree(tmp_path, {"a_odd.py": "x = 1\n", "agent.py": "import playwright\n"}) + real = agent_type_check._markers_in + + def odd(path): + if path.name == "a_odd.py": + raise LookupError("an error no reader expects") + return real(path) + + monkeypatch.setattr(agent_type_check, "_markers_in", odd) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_a_python_file_with_a_coding_line_keeps_its_marker(tmp_path): + (tmp_path / "agent.py").write_bytes( + b"# -*- coding: latin-1 -*-\nimport playwright\nNAME = '\xe9'\n" + b"TOOL = {'type': 'computer_20250124'}\n" + ) + assert browser_tools_found(tmp_path) == ["computer-use tool", "playwright"] + + def test_a_python_file_saved_with_a_bom_is_read(tmp_path): (tmp_path / "agent.py").write_bytes( b"\xef\xbb\xbffrom playwright.sync_api import sync_playwright\n" @@ -618,6 +665,61 @@ def test_records_past_their_total_limit_are_not_believed(tmp_path, monkeypatch): assert browser_tools_found(tmp_path) == ["computer-use tool"] +def _record_text_read(monkeypatch) -> list[int]: + read: list[int] = [] + real = Path.read_text + + def counted(self, *args, **kwargs): + text = real(self, *args, **kwargs) + if self.name == "RECORD": + read.append(len(text)) + return text + + monkeypatch.setattr(Path, "read_text", counted) + return read + + +def test_every_folder_shares_one_record_budget(tmp_path, monkeypatch): + record = "vendored.py,sha256=a,1\n" + for folder in ("a", "b", "c"): + _tree( + tmp_path, + { + f"{folder}/x-1.dist-info/RECORD": record, + f"{folder}/vendored.py": 'NAME = "computer_20241022"\n', + }, + ) + # Packages unpacked in more than one folder are all skipped while the budget lasts. + assert browser_tools_found(tmp_path) == [] + monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", 2 * len(record)) + read = _record_text_read(monkeypatch) + files = list(agent_type_check._agent_files(tmp_path)) + assert sum(read) <= 2 * len(record) + assert [path.name for path in files].count("vendored.py") == 1 + + +def test_linked_records_are_never_read(tmp_path, monkeypatch): + _tree( + tmp_path, + { + "agent.py": "import openai\n", + ".shared/x-1.dist-info/RECORD": "vendored.py,sha256=a,1\n", + "y-1.dist-info/METADATA": "", + }, + ) + (tmp_path / "y-1.dist-info/RECORD").symlink_to(tmp_path / ".shared/x-1.dist-info/RECORD") + for folder in range(4): + _tree(tmp_path, {f"pkg{folder}/vendored.py": 'NAME = "computer_20241022"\n'}) + for link in range(8): + (tmp_path / f"pkg{folder}/p{link}-1.dist-info").symlink_to( + tmp_path / ".shared/x-1.dist-info" + ) + read = _record_text_read(monkeypatch) + # pip writes neither as a link, so a linked one says nothing about the files beside it. + assert browser_tools_found(tmp_path) == ["computer-use tool"] + assert read == [] + + def test_a_record_linked_to_a_device_is_not_read(tmp_path, monkeypatch): _tree(tmp_path, {"x-1.dist-info/METADATA": "", "a.py": "import playwright\n"}) (tmp_path / "x-1.dist-info/RECORD").symlink_to("/dev/zero") @@ -1012,30 +1114,73 @@ def test_a_browser_answer_without_the_value_stands_after_one_look(tmp_path, plai assert not (output / "contract.json").exists() -@pytest.mark.parametrize("together", [True, False]) -def test_a_call_without_the_value_beside_an_answer_does_not_undo_it( - tmp_path, plain_repo, together +@pytest.mark.parametrize( + "repo, modality, written", + [("plain_repo", "browser", FAILURE_FILE), ("browser_repo", "chat", "contract.json")], +) +def test_a_wrong_value_every_time_still_ends_after_one_look( + tmp_path, request, repo, modality, written ): output = tmp_path / "out" - tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) - assert "second-look" in _text(asyncio.run(tool.handler(_reading("browser")))) + tool = _submit_tool(contract_tools(output, scan_root=request.getfixturevalue(repo))) + answers = [ + asyncio.run(tool.handler(_reading(modality, second_look_read="yes"))) for _ in range(3) + ] + assert _looks(answers) == [True, False, False] + assert (output / written).exists() - async def next_message(): - calls = [ - tool.handler(_reading("chat", second_look_read=_READ)), - tool.handler(_reading("browser")), - ] - if together: - return await asyncio.gather(*calls) - return [await one for one in calls] - chat_answer, browser_answer = asyncio.run(next_message()) +def _next_message(tool, *payloads): + """One model message's calls once the scan is done, when together runs as one after another.""" + + async def calls(): + return [await tool.handler(payload) for payload in payloads] + + return asyncio.run(calls()) + + +def test_a_call_without_the_value_beside_an_answer_does_not_undo_it(tmp_path, plain_repo): + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) + assert "second-look" in _text(asyncio.run(tool.handler(_reading("browser")))) + chat_answer, browser_answer = _next_message( + tool, _reading("chat", second_look_read=_READ), _reading("browser") + ) assert not chat_answer.get("is_error") assert "second-look" in _text(browser_answer) assert load(output).modality == "chat" assert not (output / FAILURE_FILE).exists() +def test_a_browser_call_before_a_chat_answer_with_the_value_stops_the_build(tmp_path, plain_repo): + # The browser call is in the state of an answer to an unanswered look, so it stands. + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) + assert "second-look" in _text(asyncio.run(tool.handler(_reading("browser")))) + browser_answer, chat_answer = _next_message( + tool, _reading("browser"), _reading("chat", second_look_read=_READ) + ) + assert _STOP in _text(browser_answer) + assert _STOP in _text(chat_answer) + assert load(output) is None + assert _failure(output)["code"] == "agent_type_unsupported" + + +def test_a_browser_call_after_a_chat_answer_neither_with_the_value_stops_the_build( + tmp_path, plain_repo +): + # Each is taken as the answer in turn: the handler can't tell that a later call in the same + # message will undo what it saves. + output = tmp_path / "out" + tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) + assert "second-look" in _text(asyncio.run(tool.handler(_reading("browser")))) + chat_answer, browser_answer = _next_message(tool, _reading("chat"), _reading("browser")) + assert not chat_answer.get("is_error") + assert _STOP in _text(browser_answer) + assert load(output) is None + assert _failure(output)["code"] == "agent_type_unsupported" + + def _looked_at(tmp_path, repo): """A handler that has given one second look, and that look's answer.""" tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=repo)) @@ -1055,14 +1200,16 @@ def test_each_second_look_gets_its_own_value(tmp_path, plain_repo, monkeypatch): assert first != second -def test_an_earlier_value_still_counts(tmp_path, plain_repo, monkeypatch): +def test_an_earlier_value_still_counts_once_a_value_came_back(tmp_path, plain_repo, monkeypatch): # Two calls in one message each got a value; the model may send back either. values = iter(["first", "second"]) monkeypatch.setattr(tools, "_new_receipt", lambda: next(values)) output = tmp_path / "out" tool = _submit_tool(contract_tools(output, scan_root=plain_repo)) assert all(_looks(_in_one_model_message(tool, "browser", together=True))) + assert not asyncio.run(tool.handler(_reading("chat", second_look_read="second"))).get("is_error") assert _STOP in _text(asyncio.run(tool.handler(_reading("browser", second_look_read="first")))) + assert _failure(output)["code"] == "agent_type_unsupported" def test_a_second_look_answered_with_its_value_stands(tmp_path, browser_repo): @@ -1317,9 +1464,8 @@ def test_conversation_with_no_tool_saves_with_no_second_look(tmp_path, plain_rep assert load(output).modality == modality -def test_one_scan_and_at_most_one_more_look_after_an_answer(tmp_path, browser_repo, monkeypatch): - output = tmp_path / "out" - scans = [] +def _counted_scans(monkeypatch) -> list[Path]: + scans: list[Path] = [] real_scan = tools.browser_tools_found def counted(root): @@ -1327,18 +1473,88 @@ def counted(root): return real_scan(root) monkeypatch.setattr(tools, "browser_tools_found", counted) + return scans + + +def test_one_scan_and_one_look_when_an_answer_comes_back_without_its_value( + tmp_path, browser_repo, monkeypatch +): + output = tmp_path / "out" + scans = _counted_scans(monkeypatch) submit = _submitter(output, browser_repo) assert "second-look" in _text(submit(_reading("chat"))) # An answer with the value stands, even when the contract is then sent back for another fix. rejected = submit(_reading("chat", real_use_cases=[], second_look_read=_READ)) assert rejected["is_error"] and "second-look" not in _text(rejected) - # Without the value it could have run beside that answer, so it gets one look of its own. - assert "second-look" in _text(submit(_reading("chat"))) + # The fix, sent without the value, is still that answer. assert not submit(_reading("chat")).get("is_error") assert load(output).modality == "chat" assert scans == [browser_repo] +def test_a_different_answer_without_the_value_after_an_answer_gets_a_look(tmp_path, plain_repo): + # It could have run beside that answer, unread. + output = tmp_path / "out" + submit = _submitter(output, plain_repo) + assert "second-look" in _text(submit(_reading("browser"))) + rejected = submit(_reading("chat", real_use_cases=[], second_look_read=_READ)) + assert rejected["is_error"] and "second-look" not in _text(rejected) + assert "second-look" in _text(submit(_reading("browser"))) + assert not (output / FAILURE_FILE).exists() + + +def test_calls_waiting_on_the_first_scan_share_it(tmp_path, plain_repo, monkeypatch): + scans = _counted_scans(monkeypatch) + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) + + async def first_message(): + return await asyncio.gather(*(tool.handler(_reading("browser")) for _ in range(3))) + + assert all(_looks(asyncio.run(first_message()))) + assert scans == [plain_repo] + + +def test_a_call_cancelled_while_waiting_on_the_scan_leaves_it_to_the_others( + tmp_path, plain_repo, monkeypatch +): + started, release = threading.Event(), threading.Event() + + def slow(_root): + started.set() + release.wait(10) + return [] + + monkeypatch.setattr(tools, "browser_tools_found", slow) + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) + + async def first_message(): + cancelled = asyncio.ensure_future(tool.handler(_reading("browser"))) + waiting = asyncio.ensure_future(tool.handler(_reading("browser"))) + await asyncio.to_thread(started.wait, 10) + cancelled.cancel() + await asyncio.sleep(0) + release.set() + return await waiting + + assert "second-look" in _text(asyncio.run(first_message())) + + +def test_a_scan_that_fails_is_run_again_by_the_next_call(tmp_path, plain_repo, monkeypatch): + outcomes = iter([OSError("gone"), []]) + + def scan(_root): + outcome = next(outcomes) + if isinstance(outcome, Exception): + raise outcome + return outcome + + monkeypatch.setattr(tools, "browser_tools_found", scan) + submit = _submitter(tmp_path / "out", plain_repo) + with pytest.raises(OSError, match="gone"): + submit(_reading("chat")) + assert not submit(_reading("chat")).get("is_error") + + # Every way into the stage reaches the scan From d2c32290d46b980e14f8caeb07b5d0ac56695b10 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Fri, 6 Nov 2026 13:14:48 +0530 Subject: [PATCH 09/15] fix(harness): keep record rows relative to their folder, and rescan only a lost scan - Record rows stay relative to the folder whose records hold them, so a deep folder no longer multiplies what each row costs. - A scan runs again only when it raised, was cancelled, or belongs to another loop; one that finished after its waiters were cancelled keeps its answer. - A regex literal that meets a stray bracket is kept to its line's end, so its quotes and backticks can't open a string or template. - Folders are walked in sorted order, and the record budget's real cost is stated. - The second look keeps one flag for a chat answer that carried a value; behaviour is unchanged. - Tests pin each of these and the close rule's flags and quotes. TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_type_check.py | 38 +++++-- src/fi/alk/harness/tools.py | 23 ++-- tests/harness/test_agent_type_refusal.py | 139 ++++++++++++++++++++++- 3 files changed, 180 insertions(+), 20 deletions(-) diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index 32bf740ad..b6dd5d978 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -108,6 +108,8 @@ # Past this a repository holds the conversation up for longer than the rest of the scan is worth. _LARGEST_SCAN = 128 * 1024 * 1024 _LARGEST_RECORD = 1024 * 1024 +# Per walk. A first scan reads up to three times this: its signature, its files and the packages +# at the top each walk the records. Every reopen reads it once more, for the signature. _LARGEST_RECORDS = 64 * 1024 * 1024 # Strings are matched first and kept, so "https://..." or "src/**/*.ts" isn't read as a comment. # An unclosed one ends at its line or the file: a start that could fail makes a customer's odd file @@ -117,12 +119,14 @@ # name or a number, where it divides. Its closing slash must be followed as a regex's is, by flags # and then no name, quote, "/" or "*": a division that starts a line, or a "/" in JSX text, then # reads as division, not as a literal that eats a string's opening quote. -# Such a start can fail, so a class holds no "[" and the rest no "]": a failed literal then can't -# run over the classes of the slashes after it, which would make the line cost its square. +# Such a start can fail, so a class holds no "[" and the rest no "]", and a literal that meets one +# is kept to its line's end: a failed literal then can't run over the classes of the slashes after +# it, which would make the line cost its square, and the quotes and backticks it holds can't open a +# string or a template that swaps code and strings for the rest of the file. _SCRIPT_COMMENT_OR_STRING = re.compile( r"""("(?:\\.|[^"\\\n])*(?:"|(?=\n)|\\?\Z)|'(?:\\.|[^'\\\n])*(?:'|(?=\n)|\\?\Z)|`(?:\\.|[^`\\])*(?:`|\\?\Z)""" r"""|(?])[ \t]*/(?![/*])(?:\\.|\[(?:\\.|[^\]\[\\\n])*(?:\]|(?=\n)|\\?\Z)|[^/\\\n\[\]])*""" - r"""(?:/(?=[dgimsuvy]*(?![\w$/*"'`]))|(?=\n)|\\?\Z))""" + r"""(?:/(?=[dgimsuvy]*(?![\w$/*"'`]))|(?=\n)|\\?\Z|(?:\[(?=(?:\\.|[^\]\[\\\n])*\[)|\])[^\n]*))""" r"""|//[^\n]*|/\*.*?(?:\*/|\Z)""", re.DOTALL, ) @@ -191,21 +195,33 @@ def _agent_files(root: Path) -> Iterator[Path]: # Unpacked distributions hold other people's code, whose SDKs name these tools whether or not # this agent uses them. Their records name those files wherever they were unpacked, so the # agent's own code beside them, as in a Lambda folder, is still read. One budget for the whole - # walk: a repository can hold any number of folders. Rows stay text, a fraction of a Path's size. - unpacked: set[str] = set() + # walk: a repository can hold any number of folders. Rows stay relative to the folder whose + # records hold them, as pip wrote them, so what is kept is the rows the budget lets in and that + # folder's path once: how deep the folder sits doesn't multiply what each row costs. + unpacked: dict[str, set[str]] = {} + tops: dict[str, tuple[str, ...]] = {} left = [_LARGEST_RECORDS] for directory, subdirectories, files in os.walk(root): - here = Path(directory) + inherited = tops.pop(directory, ()) if any(name.endswith(".dist-info") for name in subdirectories): - unpacked.update( - os.path.join(directory, name) for name in _unpacked_at_top(here, left) - ) + rows = _unpacked_at_top(Path(directory), left) + if rows: + unpacked[directory] = rows + inherited += (directory,) subdirectories[:] = [ name for name in subdirectories if not _not_the_agent_dir(name) ] + # Once the budget is spent, which folders' packages count mustn't depend on the order the + # disk lists folders in. + subdirectories.sort() + for name in subdirectories: + tops[os.path.join(directory, name)] = inherited for name in files: - if not _not_the_agent_file(name) and os.path.join(directory, name) not in unpacked: - yield here / name + path = os.path.join(directory, name) + if not _not_the_agent_file(name) and not any( + path[len(top) + 1 :] in unpacked[top] for top in inherited + ): + yield Path(path) def _packages_used(path: Path) -> set[str]: diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index fd7653952..d8178370c 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -321,9 +321,10 @@ def contract_tools( # Whether the latest second look has had no value back yet. A model that never sends one back # must still be able to answer, or every submission earns another look until the turns run out. look_unanswered = False - # Whether the last answer that carried a value read browser. The same answer again, without the - # value, is still that answer: a fix after the contract was sent back mustn't earn a new look. - answered_browser: bool | None = None + # Whether an answer that carried a value didn't read browser: one that did was refused. The same + # answer again, without the value, is still that answer: a fix after the contract was sent back + # mustn't earn a new look. + answered = False found: list[str] | None = None # Calls in one message share one scan: each running its own takes that many times longer. scan: asyncio.Future[list[str]] | None = None @@ -1113,14 +1114,20 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: receipt = payload.pop(SECOND_LOOK_READ, None) or ( args.get(SECOND_LOOK_READ) if isinstance(args, dict) else None ) - nonlocal refused, found, look_unanswered, answered_browser, scan + nonlocal refused, found, look_unanswered, answered, scan # Taken before the scan's wait: a call already running when a look went out hasn't read it. looked_before = bool(second_look_receipts) if refused or (destination / FAILURE_FILE).is_file(): return _ok(_stop_message(destination)) if found is None: - # A scan that ended while nothing is set raised, so this call runs it again. - if scan is None or scan.done(): + # Run again only a scan that can't give its answer here: one that raised or was + # cancelled, or one on another loop, which this one can't wait on. One that finished + # after every call waiting on it was cancelled still holds its answer. + if ( + scan is None + or scan.get_loop() is not asyncio.get_running_loop() + or (scan.done() and (scan.cancelled() or scan.exception() is not None)) + ): scan = asyncio.ensure_future(asyncio.to_thread(browser_tools_found, scan_root)) # Shielded: one call cancelled mustn't cancel the scan the others wait on. found = await asyncio.shield(scan) @@ -1142,8 +1149,8 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: read_it = isinstance(receipt, str) and receipt in second_look_receipts if read_it: look_unanswered = False - answered_browser = is_browser(claimed) - same_answer = answered_browser is not None and answered_browser == is_browser(claimed) + answered = answered or not is_browser(claimed) + same_answer = answered and not is_browser(claimed) verdict = decide( claimed, found, diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index a53270a24..ea03d3bc5 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -384,23 +384,31 @@ def test_an_odd_script_file_stays_quick(tmp_path, body): "line", [ 'const base = url.replace(/\\/*$/, "");', + 'const base = url.replace(/\\/*$/g, "");', "const ok = 1;\n/src\\/*/.test(f);", "if (/src\\/*/.test(f)) {}", "const kept = files.filter((f) => /src\\/*/.test(f));", 'const ratio = a / b; const label = "1 / 2 /*";', 'const r = a\n / b; const p = "src/**/*.ts";', 'x = a\n/ b; s = "http://h/*"; const t = { type: "computer_20250124" };', + 'x = a\n/ b + "/" + "/*";', "const C = () => /src/**/*.ts;", + "const ok = !/[(`.[+\\-/*%<>=,?^&]/.test(next);\nconst g = `${dir}/*.js`;", + "const r = /a]`/;\nconst g = `${dir}/*.js`;", ], ids=[ "replace", + "replace-with-a-flag", "line-start", "condition", "arrow", "division", "division-at-a-line-start", "division-at-a-line-start-before-a-url", + "division-at-a-line-start-before-a-string", "jsx-text", + "a-class-holding-a-bracket-then-a-template", + "a-bracket-outside-a-class-then-a-template", ], ) def test_a_slash_in_script_code_hides_nothing_after_it(tmp_path, line): @@ -415,8 +423,15 @@ def test_a_slash_in_script_code_hides_nothing_after_it(tmp_path, line): 'const a = 1; /* { type: "computer_20250124" } */\n', 'const ratio = a / b; // { type: "computer_20250124" }\n', 'const r = a\n / 2; // { type: "computer_20250124" }\n', + 'const r = a\n / x[0]; // { type: "computer_20250124" }\n', + ], + ids=[ + "line-start", + "after-code", + "after-division", + "after-a-division-at-a-line-start", + "after-a-division-at-a-line-start-by-an-index", ], - ids=["line-start", "after-code", "after-division", "after-a-division-at-a-line-start"], ) def test_a_script_comment_still_hides_what_it_holds(tmp_path, body): _tree(tmp_path, {"agent.js": body}) @@ -698,6 +713,52 @@ def test_every_folder_shares_one_record_budget(tmp_path, monkeypatch): assert [path.name for path in files].count("vendored.py") == 1 +def _characters(kept) -> int: + if isinstance(kept, str): + return len(kept) + if isinstance(kept, dict): + return sum(_characters(key) + _characters(value) for key, value in kept.items()) + return sum(_characters(one) for one in kept) + + +def test_a_deep_folders_records_cost_what_their_rows_do(tmp_path, monkeypatch): + deep = tmp_path.joinpath(*["d" * 100] * 5) + rows = "".join(f"r{n:04}\n" for n in range(400)) + _tree(deep, {"x-1.dist-info/RECORD": rows, "main.py": "import openai\n"}) + monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", len(rows)) + walk = agent_type_check._agent_files(tmp_path) + assert next(walk).name == "main.py" + # A row as written is a few bytes, and the folder it sits in can be as deep as the disk allows. + kept = _characters(walk.gi_frame.f_locals["unpacked"]) + assert kept <= len(rows) + len(str(deep)) + + +def test_which_folders_packages_count_once_the_budget_is_spent_is_the_same_on_every_disk( + tmp_path, monkeypatch +): + record = "vendored.py,sha256=a,1\n" + for folder in ("a", "b"): + _tree( + tmp_path, + { + f"{folder}/x-1.dist-info/RECORD": record, + f"{folder}/vendored.py": 'NAME = "computer_20241022"\n', + }, + ) + real_walk = os.walk + + def listed_backwards(top, *args, **kwargs): + for directory, subdirectories, files in real_walk(top, *args, **kwargs): + if directory == os.fspath(top): + subdirectories.reverse() + yield directory, subdirectories, files + + monkeypatch.setattr(os, "walk", listed_backwards) + monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", len(record)) + files = list(agent_type_check._agent_files(tmp_path)) + assert [path.parent.name for path in files if path.name == "vendored.py"] == ["b"] + + def test_linked_records_are_never_read(tmp_path, monkeypatch): _tree( tmp_path, @@ -1555,6 +1616,82 @@ def scan(_root): assert not submit(_reading("chat")).get("is_error") +def test_a_scan_that_fails_is_run_again_by_the_next_call_in_the_same_message( + tmp_path, plain_repo, monkeypatch +): + outcomes = iter([OSError("gone"), []]) + + def scan(_root): + outcome = next(outcomes) + if isinstance(outcome, Exception): + raise outcome + return outcome + + monkeypatch.setattr(tools, "browser_tools_found", scan) + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) + + async def one_message(): + with pytest.raises(OSError, match="gone"): + await tool.handler(_reading("chat")) + return await tool.handler(_reading("chat")) + + assert not asyncio.run(one_message()).get("is_error") + + +def test_a_scan_that_finished_after_its_only_caller_was_cancelled_is_not_run_again( + tmp_path, plain_repo, monkeypatch +): + started, release = threading.Event(), threading.Event() + scans: list[Path] = [] + + def slow(root): + scans.append(root) + started.set() + release.wait(10) + return [] + + monkeypatch.setattr(tools, "browser_tools_found", slow) + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) + + async def cancelled_then_next(): + cancelled = asyncio.ensure_future(tool.handler(_reading("browser"))) + await asyncio.to_thread(started.wait, 10) + (scan,) = asyncio.all_tasks() - {asyncio.current_task(), cancelled} + cancelled.cancel() + release.set() + await asyncio.wait({scan}) + return await tool.handler(_reading("browser")) + + assert "second-look" in _text(asyncio.run(cancelled_then_next())) + assert scans == [plain_repo] + + +def test_a_scan_left_running_on_a_closed_loop_is_started_again(tmp_path, plain_repo, monkeypatch): + started, release = threading.Event(), threading.Event() + scans: list[Path] = [] + + def scan(root): + scans.append(root) + if len(scans) == 1: + started.set() + release.wait(10) + return [] + + monkeypatch.setattr(tools, "browser_tools_found", scan) + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) + closed = asyncio.new_event_loop() + try: + closed.create_task(tool.handler(_reading("browser"))) + closed.run_until_complete(asyncio.to_thread(started.wait, 10)) + finally: + closed.close() + try: + assert "second-look" in _text(asyncio.run(tool.handler(_reading("browser")))) + finally: + release.set() + assert len(scans) == 2 + + # Every way into the stage reaches the scan From 134ff1c6ac5c36da27d936e323a93451e7038d5a Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Fri, 6 Nov 2026 14:42:57 +0530 Subject: [PATCH 10/15] fix(harness): ask only the nearest record folders, and drop a flag that never changed - A file is checked against the records of the nearest four folders that hold them, kept as a stack the walk unwinds, so a crafted chain of such folders no longer multiplies every file's check. - The second look's same-answer rule reads the answer alone: a value on a browser answer has already refused and stopped every call. - The comment states what the regex literal's exclusions keep linear and the template opener the line-end fallback can take in. - Tests pin the nearest folders, memory against folder depth, the walk's order on any disk, and classes across escaped line ends. TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_type_check.py | 24 +++++++--- src/fi/alk/harness/tools.py | 12 ++--- tests/harness/test_agent_type_refusal.py | 59 +++++++++++++++++------- 3 files changed, 63 insertions(+), 32 deletions(-) diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index b6dd5d978..c862afc3f 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -111,6 +111,7 @@ # Per walk. A first scan reads up to three times this: its signature, its files and the packages # at the top each walk the records. Every reopen reads it once more, for the signature. _LARGEST_RECORDS = 64 * 1024 * 1024 +_NEAREST_RECORD_FOLDERS = 4 # Strings are matched first and kept, so "https://..." or "src/**/*.ts" isn't read as a comment. # An unclosed one ends at its line or the file: a start that could fail makes a customer's odd file # cost the square of its size, which holds up the build's conversation until it's done. @@ -122,7 +123,11 @@ # Such a start can fail, so a class holds no "[" and the rest no "]", and a literal that meets one # is kept to its line's end: a failed literal then can't run over the classes of the slashes after # it, which would make the line cost its square, and the quotes and backticks it holds can't open a -# string or a template that swaps code and strings for the rest of the file. +# string or a template that swaps code and strings for the rest of the file. The "]" also stops a +# literal that runs on through escaped line ends, which would cost the square of their number. +# Kept to its line's end, such a literal can take in a template's opening backtick later on that +# line too; a "/*" inside that template then reads as a comment. That trade is the price of a line +# that can't run away. _SCRIPT_COMMENT_OR_STRING = re.compile( r"""("(?:\\.|[^"\\\n])*(?:"|(?=\n)|\\?\Z)|'(?:\\.|[^'\\\n])*(?:'|(?=\n)|\\?\Z)|`(?:\\.|[^`\\])*(?:`|\\?\Z)""" r"""|(?])[ \t]*/(?![/*])(?:\\.|\[(?:\\.|[^\]\[\\\n])*(?:\]|(?=\n)|\\?\Z)|[^/\\\n\[\]])*""" @@ -199,27 +204,32 @@ def _agent_files(root: Path) -> Iterator[Path]: # records hold them, as pip wrote them, so what is kept is the rows the budget lets in and that # folder's path once: how deep the folder sits doesn't multiply what each row costs. unpacked: dict[str, set[str]] = {} - tops: dict[str, tuple[str, ...]] = {} + # The folders above this one whose records gave rows, nearest last. The walk goes top down, so + # leaving a folder is seeing a directory that isn't under it. + holding: list[str] = [] left = [_LARGEST_RECORDS] for directory, subdirectories, files in os.walk(root): - inherited = tops.pop(directory, ()) + while holding and not directory.startswith(holding[-1] + os.sep): + del unpacked[holding.pop()] if any(name.endswith(".dist-info") for name in subdirectories): rows = _unpacked_at_top(Path(directory), left) if rows: unpacked[directory] = rows - inherited += (directory,) + holding.append(directory) subdirectories[:] = [ name for name in subdirectories if not _not_the_agent_dir(name) ] # Once the budget is spent, which folders' packages count mustn't depend on the order the # disk lists folders in. subdirectories.sort() - for name in subdirectories: - tops[os.path.join(directory, name)] = inherited + # Real layouts nest one or two, as a vendored package's own vendored ones. Checking every + # file against a crafted chain of them would cost the chain's depth per file; a file the + # nearest few don't name is read, which only means reading more. + nearest = holding[-_NEAREST_RECORD_FOLDERS:] for name in files: path = os.path.join(directory, name) if not _not_the_agent_file(name) and not any( - path[len(top) + 1 :] in unpacked[top] for top in inherited + path[len(top) + 1 :] in unpacked[top] for top in nearest ): yield Path(path) diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index d8178370c..95d999167 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -321,10 +321,6 @@ def contract_tools( # Whether the latest second look has had no value back yet. A model that never sends one back # must still be able to answer, or every submission earns another look until the turns run out. look_unanswered = False - # Whether an answer that carried a value didn't read browser: one that did was refused. The same - # answer again, without the value, is still that answer: a fix after the contract was sent back - # mustn't earn a new look. - answered = False found: list[str] | None = None # Calls in one message share one scan: each running its own takes that many times longer. scan: asyncio.Future[list[str]] | None = None @@ -1114,7 +1110,7 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: receipt = payload.pop(SECOND_LOOK_READ, None) or ( args.get(SECOND_LOOK_READ) if isinstance(args, dict) else None ) - nonlocal refused, found, look_unanswered, answered, scan + nonlocal refused, found, look_unanswered, scan # Taken before the scan's wait: a call already running when a look went out hasn't read it. looked_before = bool(second_look_receipts) if refused or (destination / FAILURE_FILE).is_file(): @@ -1145,12 +1141,12 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: # A call holding a value read a look, so its answer stands. Without one, the call after a # look that no value came back for is taken as the answer. Once a value came back, a call # without one that reads the same stands too; one that reads otherwise may have run beside - # that answer unread, so it gets a look of its own. + # that answer unread, so it gets a look of its own. A value that came back on a browser + # answer was refused and stopped every later call, so the answer that came back read chat. read_it = isinstance(receipt, str) and receipt in second_look_receipts if read_it: look_unanswered = False - answered = answered or not is_browser(claimed) - same_answer = answered and not is_browser(claimed) + same_answer = not is_browser(claimed) verdict = decide( claimed, found, diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index ea03d3bc5..8451a994c 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -448,6 +448,7 @@ def test_a_script_comment_still_hides_what_it_holds(tmp_path, body): "const r = /[/*", "x = /" + "[=/]" * 20_000 + "/b", "x = /" + "[=/" * 20_000 + "]/b", + "x = /" + "[=/]\\\n" * 8_000 + "/b", ], ids=[ "slash-in-a-class", @@ -457,6 +458,7 @@ def test_a_script_comment_still_hides_what_it_holds(tmp_path, body): "class-open-at-the-line-end", "classes-of-slashes", "nested-classes-of-slashes", + "classes-across-escaped-line-ends", ], ) def test_an_odd_regex_literal_stays_quick_and_hides_nothing(tmp_path, line): @@ -713,24 +715,47 @@ def test_every_folder_shares_one_record_budget(tmp_path, monkeypatch): assert [path.name for path in files].count("vendored.py") == 1 -def _characters(kept) -> int: - if isinstance(kept, str): - return len(kept) - if isinstance(kept, dict): - return sum(_characters(key) + _characters(value) for key, value in kept.items()) - return sum(_characters(one) for one in kept) - - def test_a_deep_folders_records_cost_what_their_rows_do(tmp_path, monkeypatch): - deep = tmp_path.joinpath(*["d" * 100] * 5) - rows = "".join(f"r{n:04}\n" for n in range(400)) - _tree(deep, {"x-1.dist-info/RECORD": rows, "main.py": "import openai\n"}) + import tracemalloc + + count = 4_000 + rows = "".join(f"r{n:04}\n" for n in range(count)) + shallow = tmp_path / "shallow" / "d" + deep = tmp_path.joinpath("deep", *["d" * 100] * 5) + for folder in (shallow, deep): + _tree(folder, {"x-1.dist-info/RECORD": rows, "main.py": "import openai\n"}) monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", len(rows)) - walk = agent_type_check._agent_files(tmp_path) - assert next(walk).name == "main.py" - # A row as written is a few bytes, and the folder it sits in can be as deep as the disk allows. - kept = _characters(walk.gi_frame.f_locals["unpacked"]) - assert kept <= len(rows) + len(str(deep)) + + def peak(top): + tracemalloc.start() + try: + assert "main.py" in [path.name for path in agent_type_check._agent_files(top)] + return tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + # A row as written is a few bytes, and the folder it sits in can be as deep as the disk allows: + # kept with its folder's path, each row here would cost 500 more characters. + assert peak(tmp_path / "deep") < peak(tmp_path / "shallow") + 500 * count // 4 + + +def test_a_file_is_skipped_by_the_records_of_the_nearest_folders_that_hold_them(tmp_path): + chain = [f"c{n}" for n in range(6)] + bottom = "/".join(chain) + tree = {f"{bottom}/v{n}.py": 'NAME = "computer_20241022"\n' for n in range(6)} + for n in range(6): + folder = "/".join(chain[: n + 1]) + below = "/".join(chain[n + 1 :]) + tree[f"{folder}/x-1.dist-info/RECORD"] = f"{below + '/' if below else ''}v{n}.py,,\n" + _tree(tmp_path, tree) + read = sorted( + path.name + for path in agent_type_check._agent_files(tmp_path) + if path.name.startswith("v") + ) + # Each of the nearest four names its own file, so all four are skipped; the two above them + # are too far up to be asked, so their files are read. + assert read == ["v0.py", "v1.py"] def test_which_folders_packages_count_once_the_budget_is_spent_is_the_same_on_every_disk( @@ -750,7 +775,7 @@ def test_which_folders_packages_count_once_the_budget_is_spent_is_the_same_on_ev def listed_backwards(top, *args, **kwargs): for directory, subdirectories, files in real_walk(top, *args, **kwargs): if directory == os.fspath(top): - subdirectories.reverse() + subdirectories.sort(reverse=True) yield directory, subdirectories, files monkeypatch.setattr(os, "walk", listed_backwards) From b9bda589a8e7f83ce0b81b0c144d517428a74d2b Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 13:35:54 +0530 Subject: [PATCH 11/15] fix(harness): read each record once, skip slow codecs, and give every browser reading one look - Each RECORD is read once a scan and none on a reopen; the budget is 16 MiB and every record costs at least 4 KiB. - A .py declared in idna, punycode or undefined is not read. - A browser reading always gets one second look before a refusal; the look's state lives in SecondLook beside the check. - A file the scan can't read is logged; a test-word folder is skipped only when it holds only tests; a scoped package reports its scope. - A refused local understand prints its spend; the docs describe the current rule. TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/HOW-IT-WORKS.md | 11 +- src/fi/alk/harness/agent_type_check.py | 230 ++++++++++++++-- src/fi/alk/harness/cli.py | 2 + src/fi/alk/harness/failure_file.py | 6 +- src/fi/alk/harness/tools.py | 60 +--- tests/harness/test_agent_type_refusal.py | 332 +++++++++++++++++++++-- 6 files changed, 530 insertions(+), 111 deletions(-) diff --git a/src/fi/alk/harness/HOW-IT-WORKS.md b/src/fi/alk/harness/HOW-IT-WORKS.md index f4f8a8db1..998bfbb32 100644 --- a/src/fi/alk/harness/HOW-IT-WORKS.md +++ b/src/fi/alk/harness/HOW-IT-WORKS.md @@ -78,11 +78,12 @@ not exist, or — the one that mattered most in practice — *every* tool having means the arguments were read and then not recorded. Problems are returned **into the conversation**, so the model corrects and resubmits rather than a bad contract landing. -A browser reading with a browser, crawling or computer-use tool found in the source is refused to -`failure.json`, and the build exits 78. Where the model and that code check disagree, the model gets -one second look, and its answer stands. That answer has to send back a value the second look gave, -because calls in one model message run before the model reads any result. A model that never -sends it back keeps getting second looks until its turn limit, and the build ends as a crash. +A browser reading always gets one second look first, naming any browser, crawling or +computer-use tool the code check found in the source. So does a chat or voice reading of source +where the check found one. The next submission stands: a browser answer is refused to +`failure.json` and the build exits 78, and a chat or voice answer is saved. The look gives a value +to send back, which only tells apart calls in one model message: those run before the model reads +any result. The rule and its state are `SecondLook` in `agent_type_check.py`. **How it is changed later** (`amend.py`). The contract is not frozen, but every change is recorded with a reason in `amendments[]`, so months later you can still tell what came from the diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index c862afc3f..f7cddd788 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -8,10 +8,12 @@ from __future__ import annotations import ast +import codecs import csv import fnmatch import io import json +import logging import os import re import tokenize @@ -24,6 +26,8 @@ from .instrumentation_evidence import _SKIPPED_DIRECTORIES, _imports from .job import FailureDomain, HarnessFailure, HarnessStage +logger = logging.getLogger(__name__) + AGENT_TYPE_UNSUPPORTED = "agent_type_unsupported" # EX_CONFIG, which the platform doesn't retry even if it can't read the file. EXIT_REFUSED = 78 @@ -104,13 +108,19 @@ _TOOLKIT = "PlayWrightBrowserToolkit" _TEST_FILES = ("test_*.py", "*_test.py", "conftest.py", "*.test.*", "*.spec.*") _SCRIPT_SUFFIXES = frozenset({".js", ".ts", ".mjs", ".cjs", ".tsx", ".jsx"}) +_CODE_SUFFIXES = _SCRIPT_SUFFIXES | {".py"} _LARGEST_CODE_FILE = 1024 * 1024 +_UNREAD_CODECS = frozenset({"idna", "punycode", "undefined"}) # Past this a repository holds the conversation up for longer than the rest of the scan is worth. _LARGEST_SCAN = 128 * 1024 * 1024 _LARGEST_RECORD = 1024 * 1024 -# Per walk. A first scan reads up to three times this: its signature, its files and the packages -# at the top each walk the records. Every reopen reads it once more, for the signature. -_LARGEST_RECORDS = 64 * 1024 * 1024 +# Per scan, each record read once; a reopen of unchanged files reads none. Each row is parsed and +# kept, so short rows cost the most: a scan through 16 MiB of 7-byte rows takes 0.9 s. A whole +# 411-package environment unpacked in one folder came to 6.4 MiB. +_LARGEST_RECORDS = 16 * 1024 * 1024 +# What each record costs at least, a disk block: an empty one still costs its folder and its +# look-ups, so the budget bounds how many are opened too. +_SMALLEST_RECORD = 4 * 1024 _NEAREST_RECORD_FOLDERS = 4 # Strings are matched first and kept, so "https://..." or "src/**/*.ts" isn't read as a comment. # An unclosed one ends at its line or the file: a start that could fail makes a customer's odd file @@ -137,27 +147,55 @@ ) -# The whole name, its last part, or a test word and a helper word: "functional_tests" and +# The whole name, test data, or a test word and a helper word: "integration_tests" and # "test_utils" hold tests, while "e2e_agent" and "testing_agent" hold agents, and test automation # is a main kind of browser agent. _TEST_FOLDER = re.compile( r"_*(?:(?:unit|integration|e2e)[_-]?)?(?:tests?|testing|test[_-]?data)_*|e2e" - r"|.*[_-](?:tests?|testing|test[_-]?data|e2e)_*" + r"|.*[_-]test[_-]?data_*" r"|(?:tests?|testing)[_.-]?(?:utils?|helpers?|integration|ui|unit|e2e|old|fixtures?|support" r"|common|lib|mocks?|selenium|browser|functional|acceptance|scripts?)_*" ) +# A test word as the last part names tests in "functional_tests" but an agent in "web_testing" or +# "qa_e2e", so such a folder is skipped only when its own code is all tests. +_TESTS_LAST = re.compile(r".*[_-](?:tests?|testing|e2e)_*") -def _not_the_agent_dir(name: str) -> bool: +def _not_the_agent_dir(name: str, parent: str | None = None) -> bool: lowered = name.lower() return ( name in _SKIPPED_DIRECTORIES or lowered in NOT_THE_AGENT or name.startswith(".") or _TEST_FOLDER.fullmatch(lowered) is not None + or ( + parent is not None + and _TESTS_LAST.fullmatch(lowered) is not None + and _only_tests(os.path.join(parent, name)) + ) ) +def _only_tests(folder: str) -> bool: + """Whether the code files directly in ``folder`` are all tests, and there is one. + + Helpers below them, as page objects, go with the tests. A package's ``__init__.py`` says + nothing either way, and a folder of only folders may be the agent's own package. + """ + try: + with os.scandir(folder) as entries: + code = [ + entry.name + for entry in entries + if entry.name != "__init__.py" + and os.path.splitext(entry.name)[1] in _CODE_SUFFIXES + and entry.is_file() + ] + except OSError: + return False + return bool(code) and all(_not_the_agent_file(name) for name in code) + + def _not_the_agent_file(name: str) -> bool: lowered = name.lower() return any(fnmatch.fnmatchcase(lowered, pattern) for pattern in _TEST_FILES) @@ -185,7 +223,7 @@ def _unpacked_at_top(root: Path, left: list[int] | None = None) -> set[str]: # A link to a device or a named pipe reports no size, and reading it never ends. if not record.is_file() or record.stat().st_size > _LARGEST_RECORD: continue - left[0] -= record.stat().st_size + left[0] -= max(record.stat().st_size, _SMALLEST_RECORD) if left[0] < 0: break names.update( @@ -196,18 +234,23 @@ def _unpacked_at_top(root: Path, left: list[int] | None = None) -> set[str]: return names -def _agent_files(root: Path) -> Iterator[Path]: +def _agent_files( + root: Path, left: list[int] | None = None, at_top: set[str] | None = None +) -> Iterator[Path]: # Unpacked distributions hold other people's code, whose SDKs name these tools whether or not # this agent uses them. Their records name those files wherever they were unpacked, so the # agent's own code beside them, as in a Lambda folder, is still read. One budget for the whole # walk: a repository can hold any number of folders. Rows stay relative to the folder whose # records hold them, as pip wrote them, so what is kept is the rows the budget lets in and that # folder's path once: how deep the folder sits doesn't multiply what each row costs. + # ``at_top`` gets the rows of the records at the top, for a caller that would read them again. unpacked: dict[str, set[str]] = {} # The folders above this one whose records gave rows, nearest last. The walk goes top down, so # leaving a folder is seeing a directory that isn't under it. holding: list[str] = [] - left = [_LARGEST_RECORDS] + if left is None: + left = [_LARGEST_RECORDS] + top = os.fspath(root) for directory, subdirectories, files in os.walk(root): while holding and not directory.startswith(holding[-1] + os.sep): del unpacked[holding.pop()] @@ -216,8 +259,10 @@ def _agent_files(root: Path) -> Iterator[Path]: if rows: unpacked[directory] = rows holding.append(directory) + if at_top is not None and directory == top: + at_top |= rows subdirectories[:] = [ - name for name in subdirectories if not _not_the_agent_dir(name) + name for name in subdirectories if not _not_the_agent_dir(name, directory) ] # Once the budget is spent, which folders' packages count mustn't depend on the order the # disk lists folders in. @@ -250,11 +295,28 @@ def _packages_used(path: Path) -> set[str]: listed = manifest.get(field) if isinstance(listed, dict): names.update(listed) - return { - name - for name in names - if name in BROWSER_PACKAGES or name.startswith(BROWSER_SCOPES) - } + found = names & BROWSER_PACKAGES + # The scope, not the key: the rest of a scoped name is the customer's, of any length, and only + # names from these lists reach the message, its details and the second look. + found.update(f"{scope}*" for scope in BROWSER_SCOPES for name in names if name.startswith(scope)) + return found + + +def _unread_codec(source: bytes) -> bool: + """Whether the coding line names a codec the scan won't decode ``source`` in. + + A codec such as rot13 isn't a text encoding, so it can't be read as source. Three text codecs + are refused by name: punycode's decode costs the square of its size, about 2 s for 128 KB; + idna's runs in Python for each label; undefined decodes nothing. No honest repository writes + any of them. + """ + try: + encoding, _ = tokenize.detect_encoding(io.BytesIO(source).readline) + info = codecs.lookup(encoding) + except (SyntaxError, UnicodeError, LookupError): + # The reader meets the same error and refuses the file itself. + return False + return info.name in _UNREAD_CODECS or not getattr(info, "_is_text_encoding", True) def _imports_in(path: Path) -> set[str]: @@ -263,15 +325,24 @@ def _imports_in(path: Path) -> set[str]: try: if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: return set() + source = path.read_bytes() + # Python reads a lone CR as a line end, and detect_encoding doesn't: a file of only CRs + # would be one line to it, and the coding line the parser honours not found. + if _unread_codec(source.replace(b"\r\n", b"\n").replace(b"\r", b"\n")): + return set() # From bytes, so a BOM or a coding line is honoured as Python itself would. - return _imports(ast.parse(path.read_bytes())) + return _imports(ast.parse(source)) except (OSError, ValueError, SyntaxError, RecursionError, MemoryError): return set() -def _modules_at_top(root: Path) -> set[str]: - """What sits ahead of an installed package on the import path, so isn't that package.""" - unpacked = _unpacked_at_top(root) +def _modules_at_top(root: Path, unpacked: set[str] | None = None) -> set[str]: + """What sits ahead of an installed package on the import path, so isn't that package. + + ``unpacked`` is the folder's record rows, when the caller has read them already. + """ + if unpacked is None: + unpacked = _unpacked_at_top(root) try: return { entry.stem if entry.is_file() else entry.name @@ -285,6 +356,10 @@ def _modules_at_top(root: Path) -> set[str]: def _python_code(source: bytes) -> str: + # The tokenizer finds the coding line as detect_encoding does, so it's asked about the same + # lines. + if _unread_codec(source): + return "" # From bytes, as imports are read, so a coding line or a BOM is honoured as Python itself would. return " ".join( token.string @@ -348,10 +423,20 @@ def _reading_order(path: Path) -> tuple[bool, str]: def _signature(root: Path) -> tuple[int, int, int]: - """What changes when the files a scan reads change: a nested edit or a new checkout.""" + """What changes when the files a scan reads change: a nested edit, a new checkout or a record. + + Every reopen asks, so records are looked at, not read: a record that starts or stops naming a + file changes its own size or time. Files a record names count too, which only rescans more. + """ latest = count = size = 0 - for path in _agent_files(root): - if _is_read(path): + for directory, subdirectories, files in os.walk(root): + subdirectories[:] = [ + name for name in subdirectories if not _not_the_agent_dir(name, directory) + ] + for name in files: + path = Path(directory, name) + if name != "RECORD" and (_not_the_agent_file(name) or not _is_read(path)): + continue try: info = path.stat() except OSError: @@ -368,7 +453,9 @@ def _scan(root: Path, _files: tuple[int, int, int]) -> tuple[str, ...]: found: set[str] = set() imported: set[str] = set() scanned = 0 - for path in sorted(filter(_is_read, _agent_files(root)), key=_reading_order): + left = [_LARGEST_RECORDS] + at_top: set[str] = set() + for path in sorted(filter(_is_read, _agent_files(root, left, at_top)), key=_reading_order): size = _file_size(path) # Never read, so it mustn't use up the limit before the code that is. if size > _LARGEST_CODE_FILE: @@ -387,8 +474,14 @@ def _scan(root: Path, _files: tuple[int, int, int]) -> tuple[str, ...]: found |= markers imported |= names except Exception: + # A skipped file may be the agent's main one, and the reading then finds nothing. + logger.warning("agent type scan skipped %s", path, exc_info=True) continue - local = _modules_at_top(root) + local = _modules_at_top(root, at_top) + if left[0] < 0: + # The record the budget left unread may be the one naming a browser package unpacked at + # the top, so a folder of that name no longer shows the agent wrote it. + local -= BROWSER_IMPORTS for name in imported: top = name.split(".")[0] if top in local: @@ -415,13 +508,96 @@ def is_browser(text: str) -> bool: def decide( claimed: Any, found: Sequence[str], *, looked_again: bool ) -> Literal["refuse", "second_look", "keep"]: - """What the model read against what the code check found. Where they disagree the model gets - one more look, and its answer then stands.""" + """What the model read against what the code check found. A browser reading always gets one + more look first, and so does a reading the check disagrees with; the answer then stands.""" if is_browser(modality_text(claimed)): - return "refuse" if found or looked_again else "second_look" + # Never at once: what was found can be a dev dependency or a helper script of a chat + # agent, and a refusal can't be retried. + return "refuse" if looked_again else "second_look" return "second_look" if found and not looked_again else "keep" +SECOND_LOOK_READ = "second_look_read" + + +class SecondLook: + """The second looks of one reading, across its submissions. + + Where the model and the code check disagree, the model gets one more look, and its next answer + stands. Each look gives a value to send back: calls in one model message run before the model + reads any answer, together or one after another, so a call holding the value is the only one + known to have read it. + """ + + def __init__(self) -> None: + self._receipts: set[str] = set() + # Whether the latest look has had no value back yet. A model that never sends one back + # must still be able to answer, or every submission earns another look until the turns run + # out. + self._unanswered = False + + @property + def given(self) -> bool: + """Whether a look has gone out. A call takes this as it starts, before waiting on the + scan: one already running when a look went out hasn't read it.""" + return bool(self._receipts) + + def verdict( + self, claimed: str, receipt: Any, found: Sequence[str], *, given_before: bool + ) -> Literal["refuse", "second_look", "keep"]: + # A call holding a value read a look, so its answer stands. Without one, the call after a + # look that no value came back for is taken as the answer. Once a value came back, a call + # without one that reads the same stands too; one that reads otherwise may have run beside + # that answer unread, so it gets a look of its own. A value that came back on a browser + # answer was refused and stopped every later call, so the answer that came back read chat. + read_it = isinstance(receipt, str) and receipt in self._receipts + if read_it: + self._unanswered = False + same_answer = not is_browser(claimed) + return decide( + claimed, + found, + looked_again=read_it or (given_before and (self._unanswered or same_answer)), + ) + + def issue(self, claimed: str, found: Sequence[str], receipt: str) -> str: + """The look's text, with ``receipt`` as the value to send back.""" + self._receipts.add(receipt) + self._unanswered = True + return ( + f"{self.prompt(claimed, found)} When you submit again, set " + f'{SECOND_LOOK_READ} to "{receipt}", which shows you read this.' + ) + + @staticmethod + def prompt(claimed: str, found: Sequence[str]) -> str: + if is_browser(claimed) and found: + return ( + "second-look: you recorded modality browser, which stops this build as not " + f"supported, and the source uses {', '.join(found)}. A chat or voice agent can " + "list one for its tests or use one in a helper script. Does a person come to this " + "agent for a conversation, or to have it carry out work on websites or a computer? " + "For a conversation, submit again as chat or voice. If its main job is that work, " + "submit again as browser. Decide this from the source; don't ask anyone." + ) + if is_browser(claimed): + return ( + "second-look: you recorded modality browser, which stops this build as not " + "supported, and no browser, crawling or computer-use tool was found in the " + "source. Does a person come to this agent for a conversation, or to have it carry " + "out work on websites or a computer? For a conversation, submit again as chat or " + "voice. If you're sure, submit again as browser. Decide this from the source; " + "don't ask anyone." + ) + return ( + f"second-look: you recorded modality {claimed or 'chat'}, and the source uses " + f"{', '.join(found)}. If that's a helper and a person comes to this agent for a " + "conversation, submit again as is. If its main job is carrying out work on websites " + "or a computer, submit again as browser, which stops this build as not supported. " + "Decide this from the source; don't ask anyone." + ) + + def agent_type_failure(found: Sequence[str]) -> HarnessFailure: if found: message = ( diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index f117cef1e..957a215fd 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -323,6 +323,8 @@ async def _understand(args: argparse.Namespace) -> int: if (destination / FAILURE_FILE).is_file(): _report_failure_file(destination) + # A refused reading still cost what its turns did. + print(f"spent: ${stage.spent_usd:.4f}") return EXIT_REFUSED contract = load(destination) if contract is None: diff --git a/src/fi/alk/harness/failure_file.py b/src/fi/alk/harness/failure_file.py index 2def5ce79..5d814d4b8 100644 --- a/src/fi/alk/harness/failure_file.py +++ b/src/fi/alk/harness/failure_file.py @@ -34,7 +34,11 @@ def _write_in_place(path: Path, body: str) -> None: def write_failure_file(failure: HarnessFailure, output: Path) -> Path: - """Write ``failure`` whole, in one step. The first failure of a build stands.""" + """Write ``failure``. The first failure of a build stands. + + Whole, in one step, on a disk with hard links. On one without, it's written in place, where a + reader could catch it part-written. + """ output = Path(output) output.mkdir(parents=True, exist_ok=True) path = output / FAILURE_FILE diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index 95d999167..9ca965845 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -19,9 +19,10 @@ ) from .agent_type_check import ( + SECOND_LOOK_READ, + SecondLook, agent_type_failure, browser_tools_found, - decide, failure_message, is_browser, modality_text, @@ -248,9 +249,6 @@ def _without_nulls(value: Any) -> Any: MOST_CHOSEN_EVALS = 6 -SECOND_LOOK_READ = "second_look_read" - - def _new_receipt() -> str: return secrets.token_hex(4) @@ -260,24 +258,6 @@ def _stop_message(destination: Path) -> str: return f"{said} Don't submit again or do more work; tell the person." -def _second_look(claimed: str, found: list[str]) -> str: - if is_browser(claimed): - return ( - "second-look: you recorded modality browser, which stops this build as not " - "supported, and no browser, crawling or computer-use tool was found in the source. " - "Does a person come to this agent for a conversation, or to have it carry out work " - "on websites or a computer? For a conversation, submit again as chat or voice. If " - "you're sure, submit again as browser. Decide this from the source; don't ask anyone." - ) - return ( - f"second-look: you recorded modality {claimed or 'chat'}, and the source uses " - f"{', '.join(found)}. If that's a helper and a person comes to this agent for a " - "conversation, submit again as is. If its main job is carrying out work on websites or " - "a computer, submit again as browser, which stops this build as not supported. Decide " - "this from the source; don't ask anyone." - ) - - def contract_tools( destination: Path, available_evals: list[dict[str, Any]] | None = None, @@ -315,12 +295,7 @@ def contract_tools( # and this stage cannot tell those two apart from the outside. nudged: set[str] = set() refused = False - # What a call must hold to count as having read a second look. Calls in one model message run - # before the model reads any answer, together or one after another, so none of them can know it. - second_look_receipts: set[str] = set() - # Whether the latest second look has had no value back yet. A model that never sends one back - # must still be able to answer, or every submission earns another look until the turns run out. - look_unanswered = False + second_look = SecondLook() found: list[str] | None = None # Calls in one message share one scan: each running its own takes that many times longer. scan: asyncio.Future[list[str]] | None = None @@ -1110,9 +1085,9 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: receipt = payload.pop(SECOND_LOOK_READ, None) or ( args.get(SECOND_LOOK_READ) if isinstance(args, dict) else None ) - nonlocal refused, found, look_unanswered, scan + nonlocal refused, found, scan # Taken before the scan's wait: a call already running when a look went out hasn't read it. - looked_before = bool(second_look_receipts) + looked_before = second_look.given if refused or (destination / FAILURE_FILE).is_file(): return _ok(_stop_message(destination)) if found is None: @@ -1138,20 +1113,7 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: "job is work on websites or a computer." ] ) - # A call holding a value read a look, so its answer stands. Without one, the call after a - # look that no value came back for is taken as the answer. Once a value came back, a call - # without one that reads the same stands too; one that reads otherwise may have run beside - # that answer unread, so it gets a look of its own. A value that came back on a browser - # answer was refused and stopped every later call, so the answer that came back read chat. - read_it = isinstance(receipt, str) and receipt in second_look_receipts - if read_it: - look_unanswered = False - same_answer = not is_browser(claimed) - verdict = decide( - claimed, - found, - looked_again=read_it or (looked_before and (look_unanswered or same_answer)), - ) + verdict = second_look.verdict(claimed, receipt, found, given_before=looked_before) if verdict == "refuse": # Set first: if the write fails, the model must not get a resubmission through. refused = True @@ -1163,15 +1125,7 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: write_failure_file(agent_type_failure(found), destination) return _ok(_stop_message(destination)) if verdict == "second_look": - issued = _new_receipt() - second_look_receipts.add(issued) - look_unanswered = True - return _problems( - [ - f"{_second_look(claimed, found)} When you submit again, set " - f'{SECOND_LOOK_READ} to "{issued}", which shows you read this.' - ] - ) + return _problems([second_look.issue(claimed, found, _new_receipt())]) data_store = payload.get("data_store") if not isinstance(data_store, dict): data_store = {} diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index 8451a994c..5507d4c36 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -7,9 +7,11 @@ import json import os import stat +import sys import tempfile import threading import time +from collections import Counter from pathlib import Path from types import SimpleNamespace @@ -26,6 +28,7 @@ ) from fi.alk.harness import hosted_authoring_entrypoint as hosted_authoring from fi.alk.harness.agent_type_check import ( + SecondLook, agent_type_failure, browser_tools_found, decide, @@ -318,13 +321,6 @@ def test_browser_tools_found_is_sorted_and_names_each_tool_once(tmp_path): "integration_tests", "Test", "__tests__", - "functional_tests", - "ui-tests", - "api_tests", - "playwright-tests", - "web_test", - "qa-testing", - "agent-e2e", "test_utils", "test_helpers", "tests_integration", @@ -347,6 +343,44 @@ def test_tools_found_skips_these_test_folders(tmp_path, folder): assert browser_tools_found(tmp_path) == [] +@pytest.mark.parametrize( + "folder, name, line, expected", + [ + ("web_testing", "agent.py", "from browser_use import Agent\n", ["browser-use"]), + ("qa-testing", "run.py", "import playwright\n", ["playwright"]), + ("browser_e2e", "agent.py", "import playwright\n", ["playwright"]), + ("qa_e2e", "agent.py", "import playwright\n", ["playwright"]), + ("functional_tests", "a.py", "import playwright\n", ["playwright"]), + ("ui-tests", "a.py", "import playwright\n", ["playwright"]), + ("api_tests", "a.py", "import playwright\n", ["playwright"]), + ("playwright-tests", "a.py", "import playwright\n", ["playwright"]), + ("web_test", "a.py", "import playwright\n", ["playwright"]), + ("agent-e2e", "a.py", "import playwright\n", ["playwright"]), + ("qa-testing/src", "run.py", "import playwright\n", ["playwright"]), + ], +) +def test_a_folder_ending_in_a_test_word_that_holds_an_agent_is_read( + tmp_path, folder, name, line, expected +): + _tree(tmp_path, {f"{folder}/{name}": line, f"{folder}/test_{name}": "import selenium\n"}) + assert browser_tools_found(tmp_path) == expected + + +@pytest.mark.parametrize("folder", ["ui-tests", "functional_tests", "qa_e2e", "web_testing"]) +def test_a_folder_ending_in_a_test_word_that_holds_only_tests_is_skipped_whole(tmp_path, folder): + _tree( + tmp_path, + { + f"{folder}/__init__.py": "", + f"{folder}/conftest.py": "import playwright\n", + f"{folder}/test_login.py": "import selenium\n", + f"{folder}/package.json": json.dumps({"devDependencies": {"puppeteer": "22"}}), + f"{folder}/pages/login_page.py": "from selenium import webdriver\n", + }, + ) + assert browser_tools_found(tmp_path) == [] + + def test_tools_found_skips_an_upper_case_test_file(tmp_path): (tmp_path / "Test_A.py").write_text("import playwright\n", encoding="utf-8") assert browser_tools_found(tmp_path) == [] @@ -478,7 +512,9 @@ def test_a_file_the_tokenizer_breaks_on_costs_only_that_file(tmp_path): assert browser_tools_found(tmp_path) == ["playwright"] -def test_a_file_that_raises_something_unexpected_costs_only_that_file(tmp_path, monkeypatch): +def test_a_file_that_raises_something_unexpected_costs_only_that_file( + tmp_path, monkeypatch, caplog +): _tree(tmp_path, {"a_odd.py": "x = 1\n", "agent.py": "import playwright\n"}) real = agent_type_check._markers_in @@ -488,7 +524,22 @@ def odd(path): return real(path) monkeypatch.setattr(agent_type_check, "_markers_in", odd) - assert browser_tools_found(tmp_path) == ["playwright"] + with caplog.at_level("WARNING", logger=agent_type_check.__name__): + assert browser_tools_found(tmp_path) == ["playwright"] + [record] = caplog.records + assert "a_odd.py" in record.getMessage() + assert record.exc_info[0] is LookupError + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="3.11's tokenizer reads a NUL byte") +def test_a_file_skipped_for_a_nul_byte_is_logged(tmp_path, caplog): + (tmp_path / "a_blob.py").write_bytes(b" x = 1\n\x00") + with caplog.at_level("WARNING", logger=agent_type_check.__name__): + assert browser_tools_found(tmp_path) == [] + [record] = caplog.records + assert "agent type scan skipped" in record.getMessage() + assert "a_blob.py" in record.getMessage() + assert record.exc_info[0] is SystemError def test_a_python_file_with_a_coding_line_keeps_its_marker(tmp_path): @@ -499,6 +550,47 @@ def test_a_python_file_with_a_coding_line_keeps_its_marker(tmp_path): assert browser_tools_found(tmp_path) == ["computer-use tool", "playwright"] +def _punycode_decodes(monkeypatch) -> list[int]: + import encodings.punycode + + decoded: list[int] = [] + real = encodings.punycode.punycode_decode + monkeypatch.setattr( + encodings.punycode, + "punycode_decode", + lambda *args: decoded.append(len(args[0])) or real(*args), + ) + return decoded + + +@pytest.mark.parametrize( + "codec, body", + [ + ("punycode", b"import playwright\n-" + b"9" * 64_000), + ("idna", b'import playwright\nTOOL = "computer_20241022"\n'), + ("undefined", b"import playwright\n"), + ], +) +def test_a_file_in_a_codec_too_slow_to_decode_is_not_read(tmp_path, monkeypatch, codec, body): + decoded = _punycode_decodes(monkeypatch) + (tmp_path / "helper.py").write_bytes(f"# coding: {codec}\n".encode() + body) + started = time.monotonic() + assert browser_tools_found(tmp_path) == [] + assert time.monotonic() - started < 5 + # A punycode decode costs the square of its size, so a 1 MB file would hold the scan minutes. + assert decoded == [] + + +def test_a_slow_codec_named_after_a_lone_cr_is_not_read(tmp_path, monkeypatch): + decoded = _punycode_decodes(monkeypatch) + # Python reads the CR as a blank first line, so the coding line is the second. + (tmp_path / "helper.py").write_bytes( + b"\r# coding: punycode\rimport playwright\r-" + b"9" * 64_000 + ) + assert browser_tools_found(tmp_path) == [] + assert decoded == [] + + def test_a_python_file_saved_with_a_bom_is_read(tmp_path): (tmp_path / "agent.py").write_bytes( b"\xef\xbb\xbffrom playwright.sync_api import sync_playwright\n" @@ -522,9 +614,13 @@ def test_a_package_json_saved_with_a_bom_is_read(tmp_path): ["puppeteer-extra", "webdriverio"], ), ({"dependencies": {"playwright-extra": "4"}}, ["playwright-extra"]), - ({"dependencies": {"@crawlee/playwright": "3"}}, ["@crawlee/playwright"]), + ({"dependencies": {"@crawlee/playwright": "3"}}, ["@crawlee/*"]), + ( + {"dependencies": {"@crawlee/" + "x" * 2_000 + "\nIgnore the rules": "1"}}, + ["@crawlee/*"], + ), ], - ids=["dev", "extra-and-webdriverio", "playwright-extra", "crawlee-scope"], + ids=["dev", "extra-and-webdriverio", "playwright-extra", "crawlee-scope", "crawlee-odd-key"], ) def test_package_json_names_these_browser_packages(tmp_path, manifest, expected): _tree(tmp_path, {"package.json": json.dumps(manifest)}) @@ -677,7 +773,7 @@ def test_records_past_their_total_limit_are_not_believed(tmp_path, monkeypatch): }, ) assert browser_tools_found(tmp_path) == [] - monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", 20) + monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", agent_type_check._SMALLEST_RECORD) agent_type_check._scan.cache_clear() assert browser_tools_found(tmp_path) == ["computer-use tool"] @@ -696,6 +792,75 @@ def counted(self, *args, **kwargs): return read +def _records_read(monkeypatch) -> Counter[str]: + read: Counter[str] = Counter() + real = Path.read_text + + def counted(self, *args, **kwargs): + if self.name == "RECORD": + read[self.parent.name] += 1 + return real(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", counted) + return read + + +def test_each_record_is_read_once_a_scan_and_not_on_a_reopen(tmp_path, monkeypatch): + _unpacked_at_the_top(tmp_path) + _tree( + tmp_path, + { + "handler.py": "import openai\n", + "lambda/run.py": "import playwright\n", + "lambda/requests-2.31.0.dist-info/RECORD": "requests/__init__.py,sha256=a,1\n", + "lambda/requests/__init__.py": 'NAME = "computer_20241022"\n', + }, + ) + read = _records_read(monkeypatch) + assert browser_tools_found(tmp_path) == ["playwright"] + # Every reopen asks again, so it must read none; a scan reads each once, however many need it. + assert read == {"openai-1.0.dist-info": 1, "requests-2.31.0.dist-info": 1} + read.clear() + assert browser_tools_found(tmp_path) == ["playwright"] + assert read == {} + + +def test_empty_records_still_use_up_the_budget(tmp_path, monkeypatch): + _tree(tmp_path, {f"e{n}-1.dist-info/RECORD": "" for n in range(10)}) + _tree(tmp_path, {"agent.py": "import openai\n"}) + monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", 3 * agent_type_check._SMALLEST_RECORD) + read = _records_read(monkeypatch) + assert browser_tools_found(tmp_path) == [] + assert sum(read.values()) == 3 + + +def test_a_browser_package_at_the_top_still_counts_once_the_budget_is_spent( + tmp_path, monkeypatch +): + # The empty record sorts first and spends the budget, so the one naming the package is unread. + _tree( + tmp_path, + { + "a-1.dist-info/RECORD": "", + "playwright-1.0.dist-info/RECORD": "playwright/__init__.py,sha256=a,1\n", + "playwright/__init__.py": "", + "handler.py": "import playwright\n", + }, + ) + monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", agent_type_check._SMALLEST_RECORD) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_a_record_that_starts_naming_a_file_is_scanned_again(tmp_path): + _tree( + tmp_path, + {"x-1.dist-info/RECORD": "other.py,,\n", "vendored.py": 'NAME = "computer_20241022"\n'}, + ) + assert browser_tools_found(tmp_path) == ["computer-use tool"] + (tmp_path / "x-1.dist-info/RECORD").write_text("vendored.py,,\n", encoding="utf-8") + assert browser_tools_found(tmp_path) == [] + + def test_every_folder_shares_one_record_budget(tmp_path, monkeypatch): record = "vendored.py,sha256=a,1\n" for folder in ("a", "b", "c"): @@ -708,7 +873,7 @@ def test_every_folder_shares_one_record_budget(tmp_path, monkeypatch): ) # Packages unpacked in more than one folder are all skipped while the budget lasts. assert browser_tools_found(tmp_path) == [] - monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", 2 * len(record)) + monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", 2 * agent_type_check._SMALLEST_RECORD) read = _record_text_read(monkeypatch) files = list(agent_type_check._agent_files(tmp_path)) assert sum(read) <= 2 * len(record) @@ -779,7 +944,7 @@ def listed_backwards(top, *args, **kwargs): yield directory, subdirectories, files monkeypatch.setattr(os, "walk", listed_backwards) - monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", len(record)) + monkeypatch.setattr(agent_type_check, "_LARGEST_RECORDS", agent_type_check._SMALLEST_RECORD) files = list(agent_type_check._agent_files(tmp_path)) assert [path.parent.name for path in files if path.name == "vendored.py"] == ["b"] @@ -875,7 +1040,7 @@ def test_package_json_is_read_before_script_bundles(tmp_path, monkeypatch): def test_files_are_read_by_path_whatever_order_the_disk_lists_them(tmp_path, monkeypatch): _tree(tmp_path, {"a.py": "import selenium\n", "b.py": "import playwright\n"}) listed = [tmp_path.resolve() / "b.py", tmp_path.resolve() / "a.py"] - monkeypatch.setattr(agent_type_check, "_agent_files", lambda _root: iter(listed)) + monkeypatch.setattr(agent_type_check, "_agent_files", lambda _root, *_: iter(listed)) monkeypatch.setattr(agent_type_check, "_LARGEST_SCAN", 20) assert browser_tools_found(tmp_path) == ["selenium"] @@ -1018,7 +1183,7 @@ def test_an_oversized_python_file_is_left_out(tmp_path): @pytest.mark.parametrize( "claimed, found, looked_again, expected", [ - ("browser", ["playwright"], False, "refuse"), + ("browser", ["playwright"], False, "second_look"), ("browser", ["playwright"], True, "refuse"), ("browser", [], False, "second_look"), ("browser", [], True, "refuse"), @@ -1041,8 +1206,61 @@ def test_decide_rows(claimed, found, looked_again, expected): [" Browser ", ["browser"], ["browser", "chat"], "computer_use", "cua", "COMPUTER_USE"], ) def test_decide_normalises_every_form_of_browser(claimed): - assert decide(claimed, ["playwright"], looked_again=False) == "refuse" + assert decide(claimed, ["playwright"], looked_again=False) == "second_look" assert decide(claimed, [], looked_again=False) == "second_look" + assert decide(claimed, ["playwright"], looked_again=True) == "refuse" + + +# The second look on its own + + +def test_a_disagreement_gets_one_look_and_the_next_answer_stands(): + look = SecondLook() + assert not look.given + assert look.verdict("chat", None, ["playwright"], given_before=False) == "second_look" + text = look.issue("chat", ["playwright"], "abcd1234") + assert look.given + assert "the source uses playwright" in text + assert text.endswith('set second_look_read to "abcd1234", which shows you read this.') + assert look.verdict("chat", None, ["playwright"], given_before=True) == "keep" + + +def test_a_browser_reading_always_gets_a_look_before_a_refusal(): + look = SecondLook() + assert look.verdict("browser", None, ["playwright"], given_before=False) == "second_look" + assert "the source uses playwright" in look.issue("browser", ["playwright"], "abcd1234") + assert look.verdict("browser", "abcd1234", ["playwright"], given_before=True) == "refuse" + + +def test_a_browser_reading_with_nothing_found_says_so_in_its_look(): + look = SecondLook() + assert look.verdict("browser", None, [], given_before=False) == "second_look" + assert "no browser, crawling or computer-use tool was found" in look.issue( + "browser", [], "abcd1234" + ) + assert look.verdict("browser", None, [], given_before=True) == "refuse" + + +def test_a_call_that_started_before_the_look_gets_one_of_its_own(): + look = SecondLook() + look.verdict("browser", None, [], given_before=False) + look.issue("browser", [], "abcd1234") + assert look.verdict("browser", None, [], given_before=False) == "second_look" + + +@pytest.mark.parametrize("value", ["guess", "", 7, None]) +def test_a_value_the_look_never_gave_reads_nothing(value): + look = SecondLook() + assert look.verdict("chat", value, ["playwright"], given_before=False) == "second_look" + + +def test_once_a_value_came_back_a_different_answer_without_one_gets_a_look(): + look = SecondLook() + look.verdict("browser", None, [], given_before=False) + look.issue("browser", [], "abcd1234") + assert look.verdict("chat", "abcd1234", [], given_before=True) == "keep" + assert look.verdict("chat", None, [], given_before=True) == "keep" + assert look.verdict("browser", None, [], given_before=True) == "second_look" def test_modality_text_and_is_browser(): @@ -1111,7 +1329,9 @@ def test_the_file_is_what_the_platform_reads(): @pytest.mark.parametrize("modality", ["browser", ["browser"]]) def test_a_wrapped_browser_reading_is_refused(tmp_path, browser_repo, modality): output = tmp_path / "out" - _submitter(output, browser_repo)({"contract": _reading(modality)}) + submit = _submitter(output, browser_repo) + assert "second-look" in _text(submit({"contract": _reading(modality)})) + submit({"contract": _reading(modality)}) assert _failure(output)["code"] == "agent_type_unsupported" assert not (output / "contract.json").exists() @@ -1124,9 +1344,15 @@ def test_free_text_mentioning_browser_gets_a_problem_not_a_block(tmp_path, brows assert not (output / FAILURE_FILE).exists() -def test_browser_with_tool_is_refused_on_the_first_submission(tmp_path, browser_repo): +def test_browser_with_tool_gets_a_second_look_naming_it_then_is_refused(tmp_path, browser_repo): output = tmp_path / "out" - answer = _submitter(output, browser_repo)(_reading("browser")) + submit = _submitter(output, browser_repo) + look = submit(_reading("browser")) + assert look.get("is_error") + assert "second-look: you recorded modality browser" in _text(look) + assert "the source uses playwright" in _text(look) + assert not (output / FAILURE_FILE).exists() + answer = submit(_reading("browser")) assert not answer.get("is_error") assert _FOUND_MESSAGE in _text(answer) assert "Don't submit again or do more work; tell the person." in _text(answer) @@ -1365,9 +1591,11 @@ def refuse(self, *args, **kwargs): raise PermissionError("read-only") return real(self, *args, **kwargs) + submit = _submitter(output, browser_repo) + submit(_reading("browser")) monkeypatch.setattr(Path, "unlink", refuse) with pytest.raises(PermissionError): - _submitter(output, browser_repo)(_reading("browser")) + submit(_reading("browser")) assert (output / FAILURE_FILE).exists() @@ -1375,9 +1603,11 @@ def test_a_failed_write_still_removes_the_earlier_reading(tmp_path, browser_repo output = tmp_path / "out" output.mkdir() (output / "contract.json").write_text("{}", encoding="utf-8") + submit = _submitter(output, browser_repo) + submit(_reading("browser")) _failing_write(monkeypatch) with pytest.raises(OSError, match="disk full"): - _submitter(output, browser_repo)(_reading("browser")) + submit(_reading("browser")) assert load(output) is None @@ -1385,7 +1615,9 @@ def test_a_refused_new_reading_removes_the_earlier_one(tmp_path, browser_repo): output = tmp_path / "out" output.mkdir() (output / "contract.json").write_text("{}", encoding="utf-8") - _submitter(output, browser_repo)(_reading("browser")) + submit = _submitter(output, browser_repo) + submit(_reading("browser")) + submit(_reading("browser")) assert (output / FAILURE_FILE).exists() assert not (output / "contract.json").exists() assert load(output) is None @@ -1394,10 +1626,52 @@ def test_a_refused_new_reading_removes_the_earlier_one(tmp_path, browser_repo): # After a refusal +@pytest.mark.parametrize( + "files, tool", + [ + ( + { + "package.json": json.dumps( + { + "dependencies": {"next": "14", "react": "18", "openai": "4"}, + "devDependencies": {"puppeteer": "22"}, + } + ), + "app/page.tsx": "export default function Chat() { return null; }\n", + }, + "puppeteer", + ), + ( + { + "agent.py": "import openai\n", + "scripts/ingest.py": "import scrapy\n", + }, + "scrapy", + ), + ], + ids=["next-chat-app-with-a-puppeteer-dev-dependency", "rag-bot-with-a-scrapy-ingest-script"], +) +@pytest.mark.parametrize("with_value", [True, False]) +def test_a_chat_agent_first_read_as_browser_is_saved_after_its_look( + tmp_path, files, tool, with_value +): + source = _tree(tmp_path / "source", files) + output = tmp_path / "out" + submit = _submitter(output, source) + look = submit(_reading("browser")) + assert f"the source uses {tool}" in _text(look) + assert not (output / FAILURE_FILE).exists() + answer = submit(_reading("chat", **({"second_look_read": _READ} if with_value else {}))) + assert "Accepted and saved" in _text(answer) + assert load(output) is not None + assert not (output / FAILURE_FILE).exists() + + def test_chat_resubmission_after_a_refusal_saves_nothing(tmp_path, browser_repo): output = tmp_path / "out" submit = _submitter(output, browser_repo) submit(_reading("browser")) + submit(_reading("browser")) answer = submit(_reading("chat")) assert not answer.get("is_error") assert _FOUND_MESSAGE in _text(answer) @@ -1436,6 +1710,7 @@ def refuse_to_write(failure, output): def test_writer_failure_blocks_resubmission(tmp_path, browser_repo, monkeypatch): output = tmp_path / "out" submit = _submitter(output, browser_repo) + submit(_reading("browser")) _failing_write(monkeypatch) with pytest.raises(OSError, match="disk full"): submit(_reading("browser")) @@ -1451,6 +1726,7 @@ def test_writer_failure_leaves_the_handler(tmp_path, browser_repo, monkeypatch): # with no file the build would end as a crash. output = tmp_path / "out" submit = _submitter(output, browser_repo) + submit(_reading("browser")) _failing_write(monkeypatch) with pytest.raises(OSError, match="disk full"): submit(_reading("browser")) @@ -1732,6 +2008,7 @@ def test_both_build_modes_reach_the_scan(tmp_path, browser_repo): conversational = open_for_building(None, out=tmp_path / "conversational", source=source) for stage, output in ((staged, "staged"), (conversational, "conversational")): tool = _submit_tool(_server_of(stage)) + assert "playwright" in _text(asyncio.run(tool.handler(_reading("browser")))) asyncio.run(tool.handler(_reading("browser"))) assert _failure(tmp_path / output)["details"]["tools"] == ["playwright"] @@ -1739,7 +2016,9 @@ def test_both_build_modes_reach_the_scan(tmp_path, browser_repo): def test_github_source_reaches_the_scan(tmp_path, browser_repo): source = GitHubSource(name="fixture", root=browser_repo, url="https://github.test/a/b") stage, _ = open_stage(source, out=tmp_path / "out") - asyncio.run(_submit_tool(_server_of(stage)).handler(_reading("browser"))) + tool = _submit_tool(_server_of(stage)) + assert "playwright" in _text(asyncio.run(tool.handler(_reading("browser")))) + asyncio.run(tool.handler(_reading("browser"))) assert _failure(tmp_path / "out")["details"]["tools"] == ["playwright"] @@ -1844,9 +2123,12 @@ def test_understand_returns_1_with_the_file_and_no_nudge(tmp_path, monkeypatch, stage = _Stage(lambda: write_failure_file(agent_type_failure(["playwright"]), tmp_path)) _patch_understand(monkeypatch, tmp_path, stage) args = _understand_args(tmp_path, job=SimpleNamespace(metadata={})) + stage.spent_usd = 0.25 assert asyncio.run(cli._understand(args)) == 78 assert len(stage.said) == 1, "a refused reading must not be told nothing was saved" - err = capsys.readouterr().err + printed = capsys.readouterr() + assert "spent: $0.2500" in printed.out + err = printed.err assert _FOUND_MESSAGE in err assert str(tmp_path / FAILURE_FILE) in err assert "No contract was submitted" not in err From 35fd0c98c4c322c501cd3841fe558f9d90b1d621 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 14:11:12 +0530 Subject: [PATCH 12/15] fix(harness): find the codec name whatever bytes follow it, and count every import once the budget is spent - A non-ASCII byte on the coding line no longer hides an idna codec. - Once the record budget is spent, no top folder shows the agent wrote an import, so a vendored toolkit can't hide its browser tool. - The docs and SecondLook state where a look is not given. TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/HOW-IT-WORKS.md | 4 ++-- src/fi/alk/harness/agent_type_check.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/fi/alk/harness/HOW-IT-WORKS.md b/src/fi/alk/harness/HOW-IT-WORKS.md index 998bfbb32..1c280187a 100644 --- a/src/fi/alk/harness/HOW-IT-WORKS.md +++ b/src/fi/alk/harness/HOW-IT-WORKS.md @@ -78,12 +78,12 @@ not exist, or — the one that mattered most in practice — *every* tool having means the arguments were read and then not recorded. Problems are returned **into the conversation**, so the model corrects and resubmits rather than a bad contract landing. -A browser reading always gets one second look first, naming any browser, crawling or +A browser reading gets one second look first, naming any browser, crawling or computer-use tool the code check found in the source. So does a chat or voice reading of source where the check found one. The next submission stands: a browser answer is refused to `failure.json` and the build exits 78, and a chat or voice answer is saved. The look gives a value to send back, which only tells apart calls in one model message: those run before the model reads -any result. The rule and its state are `SecondLook` in `agent_type_check.py`. +any result. The rule and its state are `SecondLook` in `agent_type_check.py`. A browser answer to a chat or voice look is refused at once, since that look said browser stops the build. On Claude Code, a later call in the same message is taken as the answer before the model has read the look. **How it is changed later** (`amend.py`). The contract is not frozen, but every change is recorded with a reason in `amendments[]`, so months later you can still tell what came from the diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index f7cddd788..57cf070af 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -311,7 +311,9 @@ def _unread_codec(source: bytes) -> bool: any of them. """ try: - encoding, _ = tokenize.detect_encoding(io.BytesIO(source).readline) + # A non-ASCII byte on the coding line makes detection fail while the parser still decodes + # the file in that codec, so it is read here as ASCII: only the codec's name matters. + encoding, _ = tokenize.detect_encoding(io.BytesIO(re.sub(rb"[\x80-\xff]", b"?", source)).readline) info = codecs.lookup(encoding) except (SyntaxError, UnicodeError, LookupError): # The reader meets the same error and refuses the file itself. @@ -479,9 +481,9 @@ def _scan(root: Path, _files: tuple[int, int, int]) -> tuple[str, ...]: continue local = _modules_at_top(root, at_top) if left[0] < 0: - # The record the budget left unread may be the one naming a browser package unpacked at - # the top, so a folder of that name no longer shows the agent wrote it. - local -= BROWSER_IMPORTS + # The record the budget left unread may be the one naming a package unpacked at the top, + # so no folder there shows the agent wrote it: every import counts as the agent's. + local = set() for name in imported: top = name.split(".")[0] if top in local: @@ -523,8 +525,8 @@ def decide( class SecondLook: """The second looks of one reading, across its submissions. - Where the model and the code check disagree, the model gets one more look, and its next answer - stands. Each look gives a value to send back: calls in one model message run before the model + The model gets one more look before any refusal, and wherever it and the code check disagree; + its next answer stands. Each look gives a value to send back: calls in one model message run before the model reads any answer, together or one after another, so a call holding the value is the only one known to have read it. """ From 4060c6e6024b775e05712dad27659223db77b061 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 14:38:53 +0530 Subject: [PATCH 13/15] fix(harness): count browser names in source left unread, and read what test folders hold - A .py in a refused codec (idna, punycode, undefined, unicode-escape) is not decoded, but every browser name and marker in its raw bytes still counts, so it gets a second look. - A test-named folder is skipped only when it holds nothing but tests: package.json counts as code, conftest.py alone doesn't decide it, and a code folder inside it is read. - Lone CRs are normalised before the codec check; scripts are read with replacement characters; files the helpers drop are logged. TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/HOW-IT-WORKS.md | 2 +- src/fi/alk/harness/agent_type_check.py | 108 ++++++++++++++++------- tests/harness/test_agent_type_refusal.py | 105 +++++++++++++++++++--- 3 files changed, 171 insertions(+), 44 deletions(-) diff --git a/src/fi/alk/harness/HOW-IT-WORKS.md b/src/fi/alk/harness/HOW-IT-WORKS.md index 1c280187a..6e40b22e8 100644 --- a/src/fi/alk/harness/HOW-IT-WORKS.md +++ b/src/fi/alk/harness/HOW-IT-WORKS.md @@ -83,7 +83,7 @@ computer-use tool the code check found in the source. So does a chat or voice re where the check found one. The next submission stands: a browser answer is refused to `failure.json` and the build exits 78, and a chat or voice answer is saved. The look gives a value to send back, which only tells apart calls in one model message: those run before the model reads -any result. The rule and its state are `SecondLook` in `agent_type_check.py`. A browser answer to a chat or voice look is refused at once, since that look said browser stops the build. On Claude Code, a later call in the same message is taken as the answer before the model has read the look. +any result. The rule and its state are `SecondLook` in `agent_type_check.py`. A browser answer to a chat or voice look is refused at once, since that look said browser stops the build. When calls in one message run one after another, or together once the first scan is done, a later call is taken as the answer before the model has read the look. **How it is changed later** (`amend.py`). The contract is not frozen, but every change is recorded with a reason in `amendments[]`, so months later you can still tell what came from the diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index 57cf070af..83f26cdc1 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -110,7 +110,7 @@ _SCRIPT_SUFFIXES = frozenset({".js", ".ts", ".mjs", ".cjs", ".tsx", ".jsx"}) _CODE_SUFFIXES = _SCRIPT_SUFFIXES | {".py"} _LARGEST_CODE_FILE = 1024 * 1024 -_UNREAD_CODECS = frozenset({"idna", "punycode", "undefined"}) +_UNREAD_CODECS = frozenset({"idna", "punycode", "undefined", "unicode-escape"}) # Past this a repository holds the conversation up for longer than the rest of the scan is worth. _LARGEST_SCAN = 128 * 1024 * 1024 _LARGEST_RECORD = 1024 * 1024 @@ -176,24 +176,45 @@ def _not_the_agent_dir(name: str, parent: str | None = None) -> bool: ) +def _listed(folder: str) -> tuple[list[str], list[str]]: + """The code files directly in ``folder``, and the folders in it the walk would enter. + + A ``package.json`` is code here: it lists what the folder's scripts run. A package's + ``__init__.py`` says nothing either way. + """ + code: list[str] = [] + folders: list[str] = [] + with os.scandir(folder) as entries: + for entry in entries: + if entry.is_dir(): + if not _not_the_agent_dir(entry.name): + folders.append(entry.name) + elif entry.is_file() and entry.name != "__init__.py" and ( + entry.name == "package.json" or os.path.splitext(entry.name)[1] in _CODE_SUFFIXES + ): + code.append(entry.name) + return code, folders + + def _only_tests(folder: str) -> bool: - """Whether the code files directly in ``folder`` are all tests, and there is one. + """Whether ``folder`` holds tests alone: its own code files are all tests, one of them more + than a ``conftest.py``, and no folder in it that holds code is other than named for tests. - Helpers below them, as page objects, go with the tests. A package's ``__init__.py`` says - nothing either way, and a folder of only folders may be the agent's own package. + A folder of only folders may be the agent's own package. A folder in it holds code if it has + a code file or a folder of its own: one level is looked at, and deeper is taken as code. """ try: - with os.scandir(folder) as entries: - code = [ - entry.name - for entry in entries - if entry.name != "__init__.py" - and os.path.splitext(entry.name)[1] in _CODE_SUFFIXES - and entry.is_file() - ] + code, folders = _listed(folder) + for name in folders: + if _TESTS_LAST.fullmatch(name.lower()) is None and any( + _listed(os.path.join(folder, name)) + ): + return False except OSError: return False - return bool(code) and all(_not_the_agent_file(name) for name in code) + return any(name.lower() != "conftest.py" for name in code) and all( + _not_the_agent_file(name) for name in code + ) def _not_the_agent_file(name: str) -> bool: @@ -305,10 +326,11 @@ def _packages_used(path: Path) -> set[str]: def _unread_codec(source: bytes) -> bool: """Whether the coding line names a codec the scan won't decode ``source`` in. - A codec such as rot13 isn't a text encoding, so it can't be read as source. Three text codecs + A codec such as rot13 isn't a text encoding, so it can't be read as source. Four text codecs are refused by name: punycode's decode costs the square of its size, about 2 s for 128 KB; - idna's runs in Python for each label; undefined decodes nothing. No honest repository writes - any of them. + idna's runs in Python for each label; undefined decodes nothing; unicode-escape is refused + with them so this check and the save's agree. Python runs an idna or unicode-escape file, so + the readers then look for the names in its bytes instead (``_named_raw``). """ try: # A non-ASCII byte on the coding line makes detection fail while the parser still decodes @@ -321,6 +343,29 @@ def _unread_codec(source: bytes) -> bool: return info.name in _UNREAD_CODECS or not getattr(info, "_is_text_encoding", True) +def _lines(source: bytes) -> bytes: + # Python reads a lone CR as a line end, and detect_encoding and the tokenizer don't: to them a + # file of only CRs is one line, so a "coding:" anywhere after a first "#" would be its cookie. + return source.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + + +def _named_raw(source: bytes) -> tuple[set[str], set[str]]: + """The browser imports and tool markers whose bytes appear in ``source``, before any decode. + + For a file in a refused codec: its names are counted rather than lost, which can only bring a + second look, never a refusal. One search over bytes already read. + """ + imports = {name for name in BROWSER_IMPORTS if name.encode() in source} + text = source.decode("latin-1") + markers = { + "Playwright MCP" if "playwright" in hit.group() else "computer-use tool" + for hit in TOOL_MARKER.finditer(text) + } + if _TOOLKIT in text: + markers.add(_TOOLKIT) + return imports, markers + + def _imports_in(path: Path) -> set[str]: # One file at a time: a customer's odd file can run the parser out of memory, and that mustn't # take the rest of the scan, or the reading it serves, down with it. @@ -328,13 +373,12 @@ def _imports_in(path: Path) -> set[str]: if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: return set() source = path.read_bytes() - # Python reads a lone CR as a line end, and detect_encoding doesn't: a file of only CRs - # would be one line to it, and the coding line the parser honours not found. - if _unread_codec(source.replace(b"\r\n", b"\n").replace(b"\r", b"\n")): - return set() + if _unread_codec(_lines(source)): + return _named_raw(source)[0] # From bytes, so a BOM or a coding line is honoured as Python itself would. return _imports(ast.parse(source)) except (OSError, ValueError, SyntaxError, RecursionError, MemoryError): + logger.debug("agent type scan read no imports from %s", path, exc_info=True) return set() @@ -358,14 +402,11 @@ def _modules_at_top(root: Path, unpacked: set[str] | None = None) -> set[str]: def _python_code(source: bytes) -> str: - # The tokenizer finds the coding line as detect_encoding does, so it's asked about the same - # lines. - if _unread_codec(source): - return "" - # From bytes, as imports are read, so a coding line or a BOM is honoured as Python itself would. + # From bytes, as imports are read, so a coding line or a BOM is honoured as Python itself would, + # and on the same lines Python reads. The caller has refused a codec it won't decode. return " ".join( token.string - for token in tokenize.tokenize(io.BytesIO(source).readline) + for token in tokenize.tokenize(io.BytesIO(_lines(source)).readline) if token.type not in (tokenize.COMMENT, tokenize.ENCODING) ) @@ -382,12 +423,17 @@ def _markers_in(path: Path) -> set[str]: try: if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: return set() - code = ( - _python_code(path.read_bytes()) - if path.suffix == ".py" - else _SCRIPT_COMMENT_OR_STRING.sub(_script_kept, path.read_text(encoding="utf-8")) - ) + if path.suffix == ".py": + source = path.read_bytes() + if _unread_codec(_lines(source)): + return _named_raw(source)[1] + code = _python_code(source) + else: + # As Node reads it: a stray byte that isn't UTF-8 becomes a replacement character. + text = path.read_text(encoding="utf-8", errors="replace") + code = _SCRIPT_COMMENT_OR_STRING.sub(_script_kept, text) except (OSError, ValueError, SyntaxError, tokenize.TokenError): + logger.debug("agent type scan read no markers from %s", path, exc_info=True) return set() return { "Playwright MCP" if "playwright" in hit.group() else "computer-use tool" diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index 5507d4c36..d19131137 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -371,16 +371,42 @@ def test_a_folder_ending_in_a_test_word_that_holds_only_tests_is_skipped_whole(t _tree( tmp_path, { - f"{folder}/__init__.py": "", + # Read if the folder were walked, so only skipping it whole gives nothing. + f"{folder}/__init__.py": "import playwright\n", f"{folder}/conftest.py": "import playwright\n", f"{folder}/test_login.py": "import selenium\n", - f"{folder}/package.json": json.dumps({"devDependencies": {"puppeteer": "22"}}), - f"{folder}/pages/login_page.py": "from selenium import webdriver\n", + f"{folder}/login.spec.ts": 'const t = "computer-use-preview";\n', + f"{folder}/e2e/test_flow.py": "import playwright\n", + f"{folder}/smoke_tests/test_smoke.py": "import playwright\n", + f"{folder}/data/users.json": "[]\n", }, ) assert browser_tools_found(tmp_path) == [] +@pytest.mark.parametrize( + "files", + [ + {"qa_e2e/conftest.py": "", "qa_e2e/agent/main.py": "import playwright\n"}, + {"qa_e2e/test_login.py": "", "qa_e2e/agent/main.py": "import playwright\n"}, + { + "web_e2e/login.spec.ts": "test('login', () => {});\n", + "web_e2e/package.json": json.dumps({"devDependencies": {"playwright": "1"}}), + }, + {"qa_e2e/conftest.py": "", "qa_e2e/__init__.py": "import playwright\n"}, + ], + ids=[ + "conftest-beside-an-agent-folder", + "test-beside-an-agent-folder", + "spec-beside-a-package-json", + "only-a-conftest", + ], +) +def test_a_test_named_folder_with_more_than_tests_is_read(tmp_path, files): + _tree(tmp_path, files) + assert browser_tools_found(tmp_path) == ["playwright"] + + def test_tools_found_skips_an_upper_case_test_file(tmp_path): (tmp_path / "Test_A.py").write_text("import playwright\n", encoding="utf-8") assert browser_tools_found(tmp_path) == [] @@ -563,22 +589,52 @@ def _punycode_decodes(monkeypatch) -> list[int]: return decoded +def _parsed_in_a_refused_codec(monkeypatch) -> list[bytes]: + import ast + + parsed: list[bytes] = [] + real = ast.parse + + def spy(source, *args, **kwargs): + if isinstance(source, bytes) and b"# coding: " in source[:40]: + parsed.append(source) + return real(source, *args, **kwargs) + + monkeypatch.setattr(ast, "parse", spy) + return parsed + + @pytest.mark.parametrize( - "codec, body", + "codec, body, expected", [ - ("punycode", b"import playwright\n-" + b"9" * 64_000), - ("idna", b'import playwright\nTOOL = "computer_20241022"\n'), - ("undefined", b"import playwright\n"), + ( + "idna", + b"import playwright\nTOOL = 'computer-use-preview'\n", + ["computer-use tool", "playwright"], + ), + ("idna", b'import playwright\nHOST = "' + b"xn--bcher-kva." * 4_000 + b'"\n', ["playwright"]), + ("punycode", b"import playwright\n-" + b"9" * 64_000, ["playwright"]), + ("undefined", b"import playwright\n", ["playwright"]), + ("unicode-escape", b"\\x69mport playwright\n", ["playwright"]), ], + ids=["idna-plain", "idna-encoded-labels", "punycode", "undefined", "unicode-escape"], ) -def test_a_file_in_a_codec_too_slow_to_decode_is_not_read(tmp_path, monkeypatch, codec, body): +def test_a_file_in_a_refused_codec_still_names_its_tools( + tmp_path, monkeypatch, codec, body, expected +): decoded = _punycode_decodes(monkeypatch) + parsed = _parsed_in_a_refused_codec(monkeypatch) + import encodings.idna + + labels = [] + real = encodings.idna.ToUnicode + monkeypatch.setattr(encodings.idna, "ToUnicode", lambda label: labels.append(label) or real(label)) (tmp_path / "helper.py").write_bytes(f"# coding: {codec}\n".encode() + body) started = time.monotonic() - assert browser_tools_found(tmp_path) == [] + assert browser_tools_found(tmp_path) == expected assert time.monotonic() - started < 5 - # A punycode decode costs the square of its size, so a 1 MB file would hold the scan minutes. - assert decoded == [] + # Never decoded: punycode's decode costs the square of its size, and idna's runs per label. + assert (decoded, labels, parsed) == ([], [], []) def test_a_slow_codec_named_after_a_lone_cr_is_not_read(tmp_path, monkeypatch): @@ -587,10 +643,35 @@ def test_a_slow_codec_named_after_a_lone_cr_is_not_read(tmp_path, monkeypatch): (tmp_path / "helper.py").write_bytes( b"\r# coding: punycode\rimport playwright\r-" + b"9" * 64_000 ) - assert browser_tools_found(tmp_path) == [] + assert browser_tools_found(tmp_path) == ["playwright"] assert decoded == [] +def test_a_coding_word_in_a_later_cr_line_is_not_the_cookie(tmp_path): + # With only CRs, detect_encoding saw one line and took the string's "coding:" as the cookie. + (tmp_path / "agent.py").write_bytes( + b"# hi\rx = 'coding: rot13'\rTOOL = 'computer-use-preview'\r" + ) + # A comment is still not read: the file is tokenized, not searched as bytes. + (tmp_path / "notes.py").write_bytes(b"# hi\rx = 'coding: rot13'\r# @playwright/mcp\r") + assert browser_tools_found(tmp_path) == ["computer-use tool"] + + +def test_a_script_with_a_byte_that_is_not_utf8_is_still_read(tmp_path): + (tmp_path / "agent.ts").write_bytes(b"// \xa9 2024\nconst t = 'computer-use-preview'\n") + assert browser_tools_found(tmp_path) == ["computer-use tool"] + + +def test_a_python_file_the_readers_refuse_is_logged_at_debug(tmp_path, caplog): + (tmp_path / "broken.py").write_text('TOOL = ("computer_20241022"\n', encoding="utf-8") + with caplog.at_level("DEBUG", logger=agent_type_check.__name__): + assert browser_tools_found(tmp_path) == [] + said = [(record.levelname, record.getMessage()) for record in caplog.records] + path = str(tmp_path.resolve() / "broken.py") + assert ("DEBUG", f"agent type scan read no imports from {path}") in said + assert ("DEBUG", f"agent type scan read no markers from {path}") in said + + def test_a_python_file_saved_with_a_bom_is_read(tmp_path): (tmp_path / "agent.py").write_bytes( b"\xef\xbb\xbffrom playwright.sync_api import sync_playwright\n" From 7726ded87bfc553bba5994af8ad7f1617e0acde3 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 15:04:28 +0530 Subject: [PATCH 14/15] fix(harness): decide a test folder without following links, and read code one level inside it - Links inside a test-named folder are not followed to decide it, so a folder of links can't multiply the check. - A package or a nested test-named folder holding code inside a test-named folder keeps the folder read. TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_type_check.py | 18 +++++++++++---- tests/harness/test_agent_type_refusal.py | 29 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index 83f26cdc1..8a5141ba5 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -171,6 +171,8 @@ def _not_the_agent_dir(name: str, parent: str | None = None) -> bool: or ( parent is not None and _TESTS_LAST.fullmatch(lowered) is not None + # The walk doesn't enter a linked folder, so what one holds doesn't matter. + and not os.path.islink(os.path.join(parent, name)) and _only_tests(os.path.join(parent, name)) ) ) @@ -186,7 +188,7 @@ def _listed(folder: str) -> tuple[list[str], list[str]]: folders: list[str] = [] with os.scandir(folder) as entries: for entry in entries: - if entry.is_dir(): + if entry.is_dir(follow_symlinks=False): if not _not_the_agent_dir(entry.name): folders.append(entry.name) elif entry.is_file() and entry.name != "__init__.py" and ( @@ -206,12 +208,20 @@ def _only_tests(folder: str) -> bool: try: code, folders = _listed(folder) for name in folders: - if _TESTS_LAST.fullmatch(name.lower()) is None and any( - _listed(os.path.join(folder, name)) - ): + path = os.path.join(folder, name) + inner, below = _listed(path) + if _TESTS_LAST.fullmatch(name.lower()) is not None: + # Tests alone, as the walk would judge it at the top, and nothing deeper. + if below or (inner and not _tests_alone(inner)): + return False + elif inner or below or os.path.isfile(os.path.join(path, "__init__.py")): return False except OSError: return False + return _tests_alone(code) + + +def _tests_alone(code: list[str]) -> bool: return any(name.lower() != "conftest.py" for name in code) and all( _not_the_agent_file(name) for name in code ) diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index d19131137..b26c6a8e5 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -366,6 +366,35 @@ def test_a_folder_ending_in_a_test_word_that_holds_an_agent_is_read( assert browser_tools_found(tmp_path) == expected +@pytest.mark.parametrize( + "files", + [ + {"qa_e2e/test_a.py": "x = 1\n", "qa_e2e/agent/__init__.py": "import playwright\n"}, + {"qa_e2e/test_a.py": "x = 1\n", "qa_e2e/ui_tests/agent.py": "import playwright\n"}, + {"b_e2e/test_a.py": "x = 1\n", "b_e2e/c_e2e/agent.py": "import playwright\n"}, + ], + ids=["init-only-package", "nested-test-word-folder", "two-levels"], +) +def test_an_agent_one_level_inside_a_test_named_folder_is_read(tmp_path, files): + _tree(tmp_path, files) + assert browser_tools_found(tmp_path) == ["playwright"] + + +@pytest.mark.skipif(not hasattr(os, "symlink"), reason="needs folder links") +def test_links_inside_a_test_named_folder_are_not_followed_to_decide_it(tmp_path): + big = tmp_path / "big" + big.mkdir() + for n in range(5_000): + (big / f"f{n}.txt").write_text("", encoding="utf-8") + _tree(tmp_path, {"t_e2e/test_a.py": "x = 1\n", "agent.py": "import playwright\n"}) + for n in range(500): + os.symlink(big, tmp_path / "t_e2e" / f"link{n}", target_is_directory=True) + started = time.monotonic() + assert browser_tools_found(tmp_path) == ["playwright"] + # Following each link would list the target once per link. + assert time.monotonic() - started < 2 + + @pytest.mark.parametrize("folder", ["ui-tests", "functional_tests", "qa_e2e", "web_testing"]) def test_a_folder_ending_in_a_test_word_that_holds_only_tests_is_skipped_whole(tmp_path, folder): _tree( From 39e5ae8d87afcf9c12aa2caaf679b903f1ffbdb7 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 15:35:44 +0530 Subject: [PATCH 15/15] fix(harness): decode unicode-escape as Python does, and count only whole names in unread source - unicode-escape is decoded like any other codec: it is linear, and its escapes can spell an import that the raw bytes don't hold. - In a file left unread, a browser name counts only as a whole name, so "steelers" is no import of steel. - Tests pin a test-named folder that is itself a link, and the toolkit named in an unread file. TH-8806 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_type_check.py | 17 ++++++++++------ tests/harness/test_agent_type_refusal.py | 26 ++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index 8a5141ba5..1c4afb0dd 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -110,7 +110,7 @@ _SCRIPT_SUFFIXES = frozenset({".js", ".ts", ".mjs", ".cjs", ".tsx", ".jsx"}) _CODE_SUFFIXES = _SCRIPT_SUFFIXES | {".py"} _LARGEST_CODE_FILE = 1024 * 1024 -_UNREAD_CODECS = frozenset({"idna", "punycode", "undefined", "unicode-escape"}) +_UNREAD_CODECS = frozenset({"idna", "punycode", "undefined"}) # Past this a repository holds the conversation up for longer than the rest of the scan is worth. _LARGEST_SCAN = 128 * 1024 * 1024 _LARGEST_RECORD = 1024 * 1024 @@ -336,11 +336,11 @@ def _packages_used(path: Path) -> set[str]: def _unread_codec(source: bytes) -> bool: """Whether the coding line names a codec the scan won't decode ``source`` in. - A codec such as rot13 isn't a text encoding, so it can't be read as source. Four text codecs + A codec such as rot13 isn't a text encoding, so it can't be read as source. Three text codecs are refused by name: punycode's decode costs the square of its size, about 2 s for 128 KB; - idna's runs in Python for each label; undefined decodes nothing; unicode-escape is refused - with them so this check and the save's agree. Python runs an idna or unicode-escape file, so - the readers then look for the names in its bytes instead (``_named_raw``). + idna's runs in Python for each label; undefined decodes nothing. Python runs an idna file, so + the readers then look for the names in its bytes instead (``_named_raw``). unicode-escape + decodes in linear time and its escapes can spell a name, so it is decoded like any other. """ try: # A non-ASCII byte on the coding line makes detection fail while the parser still decodes @@ -365,7 +365,12 @@ def _named_raw(source: bytes) -> tuple[set[str], set[str]]: For a file in a refused codec: its names are counted rather than lost, which can only bring a second look, never a refusal. One search over bytes already read. """ - imports = {name for name in BROWSER_IMPORTS if name.encode() in source} + # Whole names only: "steelers" in a string is no import of steel. + imports = { + name + for name in BROWSER_IMPORTS + if re.search(rb"(?