diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index c2256018b..2853b6220 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -66,6 +66,8 @@ jobs: tests/harness/test_scenario_source_provision_agent.py tests/harness/test_hosted_entrypoint_provision_retry.py tests/harness/test_instrumentation_evidence.py + tests/harness/test_agent_type_refusal.py + tests/harness/test_executor_completed_counts.py tests/test_bump_deployment_values.py tests/test_release_workflow.py tests/test_release_pr_step.py diff --git a/src/fi/alk/harness/HOW-IT-WORKS.md b/src/fi/alk/harness/HOW-IT-WORKS.md index d5a788b67..6e40b22e8 100644 --- a/src/fi/alk/harness/HOW-IT-WORKS.md +++ b/src/fi/alk/harness/HOW-IT-WORKS.md @@ -78,6 +78,13 @@ not exist, or — the one that mattered most in practice — *every* tool having means the arguments were read and then not recorded. Problems are returned **into the conversation**, so the model corrects and resubmits rather than a bad contract landing. +A browser reading gets one second look first, naming any browser, crawling or +computer-use tool the code check found in the source. So does a chat or voice reading of source +where the check found one. The next submission stands: a browser answer is refused to +`failure.json` and the build exits 78, and a chat or voice answer is saved. The look gives a value +to send back, which only tells apart calls in one model message: those run before the model reads +any result. The rule and its state are `SecondLook` in `agent_type_check.py`. A browser answer to a chat or voice look is refused at once, since that look said browser stops the build. When calls in one message run one after another, or together once the first scan is done, a later call is taken as the answer before the model has read the look. + **How it is changed later** (`amend.py`). The contract is not frozen, but every change is recorded with a reason in `amendments[]`, so months later you can still tell what came from the agent and what came from us: diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py new file mode 100644 index 000000000..1c4afb0dd --- /dev/null +++ b/src/fi/alk/harness/agent_type_check.py @@ -0,0 +1,692 @@ +"""Does an agent use a browser, crawling or computer-use tool, and what follows from that. + +No code can say for certain what an agent's main job is, so this is one of two opinions: what the +source uses, set against what the model read. It calls no model and runs none of the agent's code, +and it needs no job or session, so a reading can be checked without a build. +""" + +from __future__ import annotations + +import ast +import codecs +import csv +import fnmatch +import io +import json +import logging +import os +import re +import tokenize +from collections.abc import Iterator, Sequence +from functools import lru_cache +from pathlib import Path +from typing import Any, Literal + +from .failure_file import FAILURE_FILE +from .instrumentation_evidence import _SKIPPED_DIRECTORIES, _imports +from .job import FailureDomain, HarnessFailure, HarnessStage + +logger = logging.getLogger(__name__) + +AGENT_TYPE_UNSUPPORTED = "agent_type_unsupported" +# EX_CONFIG, which the platform doesn't retry even if it can't read the file. +EXIT_REFUSED = 78 +BROWSER_IMPORTS = frozenset( + { + "playwright", + "patchright", + "camoufox", + "playwright_stealth", + "selenium", + "seleniumbase", + "seleniumwire", + "undetected_chromedriver", + "splinter", + "pyppeteer", + "pydoll", + "DrissionPage", + "zendriver", + "nodriver", + "botasaurus", + "browser_use", + "stagehand", + "browserbase", + "steel", + "hyperbrowser", + "skyvern", + "anchorbrowser", + "agentql", + "scrapy", + "crawl4ai", + "firecrawl", + "crawlee", + "scrapegraphai", + "apify_client", + "pyautogui", + "pywinauto", + "e2b_desktop", + "nova_act", + } +) +BROWSER_PACKAGES = frozenset( + { + "playwright", + "playwright-core", + "@playwright/mcp", + "patchright", + "puppeteer", + "puppeteer-core", + "selenium-webdriver", + "browser-use", + "stagehand", + "@browserbasehq/stagehand", + "@browserbasehq/sdk", + "crawlee", + "firecrawl", + "@mendable/firecrawl-js", + "apify-client", + "puppeteer-extra", + "playwright-extra", + "webdriverio", + } +) +# Every package in these scopes drives a browser. +BROWSER_SCOPES = ("@crawlee/",) +# The tools' own API names only: the bare words turn up in prompts and tracing attributes. +TOOL_MARKER = re.compile( + r"\bcomputer[_-]use[_-]preview\b|\bcomputer-use-20\d\d-\d\d-\d\d\b|\bComputerUse\b" + r"""|\bcomputer_?[uU]se\b["']?\s*[:=]\s*\{\s*["']?environment\b""" + r"|\bcomputer_\d{8}\b|@playwright/mcp|mcp-server-playwright" +) +# Hidden folders and folders named for tests or e2e describe the developer's setup too. +NOT_THE_AGENT = frozenset( + {"docs", "doc", "examples", "example", "samples", "vendor", "third_party", "evals"} +) +# What the world registry already treats as a browser. +BROWSER_MODALITIES = frozenset({"browser", "computer_use", "cua"}) + +_TOOLKIT = "PlayWrightBrowserToolkit" +_TEST_FILES = ("test_*.py", "*_test.py", "conftest.py", "*.test.*", "*.spec.*") +_SCRIPT_SUFFIXES = frozenset({".js", ".ts", ".mjs", ".cjs", ".tsx", ".jsx"}) +_CODE_SUFFIXES = _SCRIPT_SUFFIXES | {".py"} +_LARGEST_CODE_FILE = 1024 * 1024 +_UNREAD_CODECS = frozenset({"idna", "punycode", "undefined"}) +# Past this a repository holds the conversation up for longer than the rest of the scan is worth. +_LARGEST_SCAN = 128 * 1024 * 1024 +_LARGEST_RECORD = 1024 * 1024 +# Per scan, each record read once; a reopen of unchanged files reads none. Each row is parsed and +# kept, so short rows cost the most: a scan through 16 MiB of 7-byte rows takes 0.9 s. A whole +# 411-package environment unpacked in one folder came to 6.4 MiB. +_LARGEST_RECORDS = 16 * 1024 * 1024 +# What each record costs at least, a disk block: an empty one still costs its folder and its +# look-ups, so the budget bounds how many are opened too. +_SMALLEST_RECORD = 4 * 1024 +_NEAREST_RECORD_FOLDERS = 4 +# Strings are matched first and kept, so "https://..." or "src/**/*.ts" isn't read as a comment. +# An unclosed one ends at its line or the file: a start that could fail makes a customer's odd file +# cost the square of its size, which holds up the build's conversation until it's done. +# A regex literal is kept the same way, so the "/*" in /\/*$/ doesn't hide the rest of the file. A +# slash where a value is expected starts one: after punctuation or at a line's start, never after a +# name or a number, where it divides. Its closing slash must be followed as a regex's is, by flags +# and then no name, quote, "/" or "*": a division that starts a line, or a "/" in JSX text, then +# reads as division, not as a literal that eats a string's opening quote. +# Such a start can fail, so a class holds no "[" and the rest no "]", and a literal that meets one +# is kept to its line's end: a failed literal then can't run over the classes of the slashes after +# it, which would make the line cost its square, and the quotes and backticks it holds can't open a +# string or a template that swaps code and strings for the rest of the file. The "]" also stops a +# literal that runs on through escaped line ends, which would cost the square of their number. +# Kept to its line's end, such a literal can take in a template's opening backtick later on that +# line too; a "/*" inside that template then reads as a comment. That trade is the price of a line +# that can't run away. +_SCRIPT_COMMENT_OR_STRING = re.compile( + r"""("(?:\\.|[^"\\\n])*(?:"|(?=\n)|\\?\Z)|'(?:\\.|[^'\\\n])*(?:'|(?=\n)|\\?\Z)|`(?:\\.|[^`\\])*(?:`|\\?\Z)""" + r"""|(?])[ \t]*/(?![/*])(?:\\.|\[(?:\\.|[^\]\[\\\n])*(?:\]|(?=\n)|\\?\Z)|[^/\\\n\[\]])*""" + r"""(?:/(?=[dgimsuvy]*(?![\w$/*"'`]))|(?=\n)|\\?\Z|(?:\[(?=(?:\\.|[^\]\[\\\n])*\[)|\])[^\n]*))""" + r"""|//[^\n]*|/\*.*?(?:\*/|\Z)""", + re.DOTALL, +) + + +# The whole name, test data, or a test word and a helper word: "integration_tests" and +# "test_utils" hold tests, while "e2e_agent" and "testing_agent" hold agents, and test automation +# is a main kind of browser agent. +_TEST_FOLDER = re.compile( + r"_*(?:(?:unit|integration|e2e)[_-]?)?(?:tests?|testing|test[_-]?data)_*|e2e" + r"|.*[_-]test[_-]?data_*" + r"|(?:tests?|testing)[_.-]?(?:utils?|helpers?|integration|ui|unit|e2e|old|fixtures?|support" + r"|common|lib|mocks?|selenium|browser|functional|acceptance|scripts?)_*" +) +# A test word as the last part names tests in "functional_tests" but an agent in "web_testing" or +# "qa_e2e", so such a folder is skipped only when its own code is all tests. +_TESTS_LAST = re.compile(r".*[_-](?:tests?|testing|e2e)_*") + + +def _not_the_agent_dir(name: str, parent: str | None = None) -> bool: + lowered = name.lower() + return ( + name in _SKIPPED_DIRECTORIES + or lowered in NOT_THE_AGENT + or name.startswith(".") + or _TEST_FOLDER.fullmatch(lowered) is not None + or ( + parent is not None + and _TESTS_LAST.fullmatch(lowered) is not None + # The walk doesn't enter a linked folder, so what one holds doesn't matter. + and not os.path.islink(os.path.join(parent, name)) + and _only_tests(os.path.join(parent, name)) + ) + ) + + +def _listed(folder: str) -> tuple[list[str], list[str]]: + """The code files directly in ``folder``, and the folders in it the walk would enter. + + A ``package.json`` is code here: it lists what the folder's scripts run. A package's + ``__init__.py`` says nothing either way. + """ + code: list[str] = [] + folders: list[str] = [] + with os.scandir(folder) as entries: + for entry in entries: + if entry.is_dir(follow_symlinks=False): + if not _not_the_agent_dir(entry.name): + folders.append(entry.name) + elif entry.is_file() and entry.name != "__init__.py" and ( + entry.name == "package.json" or os.path.splitext(entry.name)[1] in _CODE_SUFFIXES + ): + code.append(entry.name) + return code, folders + + +def _only_tests(folder: str) -> bool: + """Whether ``folder`` holds tests alone: its own code files are all tests, one of them more + than a ``conftest.py``, and no folder in it that holds code is other than named for tests. + + A folder of only folders may be the agent's own package. A folder in it holds code if it has + a code file or a folder of its own: one level is looked at, and deeper is taken as code. + """ + try: + code, folders = _listed(folder) + for name in folders: + path = os.path.join(folder, name) + inner, below = _listed(path) + if _TESTS_LAST.fullmatch(name.lower()) is not None: + # Tests alone, as the walk would judge it at the top, and nothing deeper. + if below or (inner and not _tests_alone(inner)): + return False + elif inner or below or os.path.isfile(os.path.join(path, "__init__.py")): + return False + except OSError: + return False + return _tests_alone(code) + + +def _tests_alone(code: list[str]) -> bool: + return any(name.lower() != "conftest.py" for name in code) and all( + _not_the_agent_file(name) for name in code + ) + + +def _not_the_agent_file(name: str) -> bool: + lowered = name.lower() + return any(fnmatch.fnmatchcase(lowered, pattern) for pattern in _TEST_FILES) + + +def _unpacked_at_top(root: Path, left: list[int] | None = None) -> set[str]: + """The files ``pip install -t .`` put beside the agent's code, by each distribution's record. + + Exact paths: one distribution shipping ``tools/x.py`` says nothing about the customer's own + ``tools/``. ``left`` is the record bytes still to read, shared across one walk's folders. + """ + names: set[str] = set() + try: + records = sorted(entry for entry in root.iterdir() if entry.name.endswith(".dist-info")) + except OSError: + return names + if left is None: + left = [_LARGEST_RECORDS] + for entry in records: + try: + record = entry / "RECORD" + # pip writes neither as a link, and following them reads one record once per link. + if entry.is_symlink() or record.is_symlink(): + continue + # A link to a device or a named pipe reports no size, and reading it never ends. + if not record.is_file() or record.stat().st_size > _LARGEST_RECORD: + continue + left[0] -= max(record.stat().st_size, _SMALLEST_RECORD) + if left[0] < 0: + break + names.update( + row[0] for row in csv.reader(io.StringIO(record.read_text(encoding="utf-8"))) if row + ) + except (OSError, ValueError, csv.Error): + continue + return names + + +def _agent_files( + root: Path, left: list[int] | None = None, at_top: set[str] | None = None +) -> Iterator[Path]: + # Unpacked distributions hold other people's code, whose SDKs name these tools whether or not + # this agent uses them. Their records name those files wherever they were unpacked, so the + # agent's own code beside them, as in a Lambda folder, is still read. One budget for the whole + # walk: a repository can hold any number of folders. Rows stay relative to the folder whose + # records hold them, as pip wrote them, so what is kept is the rows the budget lets in and that + # folder's path once: how deep the folder sits doesn't multiply what each row costs. + # ``at_top`` gets the rows of the records at the top, for a caller that would read them again. + unpacked: dict[str, set[str]] = {} + # The folders above this one whose records gave rows, nearest last. The walk goes top down, so + # leaving a folder is seeing a directory that isn't under it. + holding: list[str] = [] + if left is None: + left = [_LARGEST_RECORDS] + top = os.fspath(root) + for directory, subdirectories, files in os.walk(root): + while holding and not directory.startswith(holding[-1] + os.sep): + del unpacked[holding.pop()] + if any(name.endswith(".dist-info") for name in subdirectories): + rows = _unpacked_at_top(Path(directory), left) + if rows: + unpacked[directory] = rows + holding.append(directory) + if at_top is not None and directory == top: + at_top |= rows + subdirectories[:] = [ + name for name in subdirectories if not _not_the_agent_dir(name, directory) + ] + # Once the budget is spent, which folders' packages count mustn't depend on the order the + # disk lists folders in. + subdirectories.sort() + # Real layouts nest one or two, as a vendored package's own vendored ones. Checking every + # file against a crafted chain of them would cost the chain's depth per file; a file the + # nearest few don't name is read, which only means reading more. + nearest = holding[-_NEAREST_RECORD_FOLDERS:] + for name in files: + path = os.path.join(directory, name) + if not _not_the_agent_file(name) and not any( + path[len(top) + 1 :] in unpacked[top] for top in nearest + ): + yield Path(path) + + +def _packages_used(path: Path) -> set[str]: + try: + if not path.is_file(): + return set() + # Windows editors often save a BOM, which json refuses. + manifest = json.loads(path.read_text(encoding="utf-8").removeprefix("\ufeff")) + except (OSError, ValueError, RecursionError): + return set() + if not isinstance(manifest, dict): + return set() + names: set[str] = set() + # A test-automation agent usually lists its browser library as a dev dependency. + for field in ("dependencies", "devDependencies"): + listed = manifest.get(field) + if isinstance(listed, dict): + names.update(listed) + found = names & BROWSER_PACKAGES + # The scope, not the key: the rest of a scoped name is the customer's, of any length, and only + # names from these lists reach the message, its details and the second look. + found.update(f"{scope}*" for scope in BROWSER_SCOPES for name in names if name.startswith(scope)) + return found + + +def _unread_codec(source: bytes) -> bool: + """Whether the coding line names a codec the scan won't decode ``source`` in. + + A codec such as rot13 isn't a text encoding, so it can't be read as source. Three text codecs + are refused by name: punycode's decode costs the square of its size, about 2 s for 128 KB; + idna's runs in Python for each label; undefined decodes nothing. Python runs an idna file, so + the readers then look for the names in its bytes instead (``_named_raw``). unicode-escape + decodes in linear time and its escapes can spell a name, so it is decoded like any other. + """ + try: + # A non-ASCII byte on the coding line makes detection fail while the parser still decodes + # the file in that codec, so it is read here as ASCII: only the codec's name matters. + encoding, _ = tokenize.detect_encoding(io.BytesIO(re.sub(rb"[\x80-\xff]", b"?", source)).readline) + info = codecs.lookup(encoding) + except (SyntaxError, UnicodeError, LookupError): + # The reader meets the same error and refuses the file itself. + return False + return info.name in _UNREAD_CODECS or not getattr(info, "_is_text_encoding", True) + + +def _lines(source: bytes) -> bytes: + # Python reads a lone CR as a line end, and detect_encoding and the tokenizer don't: to them a + # file of only CRs is one line, so a "coding:" anywhere after a first "#" would be its cookie. + return source.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + + +def _named_raw(source: bytes) -> tuple[set[str], set[str]]: + """The browser imports and tool markers whose bytes appear in ``source``, before any decode. + + For a file in a refused codec: its names are counted rather than lost, which can only bring a + second look, never a refusal. One search over bytes already read. + """ + # Whole names only: "steelers" in a string is no import of steel. + imports = { + name + for name in BROWSER_IMPORTS + if re.search(rb"(? set[str]: + # One file at a time: a customer's odd file can run the parser out of memory, and that mustn't + # take the rest of the scan, or the reading it serves, down with it. + try: + if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: + return set() + source = path.read_bytes() + if _unread_codec(_lines(source)): + return _named_raw(source)[0] + # From bytes, so a BOM or a coding line is honoured as Python itself would. + return _imports(ast.parse(source)) + except (OSError, ValueError, SyntaxError, RecursionError, MemoryError): + logger.debug("agent type scan read no imports from %s", path, exc_info=True) + return set() + + +def _modules_at_top(root: Path, unpacked: set[str] | None = None) -> set[str]: + """What sits ahead of an installed package on the import path, so isn't that package. + + ``unpacked`` is the folder's record rows, when the caller has read them already. + """ + if unpacked is None: + unpacked = _unpacked_at_top(root) + try: + return { + entry.stem if entry.is_file() else entry.name + for entry in root.iterdir() + if entry.name not in unpacked + and f"{entry.name}/__init__.py" not in unpacked + and (entry.suffix == ".py" or (entry / "__init__.py").is_file()) + } + except OSError: + return set() + + +def _python_code(source: bytes) -> str: + # From bytes, as imports are read, so a coding line or a BOM is honoured as Python itself would, + # and on the same lines Python reads. The caller has refused a codec it won't decode. + return " ".join( + token.string + for token in tokenize.tokenize(io.BytesIO(_lines(source)).readline) + if token.type not in (tokenize.COMMENT, tokenize.ENCODING) + ) + + +def _script_kept(hit: re.Match[str]) -> str: + kept = hit.group(1) or "" + # A lone apostrophe, as in JSX text, isn't a string, so it can't protect a comment after it. + if kept.startswith("'") and not kept.endswith("'"): + return kept.split("//", 1)[0] + return kept + + +def _markers_in(path: Path) -> set[str]: + try: + if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: + return set() + if path.suffix == ".py": + source = path.read_bytes() + if _unread_codec(_lines(source)): + return _named_raw(source)[1] + code = _python_code(source) + else: + # As Node reads it: a stray byte that isn't UTF-8 becomes a replacement character. + text = path.read_text(encoding="utf-8", errors="replace") + code = _SCRIPT_COMMENT_OR_STRING.sub(_script_kept, text) + except (OSError, ValueError, SyntaxError, tokenize.TokenError): + logger.debug("agent type scan read no markers from %s", path, exc_info=True) + return set() + return { + "Playwright MCP" if "playwright" in hit.group() else "computer-use tool" + for hit in TOOL_MARKER.finditer(code) + } + + +def _file_size(path: Path) -> int: + try: + return path.stat().st_size + except OSError: + return 0 + + +def browser_tools_found(source_root: Path | None) -> list[str]: + """The browser, crawling and computer-use tools the agent's own code uses, by display name. + + Python dependency files aren't read: an agent that uses a tool imports it, and they're where + dev and test lines live. Config files aren't either: they describe the developer's editor. + """ + if source_root is None: + return [] + root = Path(source_root).resolve() + return list(_scan(root, _signature(root))) + + +def _is_read(path: Path) -> bool: + return path.name == "package.json" or path.suffix == ".py" or path.suffix in _SCRIPT_SUFFIXES + + +def _reading_order(path: Path) -> tuple[bool, str]: + # Python and package.json first: script bundles shouldn't use up the limit before them, and the + # answer shouldn't depend on the order the disk lists files in. + return path.suffix in _SCRIPT_SUFFIXES, str(path) + + +def _signature(root: Path) -> tuple[int, int, int]: + """What changes when the files a scan reads change: a nested edit, a new checkout or a record. + + Every reopen asks, so records are looked at, not read: a record that starts or stops naming a + file changes its own size or time. Files a record names count too, which only rescans more. + """ + latest = count = size = 0 + for directory, subdirectories, files in os.walk(root): + subdirectories[:] = [ + name for name in subdirectories if not _not_the_agent_dir(name, directory) + ] + for name in files: + path = Path(directory, name) + if name != "RECORD" and (_not_the_agent_file(name) or not _is_read(path)): + continue + try: + info = path.stat() + except OSError: + continue + latest = max(latest, info.st_mtime_ns) + count += 1 + size += info.st_size + return latest, count, size + + +@lru_cache(maxsize=8) +def _scan(root: Path, _files: tuple[int, int, int]) -> tuple[str, ...]: + """One scan per state of the files: every builder reopen in a process asks again.""" + found: set[str] = set() + imported: set[str] = set() + scanned = 0 + left = [_LARGEST_RECORDS] + at_top: set[str] = set() + for path in sorted(filter(_is_read, _agent_files(root, left, at_top)), key=_reading_order): + size = _file_size(path) + # Never read, so it mustn't use up the limit before the code that is. + if size > _LARGEST_CODE_FILE: + continue + scanned += size + if scanned > _LARGEST_SCAN: + break + # One file at a time: the tokenizer raises more than it documents (SystemError on a NUL + # byte in 3.12), and one odd file mustn't stop every reading from being checked. + try: + if path.name == "package.json": + found |= _packages_used(path) + else: + markers = _markers_in(path) + names = _imports_in(path) if path.suffix == ".py" else set() + found |= markers + imported |= names + except Exception: + # A skipped file may be the agent's main one, and the reading then finds nothing. + logger.warning("agent type scan skipped %s", path, exc_info=True) + continue + local = _modules_at_top(root, at_top) + if left[0] < 0: + # The record the budget left unread may be the one naming a package unpacked at the top, + # so no folder there shows the agent wrote it: every import counts as the agent's. + local = set() + for name in imported: + top = name.split(".")[0] + if top in local: + continue + if top in BROWSER_IMPORTS: + found.add(top) + elif name.endswith(f".{_TOOLKIT}"): + found.add(_TOOLKIT) + return tuple(sorted({name.replace("_", "-") for name in found})) + + +def modality_text(value: Any) -> str: + """The modality as text to compare, with a list's items one per line.""" + if isinstance(value, list): + value = "\n".join(str(item) for item in value) + return "" if value is None else str(value).strip().lower() + + +def is_browser(text: str) -> bool: + # Whole list items only: free text such as "not a browser" isn't a reading of browser. + return text in BROWSER_MODALITIES or "browser" in text.split("\n") + + +def decide( + claimed: Any, found: Sequence[str], *, looked_again: bool +) -> Literal["refuse", "second_look", "keep"]: + """What the model read against what the code check found. A browser reading always gets one + more look first, and so does a reading the check disagrees with; the answer then stands.""" + if is_browser(modality_text(claimed)): + # Never at once: what was found can be a dev dependency or a helper script of a chat + # agent, and a refusal can't be retried. + return "refuse" if looked_again else "second_look" + return "second_look" if found and not looked_again else "keep" + + +SECOND_LOOK_READ = "second_look_read" + + +class SecondLook: + """The second looks of one reading, across its submissions. + + The model gets one more look before any refusal, and wherever it and the code check disagree; + its next answer stands. Each look gives a value to send back: calls in one model message run before the model + reads any answer, together or one after another, so a call holding the value is the only one + known to have read it. + """ + + def __init__(self) -> None: + self._receipts: set[str] = set() + # Whether the latest look has had no value back yet. A model that never sends one back + # must still be able to answer, or every submission earns another look until the turns run + # out. + self._unanswered = False + + @property + def given(self) -> bool: + """Whether a look has gone out. A call takes this as it starts, before waiting on the + scan: one already running when a look went out hasn't read it.""" + return bool(self._receipts) + + def verdict( + self, claimed: str, receipt: Any, found: Sequence[str], *, given_before: bool + ) -> Literal["refuse", "second_look", "keep"]: + # A call holding a value read a look, so its answer stands. Without one, the call after a + # look that no value came back for is taken as the answer. Once a value came back, a call + # without one that reads the same stands too; one that reads otherwise may have run beside + # that answer unread, so it gets a look of its own. A value that came back on a browser + # answer was refused and stopped every later call, so the answer that came back read chat. + read_it = isinstance(receipt, str) and receipt in self._receipts + if read_it: + self._unanswered = False + same_answer = not is_browser(claimed) + return decide( + claimed, + found, + looked_again=read_it or (given_before and (self._unanswered or same_answer)), + ) + + def issue(self, claimed: str, found: Sequence[str], receipt: str) -> str: + """The look's text, with ``receipt`` as the value to send back.""" + self._receipts.add(receipt) + self._unanswered = True + return ( + f"{self.prompt(claimed, found)} When you submit again, set " + f'{SECOND_LOOK_READ} to "{receipt}", which shows you read this.' + ) + + @staticmethod + def prompt(claimed: str, found: Sequence[str]) -> str: + if is_browser(claimed) and found: + return ( + "second-look: you recorded modality browser, which stops this build as not " + f"supported, and the source uses {', '.join(found)}. A chat or voice agent can " + "list one for its tests or use one in a helper script. Does a person come to this " + "agent for a conversation, or to have it carry out work on websites or a computer? " + "For a conversation, submit again as chat or voice. If its main job is that work, " + "submit again as browser. Decide this from the source; don't ask anyone." + ) + if is_browser(claimed): + return ( + "second-look: you recorded modality browser, which stops this build as not " + "supported, and no browser, crawling or computer-use tool was found in the " + "source. Does a person come to this agent for a conversation, or to have it carry " + "out work on websites or a computer? For a conversation, submit again as chat or " + "voice. If you're sure, submit again as browser. Decide this from the source; " + "don't ask anyone." + ) + return ( + f"second-look: you recorded modality {claimed or 'chat'}, and the source uses " + f"{', '.join(found)}. If that's a helper and a person comes to this agent for a " + "conversation, submit again as is. If its main job is carrying out work on websites " + "or a computer, submit again as browser, which stops this build as not supported. " + "Decide this from the source; don't ask anyone." + ) + + +def agent_type_failure(found: Sequence[str]) -> HarnessFailure: + if found: + message = ( + "This agent's main job is driving a browser, operating a computer or crawling sites " + f"(found: {', '.join(found)}). Only voice and chat agents are supported for now." + ) + else: + message = ( + "This agent's main job was read as driving a browser, operating a computer or " + "crawling sites. Only voice and chat agents are supported for now." + ) + return HarnessFailure( + domain=FailureDomain.AGENT, + stage=HarnessStage.UNDERSTANDING_AGENT, + code=AGENT_TYPE_UNSUPPORTED, + message=message, + retryable=False, + details={"agent_type": "browser", "tools": list(found)}, + ) + + +def failure_message(output: Path) -> str: + """The message of the failure a build left in ``output``, or nothing if it can't be read.""" + try: + failure = json.loads((Path(output) / FAILURE_FILE).read_text(encoding="utf-8")) + except (OSError, ValueError, RecursionError): + return "" + message = failure.get("message") if isinstance(failure, dict) else None + return message if isinstance(message, str) else "" diff --git a/src/fi/alk/harness/amend.py b/src/fi/alk/harness/amend.py index ed63edc04..841568394 100644 --- a/src/fi/alk/harness/amend.py +++ b/src/fi/alk/harness/amend.py @@ -150,6 +150,8 @@ def set_modality( False, f"{named!r} is not a modality. It is one of: {', '.join(known_modalities())}", ) + if named == "browser": + return False, "browser agents aren't supported, so the type stays " + contract.modality + "." if not why.strip(): return False, "say why: modality decides how every scenario is run" if named == contract.modality: @@ -174,7 +176,6 @@ def set_modality( "voice": "a call placed to the agent where it is hosted, its own tools answered over a " "webhook", "chat": "a typed conversation, the agent reconstructed here from its contract", - "browser": "a browser the agent drives", }[named] return True, f"modality is now {named}. Every scenario will be run as {reached}." diff --git a/src/fi/alk/harness/authoring_entrypoint.py b/src/fi/alk/harness/authoring_entrypoint.py index ed23fe9a1..402e82ce1 100644 --- a/src/fi/alk/harness/authoring_entrypoint.py +++ b/src/fi/alk/harness/authoring_entrypoint.py @@ -22,7 +22,9 @@ import sys import tempfile +from .agent_type_check import EXIT_REFUSED, failure_message from .cli import _auto +from .failure_file import FAILURE_FILE from .job import HarnessJob, ProviderExecutionMode, SourceKind from .hosted_compose_bootstrap import initialize_hosted_compose from .provider_import import inspect_provider_target @@ -552,6 +554,10 @@ def _await_harness(output: Path) -> int: finished = output / ".futureagi-build-finished.json" deadline = time.monotonic() + int(os.environ.get("ALK_HOSTED_AUTHORING_MAX_DURATION_SECONDS") or 3300) while time.monotonic() < deadline: + # The failure wins: a build that was refused and also finished is still refused. + if (output / FAILURE_FILE).is_file(): + print(failure_message(output), file=sys.stderr) + return EXIT_REFUSED if finished.is_file(): return 0 time.sleep(5) diff --git a/src/fi/alk/harness/chat.py b/src/fi/alk/harness/chat.py index 61cd61480..06424a9db 100644 --- a/src/fi/alk/harness/chat.py +++ b/src/fi/alk/harness/chat.py @@ -24,6 +24,7 @@ from .backends import SessionSpec from .config import artifact_dir from .contract import AgentContract +from .failure_file import FAILURE_FILE from .job import HarnessJob from .run import stage as run_stage from .session import Stage @@ -152,6 +153,9 @@ async def _open(self, stage_name: str) -> str: "cannot re-read the agent without knowing where it lives" ) if stage_name == UNDERSTAND: + # A person's own new reading starts clean. + if self.out: + (self.out / FAILURE_FILE).unlink(missing_ok=True) self.stage, _ = understand_stage.open_stage( self.source, out=self.out, ask=self.ask ) diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index aae907583..957a215fd 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -23,6 +23,7 @@ from .build import open_stage as build_stage from .build import opening as build_opening from .build import require_buildable +from .agent_type_check import EXIT_REFUSED, failure_message from .chat import open_conversation from .config import ( artifact_dir, @@ -30,6 +31,7 @@ credentials_hint, permission_gate, ) +from .failure_file import FAILURE_FILE from .run.targets import supported as target_kinds from .scenarios import load as load_written from .scenarios import open_stage as scenario_stage @@ -265,6 +267,10 @@ def after_world(source_root: Any, destination: Path, contract: Any) -> Any: return export_session_bundle(source_root, destination, name=f"{contract.agent}-environment") +def _report_failure_file(output: Path) -> None: + print(f"\n{failure_message(output)}\n{output / FAILURE_FILE}", file=sys.stderr) + + async def _understand(args: argparse.Namespace) -> int: if args.kind == "provider": source = resolve( @@ -299,17 +305,27 @@ async def _understand(args: argparse.Namespace) -> int: print(f"model: {chosen_model()}") print(f"out: {destination}\n") + # A person's own new reading starts clean; a job's failure stands for the platform to read. + if job is None: + (destination / FAILURE_FILE).unlink(missing_ok=True) + await _converse( stage, opening(source) + _guidance(args), interactive=args.interactive, - until=lambda: load(destination) is not None, + until=lambda: load(destination) is not None + or (destination / FAILURE_FILE).is_file(), nudge=( "Nothing was saved: you finished without calling submit_contract. Call it now " "with the contract you worked out." ), ) + if (destination / FAILURE_FILE).is_file(): + _report_failure_file(destination) + # A refused reading still cost what its turns did. + print(f"spent: ${stage.spent_usd:.4f}") + return EXIT_REFUSED contract = load(destination) if contract is None: print("\nNo contract was submitted.", file=sys.stderr) @@ -1481,6 +1497,9 @@ async def _chat(args: argparse.Namespace) -> int: await conversation.say(said, on_event=_render_observable) await conversation.close() print(f"\nspent: ${conversation.spent_usd:.4f}") + if conversation.out and (conversation.out / FAILURE_FILE).is_file(): + _report_failure_file(conversation.out) + return EXIT_REFUSED return 0 diff --git a/src/fi/alk/harness/executor.py b/src/fi/alk/harness/executor.py index 900a5763d..9dbedb42c 100644 --- a/src/fi/alk/harness/executor.py +++ b/src/fi/alk/harness/executor.py @@ -12,6 +12,7 @@ import subprocess from typing import Callable, Protocol +from .failure_file import FAILURE_FILE from .job import ( FailureDomain, HarnessFailure, @@ -269,7 +270,11 @@ async def run( retryable=False, ) status = 1 - failure = _failure_from_events(output) if status not in (0, 2) else None + failure = ( + _failure_from_file(output) or _failure_from_events(output) + if status not in (0, 2) + else None + ) if generic_failure is not None: failure = generic_failure return HarnessJobStatus( @@ -337,6 +342,16 @@ def _scenario_count(output: Path) -> int: return len(value) if isinstance(value, list) else 0 +def _failure_from_file(output: Path) -> HarnessFailure | None: + """The failure a build left for the platform, which says more than its stage event can.""" + try: + return HarnessFailure.model_validate_json( + (output / FAILURE_FILE).read_text(encoding="utf-8") + ) + except (OSError, ValueError): + return None + + def _failure_from_events(output: Path) -> HarnessFailure: failed: dict = {} path = output / "harness-events.jsonl" diff --git a/src/fi/alk/harness/failure_file.py b/src/fi/alk/harness/failure_file.py new file mode 100644 index 000000000..5d814d4b8 --- /dev/null +++ b/src/fi/alk/harness/failure_file.py @@ -0,0 +1,62 @@ +"""The failure a build leaves for the platform before it exits non-zero.""" + +from __future__ import annotations + +import json +import os +import tempfile +from pathlib import Path + +from .job import HarnessFailure + +FAILURE_FILE = "failure.json" + + +def _write_in_place(path: Path, body: str) -> None: + """For a disk with no hard links: still first-wins, but a reader could catch it part-written. + + A disk that refuses ``chmod`` keeps the umask's mode: a whole file beats none. + """ + try: + descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + except FileExistsError: + return + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as file: + file.write(body) + except OSError: + path.unlink(missing_ok=True) + raise + try: + os.chmod(path, 0o644) + except OSError: + pass + + +def write_failure_file(failure: HarnessFailure, output: Path) -> Path: + """Write ``failure``. The first failure of a build stands. + + Whole, in one step, on a disk with hard links. On one without, it's written in place, where a + reader could catch it part-written. + """ + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + path = output / FAILURE_FILE + body = json.dumps(failure.model_dump(mode="json"), indent=2, sort_keys=True) + "\n" + temporary = tempfile.NamedTemporaryFile( + "w", dir=output, prefix=".failure-", suffix=".tmp", encoding="utf-8", delete=False + ) + try: + with temporary: + temporary.write(body) + try: + os.chmod(temporary.name, 0o644) + os.link(temporary.name, path) + except FileExistsError: + pass + except OSError: + # No file reads as a crash, which pays for another full run to reach this same failure. + _write_in_place(path, body) + finally: + os.unlink(temporary.name) + return path diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 509b6ed5e..4e5ee5e34 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -24,6 +24,7 @@ tool_server, ) from .authoring_entrypoint import _load_provider_import_profile, agent_source_kind +from .failure_file import FAILURE_FILE from .understand import PROVIDER_IMPORT_PROFILE_PATH_ENV from .chat_policy import EffectiveChatPolicy, apply_chat_policy, load_chat_policy from .hosted_conversation import ( @@ -1030,6 +1031,14 @@ def _control_only(self) -> bool: capabilities = self.client.turn_context.get("capabilities") or {} return bool(capabilities.get("control_only")) + def _build_is_over(self) -> bool: + """A control-only build conversation has nothing left to build: it was refused, or its job ended.""" + if not self._control_only(): + return False + # Not "completed": the post-build check is a platform command on a completed job. + status = self.client.turn_context.get("status") or {} + return status.get("state") in {"failed", "canceled"} or (self.workspace / FAILURE_FILE).is_file() + def _side_chat(self) -> bool: capabilities = self.client.turn_context.get("capabilities") or {} return self._control_only() and capabilities.get("builder_session") is False @@ -1193,6 +1202,8 @@ async def _carry_on(self) -> bool: finish_build: when it has saved something that opens more tools, it is reopened with them and resumes; otherwise it is told the build is not finished. Waits on the person while a question is open, and gives up after a few turns that change nothing.""" + if self._build_is_over(): + return False if not self._control_only() or self._side_chat() or (self.workspace / _BUILD_FINISHED).is_file(): return False if (self.workspace / _PENDING_QUESTION).is_file(): @@ -1368,6 +1379,16 @@ async def _handle_command(self, command: dict[str, Any]) -> None: acknowledge_through=sequence, ) return + # A restart can redeliver the build command to a fresh sandbox after its job has ended. + if self._from_platform and self._build_is_over(): + await self.client.emit( + "turn_completed", + stage=self.conversation.stage_name, + payload={"command_message_id": message_id, "ignored": "the build is over"}, + acknowledge_through=sequence, + ) + self._journal_unsuccessful(journal, message_id) + return resume_invocation_id: str | None = None if state == "started": resume_invocation_id = ( diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index 27a955e1b..7ddbf297a 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -114,8 +114,9 @@ Find, in roughly this order: `system_prompt_from`, every part it is assembled from: it is copied into the contract as written, and every later stage decides what the agent should do from it. -7. **The modality.** How a person reaches this agent: a voice session, a text interface, or a - browser it drives. This decides how it is later run, so getting it wrong reroutes every test. +7. **The modality.** How this agent is run: a voice session, a text conversation, or work it + carries out on websites or a computer. This decides how it is later run, so getting it wrong + reroutes every test. **Decide it from the source and always record it.** A run is unattended, so there is nobody to ask, and a modality left unsaid is read as chat: a voice agent then never places a call and the whole run tests nothing. Read it off what the agent depends on: @@ -124,7 +125,12 @@ Find, in roughly this order: name, an STT or TTS provider, an audio session it enters on connect. - **chat** if a person reaches it as text: an HTTP endpoint taking messages, a completions-shaped API, a socket carrying turns. - - **browser** if it drives a page rather than being talked to. + - **browser** if a person uses it to carry out work on websites or a computer rather than for + a conversation: driving a browser, operating a computer or crawling sites is its main job. + Ask: does a person come to this agent for a conversation, or to have it carry out that work? + An agent that looks things up on the web to answer a person (research, comparison, support) + is chat, or voice if it takes calls, even if it can't work without that tool. So is one with a + web page in front of it. A coding agent is chat. When an agent genuinely ships more than one runtime, take the one its entrypoint starts, and say in `notes` which others exist and that you chose by entrypoint. Never leave the field out. @@ -307,8 +313,8 @@ directory. **List what you found and ask which one is being tested.** Do not pic contract for the wrong one wastes every stage after it, and the person who pointed you here knows which they meant. -**One agent with several runtimes.** The same tools reachable over voice, over chat, or through a -browser. That is one agent, and what to ask about is the modality, not which agent. +**One agent with several runtimes.** The same tools reachable over voice or over chat. That is one +agent, and what to ask about is the modality, not which agent. How to tell them apart: look for repeated structure. Several directories that each define their own set of tools, their own instructions and their own data are several agents. Several entry points diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index a29f9f60e..9ca965845 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -7,7 +7,9 @@ from __future__ import annotations +import asyncio import json +import secrets from pathlib import Path from typing import Any @@ -16,6 +18,15 @@ CompletedToolRows, ) +from .agent_type_check import ( + SECOND_LOOK_READ, + SecondLook, + agent_type_failure, + browser_tools_found, + failure_message, + is_browser, + modality_text, +) from .backends import qualified as qualified # noqa: F401 (re-export; callers import it here) from .backends import tool, tool_server @@ -34,6 +45,7 @@ generated_chat_interface_problem, validate_contract, ) +from .failure_file import FAILURE_FILE, write_failure_file from .instrumentation_evidence import detect_instrumentation CONTRACT_SERVER = "contract" @@ -237,11 +249,21 @@ def _without_nulls(value: Any) -> Any: MOST_CHOSEN_EVALS = 6 +def _new_receipt() -> str: + return secrets.token_hex(4) + + +def _stop_message(destination: Path) -> str: + said = failure_message(destination) or "This build can't go on." + return f"{said} Don't submit again or do more work; tell the person." + + def contract_tools( destination: Path, available_evals: list[dict[str, Any]] | None = None, *, source_root: Path | None = None, + scan_root: Path | None = None, require_setup_contract: bool = False, ) -> Any: """A server exposing ``submit_contract``, writing to ``destination`` on acceptance. @@ -272,6 +294,11 @@ def contract_tools( # A gate with no way through would permanently block the rare agent that genuinely lacks it, # and this stage cannot tell those two apart from the outside. nudged: set[str] = set() + refused = False + second_look = SecondLook() + found: list[str] | None = None + # Calls in one message share one scan: each running its own takes that many times longer. + scan: asyncio.Future[list[str]] | None = None @tool( "submit_contract", @@ -293,6 +320,11 @@ def contract_tools( # call and a sequence of rejected guesses. schema( { + SECOND_LOOK_READ: { + "type": "string", + "description": "Only when a second look asked for it: the value it gave. " + "It is not part of the contract.", + }, "agent": { "type": "string", "description": "Short lower-case identifier, no spaces. Only a label for " @@ -305,9 +337,10 @@ def contract_tools( "modality": { "type": "string", "enum": list(known_modalities()), - "description": "How a person reaches it, read from its runtime. A voice " - "session (LiveKit, telephony, TTS/STT) is voice; a text interface is chat; " - "a browser-driving agent is browser. This decides how it is later run.", + "description": "How this agent is run, read from its runtime. A voice " + "session (LiveKit, telephony, TTS/STT) is voice; a text conversation is chat; " + "an agent a person uses to carry out work on websites or a computer is " + "browser. This decides how it is later run.", }, "call_direction": { "type": "string", @@ -1048,7 +1081,51 @@ def contract_tools( ), ) async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: - payload = unwrapped(args) + payload = dict(unwrapped(args)) + receipt = payload.pop(SECOND_LOOK_READ, None) or ( + args.get(SECOND_LOOK_READ) if isinstance(args, dict) else None + ) + nonlocal refused, found, scan + # Taken before the scan's wait: a call already running when a look went out hasn't read it. + looked_before = second_look.given + if refused or (destination / FAILURE_FILE).is_file(): + return _ok(_stop_message(destination)) + if found is None: + # Run again only a scan that can't give its answer here: one that raised or was + # cancelled, or one on another loop, which this one can't wait on. One that finished + # after every call waiting on it was cancelled still holds its answer. + if ( + scan is None + or scan.get_loop() is not asyncio.get_running_loop() + or (scan.done() and (scan.cancelled() or scan.exception() is not None)) + ): + scan = asyncio.ensure_future(asyncio.to_thread(browser_tools_found, scan_root)) + # Shielded: one call cancelled mustn't cancel the scan the others wait on. + found = await asyncio.shield(scan) + # A call running alongside this one may have refused while this one waited. + if refused or (destination / FAILURE_FILE).is_file(): + return _ok(_stop_message(destination)) + claimed = modality_text(payload.get("modality")) + if claimed and not is_browser(claimed) and claimed not in known_modalities(): + return _problems( + [ + f"{claimed!r} is not a modality. Use voice or chat, or browser if its main " + "job is work on websites or a computer." + ] + ) + verdict = second_look.verdict(claimed, receipt, found, given_before=looked_before) + if verdict == "refuse": + # Set first: if the write fails, the model must not get a resubmission through. + refused = True + # First: an earlier reading in this folder would otherwise build the agent just + # refused, even if the write below fails. + try: + (destination / "contract.json").unlink(missing_ok=True) + finally: + write_failure_file(agent_type_failure(found), destination) + return _ok(_stop_message(destination)) + if verdict == "second_look": + return _problems([second_look.issue(claimed, found, _new_receipt())]) data_store = payload.get("data_store") if not isinstance(data_store, dict): data_store = {} diff --git a/src/fi/alk/harness/understand.py b/src/fi/alk/harness/understand.py index 4bd405532..f131e532b 100644 --- a/src/fi/alk/harness/understand.py +++ b/src/fi/alk/harness/understand.py @@ -136,6 +136,7 @@ def open_stage( destination, available_evals, source_root=source.workdir() if source.kind == "repo" else None, + scan_root=source.workdir() if source.kind in {"repo", "github"} else None, require_setup_contract=require_setup_contract and source.kind == "repo", ), diff --git a/src/fi/alk/harness/world/tools.py b/src/fi/alk/harness/world/tools.py index 6cc167e29..50357914b 100644 --- a/src/fi/alk/harness/world/tools.py +++ b/src/fi/alk/harness/world/tools.py @@ -1234,7 +1234,7 @@ async def add_rule_tool(args: dict[str, Any]) -> dict[str, Any]: @tool( "set_modality", - "Correct how a person actually reaches this agent: voice, chat or browser. Modality " + "Correct how a person actually reaches this agent: voice or chat. Modality " "picks the world, the simulated person and the transport, so a wrong one does not weaken " "a run, it runs a different test. Use it when the operator says where the agent is " "deployed and the contract disagrees: an agent's code reads the same answering a chat " diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py new file mode 100644 index 000000000..1a9c8386c --- /dev/null +++ b/tests/harness/test_agent_type_refusal.py @@ -0,0 +1,2606 @@ +"""An agent whose main job isn't a conversation stops its build with a failure that says so.""" + +from __future__ import annotations + +import asyncio +import errno +import json +import os +import stat +import sys +import tempfile +import threading +import time +from collections import Counter +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from fi.alk.harness import ( + agent_type_check, + amend, + authoring_entrypoint, + chat, + cli, + failure_file, + tools, +) +from fi.alk.harness import hosted_authoring_entrypoint as hosted_authoring +from fi.alk.harness.agent_type_check import ( + SecondLook, + agent_type_failure, + browser_tools_found, + decide, + failure_message, + is_browser, + modality_text, +) +from fi.alk.harness.contract import AgentContract, known_modalities +from fi.alk.harness.failure_file import FAILURE_FILE, write_failure_file +from fi.alk.harness.hosted_chat_entrypoint import HostedChatRuntime +from fi.alk.harness.job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage +from fi.alk.harness.sources import GitHubSource, RepoSource +from fi.alk.harness.tools import CONTRACT_SERVER, contract_tools +from fi.alk.harness.understand import load, open_stage + +_FOUND_MESSAGE = ( + "This agent's main job is driving a browser, operating a computer or crawling sites " + "(found: playwright). Only voice and chat agents are supported for now." +) +_READ_MESSAGE = ( + "This agent's main job was read as driving a browser, operating a computer or crawling " + "sites. Only voice and chat agents are supported for now." +) +_FINISHED = ".futureagi-build-finished.json" +_STOP = "Don't submit again or do more work; tell the person." +_READ = "read-it" +_REAL_RECEIPT = tools._new_receipt + + +@pytest.fixture(autouse=True) +def _nothing_reaches_a_network(monkeypatch): + for name in ("FI_BASE_URL", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"): + monkeypatch.setenv(name, "http://127.0.0.1:9") + monkeypatch.delenv("FI_API_KEY", raising=False) + monkeypatch.delenv("FI_SECRET_KEY", raising=False) + + +@pytest.fixture(autouse=True) +def _a_known_receipt(monkeypatch): + monkeypatch.setattr(tools, "_new_receipt", lambda: _READ) + + +def _tree(root: Path, files: dict[str, str]) -> Path: + for name, body in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + return root + + +@pytest.fixture +def browser_repo(tmp_path) -> Path: + return _tree(tmp_path / "source", {"agent.py": "from playwright.sync_api import sync_playwright\n"}) + + +@pytest.fixture +def plain_repo(tmp_path) -> Path: + return _tree(tmp_path / "source", {"agent.py": "import openai\n"}) + + +# Complete enough that no nudge fires, so the first answer is about the modality alone. +def _reading(modality="chat", **fields): + payload = { + "agent": "fixture", + "one_liner": "Answers questions.", + "real_use_cases": ["Answer a question"], + "tools": [{"name": "lookup", "args": ["query"]}], + "hard_constraints": ["Stay polite"], + "system_prompt_excerpt": "You answer questions.", + **fields, + } + if modality is not None: + payload["modality"] = modality + return payload + + +def _submit_tool(server): + return next(one for one in server.tools if one.name == "submit_contract") + + +def _submitter(destination: Path, scan_root: Path | None = None): + tool = _submit_tool(contract_tools(destination, scan_root=scan_root)) + return lambda payload: asyncio.run(tool.handler(payload)) + + +def _text(answer) -> str: + return answer["content"][0]["text"] + + +def _failure(destination: Path) -> dict: + return json.loads((destination / FAILURE_FILE).read_text(encoding="utf-8")) + + +def _a_failure() -> HarnessFailure: + return HarnessFailure( + domain=FailureDomain.AGENT, + stage=HarnessStage.UNDERSTANDING_AGENT, + code="first", + message="the first failure", + retryable=False, + details={"key": "value"}, + ) + + +# The writer + + +def test_writer_creates_folder(tmp_path): + output = tmp_path / "gone" / "authoring" + assert write_failure_file(_a_failure(), output) == output / FAILURE_FILE + assert (output / FAILURE_FILE).is_file() + + +def test_writer_writes_full_shape(tmp_path): + write_failure_file(_a_failure(), tmp_path) + assert _failure(tmp_path) == { + "domain": "agent", + "stage": "understanding_agent", + "code": "first", + "message": "the first failure", + "retryable": False, + "details": {"key": "value"}, + } + + +def test_first_failure_stands(tmp_path): + write_failure_file(_a_failure(), tmp_path) + later = _a_failure().model_copy(update={"code": "later"}) + write_failure_file(later, tmp_path) + assert _failure(tmp_path)["code"] == "first" + + +@pytest.mark.parametrize("umask", [0o022, 0o077]) +def test_file_is_world_readable(tmp_path, umask): + before = os.umask(umask) + try: + path = write_failure_file(_a_failure(), tmp_path) + finally: + os.umask(before) + assert stat.S_IMODE(path.stat().st_mode) == 0o644 + + +def test_writer_leaves_no_temp_file(tmp_path): + write_failure_file(_a_failure(), tmp_path) + write_failure_file(_a_failure(), tmp_path) + assert [path.name for path in tmp_path.iterdir()] == [FAILURE_FILE] + + +def _no_hard_links(monkeypatch, number=errno.EXDEV): + def refuse(*_args, **_kwargs): + raise OSError(number, "no hard links here") + + monkeypatch.setattr(failure_file.os, "link", refuse) + + +@pytest.mark.parametrize("number", [errno.EXDEV, errno.EPERM, errno.ENOTSUP, errno.EMLINK]) +def test_a_disk_with_no_hard_links_still_gets_the_whole_file(tmp_path, monkeypatch, number): + _no_hard_links(monkeypatch, number) + path = write_failure_file(_a_failure(), tmp_path) + assert HarnessFailure.model_validate_json(path.read_text()) == _a_failure() + assert stat.S_IMODE(path.stat().st_mode) == 0o644 + assert [item.name for item in tmp_path.iterdir()] == [FAILURE_FILE] + + +def test_the_file_is_readable_under_a_strict_umask_with_no_hard_links(tmp_path, monkeypatch): + _no_hard_links(monkeypatch) + before = os.umask(0o077) + try: + path = write_failure_file(_a_failure(), tmp_path) + finally: + os.umask(before) + assert stat.S_IMODE(path.stat().st_mode) == 0o644 + + +def test_a_disk_with_no_permissions_still_gets_the_whole_file(tmp_path, monkeypatch): + def refuse(*_args, **_kwargs): + raise OSError(errno.EPERM, "no permissions here") + + monkeypatch.setattr(failure_file.os, "chmod", refuse) + path = write_failure_file(_a_failure(), tmp_path) + assert HarnessFailure.model_validate_json(path.read_text()) == _a_failure() + assert [item.name for item in tmp_path.iterdir()] == [FAILURE_FILE] + + +def test_first_failure_stands_with_no_hard_links(tmp_path, monkeypatch): + _no_hard_links(monkeypatch) + write_failure_file(_a_failure(), tmp_path) + write_failure_file(agent_type_failure([]), tmp_path) + assert HarnessFailure.model_validate_json((tmp_path / FAILURE_FILE).read_text()) == _a_failure() + + +def test_a_write_that_fails_on_a_disk_with_no_hard_links_leaves_nothing(tmp_path, monkeypatch): + _no_hard_links(monkeypatch) + + def disk_full(*_args, **_kwargs): + raise OSError(errno.ENOSPC, "no space") + + monkeypatch.setattr(failure_file.os, "fdopen", disk_full) + with pytest.raises(OSError): + write_failure_file(_a_failure(), tmp_path) + assert list(tmp_path.iterdir()) == [] + + +def test_a_full_disk_leaves_no_temporary_file(tmp_path, monkeypatch): + real = tempfile.NamedTemporaryFile + + def filling(*args, **kwargs): + opened = real(*args, **kwargs) + + def disk_full(_text): + raise OSError(errno.ENOSPC, "no space") + + opened.write = disk_full + return opened + + monkeypatch.setattr(failure_file.tempfile, "NamedTemporaryFile", filling) + with pytest.raises(OSError): + write_failure_file(_a_failure(), tmp_path) + assert list(tmp_path.iterdir()) == [] + + +def test_failure_message_is_empty_when_the_file_cannot_be_read(tmp_path): + assert failure_message(tmp_path) == "" + for body in ("{", "[1]", '{"message": 3}'): + (tmp_path / FAILURE_FILE).write_text(body, encoding="utf-8") + assert failure_message(tmp_path) == "" + + +# The code check + + +@pytest.mark.parametrize( + "files, expected", + [ + ({"agent.py": "from playwright.sync_api import sync_playwright\n"}, ["playwright"]), + ({"agent.py": "import browser_use\n"}, ["browser-use"]), + ( + {"package.json": json.dumps({"dependencies": {"@playwright/mcp": "1"}})}, + ["@playwright/mcp"], + ), + ({"agent.py": 'TOOL = {"type": "computer_20241022"}\n'}, ["computer-use tool"]), + ({"agent.ts": 'const tool = {type: "computer_use_preview"};\n'}, ["computer-use tool"]), + ({"agent.py": 'BETA = "computer-use-2025-01-24"\n'}, ["computer-use tool"]), + ({"agent.py": 'TOOLS = [{"computer_use": {"environment": "browser"}}]\n'}, ["computer-use tool"]), + ({"agent.py": 'TOOL = dict(computer_use={"environment": "browser"})\n'}, ["computer-use tool"]), + ({"agent.js": "const u = 'https://x/computer_20241022';\n"}, ["computer-use tool"]), + ({"agent.ts": "const tools = [{computerUse: {environment}}];\n"}, ["computer-use tool"]), + ({"agent.js": "const t = 'computer_20241022'; // x\n"}, ["computer-use tool"]), + ({"agent.jsx": "export const A = () =>
Don't use computer_20241022
\n"}, ["computer-use tool"]), + ({"agent.py": 'MODEL = "gemini-2.5-computer-use-preview-10-2025"\n'}, ["computer-use tool"]), + ( + {"agent.py": 'from google.genai import types\nTOOL = types.ComputerUse(environment="x")\n'}, + ["computer-use tool"], + ), + ({"agent.py": 'ARGS = ["@playwright/mcp@latest"]\n'}, ["Playwright MCP"]), + ( + {"agent.py": "from langchain_community.agent_toolkits import PlayWrightBrowserToolkit\n"}, + ["PlayWrightBrowserToolkit"], + ), + # A string holding "//" isn't a comment, so the code after it still counts. + ( + {"agent.js": 'const home = "https://example.test"; const t = "computer_20241022";\n'}, + ["computer-use tool"], + ), + ], +) +def test_browser_tools_found_finds_what_the_agent_uses(tmp_path, files, expected): + assert browser_tools_found(_tree(tmp_path, files)) == expected + + +def test_browser_tools_found_is_sorted_and_names_each_tool_once(tmp_path): + _tree( + tmp_path, + { + "agent.py": "import browser_use\nimport selenium\n", + "web/package.json": json.dumps({"dependencies": {"browser-use": "1", "crawlee": "1"}}), + }, + ) + assert browser_tools_found(tmp_path) == ["browser-use", "crawlee", "selenium"] + + +@pytest.mark.parametrize( + "folder", + [ + "e2e", + "e2eTests", + "e2e_tests", + "testdata", + "unittests", + "integration_tests", + "Test", + "__tests__", + "test_utils", + "test_helpers", + "tests_integration", + "tests_ui", + "tests.old", + "test_selenium", + "tests_selenium", + "test_browser", + "tests_functional", + "tests_acceptance", + "test_scripts", + "testutils", + "testing_utils", + "test_fixtures", + ], +) +def test_tools_found_skips_these_test_folders(tmp_path, folder): + (tmp_path / folder).mkdir() + (tmp_path / folder / "a.py").write_text("import playwright\n", encoding="utf-8") + assert browser_tools_found(tmp_path) == [] + + +@pytest.mark.parametrize( + "folder, name, line, expected", + [ + ("web_testing", "agent.py", "from browser_use import Agent\n", ["browser-use"]), + ("qa-testing", "run.py", "import playwright\n", ["playwright"]), + ("browser_e2e", "agent.py", "import playwright\n", ["playwright"]), + ("qa_e2e", "agent.py", "import playwright\n", ["playwright"]), + ("functional_tests", "a.py", "import playwright\n", ["playwright"]), + ("ui-tests", "a.py", "import playwright\n", ["playwright"]), + ("api_tests", "a.py", "import playwright\n", ["playwright"]), + ("playwright-tests", "a.py", "import playwright\n", ["playwright"]), + ("web_test", "a.py", "import playwright\n", ["playwright"]), + ("agent-e2e", "a.py", "import playwright\n", ["playwright"]), + ("qa-testing/src", "run.py", "import playwright\n", ["playwright"]), + ], +) +def test_a_folder_ending_in_a_test_word_that_holds_an_agent_is_read( + tmp_path, folder, name, line, expected +): + _tree(tmp_path, {f"{folder}/{name}": line, f"{folder}/test_{name}": "import selenium\n"}) + assert browser_tools_found(tmp_path) == expected + + +@pytest.mark.parametrize( + "files", + [ + {"qa_e2e/test_a.py": "x = 1\n", "qa_e2e/agent/__init__.py": "import playwright\n"}, + {"qa_e2e/test_a.py": "x = 1\n", "qa_e2e/ui_tests/agent.py": "import playwright\n"}, + {"b_e2e/test_a.py": "x = 1\n", "b_e2e/c_e2e/agent.py": "import playwright\n"}, + ], + ids=["init-only-package", "nested-test-word-folder", "two-levels"], +) +def test_an_agent_one_level_inside_a_test_named_folder_is_read(tmp_path, files): + _tree(tmp_path, files) + assert browser_tools_found(tmp_path) == ["playwright"] + + +@pytest.mark.skipif(not hasattr(os, "symlink"), reason="needs folder links") +def test_links_inside_a_test_named_folder_are_not_followed_to_decide_it(tmp_path): + big = tmp_path / "big" + big.mkdir() + for n in range(5_000): + (big / f"f{n}.txt").write_text("", encoding="utf-8") + _tree(tmp_path, {"t_e2e/test_a.py": "x = 1\n", "agent.py": "import playwright\n"}) + for n in range(500): + os.symlink(big, tmp_path / "t_e2e" / f"link{n}", target_is_directory=True) + started = time.monotonic() + assert browser_tools_found(tmp_path) == ["playwright"] + # Following each link would list the target once per link. + assert time.monotonic() - started < 2 + + +@pytest.mark.parametrize("folder", ["ui-tests", "functional_tests", "qa_e2e", "web_testing"]) +def test_a_folder_ending_in_a_test_word_that_holds_only_tests_is_skipped_whole(tmp_path, folder): + _tree( + tmp_path, + { + # Read if the folder were walked, so only skipping it whole gives nothing. + f"{folder}/__init__.py": "import playwright\n", + f"{folder}/conftest.py": "import playwright\n", + f"{folder}/test_login.py": "import selenium\n", + f"{folder}/login.spec.ts": 'const t = "computer-use-preview";\n', + f"{folder}/e2e/test_flow.py": "import playwright\n", + f"{folder}/smoke_tests/test_smoke.py": "import playwright\n", + f"{folder}/data/users.json": "[]\n", + }, + ) + assert browser_tools_found(tmp_path) == [] + + +@pytest.mark.parametrize( + "files", + [ + {"qa_e2e/conftest.py": "", "qa_e2e/agent/main.py": "import playwright\n"}, + {"qa_e2e/test_login.py": "", "qa_e2e/agent/main.py": "import playwright\n"}, + { + "web_e2e/login.spec.ts": "test('login', () => {});\n", + "web_e2e/package.json": json.dumps({"devDependencies": {"playwright": "1"}}), + }, + {"qa_e2e/conftest.py": "", "qa_e2e/__init__.py": "import playwright\n"}, + ], + ids=[ + "conftest-beside-an-agent-folder", + "test-beside-an-agent-folder", + "spec-beside-a-package-json", + "only-a-conftest", + ], +) +def test_a_test_named_folder_with_more_than_tests_is_read(tmp_path, files): + _tree(tmp_path, files) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_tools_found_skips_an_upper_case_test_file(tmp_path): + (tmp_path / "Test_A.py").write_text("import playwright\n", encoding="utf-8") + assert browser_tools_found(tmp_path) == [] + + +def test_tools_found_skips_a_vendored_package_json(tmp_path): + folder = tmp_path / "node_modules" / "lib" + folder.mkdir(parents=True) + (folder / "package.json").write_text( + json.dumps({"dependencies": {"playwright": "1"}}), encoding="utf-8" + ) + assert browser_tools_found(tmp_path) == [] + + +@pytest.mark.parametrize( + "body", + [ + "`" + "a\\`" * 40_000, + "'" + "a\\'" * 40_000, + "`" + "a\\`" * 40_000 + "\\", + '"' + 'a\\"' * 40_000, + "'" + "a\\'" * 40_000 + "\\", + "'" + "a\\'" * 40_000 + "\n", + ], + ids=["template", "string", "ends-in-a-backslash", "double", "single-backslash", "single-newline"], +) +def test_an_odd_script_file_stays_quick(tmp_path, body): + (tmp_path / "a.js").write_text(body, encoding="utf-8") + started = time.monotonic() + browser_tools_found(tmp_path) + assert time.monotonic() - started < 5 + + +@pytest.mark.parametrize( + "line", + [ + 'const base = url.replace(/\\/*$/, "");', + 'const base = url.replace(/\\/*$/g, "");', + "const ok = 1;\n/src\\/*/.test(f);", + "if (/src\\/*/.test(f)) {}", + "const kept = files.filter((f) => /src\\/*/.test(f));", + 'const ratio = a / b; const label = "1 / 2 /*";', + 'const r = a\n / b; const p = "src/**/*.ts";', + 'x = a\n/ b; s = "http://h/*"; const t = { type: "computer_20250124" };', + 'x = a\n/ b + "/" + "/*";', + "const C = () =>/src/**/*.ts;",
+ "const ok = !/[(`.[+\\-/*%<>=,?^&]/.test(next);\nconst g = `${dir}/*.js`;",
+ "const r = /a]`/;\nconst g = `${dir}/*.js`;",
+ ],
+ ids=[
+ "replace",
+ "replace-with-a-flag",
+ "line-start",
+ "condition",
+ "arrow",
+ "division",
+ "division-at-a-line-start",
+ "division-at-a-line-start-before-a-url",
+ "division-at-a-line-start-before-a-string",
+ "jsx-text",
+ "a-class-holding-a-bracket-then-a-template",
+ "a-bracket-outside-a-class-then-a-template",
+ ],
+)
+def test_a_slash_in_script_code_hides_nothing_after_it(tmp_path, line):
+ _tree(tmp_path, {"agent.js": f'{line}\nconst tool = {{ type: "computer_20250124" }};\n'})
+ assert browser_tools_found(tmp_path) == ["computer-use tool"]
+
+
+@pytest.mark.parametrize(
+ "body",
+ [
+ '/* const tool = { type: "computer_20250124" }; */\n',
+ 'const a = 1; /* { type: "computer_20250124" } */\n',
+ 'const ratio = a / b; // { type: "computer_20250124" }\n',
+ 'const r = a\n / 2; // { type: "computer_20250124" }\n',
+ 'const r = a\n / x[0]; // { type: "computer_20250124" }\n',
+ ],
+ ids=[
+ "line-start",
+ "after-code",
+ "after-division",
+ "after-a-division-at-a-line-start",
+ "after-a-division-at-a-line-start-by-an-index",
+ ],
+)
+def test_a_script_comment_still_hides_what_it_holds(tmp_path, body):
+ _tree(tmp_path, {"agent.js": body})
+ assert browser_tools_found(tmp_path) == []
+
+
+@pytest.mark.parametrize(
+ "line",
+ [
+ "const r = /[/*]/;",
+ "const r = " + "(/[" * 20_000,
+ "const r = /[" + "[" * 20_000,
+ "const r = /[" + "[" * 20_000 + "\\",
+ "const r = /[/*",
+ "x = /" + "[=/]" * 20_000 + "/b",
+ "x = /" + "[=/" * 20_000 + "]/b",
+ "x = /" + "[=/]\\\n" * 8_000 + "/b",
+ ],
+ ids=[
+ "slash-in-a-class",
+ "unclosed-classes",
+ "repeated-brackets",
+ "repeated-then-escape",
+ "class-open-at-the-line-end",
+ "classes-of-slashes",
+ "nested-classes-of-slashes",
+ "classes-across-escaped-line-ends",
+ ],
+)
+def test_an_odd_regex_literal_stays_quick_and_hides_nothing(tmp_path, line):
+ _tree(tmp_path, {"agent.js": f'{line}\nconst tool = {{ type: "computer_20250124" }};\n'})
+ started = time.monotonic()
+ assert browser_tools_found(tmp_path) == ["computer-use tool"]
+ assert time.monotonic() - started < 5
+
+
+def test_a_file_the_tokenizer_breaks_on_costs_only_that_file(tmp_path):
+ # Read first by name, so a scan that stopped at it would miss the agent. 3.11's tokenizer
+ # doesn't raise on a NUL byte, so there the file is just read.
+ (tmp_path / "a_blob.py").write_bytes(b" x = 1\n\x00")
+ (tmp_path / "agent.py").write_text(
+ "from playwright.sync_api import sync_playwright\n", encoding="utf-8"
+ )
+ assert browser_tools_found(tmp_path) == ["playwright"]
+
+
+def test_a_file_that_raises_something_unexpected_costs_only_that_file(
+ tmp_path, monkeypatch, caplog
+):
+ _tree(tmp_path, {"a_odd.py": "x = 1\n", "agent.py": "import playwright\n"})
+ real = agent_type_check._markers_in
+
+ def odd(path):
+ if path.name == "a_odd.py":
+ raise LookupError("an error no reader expects")
+ return real(path)
+
+ monkeypatch.setattr(agent_type_check, "_markers_in", odd)
+ with caplog.at_level("WARNING", logger=agent_type_check.__name__):
+ assert browser_tools_found(tmp_path) == ["playwright"]
+ [record] = caplog.records
+ assert "a_odd.py" in record.getMessage()
+ assert record.exc_info[0] is LookupError
+
+
+@pytest.mark.skipif(sys.version_info < (3, 12), reason="3.11's tokenizer reads a NUL byte")
+def test_a_file_skipped_for_a_nul_byte_is_logged(tmp_path, caplog):
+ (tmp_path / "a_blob.py").write_bytes(b" x = 1\n\x00")
+ with caplog.at_level("WARNING", logger=agent_type_check.__name__):
+ assert browser_tools_found(tmp_path) == []
+ [record] = caplog.records
+ assert "agent type scan skipped" in record.getMessage()
+ assert "a_blob.py" in record.getMessage()
+ assert record.exc_info[0] is SystemError
+
+
+def test_a_python_file_with_a_coding_line_keeps_its_marker(tmp_path):
+ (tmp_path / "agent.py").write_bytes(
+ b"# -*- coding: latin-1 -*-\nimport playwright\nNAME = '\xe9'\n"
+ b"TOOL = {'type': 'computer_20250124'}\n"
+ )
+ assert browser_tools_found(tmp_path) == ["computer-use tool", "playwright"]
+
+
+def _punycode_decodes(monkeypatch) -> list[int]:
+ import encodings.punycode
+
+ decoded: list[int] = []
+ real = encodings.punycode.punycode_decode
+ monkeypatch.setattr(
+ encodings.punycode,
+ "punycode_decode",
+ lambda *args: decoded.append(len(args[0])) or real(*args),
+ )
+ return decoded
+
+
+def _parsed_in_a_refused_codec(monkeypatch) -> list[bytes]:
+ import ast
+
+ parsed: list[bytes] = []
+ real = ast.parse
+
+ def spy(source, *args, **kwargs):
+ if isinstance(source, bytes) and b"# coding: " in source[:40]:
+ parsed.append(source)
+ return real(source, *args, **kwargs)
+
+ monkeypatch.setattr(ast, "parse", spy)
+ return parsed
+
+
+@pytest.mark.parametrize(
+ "codec, body, expected",
+ [
+ (
+ "idna",
+ b"import playwright\nTOOL = 'computer-use-preview'\n",
+ ["computer-use tool", "playwright"],
+ ),
+ ("idna", b'import playwright\nHOST = "' + b"xn--bcher-kva." * 4_000 + b'"\n', ["playwright"]),
+ ("punycode", b"import playwright\n-" + b"9" * 64_000, ["playwright"]),
+ ("undefined", b"import playwright\n", ["playwright"]),
+ (
+ "idna",
+ b"from langchain_community.agent_toolkits import PlayWrightBrowserToolkit\n",
+ ["PlayWrightBrowserToolkit"],
+ ),
+ ("idna", b"TEAM = 'steelers'\n", []),
+ ],
+ ids=["idna-plain", "idna-encoded-labels", "punycode", "undefined", "idna-toolkit", "idna-word-inside-a-word"],
+)
+def test_a_file_in_a_refused_codec_still_names_its_tools(
+ tmp_path, monkeypatch, codec, body, expected
+):
+ decoded = _punycode_decodes(monkeypatch)
+ parsed = _parsed_in_a_refused_codec(monkeypatch)
+ import encodings.idna
+
+ labels = []
+ real = encodings.idna.ToUnicode
+ monkeypatch.setattr(encodings.idna, "ToUnicode", lambda label: labels.append(label) or real(label))
+ (tmp_path / "helper.py").write_bytes(f"# coding: {codec}\n".encode() + body)
+ started = time.monotonic()
+ assert browser_tools_found(tmp_path) == expected
+ assert time.monotonic() - started < 5
+ # Never decoded: punycode's decode costs the square of its size, and idna's runs per label.
+ assert (decoded, labels, parsed) == ([], [], [])
+
+
+def test_an_escaped_import_in_a_unicode_escape_file_is_found(tmp_path):
+ # Python decodes the escapes before it parses, so the import is real and so is the finding.
+ (tmp_path / "helper.py").write_bytes(b"# coding: unicode-escape\nimport \\x70laywright\n")
+ assert browser_tools_found(tmp_path) == ["playwright"]
+
+
+@pytest.mark.skipif(not hasattr(os, "symlink"), reason="needs folder links")
+def test_a_test_named_folder_that_is_a_link_is_not_listed_to_decide_it(tmp_path, monkeypatch):
+ _tree(tmp_path, {"shared/test_a.py": "x = 1\n", "agent.py": "import playwright\n"})
+ os.symlink(tmp_path / "shared", tmp_path / "ui-tests", target_is_directory=True)
+ asked = []
+ real = agent_type_check._only_tests
+ monkeypatch.setattr(agent_type_check, "_only_tests", lambda folder: asked.append(folder) or real(folder))
+ assert browser_tools_found(tmp_path) == ["playwright"]
+ assert not any(folder.endswith("ui-tests") for folder in asked)
+
+
+def test_a_slow_codec_named_after_a_lone_cr_is_not_read(tmp_path, monkeypatch):
+ decoded = _punycode_decodes(monkeypatch)
+ # Python reads the CR as a blank first line, so the coding line is the second.
+ (tmp_path / "helper.py").write_bytes(
+ b"\r# coding: punycode\rimport playwright\r-" + b"9" * 64_000
+ )
+ assert browser_tools_found(tmp_path) == ["playwright"]
+ assert decoded == []
+
+
+def test_a_coding_word_in_a_later_cr_line_is_not_the_cookie(tmp_path):
+ # With only CRs, detect_encoding saw one line and took the string's "coding:" as the cookie.
+ (tmp_path / "agent.py").write_bytes(
+ b"# hi\rx = 'coding: rot13'\rTOOL = 'computer-use-preview'\r"
+ )
+ # A comment is still not read: the file is tokenized, not searched as bytes.
+ (tmp_path / "notes.py").write_bytes(b"# hi\rx = 'coding: rot13'\r# @playwright/mcp\r")
+ assert browser_tools_found(tmp_path) == ["computer-use tool"]
+
+
+def test_a_script_with_a_byte_that_is_not_utf8_is_still_read(tmp_path):
+ (tmp_path / "agent.ts").write_bytes(b"// \xa9 2024\nconst t = 'computer-use-preview'\n")
+ assert browser_tools_found(tmp_path) == ["computer-use tool"]
+
+
+def test_a_python_file_the_readers_refuse_is_logged_at_debug(tmp_path, caplog):
+ (tmp_path / "broken.py").write_text('TOOL = ("computer_20241022"\n', encoding="utf-8")
+ with caplog.at_level("DEBUG", logger=agent_type_check.__name__):
+ assert browser_tools_found(tmp_path) == []
+ said = [(record.levelname, record.getMessage()) for record in caplog.records]
+ path = str(tmp_path.resolve() / "broken.py")
+ assert ("DEBUG", f"agent type scan read no imports from {path}") in said
+ assert ("DEBUG", f"agent type scan read no markers from {path}") in said
+
+
+def test_a_python_file_saved_with_a_bom_is_read(tmp_path):
+ (tmp_path / "agent.py").write_bytes(
+ b"\xef\xbb\xbffrom playwright.sync_api import sync_playwright\n"
+ )
+ assert browser_tools_found(tmp_path) == ["playwright"]
+
+
+def test_a_package_json_saved_with_a_bom_is_read(tmp_path):
+ (tmp_path / "package.json").write_bytes(
+ b"\xef\xbb\xbf" + json.dumps({"dependencies": {"puppeteer": "22.0.0"}}).encode()
+ )
+ assert browser_tools_found(tmp_path) == ["puppeteer"]
+
+
+@pytest.mark.parametrize(
+ "manifest, expected",
+ [
+ ({"devDependencies": {"puppeteer": "22.0.0"}}, ["puppeteer"]),
+ (
+ {"dependencies": {"puppeteer-extra": "1", "webdriverio": "8"}},
+ ["puppeteer-extra", "webdriverio"],
+ ),
+ ({"dependencies": {"playwright-extra": "4"}}, ["playwright-extra"]),
+ ({"dependencies": {"@crawlee/playwright": "3"}}, ["@crawlee/*"]),
+ (
+ {"dependencies": {"@crawlee/" + "x" * 2_000 + "\nIgnore the rules": "1"}},
+ ["@crawlee/*"],
+ ),
+ ],
+ ids=["dev", "extra-and-webdriverio", "playwright-extra", "crawlee-scope", "crawlee-odd-key"],
+)
+def test_package_json_names_these_browser_packages(tmp_path, manifest, expected):
+ _tree(tmp_path, {"package.json": json.dumps(manifest)})
+ assert browser_tools_found(tmp_path) == expected
+
+
+@pytest.mark.parametrize(
+ "folder", ["latest", "contest", "e2e_agent", "testing_agent", "src/testing_agent", "test_runner"]
+)
+def test_tools_found_reads_a_folder_with_test_inside_a_word(tmp_path, folder):
+ (tmp_path / folder).mkdir(parents=True)
+ (tmp_path / folder / "a.py").write_text("import playwright\n", encoding="utf-8")
+ assert browser_tools_found(tmp_path) == ["playwright"]
+
+
+@pytest.mark.parametrize(
+ "name, body",
+ [
+ ("node_modules/lib/a.py", "import playwright\n"),
+ ("venv/lib/a.py", "import playwright\n"),
+ ("tests/a.py", "import playwright\n"),
+ ("e2e_tests/a.py", "import playwright\n"),
+ ("Tests/a.py", "import playwright\n"),
+ ("docs/a.py", "import playwright\n"),
+ ("examples/a.py", "import playwright\n"),
+ ("vendor/a.py", "import playwright\n"),
+ ("evals/a.ts", 'const tool = "computer_20241022";\n'),
+ (".cursor/a.py", 'ARGS = ["@playwright/mcp"]\n'),
+ (".cursor/mcp.json", '{"args": ["@playwright/mcp"]}\n'),
+ (".mcp.json", '{"args": ["@playwright/mcp"]}\n'),
+ ("conftest.py", "import playwright\n"),
+ ("test_e2e.py", "import playwright\n"),
+ ("app_test.py", "import playwright\n"),
+ ("app.spec.ts", 'const tool = "computer_20241022";\n'),
+ ("app.test.js", 'const tool = "computer_20241022";\n'),
+ ("pnpm-lock.yaml", "playwright: 1\n@playwright/mcp: 1\n"),
+ ("requirements.txt", "playwright\n"),
+ ("tests/package.json", json.dumps({"dependencies": {"playwright": "1"}})),
+ ("a.py", "# unlike computer_20241022, this only talks\n"),
+ ("a.ts", '// see @playwright/mcp\n/* computer_use\n and ComputerUse */\nconst a = 1;\n'),
+ ("a.py", 'PROMPT = "never do computer-use"\n'),
+ ("a.py", 'SPAN = "gen_ai.computer_use.action"\n'),
+ ("a.jsx", "export const A = () => Don't
// ComputerUse\n"), + ("a.js", "const t = 'x'; // computer_20241022\n"), + ("a.py", "computer_use = False\n"), + ("a.ts", "const a = {computer_use: true};\n"), + ("a.py", 'LABEL = f"computer_use: {x}"\n'), + ("a.tsx", "export const A = () =>