From 433098437a3f8964c16b8b67147cdf0bf3b15fb5 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Wed, 11 Nov 2026 12:15:21 +0530 Subject: [PATCH 1/4] fix(harness): log why a seed contract couldn't be compared or taken back (TH-8952) Dileep's two P3s on #269, which merged before they could land there: - holds_seed prints the traceback before returning False, so a restart that can't compare contract.json with the seed says why it reports no seed-result.json; - _take_back prints the traceback when contract.json can't be removed, since a contract left behind is what makes the outcome `used`. Both catches see only the kit's own files (/work/authoring/contract.json and the platform-placed seed, which read_seed already parses safely), so no customer source reaches the trace. The two tests assert the trace reaches stderr. One fresh reviewer: clear; test_seed_contract 108 passed on 3.12.3, and each planted removal fails its named test. Signed-off-by: khushalsonawat --- src/fi/alk/harness/seed_contract.py | 8 +++++--- tests/harness/test_seed_contract.py | 8 ++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/fi/alk/harness/seed_contract.py b/src/fi/alk/harness/seed_contract.py index cc930a721..be52bd3ac 100644 --- a/src/fi/alk/harness/seed_contract.py +++ b/src/fi/alk/harness/seed_contract.py @@ -367,7 +367,8 @@ def holds_seed(contract: Path, seed: str | Path) -> bool: if problem is not None or raw is None: return False return contract.stat().st_size == len(raw) and contract.read_bytes() == raw - except Exception: # noqa: BLE001 - a contract that can't be compared isn't claimed as the seed + except Exception as exc: # noqa: BLE001 - a contract that can't be compared isn't claimed as the seed + print_trace(exc) return False @@ -455,5 +456,6 @@ def _take_back(output: Path, had_model: bool) -> None: (output / _CONTRACT).unlink(missing_ok=True) if not had_model: (output / _SOURCE_MODEL).unlink(missing_ok=True) - except OSError: - pass + except OSError as exc: + # The outcome turns on this: a contract left behind is reported `used`. + print_trace(exc) diff --git a/tests/harness/test_seed_contract.py b/tests/harness/test_seed_contract.py index 76c1bddf0..da56a2ce0 100644 --- a/tests/harness/test_seed_contract.py +++ b/tests/harness/test_seed_contract.py @@ -1368,7 +1368,7 @@ def replace(src, dst): assert "Traceback" in capsys.readouterr().err -def test_a_contract_that_cannot_be_compared_is_never_claimed(tmp_path, monkeypatch): +def test_a_contract_that_cannot_be_compared_is_never_claimed(tmp_path, monkeypatch, capsys): runtime = _real_world(_builder(tmp_path, monkeypatch)) workspace = tmp_path / "authoring" (workspace / "contract.json").write_bytes(_seed_path(tmp_path).read_bytes()) @@ -1381,6 +1381,8 @@ def breaks(path): assert _open(runtime).contract is not None assert not (workspace / SEED_RESULT).exists() + # Why the restart didn't claim the seed reaches the log. + assert "RuntimeError: boom-7" in capsys.readouterr().err def test_every_file_is_written_beside_its_place(tmp_path, monkeypatch): @@ -1503,7 +1505,7 @@ def test_a_mark_that_cannot_be_removed_still_leaves_the_outcome( assert "PermissionError" in err -def test_a_contract_that_cannot_be_taken_back_is_used(tmp_path, monkeypatch): +def test_a_contract_that_cannot_be_taken_back_is_used(tmp_path, monkeypatch, capsys): runtime = _real_world(_builder(tmp_path, monkeypatch)) _setup_fails(tmp_path, monkeypatch, runtime) _unlink_fails_for(monkeypatch, "contract.json") @@ -1513,6 +1515,8 @@ def test_a_contract_that_cannot_be_taken_back_is_used(tmp_path, monkeypatch): assert _outcome(tmp_path) == {"seed": "used", "differs": []} installed = (tmp_path / "authoring" / "contract.json").read_bytes() assert installed == _seed_path(tmp_path).read_bytes() + # The failed take-back, which is what makes the outcome `used`, reaches the log. + assert "PermissionError: [Errno 13] Permission denied" in capsys.readouterr().err # -- the round trip ---------------------------------------------------------------------------- From df2742cfb803f073e1b613e7f7f24cf8a71b63a5 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Wed, 11 Nov 2026 16:43:32 +0530 Subject: [PATCH 2/4] fix(harness): read a decode-first Python file as its script runs it too (TH-9490) - A coding line whose own bytes don't decode to it (EBCDIC, utf-16, utf-32) is read as `python file.py` reads it: the bytes after the coding line, decoded. A decode-first file whose head doesn't read as itself, or that holds any CR, is read both ways; the two texts are never joined, and the escape budget charges such a file at most twice. - Past the decode total, a WARNING is logged once per budget, naming the reader, with counts only. A file the whole decode refuses at the total's edge is read as any file past it is. - Tests skip a decoded-CR row when this Python no longer keeps the CR, and allow 3.14's bounded decode of the cut coding line; the walk parses each file's own readings only; the tool-evidence scan finds a tool spelled in utf-7. A docstring is rewrapped at 99. TH-9490 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_type_check.py | 13 +- src/fi/alk/harness/credentials.py | 10 +- .../alk/harness/instrumentation_evidence.py | 12 +- src/fi/alk/harness/python_source.py | 188 +++++- src/fi/alk/harness/source_tool_evidence.py | 2 +- tests/harness/test_python_source_codecs.py | 611 +++++++++++++++++- tests/harness/test_source_tool_evidence.py | 33 + 7 files changed, 819 insertions(+), 50 deletions(-) diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index bcc9826b0..c2f0b8dd3 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -588,7 +588,7 @@ def _tools_in( scanned = scripts = python = 0 budget = ParseBudget() tokens = TokenBudget() - escapes = EscapeBudget() + escapes = EscapeBudget(reader="browser scan") searched = ParseBudget(_LARGEST_RAW_SEARCH) skipped = 0 for path in sorted( @@ -619,13 +619,14 @@ def _tools_in( found |= _markers_in(path) elif ( text := read_module( - path, _largest_read(path), _PYTHON_NAMES, undecoded := [], escapes + path, _largest_read(path), _PYTHON_NAMES, undecoded := [], escapes, also := [] ) ) is not None: - markers = _python_markers(text, tokens, path) - names = _imports_in(text, path) if budget.spend(text) else set() - found |= markers - imported |= names + # An import and `python file.py` can read a crafted file two ways: each reading + # on its own, charged as it's read. + for reading in (text, *also): + found |= _python_markers(reading, tokens, path) + imported |= _imports_in(reading, path) if budget.spend(reading) else set() elif undecoded: # A codec refused by name, which Python still runs the file in: the browser # packages and tool markers it names, as bytes, count. That leads to a second diff --git a/src/fi/alk/harness/credentials.py b/src/fi/alk/harness/credentials.py index 2a31c61ef..c8f68aeec 100644 --- a/src/fi/alk/harness/credentials.py +++ b/src/fi/alk/harness/credentials.py @@ -1490,7 +1490,7 @@ def script_module(path: Path, name: str) -> Path | None: # One total for the whole walk: bundling has no lookup limit and no parse total, so this is # all that bounds what decoding first costs there, a unicode-escape file's second reading # included (at most this much more parsing per walk). - escapes = EscapeBudget() + escapes = EscapeBudget(reader="import walk") seen: set[Path] = set() pending = deque(sorted(_code_in(folder, root, unread))) own = set(pending) # Read once already, so importing one doesn't queue it again. @@ -1522,8 +1522,9 @@ def script_module(path: Path, name: str) -> Path | None: bytes_read += len(raw) # A .py in a codec that can spell a name in other bytes is read as Python reads it while # the walk's total lasts: decoded after its line ends, so hz's `~` before a CRLF joins. + also: list[str] = [] taken, decoded, encoding = ( - decoded_first(raw, escapes) if path.suffix == ".py" else (False, None, None) + decoded_first(raw, escapes, also) if path.suffix == ".py" else (False, None, None) ) read_before = None if taken: @@ -1554,11 +1555,14 @@ def script_module(path: Path, name: str) -> Path | None: content = text.replace("\r\n", "\n").replace("\r", "\n") hits: list[Path] = [] if path.suffix == ".py": + # A crafted coding line: what `python file.py` reads, where an import reads the first. + others = [other.replace("\r\n", "\n").replace("\r", "\n") for other in also] # A file with no import in it can't lead anywhere, so it doesn't use up the parse total # before the files that do; a file the total cuts says so. - if "import" not in content: + if "import" not in content and not any("import" in other for other in others): continue readings = [content] if read_before is None else [content, read_before] + readings += others # Each reading is charged as it's parsed: a file parsed twice and charged once let a # crafted pair cost the walk twice its total. results = [_python_imports(reading, parse_limit, parse_budget) for reading in readings] diff --git a/src/fi/alk/harness/instrumentation_evidence.py b/src/fi/alk/harness/instrumentation_evidence.py index ba0e2c3ce..da41ac3e7 100644 --- a/src/fi/alk/harness/instrumentation_evidence.py +++ b/src/fi/alk/harness/instrumentation_evidence.py @@ -62,7 +62,7 @@ def _python_imports(root: Path, reached: Iterable[Path] = ()) -> set[str]: imported: set[str] = set() local: set[str] = set() budget = ParseBudget() - escapes = EscapeBudget() + escapes = EscapeBudget(reader="instrumentation scan") for directory, subdirectories, files in os.walk(root): if Path(directory) != root and any( name.endswith(".dist-info") for name in subdirectories @@ -95,11 +95,13 @@ def _python_imports(root: Path, reached: Iterable[Path] = ()) -> set[str]: def _read_imports( path: Path, budget: ParseBudget, escapes: EscapeBudget, imported: set[str] ) -> None: - text = read_module(path, needles=_NAMES, escapes=escapes) - if text is None or not budget.spend(text): + text = read_module(path, needles=_NAMES, escapes=escapes, also=(also := [])) + if text is None: return - if (tree := parsed(text)) is not None: - imported.update(_imports(tree)) + # An import and `python file.py` can read a crafted file two ways: each reading on its own. + for reading in (text, *also): + if budget.spend(reading) and (tree := parsed(reading)) is not None: + imported.update(_imports(tree)) def _imports(tree: ast.AST) -> set[str]: diff --git a/src/fi/alk/harness/python_source.py b/src/fi/alk/harness/python_source.py index 06de87d4b..57fd362d2 100644 --- a/src/fi/alk/harness/python_source.py +++ b/src/fi/alk/harness/python_source.py @@ -12,12 +12,15 @@ import codecs import functools import io +import logging import re import tokenize import unicodedata from collections.abc import Iterable from pathlib import Path +logger = logging.getLogger(__name__) + # The worst crafted file this size parses in about 0.3 s on 3.12; an honest one in about 0.03 s. LARGEST_PARSED_MODULE = 128 * 1024 # What one scan parses in all: two crafted files' worth, about 0.55 s, and far more than an honest @@ -137,6 +140,9 @@ ) # Refused by name in _coding: never decoded, whatever the total. _NEVER_DECODED_FIRST = frozenset({"idna", "punycode", "undefined"}) +# The longest first lines, up to the coding line's end, decoded to ask whether they read as +# themselves: an honest coding line is far shorter, and a longer one is read both ways, charged. +_LONGEST_HEAD_CHECKED = 4096 # Python reads a lone CR as a line end before it decodes, so hz's `~` and a line end join. _CR_TO_LF = bytes.maketrans(b"\r", b"\n") _NON_ASCII_RUN = re.compile(r"[^\x00-\x7f]{2,}") @@ -160,15 +166,32 @@ def spend(self, text: str) -> bool: class EscapeBudget: """What is left for one scan to decode before searching, in bytes: every file read that way (a - text codec not proven safe to search as bytes, unicode-escape among them) is charged here.""" + text codec not proven safe to search as bytes, unicode-escape among them) is charged here, for + each decode. ``reader`` names the scan in the WARNING its total running out gives.""" - def __init__(self, total: int | None = None) -> None: - self.left = LARGEST_ESCAPED_TOTAL if total is None else total + def __init__(self, total: int | None = None, reader: str = "scan") -> None: + self.total = LARGEST_ESCAPED_TOTAL if total is None else total + self.left = self.total + self.reader = reader + self.decoded = 0 def spend(self, raw: bytes) -> bool: - """Charge ``raw``, and say whether the scan may still decode it.""" + """Charge ``raw``, and say whether the scan may still decode it. The first time the total + runs out, a WARNING says so, in counts alone: a file past it isn't decoded as Python + does, so a name its codec spells in other bytes can be missed.""" self.left -= len(raw) - return self.left >= 0 + if self.left >= 0: + self.decoded += 1 + return True + if self.left + len(raw) >= 0: + logger.warning( + "%s: decoding Python as Python does stopped at its total of %d bytes, after %d " + "decodes; a name the rest spell in their codec can be missed", + self.reader, + self.total, + self.decoded, + ) + return False class TokenBudget(ParseBudget): @@ -204,6 +227,7 @@ def read_module( needles: Iterable[bytes] = (), undecoded: list[bytes] | None = None, escapes: EscapeBudget | None = None, + also: list[str] | None = None, ) -> str | None: """The file as text, or None: not a regular file, larger than ``limit`` bytes, not readable in the encoding its coding line or byte-order mark names (UTF-8 without one), or, given @@ -211,10 +235,13 @@ def read_module( Given ``escapes``, a file in a codec not proven safe to search as bytes (unicode-escape, utf-7, hz and others) is decoded as Python does while the scan's total lasts, and a needle - counts in what it decodes to, or in its bytes when that isn't ASCII. Given ``undecoded``, a file that names a needle - but isn't read because its coding line names a codec refused by name (or unicode-escape past - the total) is put there as bytes: Python still runs it (plain ASCII under idna as it would - without the line), so a scan looking for names can still find them. + counts in what it decodes to, or in its bytes when that isn't ASCII. Given ``undecoded``, a + file that names a needle but isn't read because its coding line names a codec refused by name + (or unicode-escape past the total) is put there as bytes: Python still runs it (plain ASCII + under idna as it would without the line), so a scan looking for names can still find them. + Given ``also``, a file Python reads two ways (``_decoded_readings``) has its second text put + there, for the caller to read on its own: joined to the first, one text's syntax error would + sink the other. """ try: # A link can name a device or a pipe, which reads without end. @@ -227,18 +254,22 @@ def read_module( if len(raw) > limit: return None coding = _coding(raw) - taken, text = _decoded_first(raw, coding, escapes) + taken, text, other = _decoded_readings(raw, coding, escapes) if taken: if text is None: return None + readings = [text] if other is None else [text, other] # The bytes too, as for any other file: the parser folds a fullwidth name the text spells # (`playwright`) to the needle, and a needle in the bytes is what lets such a file in. On # ASCII text nothing folds, so whatever a scan can find is in the text itself. if needles and not any( - needle.decode("ascii") in text or not text.isascii() and needle in raw + needle.decode("ascii") in reading or not reading.isascii() and needle in raw + for reading in readings for needle in needles ): return None + if other is not None and also is not None: + also.append(other) return text # A literal match: a name spelled in other characters that Python folds to the same one, # such as a fullwidth `playwright`, isn't found. @@ -273,14 +304,16 @@ def source_encoding(raw: bytes) -> str | None: def decoded_first( - raw: bytes, escapes: EscapeBudget | None + raw: bytes, escapes: EscapeBudget | None, also: list[str] | None = None ) -> tuple[bool, str | None, str | None]: """``raw`` read as ``read_module`` reads a file before searching it, for a reader that searches no needles: whether it was decoded first, its text (None when Python refuses it), and the encoding ``source_encoding`` gives, to read a file that wasn't. The coding line is - read once for both.""" + read once for both. ``also`` is as ``read_module``'s.""" coding = _coding(raw) - taken, text = _decoded_first(raw, coding, escapes) + taken, text, other = _decoded_readings(raw, coding, escapes) + if other is not None and also is not None: + also.append(other) encoding = None if coding is None or coding[1].name in _UNREAD_CODECS else coding[0] return taken, text, encoding @@ -289,14 +322,29 @@ def _decoded_first( raw: bytes, coding: tuple[str, codecs.CodecInfo] | None, escapes: EscapeBudget | None ) -> tuple[bool, str | None]: """Whether ``raw``, in the codec ``coding`` names, is decoded before it's searched, and its - text as Python reads it: None when Python refuses it. + text as Python reads it: None when Python refuses it. The first of ``_decoded_readings``.""" + taken, text, _ = _decoded_readings(raw, coding, escapes) + return taken, text + + +def _decoded_readings( + raw: bytes, coding: tuple[str, codecs.CodecInfo] | None, escapes: EscapeBudget | None +) -> tuple[bool, str | None, str | None]: + """Whether ``raw``, in the codec ``coding`` names, is decoded before it's searched, its text + as Python reads it (None when Python refuses it), and a second text, or None. A codec not proven safe to search as bytes can spell a name in other bytes, so its file is decoded while ``escapes`` lasts. Past it, or with no total, it isn't taken, and the caller reads it as any other file. + + An import decodes the whole file, and `python file.py` the lines after the coding line, the + ones up to it read as they are. The two agree when those lines decode to themselves. When + they don't (`# coding: utf-7 +`, hz's `~{`, an iso2022 escape, a backslash at the end), the + script's reading is the text when the whole file doesn't decode, and the second text when + both do and differ. Each decode is charged. """ if coding is None or escapes is None: - return False, None + return False, None, None encoding, info = coding if ( info.name in _RAW_SAFE_CODECS @@ -304,9 +352,41 @@ def _decoded_first( # Not a text encoding, such as rot13 or base64: bytes.decode refuses it before reading. or not getattr(info, "_is_text_encoding", True) ): - return False, None + return False, None, None + if not escapes.spend(raw): + return False, None, None + if not _reads_its_cookie(encoding): + return True, _read_as_a_script(raw, encoding), None + whole = _read_whole(raw, encoding) + # An import also ends lines before it decodes, and a script after, so any CR can split the + # two (a backslash or iso2022_jp_2's single shift before one joins the next line in one only). + if b"\r" not in raw and _head_reads_as_itself(raw, encoding): + return True, whole, None if not escapes.spend(raw): - return False, None + # Past the total, a file the whole decode refuses is read as any file past it is. + return (True, whole, None) if whole is not None else (False, None, None) + script = _read_as_a_script(raw, encoding) + if whole is None: + return True, script, None + return True, whole, None if script == whole else script + + +def _head_reads_as_itself(raw: bytes, encoding: str) -> bool: + """Whether the lines of ``raw`` up to and including its coding line decode to themselves in + ``encoding``, leaving its decoder as a fresh one would be: then an import and a script read + the file alike.""" + head = raw[: _cookie_line_end(raw)] + if len(head) > _LONGEST_HEAD_CHECKED: + # Not decoded uncharged: read both ways, as a crafted line is, and charged for it. + return False + try: + return head.decode(encoding) == head.decode("latin-1") + except (UnicodeError, LookupError): + return False + + +def _read_whole(raw: bytes, encoding: str) -> str | None: + """``raw`` as an import reads it, decoded whole, or None when Python refuses it.""" if b"\r" in raw: # Python ends lines before it decodes, so a backslash or hz's `~` before a CRLF joins # the lines it ends. @@ -318,10 +398,71 @@ def _decoded_first( text = raw.decode(encoding) except (UnicodeError, LookupError): # LookupError only from a codec registered at run time: Python's text codecs don't. - return True, None + return None # Python reads nothing past a NUL its decode makes at a line's start, and refuses the file # for one elsewhere: so a name after it is never run. - return True, text.partition("\0")[0] + return text.partition("\0")[0] + + +@functools.lru_cache(maxsize=64) +def _reads_its_cookie(encoding: str) -> bool: + """Whether ``encoding`` decodes an ASCII coding line to that line. Every codec compile() and + import accept does. cp037 and the other EBCDIC codecs don't, nor UTF-16 and UTF-32, whose + characters are wider than a byte, and only `python file.py` runs a file in one of those + (``_read_as_a_script``).""" + cookie = f"# -*- coding: {encoding} -*-\n" + try: + return cookie.encode("ascii").decode(encoding) == cookie + except (UnicodeError, LookupError): + return False + + +def _read_as_a_script(raw: bytes, encoding: str) -> str | None: + """``raw`` as `python file.py` reads it, or None when Python refuses it: what an import reads + too unless the coding line, or the codec, doesn't decode the lines up to it to themselves. + + Python reads the lines up to and including the cookie's as they are. It then opens the file + again in the codec one byte before that line's end, and throws away the first line it reads + there. In an EBCDIC codec that byte is no line end unless it's a CR, so after an LF the next + line goes with it; in UTF-16 or UTF-32 it starts a character with the bytes after it. The + rest is decoded, and its CRs end lines. A blank line stands for a line of the file thrown + away, so each line keeps its number in the file. + """ + end = _cookie_line_end(raw) + head = raw[:end].replace(b"\r\n", b"\n").translate(_CR_TO_LF) + try: + rest = raw[end - 1 :].decode(encoding) + except (UnicodeError, LookupError): + return None + ends = [at for at in (rest.find("\n"), rest.find("\r")) if at != -1] + thrown = len(rest) + if ends: + thrown = min(ends) + (2 if rest[min(ends) : min(ends) + 2] == "\r\n" else 1) + body = rest[thrown:].replace("\r\n", "\n").replace("\r", "\n") + text = head.decode("utf-8", "replace") + ("\n" if thrown > 1 else "") + body + if not text.endswith("\n"): + text += "\n" + # As for any decoded file: nothing past a NUL is read. + return text.partition("\0")[0] + + +def _cookie_line_end(raw: bytes) -> int: + """Where the line holding the coding line's cookie ends in ``raw``, past its line end: the + first line's, or the second's when the first names no cookie.""" + end = _line_end(raw, 0) + part = _cookie_part(raw[:end].rstrip(b"\r\n")) + if part is not None and part.startswith(b"#coding"): + return end + return _line_end(raw, end) + + +def _line_end(raw: bytes, start: int) -> int: + """Where the line at ``start`` ends, past its LF, CRLF or lone CR, as Python ends a line.""" + ends = [at for at in (raw.find(b"\n", start), raw.find(b"\r", start)) if at != -1] + if not ends: + return len(raw) + end = min(ends) + return end + (2 if raw[end : end + 2] == b"\r\n" else 1) def _first_lines(raw: bytes) -> bytes: @@ -407,7 +548,12 @@ def _coding(raw: bytes) -> tuple[str, codecs.CodecInfo] | None: if None in parts: # Read as a file in a refused codec is: searched raw, never decoded. return "undefined", codecs.lookup("undefined") - cut = b"\n".join(parts) + # From 3.14 detect_encoding decodes the lines it's handed in their codec, as Python decodes + # the real ones, line ends included: each cut line keeps its real length mod 4, so a UTF-16 + # or UTF-32 line decodes in the cut as it does whole. + # A blank line padded with spaces is still blank. + parts = [part + b" " * ((len(line) - len(part)) % 4) for part, line in zip(parts, lines_read)] + cut = b"\n".join(parts) + b"\n" lines = io.BytesIO(head[:bom] + cut).readline try: encoding, _ = tokenize.detect_encoding(lines) diff --git a/src/fi/alk/harness/source_tool_evidence.py b/src/fi/alk/harness/source_tool_evidence.py index 394fd1b45..7d1cf88ce 100644 --- a/src/fi/alk/harness/source_tool_evidence.py +++ b/src/fi/alk/harness/source_tool_evidence.py @@ -49,7 +49,7 @@ def tool_evidence_problems( commented: dict[Path, list[str] | None] = {} budget = ParseBudget() tokens = TokenBudget() - escapes = EscapeBudget() + escapes = EscapeBudget(reader="tool check") for entry in contract.tool_entrypoints: if entry.mode not in {"import", "construct"} or not entry.module: continue diff --git a/tests/harness/test_python_source_codecs.py b/tests/harness/test_python_source_codecs.py index e096cfa4c..2506cccc4 100644 --- a/tests/harness/test_python_source_codecs.py +++ b/tests/harness/test_python_source_codecs.py @@ -1,6 +1,7 @@ """A Python file whose codec can spell a name in other bytes is read as Python reads it. -Each row that says Python runs a name checks it with ``compile``, which is what an import runs. +Each row that says Python runs a name checks it with ``compile``, which is what an import runs, +or, in a codec only `python file.py` runs a file in, by running the file with a stub on the path. """ from __future__ import annotations @@ -10,9 +11,14 @@ import collections import encodings import encodings.aliases +import io +import logging +import os import pkgutil import string +import subprocess import sys +import tokenize import types from pathlib import Path @@ -31,6 +37,7 @@ EscapeBudget, _coding, _decoded_first, + _decoded_readings, read_module, ) @@ -38,6 +45,24 @@ _FROM_312 = pytest.mark.skipif( sys.version_info < (3, 12), reason="Python reads a decoded CR this way from 3.12" ) + + +def _keeps_a_decoded_cr() -> bool: + """Whether Python keeps a decoded CR inside a line: 3.13.13 and 3.14.7 end the line there.""" + try: + compile( + b'# coding: unicode-escape\nx = "\\r"\n', "", "exec", dont_inherit=True + ) + except SyntaxError: + return False + return True + + +# The rows whose premise is a decoded CR Python keeps inside a line: where it ends the line +# there instead, it refuses the file, and the row has nothing left to show. +_KEEPS_A_DECODED_CR = pytest.mark.skipif( + not _keeps_a_decoded_cr(), reason="this Python ends a line at a decoded CR" +) _ISO2022 = ( "iso2022_jp", "iso2022_jp_1", @@ -176,6 +201,378 @@ def test_a_browser_import_spelled_in_a_codec_that_can_spell_one_is_found( assert browser_tools_found(tmp_path) == [name.replace("_", "-")] +# The EBCDIC codecs, each by a name 3.14 can decode its coding line in (it refuses `cp424`, +# whose `p` cp424 lacks). compile() and import decode the coding line too, and in these it +# decodes to other characters, so only `python file.py` runs a file in one. +_EBCDIC = { + "cp037": "cp037", + "cp1026": "cp1026", + "cp1140": "cp1140", + "cp273": "cp273", + "cp424": "ibm424", + "cp500": "cp500", + "cp875": "cp875", +} + + +def _script_imports(folder: Path, raw: bytes) -> bool: + """Whether `python main.py`, for a main.py in ``folder`` holding ``raw``, imports + playwright: a stub on the path says so.""" + stub = folder.parent / f"{folder.name}-stub" + _write(stub, b"print('stub imported')\n", "playwright.py") + _write(folder, raw, "main.py") + run = subprocess.run( + [sys.executable, "-s", "main.py"], + cwd=folder, + env={**os.environ, "PYTHONPATH": str(stub), "PYTHONDONTWRITEBYTECODE": "1"}, + capture_output=True, + timeout=60, + ) + return run.returncode == 0 and b"stub imported" in run.stdout + + +@pytest.mark.parametrize( + "raw, lines", + [ + # Python throws away the line after the coding line; a blank line stands for it. + *( + pytest.param( + _in(name, "x = 0\nimport playwright\n".encode(codec)), [3], id=codec + ) + for codec, name in sorted(_EBCDIC.items()) + ), + # A NUL makes the byte before the coding line's end an LF; nothing else is thrown away. + pytest.param( + _in("utf_16_le", b"\0" + "x = 0\nimport playwright\n".encode("utf_16_le")), + [2, 3], + id="utf-16-le", + ), + # Here that byte starts a character with the NUL, which the LF after it ends. + pytest.param( + _in("utf_16_be", b"\0" + "\nimport playwright\n".encode("utf_16_be")), + [3], + id="utf-16-be", + ), + ], +) +def test_a_browser_import_in_a_codec_only_a_script_runs_is_found(tmp_path, raw, lines): + folder = tmp_path / "agent" + assert _script_imports(folder, raw) + assert browser_tools_found(folder) == ["playwright"] + budget = EscapeBudget() + text = read_module(folder / "main.py", escapes=budget) + assert budget.left == python_source.LARGEST_ESCAPED_TOTAL - len(raw) + # Each line keeps its number in the file. + assert [node.lineno for node in ast.parse(text).body] == lines + + +@pytest.mark.parametrize( + "head", + [ + "# coding: utf_16_le\n", + "# coding: utf_16_le\r\n", + "# -*- coding: utf_16_le -*-\n", + "#!x\n# coding: utf_16_le\n", + # A blank first line of one space: padded too, it keeps the pair's length even. + " \n# coding: utf_16_le\n", + # An odd number of bytes, which 3.14 can't decode in UTF-16: it refuses the file. + "# coding: utf_16_le\n", + ], +) +def test_a_utf_16_coding_line_of_any_length_is_read(tmp_path, head): + raw = head.encode() + b"\0" + "x = 0\nimport playwright\n".encode("utf_16_le") + folder = tmp_path / "agent" + if not _script_imports(folder, raw): + pytest.skip("this Python refuses the coding line in UTF-16") + assert browser_tools_found(folder) == ["playwright"] + + +@pytest.mark.parametrize( + "raw", + [ + b"# coding: utf_16_le\n", + b"# coding: utf_16_le\r\n", + b"# -*- coding: utf_16_le -*- \n", + b"#!xy\n# coding: utf_16_le\n", + b"\n# coding: utf_16_le\n", + b" \n# coding: utf_16_le\n", + b"\t \n# coding: utf_16_le\n", + ], +) +def test_detect_encoding_is_handed_lines_as_long_as_the_real_ones_mod_4( + monkeypatch, raw +): + # From 3.14 it decodes them in their codec, as Python decodes the real ones: a UTF-16 or + # UTF-32 line of the wrong length would be refused. + handed: list[bytes] = [] + real = python_source.tokenize.detect_encoding + + def spied(readline): + def line(): + got = readline() + handed.append(got) + return got + + return real(line) + + monkeypatch.setattr(python_source.tokenize, "detect_encoding", spied) + assert _coding(raw + b"\0x\0") is not None + # Python ends each with an LF, a CRLF made one. + real = [len(line) + 1 for line in raw.replace(b"\r\n", b"\n").split(b"\n")[:-1]] + assert [len(line) % 4 for line in handed] == [length % 4 for length in real][ + : len(handed) + ] + + +def test_a_long_first_line_is_read_both_ways_and_charged_for_both(): + # Asking whether a coding line this long reads as itself would decode it uncharged. + raw = ( + b"# coding: utf-7 " + + b"x" * python_source._LONGEST_HEAD_CHECKED + + b"\nimport os\n" + ) + budget = EscapeBudget() + assert _decoded_readings(raw, _coding(raw), budget) == ( + True, + raw.decode("utf-7"), + None, + ) + assert budget.left == python_source.LARGEST_ESCAPED_TOTAL - 2 * len(raw) + + +@pytest.mark.parametrize( + "head, body, imports", + [ + # Python opens the file again one byte before the coding line's end and throws away + # the line it reads there: the LF, in cp037 no line end, and the line after it. + pytest.param("# coding: cp037\n", "import playwright\n", False, id="lf"), + pytest.param("# coding: cp037\r\n", "import playwright\n", False, id="crlf"), + pytest.param( + "#!/usr/bin/env python\n# coding: cp037\n", + "x = 0\nimport playwright\n", + True, + id="second-line", + ), + # The CR is a line end in cp037 too, so nothing after it is thrown away. + pytest.param("# coding: cp037\r", "import playwright\n", True, id="cr"), + pytest.param( + "# coding: cp037\r", "\nimport playwright\n", True, id="cr-then-lf" + ), + # Decoded, a CR ends a line, as Python reads a file it decodes as it goes. + pytest.param( + "# coding: cp037\n", + "x = 0\nimport os\rimport playwright\n", + True, + id="body-cr", + ), + # The coding line on the second line: its own end is the byte decoded first. A `%`, + # cp037's LF, on it would end the thrown line there were the first line's end taken. + pytest.param( + "#!x\n# coding: cp037 %\n", + "x = 0\nimport playwright\n", + True, + id="second-line-%", + ), + pytest.param( + "#!x\r# coding: cp037 %\n", + "x = 0\nimport playwright\n", + True, + id="second-line-%-after-cr", + ), + # A CRLF is one line end: the byte decoded first is its LF, not its CR. + pytest.param( + "# coding: cp037\r\n", + "x = 0\nimport playwright\n", + True, + id="crlf-x-first", + ), + ], +) +def test_a_script_only_codec_reads_as_python_runs_the_script( + tmp_path, head, body, imports +): + raw = head.encode() + body.encode("cp037") + folder = tmp_path / "agent" + assert _script_imports(folder, raw) is imports + assert browser_tools_found(folder) == (["playwright"] if imports else []) + + +def test_an_ascii_compatible_codec_reads_as_before(): + raw = b"# coding: utf-7\nimport os\nx = '+AA0-'\n" + budget = EscapeBudget() + assert _decoded_readings(raw, _coding(raw), budget) == ( + True, + raw.replace(b"\r\n", b"\n").decode("utf-7"), + None, + ) + assert budget.left == python_source.LARGEST_ESCAPED_TOTAL - len(raw) + + +def test_a_coding_line_that_reads_as_another_gives_both_readings(): + # An import reads the coding line's escape, and the decoded CR in the string; a script + # keeps the escape as written, and ends the string's line at the CR. + raw = b"# coding: utf-7 +AHA-\r\nimport os\r\nx = '+AA0-'\r\n" + budget = EscapeBudget() + assert _decoded_readings(raw, _coding(raw), budget) == ( + True, + raw.replace(b"\r\n", b"\n").decode("utf-7"), + "# coding: utf-7 +AHA-\nimport os\nx = '\n'\n", + ) + # Each decode is charged. + assert budget.left == python_source.LARGEST_ESCAPED_TOTAL - 2 * len(raw) + + +def _compiled_imports(raw: bytes, name: str) -> bool: + """Whether importing the file, which compile() stands for, imports ``name``.""" + try: + return name in _names_python_runs(raw) + except (SyntaxError, ValueError): + return False + + +# Coding lines in a codec that reads its own name as itself, each leaving its decoder in another +# state: an import decodes the rest in that state, `python file.py` with a fresh decoder. +_CRAFTED_CODING_LINES = [ + pytest.param(b"# coding: utf-7 +\n", id="utf-7-open-shift"), + pytest.param(b"# coding: utf-7 +\r\n", id="utf-7-open-shift-crlf"), + pytest.param(b"#!/usr/bin/env python\n# coding: utf-7 +\n", id="utf-7-second-line"), + pytest.param(b"# coding: hz ~{\n", id="hz-gb-mode"), + pytest.param(b"# coding: iso2022_jp \x1b$B\n", id="iso2022_jp-escape"), + pytest.param(b"# coding: iso2022_jp_2 \x1b$A\n", id="iso2022_jp_2-escape"), + pytest.param(b"# coding: unicode-escape \\\n", id="unicode-escape-joins-the-line"), + pytest.param(b"# coding: raw-unicode-escape \\u\n", id="raw-unicode-escape-open"), + # An import ends lines before it decodes, so the backslash joins the next line to the + # comment; a script decodes first, and a backslash before a CR joins nothing. + pytest.param( + b"# coding: unicode-escape \\\r\n", id="unicode-escape-backslash-crlf" + ), + pytest.param(b"# coding: unicode-escape \\\r", id="unicode-escape-backslash-cr"), + pytest.param( + b"# coding: unicode-escape\r\nx = 1 # \\\r\n", + id="unicode-escape-backslash-crlf-body", + ), + # A single shift reads the byte after it: an import's LF, a script's CR. + pytest.param( + b"# coding: iso2022_jp_2\r\n# \x1b.A\x1bN\r\n", id="iso2022_jp_2-single-shift-crlf" + ), +] + + +@pytest.mark.parametrize("head", _CRAFTED_CODING_LINES) +@pytest.mark.parametrize( + "body", + [ + pytest.param(b"import playwright\n", id="plain"), + # Joined to the coding line, the import's line opens a string nothing closes. + pytest.param(b'import playwright; x = """\n"""\n', id="string"), + ], +) +def test_an_import_after_a_crafted_coding_line_is_found(tmp_path, head, body): + raw = head + body + folder = tmp_path / "agent" + if not (_script_imports(folder, raw) or _compiled_imports(raw, "playwright")): + pytest.skip("this Python runs neither reading") + assert browser_tools_found(folder) == ["playwright"] + + +@pytest.mark.parametrize( + "raw, script", + [ + # Only an import reads the line end the escape spells; a script reads a comment. + pytest.param( + b"# coding: utf-7 +AAo-import playwright\nx = 1\n", False, id="import-only" + ), + # Only a script reads the import: an import joins it to the coding line. + pytest.param( + b"# coding: unicode-escape \\\nimport playwright\n", True, id="script-only" + ), + # An import decodes the `\0` to a NUL and reads no further: only the second text names + # playwright, or says `import`. + pytest.param( + b"# coding: unicode-escape \\0\nimport playwright\n", True, id="nul-head" + ), + # After this designation and a shift-out, the next line decodes after a CR and not + # after an LF: an import, which makes the CR an LF first, refuses the file. + pytest.param( + b"# coding: iso2022_kr\n# \x1b$(C\x0e\rimport playwright\n", + True, + id="iso2022_kr-shift-out-cr", + ), + ], +) +def test_each_reading_of_a_crafted_coding_line_is_searched(tmp_path, raw, script): + folder = tmp_path / "agent" + assert _script_imports(folder, raw) is script + assert _compiled_imports(raw, "playwright") is not script + assert browser_tools_found(folder) == ["playwright"] + + +# Only a script reads what follows each: an import joins it to the coding line, or reads no +# further than the NUL the `\0` decodes to, where no `import` is left either. +_SCRIPT_ONLY_HEADS = [ + pytest.param(b"# coding: unicode-escape \\\n", id="joined"), + pytest.param(b"# coding: unicode-escape \\0\n", id="nul"), +] + + +@pytest.mark.parametrize("head", _SCRIPT_ONLY_HEADS) +def test_the_other_readers_search_both_readings(tmp_path, head): + traced = head + b"import traceai_openai\n" + assert "traceai_openai" not in _names_python_runs(traced) + _write(tmp_path / "traced", traced) + assert detect_instrumentation(tmp_path / "traced") == "traceai" + agent = head + b"import common.web\n" + assert "common" not in _names_python_runs(agent) + repo = _shared(tmp_path / "walk", agent) + assert repo / "common" / "web.py" in imported_by_agent(repo, "agents/a") + + +def test_a_browser_marker_only_the_second_reading_holds_is_found(tmp_path): + _write(tmp_path, b'# coding: unicode-escape \\0\nMCP = "@playwright/mcp"\n') + assert browser_tools_found(tmp_path) == ["Playwright MCP"] + + +def _parse_charges(monkeypatch) -> list[str]: + """The texts each ParseBudget, and not a TokenBudget, is charged for.""" + charged: list[str] = [] + real = python_source.ParseBudget.spend + + def spend(self, text): + if type(self) is python_source.ParseBudget: + charged.append(text) + return real(self, text) + + monkeypatch.setattr(python_source.ParseBudget, "spend", spend) + return charged + + +@pytest.mark.parametrize( + "reader, name", + [ + ("browser", b"playwright"), + ("instrumentation", b"traceai_openai"), + ("walk", b"common.web"), + ], +) +def test_each_reading_is_charged_to_the_parse_total( + tmp_path, monkeypatch, reader, name +): + # Each reader searches for its own names: the file must name one to be read at all. + raw = b"# coding: unicode-escape \\0\nimport " + name + b"\n" + _, first, second = _decoded_readings(raw, _coding(raw), EscapeBudget()) + assert second is not None + charged = _parse_charges(monkeypatch) + repo = _shared(tmp_path, raw) + if reader == "browser": + browser_tools_found(repo / "agents" / "a") + elif reader == "instrumentation": + detect_instrumentation(repo / "agents" / "a") + else: + imported_by_agent(repo, "agents/a", parse_budget=python_source.ParseBudget()) + assert first in charged + assert second in charged + + @pytest.mark.parametrize( "raw", [ @@ -376,19 +773,20 @@ def test_a_decoded_cr_reads_as_todays_code_reads_it(tmp_path, raw, name): _ESCAPED + b'x = "\\r"\nimport playwright\n', "playwright", id="cr-in-string", + marks=_KEEPS_A_DECODED_CR, ), # Python ends the comment at the CR and reads on; the parser keeps it in the comment. pytest.param( _UTF7 + b"import os # hi+AA0-, playwright\n", "playwright", id="comment-cr", - marks=_FROM_312, + marks=(_FROM_312, _KEEPS_A_DECODED_CR), ), pytest.param( _ESCAPED + b"x = 1 # hi\\r; import playwright\n", "playwright", id="ue-comment-cr", - marks=_FROM_312, + marks=(_FROM_312, _KEEPS_A_DECODED_CR), ), # Python reads a line a decoded CR starts as blank, inside the block; the parser as # a line at column 0, which ends it. @@ -396,27 +794,27 @@ def test_a_decoded_cr_reads_as_todays_code_reads_it(tmp_path, raw, name): _UTF7 + b"def f():\n a = 1\n+AA0-(1)\n import +AHA-laywright\n", "playwright", id="cr-in-a-block", - marks=_FROM_312, + marks=(_FROM_312, _KEEPS_A_DECODED_CR), ), pytest.param( _UTF7 + b"try:\n import +AHA-laywright\n+AA0-[0]\nexcept ImportError:\n pass\n", "playwright", id="cr-in-a-try", - marks=_FROM_312, + marks=(_FROM_312, _KEEPS_A_DECODED_CR), ), pytest.param( _UTF7 + b"try:\n import +AHQ-raceai_openai\n+AA0-[0]\nexcept ImportError:\n pass\n", "traceai_openai", id="cr-in-a-try-traceai", - marks=_FROM_312, + marks=(_FROM_312, _KEEPS_A_DECODED_CR), ), pytest.param( _UTF7 + b"+AA0-\\+AA0-\n import playwright\n", "playwright", id="cr-then-continued", - marks=_FROM_312, + marks=(_FROM_312, _KEEPS_A_DECODED_CR), ), # More failed cookie places than _coding reads before the cookie: read as a refused # codec's file, searched as bytes. @@ -475,6 +873,68 @@ def test_past_the_total_a_file_reads_as_today(tmp_path, monkeypatch): assert browser_tools_found(tmp_path / "spelled") == [] +def _tool_check(folder: Path) -> object: + """The tool check over three named modules, m0 to m2, each a file in ``folder``.""" + from fi.alk.harness.contract import AgentContract + from fi.alk.harness.source_tool_evidence import tool_evidence_problems + + entries = [ + {"tool": f"t{n}", "mode": "import", "module": f"m{n}", "callable": "f"} + for n in range(3) + ] + contract = AgentContract.model_validate({"agent": "a", "tool_entrypoints": entries}) + return tool_evidence_problems(contract, folder) + + +@pytest.mark.parametrize( + "reader, scan, names", + [ + ( + "browser scan", + browser_tools_found, + ["a_pad00.py", "a_pad01.py", "a_pad02.py"], + ), + ("instrumentation scan", detect_instrumentation, ["a0.py", "a1.py", "a2.py"]), + ("tool check", _tool_check, ["m0.py", "m1.py", "m2.py"]), + ( + "import walk", + lambda folder: imported_by_agent( + folder.parent.parent.resolve(), "agents/a" + ), + ["a0.py", "a1.py", "a2.py"], + ), + ], +) +def test_the_total_running_out_is_logged_once_in_counts( + tmp_path, monkeypatch, caplog, reader, scan, names +): + pad = _UTF7 + b"x = 1\n" * 10 + monkeypatch.setattr(python_source, "LARGEST_ESCAPED_TOTAL", 2 * len(pad)) + folder = tmp_path / "agents" / "a" + # Read in name order: two pads fill the total, and the rest are past it. + for name in names: + _write(folder, pad, name) + _write(folder, _UTF7 + b"import +AHA-laywright\n", "z_main.py") + with caplog.at_level(logging.WARNING, logger=python_source.__name__): + scan(folder) + logged = [r for r in caplog.records if r.name == python_source.__name__] + assert [(r.levelno, r.args) for r in logged] == [ + (logging.WARNING, (reader, 2 * len(pad), 2)) + ] + # Counts alone: never a path or the code. Past the total some readers skip a file, and + # others search its bytes: either way a name can be missed. + message = logged[0].getMessage() + assert message.endswith("a name the rest spell in their codec can be missed") + assert all(part not in message for part in ("a_pad", "z_main", "laywright", "tmp")) + + +def test_a_total_that_lasts_logs_nothing(caplog): + budget = EscapeBudget(10) + with caplog.at_level(logging.WARNING, logger=python_source.__name__): + assert budget.spend(b"x" * 10) + assert caplog.records == [] + + @pytest.mark.parametrize( "codec, word", [ @@ -508,6 +968,11 @@ def test_without_a_total_every_codec_reads_as_today(tmp_path): @pytest.mark.parametrize("codec", ["rot13", "base64"]) def test_a_non_text_codec_is_never_charged(tmp_path, codec): raw = _in(codec, b"import playwright\n") + try: + tokenize.detect_encoding(io.BytesIO(raw).readline) + except (SyntaxError, LookupError): + # From 3.14 detect_encoding decodes the coding line in its codec, which refuses it. + pytest.skip(f"this Python refuses a {codec} coding line before reading it") assert _coding(raw) is not None budget = EscapeBudget() assert read_module(_write(tmp_path, raw), escapes=budget) is None @@ -584,15 +1049,29 @@ def test_a_failed_decode_is_still_charged(): @pytest.mark.parametrize( "agent", [ - pytest.param(_ESCAPED + b"import common.web # see \\r, x\n", id="comment"), - pytest.param(_ESCAPED + b'SEP = "\\r"\nimport common.web\n', id="string"), - pytest.param(_ESCAPED + b"CR = b'\\r'\nimport common.web\n", id="bytes"), + pytest.param( + _ESCAPED + b"import common.web # see \\r, x\n", + id="comment", + marks=_KEEPS_A_DECODED_CR, + ), + pytest.param( + _ESCAPED + b'SEP = "\\r"\nimport common.web\n', + id="string", + marks=_KEEPS_A_DECODED_CR, + ), + pytest.param( + _ESCAPED + b"CR = b'\\r'\nimport common.web\n", + id="bytes", + marks=_KEEPS_A_DECODED_CR, + ), + # A triple-quoted string holds a line end, so Python runs this one either way. pytest.param( _ESCAPED + b'"""\\r"""\nimport \\x63ommon.web\n', id="and-an-escaped-import" ), pytest.param( _ESCAPED + b'SEP = "\\r" # caf\xe9\nimport common.web\n', id="and-a-latin-1-byte", + marks=_KEEPS_A_DECODED_CR, ), ], ) @@ -651,15 +1130,15 @@ def test_the_walks_total_is_spent_across_one_walk(tmp_path, monkeypatch): ) is followed -def _punycode_decodes(monkeypatch) -> list[int]: +def _punycode_decodes(monkeypatch) -> list[bytes]: import encodings.punycode - decoded: list[int] = [] + decoded: list[bytes] = [] real = encodings.punycode.punycode_decode monkeypatch.setattr( encodings.punycode, "punycode_decode", - lambda *args: decoded.append(len(args[0])) or real(*args), + lambda *args: decoded.append(bytes(args[0])) or real(*args), ) return decoded @@ -680,7 +1159,11 @@ def test_the_walk_never_decodes_a_refused_codec(tmp_path, monkeypatch): "host.py", ) imported_by_agent(repo, "agents/a") - assert (decoded, labels) == ([], []) + # From 3.14 detect_encoding decodes the coding line it's handed in its codec: _coding hands + # it the line cut to its cookie, padded with spaces and ended with an LF, a few bytes. The + # file's body is never decoded. + assert all(line.rstrip(b" \n") == b"#coding:punycode" for line in decoded) + assert labels == [] def test_the_walk_reads_each_coding_line_once(tmp_path, monkeypatch): @@ -702,6 +1185,27 @@ def test_the_walk_reads_each_coding_line_once(tmp_path, monkeypatch): assert sorted(asked) == sorted(path.read_bytes() for path in python) +def test_the_walk_parses_each_files_own_readings(tmp_path, monkeypatch): + # The second text a crafted coding line gives belongs to its own file: a later file never + # parses it again. + from fi.alk.harness import credentials + + parsed: list[str] = [] + real = credentials._python_imports + monkeypatch.setattr( + credentials, + "_python_imports", + lambda reading, *args: parsed.append(reading) or real(reading, *args), + ) + repo = _shared(tmp_path, b"") + _write(repo / "agents" / "a", b"# coding: unicode-escape \\0\nimport common.web\n", "a.py") + _write(repo / "agents" / "a", b"import os\n", "b.py") + assert repo / "common" / "web.py" in imported_by_agent(repo, "agents/a") + # a.py is read twice, the decoded text cut at its NUL and the file as UTF-8; b.py once. + assert sum("unicode-escape" in reading for reading in parsed) == 2 + assert sum("import os" in reading for reading in parsed) == 1 + + def test_ci_runs_this_file(): workflows = ( Path(__file__).resolve().parents[2] / ".github" / "workflows" / "sdk-smoke.yml" @@ -923,6 +1427,76 @@ def test_the_accepted_codecs_are_exactly_the_three_groups(): assert _accepted_codecs() == _RAW_SAFE_CODECS | _SPELLING_CODECS | {"idna"} +def _codec_names() -> list[str]: + """Every name a coding line may give a codec in.""" + names = set(encodings.aliases.aliases) | set(encodings.aliases.aliases.values()) + names |= {module.name for module in pkgutil.iter_modules(encodings.__path__)} + return sorted(names - {"aliases"}) + + +def _compiles(name: str) -> bool: + try: + compile( + f"# coding: {name}\nx = 1\n".encode(), "", "exec", dont_inherit=True + ) + except (LookupError, SyntaxError, ValueError): + return False + return True + + +def _script_only_codecs(folder: Path) -> dict[str, bytes]: + """Every codec `python file.py` runs an import in whose coding line compile() refuses, by + its codec info's name, with the file it ran. Python decodes such a file from the byte + before the coding line's end and throws the first line it reads away, so the body is tried + after each count of NUL bytes that could make that byte part of a whole character.""" + found: dict[str, bytes] = {} + for name in _codec_names(): + try: + codec = codecs.lookup(name) + if not getattr(codec, "_is_text_encoding", True): + # bytes.decode refuses it, as Python does on a coding line. + continue + encoder = codecs.getincrementalencoder(name)() + # Past its first call an encoder writes no byte-order mark. + encoder.encode("") + body = encoder.encode("\nimport playwright\n", final=True) + unit = len("\n\n".encode(name)) - len("\n".encode(name)) + except (LookupError, UnicodeError): + continue + if codec.name in found or _compiles(name): + continue + for pad in range(unit): + raw = _in(name, b"\0" * pad + body) + if _script_imports(folder, raw): + found[codec.name] = raw + break + return found + + +# What a script runs a file in on 3.12 to 3.14 besides the EBCDIC codecs. Which of these runs +# varies by version: 3.14 also decodes the coding line in its codec, and refuses it when that +# fails. +_WIDE_OR_PUNYCODE = frozenset( + {"utf-16", "utf-16-le", "utf-16-be", "utf-32", "utf-32-le", "punycode"} +) + + +def test_every_codec_a_script_runs_is_read_as_it_runs(tmp_path): + ran = _script_only_codecs(tmp_path / "probe") + # A codec a new Python runs a script in fails here until it's put in one group or the other. + assert set(_EBCDIC) <= set(ran) <= set(_EBCDIC) | _WIDE_OR_PUNYCODE + for codec, raw in sorted(ran.items()): + # punycode is refused by name and searched as bytes, where its letters are as written. + _write(tmp_path / codec, raw) + assert browser_tools_found(tmp_path / codec) == ["playwright"], codec + # Every codec compile() accepts reads as before; the others are read as a script reads them. + for name in _codec_names(): + if _compiles(name): + assert python_source._reads_its_cookie(name), name + for codec in set(ran) - {"punycode"}: + assert not python_source._reads_its_cookie(codec), codec + + # Each holds a needle in bytes its codec decodes to other characters, beside an import of # playwright Python runs: the bytes let the file in, and the text read is Python's. Sorted by # their bytes; two need Python 3.12's reading of a decoded CR. @@ -1351,3 +1925,12 @@ def test_no_decoy_case_regresses(tmp_path, raw): ) assert text is not None assert "playwright" in _imported(ast.parse(text)) + + +def test_a_file_the_whole_decode_refuses_at_the_totals_edge_is_read_as_past_it(tmp_path): + # The total lasts for the first decode and not the second: the file is read as any file + # past the total is, not dropped. + raw = b"# coding: iso2022_kr\n# \x1b$(C\x0e\rimport playwright\n" + path = _write(tmp_path, raw) + text = read_module(path, needles=(b"playwright",), escapes=EscapeBudget(len(raw) + 1)) + assert text is not None and "import playwright" in text diff --git a/tests/harness/test_source_tool_evidence.py b/tests/harness/test_source_tool_evidence.py index f896bc7d2..ea15dad77 100644 --- a/tests/harness/test_source_tool_evidence.py +++ b/tests/harness/test_source_tool_evidence.py @@ -836,6 +836,39 @@ def test_tool_check_decodes_a_unicode_escape_module_as_python_does(tmp_path): assert tool_evidence_problems(_entry_contract(), tmp_path) +def test_tool_check_decodes_a_utf_7_module_as_python_does(tmp_path): + # +AAo- is a line end and +AEA- an "@": Python reads a decorated lookup after the comment, + # where the bytes hold one comment line with the definition in it. + (tmp_path / "agent.py").write_bytes( + b"# coding: utf-7\nfrom agents import tool\n" + b"# def lookup(): an example+AAo-+AEA-tool+AAo-def lookup(): return 1\n" + ) + + assert tool_evidence_problems(_entry_contract(), tmp_path) == [] + + +def test_tool_check_reads_the_text_an_import_reads(tmp_path): + # The runtime imports a tool's module. An import joins the definition to the coding line's + # comment, where `python file.py` would read it as code. + (tmp_path / "agent.py").write_bytes(b"# coding: unicode-escape \\\ndef lookup(): return 1\n") + + problems = tool_evidence_problems(_entry_contract(), tmp_path) + assert [p.split(" — ")[0] for p in problems] == [ + "tool[lookup]:commented-only-entrypoint:agent.lookup" + ] + + +def test_tool_check_finds_a_def_a_utf_7_comment_hides(tmp_path): + # +ACM- is a "#": Python reads the definition as a comment. Read as UTF-8 the line doesn't + # parse, and a file not read gives no problem either. + (tmp_path / "agent.py").write_bytes(b"# coding: utf-7\n+ACM- def lookup(): return 1\n") + + problems = tool_evidence_problems(_entry_contract(), tmp_path) + assert [p.split(" — ")[0] for p in problems] == [ + "tool[lookup]:commented-only-entrypoint:agent.lookup" + ] + + _COMMENTED_METHOD = ( "class Assistant:\n def __init__(self): pass\n" " # async def lookup_weather(self, location): pass\n" From a8445f18065fe735b6e93ba2acd6070fb596eb8b Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Wed, 11 Nov 2026 18:23:24 +0530 Subject: [PATCH 3/4] fix(harness): say a file with a CR is read both ways, and pin the total's edge (TH-9490) - The _decoded_readings and _head_reads_as_itself docstrings say an import and a script read a file alike only when it holds no CR. - A test pins what comes back when the total lasts for the first decode only: the import's reading, decoded, alone. - The walk test's comment names a.py's second reading as the script's. TH-9490 Signed-off-by: khushalsonawat --- src/fi/alk/harness/python_source.py | 9 +++++---- tests/harness/test_python_source_codecs.py | 14 +++++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/fi/alk/harness/python_source.py b/src/fi/alk/harness/python_source.py index 83ebbc503..df8117b79 100644 --- a/src/fi/alk/harness/python_source.py +++ b/src/fi/alk/harness/python_source.py @@ -338,8 +338,9 @@ def _decoded_readings( reads it as any other file. An import decodes the whole file, and `python file.py` the lines after the coding line, the - ones up to it read as they are. The two agree when those lines decode to themselves. When - they don't (`# coding: utf-7 +`, hz's `~{`, an iso2022 escape, a backslash at the end), the + ones up to it read as they are. The two agree when those lines decode to themselves and the + file holds no CR: an import ends lines before it decodes, a script after. When they don't + (`# coding: utf-7 +`, hz's `~{`, an iso2022 escape, a backslash at the end, a CR), the script's reading is the text when the whole file doesn't decode, and the second text when both do and differ. Each decode is charged. """ @@ -373,8 +374,8 @@ def _decoded_readings( def _head_reads_as_itself(raw: bytes, encoding: str) -> bool: """Whether the lines of ``raw`` up to and including its coding line decode to themselves in - ``encoding``, leaving its decoder as a fresh one would be: then an import and a script read - the file alike.""" + ``encoding``, leaving its decoder as a fresh one would be: then, in a file with no CR, an + import and a script read it alike.""" head = raw[: _cookie_line_end(raw)] if len(head) > _LONGEST_HEAD_CHECKED: # Not decoded uncharged: read both ways, as a crafted line is, and charged for it. diff --git a/tests/harness/test_python_source_codecs.py b/tests/harness/test_python_source_codecs.py index 2506cccc4..d29a5f73b 100644 --- a/tests/harness/test_python_source_codecs.py +++ b/tests/harness/test_python_source_codecs.py @@ -408,6 +408,18 @@ def test_an_ascii_compatible_codec_reads_as_before(): assert budget.left == python_source.LARGEST_ESCAPED_TOTAL - len(raw) +def test_a_file_whose_second_reading_is_past_the_total_keeps_the_imports_reading(): + # A CR asks for the script's reading too, and the total lasts for the first decode only: the + # import's reading comes back alone, decoded, the escape on the coding line included. + raw = b"# coding: unicode-escape # \\x41\r\nimport playwright\r\n" + budget = EscapeBudget(len(raw) + 1) + assert _decoded_readings(raw, _coding(raw), budget) == ( + True, + raw.replace(b"\r\n", b"\n").decode("unicode-escape"), + None, + ) + + def test_a_coding_line_that_reads_as_another_gives_both_readings(): # An import reads the coding line's escape, and the decoded CR in the string; a script # keeps the escape as written, and ends the string's line at the CR. @@ -1201,7 +1213,7 @@ def test_the_walk_parses_each_files_own_readings(tmp_path, monkeypatch): _write(repo / "agents" / "a", b"# coding: unicode-escape \\0\nimport common.web\n", "a.py") _write(repo / "agents" / "a", b"import os\n", "b.py") assert repo / "common" / "web.py" in imported_by_agent(repo, "agents/a") - # a.py is read twice, the decoded text cut at its NUL and the file as UTF-8; b.py once. + # a.py is read twice, the decoded text cut at its NUL and the text a script reads; b.py once. assert sum("unicode-escape" in reading for reading in parsed) == 2 assert sum("import os" in reading for reading in parsed) == 1 From bddd26680c38d0cc17f0f4753bc6adb0ccfd4f11 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Thu, 12 Nov 2026 12:46:19 +0530 Subject: [PATCH 4/4] fix(harness): drop the unused _decoded_first, and skip UTF-16 rows this Python refuses (TH-9490) From 3.14, Python refuses some UTF-16 coding lines itself, so no coding is right there: the length test skips those rows rather than asserting one. _decoded_first had no caller left but tests, which now take the first two items of _decoded_readings. A noqa reason moves above its line, to fit 99. Signed-off-by: khushalsonawat --- src/fi/alk/harness/python_source.py | 9 --------- src/fi/alk/harness/seed_contract.py | 3 ++- tests/harness/test_python_source_codecs.py | 13 +++++++------ 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/fi/alk/harness/python_source.py b/src/fi/alk/harness/python_source.py index df8117b79..d1d1ba443 100644 --- a/src/fi/alk/harness/python_source.py +++ b/src/fi/alk/harness/python_source.py @@ -318,15 +318,6 @@ def decoded_first( return taken, text, encoding -def _decoded_first( - raw: bytes, coding: tuple[str, codecs.CodecInfo] | None, escapes: EscapeBudget | None -) -> tuple[bool, str | None]: - """Whether ``raw``, in the codec ``coding`` names, is decoded before it's searched, and its - text as Python reads it: None when Python refuses it. The first of ``_decoded_readings``.""" - taken, text, _ = _decoded_readings(raw, coding, escapes) - return taken, text - - def _decoded_readings( raw: bytes, coding: tuple[str, codecs.CodecInfo] | None, escapes: EscapeBudget | None ) -> tuple[bool, str | None, str | None]: diff --git a/src/fi/alk/harness/seed_contract.py b/src/fi/alk/harness/seed_contract.py index be52bd3ac..3a83ac023 100644 --- a/src/fi/alk/harness/seed_contract.py +++ b/src/fi/alk/harness/seed_contract.py @@ -367,7 +367,8 @@ def holds_seed(contract: Path, seed: str | Path) -> bool: if problem is not None or raw is None: return False return contract.stat().st_size == len(raw) and contract.read_bytes() == raw - except Exception as exc: # noqa: BLE001 - a contract that can't be compared isn't claimed as the seed + # A contract that can't be compared isn't claimed as the seed. + except Exception as exc: # noqa: BLE001 print_trace(exc) return False diff --git a/tests/harness/test_python_source_codecs.py b/tests/harness/test_python_source_codecs.py index d29a5f73b..d1d295d83 100644 --- a/tests/harness/test_python_source_codecs.py +++ b/tests/harness/test_python_source_codecs.py @@ -36,7 +36,6 @@ _RAW_SAFE_CODECS, EscapeBudget, _coding, - _decoded_first, _decoded_readings, read_module, ) @@ -316,7 +315,9 @@ def line(): return real(line) monkeypatch.setattr(python_source.tokenize, "detect_encoding", spied) - assert _coding(raw + b"\0x\0") is not None + if _coding(raw + b"\0x\0") is None: + # From 3.14 Python refuses some of these files itself, so no coding is right there. + pytest.skip("this Python refuses this coding line") # Python ends each with an LF, a CRLF made one. real = [len(line) + 1 for line in raw.replace(b"\r\n", b"\n").split(b"\n")[:-1]] assert [len(line) % 4 for line in handed] == [length % 4 for length in real][ @@ -961,7 +962,7 @@ def test_a_raw_safe_codec_reads_as_today(tmp_path, codec, word): # CRLF and no final LF: decoding first would end both lines with an LF. raw = f"# coding: {codec}\r\nimport playwright\r\nNAME = '{word}'".encode(codec) budget = EscapeBudget() - assert _decoded_first(raw, _coding(raw), budget) == (False, None) + assert _decoded_readings(raw, _coding(raw), budget)[:2] == (False, None) path = _write(tmp_path, raw) assert read_module(path, needles=(b"playwright",), escapes=budget) == raw.decode( codec @@ -971,7 +972,7 @@ def test_a_raw_safe_codec_reads_as_today(tmp_path, codec, word): def test_without_a_total_every_codec_reads_as_today(tmp_path): unended = _UTF7 + b"import +AHA-laywright" - assert _decoded_first(unended, _coding(unended), None) == (False, None) + assert _decoded_readings(unended, _coding(unended), None)[:2] == (False, None) assert read_module(_write(tmp_path / "utf-7", unended)) == unended.decode("utf-7") escaped = _ESCAPED + b"import \\x70laywright\n" assert read_module(_write(tmp_path / "unicode-escape", escaped)) is None @@ -1003,7 +1004,7 @@ def search(name): codecs.register(search) try: coding = ("lookup-failing", codecs.lookup("lookup-failing")) - assert _decoded_first(b"x = 1\n", coding, EscapeBudget()) == (True, None) + assert _decoded_readings(b"x = 1\n", coding, EscapeBudget())[:2] == (True, None) finally: codecs.unregister(search) @@ -1054,7 +1055,7 @@ def test_the_union_is_for_unicode_escape_only(tmp_path): def test_a_failed_decode_is_still_charged(): bad = _UTF7 + b"import os\n# +\xff\n" budget = EscapeBudget() - assert _decoded_first(bad, _coding(bad), budget) == (True, None) + assert _decoded_readings(bad, _coding(bad), budget)[:2] == (True, None) assert budget.left == python_source.LARGEST_ESCAPED_TOTAL - len(bad)