From ca54d72fd9050bfcf8345053b938e8e737fe7669 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 14:55:43 +0530 Subject: [PATCH 01/10] feat(harness): send the model found in the agent's code, and ask once for the prompt's lines (TH-8953) The agent document always sent "model": null, and the prompt was only the model's quote whenever it didn't name the prompt's lines. The description now records the model the agent talks to the person with: provider, name, and the file and line the name is written on. The save keeps it only when that exact name is on that line as a whole name; otherwise it says what to fix. The cited path is resolved within the save's shared lookup budget and the file is read as UTF-8, at most 1 MiB, so a crafted repository can't stall the check. With no source to check against, the model is dropped. build_authored_agent copies provider and name into the document's model. A description without a model saves and builds to the same bytes and setup hash as before, so stored agent versions are unaffected. When the model quotes the prompt without naming its lines, the save asks once for the lines; a second identical save keeps the quote. Signed-off-by: khushalsonawat --- src/fi/alk/harness/contract.py | 26 + .../harness/skills/understand-agent/SKILL.md | 17 +- src/fi/alk/harness/tools.py | 159 +++- src/fi/alk/providers/agent.py | 13 +- tests/harness/test_contract_model.py | 775 ++++++++++++++++++ .../test_providers_agent_authored.py | 83 ++ 6 files changed, 1070 insertions(+), 3 deletions(-) create mode 100644 tests/harness/test_contract_model.py diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index 57932763d..c104594fd 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -208,6 +208,21 @@ class PromptSource(BaseModel): end_line: int +class ModelSource(BaseModel): + """The model the agent talks to the person with, and the line its name is written on.""" + + provider: str = "" + name: str = "" + path: str = "" + line: int = 0 + + @model_validator(mode="before") + @classmethod + def _bare_name(cls, value: Any) -> Any: + # A bare string is the name alone; the save then asks for path and line. + return {"name": value} if isinstance(value, str) else value + + class DataStore(BaseModel): """What the agent's tools read and write, and how to be there instead of it. @@ -1127,6 +1142,8 @@ def _normalize_shapes(cls, payload: Any) -> Any: system_prompt_excerpt: str = "" # When given, the excerpt is replaced by these lines copied from the source as written. system_prompt_from: list[PromptSource] = Field(default_factory=list) + # None is unknown. Kept only when its name was found on the line cited in the source. + model: ModelSource | None = None hard_constraints: list[str] = Field(default_factory=list) tools: list[ToolSpec] = Field(default_factory=list) data_schema: dict[str, Any] = Field(default_factory=dict) @@ -1161,6 +1178,15 @@ def _normalize_shapes(cls, payload: Any) -> Any: amendments: list[str] = Field(default_factory=list) instrumentation: str = "none" + @model_serializer(mode="wrap") + def _omit_absent_model(self, handler: Any) -> dict[str, Any]: + # A contract with no model writes the same bytes as before the field existed, so its + # setup fingerprint is unchanged. + data = handler(self) + if data.get("model") is None: + data.pop("model", None) + return data + def tool_names(self) -> set[str]: return {tool.name for tool in self.tools} diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index 7ddbf297a..d36859c7e 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -112,7 +112,9 @@ Find, in roughly this order: often kept away from the main agent file, so search the whole source for a long instructions string before concluding there are none. Name where the whole prompt is written in `system_prompt_from`, every part it is assembled from: it is copied into the contract as written, - and every later stage decides what the agent should do from it. + and every later stage decides what the agent should do from it. If the prompt is built or + fetched at run time and has no lines to name, quote it in `system_prompt_excerpt`; the save + asks once whether lines can be named. 7. **The modality.** How this agent is run: a voice session, a text conversation, or work it carries out on websites or a computer. This decides how it is later run, so getting it wrong @@ -299,6 +301,19 @@ Find, in roughly this order: agent's own text says it does; a capability stretched from a passing word or a tool name becomes a whole slice of scenarios testing what the agent was never built to do. +16. **The model.** Record `model` only from the agent's own source: the one model the agent talks + to the person with. A voice agent's speech-to-text and text-to-speech models don't count. Its + provider is the company or service as the source names it (openai, anthropic, google, azure), + left out when the source doesn't say. Its name is written exactly and wholly as it appears on + the line you cite (openai/gpt-4o, if that is what is written); its path is relative to the + source root, as for the prompt's lines; its line is the 1-based line the name is written on. A name + or provider is made only of letters, digits and . _ : / @ -, so a name with a space in it + can't be recorded; leave `model` out then. + + Leave `model` out when the name is only set at run time: an environment variable with no + default in the code, or a value fetched from elsewhere. Where the code gives an environment + variable a default, record the default. + ## A repository may not hold one agent What you are pointed at is a directory, not necessarily a single agent. Before reading anything in diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index ad220695a..50e025498 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -11,6 +11,7 @@ import json import os import secrets +import string import sys import threading from pathlib import Path @@ -34,6 +35,7 @@ from .backends import tool, tool_server from .contract import ( + _ALIASES, CALL_DIRECTIONS, AgentContract, Runtime, @@ -236,8 +238,12 @@ def _checked_against_source( try: problems.extend(tool_evidence_problems(contract, source_root, subfolder)) problems.extend(_copy_prompt(contract, source_root)) + problems.extend(_model_problems(contract, source_root)) except PathLookupsSpent as spent: return _problems([*problems, str(spent)], arrived) + else: + # With no source there is no line to find the name on, so the model is unknown. + contract.model = None runtime = contract.runtime if subfolder and source_root is not None: try: @@ -381,6 +387,121 @@ def _copy_prompt(contract: AgentContract, source_root: Path) -> list[str]: return problems +# The platform refuses the whole agent document past this, for the model's name and provider. +_LONGEST_MODEL_FIELD = 255 +# One path as written: charged_path doesn't refuse a long path itself. +_LONGEST_MODEL_PATH = 4096 +# What a model name is made of. A match must have none of these right before or after it, so +# "gpt-4o" is not found inside "gpt-4o-mini". +_MODEL_NAME_CHARS = frozenset(string.ascii_letters + string.digits + "._:/@-") + + +def _resolved_in_source(root: Path, path: str) -> Path | None: + """``root / path`` resolved, each lookup charged to the save's budget, or None: a link + loop, a NUL, a path the system refuses. PathLookupsSpent is not caught here: past the + budget the save answers with its own message.""" + try: + return charged_path(root / path) + except (OSError, RuntimeError, ValueError): + return None + + +def _source_lines(path: Path, limit: int) -> list[str] | None: + """The lines of a file the way _copy_prompt reads them, or None: not a regular file, not + readable, or larger than ``limit`` bytes.""" + try: + # Before the open: opening a FIFO or a device waits on it, or reads without end. + if not path.is_file(): + return None + with path.open("rb") as handle: + raw = handle.read(limit + 1) + except (OSError, ValueError): + return None + if len(raw) > limit: + return None + # As _copy_prompt decodes, whatever coding line the file carries: one file then has one + # numbering in both checks, and no codec the file names gets to run on it. + return raw.decode("utf-8", errors="replace").splitlines() + + +def _named_on(line: str, name: str) -> bool: + """Whether ``name`` is on ``line`` as a whole name. ``name`` is made only of + _MODEL_NAME_CHARS, so an occurrence starting inside one that failed has a name character + just before it, and the search resumes after the failed one.""" + if not name: + # find("") matches at every position without moving on. + return False + at = line.find(name) + while at >= 0: + end = at + len(name) + if (at == 0 or line[at - 1] not in _MODEL_NAME_CHARS) and ( + end == len(line) or line[end] not in _MODEL_NAME_CHARS + ): + return True + at = line.find(name, end) + return False + + +def _model_problems(contract: AgentContract, source_root: Path) -> list[str]: + """Problems with contract.model, checked against the cited line. On success, strips name and + provider and stores path as the resolved file relative to the root, in POSIX form. Resolves + one path (charged to the save's budget) and reads at most one file.""" + model = contract.model + if model is None: + return [] + name, provider, path = model.name.strip(), model.provider.strip(), model.path.strip() + shown_name, shown_path = name[:100], path[:200] + problems: list[str] = [] + if not name: + problems.append( + "model: name is empty; give the model's name as written in the source, or leave " + "model out" + ) + elif not _MODEL_NAME_CHARS.issuperset(name): + problems.append( + "model: a model name is letters, digits and . _ : / @ -; write it as it appears in " + "the source, or leave model out" + ) + if not _MODEL_NAME_CHARS.issuperset(provider): + problems.append( + "model: a provider is letters, digits and . _ : / @ -; write it as the source names " + "it, or leave it out" + ) + if len(name) > _LONGEST_MODEL_FIELD or len(provider) > _LONGEST_MODEL_FIELD: + problems.append("model: name and provider are at most 255 characters") + if not path or model.line < 1: + problems.append( + "model: give path and line, the file and the line the model's name is written on" + ) + if len(path) > _LONGEST_MODEL_PATH: + problems.append("model: path is too long") + # All of these at once, and before the disk: a citation wrong in its shape costs no lookup. + if problems: + return problems + root = source_root.resolve() + resolved = _resolved_in_source(root, path) + if resolved is None or not is_within(resolved, root): + return [f"model: {shown_path} is not a file in the source"] + lines = _source_lines(resolved, _LARGEST_PROMPT_FILE) + if lines is None: + return [ + f"model: {shown_path} can't be read as a file of at most 1 MiB; check the path, and " + "cite the line in the file the name is written in" + ] + if model.line > len(lines): + return [f"model: line {model.line} is outside {shown_path}, which has {len(lines)} lines"] + if not _named_on(lines[model.line - 1], name): + return [ + f"model: the name {shown_name} is not on line {model.line} of {shown_path} as a " + "whole name; write it exactly as it appears there, or leave model out if the name " + "is only set at run time" + ] + model.name, model.provider = name, provider + # The file the path leads to, so a link inside the source is stored as its target. + model.path = resolved.relative_to(root).as_posix() + return [] + + def _without_nulls(value: Any) -> Any: """Treat JSON null like an omitted optional field, including in nested records.""" if isinstance(value, dict): @@ -619,6 +740,24 @@ def contract_tools( "assembled from, in order, as a path relative to the source root and the first " "and last line of each. These lines replace system_prompt_excerpt as written.", }, + # Nothing required inside it either: a model sent without its line reaches the + # save, which says what is missing, instead of failing the whole call here. + "model": { + "type": "object", + "properties": { + "provider": {"type": "string"}, + "name": {"type": "string"}, + "path": {"type": "string"}, + "line": {"type": "integer"}, + }, + "description": "The model the agent talks to the person with, read from its " + "source: provider as the source names it (openai, anthropic, google...; leave " + "out when it doesn't say), name exactly as written (letters, digits and " + ". _ : / @ - only), and the file (relative to " + "the source root) and line the name is written on. One model; a voice agent's " + "speech-to-text and text-to-speech models don't count. Leave model out when " + "the name is only set at run time.", + }, "hard_constraints": { "type": "array", "items": {"type": "string"}, @@ -1554,12 +1693,19 @@ def rows_for(collection: str) -> list[dict[str, Any]]: ) payload["chosen_evals"] = wanted + # Under any name the contract takes the quote from, so a quote under an alias is asked + # about too. + quoted = ( + payload.get(key) + for key in ("system_prompt_excerpt", *_ALIASES["system_prompt_excerpt"]) + ) + excerpt = next((one for one in quoted if isinstance(one, str) and one.strip()), "") thin = [ ( "prompt", bool(payload.get("conversational", True)) and not payload.get("hard_constraints") - and not str(payload.get("system_prompt_excerpt") or "").strip(), + and not excerpt, "no hard_constraints and no system_prompt_excerpt, for a conversational agent. " "Its prompt usually exists and often lives away from the main agent file — " "search the whole source for a long instructions string before deciding there " @@ -1602,6 +1748,17 @@ def rows_for(collection: str) -> list[dict[str, Any]]: "values. Semantic summaries such as rooms, notable_bookings or restaurant_slots " "cannot seed or verify the submitted store.", ), + ( + # A quote may stop short of the whole prompt; lines are copied whole. Asked once, + # since a prompt built at run time has no lines to name, and its quote is kept. + "prompt-lines", + source_root is not None + and bool(excerpt) + and not payload.get("system_prompt_from"), + "system_prompt_excerpt is a quote, and system_prompt_from names no lines. If the " + "prompt is written in the source, name its lines in system_prompt_from so it is " + "copied whole.", + ), ] # All of them together, and each only once. Nudging in sequence would cost a turn per # nudge and read as though the requirements were being invented one at a time. diff --git a/src/fi/alk/providers/agent.py b/src/fi/alk/providers/agent.py index e0cb43529..b05edfd45 100644 --- a/src/fi/alk/providers/agent.py +++ b/src/fi/alk/providers/agent.py @@ -177,7 +177,7 @@ def build_authored_agent( } ], "tools": tools, - "model": None, + "model": _authored_model(contract.get("model")), "voice": None, "telephony": [], "supports": [], @@ -229,6 +229,17 @@ def _authored_tool( } +def _authored_model(value: Any) -> dict[str, Any] | None: + """The document's model: the contract's provider and name, or None when it has no name.""" + if not isinstance(value, Mapping): + return None + name = str(value.get("name") or "").strip()[:_NAME_MAX] + if not name: + return None + provider = str(value.get("provider") or "").strip()[:_NAME_MAX] or None + return {"provider": provider, "name": name, "temperature": None, "max_tokens": None} + + def _redact(value: Any, key: str = "") -> Any: lowered = key.lower() if any(marker in lowered for marker in _REDACTION_MARKERS): diff --git a/tests/harness/test_contract_model.py b/tests/harness/test_contract_model.py new file mode 100644 index 000000000..eff6af7db --- /dev/null +++ b/tests/harness/test_contract_model.py @@ -0,0 +1,775 @@ +"""The model an agent talks to the person with: recorded only where its name is on a cited line. + +The save is the only gate. A name it can't find whole on the line the model cites is refused with +a message saying what to do, and a contract without a model is saved exactly as before. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import threading +import time +from pathlib import Path + +import pytest + +from fi.alk.harness.contract import AgentContract +from fi.alk.harness.runtime_selection import PathLookupsSpent, path_lookup_budget +from fi.alk.harness.tools import ( + _model_problems, + _named_on, + accept_contract, + contract_tools, +) + +MIB = 1024 * 1024 +FIXTURES = Path(__file__).resolve().parents[1] / "providers" / "fixtures" / "authored" + +EMPTY_NAME = ( + "model: name is empty; give the model's name as written in the source, or leave model out" +) +NAME_CHARACTERS = ( + "model: a model name is letters, digits and . _ : / @ -; write it as it appears in the " + "source, or leave model out" +) +PROVIDER_CHARACTERS = ( + "model: a provider is letters, digits and . _ : / @ -; write it as the source names it, or " + "leave it out" +) +TOO_LONG = "model: name and provider are at most 255 characters" +NO_LINE = "model: give path and line, the file and the line the model's name is written on" +LONG_PATH = "model: path is too long" +REFUSED = "Not accepted. Fix all of these and call submit_contract again:\n - " +PROMPT_LINES = ( + "system_prompt_excerpt is a quote, and system_prompt_from names no lines. If the prompt is " + "written in the source, name its lines in system_prompt_from so it is copied whole." +) +ASKED_FOR_LINES = ( + REFUSED + + PROMPT_LINES + + "\n - If any of these genuinely does not apply, submit again as is." +) + + +def not_a_file(path: str) -> str: + return f"model: {path[:200]} is not a file in the source" + + +def unreadable(path: str) -> str: + return ( + f"model: {path[:200]} can't be read as a file of at most 1 MiB; check the path, and cite " + "the line in the file the name is written in" + ) + + +def outside(line: int, path: str, count: int) -> str: + return f"model: line {line} is outside {path[:200]}, which has {count} lines" + + +def not_named(name: str, line: int, path: str) -> str: + return ( + f"model: the name {name[:100]} is not on line {line} of {path[:200]} as a whole name; " + "write it exactly as it appears there, or leave model out if the name is only set at " + "run time" + ) + + +def payload(**fields): + return { + "agent": "pair", + "modality": "chat", + "real_use_cases": ["order"], + "tools": [{"name": "add", "args": ["item_id"]}], + **fields, + } + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + folder = tmp_path / "src" + folder.mkdir() + (folder / "agent.py").write_text('AGENT = "pair"\nMODEL = "gpt-4o"\n', encoding="utf-8") + return folder + + +def checked(root: Path, model) -> tuple[list[str], AgentContract, float]: + """The model check alone, under the save's budget, and how long it took.""" + contract = AgentContract.model_validate(payload(model=model)) + with path_lookup_budget(): + started = time.perf_counter() + problems = _model_problems(contract, root) + took = time.perf_counter() - started + return problems, contract, took + + +def saved(destination: Path) -> dict: + return json.loads((destination / "contract.json").read_text(encoding="utf-8")) + + +def link(target: str, at: Path) -> None: + try: + os.symlink(target, at) + except (OSError, NotImplementedError, AttributeError) as unable: + pytest.skip(f"links can't be made here: {unable}") + + +def chain(root: Path, prefix: str, count: int) -> None: + """Links each costing about a thousand lookups to follow, the last leading to agent.py.""" + for index in range(count): + following = f"{prefix}{index + 1}" if index < count - 1 else "agent.py" + link("." + "/" * 999 + following, root / f"{prefix}{index}") + + +def text_of(result: dict) -> str: + return result["content"][0]["text"] + + +def submitter(destination: Path, source_root: Path | None = None): + server = contract_tools(destination, source_root=source_root) + tool = next(one for one in server.tools if one.name == "submit_contract") + return lambda arguments: asyncio.run(tool.handler(dict(arguments))) + + +# P1, P8 + + +def test_a_contract_without_a_model_saves_and_builds_as_before(tmp_path): + from fi.alk.providers.agent import build_authored_agent + + contract = json.loads((FIXTURES / "chat_source.json").read_text(encoding="utf-8"))["contract"] + + result = accept_contract(dict(contract), tmp_path / "out") + + assert not result.get("is_error"), result + raw = (tmp_path / "out" / "contract.json").read_bytes() + assert hashlib.sha256(raw).hexdigest() == ( + "7a61d8e863f75541c6d32c287fea02fb7a605d3b440a91f4c1b4212a3e1832b6" + ) + assert "model" not in json.loads(raw) + document = build_authored_agent( + AgentContract.model_validate(json.loads(raw)).model_dump(mode="json") + ) + assert document["setup_hash"] == ( + "sh1:3854d6ecb87a1ee2890c5c16f25e854e20be84c109dc80311ad292e6f656d622" + ) + assert document["model"] is None + + +# P2 + + +def test_model_without_a_source_is_dropped(tmp_path): + for index, model in enumerate( + ( + {"provider": "openai", "name": "gpt-4o", "path": "agent.py", "line": 2}, + # Unchecked shapes too: with no source nothing about the model is known. + {"name": "gpt 4o"}, + ) + ): + destination = tmp_path / f"out{index}" + + result = accept_contract(payload(model=model), destination) + + assert not result.get("is_error"), result + assert "model" not in saved(destination) + + +# P3a-P3d + + +@pytest.mark.parametrize( + "model, expected", + [ + pytest.param({"name": "", "path": "agent.py", "line": 2}, [EMPTY_NAME], id="empty-name"), + pytest.param({"name": " ", "path": "agent.py", "line": 2}, [EMPTY_NAME], id="blank-name"), + pytest.param( + {"name": "gpt 4o", "path": "agent.py", "line": 2}, + [NAME_CHARACTERS], + id="name-with-a-space", + ), + pytest.param( + {"name": 'gpt-4o"', "path": "agent.py", "line": 2}, + [NAME_CHARACTERS], + id="name-with-a-quote", + ), + pytest.param( + {"provider": "open\x00ai", "name": "gpt-4o", "path": "agent.py", "line": 2}, + [PROVIDER_CHARACTERS], + id="provider-with-a-nul", + ), + pytest.param( + {"provider": "http://[x", "name": "gpt-4o", "path": "agent.py", "line": 2}, + [PROVIDER_CHARACTERS], + id="provider-url-bracket", + ), + pytest.param("gpt-4o", [NO_LINE], id="bare-string"), + pytest.param( + {"name": "a" * 256, "path": "agent.py", "line": 2}, [TOO_LONG], id="long-name" + ), + pytest.param( + {"provider": "a" * 256, "name": "gpt-4o", "path": "agent.py", "line": 2}, + [TOO_LONG], + id="long-provider", + ), + pytest.param({"name": "gpt-4o", "line": 2}, [NO_LINE], id="no-path"), + pytest.param({"name": "gpt-4o", "path": " ", "line": 2}, [NO_LINE], id="blank-path"), + pytest.param( + {"name": "gpt-4o", "path": "agent.py", "line": 0}, [NO_LINE], id="line-zero" + ), + pytest.param( + {"name": "gpt-4o", "path": "a" * 4097, "line": 2}, [LONG_PATH], id="long-path" + ), + ], +) +def test_model_shape_problems(root, model, expected): + problems, _, _ = checked(root, model) + + assert problems == expected + + +# P3e + + +def _parent(root: Path) -> str: + return "../outside.py" + + +def _absolute_outside(root: Path) -> str: + return str(root.parent / "outside.py") + + +def _link_outside(root: Path) -> str: + link(str(root.parent / "outside.py"), root / "out.py") + return "out.py" + + +def _link_to_dev_zero(root: Path) -> str: + if not Path("/dev/zero").exists(): + pytest.skip("no /dev/zero here") + # Only made, never read: it must be refused before anything opens it. + link("/dev/zero", root / "zero.py") + return "zero.py" + + +def _link_loop(root: Path) -> str: + link("b.py", root / "a.py") + link("a.py", root / "b.py") + return "a.py" + + +def _nul(root: Path) -> str: + return "agent\x00.py" + + +def _unwinding_chain(root: Path) -> str: + # Each link leaves a long part behind as it unwinds, so the path grows past any limit. + for index in range(900): + target = f"L{index + 1}/" + "A" * 990 if index < 899 else "." + link(target, root / f"L{index}") + return "L0/agent.py" + + +@pytest.mark.parametrize( + "cited", + [ + pytest.param(_parent, id="parent"), + pytest.param(_absolute_outside, id="absolute-outside"), + pytest.param(_link_outside, id="link-outside"), + pytest.param(_link_to_dev_zero, id="link-to-dev-zero"), + pytest.param(_link_loop, id="link-loop"), + pytest.param(_nul, id="nul"), + pytest.param(_unwinding_chain, id="unwinding-chain"), + ], +) +def test_model_path_outside_the_source_is_refused(root, cited): + # Holds the name on the cited line, so only where the file is can refuse it. + (root.parent / "outside.py").write_text('AGENT = "pair"\nMODEL = "gpt-4o"\n') + path = cited(root) + + problems, _, took = checked(root, {"name": "gpt-4o", "path": path, "line": 2}) + + assert problems == [not_a_file(path)] + assert took < 0.5 + + +# The save's one budget + + +def test_model_path_spending_the_save_budget_is_refused(tmp_path, root): + chain(root, "E", 25) + destination = tmp_path / "out" + + started = time.perf_counter() + result = accept_contract( + payload(model={"name": "gpt-4o", "path": "E0", "line": 2}), + destination, + source_root=root, + ) + took = time.perf_counter() - started + + assert result.get("is_error") + assert text_of(result) == REFUSED + str(PathLookupsSpent()) + assert not (destination / "contract.json").exists() + assert took < 0.5 + + +@pytest.mark.parametrize("links", [pytest.param(11, id="lookup-budget-real")]) +def test_model_path_charged_to_the_same_budget_as_the_prompt(tmp_path, root, links): + chain(root, "P", links) + chain(root, "M", links) + model = {"name": "gpt-4o", "path": "M0", "line": 2} + # Either half alone fits the budget, so only charging both to one budget refuses them. + alone, _, _ = checked(root, model) + assert alone == [] + destination = tmp_path / "out" + + result = accept_contract( + payload( + system_prompt_from=[{"path": "P0", "start_line": 1, "end_line": 2}], + model=model, + ), + destination, + source_root=root, + ) + + assert result.get("is_error") + assert text_of(result) == REFUSED + str(PathLookupsSpent()) + assert not (destination / "contract.json").exists() + + +def test_model_path_through_an_honest_link_is_kept(tmp_path, root): + (root / "pkg").mkdir() + (root / "pkg" / "settings.py").write_text('MODEL = "gpt-4o"\n') + link("pkg/settings.py", root / "alias.py") + destination = tmp_path / "out" + + result = accept_contract( + payload(model={"provider": "openai", "name": "gpt-4o", "path": "alias.py", "line": 1}), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["model"] == { + "provider": "openai", + "name": "gpt-4o", + "path": "pkg/settings.py", + "line": 1, + } + + +# P3f + + +def _over_a_mib(root: Path) -> str: + head = b'MODEL = "gpt-4o"\n' + (root / "big.py").write_bytes(head + b"#" * (MIB + 1 - len(head))) + return "big.py" + + +def _fifo_inside(root: Path) -> str: + if not hasattr(os, "mkfifo"): + pytest.skip("no FIFOs here") + try: + os.mkfifo(root / "pipe.py") + except OSError as unable: + pytest.skip(f"a FIFO can't be made here: {unable}") + return "pipe.py" + + +def _directory(root: Path) -> str: + (root / "pkg").mkdir() + return "pkg" + + +def _long_component(root: Path) -> str: + return "a" * 300 + + +@pytest.mark.parametrize( + "cited", + [ + pytest.param(_over_a_mib, id="over-1-mib"), + pytest.param(_fifo_inside, id="fifo-inside"), + pytest.param(_directory, id="directory"), + pytest.param(_long_component, id="long-component"), + ], +) +def test_model_file_that_cannot_be_read_is_refused(root, cited): + path = cited(root) + # On a thread that can't hold the run open: a check that opens a FIFO waits on it forever. + answer: list = [] + checking = threading.Thread( + target=lambda: answer.append(checked(root, {"name": "gpt-4o", "path": path, "line": 1})), + daemon=True, + ) + checking.start() + checking.join(timeout=5) + assert answer, "the check did not return" + problems, _, took = answer[0] + + assert problems == [unreadable(path)] + assert took < 0.5 + + +# P3g + + +def test_model_line_outside_the_file_is_refused(root): + (root / "three.py").write_text('A = 1\nB = 2\nMODEL = "gpt-4o"\n') + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "three.py", "line": 4}) + + assert problems == [outside(4, "three.py", 3)] + + +# P3h + + +@pytest.mark.parametrize( + "written, name", + [ + pytest.param("abc", "", id="empty-name-direct"), + pytest.param('MODEL = "claude"', "gpt-4o", id="absent"), + pytest.param('model="gpt-4o-mini"', "gpt-4o", id="prefix-of-longer"), + pytest.param('model="gpt-4o"', "o", id="inside-longer"), + pytest.param( + 'MODEL = "llama3:8b-instruct-q4_K_M"', "llama3:8b-instruct-q4", id="underscore-suffix" + ), + pytest.param('model="GPT-4o"', "gpt-4o", id="case"), + ], +) +def test_model_name_not_on_the_cited_line_is_refused(root, written, name): + if not name: + # On a thread that can't hold the run open: a search that never moves on fails here. + answer: list[bool] = [] + searching = threading.Thread( + target=lambda: answer.append(_named_on(written, name)), daemon=True + ) + searching.start() + searching.join(timeout=2) + assert answer == [False] + return + (root / "named.py").write_text(f"# the model\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": "named.py", "line": 2}) + + assert problems == [not_named(name, 2, "named.py")] + + +@pytest.mark.parametrize( + "written, name", + [ + pytest.param('model="gpt-4o"', "gpt-4o", id="quoted"), + pytest.param("model: gpt-4o", "gpt-4o", id="line-end"), + pytest.param("gpt-4o is the model", "gpt-4o", id="line-start"), + pytest.param('model = "openai/gpt-4o"', "openai/gpt-4o", id="with-provider-prefix"), + pytest.param('x = "gpt-4o-mini" or "gpt-4o"', "gpt-4o", id="second-occurrence"), + pytest.param("«gpt-4o»", "gpt-4o", id="non-ascii-edge"), + ], +) +def test_model_whole_name_is_found(root, written, name): + (root / "named.py").write_text(f"# the model\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": "named.py", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +# P4 + + +def test_model_on_the_cited_line_is_saved_stripped(tmp_path, root): + destination = tmp_path / "out" + + result = accept_contract( + payload( + model={ + "provider": " openai ", + "name": " gpt-4o ", + "path": f" {root / 'agent.py'} ", + "line": 2, + } + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["model"] == { + "provider": "openai", + "name": "gpt-4o", + "path": "agent.py", + "line": 2, + } + + +# P5 + + +@pytest.mark.parametrize( + "written, line", + [ + pytest.param(b'# head\nA = 1\x0cB = 2\nMODEL = "gpt-4o"\n', 4, id="form-feed"), + # Read as latin-1, \x85 is a line break and the model's line moves down one. + pytest.param(b'# coding: latin-1\nA = "\x85"\nMODEL = "gpt-4o"\n', 3, id="latin-1-c1"), + ], +) +def test_model_line_numbering_matches_the_prompt_copy(tmp_path, root, written, line): + (root / "numbered.py").write_bytes(written) + destination = tmp_path / "out" + + result = accept_contract( + payload( + system_prompt_from=[{"path": "numbered.py", "start_line": line, "end_line": line}], + model={"name": "gpt-4o", "path": "numbered.py", "line": line}, + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + contract = saved(destination) + assert contract["system_prompt_excerpt"] == 'MODEL = "gpt-4o"' + assert contract["model"]["line"] == line + + +# P6 + + +def _ace_label() -> bytes: + written = "".join(chr(0x4E00 + index * 37) for index in range(40)) + while written: + try: + label = written.encode("idna") + except UnicodeError: + label = b"" + if label and len(label) <= 63: + return label + written = written[:-1] + raise AssertionError("no ACE label") + + +def _filled(head: bytes, unit: bytes) -> bytes: + return head + unit * ((MIB - len(head)) // len(unit)) + + +def _one_line() -> bytes: + head = b'MODEL = "gpt-4o" #' + return head + b"x" * (MIB - len(head)) + + +def _idna_ace() -> bytes: + return _filled(b"# coding: idna\nMODEL = 'gpt-4o'\n", _ace_label() + b".") + + +def _punycode() -> bytes: + return _filled(b"# coding: punycode\nMODEL = 'gpt-4o'\n", b"a" * 60 + b"-" + b"kva" + b"\n") + + +def _line_of_m() -> bytes: + return b"m" * MIB + + +def _name_at_the_end() -> bytes: + tail = b'="' + b"m" * 255 + b'"' + return b"m" * (MIB - len(tail)) + tail + + +@pytest.mark.parametrize( + "written, line, name, refused", + [ + pytest.param(_one_line, 1, "gpt-4o", False, id="one-line-1-mib"), + pytest.param(_idna_ace, 2, "gpt-4o", False, id="idna-ace-1-mib"), + pytest.param(_punycode, 2, "gpt-4o", False, id="punycode-1-mib"), + pytest.param(_line_of_m, 1, "m" * 255, True, id="long-name-on-a-line-of-it"), + pytest.param(_name_at_the_end, 1, "m" * 255, False, id="name-at-the-end-of-a-line-of-it"), + ], +) +def test_a_crafted_model_file_does_not_hold_up_the_save(root, written, line, name, refused): + content = written() + assert len(content) <= MIB + (root / "crafted.py").write_bytes(content) + + problems, _, took = checked(root, {"name": name, "path": "crafted.py", "line": line}) + + assert problems == ([not_named(name, line, "crafted.py")] if refused else []) + assert took < 0.5 + + +# P7 + + +def test_a_quote_without_lines_is_asked_for_lines_once(tmp_path, root): + destination = tmp_path / "out" + submit = submitter(destination, root) + quoted = payload( + hard_constraints=["Never share a card number."], + system_prompt_excerpt="You take orders for the cafe.", + ) + + first = submit(quoted) + + assert first.get("is_error") + assert text_of(first) == ASKED_FOR_LINES + assert not (destination / "contract.json").exists() + + second = submit(quoted) + + assert not second.get("is_error"), second + assert PROMPT_LINES not in text_of(second) + assert saved(destination)["system_prompt_excerpt"] == "You take orders for the cafe." + + +@pytest.mark.parametrize("alias", ["system_prompt", "prompt", "instructions"]) +def test_quote_under_an_alias_is_asked_too(tmp_path, root, alias): + destination = tmp_path / "out" + submit = submitter(destination, root) + quoted = payload( + hard_constraints=["Never share a card number."], + **{alias: "You take orders for the cafe."}, + ) + + first = submit(quoted) + + assert text_of(first) == ASKED_FOR_LINES + assert not (destination / "contract.json").exists() + second = submit(quoted) + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "You take orders for the cafe." + + +def test_lines_given_are_not_asked(tmp_path, root): + destination = tmp_path / "out" + submit = submitter(destination, root) + + first = submit( + payload( + hard_constraints=["Never share a card number."], + system_prompt_excerpt="You take orders", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 2}], + ) + ) + + assert not first.get("is_error"), first + assert PROMPT_LINES not in text_of(first) + assert saved(destination)["system_prompt_excerpt"] == 'AGENT = "pair"\nMODEL = "gpt-4o"' + + +def test_no_source_is_not_asked(tmp_path): + destination = tmp_path / "out" + submit = submitter(destination) + + first = submit( + payload( + hard_constraints=["Never share a card number."], + system_prompt_excerpt="You take orders for the cafe.", + ) + ) + + assert not first.get("is_error"), first + assert PROMPT_LINES not in text_of(first) + assert saved(destination)["system_prompt_excerpt"] == "You take orders for the cafe." + + +# The schema the model writes against + + +def test_submit_contract_schema_describes_model(tmp_path): + server = contract_tools(tmp_path) + schema = next(one for one in server.tools if one.name == "submit_contract").input_schema + + model = schema["properties"]["model"] + + assert set(model["properties"]) == {"provider", "name", "path", "line"} + assert "required" not in model + assert "model" not in schema["required"] + + +def test_saved_model_and_whole_prompt_reach_the_document(tmp_path): + # The whole path the platform sees: the save copies the prompt's lines over the quote and + # records the model, the build reads contract.json back, and the document carries both. + from fi.alk.providers.agent import build_authored_agent + + source = tmp_path / "source" + source.mkdir() + (source / "agent.py").write_text( + "import openai\n" + "PROMPT = '''\n" + "You book rides for riders.\n" + "Never share a rider's PIN.\n" + "Confirm the pickup before booking.\n" + "'''\n" + "\n" + "client = openai.OpenAI()\n" + 'model="gpt-4o"\n' + ) + payload = json.loads((FIXTURES / "chat_source.json").read_text())["contract"] + payload.update( + system_prompt_excerpt="short quote", + system_prompt_from=[{"path": "agent.py", "start_line": 3, "end_line": 5}], + model={"provider": "openai", "name": "gpt-4o", "path": "agent.py", "line": 9}, + ) + out = tmp_path / "out" + + result = accept_contract(payload, out, source_root=source) + + assert not result.get("is_error"), result + contract = AgentContract.model_validate( + json.loads((out / "contract.json").read_text(encoding="utf-8")) + ).model_dump(mode="json") + document = build_authored_agent(contract) + assert document["behaviour"][0]["instructions"] == ( + "You book rides for riders.\n" + "Never share a rider's PIN.\n" + "Confirm the pickup before booking." + ) + assert document["model"] == { + "provider": "openai", + "name": "gpt-4o", + "temperature": None, + "max_tokens": None, + } + assert document["extension"]["contract"]["model"]["path"] == "agent.py" + + +@pytest.mark.parametrize( + "model, said", + [ + pytest.param( + {"name": "gpt-5", "path": "agent.py", "line": 2}, + lambda: not_named("gpt-5", 2, "agent.py"), + id="not-on-line", + ), + pytest.param( + {"name": "gpt-4o", "path": "../outside.py", "line": 2}, + lambda: not_a_file("../outside.py"), + id="parent", + ), + ], +) +def test_a_refused_model_is_not_saved(tmp_path, root, model, said): + # The check's answer has to reach the save: a model it refuses is never written. + (tmp_path / "outside.py").write_text('AGENT = "pair"\nMODEL = "gpt-4o"\n', encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract(payload(model=model), destination, source_root=root) + + assert result.get("is_error") + assert text_of(result) == REFUSED + said() + assert not (destination / "contract.json").exists() + + +def test_a_quote_under_an_alias_is_not_called_missing(tmp_path, root): + # A quote under another name is a quote: the save asks for its lines, and never says in the + # same answer that there is no prompt at all. + destination = tmp_path / "out" + submit = submitter(destination, root) + + first = submit(payload(prompt="You take orders for the cafe.")) + + assert text_of(first) == ASKED_FOR_LINES diff --git a/tests/providers/test_providers_agent_authored.py b/tests/providers/test_providers_agent_authored.py index 312b9f9ff..9cf867b21 100644 --- a/tests/providers/test_providers_agent_authored.py +++ b/tests/providers/test_providers_agent_authored.py @@ -187,3 +187,86 @@ def test_no_framework_leaves_extension_and_hash() -> None: assert agent == recorded["agent"] assert set(agent["extension"]) == {"contract"} + + +_ABSENT = object() + + +def _chat_contract() -> dict: + # A raw dict, not through AgentContract, so a model of any shape reaches + # build_authored_agent unchanged. + return copy.deepcopy( + json.loads((AUTHORED / "chat_source.json").read_text())["contract"] + ) + + +@pytest.mark.parametrize( + "model", + [_ABSENT, None, "gpt-4o", {"name": " "}], + ids=["absent", "none", "raw-string", "blank-name"], +) +def test_no_model_leaves_the_document_and_hash(model: object) -> None: + recorded = json.loads((AUTHORED / "chat_source.json").read_text()) + contract = _chat_contract() + expected = copy.deepcopy(recorded["agent"]) + if model is not _ABSENT: + contract["model"] = model + # The stored contract echoes the value whatever its shape, so only that + # echo and the hash over it may differ from the recorded document. + expected["extension"]["contract"]["model"] = model + expected["setup_hash"] = compute_setup_hash(expected) + + agent = build_authored_agent(contract) + + assert agent["model"] is None + assert agent == expected + + +def test_model_goes_to_the_document() -> None: + def built(model: dict) -> dict: + return build_authored_agent({**_chat_contract(), "model": model}) + + without = build_authored_agent(_chat_contract()) + saved = {"provider": " openai ", "name": " gpt-4o ", "path": "agent.py", "line": 9} + agent = built(saved) + + assert agent["model"] == { + "provider": "openai", + "name": "gpt-4o", + "temperature": None, + "max_tokens": None, + } + assert agent["extension"]["contract"]["model"] == saved + assert set(agent["extension"]) == {"contract"} + assert agent["setup_hash"] == compute_setup_hash(agent) + assert agent["setup_hash"] != without["setup_hash"] + + for provider in ("", " ", None): + assert built({"provider": provider, "name": "gpt-4o"})["model"] == { + "provider": None, + "name": "gpt-4o", + "temperature": None, + "max_tokens": None, + } + + long = built({"provider": "p" * 300, "name": "m" * 300, "path": "a.py", "line": 1}) + assert long["model"]["provider"] == "p" * 255 + assert long["model"]["name"] == "m" * 255 + + with_url = built( + { + "provider": "openai", + "name": "https://u:p@host.example:8443/v1/m?token=abc", + "path": "agent.py", + "line": 9, + } + ) + assert with_url["model"]["name"] == "https://host.example:8443/v1/m" + assert with_url["extension"]["contract"]["model"] == { + "provider": "openai", + "name": "https://host.example:8443/v1/m", + "path": "agent.py", + "line": 9, + } + assert "token=abc" not in json.dumps(with_url) + assert "u:p@" not in json.dumps(with_url) From eadebfc09e081d1eb4ad34c63fdbff75b07c5a70 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 16:52:54 +0530 Subject: [PATCH 02/10] fix(harness): record the model only when it is the value on the cited line, and check named prompt lines against the quote (TH-8953) Answers the review of #268. - The model check moves to model_citation.py. A name counts only when it is the value written on the line: between matching quotes and not a key or a lookup (a subscript, get(), getenv()), or bare after : or = in a configuration file, and never inside a comment. A name shaped like a setting (OPENAI_MODEL, openai_model, model) is refused. - When a quote and lines are both given, the lines must hold the quote, compared by letters and digits with escapes decoded. A second mismatch in one reading keeps the quote and drops the lines. - A refused model citation is refused once per reading, then dropped, and the prompt nudges come one at a time. - _copy_prompt reads through the same bounded helpers, so an unreadable prompt file is refused instead of raising. Signed-off-by: khushalsonawat --- src/fi/alk/harness/model_citation.py | 282 +++++++++ src/fi/alk/harness/tools.py | 203 ++----- tests/harness/test_contract_model.py | 573 ++++++++++++++++-- .../harness/test_contract_tools_subfolder.py | 8 +- 4 files changed, 888 insertions(+), 178 deletions(-) create mode 100644 src/fi/alk/harness/model_citation.py diff --git a/src/fi/alk/harness/model_citation.py b/src/fi/alk/harness/model_citation.py new file mode 100644 index 000000000..d8eb35a5c --- /dev/null +++ b/src/fi/alk/harness/model_citation.py @@ -0,0 +1,282 @@ +"""Reading the model an agent talks to the person with, and the prompt it is given, from the +agent's own files, while a conversation waits on the answer. + +A model is kept only when its name is the value written on the line the reader cites: the +Agents page shows it as fact, and an unknown model is better left empty than guessed. The files +can be built to stall whoever reads them, so every path is resolved within the save's one lookup +budget, every read is bounded in size and decoded as UTF-8 whatever coding line the file carries, +and every check of a line costs at most a few passes over it. +""" + +from __future__ import annotations + +import re +import string +from pathlib import Path + +from .contract import AgentContract +from .runtime_selection import charged_path +from .source_folder import is_within + +# The platform refuses the whole agent document past this, for the model's name and provider. +LONGEST_MODEL_FIELD = 255 +# One path as written: charged_path doesn't refuse a long path itself. +LONGEST_MODEL_PATH = 4096 +# The cited file, as for a prompt file. +LARGEST_MODEL_FILE = 1024 * 1024 +# What a model name is made of. A match must have none of these right before or after it, so +# "gpt-4o" is not found inside "gpt-4o-mini". +MODEL_NAME_CHARS = frozenset(string.ascii_letters + string.digits + "._:/@-") +# Files whose values are written bare after a key: `model: gpt-4o`, `MODEL=gpt-4o`, +# `ENV MODEL=gpt-4o`, `export MODEL=gpt-4o`. +CONFIG_SUFFIXES = frozenset( + { + ".yaml", ".yml", ".toml", ".json", ".ini", ".cfg", ".conf", ".env", ".properties", + ".sh", ".bash", ".zsh", ".dockerfile", + } +) +_YAML_SUFFIXES = frozenset({".yaml", ".yml"}) +# Names a setting goes by, never a model: `cfg.get("model")`, `os.getenv("llm")`. +_SETTING_WORDS = frozenset({"model", "llm", "name", "default", "engine", "deployment", "model_name"}) +_QUOTES = frozenset("\"'`") +_SETTING_NAME_CHARS = frozenset(string.ascii_uppercase + string.digits + "_") +# A \uXXXX or \xXX escape in a string, as json.dump writes any non-ASCII character. +_CHAR_ESCAPE = re.compile(r"\\(?:u([0-9a-fA-F]{4})|x([0-9a-fA-F]{2}))") +# How much of a quote the named lines must hold, once spacing is set aside. +_QUOTE_START = 80 + + +def resolved_in_source(root: Path, path: str) -> Path | None: + """``root / path`` resolved, each lookup charged to the save's budget, or None: outside + ``root``, a link loop, a NUL, a path the system refuses. ``root`` is already resolved. + PathLookupsSpent is not caught here: past the budget the save answers with its own message.""" + try: + resolved = charged_path(root / path) + except (OSError, RuntimeError, ValueError): + return None + return resolved if is_within(resolved, root) else None + + +def read_source(path: Path, limit: int) -> bytes | None: + """At most ``limit + 1`` bytes of the file, or None: not a regular file, or not readable. + The caller judges the size.""" + try: + # Before the open: opening a FIFO or a device waits on it, or reads without end. + if not path.is_file(): + return None + with path.open("rb") as handle: + return handle.read(limit + 1) + except (OSError, ValueError): + return None + + +def source_lines(raw: bytes) -> list[str]: + """The file's lines, whatever coding line it carries: no codec the file names gets to run + on it, and one file has one numbering wherever it is read.""" + return raw.decode("utf-8", errors="replace").splitlines() + + +def _comment_start(line: str) -> int: + """Where a comment starts on the line, or its length: a line that is a comment, or the first + ` #` or ` //`. Quotes aren't tracked, so a ` #` inside a string reads as a comment too; that + can only leave a model out, never record a wrong one.""" + stripped = line.lstrip() + if stripped.startswith(("#", "//", "/*", "', "gpt-4o-mini", id="markup-comment"), + pytest.param("llm.html", 'x ', "gpt-4o-mini", id="trailing-markup-comment"), + ], +) +def test_a_value_in_a_block_comment_is_not_the_model(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +def test_a_prompt_with_escaped_characters_holds_the_quote(tmp_path, root): + # json.dump writes every non-ASCII character as an escape; the quote has the character. + written = '{"prompt": "You are pair \\u2014 a helpful assistant who books tables."}' + (root / "prompt.json").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair — a helpful assistant who books tables.", + system_prompt_from=[{"path": "prompt.json", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written diff --git a/tests/harness/test_contract_tools_subfolder.py b/tests/harness/test_contract_tools_subfolder.py index 0e06e7ff4..3412b5684 100644 --- a/tests/harness/test_contract_tools_subfolder.py +++ b/tests/harness/test_contract_tools_subfolder.py @@ -1772,12 +1772,14 @@ def test_a_tool_module_through_links_is_charged_to_the_saves_budget( def test_a_prompt_path_named_again_is_resolved_once(tmp_path, repo, monkeypatch): - from fi.alk.harness import tools + from fi.alk.harness import model_citation (repo / "agents" / "a" / "prompt.txt").write_text("Be polite.\nBe brief.\n") resolved: list[Path] = [] - real = tools.charged_path - monkeypatch.setattr(tools, "charged_path", lambda path: resolved.append(path) or real(path)) + real = model_citation.charged_path + monkeypatch.setattr( + model_citation, "charged_path", lambda path: resolved.append(path) or real(path) + ) result = accept_contract( payload( From 3a053176f60b90752972c5ecf037b80b3ee8dbbe Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 18:59:29 +0530 Subject: [PATCH 03/10] fix(harness): answer Dileep's second review of the model and prompt checks (TH-8953) - The named prompt lines must hold every piece of the quote, in order; the quote is split at ... and compared by letters and digits after escapes, HTML entities, NFKC and case folding. The 80-letter window and the length check go. - A refused quote is held for the reading, so lines named later without a quote are still checked against it. - Settings: capitals need an underscore to read as an environment variable; a setting word among the name's parts counts when it has an underscore or no digit; a dotted name with no digit is a setting. GPT4O, O1, mistral_large and gpt4o-deployment are models again. - Lookups: more lookup calls, spaces before the bracket, a generic type. - Comments: one scan that skips quoted strings; markers without spaces, a lone --, and block-comment middle lines count. - Prose files: backticks are not quotes. Makefile, .mk, .flaskenv and ${VAR:-default} are configuration. - read_source opens with O_NONBLOCK, O_NOFOLLOW and O_CLOEXEC, checks S_ISREG on the descriptor, reads to the limit in a loop and logs a failure by path and error kind. #225's two prompt-reader tests now watch os.open and os.read. Signed-off-by: khushalsonawat --- src/fi/alk/harness/model_citation.py | 213 ++++++--- .../harness/skills/understand-agent/SKILL.md | 3 +- src/fi/alk/harness/tools.py | 15 +- tests/harness/test_contract_model.py | 446 +++++++++++++++++- .../harness/test_contract_tools_subfolder.py | 50 +- 5 files changed, 637 insertions(+), 90 deletions(-) diff --git a/src/fi/alk/harness/model_citation.py b/src/fi/alk/harness/model_citation.py index d8eb35a5c..a0757ff62 100644 --- a/src/fi/alk/harness/model_citation.py +++ b/src/fi/alk/harness/model_citation.py @@ -10,14 +10,20 @@ from __future__ import annotations +import html +import logging +import os import re +import stat import string +import unicodedata from pathlib import Path from .contract import AgentContract from .runtime_selection import charged_path from .source_folder import is_within +_LOG = logging.getLogger(__name__) # The platform refuses the whole agent document past this, for the model's name and provider. LONGEST_MODEL_FIELD = 255 # One path as written: charged_path doesn't refuse a long path itself. @@ -32,18 +38,46 @@ CONFIG_SUFFIXES = frozenset( { ".yaml", ".yml", ".toml", ".json", ".ini", ".cfg", ".conf", ".env", ".properties", - ".sh", ".bash", ".zsh", ".dockerfile", + ".sh", ".bash", ".zsh", ".dockerfile", ".mk", } ) +_CONFIG_NAMES = frozenset({"makefile", "gnumakefile", ".flaskenv"}) _YAML_SUFFIXES = frozenset({".yaml", ".yml"}) -# Names a setting goes by, never a model: `cfg.get("model")`, `os.getenv("llm")`. -_SETTING_WORDS = frozenset({"model", "llm", "name", "default", "engine", "deployment", "model_name"}) -_QUOTES = frozenset("\"'`") +# Files written for people: a backticked word there names something, it doesn't set it. +PROSE_SUFFIXES = frozenset({".md", ".markdown", ".rst", ".txt", ".adoc"}) +# Words a setting is named with, never a model: `cfg.get("model")`, OPENAI_MODEL, llm.model. +_SETTING_WORDS = frozenset( + {"model", "llm", "name", "default", "engine", "deployment", "provider", "key", "env", "id"} +) +_CODE_QUOTES = frozenset("\"'`") +_PROSE_QUOTES = frozenset("\"'") _SETTING_NAME_CHARS = frozenset(string.ascii_uppercase + string.digits + "_") +_SETTING_PARTS = re.compile(r"[_\-.:/@]") # A \uXXXX or \xXX escape in a string, as json.dump writes any non-ASCII character. _CHAR_ESCAPE = re.compile(r"\\(?:u([0-9a-fA-F]{4})|x([0-9a-fA-F]{2}))") -# How much of a quote the named lines must hold, once spacing is set aside. -_QUOTE_START = 80 +# Where a quote leaves text out: ... or …, also inside brackets, which only letters are compared. +_ELISION = re.compile(r"\.{3,}|…") + + +def _comment_finder(quotes: frozenset[str]) -> re.Pattern[str]: + """One pass over a line: a quoted string is matched whole (to the line's end if it isn't + closed), so a marker inside it is passed over; the first marker outside a string starts the + comment. The string alternatives can't overlap, so the search never backtracks.""" + strings = [rf"{re.escape(q)}(?:[^{re.escape(q)}\\]|\\.)*{re.escape(q)}?" for q in sorted(quotes)] + markers = [ + # Not $# (a shell count) or &# (an HTML character). + r"(? Path | None: @@ -57,17 +91,44 @@ def resolved_in_source(root: Path, path: str) -> Path | None: return resolved if is_within(resolved, root) else None +_OPEN_FLAGS = ( + os.O_RDONLY + # A pipe or a device put at the path after it was resolved is opened without waiting on it, + # then refused below; a link put there since isn't followed out of the source. + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) +) + + def read_source(path: Path, limit: int) -> bytes | None: """At most ``limit + 1`` bytes of the file, or None: not a regular file, or not readable. - The caller judges the size.""" + ``path`` is already resolved. The caller judges the size. A file that can't be read is + logged by its path and the error's kind, never its content.""" + try: + descriptor = os.open(path, _OPEN_FLAGS) + except (OSError, ValueError) as failed: + _LOG.info("source file not read: %s (%s)", path, type(failed).__name__) + return None try: - # Before the open: opening a FIFO or a device waits on it, or reads without end. - if not path.is_file(): + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + _LOG.info("source file not read: %s (not a regular file)", path) return None - with path.open("rb") as handle: - return handle.read(limit + 1) - except (OSError, ValueError): + chunks: list[bytes] = [] + held = 0 + # A read can return less than asked for; what counts is limit + 1 or the end. + while held <= limit: + chunk = os.read(descriptor, limit + 1 - held) + if not chunk: + break + chunks.append(chunk) + held += len(chunk) + return b"".join(chunks) + except OSError as failed: + _LOG.info("source file not read: %s (%s)", path, type(failed).__name__) return None + finally: + os.close(descriptor) def source_lines(raw: bytes) -> list[str]: @@ -76,21 +137,17 @@ def source_lines(raw: bytes) -> list[str]: return raw.decode("utf-8", errors="replace").splitlines() -def _comment_start(line: str) -> int: - """Where a comment starts on the line, or its length: a line that is a comment, or the first - ` #` or ` //`. Quotes aren't tracked, so a ` #` inside a string reads as a comment too; that - can only leave a model out, never record a wrong one.""" +def _comment_start(line: str, *, prose: bool) -> int: + """Where a comment starts on the line, or its length: a line that is a comment (also the + `* ` middle of a block comment, outside prose), or the first #, //, /*, ', "gpt-4o", id="markup"), + ], +) +def test_comments_without_spaces_are_comments(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +# Prose names models, it doesn't set them + + +@pytest.mark.parametrize( + "file, written", + [ + pytest.param("README.md", "This agent uses `gpt-4o` (we tried `claude-3-opus`)", id="readme"), + pytest.param("README.md", "| fallback | `claude-3-opus` |", id="table-cell"), + pytest.param("notes.txt", "tried `claude-3-opus`", id="text"), + pytest.param("index.rst", "uses `claude-3-opus` here", id="rst"), + ], +) +def test_backticks_in_prose_are_not_values(root, file, written): + (root / file).write_text(f"# notes\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "claude-3-opus", "path": file, "line": 2}) + + assert problems == [not_named("claude-3-opus", 2, file)] + + +def test_a_list_item_in_prose_is_not_a_comment(root): + (root / "README.md").write_text('# notes\n* default model: "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "README.md", "line": 2}) + + assert problems == [] + + +def test_backticks_in_code_are_values(root): + (root / "llm.ts").write_text("// the model\nconst model = `claude-3-opus`;\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "claude-3-opus", "path": "llm.ts", "line": 2}) + + assert problems == [] + + +@pytest.mark.parametrize( + "file, written", + [ + pytest.param("Makefile", "MODEL ?= gpt-4o", id="makefile"), + pytest.param("rules.mk", "MODEL := gpt-4o", id="make-include"), + pytest.param(".flaskenv", "MODEL=gpt-4o", id="flaskenv"), + pytest.param("run.sh", 'python run.py --model "${MODEL:-gpt-4o}"', id="shell-default"), + pytest.param(".env", "MODEL=${MODEL:-gpt-4o}", id="env-default"), + ], +) +def test_more_configuration_forms(root, file, written): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": "gpt-4o", "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == "gpt-4o" + + +def test_a_shell_default_is_only_found_in_configuration(root): + (root / "llm.py").write_text('# the model\nx = ${MODEL:-gpt-4o}\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "llm.py")] + + +# The reader never waits on a pipe, and leaves a trace when it can't read + + +def test_a_pipe_is_not_waited_on(tmp_path): + from fi.alk.harness.model_citation import read_source + + if not hasattr(os, "mkfifo"): + pytest.skip("no named pipes here") + pipe = tmp_path / "pipe" + os.mkfifo(pipe) + answer: list[bytes | None] = [] + reading = threading.Thread(target=lambda: answer.append(read_source(pipe, 10)), daemon=True) + reading.start() + reading.join(timeout=2) + + assert answer == [None] + + +def test_a_file_read_in_short_pieces_is_read_whole(tmp_path, monkeypatch): + from fi.alk.harness import model_citation + + (tmp_path / "agent.py").write_bytes(b"x" * 100) + real_read = os.read + monkeypatch.setattr(model_citation.os, "read", lambda fd, n: real_read(fd, min(n, 7))) + + assert model_citation.read_source(tmp_path / "agent.py", 50) == b"x" * 51 + assert model_citation.read_source(tmp_path / "agent.py", 200) == b"x" * 100 + + +def test_an_unreadable_file_is_logged_without_its_content(tmp_path, caplog): + from fi.alk.harness.model_citation import read_source + + locked = tmp_path / "locked.py" + locked.write_text('SECRET = "do-not-log"\n', encoding="utf-8") + locked.chmod(0) + try: + if os.access(locked, os.R_OK): + pytest.skip("this user can read a file with no permissions") + with caplog.at_level("INFO", logger="fi.alk.harness.model_citation"): + assert read_source(locked, 100) is None + finally: + locked.chmod(0o600) + + assert [record.getMessage() for record in caplog.records] == [ + f"source file not read: {locked} (PermissionError)" + ] + assert "do-not-log" not in caplog.text + + +def test_a_link_put_in_place_of_the_file_is_not_followed(tmp_path): + from fi.alk.harness.model_citation import read_source + + (tmp_path / "outside.py").write_text("SECRET\n", encoding="utf-8") + link("outside.py", tmp_path / "agent.py") + + assert read_source(tmp_path / "agent.py", 100) is None + + def test_mismatched_quotes_are_not_a_value(root): (root / "llm.py").write_text("# settings\nmodel = \"gpt-4o'\n", encoding="utf-8") @@ -1116,9 +1351,11 @@ def test_honest_prompt_forms_hold_the_quote(tmp_path, root, file, written): @pytest.mark.parametrize( "quote, refused", [ - # The first 80 letters match and the rest of the quote differs: a quote may trail off. - pytest.param(_LONG_PROMPT[:120] + " and something else entirely", False, id="same-start"), - # The quote differs from the lines within its first 80 letters. + # A quote that says where it trails off fits. + pytest.param(_LONG_PROMPT[:120] + " ...", False, id="trails-off-with-dots"), + # The start matches and the rest differs: every part of the quote must be in the lines. + pytest.param(_LONG_PROMPT[:120] + " and something else entirely", True, id="differs-late"), + # The quote differs from the lines near its start. pytest.param("You are pair, a helpful assistant who books rooms for guests.", True, id="differs-early"), # The lines hold the quote's start but are shorter than the quote. pytest.param(_LONG_PROMPT * 3, True, id="lines-shorter-than-the-quote"), @@ -1168,6 +1405,177 @@ def test_lines_refused_twice_leave_the_quote(tmp_path, root): assert saved(destination)["system_prompt_from"] == [] +# Every piece of the quote must be in the lines named, in order + +_BOOKING = ( + "import os\n" + "import json\n" + "from acme.tools import book\n" + "from acme.tools import cancel\n" + "\n" + "PROMPT = (\n" + ' "You are Acme\'s booking agent. "\n' + ' "Help guests book tables. "\n' + ' "Never share other guests\' bookings. "\n' + ' "Confirm every booking before saving it. "\n' + ' "Be warm and brief."\n' + ")\n" +) +_BOOKING_QUOTE = ( + "You are Acme's booking agent. Help guests book tables. Never share other guests' bookings. " + "Confirm every booking before saving it. Be warm and brief." +) + + +@pytest.mark.parametrize( + "quote, lines, refused", + [ + # Starting early pads the range, but its last line is still missing. + pytest.param(_BOOKING_QUOTE, (1, 10), True, id="starts-early-stops-short"), + pytest.param(_BOOKING_QUOTE, (6, 12), False, id="the-prompt"), + pytest.param("You are Acme's booking agent. ... Be warm and brief.", (6, 12), False, id="elided"), + pytest.param("You are Acme's booking agent. ... Be warm and brief.", (1, 10), True, id="elided-stops-short"), + pytest.param("Be warm and brief. ... You are Acme's booking agent.", (6, 12), True, id="out-of-order"), + pytest.param("You are Acme's booking agent. [...] Be warm and brief.", (6, 12), False, id="bracketed-dots"), + pytest.param("You are Acme's booking agent. … Be warm and brief.", (6, 12), False, id="ellipsis-character"), + pytest.param("YOU ARE ACME'S BOOKING AGENT. ... BE WARM AND BRIEF.", (6, 12), False, id="case"), + pytest.param("... Be warm and brief.", (6, 12), False, id="starts-with-dots"), + pytest.param("...", (1, 4), False, id="only-dots"), + # One line can't hold a piece the quote has twice. + pytest.param("Be warm and brief. ... Be warm and brief.", (6, 12), True, id="a-piece-twice"), + ], +) +def test_a_range_that_stops_short_is_refused(tmp_path, root, quote, lines, refused): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "agent.py", "start_line": lines[0], "end_line": lines[1]}], + ), + destination, + source_root=root, + ) + + if refused: + assert text_of(result) == REFUSED + not_the_quote(f"agent.py {lines[0]}-{lines[1]}") + else: + assert not result.get("is_error"), result + copied = "\n".join(_BOOKING.splitlines()[lines[0] - 1 : lines[1]]) + assert saved(destination)["system_prompt_excerpt"] == copied + + +@pytest.mark.parametrize( + "written, quote", + [ + pytest.param('PROMPT = "Tom & Jerry\'s"', "Tom & Jerry's", id="entity-in-the-source"), + pytest.param('PROMPT = "Tom & Jerry\'s"', "Tom & Jerry's", id="entity-in-the-quote"), + pytest.param('PROMPT = "Order a café"', "Order a café", id="decomposed-accent"), + ], +) +def test_an_honest_quote_in_another_form_fits(tmp_path, root, written, quote): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written + + +def test_lines_sent_without_the_refused_quote_are_still_checked(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=imports, + ) + ) + assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 1-4")) + + # The quote left out: the imports are still checked against it, and the quote is what's sent. + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["system_prompt_from"] == [] + + +def test_the_right_lines_after_a_refusal_are_copied(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + ) + assert first.get("is_error"), first + + second = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}], + ) + ) + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + + +def test_a_new_quote_replaces_the_refused_one(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant.", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + ) + assert first.get("is_error"), first + + second = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}], + ) + ) + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + + +def test_a_quote_of_many_pieces_does_not_hold_up_the_check(root): + from fi.alk.harness.model_citation import quote_problem + + copied = "a" * MIB + started = time.perf_counter() + answer = quote_problem("a ... " * 50_000 + "b", copied, "agent.py 1-1") + took = time.perf_counter() - started + + assert answer is not None + assert took < 0.5 + + def test_leaving_the_quote_out_is_not_asked_about(tmp_path, root): # Following the advice to leave the quote out must not set off the "no prompt" question. (root / "prompt.py").write_text('PROMPT = "You are pair."\n', encoding="utf-8") @@ -1206,10 +1614,18 @@ def test_a_prompt_path_through_a_link_outside_is_refused(tmp_path, root): @pytest.mark.parametrize( "written, name", [ - pytest.param('MODEL = os.getenv("model-name")', "model-name", id="getenv-hyphenated"), - pytest.param('MODEL = cfg.get("llm.model")', "llm.model", id="get-dotted"), - pytest.param('MODEL = os.getenv("openai_model_v2")', "openai_model_v2", id="getenv-with-a-digit"), - pytest.param('MODEL = os.environ["llm-model"]', "llm-model", id="subscript"), + # Keys shaped like model names, so the setting-shape rule passes them and the lookup is + # what refuses them. + pytest.param('MODEL = os.getenv("fast-4")', "fast-4", id="getenv-hyphenated"), + pytest.param('MODEL = cfg.get("chat.v2")', "chat.v2", id="get-dotted"), + pytest.param('MODEL = os.getenv("gpt_4o")', "gpt_4o", id="getenv-with-a-digit"), + pytest.param('MODEL = os.environ["gpt-4o"]', "gpt-4o", id="subscript"), + pytest.param('model = this.config.get("gpt-4o");', "gpt-4o", id="get-generic"), + pytest.param('model = configService.getOrThrow("gpt-4o");', "gpt-4o", id="get-or-throw"), + pytest.param('model = props.getProperty("gpt-4o");', "gpt-4o", id="get-property"), + pytest.param('MODEL = os.environ.get ("gpt-4o")', "gpt-4o", id="space-before-the-bracket"), + pytest.param('MODEL = env.str("gpt-4o")', "gpt-4o", id="env-str"), + pytest.param("$model = config('gpt-4o');", "gpt-4o", id="config-call"), ], ) def test_a_lookup_key_is_not_the_model(root, written, name): diff --git a/tests/harness/test_contract_tools_subfolder.py b/tests/harness/test_contract_tools_subfolder.py index 72423fb7c..581cd0346 100644 --- a/tests/harness/test_contract_tools_subfolder.py +++ b/tests/harness/test_contract_tools_subfolder.py @@ -1173,25 +1173,32 @@ def test_a_strings_length_is_charged_though_it_builds_nothing(repo, copies, refu def test_a_prompt_file_without_end_is_read_only_to_its_limit(tmp_path, repo, monkeypatch): - import io - - class Endless(io.RawIOBase): - def readable(self): - return True + import os - def read(self, size=-1): - assert size >= 0, "read to the end" - return b"#" * size + from fi.alk.harness import model_citation (repo / "agents" / "a" / "prompt.txt").write_text("") - opened = Path.open - monkeypatch.setattr( - Path, - "open", - lambda path, *args, **kwargs: Endless() - if path.name == "prompt.txt" - else opened(path, *args, **kwargs), - ) + # The prompt is read through model_citation.read_source, by descriptor. + endless: set[int] = set() + real_open, real_read, real_close = os.open, os.read, os.close + + def open_(path, flags, *args, **kwargs): + descriptor = real_open(path, flags, *args, **kwargs) + if Path(path).name == "prompt.txt": + endless.add(descriptor) + return descriptor + + def read(descriptor, size): + # Like a link to /proc/self/pagemap: its size says 0 and it reads without end. + return b"#" * size if descriptor in endless else real_read(descriptor, size) + + def close(descriptor): + endless.discard(descriptor) + real_close(descriptor) + + monkeypatch.setattr(model_citation.os, "open", open_) + monkeypatch.setattr(model_citation.os, "read", read) + monkeypatch.setattr(model_citation.os, "close", close) result = accept_contract( payload( @@ -1443,10 +1450,17 @@ def test_a_prompt_file_named_again_is_not_read_again(tmp_path, repo): def test_a_prompt_file_past_the_limit_is_read_once(tmp_path, repo, monkeypatch): large = repo / "agents" / "a" / "p.txt" large.write_text("Be brief.\n" + "#" * (1024 * 1024)) + import os + + from fi.alk.harness import model_citation + opened: list[Path] = [] - real = Path.open + real = os.open + # The prompt is opened through model_citation.read_source, by descriptor. monkeypatch.setattr( - Path, "open", lambda path, *args, **kwargs: opened.append(path) or real(path, *args, **kwargs) + model_citation.os, + "open", + lambda path, *args, **kwargs: opened.append(Path(path)) or real(path, *args, **kwargs), ) result = accept_contract( From 215c59fb6edc1ef2c12a06591a343d392d692322 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 20:47:43 +0530 Subject: [PATCH 04/10] fix(harness): answer review rounds 1 and 2 of the second-review fixes, and close two known limits (TH-8953) Local, not pushed: review round 3 did not clear (two Highs, reports in the ticket folder); the push waits on Khushal's call. - The quote's words are matched in order within 24 letters of each other, any gap at "...", with a charged budget; the reading's latest quote is held at the top of submit_contract; lines sent alone are refused once more, then the quote is kept and the lines recorded in unconfirmed_prompt_from. - One line scanner for strings, comments and triple-quoted strings, per language; prose and prompt templates hold no model; more lookups and configuration forms; the reader logs the error kind only. Signed-off-by: khushalsonawat --- src/fi/alk/harness/contract.py | 6 + src/fi/alk/harness/model_citation.py | 429 +++++-- .../harness/skills/understand-agent/SKILL.md | 2 +- src/fi/alk/harness/tools.py | 75 +- tests/harness/test_contract_model.py | 1023 ++++++++++++++++- .../harness/test_contract_tools_subfolder.py | 8 +- tests/test_harness.py | 2 +- 7 files changed, 1409 insertions(+), 136 deletions(-) diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index e6936dcbe..fd0ffee1e 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -1142,6 +1142,9 @@ def _normalize_shapes(cls, payload: Any) -> Any: system_prompt_excerpt: str = "" # When given, the excerpt is replaced by these lines copied from the source as written. system_prompt_from: list[PromptSource] = Field(default_factory=list) + # Set by the save, never by the reading: lines named for the prompt that twice didn't hold + # the quote given for it, so the quote went out instead. Empty for nearly every contract. + unconfirmed_prompt_from: list[PromptSource] = Field(default_factory=list) # None is unknown. Kept only when its name was found on the line cited in the source. model: ModelSource | None = None hard_constraints: list[str] = Field(default_factory=list) @@ -1185,6 +1188,9 @@ def _omit_absent_model(self, handler: Any) -> dict[str, Any]: data = handler(self) if data.get("model") is None: data.pop("model", None) + # The same for the record of unconfirmed prompt lines, kept only when there are some. + if not data.get("unconfirmed_prompt_from"): + data.pop("unconfirmed_prompt_from", None) return data def tool_names(self) -> set[str]: diff --git a/src/fi/alk/harness/model_citation.py b/src/fi/alk/harness/model_citation.py index a0757ff62..2b88eacf8 100644 --- a/src/fi/alk/harness/model_citation.py +++ b/src/fi/alk/harness/model_citation.py @@ -38,46 +38,201 @@ CONFIG_SUFFIXES = frozenset( { ".yaml", ".yml", ".toml", ".json", ".ini", ".cfg", ".conf", ".env", ".properties", - ".sh", ".bash", ".zsh", ".dockerfile", ".mk", + ".sh", ".bash", ".zsh", ".dockerfile", ".mk", ".prompt", } ) _CONFIG_NAMES = frozenset({"makefile", "gnumakefile", ".flaskenv"}) +# Languages where # isn't a comment: C and C++ (#define), JavaScript and TypeScript (#field), +# Java, Go, Rust (#[attr]), Swift (#if), Kotlin, C#, Scala, Dart, Objective-C. +_NO_HASH_COMMENT_SUFFIXES = frozenset( + { + ".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", + ".java", ".go", ".rs", ".swift", ".kt", ".kts", ".cs", ".scala", ".dart", ".m", ".mm", + } +) _YAML_SUFFIXES = frozenset({".yaml", ".yml"}) -# Files written for people: a backticked word there names something, it doesn't set it. -PROSE_SUFFIXES = frozenset({".md", ".markdown", ".rst", ".txt", ".adoc"}) -# Words a setting is named with, never a model: `cfg.get("model")`, OPENAI_MODEL, llm.model. -_SETTING_WORDS = frozenset( - {"model", "llm", "name", "default", "engine", "deployment", "provider", "key", "env", "id"} +# INI files: a line starting with ; is a comment. +_INI_SUFFIXES = frozenset({".ini", ".cfg", ".conf"}) +# Files where -- starts a comment: SQL, Lua, Haskell. +_DASH_COMMENT_SUFFIXES = frozenset({".sql", ".lua", ".hs"}) +# Files written for people: they say which model is used; they don't set it. +# And prompt templates (.j2, .hbs): their text is the agent's prompt, which may name a model +# without setting it. +PROSE_SUFFIXES = frozenset( + { + ".md", ".markdown", ".mdx", ".mdc", ".rst", ".txt", ".adoc", ".org", ".j2", ".jinja", + ".jinja2", ".hbs", ".mustache", + } ) -_CODE_QUOTES = frozenset("\"'`") -_PROSE_QUOTES = frozenset("\"'") +# Files written for people, by name: README, CHANGELOG, LICENSE, CONTRIBUTING, with or without a +# suffix (not readme_writer.py). +_PROSE_STEMS = frozenset({"readme", "changelog", "license", "contributing"}) +# Words a setting is named with, never a model: `cfg.get("model")`, OPENAI_MODEL, llm.model. +# Not "deployment": an Azure deployment's own name often has it (gpt-4o_deployment). +_SETTING_WORDS = frozenset({"model", "llm", "name", "default", "engine", "provider", "key", "env", "id"}) +_QUOTES = frozenset("\"'`") _SETTING_NAME_CHARS = frozenset(string.ascii_uppercase + string.digits + "_") _SETTING_PARTS = re.compile(r"[_\-.:/@]") -# A \uXXXX or \xXX escape in a string, as json.dump writes any non-ASCII character. -_CHAR_ESCAPE = re.compile(r"\\(?:u([0-9a-fA-F]{4})|x([0-9a-fA-F]{2}))") -# Where a quote leaves text out: ... or …, also inside brackets, which only letters are compared. +# An escape in a string: \uXXXX and \xXX (as json.dump writes any non-ASCII character), +# \UXXXXXXXX, \N{NAME} and octal. +_CHAR_ESCAPE = re.compile( + r"\\(?:u([0-9a-fA-F]{4})|x([0-9a-fA-F]{2})|U([0-9a-fA-F]{8})|N\{([A-Za-z0-9 \-]{1,88})\}" + r"|([0-7]{1,3}))" +) +_NOT_A_LETTER = re.compile(r"[\W_]+") +# Full-width letters and digits (3, A) as their ASCII forms: what NFKC would do for them, +# without NFKC's growth elsewhere. +_FULL_WIDTH = {code: code - 0xFEE0 for code in range(0xFF01, 0xFF5F)} +# Scripts written without spaces between words (Thai, Lao, Burmese, Khmer, Japanese kana, +# Chinese and Japanese ideographs): each character is a word, so the gap below applies between +# characters there too. +_UNSPACED = "\u0e00-\u0eff\u1000-\u109f\u1780-\u17ff\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff" +_UNSPACED_CHAR = re.compile(f"[{_UNSPACED}]") +_WORD = re.compile(f"[{_UNSPACED}]|[^\\W_{_UNSPACED}]+") +# A quote of fewer letters than this says nothing to check lines against ("n/a", "TBD", "see +# lines"): a few short words are found in almost any lines. A character of an unspaced script +# counts as three. +_SHORTEST_QUOTE = 16 +# Languages with triple-quoted strings ("""...""" or '''...'''): Python, TOML, Kotlin, Scala, +# Swift, Groovy and Gradle, Java and C# text blocks, Julia, Dart, Elixir. +_TRIPLE_SUFFIXES = frozenset( + { + ".py", ".pyi", ".pyw", ".toml", ".kt", ".kts", ".scala", ".sc", ".swift", ".groovy", + ".gradle", ".java", ".cs", ".jl", ".dart", ".ex", ".exs", + } +) +# Where a quote leaves text out: ... or …, also inside brackets, since only letters are compared. _ELISION = re.compile(r"\.{3,}|…") - - -def _comment_finder(quotes: frozenset[str]) -> re.Pattern[str]: - """One pass over a line: a quoted string is matched whole (to the line's end if it isn't - closed), so a marker inside it is passed over; the first marker outside a string starts the - comment. The string alternatives can't overlap, so the search never backtracks.""" - strings = [rf"{re.escape(q)}(?:[^{re.escape(q)}\\]|\\.)*{re.escape(q)}?" for q in sorted(quotes)] - markers = [ - # Not $# (a shell count) or &# (an HTML character). - r"(? re.Pattern[str]: + """What the line scan stops at: a triple quote (in languages that have them), a + double-quoted or backticked string, matched whole (to the line's end if it isn't closed); an + apostrophe, which the scan pairs itself; or a comment marker. # not in $# (a shell count), + &# (an HTML character) or ${# (a shell length); // not in a URL (http://), floor division + (x//2) or after a name; /* not in a glob (data/*).""" + alternatives = ['"""', "'''"] if triple else [] + alternatives += [r'"(?:[^"\\]|\\.)*"?', r"`(?:[^`\\]|\\.)*`?", "'"] + if hash_comments == _HASH_ANYWHERE: + alternatives.append(r"(? bool: + """Whether the apostrophe at ``at`` can open a string: not inside a word (Pair's, it's, x' + in Haskell) and not after a backslash (it\\'s), unless the word is a string prefix (r'...', + rb'...'). Looks at three characters at most.""" + start = at + while start > 0 and at - start < 3 and line[start - 1].isalnum(): + start -= 1 + if start == at: + return at == 0 or line[at - 1] != "\\" + whole = start == 0 or not (line[start - 1].isalnum() or line[start - 1] == "_") + return whole and line[start:at].lower() in _STRING_PREFIXES + + +def _line_scan( + line: str, + block: str | None, + *, + dashes: bool = False, + ini: bool = False, + triple: bool = False, + hash_comments: str = _HASH_ANYWHERE, +) -> tuple[int, list[tuple[int, int]], str | None]: + """One pass over a line: where its comment starts (or its length), the stretches of it that + are text (inside a triple-quoted string, or a block comment closed on the line), and the + triple quote still open at its end. ``block`` is the one open where the line starts. A line + is a comment when it is the `* ` middle of a block comment or, + in INI, starts with ;. Strings are passed over whole, so a marker or a triple quote inside one + doesn't count; an apostrophe opens a string only when another closes it on the line.""" + inside: list[tuple[int, int]] = [] + at = 0 + if block is not None: + close = line.find(block) + if close < 0: + return len(line), [(0, len(line))], block + inside.append((0, close)) + at = close + 3 + else: + stripped = line.lstrip() + if stripped.startswith(("* ", "*\t")) or (ini and stripped.startswith(";")): + return 0, inside, None + tokens = _TOKENS[dashes, triple, hash_comments] + while (found := tokens.search(line, at)) is not None: + token, start = found.group(), found.start() + at = found.end() + if token in ('"""', "'''"): + close = line.find(token, at) + if close < 0: + inside.append((at, len(line))) + return len(line), inside, token + inside.append((at, close)) + at = close + 3 + elif token == "'": + if _opens_a_string(line, start): + # Unclosed, it is an apostrophe: every later one is then after a backslash, so + # none of them opens a string either, and the line is read about once. + closed = _REST_OF_SINGLE.match(line, at) + if closed is not None: + at = closed.end() + elif token in ("/*", "", at) + if close < 0: + return start, inside, None + # From the marker itself, so the stretch can never be a quoted value. + inside.append((start, close)) + at = close + (2 if token == "/*" else 3) + elif token[0] not in "\"`": + return start, inside, None + return len(line), inside, None + + +def block_open_before( + lines: list[str], *, dashes: bool = False, ini: bool = False, hash_comments: str = _HASH_ANYWHERE +) -> str | None: + """The triple quote open where the line after ``lines`` starts, the lines read with the same + scan as the cited one. A line with no triple quote, outside a block, is passed over without + a scan; inside one, only the closing quote is looked for.""" + block: str | None = None + for line in lines: + if block is None and '"""' not in line and "'''" not in line: + continue + if block is not None and block not in line: + continue + _, _, block = _line_scan( + line, block, dashes=dashes, ini=ini, triple=True, hash_comments=hash_comments + ) + return block def resolved_in_source(root: Path, path: str) -> Path | None: @@ -94,7 +249,7 @@ def resolved_in_source(root: Path, path: str) -> Path | None: _OPEN_FLAGS = ( os.O_RDONLY # A pipe or a device put at the path after it was resolved is opened without waiting on it, - # then refused below; a link put there since isn't followed out of the source. + # then refused below; a link put in the path's last part since isn't followed. | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) @@ -104,15 +259,15 @@ def resolved_in_source(root: Path, path: str) -> Path | None: def read_source(path: Path, limit: int) -> bytes | None: """At most ``limit + 1`` bytes of the file, or None: not a regular file, or not readable. ``path`` is already resolved. The caller judges the size. A file that can't be read is - logged by its path and the error's kind, never its content.""" + logged by the error's kind alone: its path and content are the customer's.""" try: descriptor = os.open(path, _OPEN_FLAGS) except (OSError, ValueError) as failed: - _LOG.info("source file not read: %s (%s)", path, type(failed).__name__) + _LOG.info("source file not read (%s)", type(failed).__name__) return None try: if not stat.S_ISREG(os.fstat(descriptor).st_mode): - _LOG.info("source file not read: %s (not a regular file)", path) + _LOG.info("source file not read (not a regular file)") return None chunks: list[bytes] = [] held = 0 @@ -125,7 +280,7 @@ def read_source(path: Path, limit: int) -> bytes | None: held += len(chunk) return b"".join(chunks) except OSError as failed: - _LOG.info("source file not read: %s (%s)", path, type(failed).__name__) + _LOG.info("source file not read (%s)", type(failed).__name__) return None finally: os.close(descriptor) @@ -137,19 +292,6 @@ def source_lines(raw: bytes) -> list[str]: return raw.decode("utf-8", errors="replace").splitlines() -def _comment_start(line: str, *, prose: bool) -> int: - """Where a comment starts on the line, or its length: a line that is a comment (also the - `* ` middle of a block comment, outside prose), or the first #, //, /*, ', "gpt-4o", id="markup"), @@ -1182,24 +1207,22 @@ def test_comments_without_spaces_are_comments(root, file, written, name): [ pytest.param("README.md", "This agent uses `gpt-4o` (we tried `claude-3-opus`)", id="readme"), pytest.param("README.md", "| fallback | `claude-3-opus` |", id="table-cell"), + pytest.param("README.md", 'This agent uses "gpt-4o" (we tried "claude-3-opus")', id="straight-quotes"), + pytest.param("README.md", '* default model: "claude-3-opus"', id="list-item"), pytest.param("notes.txt", "tried `claude-3-opus`", id="text"), pytest.param("index.rst", "uses `claude-3-opus` here", id="rst"), + pytest.param("guide.markdown", 'model: "claude-3-opus"', id="markdown"), + pytest.param("guide.adoc", 'model: "claude-3-opus"', id="asciidoc"), + pytest.param("page.mdx", 'model: "claude-3-opus"', id="mdx"), + pytest.param("README", 'model: "claude-3-opus"', id="readme-without-a-suffix"), ], ) -def test_backticks_in_prose_are_not_values(root, file, written): +def test_prose_files_hold_no_model(root, file, written): (root / file).write_text(f"# notes\n{written}\n", encoding="utf-8") problems, _, _ = checked(root, {"name": "claude-3-opus", "path": file, "line": 2}) - assert problems == [not_named("claude-3-opus", 2, file)] - - -def test_a_list_item_in_prose_is_not_a_comment(root): - (root / "README.md").write_text('# notes\n* default model: "gpt-4o"\n', encoding="utf-8") - - problems, _, _ = checked(root, {"name": "gpt-4o", "path": "README.md", "line": 2}) - - assert problems == [] + assert problems == [prose(file)] def test_backticks_in_code_are_values(root): @@ -1281,8 +1304,9 @@ def test_an_unreadable_file_is_logged_without_its_content(tmp_path, caplog): locked.chmod(0o600) assert [record.getMessage() for record in caplog.records] == [ - f"source file not read: {locked} (PermissionError)" + "source file not read (PermissionError)" ] + assert str(locked) not in caplog.text assert "do-not-log" not in caplog.text @@ -1396,13 +1420,15 @@ def test_lines_refused_twice_leave_the_quote(tmp_path, root): assert text_of(first) == ( REFUSED + not_the_quote("prompt.py 1-2") - + "\n - system_prompt_from: if the lines are refused again, the quote is kept and the " - "lines are left out" + + "\n - system_prompt_from: lines named later are checked against this quote until you " + "give another; given again with lines that don't hold it, the quote is kept and the lines " + "are recorded as unconfirmed" ) second = submit(wrong) assert not second.get("is_error"), second assert saved(destination)["system_prompt_excerpt"] == "You are pair, a helpful assistant." assert saved(destination)["system_prompt_from"] == [] + assert saved(destination)["unconfirmed_prompt_from"] == wrong["system_prompt_from"] # Every piece of the quote must be in the lines named, in order @@ -1439,7 +1465,7 @@ def test_lines_refused_twice_leave_the_quote(tmp_path, root): pytest.param("You are Acme's booking agent. [...] Be warm and brief.", (6, 12), False, id="bracketed-dots"), pytest.param("You are Acme's booking agent. … Be warm and brief.", (6, 12), False, id="ellipsis-character"), pytest.param("YOU ARE ACME'S BOOKING AGENT. ... BE WARM AND BRIEF.", (6, 12), False, id="case"), - pytest.param("... Be warm and brief.", (6, 12), False, id="starts-with-dots"), + pytest.param("... Confirm every booking before saving it. Be warm and brief.", (6, 12), False, id="starts-with-dots"), pytest.param("...", (1, 4), False, id="only-dots"), # One line can't hold a piece the quote has twice. pytest.param("Be warm and brief. ... Be warm and brief.", (6, 12), True, id="a-piece-twice"), @@ -1470,9 +1496,9 @@ def test_a_range_that_stops_short_is_refused(tmp_path, root, quote, lines, refus @pytest.mark.parametrize( "written, quote", [ - pytest.param('PROMPT = "Tom & Jerry\'s"', "Tom & Jerry's", id="entity-in-the-source"), - pytest.param('PROMPT = "Tom & Jerry\'s"', "Tom & Jerry's", id="entity-in-the-quote"), - pytest.param('PROMPT = "Order a café"', "Order a café", id="decomposed-accent"), + pytest.param('PROMPT = "Book at Tom & Jerry\'s diner tonight"', "Book at Tom & Jerry's diner tonight", id="entity-in-the-source"), + pytest.param('PROMPT = "Book at Tom & Jerry\'s diner tonight"', "Book at Tom & Jerry's diner tonight", id="entity-in-the-quote"), + pytest.param('PROMPT = "Order a caf\u00e9 and wait for the bill"', "Order a cafe\u0301 and wait for the bill", id="decomposed-accent"), ], ) def test_an_honest_quote_in_another_form_fits(tmp_path, root, written, quote): @@ -1508,11 +1534,17 @@ def test_lines_sent_without_the_refused_quote_are_still_checked(tmp_path, root): ) assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 1-4")) - # The quote left out: the imports are still checked against it, and the quote is what's sent. + # The quote left out: the imports are still checked against it, asked about once more... second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) - assert not second.get("is_error"), second + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) + assert not (destination / "contract.json").exists() + + # ... and then the quote is what's sent, with the lines on record as unconfirmed. + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert not third.get("is_error"), third assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE assert saved(destination)["system_prompt_from"] == [] + assert saved(destination)["unconfirmed_prompt_from"] == imports def test_the_right_lines_after_a_refusal_are_copied(tmp_path, root): @@ -1576,6 +1608,134 @@ def test_a_quote_of_many_pieces_does_not_hold_up_the_check(root): assert took < 0.5 +# A prompt written with Python string prefixes holds its quote + + +@pytest.mark.parametrize( + "written", + [ + pytest.param('PROMPT = ("You are pair, " f"a helpful assistant who books tables.")', id="f-string-piece"), + pytest.param('PROMPT = ("You are pair, " r"a helpful assistant who books tables.")', id="raw-piece"), + pytest.param("PROMPT = ('You are pair, ' rb'a helpful assistant who books tables.')", id="two-letter-prefix"), + pytest.param('PROMPT = ("You are pair, " F"a helpful assistant who books tables.")', id="capital-prefix"), + pytest.param('PROMPT = f"You are pair, " "a helpful assistant who books tables."', id="prefix-on-the-first-piece"), + pytest.param('PROMPT = (u"You are pair, " fr"a helpful assistant who books tables.")', id="u-and-fr"), + ], +) +def test_a_prompt_with_string_prefixes_holds_the_quote(tmp_path, root, written): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant who books tables.", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written + + +@pytest.mark.parametrize( + "written, quote, refused", + [ + # A word ending in a prefix letter keeps that letter, on both sides. + pytest.param('PROMPT = "He said" " hello and goodbye."', 'He said hello and goodbye.', False, id="word-before-a-quote"), + pytest.param('PROMPT = "Pick one of" "f the options."', "Pick one off the options.", False, id="prefix-letter-inside-a-string"), + ], +) +def test_a_word_before_a_quote_is_not_a_prefix(tmp_path, root, written, quote, refused): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + if refused: + assert text_of(result) == REFUSED + not_the_quote("prompt.py 1-1") + else: + assert not result.get("is_error"), result + + +# A value inside a docstring or a multi-line string is not the model + +_DOCSTRING = ( + "def build():\n" + ' """Build the agent.\n' + "\n" + ' Example: model = "gpt-4o"\n' + ' """\n' + ' return Agent(model="claude-3-opus")\n' +) + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("llm.py", _DOCSTRING, 4, "gpt-4o", id="docstring-line"), + pytest.param("llm.py", "X = '''\nmodel = \"gpt-4o\"\n'''\n", 2, "gpt-4o", id="single-quote-block"), + pytest.param("llm.py", 'PROMPT = """You are pair.\nUse "gpt-4o" for hard cases.\n"""\n', 2, "gpt-4o", id="prompt-line"), + pytest.param("llm.py", 'PROMPT = """Use "gpt-4o" for hard cases."""\n', 1, "gpt-4o", id="opened-on-the-line"), + pytest.param("llm.toml", 'notes = """\nmodel = "gpt-4o"\n"""\n', 2, "gpt-4o", id="toml-multiline"), + pytest.param("llm.py", '"""Docs with \'\'\' inside.\nmodel = "gpt-4o"\n"""\n', 2, "gpt-4o", id="other-kind-does-not-close"), + pytest.param("llm.py", 'X = """docs \'\'\' and "gpt-4o"\n"""\n', 1, "gpt-4o", id="other-kind-on-the-line-does-not-close"), + pytest.param("llm.py", 'X = """gpt-4o" is the default"""\n', 1, "gpt-4o", id="a-block-that-opens-at-the-name"), + ], +) +def test_a_value_inside_a_triple_quoted_block_is_not_the_model(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [not_named(name, line, file)] + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("llm.py", _DOCSTRING, 6, "claude-3-opus", id="closed-before"), + pytest.param("llm.py", 'DOC = """notes"""; MODEL = "gpt-4o"\n', 1, "gpt-4o", id="closed-on-the-line"), + pytest.param("llm.py", 'MODEL = """gpt-4o"""\n', 1, "gpt-4o", id="triple-quoted-value"), + pytest.param("llm.py", "MODEL = '''gpt-4o'''\n", 1, "gpt-4o", id="triple-single-quoted-value"), + pytest.param("llm.py", 'X = """a\nb"""\nMODEL = "gpt-4o"\n', 3, "gpt-4o", id="block-closed-on-an-earlier-line"), + ], +) +def test_a_value_after_a_closed_block_is_the_model(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "written", + [ + pytest.param('"""' * (MIB // 3 - 20) + '\nMODEL = "gpt-4o"\n', id="a-file-of-triple-quotes"), + pytest.param('"""' + '""" "gpt-4o" """' * (MIB // 20), id="a-line-of-blocks"), + ], +) +def test_a_crafted_file_of_triple_quotes_does_not_hold_up_the_check(root, written): + (root / "crafted.py").write_text(written, encoding="utf-8") + line = written.count("\n") + (0 if written.endswith("\n") else 1) + + _, _, took = checked(root, {"name": "gpt-4o", "path": "crafted.py", "line": line}) + + assert took < 0.5 + + def test_leaving_the_quote_out_is_not_asked_about(tmp_path, root): # Following the advice to leave the quote out must not set off the "no prompt" question. (root / "prompt.py").write_text('PROMPT = "You are pair."\n', encoding="utf-8") @@ -1626,6 +1786,10 @@ def test_a_prompt_path_through_a_link_outside_is_refused(tmp_path, root): pytest.param('MODEL = os.environ.get ("gpt-4o")', "gpt-4o", id="space-before-the-bracket"), pytest.param('MODEL = env.str("gpt-4o")', "gpt-4o", id="env-str"), pytest.param("$model = config('gpt-4o');", "gpt-4o", id="config-call"), + pytest.param('model = env.getOrDefault("gpt-4o", x);', "gpt-4o", id="get-or-default"), + pytest.param('model = bundle.getString("gpt-4o");', "gpt-4o", id="get-string"), + pytest.param('model = config.getValue("gpt-4o");', "gpt-4o", id="get-value"), + pytest.param("$model = env('gpt-4o');", "gpt-4o", id="env-call"), ], ) def test_a_lookup_key_is_not_the_model(root, written, name): @@ -1643,6 +1807,8 @@ def test_a_lookup_key_is_not_the_model(root, written, name): pytest.param('model = genai.GenerativeModel("gemini-1.5-pro")', "gemini-1.5-pro", id="constructor"), pytest.param('MODEL = os.getenv("OPENAI_MODEL", "gpt-4o")', "gpt-4o", id="getenv-default"), pytest.param('llm = target("gpt-4o")', "gpt-4o", id="a-call-ending-in-get"), + pytest.param('llm = my_get("gpt-4o")', "gpt-4o", id="a-call-ending-in-_get"), + pytest.param('["gpt-4o", "gpt-4o-mini"]', "gpt-4o", id="a-list-at-the-line-start"), ], ) def test_a_value_passed_to_a_call_is_the_model(root, written, name): @@ -1688,3 +1854,822 @@ def test_a_prompt_with_escaped_characters_holds_the_quote(tmp_path, root): assert not result.get("is_error"), result assert saved(destination)["system_prompt_excerpt"] == written + + +# Code between the string pieces of a prompt is passed over, a little at a time + +_LONG_OPENING = ( + "You are Acme's booking agent. Help guests book tables, change bookings and cancel them. " +) + + +@pytest.mark.parametrize( + "written, quote", + [ + pytest.param( + f'PROMPT = (\n "{_LONG_OPENING}"\n f"Today is {{today}}. "\n "Be warm and brief."\n)', + _LONG_OPENING + "Today is {today}. Be warm and brief.", + id="f-string-piece-late", + ), + pytest.param( + f'PROMPT = (\n "{_LONG_OPENING}" # the rules\n "Be warm and brief."\n)', + _LONG_OPENING + "Be warm and brief.", + id="comment-between-pieces", + ), + pytest.param( + f'PROMPT = (\n "{_LONG_OPENING}"\n r"Never share \\d+ ids. "\n "Be warm and brief."\n)', + _LONG_OPENING + "Never share \\d+ ids. Be warm and brief.", + id="raw-piece-late", + ), + pytest.param( + f'PROMPT = "{_LONG_OPENING}" + NAME + " Be warm and brief."', + _LONG_OPENING + "Be warm and brief.", + id="a-variable-left-out", + ), + pytest.param( + f'PROMPT = "{_LONG_OPENING}Be warm and brief."', + _LONG_OPENING + "Be warm and brief.", + id="markup-left-out", + ), + pytest.param( + f'PROMPT = "{_LONG_OPENING}{{{{ guest_name }}}}, be warm and brief."', + _LONG_OPENING + "Be warm and brief.", + id="a-placeholder-left-out", + ), + ], +) +def test_code_between_string_pieces_is_passed_over(tmp_path, root, written, quote): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + count = len(written.splitlines()) + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": count}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written + + +def test_a_long_gap_between_words_is_not_passed_over(tmp_path, root): + written = ( + f'PROMPT = (\n "{_LONG_OPENING}" # twenty-five letters of a comment here\n' + ' "Be warm and brief."\n)' + ) + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_LONG_OPENING + "Be warm and brief.", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 4}], + ), + tmp_path / "out", + source_root=root, + ) + + assert text_of(result) == REFUSED + not_the_quote("prompt.py 1-4") + + +def test_a_quote_that_passed_replaces_the_held_one(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + prompt = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant.", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + ) + assert first.get("is_error"), first + second = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=prompt, + ) + ) + assert not second.get("is_error"), second + + # The lines again, without a quote: checked against the quote that passed, not the old one. + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=prompt)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + assert saved(destination)["system_prompt_from"] == prompt + + +@pytest.mark.parametrize("placeholder", ["...", "…", "[...]", "-", "—", "?"]) +def test_a_quote_without_words_does_not_skip_the_held_one(tmp_path, root, placeholder): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=imports, + ) + ) + assert first.get("is_error"), first + + again = payload( + hard_constraints=["Be polite."], system_prompt_excerpt=placeholder, system_prompt_from=imports + ) + second = submit(again) + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) + + third = submit(again) + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["system_prompt_from"] == [] + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +@pytest.mark.parametrize( + "written", + [ + pytest.param('PROMPT = "Order a caf\\u00e9 here, then wait for the bill."', id="short-unicode-escape"), + pytest.param('PROMPT = "Order a caf\\xe9 here, then wait for the bill."', id="hex-escape"), + pytest.param('PROMPT = "Order a caf\\U000000e9 here, then wait for the bill."', id="long-unicode-escape"), + pytest.param('PROMPT = "Order a caf\\N{LATIN SMALL LETTER E WITH ACUTE} here, then wait for the bill."', id="named-escape"), + pytest.param('PROMPT = "Order a caf\\351 here, then wait for the bill."', id="octal-escape"), + ], +) +def test_more_escapes_hold_the_quote(tmp_path, root, written): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="Order a café here, then wait for the bill.", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + + +def test_a_quote_that_grows_when_normalised_costs_little(root): + from fi.alk.harness.model_citation import quote_problem + + import tracemalloc + + # U+FDFA grows eighteen times under NFKC; NFC leaves it as it is. + copied = "\ufdfa" * (MIB // 3) + tracemalloc.start() + try: + started = time.perf_counter() + quote_problem("\ufdfa" * 1000 + " b", copied, "agent.py 1-1") + took = time.perf_counter() - started + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert took < 0.5 + assert peak < 32 * MIB + + +# An apostrophe in bare text doesn't hide the comment after it + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("cfg.yaml", "description: Pair's booking agent # model: gpt-4o-mini", "gpt-4o-mini", id="yaml"), + pytest.param(".env", "AGENT_NAME=Bob's agent # MODEL=gpt-4o-mini", "gpt-4o-mini", id="env"), + pytest.param("llm.sh", "echo it\\'s # 'gpt-4o-mini'", "gpt-4o-mini", id="shell-escaped"), + pytest.param("lib.rs", "fn f<'a>(x: &'a str) -> &'a str { x } // \"gpt-4o-mini\"", "gpt-4o-mini", id="rust-lifetimes"), + pytest.param("cfg.yaml", "note: Pair's agent # it's \"gpt-4o-mini\"", "gpt-4o-mini", id="apostrophes-around-the-marker"), + pytest.param("llm.ini", "; model = gpt-4o-mini", "gpt-4o-mini", id="ini-comment"), + ], +) +def test_an_apostrophe_does_not_hide_a_comment(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("run.sh", "curl -s http://localhost:8000/v1/chat/completions -d '{\"model\": \"gpt-4o\"}'", "gpt-4o", id="curl"), + pytest.param("run.sh", "python app.py --base-url=http://localhost:4000 --model=gpt-4o", "gpt-4o", id="url-flag"), + pytest.param("Dockerfile", "CMD python app.py --base-url=http://ollama:11434 --model=llama3", "llama3", id="dockerfile-cmd"), + pytest.param("cfg.yaml", "llm: {base_url: http://localhost:11434, model: llama3}", "llama3", id="yaml-flow-map"), + pytest.param("run.sh", 'exec uv run -- python -m app --model "gpt-4o"', "gpt-4o", id="uv-run"), + pytest.param("Dockerfile", "CMD tini -- python app.py --model=gpt-4o", "gpt-4o", id="tini"), + pytest.param("run.sh", "for f in data/*; do MODEL=gpt-4o; done", "gpt-4o", id="glob"), + pytest.param("run.sh", "[ ${#MODELS[@]} -eq 0 ] && MODEL=gpt-4o", "gpt-4o", id="shell-length"), + pytest.param("llm.py", "n = len(x)//2; model = 'gpt-4o'", "gpt-4o", id="floor-division"), + pytest.param("llm.ts", ' */ const model = "gpt-4o";', "gpt-4o", id="after-a-block-comment-closes"), + ], +) +def test_urls_globs_and_dashes_are_not_comments(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "written, name", + [ + pytest.param('DEPLOYMENT = "gpt-4o_deployment"', "gpt-4o_deployment", id="underscore-deployment"), + pytest.param('DEPLOYMENT = "chat-deployment"', "chat-deployment", id="deployment-without-a-digit"), + pytest.param('MODEL = "azure/gpt4o_deployment"', "azure/gpt4o_deployment", id="provider-prefixed-deployment"), + # A setting's word with a version and no underscore: a deployment's own name. + pytest.param('DEPLOYMENT = "gpt4o-default"', "gpt4o-default", id="versioned-with-a-setting-word"), + ], +) +def test_deployment_ids_are_models(root, written, name): + (root / "llm.py").write_text(f"# the model\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": "llm.py", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +def test_a_crafted_line_of_generic_calls_does_not_hold_up_the_check(root): + # Each occurrence looks back for a generic's < only a few characters, not to the first get<. + (root / "crafted.ts").write_text('get("a") ' + '>("a") ' * 100_000, encoding="utf-8") + + _, _, took = checked(root, {"name": "a", "path": "crafted.ts", "line": 1}) + + assert took < 0.5 + + +def test_a_crafted_line_of_apostrophes_does_not_hold_up_the_check(root): + # Once one apostrophe finds no partner, none after it is paired again. + # Small enough that pairing each apostrophe again would show as seconds, not hang. + (root / "crafted.py").write_text("'" + "\\'" * 40_000 + "\n", encoding="utf-8") + + problems, _, took = checked(root, {"name": "a", "path": "crafted.py", "line": 1}) + + assert problems == [not_named("a", 1, "crafted.py")] + assert took < 0.5 + + +# Lines sent alone after a refusal are asked about once more before either side is given up + + +def test_right_lines_sent_alone_after_a_refused_paraphrase_are_kept(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + prompt = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + paraphrase = "Acme booking agent: helps guests reserve tables and never shares bookings." + + first = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=paraphrase, system_prompt_from=prompt) + ) + assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 6-12")) + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=prompt)) + assert text_of(second) == REFUSED + lines_alone("agent.py 6-12", paraphrase) + + # Quoted word for word from the lines, as asked: the lines are copied, nothing unconfirmed. + third = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, system_prompt_from=prompt) + ) + + assert not third.get("is_error"), third + contract = saved(destination) + assert contract["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + assert contract["system_prompt_from"] == prompt + assert "unconfirmed_prompt_from" not in contract + + +def test_unconfirmed_lines_reach_the_document(tmp_path, root): + from fi.alk.providers.agent import build_authored_agent + + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + quoted = payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, system_prompt_from=imports) + + assert submit(quoted).get("is_error") + assert submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)).get("is_error") + assert not submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)).get("is_error") + + contract = AgentContract.model_validate(saved(destination)).model_dump(mode="json") + document = build_authored_agent(contract) + assert document["behaviour"][0]["instructions"] == _BOOKING_QUOTE + assert document["extension"]["contract"]["unconfirmed_prompt_from"] == imports + + +def test_only_the_save_records_unconfirmed_lines(tmp_path, root): + destination = tmp_path / "out" + claimed = [{"path": "agent.py", "start_line": 1, "end_line": 1}] + + result = accept_contract( + payload(hard_constraints=["Be polite."], unconfirmed_prompt_from=claimed), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert "unconfirmed_prompt_from" not in saved(destination) + + +def test_a_quote_whose_start_matches_everywhere_does_not_hold_up_the_check(root): + from fi.alk.harness.model_citation import quote_problem + + # Every start matches a thousand words before failing: the attempts are counted and stop. + started = time.perf_counter() + answer = quote_problem("a " * 1000 + "b", "a" * MIB, "agent.py 1-1") + took = time.perf_counter() - started + + assert answer is not None + assert took < 0.5 + + +def test_a_first_word_found_early_is_tried_again(tmp_path, root): + # "you" first appears in the comment, too far from "are"; the prompt's own "You are" follows. + written = "# you should keep this comment long enough to break the match\n" + _BOOKING + (root / "agent.py").write_text(written, encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 13}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == "\n".join(written.splitlines()) + + +def test_wrong_lines_alone_after_a_passed_quote_are_asked_about_then_left_out(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}], + ) + ) + assert not first.get("is_error"), first + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) + + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +# A quote is held from the moment it is sent, whatever sends the call back + + +@pytest.mark.parametrize( + "first", + [ + pytest.param({}, id="asked-for-lines"), + pytest.param({"modality": "telepathy"}, id="refused-before-the-checks"), + ], +) +def test_a_quote_sent_without_lines_is_held(tmp_path, root, first): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + sent_back = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, **first) + ) + assert sent_back.get("is_error"), sent_back + + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +def test_the_quote_shown_with_lines_sent_alone_is_on_one_line(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + quote = "You are Acme's booking agent.\n\nHelp guests book tables." + + submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=quote, system_prompt_from=imports)) + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + assert "(You are Acme's booking agent. Help guests book tables.)" in text_of(second) + + +@pytest.mark.parametrize("placeholder", ["n/a", "N/A", "TBD", "none", "see lines", "as above"]) +def test_a_quote_of_a_few_letters_says_nothing(tmp_path, root, placeholder): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + assert submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, system_prompt_from=imports) + ).get("is_error") + again = payload(hard_constraints=["Be polite."], system_prompt_excerpt=placeholder, system_prompt_from=imports) + + assert text_of(submit(again)) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) + assert not submit(again).get("is_error") + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +# The gap between two words of a quote is 24 letters, no more + + +@pytest.mark.parametrize("letters, refused", [(24, False), (25, True)]) +def test_the_gap_between_words_is_24_letters(letters, refused): + from fi.alk.harness.model_citation import quote_problem + + copied = "Confirm every booking " + "x" * letters + " before saving it." + + answer = quote_problem("Confirm every booking before saving it.", copied, "agent.py 1-1") + + assert (answer is not None) is refused + + +# Each search is charged what it reads + + +def test_an_elided_quote_of_a_long_prompt_holds(tmp_path, root): + from fi.alk.harness.model_citation import quote_problem + + filler = "A guest asks a question and a host answers it at a table. " * 3_000 + copied = f'PROMPT = """You are Acme\'s booking agent. {filler}A short answer is best."""' + + answer = quote_problem( + "You are Acme's booking agent. ... A short answer is best.", copied, "agent.py 1-1" + ) + + assert answer is None + + +def test_a_crafted_long_word_costs_little(root): + from fi.alk.harness.model_citation import quote_problem + + started = time.perf_counter() + answer = quote_problem("a " + "a" * MIB + "b", "a" * MIB, "agent.py 1-1") + took = time.perf_counter() - started + + assert answer is not None + assert took < 0.25 + + +# Scripts written without spaces: each character is a word + + +@pytest.mark.parametrize( + "written, quote", + [ + pytest.param('PROMPT = ("你是一个乐于助人的订餐" f"助手,今天是{today}。")', "你是一个乐于助人的订餐助手,今天是{today}。", id="chinese-prefix-mid-clause"), + pytest.param('PROMPT = ("你是一个乐于助人的订餐" # 角色\n "助手,请礼貌地回答。")', "你是一个乐于助人的订餐助手,请礼貌地回答。", id="chinese-comment-mid-clause"), + pytest.param('PROMPT = ("あなたは親切な予約" f"アシスタントです。{name}")', "あなたは親切な予約アシスタントです。", id="japanese-prefix-mid-clause"), + ], +) +def test_an_unspaced_prompt_holds_its_quote(tmp_path, root, written, quote): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + count = len(written.splitlines()) + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": count}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + + +def test_an_escape_that_names_no_character_is_kept_as_written(tmp_path, root): + from fi.alk.harness.model_citation import quote_problem + + quote = "Order a table \\UFFFFFFFF and \\N{NOT A REAL NAME} for two." + + assert quote_problem(quote, 'PROMPT = "' + quote + '"', "prompt.py 1-1") is None + + +# The reader: what counts is limit + 1, and every descriptor is closed + + +def test_a_read_that_lands_on_the_limit_reads_one_more(tmp_path, monkeypatch): + from fi.alk.harness import model_citation + + (tmp_path / "agent.py").write_bytes(b"x" * 100) + real_read = os.read + monkeypatch.setattr(model_citation.os, "read", lambda fd, n: real_read(fd, min(n, 7))) + + assert model_citation.read_source(tmp_path / "agent.py", 49) == b"x" * 50 + + +def test_the_reader_closes_what_it_opens(tmp_path, monkeypatch): + from fi.alk.harness import model_citation + + (tmp_path / "agent.py").write_bytes(b"x" * 10) + (tmp_path / "folder").mkdir() + opened: list[int] = [] + closed: list[int] = [] + real_open, real_close = os.open, os.close + monkeypatch.setattr(model_citation.os, "open", lambda *a: opened.append(real_open(*a)) or opened[-1]) + monkeypatch.setattr(model_citation.os, "close", lambda fd: closed.append(fd) or real_close(fd)) + + model_citation.read_source(tmp_path / "agent.py", 100) + model_citation.read_source(tmp_path / "folder", 100) + + assert sorted(opened) == sorted(closed) and len(opened) == 2 + + +# Triple quotes count only in languages that have them, and never inside a string or comment + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("llm.py", "parts = r.split('\"\"\"')\nMODEL = \"gpt-4o\"\n", 2, "gpt-4o", id="in-a-one-line-string"), + pytest.param("llm.py", "# docstrings use \"\"\"\nMODEL = \"gpt-4o\"\n", 2, "gpt-4o", id="in-a-comment"), + pytest.param("llm.py", "SEP = \"'''\"; MODEL = \"gpt-4o\"\n", 1, "gpt-4o", id="in-a-string-on-the-line"), + pytest.param("llm.ts", "const prompt = 'Use \"\"\" for blocks';\nconst model = \"gpt-4o\";\n", 2, "gpt-4o", id="typescript"), + pytest.param("main.go", "var sep = `\"\"\"`\nvar model = \"gpt-4o\"\n", 2, "gpt-4o", id="go"), + pytest.param("cfg.yaml", "a: '''quoted'''\nmodel: gpt-4o\n", 2, "gpt-4o", id="yaml"), + pytest.param("llm.py", 'X = """a\nb"""; MODEL = "gpt-4o"\n', 2, "gpt-4o", id="a-block-closing-before-the-value"), + pytest.param("llm.py", "msg = f'Agent #{n}'; MODEL = \"gpt-4o\"\n", 1, "gpt-4o", id="prefixed-string-with-a-hash"), + pytest.param("llm.py", "x = a[1]//2; m = \"gpt-4o\"\n", 1, "gpt-4o", id="floor-division-after-a-subscript"), + pytest.param("llm.py", "n = a//2; model = 'gpt-4o'\n", 1, "gpt-4o", id="floor-division-after-a-name"), + ], +) +def test_stray_triple_quotes_do_not_hide_the_model(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("llm.py", '"""Uses "gpt-4o".\n"""\n', 1, "gpt-4o", id="a-docstring-opening-on-the-line"), + pytest.param("llm.py", "X = '''It's a''' # it's \"gpt-4o\"\n", 1, "gpt-4o", id="a-comment-after-a-block"), + pytest.param("llm.py", 'X = """a\nb""" # was "gpt-4o"\n', 2, "gpt-4o", id="a-comment-after-a-block-closes"), + pytest.param("lib.hs", "x' = 1 -- \"gpt-4o\"\n", 1, "gpt-4o", id="haskell-prime"), + pytest.param("llm.ts", '\t*\tmodel: "gpt-4o"\n', 1, "gpt-4o", id="a-tabbed-block-comment-line"), + pytest.param("llm.cfg", "; model = gpt-4o\n", 1, "gpt-4o", id="cfg-comment"), + pytest.param("llm.conf", "; model = gpt-4o\n", 1, "gpt-4o", id="conf-comment"), + ], +) +def test_more_comments_and_docstrings_are_not_the_model(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [not_named(name, line, file)] + + +def test_a_crafted_file_of_one_byte_lines_and_triple_quotes_costs_little(root): + # Lines with no triple quote, and lines inside a block without its closing quote, are passed + # over without a scan. + (root / "crafted.py").write_text( + '"""\n' + "x\n" * (MIB // 4) + '"""\n' + "y\n" * (MIB // 4) + 'MODEL = "gpt-4o"\n', + encoding="utf-8", + ) + line = 2 * (MIB // 4) + 3 + + _, _, took = checked(root, {"name": "gpt-4o", "path": "crafted.py", "line": line}) + + assert took < 0.25 + + +@pytest.mark.parametrize( + "file", + ["notes.org", "rules.mdc", "CHANGELOG", "CHANGELOG.md", "LICENSE", "CONTRIBUTING.rst"], +) +def test_more_prose_files_hold_no_model(root, file): + (root / file).write_text('# notes\nmodel: "claude-3-opus"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "claude-3-opus", "path": file, "line": 2}) + + assert problems == [prose(file)] + + +def test_a_file_named_after_readme_is_code(root): + (root / "readme_writer.py").write_text('# writes READMEs\nMODEL = "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "readme_writer.py", "line": 2}) + + assert problems == [] + + +@pytest.mark.parametrize( + "quote, lines, refused", + [ + pytest.param("You are pair.", (1, 4), True, id="short-and-not-in-the-lines"), + pytest.param("Be warm and brief.", (6, 12), False, id="short-and-in-the-lines"), + pytest.param("TBD", (1, 4), True, id="a-placeholder-with-nothing-held"), + ], +) +def test_a_short_quote_is_looked_for_whole(tmp_path, root, quote, lines, refused): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "agent.py", "start_line": lines[0], "end_line": lines[1]}], + ), + tmp_path / "out", + source_root=root, + ) + + if refused: + assert text_of(result) == REFUSED + not_the_quote(f"agent.py {lines[0]}-{lines[1]}") + else: + assert not result.get("is_error"), result + + +def test_full_width_digits_hold_their_quote(tmp_path, root): + (root / "prompt.py").write_text('PROMPT = "Seat at most 8 guests per table tonight."\n', encoding="utf-8") + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="Seat at most 8 guests per table tonight.", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + tmp_path / "out", + source_root=root, + ) + + assert not result.get("is_error"), result + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("agent.ts", 'class A { #model = "gpt-4o"; }', "gpt-4o", id="js-private-field"), + pytest.param("agent.c", '#define MODEL "gpt-4o"', "gpt-4o", id="c-define"), + pytest.param("main.rs", '#[default = "gpt-4o"] model: String,', "gpt-4o", id="rust-attribute"), + pytest.param("cfg.yaml", "llm: {url: http://x/#top, model: gpt-4o}", "gpt-4o", id="yaml-url-fragment"), + pytest.param("cfg.yaml", "llm: {a: 1,#b, model: gpt-4o}", "gpt-4o", id="yaml-hash-without-a-space"), + pytest.param("agent.ts", '/* the default */ const model = "gpt-4o";', "gpt-4o", id="a-closed-block-comment-before"), + pytest.param("index.html", ' ', "gpt-4o", id="a-closed-markup-comment-before"), + pytest.param("llm.py", 'm = str("gpt-4o")', "gpt-4o", id="str-as-a-function"), + pytest.param("GNUmakefile", "MODEL ?= gpt-4o", "gpt-4o", id="gnu-makefile"), + pytest.param("chat.prompt", "model: googleai/gemini-1.5-pro", "googleai/gemini-1.5-pro", id="dotprompt-frontmatter"), + ], +) +def test_more_honest_forms_by_language(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("agent.ts", 'x = 1; /* "gpt-4o" */ y = 2;', "gpt-4o", id="inside-a-closed-block-comment"), + pytest.param("agent.py", "x = 1 # it's \"gpt-4o\"", "gpt-4o", id="python-hash"), + pytest.param("cfg.yaml", 'model: gpt-4o-mini # was "gpt-4o"', "gpt-4o", id="yaml-spaced-hash"), + pytest.param("agent.ts", 'const m = config?.["gpt-4o"];', "gpt-4o", id="optional-chained-lookup"), + pytest.param("agent.py", 'm = env.str("gpt-4o")', "gpt-4o", id="str-as-a-method"), + ], +) +def test_more_comments_and_lookups_by_language(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +@pytest.mark.parametrize("file", ["prompt.j2", "prompt.jinja", "prompt.jinja2", "prompt.hbs", "prompt.mustache"]) +def test_a_prompt_template_holds_no_model(root, file): + (root / file).write_text('You are an assistant.\nYou run on "gpt-4o".\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": file, "line": 2}) + + assert problems == [prose(file)] + + +def test_a_short_quote_with_nothing_held_is_looked_for_whole(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + + result = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair.", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + ) + + assert text_of(result).startswith(REFUSED + not_the_quote("agent.py 1-4")) + + +def test_a_long_first_word_is_charged_on_every_try(root): + from fi.alk.harness.model_citation import quote_problem + + started = time.perf_counter() + # Without the charge, each of 200,000 tries would compare 200,000 letters. + answer = quote_problem("a" * 200_000 + " b", "a" * 400_000, "agent.py 1-1") + took = time.perf_counter() - started + + assert answer is not None + assert took < 0.25 + + +def test_a_short_unspaced_quote_is_still_searched_with_gaps(tmp_path, root): + # Six characters, each counted as three letters: long enough to be searched with gaps. + (root / "prompt.py").write_text('PROMPT = ("你是订餐" f"助手")\n', encoding="utf-8") + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="你是订餐助手", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + tmp_path / "out", + source_root=root, + ) + + assert not result.get("is_error"), result + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("cfg.yaml", 'note: x """ y\nmodel: gpt-4o\n', 2, "gpt-4o", id="yaml-bare-triple"), + pytest.param("main.go", 'var s = x + """\nvar model = "gpt-4o"\n', 2, "gpt-4o", id="go-bare-triple"), + ], +) +def test_triple_quotes_are_tracked_only_where_they_are_strings(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [] + + +def test_a_value_before_a_block_closes_is_text(root): + (root / "llm.py").write_text('X = """a\nuse "gpt-4o" here"""\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "llm.py")] + + +def test_lines_without_a_triple_quote_are_not_scanned(root, monkeypatch): + from fi.alk.harness import model_citation + + written = '"""\n' + "x\n" * 499 + '"""\n' + "y\n" * 499 + 'MODEL = "gpt-4o"\n' + (root / "llm.py").write_text(written, encoding="utf-8") + scanned: list[str] = [] + real_scan = model_citation._line_scan + monkeypatch.setattr( + model_citation, "_line_scan", lambda line, *a, **k: scanned.append(line) or real_scan(line, *a, **k) + ) + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 1001}) + + assert problems == [] + # The block's opening and closing lines, and the cited line: nothing between. + assert scanned == ['"""', '"""', 'MODEL = "gpt-4o"'] diff --git a/tests/harness/test_contract_tools_subfolder.py b/tests/harness/test_contract_tools_subfolder.py index 581cd0346..16e3374c9 100644 --- a/tests/harness/test_contract_tools_subfolder.py +++ b/tests/harness/test_contract_tools_subfolder.py @@ -1188,9 +1188,15 @@ def open_(path, flags, *args, **kwargs): endless.add(descriptor) return descriptor + given: list[int] = [] + def read(descriptor, size): # Like a link to /proc/self/pagemap: its size says 0 and it reads without end. - return b"#" * size if descriptor in endless else real_read(descriptor, size) + if descriptor not in endless: + return real_read(descriptor, size) + given.append(size) + assert sum(given) <= 8 * 1024 * 1024, "read to the end" + return b"#" * size def close(descriptor): endless.discard(descriptor) diff --git a/tests/test_harness.py b/tests/test_harness.py index 8b9246a86..e8ca3ea5b 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -3968,7 +3968,7 @@ def visit(node): advertised = asyncio.run(advertised_fields()) # Set by the amendment tools during later stages, never by whoever submits the contract. - ours = {"amendments", "instrumentation", "source_subfolder"} + ours = {"amendments", "instrumentation", "source_subfolder", "unconfirmed_prompt_from"} def named(model: type[BaseModel]) -> list[str]: """Every field the model would have to fill, nested ones included. From 555e5f0a28ce2ee805fbd5a5ca7bc84a9cd5fada Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 20:56:54 +0530 Subject: [PATCH 05/10] fix(harness): answer review round 3 of the second-review fixes (TH-8953) Local, not pushed: review round 4 runs before the push (Khushal: go with A). - Unspaced scripts: only a letter is a word. - The quote is read as the contract reads it for the hold and the nudges; short quotes are held while nothing longer is; a quote under 6 letters says nothing; at the second mismatch a short quote keeps the lines, recorded unconfirmed. - Each call decides from what was asked before it was sent. - // and /* only in languages with block comments; # per language; config templates read as their inner kind; env.py is code. - SKILL item 8 records the call-direction line in notes. Signed-off-by: khushalsonawat --- src/fi/alk/harness/contract.py | 17 +- src/fi/alk/harness/model_citation.py | 151 +++++++--- .../harness/skills/understand-agent/SKILL.md | 9 +- src/fi/alk/harness/tools.py | 61 ++-- tests/harness/test_contract_model.py | 260 +++++++++++++++++- 5 files changed, 431 insertions(+), 67 deletions(-) diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index fd0ffee1e..3fc5423bf 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -1084,6 +1084,20 @@ def _reached(one: Dependency) -> str: return "; ".join(said) +def prompt_quote(payload: Any) -> str: + """The quote of the prompt as AgentContract reads it from a payload: under its own name or + the first alias that has one, a list joined by lines. The save holds it before validating.""" + if not isinstance(payload, dict): + return "" + for key in ("system_prompt_excerpt", *_ALIASES["system_prompt_excerpt"]): + value = payload.get(key) + if value: + if isinstance(value, list): + return "\n".join(str(item) for item in value) + return value if isinstance(value, str) else str(value) + return "" + + class AgentContract(BaseModel): """What the agent verifiably is. Nothing downstream may contradict this.""" @@ -1143,7 +1157,8 @@ def _normalize_shapes(cls, payload: Any) -> Any: # When given, the excerpt is replaced by these lines copied from the source as written. system_prompt_from: list[PromptSource] = Field(default_factory=list) # Set by the save, never by the reading: lines named for the prompt that twice didn't hold - # the quote given for it, so the quote went out instead. Empty for nearly every contract. + # the quote given for it. The quote went out instead, or, when the quote was too short to + # stand for the prompt, the lines did, unconfirmed. Empty for nearly every contract. unconfirmed_prompt_from: list[PromptSource] = Field(default_factory=list) # None is unknown. Kept only when its name was found on the line cited in the source. model: ModelSource | None = None diff --git a/src/fi/alk/harness/model_citation.py b/src/fi/alk/harness/model_citation.py index 2b88eacf8..5183d111e 100644 --- a/src/fi/alk/harness/model_citation.py +++ b/src/fi/alk/harness/model_citation.py @@ -10,6 +10,7 @@ from __future__ import annotations +import functools import html import logging import os @@ -43,13 +44,23 @@ ) _CONFIG_NAMES = frozenset({"makefile", "gnumakefile", ".flaskenv"}) # Languages where # isn't a comment: C and C++ (#define), JavaScript and TypeScript (#field), -# Java, Go, Rust (#[attr]), Swift (#if), Kotlin, C#, Scala, Dart, Objective-C. +# Java, Go, Rust (#[attr]), Swift (#if), Kotlin, C#, Scala, Dart, Objective-C, Groovy, Gradle, +# Lua (#t is a length). _NO_HASH_COMMENT_SUFFIXES = frozenset( { ".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".java", ".go", ".rs", ".swift", ".kt", ".kts", ".cs", ".scala", ".dart", ".m", ".mm", + ".groovy", ".gradle", ".lua", } ) +# Files where # starts a comment only at a line's start or after a space: YAML, shell, Makefile +# and Dockerfile (a#b is a word there), and .env files. +_SPACED_HASH_SUFFIXES = frozenset({".yaml", ".yml", ".sh", ".bash", ".zsh", ".mk", ".dockerfile"}) +# Files with no block comments, nor //: there `* ` starts no comment (a Python continuation +# line), and /* and // are text (ls /*, a // 2). +_NO_BLOCK_COMMENT_SUFFIXES = frozenset( + {".py", ".pyi", ".pyw", ".rb", ".pl", ".r", ".jl", ".ex", ".exs", ".nim", ".lua"} +) _YAML_SUFFIXES = frozenset({".yaml", ".yml"}) # INI files: a line starting with ; is a comment. _INI_SUFFIXES = frozenset({".ini", ".cfg", ".conf"}) @@ -64,8 +75,8 @@ ".jinja2", ".hbs", ".mustache", } ) -# Files written for people, by name: README, CHANGELOG, LICENSE, CONTRIBUTING, with or without a -# suffix (not readme_writer.py). +# Files written for people, by name with no suffix: README, CHANGELOG, LICENSE, CONTRIBUTING +# (README.md is prose by its suffix; readme.py is code). _PROSE_STEMS = frozenset({"readme", "changelog", "license", "contributing"}) # Words a setting is named with, never a model: `cfg.get("model")`, OPENAI_MODEL, llm.model. # Not "deployment": an Azure deployment's own name often has it (gpt-4o_deployment). @@ -88,11 +99,15 @@ # characters there too. _UNSPACED = "\u0e00-\u0eff\u1000-\u109f\u1780-\u17ff\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff" _UNSPACED_CHAR = re.compile(f"[{_UNSPACED}]") -_WORD = re.compile(f"[{_UNSPACED}]|[^\\W_{_UNSPACED}]+") +# Only a letter of those scripts is a word: their vowel signs and tone marks, and ・, are not. +_WORD = re.compile(f"(?![\\W_])[{_UNSPACED}]|[^\\W_{_UNSPACED}]+") # A quote of fewer letters than this says nothing to check lines against ("n/a", "TBD", "see # lines"): a few short words are found in almost any lines. A character of an unspaced script # counts as three. _SHORTEST_QUOTE = 16 +# Fewer letters than this and a quote says nothing at all ("n/a", "TBD", "none"): it isn't held +# and isn't checked. +_SAYS_NOTHING = 6 # Languages with triple-quoted strings ("""...""" or '''...'''): Python, TOML, Kotlin, Scala, # Swift, Groovy and Gradle, Java and C# text blocks, Julia, Dart, Elixir. _TRIPLE_SUFFIXES = frozenset( @@ -113,37 +128,35 @@ _HASH_ANYWHERE, _HASH_SPACED, _HASH_NONE = "anywhere", "spaced", "none" -def _comment_tokens(*, dashes: bool, triple: bool, hash_comments: str) -> re.Pattern[str]: +@functools.cache +def _comment_tokens( + *, dashes: bool, triple: bool, hash_comments: str, block_comments: bool +) -> re.Pattern[str]: """What the line scan stops at: a triple quote (in languages that have them), a double-quoted or backticked string, matched whole (to the line's end if it isn't closed); an apostrophe, which the scan pairs itself; or a comment marker. # not in $# (a shell count), - &# (an HTML character) or ${# (a shell length); // not in a URL (http://), floor division - (x//2) or after a name; /* not in a glob (data/*).""" + &# (an HTML character) or ${# (a shell length).""" alternatives = ['"""', "'''"] if triple else [] alternatives += [r'"(?:[^"\\]|\\.)*"?', r"`(?:[^`\\]|\\.)*`?", "'"] if hash_comments == _HASH_ANYWHERE: alternatives.append(r"(? bool: @@ -155,7 +168,7 @@ def _opens_a_string(line: str, at: int) -> bool: start -= 1 if start == at: return at == 0 or line[at - 1] != "\\" - whole = start == 0 or not (line[start - 1].isalnum() or line[start - 1] == "_") + whole = start == 0 or not line[start - 1].isalnum() return whole and line[start:at].lower() in _STRING_PREFIXES @@ -167,6 +180,7 @@ def _line_scan( ini: bool = False, triple: bool = False, hash_comments: str = _HASH_ANYWHERE, + block_comments: bool = True, ) -> tuple[int, list[tuple[int, int]], str | None]: """One pass over a line: where its comment starts (or its length), the stretches of it that are text (inside a triple-quoted string, or a block comment closed on the line), and the @@ -184,9 +198,13 @@ def _line_scan( at = close + 3 else: stripped = line.lstrip() - if stripped.startswith(("* ", "*\t")) or (ini and stripped.startswith(";")): + if (block_comments and stripped.startswith(("* ", "*\t"))) or ( + ini and stripped.startswith(";") + ): return 0, inside, None - tokens = _TOKENS[dashes, triple, hash_comments] + tokens = _comment_tokens( + dashes=dashes, triple=triple, hash_comments=hash_comments, block_comments=block_comments + ) while (found := tokens.search(line, at)) is not None: token, start = found.group(), found.start() at = found.end() @@ -218,7 +236,7 @@ def _line_scan( def block_open_before( - lines: list[str], *, dashes: bool = False, ini: bool = False, hash_comments: str = _HASH_ANYWHERE + lines: list[str], *, hash_comments: str = _HASH_ANYWHERE, block_comments: bool = True ) -> str | None: """The triple quote open where the line after ``lines`` starts, the lines read with the same scan as the cited one. A line with no triple quote, outside a block, is passed over without @@ -230,7 +248,11 @@ def block_open_before( if block is not None and block not in line: continue _, _, block = _line_scan( - line, block, dashes=dashes, ini=ini, triple=True, hash_comments=hash_comments + line, + block, + triple=True, + hash_comments=hash_comments, + block_comments=block_comments, ) return block @@ -371,6 +393,7 @@ def named_on( ini: bool = False, triple: bool = False, hash_comments: str = _HASH_ANYWHERE, + block_comments: bool = True, block: str | None = None, ) -> bool: """Whether ``name`` is the value written on ``line``: whole, outside a comment and outside @@ -385,7 +408,13 @@ def named_on( # find("") matches at every position without moving on. return False comment, inside, _ = _line_scan( - line, block, dashes=dashes, ini=ini, triple=triple, hash_comments=hash_comments + line, + block, + dashes=dashes, + ini=ini, + triple=triple, + hash_comments=hash_comments, + block_comments=block_comments, ) stretches = iter(inside) stretch = next(stretches, None) @@ -413,18 +442,38 @@ def named_on( return False +# Prompt and page templates: what they hold is text, unless the name says they render a +# configuration file (config.yaml.j2), which is then read as that file. +_TEMPLATE_SUFFIXES = frozenset({".j2", ".jinja", ".jinja2", ".hbs", ".mustache"}) + + +def kind_suffix(path: Path) -> str: + """The suffix that says what a file is: config.yaml.j2 is YAML.""" + suffix = path.suffix.lower() + if suffix in _TEMPLATE_SUFFIXES: + inner = Path(path.stem).suffix.lower() + if inner in CONFIG_SUFFIXES: + return inner + return suffix + + def is_config_file(path: Path) -> bool: name = path.name.lower() + suffix = kind_suffix(path) + # env.py and dockerfile_gen.py are code, whatever their names start with. + code = suffix in _TRIPLE_SUFFIXES | _NO_HASH_COMMENT_SUFFIXES | _NO_BLOCK_COMMENT_SUFFIXES return ( - path.suffix.lower() in CONFIG_SUFFIXES + suffix in CONFIG_SUFFIXES or name in _CONFIG_NAMES - or name.startswith((".env", "env.", "dockerfile")) + or (not code and name.startswith((".env", "env.", "dockerfile"))) or name.endswith(".env") ) def is_prose_file(path: Path) -> bool: - return path.suffix.lower() in PROSE_SUFFIXES or path.stem.lower() in _PROSE_STEMS + return kind_suffix(path) in PROSE_SUFFIXES or ( + not path.suffix and path.name.lower() in _PROSE_STEMS + ) def _reads_as_a_setting(name: str) -> bool: @@ -507,28 +556,38 @@ def model_problems(contract: AgentContract, source_root: Path) -> list[str]: lines = source_lines(raw) if model.line > len(lines): return [f"model: line {model.line} is outside {shown_path}, which has {len(lines)} lines"] - suffix = resolved.suffix.lower() + suffix = kind_suffix(resolved) dashes, ini = suffix in _DASH_COMMENT_SUFFIXES, suffix in _INI_SUFFIXES triple = suffix in _TRIPLE_SUFFIXES - hash_comments = ( - _HASH_NONE - if suffix in _NO_HASH_COMMENT_SUFFIXES - else _HASH_SPACED - if suffix in _YAML_SUFFIXES - else _HASH_ANYWHERE - ) + config = is_config_file(resolved) + name_lower = resolved.name.lower() + if suffix in _NO_HASH_COMMENT_SUFFIXES: + hash_comments = _HASH_NONE + elif ( + suffix in _SPACED_HASH_SUFFIXES + or name_lower in _CONFIG_NAMES + or name_lower.startswith((".env", "dockerfile")) + ): + hash_comments = _HASH_SPACED + else: + hash_comments = _HASH_ANYWHERE + # Configuration files have no block comments either. + block_comments = not (config or suffix in _NO_BLOCK_COMMENT_SUFFIXES) if not named_on( lines[model.line - 1], name, - config=is_config_file(resolved), + config=config, yaml=suffix in _YAML_SUFFIXES, dashes=dashes, ini=ini, triple=triple, hash_comments=hash_comments, + block_comments=block_comments, block=( block_open_before( - lines[: model.line - 1], dashes=dashes, ini=ini, hash_comments=hash_comments + lines[: model.line - 1], + hash_comments=hash_comments, + block_comments=block_comments, ) if triple else None @@ -575,10 +634,20 @@ def quote_pieces(quote: str) -> list[list[str]]: quote has fewer than _SHORTEST_QUOTE letters: then it says nothing to check lines against.""" pieces = [_WORD.findall(_comparable(piece)) for piece in _ELISION.split(quote)] pieces = [words for words in pieces if words] - letters = sum( + return pieces if _weighed(pieces) >= _SHORTEST_QUOTE else [] + + +def _weighed(pieces: list[list[str]]) -> int: + """How many letters a quote's words come to, a character of an unspaced script as three.""" + return sum( 3 if _UNSPACED_CHAR.fullmatch(word) else len(word) for words in pieces for word in words ) - return pieces if letters >= _SHORTEST_QUOTE else [] + + +def says_something(quote: str) -> bool: + """Whether a quote has enough letters to say anything: "n/a" and "TBD" don't.""" + pieces = [_WORD.findall(_comparable(piece)) for piece in _ELISION.split(quote)] + return _weighed([words for words in pieces if words]) >= _SAYS_NOTHING def _holds(found: str, pieces: list[list[str]]) -> bool: @@ -622,9 +691,9 @@ def quote_problem(quote: str, copied: str, named: str) -> str | None: if pieces: held = _holds(_letters(copied), pieces) else: - # A quote too short to search for with gaps must be in the lines letter for letter. - letters = _letters(quote) - held = not letters or letters in _letters(copied) + # A quote too short to search for with gaps must be in the lines letter for letter; one + # that says nothing isn't looked for. + held = not says_something(quote) or _letters(quote) in _letters(copied) if held: return None return ( diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index 6ea26464c..e431a1d9d 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -114,8 +114,10 @@ Find, in roughly this order: `system_prompt_from`, every part it is assembled from: it is copied into the contract as written, and every later stage decides what the agent should do from it. If the prompt is built or fetched at run time and has no lines to name, quote it in `system_prompt_excerpt`; the save - asks once whether lines can be named. A quote is word for word, with ... where you leave text - out. The lines you name are checked against your latest quote, given with them or before. + asks once whether lines can be named. A quote is word for word from the prompt, with ... where + you leave text out. The lines you name are checked against your last quote, given with them or + before; lines that don't hold it are asked about once more, and after that the quote is kept + and the lines are recorded as unconfirmed. 7. **The modality.** How this agent is run: a voice session, a text conversation, or work it carries out on websites or a computer. This decides how it is later run, so getting it wrong @@ -186,7 +188,8 @@ Find, in roughly this order: This is not who speaks first: an outbound agent usually still greets. It decides how the simulated person is briefed, and briefing someone who did not dial as though they had an errand to raise tests nothing about how the agent opens a call it placed. Two builds of the same agent - can differ only in this, so quote the line you read it from in `system_prompt_excerpt`. + can differ only in this, so quote the line you read it from in `notes`, unless it is already in + the prompt you name or quote. 9. **What it depends on.** Everything the agent reaches for that has to exist before it can work: a datastore, a service it calls over HTTP, a file it reads, a queue. Record each one, diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index c5f656d96..9818bba55 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -36,7 +36,7 @@ from .backends import tool, tool_server from .contract import ( - _ALIASES, + prompt_quote, CALL_DIRECTIONS, AgentContract, Runtime, @@ -65,6 +65,7 @@ model_problems, quote_pieces, quote_problem, + says_something, read_source, resolved_in_source, source_lines, @@ -270,9 +271,13 @@ def _checked_against_source( and contract.system_prompt_from and not quote_pieces(quote) and held.get("prompt-quote") + # A short quote held for this very call is this call's own, not one given before. + and held["prompt-quote"] != quote ): quote = held["prompt-quote"] alone = True + # Said when the fallback below sets the lines aside in a call refused for something else. + set_aside: list[str] = [] try: problems.extend(tool_evidence_problems(contract, source_root, subfolder)) copy_problems = _copy_prompt(contract, source_root) @@ -298,18 +303,31 @@ def _checked_against_source( "the lines are recorded as unconfirmed" ) elif asked & {"prompt-quote", "prompt-lines-alone"}: - # The quote, which is what was read, rather than lines nobody confirmed; - # the lines are kept on record as unconfirmed. + # Either way the lines are kept on record as unconfirmed. contract.unconfirmed_prompt_from = list(contract.system_prompt_from) - contract.system_prompt_excerpt = quote - contract.system_prompt_from = [] + if quote_pieces(quote): + # The quote, which is what was read, rather than lines nobody + # confirmed. + contract.system_prompt_excerpt = quote + contract.system_prompt_from = [] + set_aside.append( + f"system_prompt_from: the lines named ({named}) are left out and " + "the quote is kept, since they don't hold it" + ) + else: + # A quote of a few words says too little to stand for the prompt: + # the lines are copied. + set_aside.append( + f"system_prompt_from: the lines named ({named}) are copied " + "unconfirmed, since they don't hold the short quote" + ) else: problems.append(mismatch) asked.add("prompt-quote") problems.append( "system_prompt_from: lines named later are checked against this " - "quote until you give another; given again with lines that don't " - "hold it, the quote is kept and the lines are recorded as unconfirmed" + "quote until you give another; a later mismatch, with any quote, " + "keeps the quote and records the lines as unconfirmed" ) if cited := model_problems(contract, source_root): if asked is not None and "model" in asked: @@ -321,6 +339,8 @@ def _checked_against_source( problems.append("model: if the model is refused again, it is left out") except PathLookupsSpent as spent: return _problems([*problems, str(spent)], arrived) + if problems: + problems.extend(set_aside) else: # With no source there is no line to find the name on, so the model is unknown. contract.model = None @@ -1404,16 +1424,18 @@ def contract_tools( ) async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: payload = dict(unwrapped(args)) - # Under any name the contract takes the quote from, so a quote under an alias counts too. - quoted = ( - payload.get(key) - for key in ("system_prompt_excerpt", *_ALIASES["system_prompt_excerpt"]) - ) - excerpt = next((one for one in quoted if isinstance(one, str) and one.strip()), "") + # The quote as the contract will read it: under any alias, a list joined. + excerpt = prompt_quote(payload) # Held before anything can send this call back (a nudge, a refusal, a schema problem), so - # lines named later without a quote are checked against the reading's latest one. - if quote_pieces(excerpt): + # lines named later without a quote are checked against the reading's latest one. A short + # quote is held only while nothing longer is: it says less. + if quote_pieces(excerpt) or ( + says_something(excerpt) and not quote_pieces(held.get("prompt-quote", "")) + ): held["prompt-quote"] = excerpt + # What this call was already asked, as it stood when the call was sent: a call sent in the + # same message as one that is refused hasn't seen that refusal yet. + asked_before = set(nudged) receipt = payload.pop(SECOND_LOOK_READ, None) or ( args.get(SECOND_LOOK_READ) if isinstance(args, dict) else None ) @@ -1691,8 +1713,9 @@ def rows_for(collection: str) -> list[dict[str, Any]]: source_root is not None and bool(excerpt) and not payload.get("system_prompt_from") - # Asked already, with the "prompt" nudge: the lines aren't asked for twice. - and "prompt" not in nudged, + # Asked already, with the "prompt" nudge or a refusal of named lines: the lines + # aren't asked for twice. + and not nudged & {"prompt", "prompt-quote", "prompt-lines-alone"}, "system_prompt_excerpt is a quote, and system_prompt_from names no lines. If the " "prompt is written in the source, name its lines in system_prompt_from so it is " "copied whole.", @@ -1716,9 +1739,11 @@ def rows_for(collection: str) -> list[dict[str, Any]]: subfolder=subfolder, # The scan's answer is for scan_root; another root's code is found again. reached=reach if scan_root == source_root else None, - asked=nudged, + asked=asked_before, held=held, ) + # Whatever this call asked is asked for the calls sent after it. + nudged.update(asked_before) if isinstance(checked, dict): return checked if refused or (destination / FAILURE_FILE).is_file(): diff --git a/tests/harness/test_contract_model.py b/tests/harness/test_contract_model.py index fc2b89a5e..e2d0c8a7f 100644 --- a/tests/harness/test_contract_model.py +++ b/tests/harness/test_contract_model.py @@ -1421,8 +1421,8 @@ def test_lines_refused_twice_leave_the_quote(tmp_path, root): REFUSED + not_the_quote("prompt.py 1-2") + "\n - system_prompt_from: lines named later are checked against this quote until you " - "give another; given again with lines that don't hold it, the quote is kept and the lines " - "are recorded as unconfirmed" + "give another; a later mismatch, with any quote, keeps the quote and records the lines as " + "unconfirmed" ) second = submit(wrong) assert not second.get("is_error"), second @@ -2357,6 +2357,10 @@ def test_a_crafted_long_word_costs_little(root): pytest.param('PROMPT = ("你是一个乐于助人的订餐" f"助手,今天是{today}。")', "你是一个乐于助人的订餐助手,今天是{today}。", id="chinese-prefix-mid-clause"), pytest.param('PROMPT = ("你是一个乐于助人的订餐" # 角色\n "助手,请礼貌地回答。")', "你是一个乐于助人的订餐助手,请礼貌地回答。", id="chinese-comment-mid-clause"), pytest.param('PROMPT = ("あなたは親切な予約" f"アシスタントです。{name}")', "あなたは親切な予約アシスタントです。", id="japanese-prefix-mid-clause"), + pytest.param('PROMPT = "あなたは予約・案内のアシスタントです。"', "あなたは予約・案内のアシスタントです。", id="japanese-middle-dot"), + pytest.param('PROMPT = ("คุณเป็นผู้ช่วย" f"จองโต๊ะที่สุภาพ")', "คุณเป็นผู้ช่วยจองโต๊ะที่สุภาพ", id="thai"), + pytest.param('PROMPT = "သင်သည် စားပွဲ ကြိုတင်မှာယူမှု အကူအညီပေးသူ ဖြစ်သည်။"', "သင်သည် စားပွဲ ကြိုတင်မှာယူမှု အကူအညီပေးသူ ဖြစ်သည်။", id="burmese"), + pytest.param('PROMPT = "អ្នកគឺជាជំនួយការកក់តុដែលគួរសម។"', "អ្នកគឺជាជំនួយការកក់តុដែលគួរសម។", id="khmer"), ], ) def test_an_unspaced_prompt_holds_its_quote(tmp_path, root, written, quote): @@ -2478,7 +2482,7 @@ def test_a_crafted_file_of_one_byte_lines_and_triple_quotes_costs_little(root): @pytest.mark.parametrize( "file", - ["notes.org", "rules.mdc", "CHANGELOG", "CHANGELOG.md", "LICENSE", "CONTRIBUTING.rst"], + ["notes.org", "rules.mdc", "CHANGELOG", "CHANGELOG.md", "LICENSE", "CONTRIBUTING", "README"], ) def test_more_prose_files_hold_no_model(root, file): (root / file).write_text('# notes\nmodel: "claude-3-opus"\n', encoding="utf-8") @@ -2501,7 +2505,8 @@ def test_a_file_named_after_readme_is_code(root): [ pytest.param("You are pair.", (1, 4), True, id="short-and-not-in-the-lines"), pytest.param("Be warm and brief.", (6, 12), False, id="short-and-in-the-lines"), - pytest.param("TBD", (1, 4), True, id="a-placeholder-with-nothing-held"), + # Too few letters to say anything: not looked for, like no quote at all. + pytest.param("TBD", (1, 4), False, id="a-placeholder-with-nothing-held"), ], ) def test_a_short_quote_is_looked_for_whole(tmp_path, root, quote, lines, refused): @@ -2673,3 +2678,250 @@ def test_lines_without_a_triple_quote_are_not_scanned(root, monkeypatch): assert problems == [] # The block's opening and closing lines, and the cited line: nothing between. assert scanned == ['"""', '"""', 'MODEL = "gpt-4o"'] + + +# Round 3: a short or list quote is held like any other + + +def test_a_short_quote_is_held_and_its_lines_asked_about(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt="Be warm and brief.", system_prompt_from=imports) + ) + assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 1-4")) + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", "Be warm and brief.") + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + # A quote this short can't stand for the prompt: the lines go out, on record as unconfirmed. + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_from"] == imports + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +@pytest.mark.parametrize( + "fields", + [ + pytest.param({"system_prompt_excerpt": _BOOKING_QUOTE.split(". ")}, id="a-list"), + pytest.param({"prompt": _BOOKING_QUOTE.split(". ")}, id="a-list-under-an-alias"), + pytest.param({"system_prompt": _BOOKING_QUOTE}, id="an-alias"), + ], +) +def test_a_quote_in_another_shape_is_held(tmp_path, root, fields): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports, **fields)) + assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 1-4")) + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + assert text_of(second).startswith(REFUSED + "system_prompt_from: the lines named (agent.py 1-4) don't hold the quote you gave before") + + +def test_a_list_quote_that_passes_replaces_the_held_one(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + prompt = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + paraphrase = "Acme booking agent: helps guests reserve tables and never shares bookings." + + assert submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=paraphrase, system_prompt_from=prompt)).get("is_error") + assert not submit( + payload(hard_constraints=["Be polite."], prompt=_BOOKING_QUOTE.split(". "), system_prompt_from=prompt) + ).get("is_error") + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=prompt)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_from"] == prompt + assert "unconfirmed_prompt_from" not in saved(destination) + + +def test_a_placeholder_with_the_right_lines_keeps_the_lines(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + prompt = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + again = payload(hard_constraints=["Be polite."], system_prompt_excerpt="see lines", system_prompt_from=prompt) + + assert submit(again).get("is_error") + second = submit(again) + + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + assert saved(destination)["unconfirmed_prompt_from"] == prompt + + +def test_two_calls_in_one_message_are_each_asked(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + server = contract_tools(destination, source_root=root) + tool = next(one for one in server.tools if one.name == "submit_contract") + paraphrase = "Acme booking agent: helps guests reserve tables and never shares bookings." + both = payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=paraphrase, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}], + ) + + async def send_together(): + return await asyncio.gather(tool.handler(dict(both)), tool.handler(dict(both))) + + first, second = asyncio.run(send_together()) + + # Neither had seen a refusal when it was sent, so both are refused. + assert first.get("is_error") and second.get("is_error") + assert not (destination / "contract.json").exists() + + +def test_a_quote_sent_alone_after_its_lines_were_refused_is_not_asked_for_lines(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + assert submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ).get("is_error") + second = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE)) + + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + + +def test_a_new_quote_with_wrong_lines_after_lines_alone_were_refused_is_kept(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + quote = "You are Acme's booking agent. Help guests book tables." + + assert not submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}]) + ).get("is_error") + assert submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)).get("is_error") + third = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=quote, system_prompt_from=imports)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == quote + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +def test_lines_set_aside_in_a_call_refused_for_something_else_are_said(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + wrong = payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + assert submit(wrong).get("is_error") + + second = submit({**wrong, "model": {"name": "gpt-4o", "path": "agent.py", "line": 1}}) + + assert "model: the name gpt-4o is not the value written on line 1" in text_of(second) + assert ( + "system_prompt_from: the lines named (agent.py 1-4) are left out and the quote is kept, " + "since they don't hold it" + ) in text_of(second) + + +# Round 3: pins for the line scan + + +@pytest.mark.parametrize("quote", ['x = "' + '\\"' * 40_000, "x = `" + "\\`" * 40_000]) +def test_an_unclosed_string_of_escapes_costs_one_pass(root, quote): + (root / "crafted.ts").write_text(quote + "\n", encoding="utf-8") + + _, _, took = checked(root, {"name": "gpt-4o", "path": "crafted.ts", "line": 1}) + + assert took < 0.25 + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("agent.ts", 'const model = "gpt-4o"; /* fast */', "gpt-4o", id="ts-comment-after"), + pytest.param("Agent.java", 'String model = "gpt-4o"; /* the default */', "gpt-4o", id="java-comment-after"), + pytest.param("llm.py", 'MODEL = "gpt-4o"; PROMPT = """', "gpt-4o", id="a-block-opening-after"), + pytest.param("llm.py", ' * 2, model="gpt-4o")', "gpt-4o", id="python-continuation-star"), + pytest.param("run.sh", "echo a#b; MODEL=gpt-4o", "gpt-4o", id="shell-hash-in-a-word"), + pytest.param("run.sh", "ls /*; MODEL=gpt-4o", "gpt-4o", id="shell-slash-star"), + pytest.param("build.groovy", 'a #b; def model = "gpt-4o"', "gpt-4o", id="groovy-hash"), + pytest.param("license.py", 'MODEL = "gpt-4o"', "gpt-4o", id="license-py-is-code"), + pytest.param("readme.py", 'MODEL = "gpt-4o"', "gpt-4o", id="readme-py-is-code"), + pytest.param("llm.py", "x = rb'Agent #1'; MODEL = \"gpt-4o\"", "gpt-4o", id="two-letter-prefix"), + pytest.param("llm.py", "x = 'a\\'b # c'; MODEL = \"gpt-4o\"", "gpt-4o", id="escaped-apostrophe"), + pytest.param("agent.ts", 'u = `${b}//v1`; m = "gpt-4o"', "gpt-4o", id="backtick-string"), + ], +) +def test_more_honest_lines_after_round_3(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "suffix", + [".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", + ".java", ".go", ".rs", ".swift", ".kt", ".kts", ".cs", ".scala", ".dart", ".m", ".mm", + ".groovy", ".gradle"], +) +def test_hash_is_no_comment_in_these_languages(root, suffix): + (root / f"agent{suffix}").write_text('\na = b #c; m = "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": f"agent{suffix}", "line": 2}) + + assert problems == [] + + +def test_a_tab_before_hash_starts_a_yaml_comment(root): + (root / "cfg.yaml").write_text('\nnote: a\t# "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "cfg.yaml", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "cfg.yaml")] + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("llm.py", 'n = a // 2; model = "gpt-4o"', "gpt-4o", id="python-floor-division-spaced"), + pytest.param("init.lua", 'local n = #t; model = "gpt-4o"', "gpt-4o", id="lua-length"), + pytest.param("llm.py", "x = t'Agent #{n}'; MODEL = \"gpt-4o\"", "gpt-4o", id="python-t-string"), + pytest.param("config.yaml.j2", "model: gpt-4o", "gpt-4o", id="a-config-template"), + ], +) +def test_more_honest_lines_after_round_3_probes(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("env.py", "model = gpt-4o", "gpt-4o", id="env-py-is-code"), + pytest.param("dockerfile_gen.py", "MODEL = gpt-4o", "gpt-4o", id="dockerfile-py-is-code"), + pytest.param("agent.js", 'call()// "gpt-4o"', "gpt-4o", id="js-comment-after-a-call"), + pytest.param("agent.js", 'x = 1// "gpt-4o"', "gpt-4o", id="js-comment-after-a-number"), + pytest.param("agent.ts", 'x = a[1]// "gpt-4o"', "gpt-4o", id="js-comment-after-a-subscript"), + ], +) +def test_more_wrong_values_after_round_3_probes(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] From 3412cbbe586171853478661057badeac0f3c7507 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 21:30:40 +0530 Subject: [PATCH 06/10] test(harness): pin the rules plants found unpinned, and drop a check that can't fail (TH-8953) Plants for round 3's fixes left eleven survivors. Four were caught by other tests. Six get pins: URLs, globs, ${#x} and $# in a script without a suffix; rt/tr string prefixes; a triple quote after floor division in Python; a quote sent alone after lines alone were refused is not asked for lines. _opens_a_string's 'whole' check could not fail: a prefix is one or two letters and the scan looks back three, so it is removed. Signed-off-by: khushalsonawat --- src/fi/alk/harness/model_citation.py | 6 +-- tests/harness/test_contract_model.py | 59 ++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/model_citation.py b/src/fi/alk/harness/model_citation.py index 5183d111e..c6d8a7d14 100644 --- a/src/fi/alk/harness/model_citation.py +++ b/src/fi/alk/harness/model_citation.py @@ -162,14 +162,14 @@ def _comment_tokens( def _opens_a_string(line: str, at: int) -> bool: """Whether the apostrophe at ``at`` can open a string: not inside a word (Pair's, it's, x' in Haskell) and not after a backslash (it\\'s), unless the word is a string prefix (r'...', - rb'...'). Looks at three characters at most.""" + rb'...'). Looks at three characters at most: a prefix is one or two letters, so three letters + before the apostrophe are a word, whatever comes before them.""" start = at while start > 0 and at - start < 3 and line[start - 1].isalnum(): start -= 1 if start == at: return at == 0 or line[at - 1] != "\\" - whole = start == 0 or not line[start - 1].isalnum() - return whole and line[start:at].lower() in _STRING_PREFIXES + return line[start:at].lower() in _STRING_PREFIXES def _line_scan( diff --git a/tests/harness/test_contract_model.py b/tests/harness/test_contract_model.py index e2d0c8a7f..63757990e 100644 --- a/tests/harness/test_contract_model.py +++ b/tests/harness/test_contract_model.py @@ -2925,3 +2925,62 @@ def test_more_wrong_values_after_round_3_probes(root, file, written, name): problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) assert problems == [not_named(name, 2, file)] + + +# The settled round: pins for rules the plants found unpinned + + +@pytest.mark.parametrize( + "written", + [ + pytest.param('curl http://x/v1; MODEL="gpt-4o"', id="a-url"), + pytest.param('cp data/* out; MODEL="gpt-4o"', id="a-glob"), + pytest.param('n=${#MODELS}; MODEL="gpt-4o"', id="a-length"), + pytest.param('echo $#; MODEL="gpt-4o"', id="a-count"), + ], +) +def test_a_script_without_a_suffix_keeps_its_model(root, written): + (root / "start").write_text(f"#!/bin/sh\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": "gpt-4o", "path": "start", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == "gpt-4o" + + +@pytest.mark.parametrize("prefix", ["rt", "tr", "RT"]) +def test_a_raw_template_string_is_a_string(root, prefix): + (root / "llm.py").write_text(f"\nx = {prefix}'Agent #{{n}}'; MODEL = \"gpt-4o\"\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [] + + +def test_a_triple_quote_after_floor_division_opens_a_block(root): + (root / "llm.py").write_text( + 'half = n // 2; DOC = """\nmodel = "gpt-4o"\n"""\n', encoding="utf-8" + ) + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "llm.py")] + + +def test_a_quote_alone_after_lines_alone_were_refused_is_not_asked_for_lines(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + assert not submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}]) + ).get("is_error") + assert submit( + payload(hard_constraints=["Be polite."], + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ).get("is_error") + third = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE From 770e92a2752b88fe815531b0109c99454e2a3c77 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 22:02:02 +0530 Subject: [PATCH 07/10] fix(harness): answer the settled round's quote and rules findings (TH-8953) Lines are numbered at each newline, as the reading's Read and Grep number them, so a form feed, a lone CR or U+2028 no longer moves a cited line. A JSON-escaped surrogate pair reads as one character. JSON keeps // and /* */ comments (JSONC). TypeScript modules and Scala scripts have no # comments; .env files read # after a space by suffix too; Python files named like a Dockerfile or .env read # anywhere. An underscore counts in the apostrophe look-back. A template named for configuration (.env.j2) is configuration. A dead clause is removed. Pins for the 6- and 16-letter boundaries, the not-a-regular-file log line, an escape past U+10FFFF, and eight rule parts. SKILL item 6 says what happens with short quotes. Signed-off-by: khushalsonawat --- src/fi/alk/harness/model_citation.py | 42 ++-- .../harness/skills/understand-agent/SKILL.md | 9 +- tests/harness/test_contract_model.py | 191 +++++++++++++++++- 3 files changed, 224 insertions(+), 18 deletions(-) diff --git a/src/fi/alk/harness/model_citation.py b/src/fi/alk/harness/model_citation.py index c6d8a7d14..54ef7a9df 100644 --- a/src/fi/alk/harness/model_citation.py +++ b/src/fi/alk/harness/model_citation.py @@ -44,18 +44,20 @@ ) _CONFIG_NAMES = frozenset({"makefile", "gnumakefile", ".flaskenv"}) # Languages where # isn't a comment: C and C++ (#define), JavaScript and TypeScript (#field), -# Java, Go, Rust (#[attr]), Swift (#if), Kotlin, C#, Scala, Dart, Objective-C, Groovy, Gradle, -# Lua (#t is a length). +# Java, Go, Rust (#[attr]), Swift (#if), Kotlin, C#, Scala (A#B), Dart, Objective-C, Groovy, +# Gradle, Lua (#t is a length). _NO_HASH_COMMENT_SUFFIXES = frozenset( { ".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".java", ".go", ".rs", ".swift", ".kt", ".kts", ".cs", ".scala", ".dart", ".m", ".mm", - ".groovy", ".gradle", ".lua", + ".groovy", ".gradle", ".lua", ".mts", ".cts", ".sc", } ) # Files where # starts a comment only at a line's start or after a space: YAML, shell, Makefile # and Dockerfile (a#b is a word there), and .env files. -_SPACED_HASH_SUFFIXES = frozenset({".yaml", ".yml", ".sh", ".bash", ".zsh", ".mk", ".dockerfile"}) +_SPACED_HASH_SUFFIXES = frozenset( + {".yaml", ".yml", ".sh", ".bash", ".zsh", ".mk", ".dockerfile", ".env"} +) # Files with no block comments, nor //: there `* ` starts no comment (a Python continuation # line), and /* and // are text (ls /*, a // 2). _NO_BLOCK_COMMENT_SUFFIXES = frozenset( @@ -91,6 +93,7 @@ r"|([0-7]{1,3}))" ) _NOT_A_LETTER = re.compile(r"[\W_]+") +_SURROGATE = re.compile("[\ud800-\udfff]") # Full-width letters and digits (3, A) as their ASCII forms: what NFKC would do for them, # without NFKC's growth elsewhere. _FULL_WIDTH = {code: code - 0xFEE0 for code in range(0xFF01, 0xFF5F)} @@ -162,10 +165,10 @@ def _comment_tokens( def _opens_a_string(line: str, at: int) -> bool: """Whether the apostrophe at ``at`` can open a string: not inside a word (Pair's, it's, x' in Haskell) and not after a backslash (it\\'s), unless the word is a string prefix (r'...', - rb'...'). Looks at three characters at most: a prefix is one or two letters, so three letters - before the apostrophe are a word, whatever comes before them.""" + rb'...'). Looks at three characters at most: a prefix is one or two letters, so three word + characters before the apostrophe (letters, digits, _) are a word, whatever comes before.""" start = at - while start > 0 and at - start < 3 and line[start - 1].isalnum(): + while start > 0 and at - start < 3 and (line[start - 1].isalnum() or line[start - 1] == "_"): start -= 1 if start == at: return at == 0 or line[at - 1] != "\\" @@ -310,8 +313,14 @@ def read_source(path: Path, limit: int) -> bytes | None: def source_lines(raw: bytes) -> list[str]: """The file's lines, whatever coding line it carries: no codec the file names gets to run - on it, and one file has one numbering wherever it is read.""" - return raw.decode("utf-8", errors="replace").splitlines() + on it, and one file has one numbering wherever it is read. Numbered as the reading's Read and + Grep number them: at each \\n, with a \\r before it dropped. Not at the other characters + str.splitlines breaks at (a form feed, a lone \\r, U+2028), which would move every line + after them.""" + lines = raw.decode("utf-8", errors="replace").split("\n") + if lines[-1] == "": + lines.pop() + return [line[:-1] if line.endswith("\r") else line for line in lines] def _bare_after_key(line: str, at: int, *, yaml: bool) -> bool: @@ -466,11 +475,13 @@ def is_config_file(path: Path) -> bool: suffix in CONFIG_SUFFIXES or name in _CONFIG_NAMES or (not code and name.startswith((".env", "env.", "dockerfile"))) - or name.endswith(".env") ) def is_prose_file(path: Path) -> bool: + if path.suffix.lower() in _TEMPLATE_SUFFIXES and is_config_file(path): + # A template that renders configuration by its name: .env.j2, Dockerfile.j2. + return False return kind_suffix(path) in PROSE_SUFFIXES or ( not path.suffix and path.name.lower() in _PROSE_STEMS ) @@ -566,13 +577,15 @@ def model_problems(contract: AgentContract, source_root: Path) -> list[str]: elif ( suffix in _SPACED_HASH_SUFFIXES or name_lower in _CONFIG_NAMES - or name_lower.startswith((".env", "dockerfile")) + # dockerfile_gen.py and .env_loader.py are Python: # anywhere there. + or (config and name_lower.startswith((".env", "dockerfile"))) ): hash_comments = _HASH_SPACED else: hash_comments = _HASH_ANYWHERE - # Configuration files have no block comments either. - block_comments = not (config or suffix in _NO_BLOCK_COMMENT_SUFFIXES) + # Configuration files have no block comments, but JSON read as JSONC (appsettings.json, + # tsconfig.json) has: outside a string, // and /* can only be comments there. + block_comments = suffix == ".json" or not (config or suffix in _NO_BLOCK_COMMENT_SUFFIXES) if not named_on( lines[model.line - 1], name, @@ -620,6 +633,9 @@ def _comparable(text: str) -> str: """``text`` as a quote and the lines it was quoted from both read it: escapes decoded, HTML entities decoded, one Unicode form (NFC, which never grows the text much), case set aside.""" unescaped = _CHAR_ESCAPE.sub(_escape, text) + if _SURROGATE.search(unescaped): + # json.dump writes a character past U+FFFF as two escapes, one per half of its pair. + unescaped = unescaped.encode("utf-16", "surrogatepass").decode("utf-16", "replace") return unicodedata.normalize("NFC", html.unescape(unescaped)).translate(_FULL_WIDTH).casefold() diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index e431a1d9d..08c16edff 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -114,10 +114,11 @@ Find, in roughly this order: `system_prompt_from`, every part it is assembled from: it is copied into the contract as written, and every later stage decides what the agent should do from it. If the prompt is built or fetched at run time and has no lines to name, quote it in `system_prompt_excerpt`; the save - asks once whether lines can be named. A quote is word for word from the prompt, with ... where - you leave text out. The lines you name are checked against your last quote, given with them or - before; lines that don't hold it are asked about once more, and after that the quote is kept - and the lines are recorded as unconfirmed. + asks once whether lines can be named. A quote is word for word from the prompt, a sentence or + more, with ... where you leave text out. The lines you name are checked against your latest + such quote, given with them or before. Lines that don't hold it are refused; the next time, + the quote is kept and the lines are recorded as unconfirmed (with a quote of a few words, the + lines are kept, also recorded as unconfirmed). 7. **The modality.** How this agent is run: a voice session, a text conversation, or work it carries out on websites or a computer. This decides how it is later run, so getting it wrong diff --git a/tests/harness/test_contract_model.py b/tests/harness/test_contract_model.py index 63757990e..b0e3f06ef 100644 --- a/tests/harness/test_contract_model.py +++ b/tests/harness/test_contract_model.py @@ -527,7 +527,8 @@ def test_model_on_the_cited_line_is_saved_stripped(tmp_path, root): @pytest.mark.parametrize( "written, line", [ - pytest.param(b'# head\nA = 1\x0cB = 2\nMODEL = "gpt-4o"\n', 4, id="form-feed"), + # A form feed is no line break to the reading's Read, nor to the save. + pytest.param(b'# head\nA = 1\x0cB = 2\nMODEL = "gpt-4o"\n', 3, id="form-feed"), # Read as latin-1, \x85 is a line break and the model's line moves down one. pytest.param(b'# coding: latin-1\nA = "\x85"\nMODEL = "gpt-4o"\n', 3, id="latin-1-c1"), ], @@ -2984,3 +2985,191 @@ def test_a_quote_alone_after_lines_alone_were_refused_is_not_asked_for_lines(tmp assert not third.get("is_error"), third assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + + +# The settled round: lines numbered as the reading numbers them, and the reviewers' pins + + +@pytest.mark.parametrize( + "written, prompt_line, model_line", + [ + pytest.param( + b'# head\nA = 1\x0cB = 2\rC = 3\nNOTE = "a\xe2\x80\xa8b"\n' + b'PROMPT = "You are Acme\'s booking agent."\nMODEL = "gpt-4o"\n', + 4, 5, id="form-feed-lone-cr-and-u2028", + ), + pytest.param( + b'# head\r\nPROMPT = "You are Acme\'s booking agent."\r\nMODEL = "gpt-4o"\r\n', + 2, 3, id="crlf", + ), + pytest.param( + b'# head\nPROMPT = "You are Acme\'s booking agent."\nMODEL = "gpt-4o"', + 2, 3, id="no-newline-at-the-end", + ), + ], +) +def test_lines_are_numbered_at_each_newline(tmp_path, root, written, prompt_line, model_line): + (root / "agent.py").write_bytes(written) + destination = tmp_path / "out" + + result = accept_contract( + payload( + system_prompt_from=[{"path": "agent.py", "start_line": prompt_line, "end_line": prompt_line}], + model={"name": "gpt-4o", "path": "agent.py", "line": model_line}, + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == 'PROMPT = "You are Acme\'s booking agent."' + assert saved(destination)["model"]["line"] == model_line + + +def test_a_file_ending_in_a_newline_has_no_line_after_it(root): + (root / "agent.py").write_text('\nMODEL = "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "agent.py", "line": 3}) + + assert problems == ["model: line 3 is outside agent.py, which has 2 lines"] + + +@pytest.mark.parametrize( + "letter, escaped", + [ + pytest.param("\U00020000", "\\ud840\\udc00", id="cjk-extension-b"), + pytest.param("\U0001d400", "\\ud835\\udc00", id="math-bold"), + ], +) +def test_a_json_escaped_letter_past_ffff_holds_its_quote(letter, escaped): + from fi.alk.harness.model_citation import quote_problem + + quote = f"Greet each guest as {letter} and seat them at once." + written = '{"prompt": "' + quote.replace(letter, escaped) + '"}' + + assert quote_problem(quote, written, "prompt.json 1-1") is None + + +def test_an_escape_past_the_last_character_is_kept_as_written(): + from fi.alk.harness.model_citation import quote_problem + + quote = "Order a table \\U00110000 for two guests tonight." + + assert quote_problem(quote, 'PROMPT = "' + quote + '"', "prompt.py 1-1") is None + + +@pytest.mark.parametrize( + "quote, says", + [ + # 6 letters is the floor of a quote that says something, looked for whole. + pytest.param("Be kind", True, id="six-letters"), + pytest.param("Be wry", False, id="five-letters"), + ], +) +def test_six_letters_say_something(quote, says): + from fi.alk.harness.model_citation import quote_problem + + assert (quote_problem(quote, 'x = "nothing here"', "a.py 1-1") is not None) == says + + +def test_sixteen_letters_are_searched_with_gaps(): + from fi.alk.harness.model_citation import quote_pieces, quote_problem + + # 16 letters: words in order, with code between them. 15 would be looked for whole. + quote = "Be warm and kind now" + written = 'P = "Be warm " + tone + " and kind now"' + + assert quote_pieces(quote) + assert quote_problem(quote, written, "a.py 1-1") is None + assert not quote_pieces("Be warm and kind no") + + +def test_a_pipe_is_logged_as_not_a_regular_file(tmp_path, caplog): + from fi.alk.harness.model_citation import read_source + + if not hasattr(os, "mkfifo"): + pytest.skip("no named pipes here") + pipe = tmp_path / "pipe" + os.mkfifo(pipe) + answer: list[bytes | None] = [] + with caplog.at_level("INFO", logger="fi.alk.harness.model_citation"): + reading = threading.Thread(target=lambda: answer.append(read_source(pipe, 10)), daemon=True) + reading.start() + reading.join(timeout=2) + + assert answer == [None] + assert [record.getMessage() for record in caplog.records] == [ + "source file not read (not a regular file)" + ] + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("init.lua", 'local n = a // 2; model = "gpt-4o"', "gpt-4o", id="lua-floor-division"), + pytest.param("Makefile", "\techo a#b; MODEL=gpt-4o", "gpt-4o", id="makefile-hash-in-a-word"), + pytest.param(".flaskenv", "URL=http://x/#a MODEL=gpt-4o", "gpt-4o", id="flaskenv-hash-in-a-word"), + pytest.param(".env.local", "URL=http://x/#a MODEL=gpt-4o", "gpt-4o", id="env-name-hash-in-a-word"), + pytest.param("Dockerfile.dev", "ENV A=a#b MODEL=gpt-4o", "gpt-4o", id="dockerfile-name-hash-in-a-word"), + pytest.param("run.bash", "echo a#b; MODEL=gpt-4o", "gpt-4o", id="bash-hash-in-a-word"), + pytest.param("run.zsh", "echo a#b; MODEL=gpt-4o", "gpt-4o", id="zsh-hash-in-a-word"), + pytest.param("rules.mk", "\techo a#b; MODEL=gpt-4o", "gpt-4o", id="mk-hash-in-a-word"), + pytest.param("app.dockerfile", "ENV A=a#b MODEL=gpt-4o", "gpt-4o", id="dockerfile-suffix-hash-in-a-word"), + pytest.param("prod.env", "URL=http://x/#a MODEL=gpt-4o", "gpt-4o", id="env-suffix-hash-in-a-word"), + pytest.param("agent.mts", 'class A { #model = "gpt-4o"; }', "gpt-4o", id="mts-private-field"), + pytest.param("agent.cts", 'this.#model = "gpt-4o";', "gpt-4o", id="cts-private-field"), + pytest.param("agent.sc", 'val t: A#B = x; val model = "gpt-4o"', "gpt-4o", id="scala-script-type-projection"), + pytest.param(".env.j2", "MODEL=gpt-4o", "gpt-4o", id="env-template"), + pytest.param("Dockerfile.j2", "ENV MODEL=gpt-4o", "gpt-4o", id="dockerfile-template"), + pytest.param("config.yaml.j2", " - gpt-4o", "gpt-4o", id="yaml-template-list-item"), + pytest.param("appsettings.json", '"ModelId": "gpt-4o", // the default', "gpt-4o", id="json-comment-after"), + pytest.param("agent.json", '"url": "http://x//y", "model": "gpt-4o"', "gpt-4o", id="json-slashes-in-a-string"), + ], +) +def test_more_honest_lines_after_the_settled_round(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("appsettings.json", '// "ModelId": "gpt-35-turbo",', "gpt-35-turbo", id="json-line-comment"), + pytest.param("appsettings.json", '"ModelId": "gpt-4o" /* was "gpt-35-turbo" */', "gpt-35-turbo", id="json-block-comment"), + pytest.param("agent.json", '"model": "gpt-4o", // "gpt-4-turbo" is slower', "gpt-4-turbo", id="json-comment-after-a-value"), + pytest.param("agent.json", '* "model": "gpt-4",', "gpt-4", id="json-block-comment-middle-line"), + pytest.param("dockerfile_gen.py", 'MODEL = DEFAULT#"gpt-4"', "gpt-4", id="python-named-like-a-dockerfile"), + pytest.param(".env_loader.py", 'MODEL = DEFAULT#"gpt-4"', "gpt-4", id="python-named-like-an-env-file"), + pytest.param("cfg.yaml", "note: the cart's total # model: 'gpt-4'", "gpt-4", id="a-word-ending-like-a-prefix"), + pytest.param("cfg.yaml", "note: it_t's ok # model: 'gpt-4'", "gpt-4", id="an-underscore-before-a-prefix"), + pytest.param("env.rb", "MODEL = gpt-4o", "gpt-4o", id="env-rb-is-code"), + ], +) +def test_more_wrong_values_after_the_settled_round(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +def test_a_python_template_is_text(root): + (root / "agent.py.j2").write_text('MODEL = "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "agent.py.j2", "line": 1}) + + assert problems == [prose("agent.py.j2")] + + +def test_a_type_projection_does_not_hide_a_triple_quote(root): + (root / "agent.scala").write_text( + 'val t: A#B = x; val p = """\nmodel = "gpt-4o"\n"""\n', encoding="utf-8" + ) + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "agent.scala", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "agent.scala")] From 4396675bccc0f86ab45763a3492a034d7db30cc2 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 22:17:07 +0530 Subject: [PATCH 08/10] fix(harness): answer the settled round's flows findings (TH-8953) A call's own short quote, found letter for letter in its own lines, passes and becomes the held quote, so a verbatim short quote sent after a refused paraphrase keeps the right lines. The quote is read after nulls are dropped, as the contract reads it, and a blank quote is none. The once-only nudges decide from what was asked before the call, like the refusals. The refusals say what the fallback will do (a quote of a few words keeps the lines), and a saved call that took the fallback says so. Signed-off-by: khushalsonawat --- src/fi/alk/harness/tools.py | 54 ++++++-- tests/harness/test_contract_model.py | 185 +++++++++++++++++++++++++-- 2 files changed, 220 insertions(+), 19 deletions(-) diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index 9818bba55..979fc947a 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -262,7 +262,7 @@ def _checked_against_source( if source_root is not None: from .source_tool_evidence import tool_evidence_problems - quote = contract.system_prompt_excerpt + quote = own = contract.system_prompt_excerpt # Lines named with no quote of their own (or one too short to say anything), checked # against the reading's latest quote, which submit_contract holds. alone = False @@ -287,7 +287,19 @@ def _checked_against_source( f"{where.path[:200]} {where.start_line}-{where.end_line}" for where in contract.system_prompt_from ) - if mismatch := quote_problem(quote, contract.system_prompt_excerpt, named): + mismatch = quote_problem(quote, contract.system_prompt_excerpt, named) + if ( + mismatch + and alone + and says_something(own) + and quote_problem(own, contract.system_prompt_excerpt, named) is None + ): + # The call's own short quote is in its lines letter for letter: that is this + # call's evidence, and the quote it passed with is the reading's latest now. + mismatch = None + if held is not None: + held["prompt-quote"] = own + if mismatch: if asked is None: problems.append(mismatch) elif alone and "prompt-lines-alone" not in asked: @@ -295,12 +307,17 @@ def _checked_against_source( # asked once to quote these lines, before either is given up. asked.add("prompt-lines-alone") shown = " ".join(quote.split())[:80] + outcome = ( + "the quote is kept and the lines are recorded as unconfirmed" + if quote_pieces(quote) + else "the lines are copied as the prompt and recorded as unconfirmed, " + "since that quote is too short to stand for it" + ) problems.append( f"system_prompt_from: the lines named ({named}) don't hold the quote " f"you gave before ({shown}); if these lines are the prompt, quote " "them word for word in system_prompt_excerpt, with ... where you " - "leave text out. If they are refused again, the quote is kept and " - "the lines are recorded as unconfirmed" + f"leave text out. If they are refused again, {outcome}" ) elif asked & {"prompt-quote", "prompt-lines-alone"}: # Either way the lines are kept on record as unconfirmed. @@ -326,8 +343,9 @@ def _checked_against_source( asked.add("prompt-quote") problems.append( "system_prompt_from: lines named later are checked against this " - "quote until you give another; a later mismatch, with any quote, " - "keeps the quote and records the lines as unconfirmed" + "quote until you give another; a later mismatch records the lines as " + "unconfirmed and keeps the quote, or the lines if the quote is only a " + "few words" ) if cited := model_problems(contract, source_root): if asked is not None and "model" in asked: @@ -406,13 +424,24 @@ def _save_contract(contract: AgentContract, destination: Path) -> dict[str, Any] os.replace(partial, path) finally: partial.unlink(missing_ok=True) - return _ok( + said = ( f"Accepted and saved to {path}.\n" f"{len(contract.tools)} tools: {', '.join(sorted(contract.tool_names()))}\n" f"{len(contract.hard_constraints)} rules, " f"{len(contract.real_use_cases)} use cases, " f"{len(contract.open_questions)} open questions." ) + if contract.unconfirmed_prompt_from: + named = ", ".join( + f"{where.path[:200]} {where.start_line}-{where.end_line}" + for where in contract.unconfirmed_prompt_from + ) + kept = "the lines are the prompt" if contract.system_prompt_from else "the quote is the prompt" + said += ( + f"\nsystem_prompt_from: the lines named ({named}) are recorded as unconfirmed, since " + f"they don't hold the quote; {kept}." + ) + return _ok(said) # One prompt file, and what the excerpt copies from all of them: the same file named a hundred @@ -1425,7 +1454,9 @@ def contract_tools( async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: payload = dict(unwrapped(args)) # The quote as the contract will read it: under any alias, a list joined. - excerpt = prompt_quote(payload) + excerpt = prompt_quote(_without_nulls(payload)) + if not excerpt.strip(): + excerpt = "" # Held before anything can send this call back (a nudge, a refusal, a schema problem), so # lines named later without a quote are checked against the reading's latest one. A short # quote is held only while nothing longer is: it says less. @@ -1715,7 +1746,7 @@ def rows_for(collection: str) -> list[dict[str, Any]]: and not payload.get("system_prompt_from") # Asked already, with the "prompt" nudge or a refusal of named lines: the lines # aren't asked for twice. - and not nudged & {"prompt", "prompt-quote", "prompt-lines-alone"}, + and not asked_before & {"prompt", "prompt-quote", "prompt-lines-alone"}, "system_prompt_excerpt is a quote, and system_prompt_from names no lines. If the " "prompt is written in the source, name its lines in system_prompt_from so it is " "copied whole.", @@ -1723,8 +1754,9 @@ def rows_for(collection: str) -> list[dict[str, Any]]: ] # All of them together, and each only once. Nudging in sequence would cost a turn per # nudge and read as though the requirements were being invented one at a time. - say = [said for key, when, said in thin if when and key not in nudged] - nudged.update(key for key, when, _ in thin if when) + say = [said for key, when, said in thin if when and key not in asked_before] + asked_before.update(key for key, when, _ in thin if when) + nudged.update(asked_before) if say: return _problems( say + ["If any of these genuinely does not apply, submit again as is."] diff --git a/tests/harness/test_contract_model.py b/tests/harness/test_contract_model.py index b0e3f06ef..b4002b7c6 100644 --- a/tests/harness/test_contract_model.py +++ b/tests/harness/test_contract_model.py @@ -843,12 +843,18 @@ def prose(path: str) -> str: ) -def lines_alone(named: str, quote: str) -> str: +def lines_alone(named: str, quote: str, *, short: bool = False) -> str: + outcome = ( + "the lines are copied as the prompt and recorded as unconfirmed, since that quote is too " + "short to stand for it" + if short + else "the quote is kept and the lines are recorded as unconfirmed" + ) return ( f"system_prompt_from: the lines named ({named}) don't hold the quote you gave before " f"({' '.join(quote.split())[:80]}); if these lines are the prompt, quote them word for " - "word in system_prompt_excerpt, with ... where you leave text out. If they are refused " - "again, the quote is kept and the lines are recorded as unconfirmed" + f"word in system_prompt_excerpt, with ... where you leave text out. If they are refused " + f"again, {outcome}" ) @@ -1422,8 +1428,8 @@ def test_lines_refused_twice_leave_the_quote(tmp_path, root): REFUSED + not_the_quote("prompt.py 1-2") + "\n - system_prompt_from: lines named later are checked against this quote until you " - "give another; a later mismatch, with any quote, keeps the quote and records the lines as " - "unconfirmed" + "give another; a later mismatch records the lines as unconfirmed and keeps the quote, or the " + "lines if the quote is only a few words" ) second = submit(wrong) assert not second.get("is_error"), second @@ -2695,11 +2701,16 @@ def test_a_short_quote_is_held_and_its_lines_asked_about(tmp_path, root): ) assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 1-4")) second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) - assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", "Be warm and brief.") + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", "Be warm and brief.", short=True) third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) - # A quote this short can't stand for the prompt: the lines go out, on record as unconfirmed. + # A quote this short can't stand for the prompt: the lines go out, on record as unconfirmed, + # and the reply says so. assert not third.get("is_error"), third + assert text_of(third).endswith( + "\nsystem_prompt_from: the lines named (agent.py 1-4) are recorded as unconfirmed, since " + "they don't hold the quote; the lines are the prompt." + ) assert saved(destination)["system_prompt_from"] == imports assert saved(destination)["unconfirmed_prompt_from"] == imports @@ -2774,7 +2785,9 @@ async def send_together(): first, second = asyncio.run(send_together()) - # Neither had seen a refusal when it was sent, so both are refused. + # Neither had seen a refusal when it was sent, so both are refused. This models a dispatcher + # that runs one message's calls side by side (ADK, the Vertex backend). The Claude backend runs + # them one after another, and there the second sees the first's refusal: a known limit. assert first.get("is_error") and second.get("is_error") assert not (destination / "contract.json").exists() @@ -3173,3 +3186,159 @@ def test_a_type_projection_does_not_hide_a_triple_quote(root): problems, _, _ = checked(root, {"name": "gpt-4o", "path": "agent.scala", "line": 2}) assert problems == [not_named("gpt-4o", 2, "agent.scala")] + + +# The settled round: the flows + + +_SHORT_PROMPT = 'import os\n\nINSTRUCTIONS = "Be warm and brief."\n' +_PARAPHRASE = "The agent books tables for Acme guests, keeps other bookings private, and confirms first." + + +@pytest.mark.parametrize( + "source, lines, quote, paraphrase", + [ + pytest.param( + _SHORT_PROMPT, (3, 3), "Be warm and brief.", + "The agent is told to be warm and to keep its answers brief.", id="a-whole-short-prompt", + ), + pytest.param(_BOOKING, (6, 12), "Be warm and brief.", _PARAPHRASE, id="one-sentence-of-a-long-prompt"), + ], +) +@pytest.mark.parametrize("then", ["the-same-again", "the-lines-alone"]) +def test_a_short_verbatim_quote_after_a_refused_paraphrase_keeps_the_lines( + tmp_path, root, source, lines, quote, paraphrase, then +): + (root / "agent.py").write_text(source, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + named = [{"path": "agent.py", "start_line": lines[0], "end_line": lines[1]}] + + assert submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=paraphrase, system_prompt_from=named) + ).get("is_error") + second = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=quote, system_prompt_from=named) + ) + assert not second.get("is_error"), second + again = ( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=quote, system_prompt_from=named) + if then == "the-same-again" + else payload(hard_constraints=["Be polite."], system_prompt_from=named) + ) + third = submit(again) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_from"] == named + assert "unconfirmed_prompt_from" not in saved(destination) + + +def test_a_short_quote_not_in_its_lines_is_still_checked_against_the_held_one(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + assert submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_PARAPHRASE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}]) + ).get("is_error") + second = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt="Be warm and brief.", system_prompt_from=imports) + ) + + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _PARAPHRASE) + + +def test_a_null_in_a_list_quote_is_dropped_before_it_is_held(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + right = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + + result = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=["Be warm", None, "and brief."], + system_prompt_from=right) + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_from"] == right + + +def test_a_blank_quote_is_no_quote(tmp_path, root): + submit = submitter(tmp_path / "out", root) + + result = submit(payload(system_prompt_excerpt=" ")) + + assert result.get("is_error") + assert "no hard_constraints and no system_prompt_excerpt" in text_of(result) + assert PROMPT_LINES not in text_of(result) + + +def test_two_quotes_alone_in_one_message_are_each_asked_for_lines(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + server = contract_tools(destination, source_root=root) + tool = next(one for one in server.tools if one.name == "submit_contract") + both = payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE) + + async def send_together(): + return await asyncio.gather(tool.handler(dict(both)), tool.handler(dict(both))) + + first, second = asyncio.run(send_together()) + + assert PROMPT_LINES in text_of(first) and PROMPT_LINES in text_of(second) + assert not (destination / "contract.json").exists() + + +def test_short_lines_set_aside_in_a_call_refused_for_something_else_are_said(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + wrong = payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="Be warm and brief.", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + assert submit(wrong).get("is_error") + + second = submit({**wrong, "model": {"name": "gpt-4o", "path": "agent.py", "line": 1}}) + + assert "model: the name gpt-4o is not the value written on line 1" in text_of(second) + assert ( + "system_prompt_from: the lines named (agent.py 1-4) are copied unconfirmed, since they " + "don't hold the short quote" + ) in text_of(second) + + +@pytest.mark.parametrize( + "fields", + [ + pytest.param({"system_prompt_excerpt": ["Be warm", "and brief."]}, id="a-list"), + pytest.param({"prompt": ["Be warm", None, "and brief."]}, id="a-list-with-a-null-under-an-alias"), + pytest.param({"system_prompt_excerpt": 42}, id="a-number"), + ], +) +def test_the_held_quote_is_the_quote_the_contract_reads(fields): + from fi.alk.harness.contract import prompt_quote + from fi.alk.harness.tools import _without_nulls + + contract = AgentContract.model_validate(_without_nulls(payload(**fields))) + + assert prompt_quote(_without_nulls(payload(**fields))) == contract.system_prompt_excerpt + + +def test_a_saved_call_that_kept_the_quote_says_so(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + wrong = payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, system_prompt_from=imports) + + assert submit(wrong).get("is_error") + second = submit(wrong) + + assert not second.get("is_error"), second + assert text_of(second).endswith( + "\nsystem_prompt_from: the lines named (agent.py 1-4) are recorded as unconfirmed, since " + "they don't hold the quote; the quote is the prompt." + ) From afc428a954c70514a73b6e0bced3b611992507e6 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 22:32:15 +0530 Subject: [PATCH 09/10] test(harness): pin the null-dropping on wrong lines, and drop a no-op update (TH-8953) Plants on 4396675b: a list quote with a null and the wrong lines is refused as the call's own quote; asked_before's update in the nudges did nothing, since a call either returns with the nudge or had nothing new to add. Signed-off-by: khushalsonawat --- src/fi/alk/harness/tools.py | 5 +++-- tests/harness/test_contract_model.py | 15 ++++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index 979fc947a..c30958e4d 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -290,6 +290,8 @@ def _checked_against_source( mismatch = quote_problem(quote, contract.system_prompt_excerpt, named) if ( mismatch + # Only lines checked against a held quote can hold their own instead (and + # the second search is spent only then). and alone and says_something(own) and quote_problem(own, contract.system_prompt_excerpt, named) is None @@ -1755,8 +1757,7 @@ def rows_for(collection: str) -> list[dict[str, Any]]: # All of them together, and each only once. Nudging in sequence would cost a turn per # nudge and read as though the requirements were being invented one at a time. say = [said for key, when, said in thin if when and key not in asked_before] - asked_before.update(key for key, when, _ in thin if when) - nudged.update(asked_before) + nudged.update(key for key, when, _ in thin if when) if say: return _problems( say + ["If any of these genuinely does not apply, submit again as is."] diff --git a/tests/harness/test_contract_model.py b/tests/harness/test_contract_model.py index b4002b7c6..7ed97596e 100644 --- a/tests/harness/test_contract_model.py +++ b/tests/harness/test_contract_model.py @@ -3250,19 +3250,24 @@ def test_a_short_quote_not_in_its_lines_is_still_checked_against_the_held_one(tm assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _PARAPHRASE) -def test_a_null_in_a_list_quote_is_dropped_before_it_is_held(tmp_path, root): +@pytest.mark.parametrize("lines", [(6, 12), (1, 4)], ids=["the-right-lines", "the-wrong-lines"]) +def test_a_null_in_a_list_quote_is_dropped_before_it_is_held(tmp_path, root, lines): (root / "agent.py").write_text(_BOOKING, encoding="utf-8") destination = tmp_path / "out" submit = submitter(destination, root) - right = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + named = [{"path": "agent.py", "start_line": lines[0], "end_line": lines[1]}] result = submit( payload(hard_constraints=["Be polite."], system_prompt_excerpt=["Be warm", None, "and brief."], - system_prompt_from=right) + system_prompt_from=named) ) - assert not result.get("is_error"), result - assert saved(destination)["system_prompt_from"] == right + if lines == (6, 12): + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_from"] == named + else: + # Refused as this call's own quote, not as one given before with "None" in it. + assert text_of(result).startswith(REFUSED + not_the_quote("agent.py 1-4")) def test_a_blank_quote_is_no_quote(tmp_path, root): From b70fc9cbdfa2a2d880b5b1ebda4982afd3bd97b7 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 23:00:02 +0530 Subject: [PATCH 10/10] fix(harness): answer the settled round 2 findings (TH-8953) A short quote of 6-15 letters is looked for piece by piece at its ..., so "Be warm ... brief." holds its lines. A call's own short quote on its own lines replaces the held quote only when it is two words or more and the held quote isn't one that lines already held, so a name like PROMPT or a sentence repeated in a comment no longer overrides a verified quote. The messages say "fewer than 16 letters" and "refused again against it"; SKILL item 6 says when the lines are asked about once more. A template is configuration only when its stem is a configuration name (.env.j2, Dockerfile.j2), and system.prompt.j2 is a prompt. Pins for .conf, config.json.j2, quote precedence, padded short quotes, case and punctuation in a correction. Signed-off-by: khushalsonawat --- src/fi/alk/harness/model_citation.py | 40 +++++- .../harness/skills/understand-agent/SKILL.md | 7 +- src/fi/alk/harness/tools.py | 16 ++- tests/harness/test_contract_model.py | 120 +++++++++++++++++- 4 files changed, 166 insertions(+), 17 deletions(-) diff --git a/src/fi/alk/harness/model_citation.py b/src/fi/alk/harness/model_citation.py index 54ef7a9df..fe274a647 100644 --- a/src/fi/alk/harness/model_citation.py +++ b/src/fi/alk/harness/model_citation.py @@ -461,7 +461,8 @@ def kind_suffix(path: Path) -> str: suffix = path.suffix.lower() if suffix in _TEMPLATE_SUFFIXES: inner = Path(path.stem).suffix.lower() - if inner in CONFIG_SUFFIXES: + # Not .prompt: system.prompt.j2 is a Jinja prompt, not a Dotprompt file. + if inner in CONFIG_SUFFIXES and inner != ".prompt": return inner return suffix @@ -478,9 +479,20 @@ def is_config_file(path: Path) -> bool: ) +def _names_configuration(name: str) -> bool: + """A file name that is configuration by itself: Makefile, .env, .env.local, Dockerfile, + Dockerfile.dev.""" + return ( + name in _CONFIG_NAMES + or name in (".env", "dockerfile") + or name.startswith((".env.", "dockerfile.")) + ) + + def is_prose_file(path: Path) -> bool: - if path.suffix.lower() in _TEMPLATE_SUFFIXES and is_config_file(path): - # A template that renders configuration by its name: .env.j2, Dockerfile.j2. + if path.suffix.lower() in _TEMPLATE_SUFFIXES and _names_configuration(Path(path.stem).name.lower()): + # A template that renders configuration by its name: .env.j2, Dockerfile.j2 (not + # dockerfile_writer.j2 or env.system.j2, which are prompts). return False return kind_suffix(path) in PROSE_SUFFIXES or ( not path.suffix and path.name.lower() in _PROSE_STEMS @@ -697,19 +709,33 @@ def _holds(found: str, pieces: list[list[str]]) -> bool: return True +def _in_order(pieces: list[str], found: str) -> bool: + """Whether each piece is in ``found``, each after the one before.""" + at = 0 + for piece in pieces: + start = found.find(piece, at) + if start < 0: + return False + at = start + len(piece) + return True + + def quote_problem(quote: str, copied: str, named: str) -> str | None: """A problem when the lines named for the prompt don't hold the quote given for it: they replace the quote, so lines that miss any of it (a range that stops short, or the wrong lines) would send other text as the agent's instructions. A quote of fewer than - _SHORTEST_QUOTE letters is looked for whole, with no gaps. Lines that hold more than the quote + _SHORTEST_QUOTE letters is looked for whole, with no gaps but at its `...`. Lines that hold more than the quote are not caught.""" pieces = quote_pieces(quote) if pieces: held = _holds(_letters(copied), pieces) else: - # A quote too short to search for with gaps must be in the lines letter for letter; one - # that says nothing isn't looked for. - held = not says_something(quote) or _letters(quote) in _letters(copied) + # A quote too short to search for with gaps must be in the lines letter for letter, each + # piece between its ... after the one before; one that says nothing isn't looked for. + held = not says_something(quote) or _in_order( + [letters for piece in _ELISION.split(quote) if (letters := _letters(piece))], + _letters(copied), + ) if held: return None return ( diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index 08c16edff..d99ddc3b5 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -116,9 +116,10 @@ Find, in roughly this order: fetched at run time and has no lines to name, quote it in `system_prompt_excerpt`; the save asks once whether lines can be named. A quote is word for word from the prompt, a sentence or more, with ... where you leave text out. The lines you name are checked against your latest - such quote, given with them or before. Lines that don't hold it are refused; the next time, - the quote is kept and the lines are recorded as unconfirmed (with a quote of a few words, the - lines are kept, also recorded as unconfirmed). + such quote, given with them or before. Lines that don't hold it are refused, and asked about + once more if they come without a quote; after that, the quote is kept and the lines are + recorded as unconfirmed (with a quote of fewer than 16 letters, the lines are kept, also + recorded as unconfirmed). 7. **The modality.** How this agent is run: a voice session, a text conversation, or work it carries out on websites or a computer. This decides how it is later run, so getting it wrong diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index c30958e4d..a86bb63c3 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -288,19 +288,25 @@ def _checked_against_source( for where in contract.system_prompt_from ) mismatch = quote_problem(quote, contract.system_prompt_excerpt, named) + confirmed = None if mismatch else quote if ( mismatch # Only lines checked against a held quote can hold their own instead (and # the second search is spent only then). and alone and says_something(own) + # Words, not a name: PROMPT or SYSTEM_PROMPT is on almost any prompt's lines. + and len(own.split()) > 1 + # A quote some lines already held stands: a short one doesn't replace it. + and held.get("prompt-quote-confirmed") != quote and quote_problem(own, contract.system_prompt_excerpt, named) is None ): # The call's own short quote is in its lines letter for letter: that is this # call's evidence, and the quote it passed with is the reading's latest now. mismatch = None - if held is not None: - held["prompt-quote"] = own + held["prompt-quote"] = confirmed = own + if held is not None and confirmed and says_something(confirmed): + held["prompt-quote-confirmed"] = confirmed if mismatch: if asked is None: problems.append(mismatch) @@ -319,7 +325,7 @@ def _checked_against_source( f"system_prompt_from: the lines named ({named}) don't hold the quote " f"you gave before ({shown}); if these lines are the prompt, quote " "them word for word in system_prompt_excerpt, with ... where you " - f"leave text out. If they are refused again, {outcome}" + f"leave text out. If they are refused again against it, {outcome}" ) elif asked & {"prompt-quote", "prompt-lines-alone"}: # Either way the lines are kept on record as unconfirmed. @@ -346,8 +352,8 @@ def _checked_against_source( problems.append( "system_prompt_from: lines named later are checked against this " "quote until you give another; a later mismatch records the lines as " - "unconfirmed and keeps the quote, or the lines if the quote is only a " - "few words" + "unconfirmed and keeps the quote, or the lines if the quote has fewer " + "than 16 letters" ) if cited := model_problems(contract, source_root): if asked is not None and "model" in asked: diff --git a/tests/harness/test_contract_model.py b/tests/harness/test_contract_model.py index 7ed97596e..6ba8e7dc8 100644 --- a/tests/harness/test_contract_model.py +++ b/tests/harness/test_contract_model.py @@ -854,7 +854,7 @@ def lines_alone(named: str, quote: str, *, short: bool = False) -> str: f"system_prompt_from: the lines named ({named}) don't hold the quote you gave before " f"({' '.join(quote.split())[:80]}); if these lines are the prompt, quote them word for " f"word in system_prompt_excerpt, with ... where you leave text out. If they are refused " - f"again, {outcome}" + f"again against it, {outcome}" ) @@ -1429,7 +1429,7 @@ def test_lines_refused_twice_leave_the_quote(tmp_path, root): + not_the_quote("prompt.py 1-2") + "\n - system_prompt_from: lines named later are checked against this quote until you " "give another; a later mismatch records the lines as unconfirmed and keeps the quote, or the " - "lines if the quote is only a few words" + "lines if the quote has fewer than 16 letters" ) second = submit(wrong) assert not second.get("is_error"), second @@ -3134,6 +3134,8 @@ def test_a_pipe_is_logged_as_not_a_regular_file(tmp_path, caplog): pytest.param("agent.sc", 'val t: A#B = x; val model = "gpt-4o"', "gpt-4o", id="scala-script-type-projection"), pytest.param(".env.j2", "MODEL=gpt-4o", "gpt-4o", id="env-template"), pytest.param("Dockerfile.j2", "ENV MODEL=gpt-4o", "gpt-4o", id="dockerfile-template"), + pytest.param(".env.local.j2", "MODEL=gpt-4o", "gpt-4o", id="env-local-template"), + pytest.param("Dockerfile.dev.j2", "ENV MODEL=gpt-4o", "gpt-4o", id="dockerfile-dev-template"), pytest.param("config.yaml.j2", " - gpt-4o", "gpt-4o", id="yaml-template-list-item"), pytest.param("appsettings.json", '"ModelId": "gpt-4o", // the default', "gpt-4o", id="json-comment-after"), pytest.param("agent.json", '"url": "http://x//y", "model": "gpt-4o"', "gpt-4o", id="json-slashes-in-a-string"), @@ -3203,6 +3205,8 @@ def test_a_type_projection_does_not_hide_a_triple_quote(root): "The agent is told to be warm and to keep its answers brief.", id="a-whole-short-prompt", ), pytest.param(_BOOKING, (6, 12), "Be warm and brief.", _PARAPHRASE, id="one-sentence-of-a-long-prompt"), + pytest.param(_BOOKING, (6, 12), "be warm and brief", _PARAPHRASE, id="another-case-no-full-stop"), + pytest.param(_BOOKING, (6, 12), "Be warm ... brief.", _PARAPHRASE, id="a-short-elided-quote"), ], ) @pytest.mark.parametrize("then", ["the-same-again", "the-lines-alone"]) @@ -3347,3 +3351,115 @@ def test_a_saved_call_that_kept_the_quote_says_so(tmp_path, root): "\nsystem_prompt_from: the lines named (agent.py 1-4) are recorded as unconfirmed, since " "they don't hold the quote; the quote is the prompt." ) + + +# The settled round 2: a short quote on its own lines, and its limits + + +@pytest.mark.parametrize( + "first_lines, short, line", + [ + # The quote held from a call with no lines; then a name on the prompt's first line. + pytest.param(None, "PROMPT", 6, id="a-name-after-a-quote-alone"), + pytest.param(None, "SYSTEM_PROMPT", 6, id="a-setting-name-after-a-quote-alone"), + # The quote held from lines that held it; then a sentence repeated in a comment. + pytest.param((6, 12), "Be warm and brief.", 14, id="a-comment-after-a-confirmed-quote"), + ], +) +def test_a_short_quote_on_its_own_lines_does_not_replace_a_stronger_one( + tmp_path, root, first_lines, short, line +): + (root / "agent.py").write_text(_BOOKING + "\n# Style: Be warm and brief.\n", encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + named = [{"path": "agent.py", "start_line": line, "end_line": line}] + first = payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE) + if first_lines: + first["system_prompt_from"] = [{"path": "agent.py", "start_line": first_lines[0], "end_line": first_lines[1]}] + submit(first) + + second = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=short, system_prompt_from=named)) + assert text_of(second).startswith(REFUSED + lines_alone(f"agent.py {line}-{line}", _BOOKING_QUOTE)) + third = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=short, system_prompt_from=named)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["unconfirmed_prompt_from"] == named + + +def test_a_padded_short_quote_is_this_calls_own(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + + result = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=" Be warm and brief. ", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ) + + assert text_of(result).startswith(REFUSED + not_the_quote("agent.py 1-4")) + + +@pytest.mark.parametrize( + "quote, held", + [ + pytest.param("Be warm ... brief.", True, id="pieces-in-order"), + pytest.param("brief ... Be warm", False, id="pieces-out-of-order"), + ], +) +def test_a_short_elided_quote_is_looked_for_piece_by_piece(quote, held): + from fi.alk.harness.model_citation import quote_problem + + assert (quote_problem(quote, 'P = "Be warm and brief."', "a.py 1-1") is None) == held + + +def test_the_quote_under_its_own_name_comes_before_an_alias(): + from fi.alk.harness.contract import prompt_quote + + both = payload(system_prompt_excerpt="You are Acme's booking agent.", prompt="Something else entirely.") + + assert prompt_quote(both) == "You are Acme's booking agent." + assert AgentContract.model_validate(both).system_prompt_excerpt == "You are Acme's booking agent." + + +@pytest.mark.parametrize( + "file", + ["prompts/dockerfile_writer.j2", "prompts/env.system.j2", "prompts/system.prompt.j2", "dockerfile.md"], +) +def test_a_prompt_named_like_configuration_is_text(root, file): + (root / file).parent.mkdir(parents=True, exist_ok=True) + (root / file).write_text("You write the deployment.\nmodel: gpt-4o\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": file, "line": 2}) + + assert problems == [prose(file)] + + +def test_a_conf_file_has_no_block_comments(root): + (root / "site.conf").write_text('\nrewrite ^/(.*)//$ /$1 permanent; set $m "gpt-4o";\n', encoding="utf-8") + + problems, contract, _ = checked(root, {"name": "gpt-4o", "path": "site.conf", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == "gpt-4o" + + +def test_a_json_template_keeps_its_comments(root): + (root / "config.json.j2").write_text('\n// "model": "gpt-4-turbo",\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4-turbo", "path": "config.json.j2", "line": 2}) + + assert problems == [not_named("gpt-4-turbo", 2, "config.json.j2")] + + +@pytest.mark.parametrize("nothing", ["n / a", "- -", "a b"]) +def test_words_that_say_nothing_do_not_replace_the_held_quote(tmp_path, root, nothing): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE)) + + result = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=nothing, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ) + + assert text_of(result) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE)