diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index f0c5766f2..3fc5423bf 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -208,6 +208,21 @@ class PromptSource(BaseModel): end_line: int +class ModelSource(BaseModel): + """The model the agent talks to the person with, and the line its name is written on.""" + + provider: str = "" + name: str = "" + path: str = "" + line: int = 0 + + @model_validator(mode="before") + @classmethod + def _bare_name(cls, value: Any) -> Any: + # A bare string is the name alone; the save then asks for path and line. + return {"name": value} if isinstance(value, str) else value + + class DataStore(BaseModel): """What the agent's tools read and write, and how to be there instead of it. @@ -1069,6 +1084,20 @@ def _reached(one: Dependency) -> str: return "; ".join(said) +def prompt_quote(payload: Any) -> str: + """The quote of the prompt as AgentContract reads it from a payload: under its own name or + the first alias that has one, a list joined by lines. The save holds it before validating.""" + if not isinstance(payload, dict): + return "" + for key in ("system_prompt_excerpt", *_ALIASES["system_prompt_excerpt"]): + value = payload.get(key) + if value: + if isinstance(value, list): + return "\n".join(str(item) for item in value) + return value if isinstance(value, str) else str(value) + return "" + + class AgentContract(BaseModel): """What the agent verifiably is. Nothing downstream may contradict this.""" @@ -1127,6 +1156,12 @@ def _normalize_shapes(cls, payload: Any) -> Any: system_prompt_excerpt: str = "" # When given, the excerpt is replaced by these lines copied from the source as written. system_prompt_from: list[PromptSource] = Field(default_factory=list) + # Set by the save, never by the reading: lines named for the prompt that twice didn't hold + # the quote given for it. The quote went out instead, or, when the quote was too short to + # stand for the prompt, the lines did, unconfirmed. Empty for nearly every contract. + unconfirmed_prompt_from: list[PromptSource] = Field(default_factory=list) + # None is unknown. Kept only when its name was found on the line cited in the source. + model: ModelSource | None = None hard_constraints: list[str] = Field(default_factory=list) tools: list[ToolSpec] = Field(default_factory=list) data_schema: dict[str, Any] = Field(default_factory=dict) @@ -1161,6 +1196,18 @@ def _normalize_shapes(cls, payload: Any) -> Any: amendments: list[str] = Field(default_factory=list) instrumentation: str = "none" + @model_serializer(mode="wrap") + def _omit_absent_model(self, handler: Any) -> dict[str, Any]: + # A contract with no model writes the same bytes as before the field existed, so its + # setup fingerprint is unchanged. + data = handler(self) + if data.get("model") is None: + data.pop("model", None) + # The same for the record of unconfirmed prompt lines, kept only when there are some. + if not data.get("unconfirmed_prompt_from"): + data.pop("unconfirmed_prompt_from", None) + return data + def tool_names(self) -> set[str]: return {tool.name for tool in self.tools} diff --git a/src/fi/alk/harness/model_citation.py b/src/fi/alk/harness/model_citation.py new file mode 100644 index 000000000..fe274a647 --- /dev/null +++ b/src/fi/alk/harness/model_citation.py @@ -0,0 +1,745 @@ +"""Reading the model an agent talks to the person with, and the prompt it is given, from the +agent's own files, while a conversation waits on the answer. + +A model is kept only when its name is the value written on the line the reader cites: the +Agents page shows it as fact, and an unknown model is better left empty than guessed. The files +can be built to stall whoever reads them, so every path is resolved within the save's one lookup +budget, every read is bounded in size and decoded as UTF-8 whatever coding line the file carries, +and every check of a line costs at most a few passes over it. +""" + +from __future__ import annotations + +import functools +import html +import logging +import os +import re +import stat +import string +import unicodedata +from pathlib import Path + +from .contract import AgentContract +from .runtime_selection import charged_path +from .source_folder import is_within + +_LOG = logging.getLogger(__name__) +# The platform refuses the whole agent document past this, for the model's name and provider. +LONGEST_MODEL_FIELD = 255 +# One path as written: charged_path doesn't refuse a long path itself. +LONGEST_MODEL_PATH = 4096 +# The cited file, as for a prompt file. +LARGEST_MODEL_FILE = 1024 * 1024 +# What a model name is made of. A match must have none of these right before or after it, so +# "gpt-4o" is not found inside "gpt-4o-mini". +MODEL_NAME_CHARS = frozenset(string.ascii_letters + string.digits + "._:/@-") +# Files whose values are written bare after a key: `model: gpt-4o`, `MODEL=gpt-4o`, +# `ENV MODEL=gpt-4o`, `export MODEL=gpt-4o`. +CONFIG_SUFFIXES = frozenset( + { + ".yaml", ".yml", ".toml", ".json", ".ini", ".cfg", ".conf", ".env", ".properties", + ".sh", ".bash", ".zsh", ".dockerfile", ".mk", ".prompt", + } +) +_CONFIG_NAMES = frozenset({"makefile", "gnumakefile", ".flaskenv"}) +# Languages where # isn't a comment: C and C++ (#define), JavaScript and TypeScript (#field), +# Java, Go, Rust (#[attr]), Swift (#if), Kotlin, C#, Scala (A#B), Dart, Objective-C, Groovy, +# Gradle, Lua (#t is a length). +_NO_HASH_COMMENT_SUFFIXES = frozenset( + { + ".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", + ".java", ".go", ".rs", ".swift", ".kt", ".kts", ".cs", ".scala", ".dart", ".m", ".mm", + ".groovy", ".gradle", ".lua", ".mts", ".cts", ".sc", + } +) +# Files where # starts a comment only at a line's start or after a space: YAML, shell, Makefile +# and Dockerfile (a#b is a word there), and .env files. +_SPACED_HASH_SUFFIXES = frozenset( + {".yaml", ".yml", ".sh", ".bash", ".zsh", ".mk", ".dockerfile", ".env"} +) +# Files with no block comments, nor //: there `* ` starts no comment (a Python continuation +# line), and /* and // are text (ls /*, a // 2). +_NO_BLOCK_COMMENT_SUFFIXES = frozenset( + {".py", ".pyi", ".pyw", ".rb", ".pl", ".r", ".jl", ".ex", ".exs", ".nim", ".lua"} +) +_YAML_SUFFIXES = frozenset({".yaml", ".yml"}) +# INI files: a line starting with ; is a comment. +_INI_SUFFIXES = frozenset({".ini", ".cfg", ".conf"}) +# Files where -- starts a comment: SQL, Lua, Haskell. +_DASH_COMMENT_SUFFIXES = frozenset({".sql", ".lua", ".hs"}) +# Files written for people: they say which model is used; they don't set it. +# And prompt templates (.j2, .hbs): their text is the agent's prompt, which may name a model +# without setting it. +PROSE_SUFFIXES = frozenset( + { + ".md", ".markdown", ".mdx", ".mdc", ".rst", ".txt", ".adoc", ".org", ".j2", ".jinja", + ".jinja2", ".hbs", ".mustache", + } +) +# Files written for people, by name with no suffix: README, CHANGELOG, LICENSE, CONTRIBUTING +# (README.md is prose by its suffix; readme.py is code). +_PROSE_STEMS = frozenset({"readme", "changelog", "license", "contributing"}) +# Words a setting is named with, never a model: `cfg.get("model")`, OPENAI_MODEL, llm.model. +# Not "deployment": an Azure deployment's own name often has it (gpt-4o_deployment). +_SETTING_WORDS = frozenset({"model", "llm", "name", "default", "engine", "provider", "key", "env", "id"}) +_QUOTES = frozenset("\"'`") +_SETTING_NAME_CHARS = frozenset(string.ascii_uppercase + string.digits + "_") +_SETTING_PARTS = re.compile(r"[_\-.:/@]") +# An escape in a string: \uXXXX and \xXX (as json.dump writes any non-ASCII character), +# \UXXXXXXXX, \N{NAME} and octal. +_CHAR_ESCAPE = re.compile( + r"\\(?:u([0-9a-fA-F]{4})|x([0-9a-fA-F]{2})|U([0-9a-fA-F]{8})|N\{([A-Za-z0-9 \-]{1,88})\}" + r"|([0-7]{1,3}))" +) +_NOT_A_LETTER = re.compile(r"[\W_]+") +_SURROGATE = re.compile("[\ud800-\udfff]") +# Full-width letters and digits (3, A) as their ASCII forms: what NFKC would do for them, +# without NFKC's growth elsewhere. +_FULL_WIDTH = {code: code - 0xFEE0 for code in range(0xFF01, 0xFF5F)} +# Scripts written without spaces between words (Thai, Lao, Burmese, Khmer, Japanese kana, +# Chinese and Japanese ideographs): each character is a word, so the gap below applies between +# characters there too. +_UNSPACED = "\u0e00-\u0eff\u1000-\u109f\u1780-\u17ff\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff" +_UNSPACED_CHAR = re.compile(f"[{_UNSPACED}]") +# Only a letter of those scripts is a word: their vowel signs and tone marks, and ・, are not. +_WORD = re.compile(f"(?![\\W_])[{_UNSPACED}]|[^\\W_{_UNSPACED}]+") +# A quote of fewer letters than this says nothing to check lines against ("n/a", "TBD", "see +# lines"): a few short words are found in almost any lines. A character of an unspaced script +# counts as three. +_SHORTEST_QUOTE = 16 +# Fewer letters than this and a quote says nothing at all ("n/a", "TBD", "none"): it isn't held +# and isn't checked. +_SAYS_NOTHING = 6 +# Languages with triple-quoted strings ("""...""" or '''...'''): Python, TOML, Kotlin, Scala, +# Swift, Groovy and Gradle, Java and C# text blocks, Julia, Dart, Elixir. +_TRIPLE_SUFFIXES = frozenset( + { + ".py", ".pyi", ".pyw", ".toml", ".kt", ".kts", ".scala", ".sc", ".swift", ".groovy", + ".gradle", ".java", ".cs", ".jl", ".dart", ".ex", ".exs", + } +) +# Where a quote leaves text out: ... or …, also inside brackets, since only letters are compared. +_ELISION = re.compile(r"\.{3,}|…") +# How many letters of code may sit between two words of a quote in the lines: a string prefix +# (f"), a short comment, `+ NAME +`, a markup tag or a placeholder the quote leaves out. +_WORD_GAP = 24 + + +# How # starts a comment: anywhere (Python, shell, Ruby, TOML...), only at a line's start or +# after a space (YAML, where `url: http://x/#top` and `{a: 1,#b}` are values), or not at all. +_HASH_ANYWHERE, _HASH_SPACED, _HASH_NONE = "anywhere", "spaced", "none" + + +@functools.cache +def _comment_tokens( + *, dashes: bool, triple: bool, hash_comments: str, block_comments: bool +) -> re.Pattern[str]: + """What the line scan stops at: a triple quote (in languages that have them), a + double-quoted or backticked string, matched whole (to the line's end if it isn't closed); an + apostrophe, which the scan pairs itself; or a comment marker. # not in $# (a shell count), + &# (an HTML character) or ${# (a shell length).""" + alternatives = ['"""', "'''"] if triple else [] + alternatives += [r'"(?:[^"\\]|\\.)*"?', r"`(?:[^`\\]|\\.)*`?", "'"] + if hash_comments == _HASH_ANYWHERE: + alternatives.append(r"(? bool: + """Whether the apostrophe at ``at`` can open a string: not inside a word (Pair's, it's, x' + in Haskell) and not after a backslash (it\\'s), unless the word is a string prefix (r'...', + rb'...'). Looks at three characters at most: a prefix is one or two letters, so three word + characters before the apostrophe (letters, digits, _) are a word, whatever comes before.""" + start = at + while start > 0 and at - start < 3 and (line[start - 1].isalnum() or line[start - 1] == "_"): + start -= 1 + if start == at: + return at == 0 or line[at - 1] != "\\" + return line[start:at].lower() in _STRING_PREFIXES + + +def _line_scan( + line: str, + block: str | None, + *, + dashes: bool = False, + ini: bool = False, + triple: bool = False, + hash_comments: str = _HASH_ANYWHERE, + block_comments: bool = True, +) -> tuple[int, list[tuple[int, int]], str | None]: + """One pass over a line: where its comment starts (or its length), the stretches of it that + are text (inside a triple-quoted string, or a block comment closed on the line), and the + triple quote still open at its end. ``block`` is the one open where the line starts. A line + is a comment when it is the `* ` middle of a block comment or, + in INI, starts with ;. Strings are passed over whole, so a marker or a triple quote inside one + doesn't count; an apostrophe opens a string only when another closes it on the line.""" + inside: list[tuple[int, int]] = [] + at = 0 + if block is not None: + close = line.find(block) + if close < 0: + return len(line), [(0, len(line))], block + inside.append((0, close)) + at = close + 3 + else: + stripped = line.lstrip() + if (block_comments and stripped.startswith(("* ", "*\t"))) or ( + ini and stripped.startswith(";") + ): + return 0, inside, None + tokens = _comment_tokens( + dashes=dashes, triple=triple, hash_comments=hash_comments, block_comments=block_comments + ) + while (found := tokens.search(line, at)) is not None: + token, start = found.group(), found.start() + at = found.end() + if token in ('"""', "'''"): + close = line.find(token, at) + if close < 0: + inside.append((at, len(line))) + return len(line), inside, token + inside.append((at, close)) + at = close + 3 + elif token == "'": + if _opens_a_string(line, start): + # Unclosed, it is an apostrophe: every later one is then after a backslash, so + # none of them opens a string either, and the line is read about once. + closed = _REST_OF_SINGLE.match(line, at) + if closed is not None: + at = closed.end() + elif token in ("/*", "", at) + if close < 0: + return start, inside, None + # From the marker itself, so the stretch can never be a quoted value. + inside.append((start, close)) + at = close + (2 if token == "/*" else 3) + elif token[0] not in "\"`": + return start, inside, None + return len(line), inside, None + + +def block_open_before( + lines: list[str], *, hash_comments: str = _HASH_ANYWHERE, block_comments: bool = True +) -> str | None: + """The triple quote open where the line after ``lines`` starts, the lines read with the same + scan as the cited one. A line with no triple quote, outside a block, is passed over without + a scan; inside one, only the closing quote is looked for.""" + block: str | None = None + for line in lines: + if block is None and '"""' not in line and "'''" not in line: + continue + if block is not None and block not in line: + continue + _, _, block = _line_scan( + line, + block, + triple=True, + hash_comments=hash_comments, + block_comments=block_comments, + ) + return block + + +def resolved_in_source(root: Path, path: str) -> Path | None: + """``root / path`` resolved, each lookup charged to the save's budget, or None: outside + ``root``, a link loop, a NUL, a path the system refuses. ``root`` is already resolved. + PathLookupsSpent is not caught here: past the budget the save answers with its own message.""" + try: + resolved = charged_path(root / path) + except (OSError, RuntimeError, ValueError): + return None + return resolved if is_within(resolved, root) else None + + +_OPEN_FLAGS = ( + os.O_RDONLY + # A pipe or a device put at the path after it was resolved is opened without waiting on it, + # then refused below; a link put in the path's last part since isn't followed. + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) +) + + +def read_source(path: Path, limit: int) -> bytes | None: + """At most ``limit + 1`` bytes of the file, or None: not a regular file, or not readable. + ``path`` is already resolved. The caller judges the size. A file that can't be read is + logged by the error's kind alone: its path and content are the customer's.""" + try: + descriptor = os.open(path, _OPEN_FLAGS) + except (OSError, ValueError) as failed: + _LOG.info("source file not read (%s)", type(failed).__name__) + return None + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + _LOG.info("source file not read (not a regular file)") + return None + chunks: list[bytes] = [] + held = 0 + # A read can return less than asked for; what counts is limit + 1 or the end. + while held <= limit: + chunk = os.read(descriptor, limit + 1 - held) + if not chunk: + break + chunks.append(chunk) + held += len(chunk) + return b"".join(chunks) + except OSError as failed: + _LOG.info("source file not read (%s)", type(failed).__name__) + return None + finally: + os.close(descriptor) + + +def source_lines(raw: bytes) -> list[str]: + """The file's lines, whatever coding line it carries: no codec the file names gets to run + on it, and one file has one numbering wherever it is read. Numbered as the reading's Read and + Grep number them: at each \\n, with a \\r before it dropped. Not at the other characters + str.splitlines breaks at (a form feed, a lone \\r, U+2028), which would move every line + after them.""" + lines = raw.decode("utf-8", errors="replace").split("\n") + if lines[-1] == "": + lines.pop() + return [line[:-1] if line.endswith("\r") else line for line in lines] + + +def _bare_after_key(line: str, at: int, *, yaml: bool) -> bool: + """Whether the text before ``at`` ends, past spaces, in ``:`` or ``=`` (or ``-``, a YAML list + item). Steps back over the spaces only: copying the line before every occurrence would cost + the square of its length.""" + before = at - 1 + while before >= 0 and line[before] in " \t": + before -= 1 + return before >= 0 and (line[before] in ":=" or (yaml and line[before] == "-")) + + +def _is_a_key(line: str, at: int, end: int) -> bool: + """Whether a quoted occurrence from ``at`` to ``end`` is a key: its closing quote is followed, + past spaces, by ``:``, as in `{"llm_model": "gpt-4o"}`, and its opening quote doesn't follow a + ``?``, as the first value of `prod ? "gpt-4o" : "gpt-4o-mini"` does.""" + before = at - 2 + while before >= 0 and line[before] in " \t": + before -= 1 + if before >= 0 and line[before] == "?": + return False + after = end + 1 + while after < len(line) and line[after] in " \t": + after += 1 + return after < len(line) and line[after] == ":" + + +# Calls that look a setting up by its name: get, os.getenv, getOrThrow, getProperty, env.str. +# str only as a method (env.str): str("gpt-4o") is a value. +_METHOD_ONLY_CALLS = frozenset({"str"}) +_LOOKUP_CALLS = ( + "getenv", "get", "getOrThrow", "getOrDefault", "getProperty", "getString", "getValue", "str", + "env", "config", +) +_NAME_END = re.compile(r"[A-Za-z0-9_]") + + +def _is_a_lookup(line: str, at: int) -> bool: + """Whether a quoted occurrence starting at ``at`` names what is looked up rather than the + value: right after a subscript (`config["model-name"]`) or as the first argument of a lookup + call (`cfg.get("llm-model")`, `os.getenv ("model-name")`, `config.get("m-x")`). A + first argument of any other call is a value: `GenerativeModel("gemini-pro")`. Looks at a few + characters only.""" + before = at - 2 + while before >= 0 and line[before] in " \t": + before -= 1 + if before < 1: + return False + if line[before] == "[": + # A subscript follows a name or a closing bracket; a list literal doesn't. + return line[before - 1].isalnum() or line[before - 1] in "_)]." + if line[before] != "(": + return False + end = before - 1 + while end >= 0 and line[end] in " \t": + end -= 1 + if end >= 0 and line[end] == ">": + # A generic type, as in get(: step back over it, a few characters at most. + opened = line.rfind("<", max(0, end - 40), end) + if opened < 0: + return False + end = opened - 1 + for call in _LOOKUP_CALLS: + start = end + 1 - len(call) + if start >= 0 and line[start : end + 1] == call: + if call in _METHOD_ONLY_CALLS: + return start > 0 and line[start - 1] == "." + return start == 0 or not _NAME_END.match(line[start - 1]) + return False + + +def named_on( + line: str, + name: str, + *, + config: bool, + yaml: bool = False, + dashes: bool = False, + ini: bool = False, + triple: bool = False, + hash_comments: str = _HASH_ANYWHERE, + block_comments: bool = True, + block: str | None = None, +) -> bool: + """Whether ``name`` is the value written on ``line``: whole, outside a comment and outside + a triple-quoted string (``block`` is the triple quote open where the line starts), and + between matching quotes (not a key, not a lookup), or in a configuration file written bare + after ``:`` or ``=`` or as a shell default, ``${MODEL:-gpt-4o}``. ``name`` is made only of + MODEL_NAME_CHARS, so an occurrence starting inside one that failed has a name character just + before it, and the search resumes after the failed one. The line is scanned once, its + triple-quoted stretches walked alongside the occurrences, and each step looks at a few + characters beside one, so a line costs about one pass.""" + if not name: + # find("") matches at every position without moving on. + return False + comment, inside, _ = _line_scan( + line, + block, + dashes=dashes, + ini=ini, + triple=triple, + hash_comments=hash_comments, + block_comments=block_comments, + ) + stretches = iter(inside) + stretch = next(stretches, None) + at = line.find(name, 0, comment) + while at >= 0: + end = at + len(name) + while stretch is not None and stretch[1] <= at: + stretch = next(stretches, None) + before = line[at - 1] if at else "" + after = line[end] if end < len(line) else "" + # Inside a triple-quoted string or a block comment the line is text, unless the string is + # the value: MODEL = """gpt-4o""". + inside_text = stretch is not None and stretch[0] <= at and stretch != (at, end) + if not inside_text and after not in MODEL_NAME_CHARS: + if before not in MODEL_NAME_CHARS: + if before and before in _QUOTES and after == before: + if not _is_a_key(line, at, end) and not _is_a_lookup(line, at): + return True + elif config and _bare_after_key(line, at, yaml=yaml): + return True + elif config and after == "}" and line[max(0, at - 2) : at] == ":-": + # The default of a shell variable: ${MODEL:-gpt-4o}. + return True + at = line.find(name, end, comment) + return False + + +# Prompt and page templates: what they hold is text, unless the name says they render a +# configuration file (config.yaml.j2), which is then read as that file. +_TEMPLATE_SUFFIXES = frozenset({".j2", ".jinja", ".jinja2", ".hbs", ".mustache"}) + + +def kind_suffix(path: Path) -> str: + """The suffix that says what a file is: config.yaml.j2 is YAML.""" + suffix = path.suffix.lower() + if suffix in _TEMPLATE_SUFFIXES: + inner = Path(path.stem).suffix.lower() + # Not .prompt: system.prompt.j2 is a Jinja prompt, not a Dotprompt file. + if inner in CONFIG_SUFFIXES and inner != ".prompt": + return inner + return suffix + + +def is_config_file(path: Path) -> bool: + name = path.name.lower() + suffix = kind_suffix(path) + # env.py and dockerfile_gen.py are code, whatever their names start with. + code = suffix in _TRIPLE_SUFFIXES | _NO_HASH_COMMENT_SUFFIXES | _NO_BLOCK_COMMENT_SUFFIXES + return ( + suffix in CONFIG_SUFFIXES + or name in _CONFIG_NAMES + or (not code and name.startswith((".env", "env.", "dockerfile"))) + ) + + +def _names_configuration(name: str) -> bool: + """A file name that is configuration by itself: Makefile, .env, .env.local, Dockerfile, + Dockerfile.dev.""" + return ( + name in _CONFIG_NAMES + or name in (".env", "dockerfile") + or name.startswith((".env.", "dockerfile.")) + ) + + +def is_prose_file(path: Path) -> bool: + if path.suffix.lower() in _TEMPLATE_SUFFIXES and _names_configuration(Path(path.stem).name.lower()): + # A template that renders configuration by its name: .env.j2, Dockerfile.j2 (not + # dockerfile_writer.j2 or env.system.j2, which are prompts). + return False + return kind_suffix(path) in PROSE_SUFFIXES or ( + not path.suffix and path.name.lower() in _PROSE_STEMS + ) + + +def _reads_as_a_setting(name: str) -> bool: + """A name shaped like where a model is read from rather than which: OPENAI_MODEL, + openai_model, model-name, llm.model. Model and deployment ids such as GPT4O, O1, + mistral_large, gpt4o-deployment and gemini-1.5-pro are not.""" + # An environment variable: capitals, digits and underscores, with an underscore. + if "_" in name and _SETTING_NAME_CHARS.issuperset(name): + return True + has_digit = any(char.isdigit() for char in name) + parts = _SETTING_PARTS.split(name.casefold()) + # A setting's words, alone, in snake case or without a version number: model, llm_model, + # model-name. + if ("_" in name or not has_digit) and _SETTING_WORDS.intersection(parts): + return True + # A dotted key with no version in it: services.openai.model, llm.provider. + return "." in name and not has_digit + + +def model_problems(contract: AgentContract, source_root: Path) -> list[str]: + """Problems with contract.model, checked against the cited line. On success, strips name and + provider and stores path as the resolved file relative to the root, in POSIX form. Resolves + one path (charged to the save's budget) and reads at most one file.""" + model = contract.model + if model is None: + return [] + name, provider, path = model.name.strip(), model.provider.strip(), model.path.strip() + shown_name, shown_path = name[:100], path[:200] + problems: list[str] = [] + if not name: + problems.append( + "model: name is empty; give the model's name as written in the source, or leave " + "model out" + ) + elif not MODEL_NAME_CHARS.issuperset(name): + problems.append( + "model: a model name is letters, digits and . _ : / @ -; write it as it appears in " + "the source, or leave model out" + ) + elif _reads_as_a_setting(name): + # OPENAI_MODEL in os.getenv("OPENAI_MODEL") is where the model is read from, not which. + problems.append( + f"model: {shown_name} reads as the name of a setting, not a model; cite the line its " + "value is written on, or leave model out" + ) + if not MODEL_NAME_CHARS.issuperset(provider): + problems.append( + "model: a provider is letters, digits and . _ : / @ -; write it as the source names " + "it, or leave it out" + ) + if len(name) > LONGEST_MODEL_FIELD or len(provider) > LONGEST_MODEL_FIELD: + problems.append("model: name and provider are at most 255 characters") + if not path or model.line < 1: + problems.append( + "model: give path and line, the file and the line the model's name is written on" + ) + if len(path) > LONGEST_MODEL_PATH: + problems.append("model: path is too long") + # All of these at once, and before the disk: a citation wrong in its shape costs no lookup. + if problems: + return problems + root = source_root.resolve() + resolved = resolved_in_source(root, path) + if resolved is None: + return [f"model: {shown_path} is not a file in the source"] + if is_prose_file(resolved): + # A README, a text file or a prompt template says which model is used; the code or its + # configuration sets it. + return [ + f"model: {shown_path} is text (for people, or a prompt), not code or configuration; " + "cite the line in the agent's code or configuration the model is set on, or leave " + "model out" + ] + raw = read_source(resolved, LARGEST_MODEL_FILE) + if raw is None or len(raw) > LARGEST_MODEL_FILE: + return [ + f"model: {shown_path} can't be read as a file of at most 1 MiB; check the path, and " + "cite the line in the file the name is written in" + ] + lines = source_lines(raw) + if model.line > len(lines): + return [f"model: line {model.line} is outside {shown_path}, which has {len(lines)} lines"] + suffix = kind_suffix(resolved) + dashes, ini = suffix in _DASH_COMMENT_SUFFIXES, suffix in _INI_SUFFIXES + triple = suffix in _TRIPLE_SUFFIXES + config = is_config_file(resolved) + name_lower = resolved.name.lower() + if suffix in _NO_HASH_COMMENT_SUFFIXES: + hash_comments = _HASH_NONE + elif ( + suffix in _SPACED_HASH_SUFFIXES + or name_lower in _CONFIG_NAMES + # dockerfile_gen.py and .env_loader.py are Python: # anywhere there. + or (config and name_lower.startswith((".env", "dockerfile"))) + ): + hash_comments = _HASH_SPACED + else: + hash_comments = _HASH_ANYWHERE + # Configuration files have no block comments, but JSON read as JSONC (appsettings.json, + # tsconfig.json) has: outside a string, // and /* can only be comments there. + block_comments = suffix == ".json" or not (config or suffix in _NO_BLOCK_COMMENT_SUFFIXES) + if not named_on( + lines[model.line - 1], + name, + config=config, + yaml=suffix in _YAML_SUFFIXES, + dashes=dashes, + ini=ini, + triple=triple, + hash_comments=hash_comments, + block_comments=block_comments, + block=( + block_open_before( + lines[: model.line - 1], + hash_comments=hash_comments, + block_comments=block_comments, + ) + if triple + else None + ), + ): + return [ + f"model: the name {shown_name} is not the value written on line {model.line} of " + f"{shown_path}; cite the line where it is written in quotes, or after : or = in a " + "configuration file, or leave model out if the name is only set at run time" + ] + model.name, model.provider = name, provider + # The file the path leads to, so a link inside the source is stored as its target. + model.path = resolved.relative_to(root).as_posix() + return [] + + +def _escape(found: re.Match[str]) -> str: + code = found.group(1) or found.group(2) or found.group(3) + try: + if code: + return chr(int(code, 16)) + if found.group(4): + return unicodedata.lookup(found.group(4)) + return chr(int(found.group(5), 8)) + except (KeyError, ValueError, OverflowError): + return found.group() + + +def _comparable(text: str) -> str: + """``text`` as a quote and the lines it was quoted from both read it: escapes decoded, HTML + entities decoded, one Unicode form (NFC, which never grows the text much), case set aside.""" + unescaped = _CHAR_ESCAPE.sub(_escape, text) + if _SURROGATE.search(unescaped): + # json.dump writes a character past U+FFFF as two escapes, one per half of its pair. + unescaped = unescaped.encode("utf-16", "surrogatepass").decode("utf-16", "replace") + return unicodedata.normalize("NFC", html.unescape(unescaped)).translate(_FULL_WIDTH).casefold() + + +def _letters(text: str) -> str: + """The letters and digits of ``text`` once comparable: what a quote and its lines share + whatever the quoting or the pieces a long string is written in.""" + return _NOT_A_LETTER.sub("", _comparable(text)) + + +def quote_pieces(quote: str) -> list[list[str]]: + """The quote's words, in the pieces it leaves text out between (..., …). Empty when the + quote has fewer than _SHORTEST_QUOTE letters: then it says nothing to check lines against.""" + pieces = [_WORD.findall(_comparable(piece)) for piece in _ELISION.split(quote)] + pieces = [words for words in pieces if words] + return pieces if _weighed(pieces) >= _SHORTEST_QUOTE else [] + + +def _weighed(pieces: list[list[str]]) -> int: + """How many letters a quote's words come to, a character of an unspaced script as three.""" + return sum( + 3 if _UNSPACED_CHAR.fullmatch(word) else len(word) for words in pieces for word in words + ) + + +def says_something(quote: str) -> bool: + """Whether a quote has enough letters to say anything: "n/a" and "TBD" don't.""" + pieces = [_WORD.findall(_comparable(piece)) for piece in _ELISION.split(quote)] + return _weighed([words for words in pieces if words]) >= _SAYS_NOTHING + + +def _holds(found: str, pieces: list[list[str]]) -> bool: + """Whether every piece's words are in ``found`` in order, each within _WORD_GAP letters of + the one before, and each piece after the one before. A piece whose first word matches in the + wrong place is tried again further on. Each search is charged what it reads, against a budget + of a few times the text, so a crafted file or quote costs a few passes at most.""" + budget = 4 * (len(found) + sum(len(word) for words in pieces for word in words)) + 10_000 + at = 0 + for words in pieces: + while True: + start = found.find(words[0], at) + if start < 0: + return False + budget -= start - at + len(words[0]) + end = start + len(words[0]) + for word in words[1:]: + hit = found.find(word, end, end + _WORD_GAP + len(word)) + if hit < 0: + # A false start usually ends here, at its second word: charged that word. + budget -= len(word) + break + budget -= _WORD_GAP + len(word) + end = hit + len(word) + else: + at = end + break + if budget < 0: + return False + at = start + 1 + return True + + +def _in_order(pieces: list[str], found: str) -> bool: + """Whether each piece is in ``found``, each after the one before.""" + at = 0 + for piece in pieces: + start = found.find(piece, at) + if start < 0: + return False + at = start + len(piece) + return True + + +def quote_problem(quote: str, copied: str, named: str) -> str | None: + """A problem when the lines named for the prompt don't hold the quote given for it: they + replace the quote, so lines that miss any of it (a range that stops short, or the wrong + lines) would send other text as the agent's instructions. A quote of fewer than + _SHORTEST_QUOTE letters is looked for whole, with no gaps but at its `...`. Lines that hold more than the quote + are not caught.""" + pieces = quote_pieces(quote) + if pieces: + held = _holds(_letters(copied), pieces) + else: + # A quote too short to search for with gaps must be in the lines letter for letter, each + # piece between its ... after the one before; one that says nothing isn't looked for. + held = not says_something(quote) or _in_order( + [letters for piece in _ELISION.split(quote) if (letters := _letters(piece))], + _letters(copied), + ) + if held: + return None + return ( + f"system_prompt_from: the lines named ({named}) don't hold system_prompt_excerpt; quote " + "the prompt word for word, with ... where you leave text out, and name the lines it is " + "written on" + ) diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index 7ddbf297a..d99ddc3b5 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -112,7 +112,14 @@ Find, in roughly this order: often kept away from the main agent file, so search the whole source for a long instructions string before concluding there are none. Name where the whole prompt is written in `system_prompt_from`, every part it is assembled from: it is copied into the contract as written, - and every later stage decides what the agent should do from it. + and every later stage decides what the agent should do from it. If the prompt is built or + fetched at run time and has no lines to name, quote it in `system_prompt_excerpt`; the save + asks once whether lines can be named. A quote is word for word from the prompt, a sentence or + more, with ... where you leave text out. The lines you name are checked against your latest + such quote, given with them or before. Lines that don't hold it are refused, and asked about + once more if they come without a quote; after that, the quote is kept and the lines are + recorded as unconfirmed (with a quote of fewer than 16 letters, the lines are kept, also + recorded as unconfirmed). 7. **The modality.** How this agent is run: a voice session, a text conversation, or work it carries out on websites or a computer. This decides how it is later run, so getting it wrong @@ -183,7 +190,8 @@ Find, in roughly this order: This is not who speaks first: an outbound agent usually still greets. It decides how the simulated person is briefed, and briefing someone who did not dial as though they had an errand to raise tests nothing about how the agent opens a call it placed. Two builds of the same agent - can differ only in this, so quote the line you read it from in `system_prompt_excerpt`. + can differ only in this, so quote the line you read it from in `notes`, unless it is already in + the prompt you name or quote. 9. **What it depends on.** Everything the agent reaches for that has to exist before it can work: a datastore, a service it calls over HTTP, a file it reads, a queue. Record each one, @@ -299,6 +307,19 @@ Find, in roughly this order: agent's own text says it does; a capability stretched from a passing word or a tool name becomes a whole slice of scenarios testing what the agent was never built to do. +16. **The model.** Record `model` only from the agent's own source: the one model the agent talks + to the person with. A voice agent's speech-to-text and text-to-speech models don't count. Its + provider is the company or service as the source names it (openai, anthropic, google, azure), + left out when the source doesn't say. Its name is written exactly and wholly as it appears on + the line you cite (openai/gpt-4o, if that is what is written); its path is relative to the + source root, as for the prompt's lines; its line is the 1-based line the name is written on. A name + or provider is made only of letters, digits and . _ : / @ -, so a name with a space in it + can't be recorded; leave `model` out then. + + Leave `model` out when the name is only set at run time: an environment variable with no + default in the code, or a value fetched from elsewhere. Where the code gives an environment + variable a default, record the default. + ## A repository may not hold one agent What you are pointed at is a directory, not necessarily a single agent. Before reading anything in diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index 57365734e..a86bb63c3 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -36,6 +36,7 @@ from .backends import tool, tool_server from .contract import ( + prompt_quote, CALL_DIRECTIONS, AgentContract, Runtime, @@ -57,12 +58,20 @@ COMPOSE_SERVICE_OUTSIDE_FOLDER, PathLookupsSpent, RuntimeSelectionError, - charged_path, path_lookup_budget, select_runtime, ) +from .model_citation import ( + model_problems, + quote_pieces, + quote_problem, + says_something, + read_source, + resolved_in_source, + source_lines, +) from .python_source import utf8_size -from .source_folder import SubfolderNotFound, agent_folder, bind_runtime, is_within +from .source_folder import SubfolderNotFound, agent_folder, bind_runtime _LOG = logging.getLogger(__name__) @@ -198,6 +207,8 @@ def _checked_contract( require_setup_contract: bool, subfolder: str | None, reached: list[Path] | None = None, + asked: set[str] | None = None, + held: dict[str, str] | None = None, ) -> AgentContract | dict[str, Any]: """The contract to save, or the problems to send back. It reads the customer's files and writes nothing, so the tool runs it off the event loop. ``reached`` is the code outside the @@ -208,6 +219,8 @@ def _checked_contract( contract = AgentContract.model_validate(payload) except Exception as invalid: return _problems([f"schema:{invalid}"[:600]], arrived) + # Only the save records lines it couldn't confirm. + contract.unconfirmed_prompt_from = [] problems = validate_contract(contract) if ( @@ -226,7 +239,7 @@ def _checked_contract( problems.append(problem) with path_lookup_budget(): return _checked_against_source( - contract, problems, arrived, source_root, subfolder, reached + contract, problems, arrived, source_root, subfolder, reached, asked, held ) @@ -237,16 +250,126 @@ def _checked_against_source( source_root: Path | None, subfolder: str | None, reached: list[Path] | None = None, + asked: set[str] | None = None, + held: dict[str, str] | None = None, ) -> AgentContract | dict[str, Any]: - """The checks that read the customer's files, under the save's one path lookup budget.""" + """The checks that read the customer's files, under the save's one path lookup budget. + ``asked`` is the reading's record of what it has been asked once. A model whose citation was + refused before is left out the second time, and lines that didn't hold the quote are left out + for the quote, rather than refused again: both are optional, and neither may cost the + reading its turns. ``held`` is the reading's latest quote (submit_contract keeps it), so + lines named later without one are still checked against what was quoted.""" if source_root is not None: from .source_tool_evidence import tool_evidence_problems + quote = own = contract.system_prompt_excerpt + # Lines named with no quote of their own (or one too short to say anything), checked + # against the reading's latest quote, which submit_contract holds. + alone = False + if ( + held is not None + and contract.system_prompt_from + and not quote_pieces(quote) + and held.get("prompt-quote") + # A short quote held for this very call is this call's own, not one given before. + and held["prompt-quote"] != quote + ): + quote = held["prompt-quote"] + alone = True + # Said when the fallback below sets the lines aside in a call refused for something else. + set_aside: list[str] = [] try: problems.extend(tool_evidence_problems(contract, source_root, subfolder)) - problems.extend(_copy_prompt(contract, source_root)) + copy_problems = _copy_prompt(contract, source_root) + problems.extend(copy_problems) + if contract.system_prompt_from and not copy_problems: + named = ", ".join( + f"{where.path[:200]} {where.start_line}-{where.end_line}" + for where in contract.system_prompt_from + ) + mismatch = quote_problem(quote, contract.system_prompt_excerpt, named) + confirmed = None if mismatch else quote + if ( + mismatch + # Only lines checked against a held quote can hold their own instead (and + # the second search is spent only then). + and alone + and says_something(own) + # Words, not a name: PROMPT or SYSTEM_PROMPT is on almost any prompt's lines. + and len(own.split()) > 1 + # A quote some lines already held stands: a short one doesn't replace it. + and held.get("prompt-quote-confirmed") != quote + and quote_problem(own, contract.system_prompt_excerpt, named) is None + ): + # The call's own short quote is in its lines letter for letter: that is this + # call's evidence, and the quote it passed with is the reading's latest now. + mismatch = None + held["prompt-quote"] = confirmed = own + if held is not None and confirmed and says_something(confirmed): + held["prompt-quote-confirmed"] = confirmed + if mismatch: + if asked is None: + problems.append(mismatch) + elif alone and "prompt-lines-alone" not in asked: + # The lines may be right and the quote they were checked against wrong: + # asked once to quote these lines, before either is given up. + asked.add("prompt-lines-alone") + shown = " ".join(quote.split())[:80] + outcome = ( + "the quote is kept and the lines are recorded as unconfirmed" + if quote_pieces(quote) + else "the lines are copied as the prompt and recorded as unconfirmed, " + "since that quote is too short to stand for it" + ) + problems.append( + f"system_prompt_from: the lines named ({named}) don't hold the quote " + f"you gave before ({shown}); if these lines are the prompt, quote " + "them word for word in system_prompt_excerpt, with ... where you " + f"leave text out. If they are refused again against it, {outcome}" + ) + elif asked & {"prompt-quote", "prompt-lines-alone"}: + # Either way the lines are kept on record as unconfirmed. + contract.unconfirmed_prompt_from = list(contract.system_prompt_from) + if quote_pieces(quote): + # The quote, which is what was read, rather than lines nobody + # confirmed. + contract.system_prompt_excerpt = quote + contract.system_prompt_from = [] + set_aside.append( + f"system_prompt_from: the lines named ({named}) are left out and " + "the quote is kept, since they don't hold it" + ) + else: + # A quote of a few words says too little to stand for the prompt: + # the lines are copied. + set_aside.append( + f"system_prompt_from: the lines named ({named}) are copied " + "unconfirmed, since they don't hold the short quote" + ) + else: + problems.append(mismatch) + asked.add("prompt-quote") + problems.append( + "system_prompt_from: lines named later are checked against this " + "quote until you give another; a later mismatch records the lines as " + "unconfirmed and keeps the quote, or the lines if the quote has fewer " + "than 16 letters" + ) + if cited := model_problems(contract, source_root): + if asked is not None and "model" in asked: + contract.model = None + else: + problems.extend(cited) + if asked is not None: + asked.add("model") + problems.append("model: if the model is refused again, it is left out") except PathLookupsSpent as spent: return _problems([*problems, str(spent)], arrived) + if problems: + problems.extend(set_aside) + else: + # With no source there is no line to find the name on, so the model is unknown. + contract.model = None runtime = contract.runtime if subfolder and source_root is not None: try: @@ -309,13 +432,24 @@ def _save_contract(contract: AgentContract, destination: Path) -> dict[str, Any] os.replace(partial, path) finally: partial.unlink(missing_ok=True) - return _ok( + said = ( f"Accepted and saved to {path}.\n" f"{len(contract.tools)} tools: {', '.join(sorted(contract.tool_names()))}\n" f"{len(contract.hard_constraints)} rules, " f"{len(contract.real_use_cases)} use cases, " f"{len(contract.open_questions)} open questions." ) + if contract.unconfirmed_prompt_from: + named = ", ".join( + f"{where.path[:200]} {where.start_line}-{where.end_line}" + for where in contract.unconfirmed_prompt_from + ) + kept = "the lines are the prompt" if contract.system_prompt_from else "the quote is the prompt" + said += ( + f"\nsystem_prompt_from: the lines named ({named}) are recorded as unconfirmed, since " + f"they don't hold the quote; {kept}." + ) + return _ok(said) # One prompt file, and what the excerpt copies from all of them: the same file named a hundred @@ -338,22 +472,20 @@ def _copy_prompt(contract: AgentContract, source_root: Path) -> list[str]: read = copied = 0 for index, where in enumerate(contract.system_prompt_from): if where.path not in found_at: - try: - # Through links, resolving one path can cost a lookup for each part of every - # target; each is charged to the save's budget. - resolved = charged_path(root / where.path) - found_at[where.path] = ( - resolved if is_within(resolved, root) and resolved.is_file() else None - ) - except (OSError, RuntimeError, ValueError): - found_at[where.path] = None + # Through links, resolving one path can cost a lookup for each part of every + # target; each is charged to the save's budget. + found_at[where.path] = resolved_in_source(root, where.path) path = found_at[where.path] + raw = None + if path is not None and path not in lines_of and path not in too_large: + raw = read_source(path, _LARGEST_PROMPT_FILE) + if raw is None: + # Not a regular file, or one that can't be opened: the same answer as missing. + found_at[where.path] = path = None if path is None: problems.append(f"system_prompt_from[{index}]: {where.path} is not a file in the source") continue - if path not in lines_of and path not in too_large: - with path.open("rb") as handle: - raw = handle.read(_LARGEST_PROMPT_FILE + 1) + if raw is not None: read += len(raw) if read > _LARGEST_PROMPT_READ: problems.append( @@ -364,7 +496,7 @@ def _copy_prompt(contract: AgentContract, source_root: Path) -> list[str]: if len(raw) > _LARGEST_PROMPT_FILE: too_large.add(path) else: - lines_of[path] = raw.decode("utf-8", errors="replace").splitlines() + lines_of[path] = source_lines(raw) if path in too_large: problems.append( f"system_prompt_from[{index}]: {where.path} is larger than 1 MiB; name the " @@ -514,6 +646,8 @@ def contract_tools( # A gate with no way through would permanently block the rare agent that genuinely lacks it, # and this stage cannot tell those two apart from the outside. nudged: set[str] = set() + # What was quoted for the prompt when the lines named for it were refused. + held: dict[str, str] = {} refused = False second_look = SecondLook() found: list[str] | None = None @@ -609,7 +743,26 @@ def contract_tools( }, "description": "Where the agent's whole prompt is written: every part it is " "assembled from, in order, as a path relative to the source root and the first " - "and last line of each. These lines replace system_prompt_excerpt as written.", + "and last line of each. Once they are checked against your quote in " + "system_prompt_excerpt, these lines replace it as written: keep the quote too.", + }, + # Nothing required inside it either: a model sent without its line reaches the + # save, which says what is missing, instead of failing the whole call here. + "model": { + "type": "object", + "properties": { + "provider": {"type": "string"}, + "name": {"type": "string"}, + "path": {"type": "string"}, + "line": {"type": "integer"}, + }, + "description": "The model the agent talks to the person with, read from its " + "source: provider as the source names it (openai, anthropic, google...; leave " + "out when it doesn't say), name exactly as written (letters, digits and " + ". _ : / @ - only), and the file (relative to " + "the source root) and line the name is written on. One model; a voice agent's " + "speech-to-text and text-to-speech models don't count. Leave model out when " + "the name is only set at run time.", }, "hard_constraints": { "type": "array", @@ -1308,6 +1461,20 @@ def contract_tools( ) async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: payload = dict(unwrapped(args)) + # The quote as the contract will read it: under any alias, a list joined. + excerpt = prompt_quote(_without_nulls(payload)) + if not excerpt.strip(): + excerpt = "" + # Held before anything can send this call back (a nudge, a refusal, a schema problem), so + # lines named later without a quote are checked against the reading's latest one. A short + # quote is held only while nothing longer is: it says less. + if quote_pieces(excerpt) or ( + says_something(excerpt) and not quote_pieces(held.get("prompt-quote", "")) + ): + held["prompt-quote"] = excerpt + # What this call was already asked, as it stood when the call was sent: a call sent in the + # same message as one that is refused hasn't seen that refusal yet. + asked_before = set(nudged) receipt = payload.pop(SECOND_LOOK_READ, None) or ( args.get(SECOND_LOOK_READ) if isinstance(args, dict) else None ) @@ -1532,11 +1699,14 @@ def rows_for(collection: str) -> list[dict[str, Any]]: "prompt", bool(payload.get("conversational", True)) and not payload.get("hard_constraints") - and not str(payload.get("system_prompt_excerpt") or "").strip(), + and not excerpt + # Lines named are the prompt, found: copied whole, with no quote needed. + and not payload.get("system_prompt_from"), "no hard_constraints and no system_prompt_excerpt, for a conversational agent. " "Its prompt usually exists and often lives away from the main agent file — " "search the whole source for a long instructions string before deciding there " - "is none.", + "is none. If you find it, name its lines in system_prompt_from so it is copied " + "whole.", ), ( "data", @@ -1575,10 +1745,24 @@ def rows_for(collection: str) -> list[dict[str, Any]]: "values. Semantic summaries such as rooms, notable_bookings or restaurant_slots " "cannot seed or verify the submitted store.", ), + ( + # A quote may stop short of the whole prompt; lines are copied whole. Asked once, + # since a prompt built at run time has no lines to name, and its quote is kept. + "prompt-lines", + source_root is not None + and bool(excerpt) + and not payload.get("system_prompt_from") + # Asked already, with the "prompt" nudge or a refusal of named lines: the lines + # aren't asked for twice. + and not asked_before & {"prompt", "prompt-quote", "prompt-lines-alone"}, + "system_prompt_excerpt is a quote, and system_prompt_from names no lines. If the " + "prompt is written in the source, name its lines in system_prompt_from so it is " + "copied whole.", + ), ] # All of them together, and each only once. Nudging in sequence would cost a turn per # nudge and read as though the requirements were being invented one at a time. - say = [said for key, when, said in thin if when and key not in nudged] + say = [said for key, when, said in thin if when and key not in asked_before] nudged.update(key for key, when, _ in thin if when) if say: return _problems( @@ -1594,7 +1778,11 @@ def rows_for(collection: str) -> list[dict[str, Any]]: subfolder=subfolder, # The scan's answer is for scan_root; another root's code is found again. reached=reach if scan_root == source_root else None, + asked=asked_before, + held=held, ) + # Whatever this call asked is asked for the calls sent after it. + nudged.update(asked_before) if isinstance(checked, dict): return checked if refused or (destination / FAILURE_FILE).is_file(): diff --git a/src/fi/alk/providers/agent.py b/src/fi/alk/providers/agent.py index 18cf6dbdd..acfa4bee9 100644 --- a/src/fi/alk/providers/agent.py +++ b/src/fi/alk/providers/agent.py @@ -180,7 +180,7 @@ def build_authored_agent( } ], "tools": tools, - "model": None, + "model": _authored_model(contract.get("model")), "voice": None, "telephony": [], "supports": [], @@ -232,6 +232,17 @@ def _authored_tool( } +def _authored_model(value: Any) -> dict[str, Any] | None: + """The document's model: the contract's provider and name, or None when it has no name.""" + if not isinstance(value, Mapping): + return None + name = str(value.get("name") or "").strip()[:_NAME_MAX] + if not name: + return None + provider = str(value.get("provider") or "").strip()[:_NAME_MAX] or None + return {"provider": provider, "name": name, "temperature": None, "max_tokens": None} + + def _redact(value: Any, key: str = "") -> Any: lowered = key.lower() if any(marker in lowered for marker in _REDACTION_MARKERS): diff --git a/tests/harness/test_contract_model.py b/tests/harness/test_contract_model.py new file mode 100644 index 000000000..6ba8e7dc8 --- /dev/null +++ b/tests/harness/test_contract_model.py @@ -0,0 +1,3465 @@ +"""The model an agent talks to the person with: recorded only where its name is the value written +on the line the reader cites. + +The save is the gate. A name that isn't that value (not quoted, a setting's name, a key, inside a +comment) is refused with a message saying what to do, once per reading; a contract without a +model is saved exactly as before. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import threading +import time +from pathlib import Path + +import pytest + +from fi.alk.harness.contract import AgentContract +from fi.alk.harness.runtime_selection import PathLookupsSpent, path_lookup_budget +from fi.alk.harness.model_citation import model_problems, named_on +from fi.alk.harness.tools import accept_contract, contract_tools + +MIB = 1024 * 1024 +FIXTURES = Path(__file__).resolve().parents[1] / "providers" / "fixtures" / "authored" + +EMPTY_NAME = ( + "model: name is empty; give the model's name as written in the source, or leave model out" +) +NAME_CHARACTERS = ( + "model: a model name is letters, digits and . _ : / @ -; write it as it appears in the " + "source, or leave model out" +) +PROVIDER_CHARACTERS = ( + "model: a provider is letters, digits and . _ : / @ -; write it as the source names it, or " + "leave it out" +) +TOO_LONG = "model: name and provider are at most 255 characters" +NO_LINE = "model: give path and line, the file and the line the model's name is written on" +LONG_PATH = "model: path is too long" +REFUSED = "Not accepted. Fix all of these and call submit_contract again:\n - " +PROMPT_LINES = ( + "system_prompt_excerpt is a quote, and system_prompt_from names no lines. If the prompt is " + "written in the source, name its lines in system_prompt_from so it is copied whole." +) +ASKED_FOR_LINES = ( + REFUSED + + PROMPT_LINES + + "\n - If any of these genuinely does not apply, submit again as is." +) + + +def not_a_file(path: str) -> str: + return f"model: {path[:200]} is not a file in the source" + + +def unreadable(path: str) -> str: + return ( + f"model: {path[:200]} can't be read as a file of at most 1 MiB; check the path, and cite " + "the line in the file the name is written in" + ) + + +def outside(line: int, path: str, count: int) -> str: + return f"model: line {line} is outside {path[:200]}, which has {count} lines" + + +def not_named(name: str, line: int, path: str) -> str: + return ( + f"model: the name {name[:100]} is not the value written on line {line} of {path[:200]}; " + "cite the line where it is written in quotes, or after : or = in a configuration file, " + "or leave model out if the name is only set at run time" + ) + + +def payload(**fields): + return { + "agent": "pair", + "modality": "chat", + "real_use_cases": ["order"], + "tools": [{"name": "add", "args": ["item_id"]}], + **fields, + } + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + folder = tmp_path / "src" + folder.mkdir() + (folder / "agent.py").write_text('AGENT = "pair"\nMODEL = "gpt-4o"\n', encoding="utf-8") + return folder + + +def checked(root: Path, model) -> tuple[list[str], AgentContract, float]: + """The model check alone, under the save's budget, and how long it took.""" + contract = AgentContract.model_validate(payload(model=model)) + with path_lookup_budget(): + started = time.perf_counter() + problems = model_problems(contract, root) + took = time.perf_counter() - started + return problems, contract, took + + +def saved(destination: Path) -> dict: + return json.loads((destination / "contract.json").read_text(encoding="utf-8")) + + +def link(target: str, at: Path) -> None: + try: + os.symlink(target, at) + except (OSError, NotImplementedError, AttributeError) as unable: + pytest.skip(f"links can't be made here: {unable}") + + +def chain(root: Path, prefix: str, count: int) -> None: + """Links each costing about a thousand lookups to follow, the last leading to agent.py.""" + for index in range(count): + following = f"{prefix}{index + 1}" if index < count - 1 else "agent.py" + link("." + "/" * 999 + following, root / f"{prefix}{index}") + + +def text_of(result: dict) -> str: + return result["content"][0]["text"] + + +def submitter(destination: Path, source_root: Path | None = None): + server = contract_tools(destination, source_root=source_root) + tool = next(one for one in server.tools if one.name == "submit_contract") + return lambda arguments: asyncio.run(tool.handler(dict(arguments))) + + +# A contract without a model saves and builds as before + + +def test_a_contract_without_a_model_saves_and_builds_as_before(tmp_path): + from fi.alk.providers.agent import build_authored_agent + + contract = json.loads((FIXTURES / "chat_source.json").read_text(encoding="utf-8"))["contract"] + + result = accept_contract(dict(contract), tmp_path / "out") + + assert not result.get("is_error"), result + raw = (tmp_path / "out" / "contract.json").read_bytes() + assert hashlib.sha256(raw).hexdigest() == ( + "7a61d8e863f75541c6d32c287fea02fb7a605d3b440a91f4c1b4212a3e1832b6" + ) + assert "model" not in json.loads(raw) + document = build_authored_agent( + AgentContract.model_validate(json.loads(raw)).model_dump(mode="json") + ) + assert document["setup_hash"] == ( + "sh1:3854d6ecb87a1ee2890c5c16f25e854e20be84c109dc80311ad292e6f656d622" + ) + assert document["model"] is None + + +# No source: nothing to check the model against + + +def test_model_without_a_source_is_dropped(tmp_path): + for index, model in enumerate( + ( + {"provider": "openai", "name": "gpt-4o", "path": "agent.py", "line": 2}, + # Unchecked shapes too: with no source nothing about the model is known. + {"name": "gpt 4o"}, + ) + ): + destination = tmp_path / f"out{index}" + + result = accept_contract(payload(model=model), destination) + + assert not result.get("is_error"), result + assert "model" not in saved(destination) + + +# The citation's shape, checked before the disk + + +@pytest.mark.parametrize( + "model, expected", + [ + pytest.param({"name": "", "path": "agent.py", "line": 2}, [EMPTY_NAME], id="empty-name"), + pytest.param({"name": " ", "path": "agent.py", "line": 2}, [EMPTY_NAME], id="blank-name"), + pytest.param( + {"name": "gpt 4o", "path": "agent.py", "line": 2}, + [NAME_CHARACTERS], + id="name-with-a-space", + ), + pytest.param( + {"name": 'gpt-4o"', "path": "agent.py", "line": 2}, + [NAME_CHARACTERS], + id="name-with-a-quote", + ), + pytest.param( + {"provider": "open\x00ai", "name": "gpt-4o", "path": "agent.py", "line": 2}, + [PROVIDER_CHARACTERS], + id="provider-with-a-nul", + ), + pytest.param( + {"provider": "http://[x", "name": "gpt-4o", "path": "agent.py", "line": 2}, + [PROVIDER_CHARACTERS], + id="provider-url-bracket", + ), + pytest.param("gpt-4o", [NO_LINE], id="bare-string"), + pytest.param( + {"name": "a" * 256, "path": "agent.py", "line": 2}, [TOO_LONG], id="long-name" + ), + pytest.param( + {"provider": "a" * 256, "name": "gpt-4o", "path": "agent.py", "line": 2}, + [TOO_LONG], + id="long-provider", + ), + pytest.param({"name": "gpt-4o", "line": 2}, [NO_LINE], id="no-path"), + pytest.param({"name": "gpt-4o", "path": " ", "line": 2}, [NO_LINE], id="blank-path"), + pytest.param( + {"name": "gpt-4o", "path": "agent.py", "line": 0}, [NO_LINE], id="line-zero" + ), + pytest.param( + {"name": "gpt-4o", "path": "a" * 4097, "line": 2}, [LONG_PATH], id="long-path" + ), + ], +) +def test_model_shape_problems(root, model, expected): + problems, _, _ = checked(root, model) + + assert problems == expected + + +# The cited path must lead to a file in the source + + +def _parent(root: Path) -> str: + return "../outside.py" + + +def _absolute_outside(root: Path) -> str: + return str(root.parent / "outside.py") + + +def _link_outside(root: Path) -> str: + link(str(root.parent / "outside.py"), root / "out.py") + return "out.py" + + +def _link_to_dev_zero(root: Path) -> str: + if not Path("/dev/zero").exists(): + pytest.skip("no /dev/zero here") + # Only made, never read: it must be refused before anything opens it. + link("/dev/zero", root / "zero.py") + return "zero.py" + + +def _link_loop(root: Path) -> str: + link("b.py", root / "a.py") + link("a.py", root / "b.py") + return "a.py" + + +def _nul(root: Path) -> str: + return "agent\x00.py" + + +def _unwinding_chain(root: Path) -> str: + # Each link leaves a long part behind as it unwinds, so the path grows past any limit. + for index in range(900): + target = f"L{index + 1}/" + "A" * 990 if index < 899 else "." + link(target, root / f"L{index}") + return "L0/agent.py" + + +@pytest.mark.parametrize( + "cited", + [ + pytest.param(_parent, id="parent"), + pytest.param(_absolute_outside, id="absolute-outside"), + pytest.param(_link_outside, id="link-outside"), + pytest.param(_link_to_dev_zero, id="link-to-dev-zero"), + pytest.param(_link_loop, id="link-loop"), + pytest.param(_nul, id="nul"), + pytest.param(_unwinding_chain, id="unwinding-chain"), + ], +) +def test_model_path_outside_the_source_is_refused(root, cited): + # Holds the name on the cited line, so only where the file is can refuse it. + (root.parent / "outside.py").write_text('AGENT = "pair"\nMODEL = "gpt-4o"\n') + path = cited(root) + + problems, _, took = checked(root, {"name": "gpt-4o", "path": path, "line": 2}) + + assert problems == [not_a_file(path)] + assert took < 0.5 + + +# The save's one budget + + +def test_model_path_spending_the_save_budget_is_refused(tmp_path, root): + chain(root, "E", 25) + destination = tmp_path / "out" + + started = time.perf_counter() + result = accept_contract( + payload(model={"name": "gpt-4o", "path": "E0", "line": 2}), + destination, + source_root=root, + ) + took = time.perf_counter() - started + + assert result.get("is_error") + assert text_of(result) == REFUSED + str(PathLookupsSpent()) + assert not (destination / "contract.json").exists() + assert took < 0.5 + + +@pytest.mark.parametrize("links", [pytest.param(11, id="lookup-budget-real")]) +def test_model_path_charged_to_the_same_budget_as_the_prompt(tmp_path, root, links): + chain(root, "P", links) + chain(root, "M", links) + model = {"name": "gpt-4o", "path": "M0", "line": 2} + # Either half alone fits the budget, so only charging both to one budget refuses them. + alone, _, _ = checked(root, model) + assert alone == [] + destination = tmp_path / "out" + + result = accept_contract( + payload( + system_prompt_from=[{"path": "P0", "start_line": 1, "end_line": 2}], + model=model, + ), + destination, + source_root=root, + ) + + assert result.get("is_error") + assert text_of(result) == REFUSED + str(PathLookupsSpent()) + assert not (destination / "contract.json").exists() + + +def test_model_path_through_an_honest_link_is_kept(tmp_path, root): + (root / "pkg").mkdir() + (root / "pkg" / "settings.py").write_text('MODEL = "gpt-4o"\n') + link("pkg/settings.py", root / "alias.py") + destination = tmp_path / "out" + + result = accept_contract( + payload(model={"provider": "openai", "name": "gpt-4o", "path": "alias.py", "line": 1}), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["model"] == { + "provider": "openai", + "name": "gpt-4o", + "path": "pkg/settings.py", + "line": 1, + } + + +# The cited file must be readable, and at most 1 MiB + + +def _over_a_mib(root: Path) -> str: + head = b'MODEL = "gpt-4o"\n' + (root / "big.py").write_bytes(head + b"#" * (MIB + 1 - len(head))) + return "big.py" + + +def _fifo_inside(root: Path) -> str: + if not hasattr(os, "mkfifo"): + pytest.skip("no FIFOs here") + try: + os.mkfifo(root / "pipe.py") + except OSError as unable: + pytest.skip(f"a FIFO can't be made here: {unable}") + return "pipe.py" + + +def _directory(root: Path) -> str: + (root / "pkg").mkdir() + return "pkg" + + +def _long_component(root: Path) -> str: + return "a" * 300 + + +@pytest.mark.parametrize( + "cited", + [ + pytest.param(_over_a_mib, id="over-1-mib"), + pytest.param(_fifo_inside, id="fifo-inside"), + pytest.param(_directory, id="directory"), + pytest.param(_long_component, id="long-component"), + ], +) +def test_model_file_that_cannot_be_read_is_refused(root, cited): + path = cited(root) + # On a thread that can't hold the run open: a check that opens a FIFO waits on it forever. + answer: list = [] + checking = threading.Thread( + target=lambda: answer.append(checked(root, {"name": "gpt-4o", "path": path, "line": 1})), + daemon=True, + ) + checking.start() + checking.join(timeout=5) + assert answer, "the check did not return" + problems, _, took = answer[0] + + assert problems == [unreadable(path)] + assert took < 0.5 + + +# The cited line must be in the file + + +def test_model_line_outside_the_file_is_refused(root): + (root / "three.py").write_text('A = 1\nB = 2\nMODEL = "gpt-4o"\n') + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "three.py", "line": 4}) + + assert problems == [outside(4, "three.py", 3)] + + +# The name must be the value written on the line + + +@pytest.mark.parametrize( + "written, name", + [ + pytest.param("abc", "", id="empty-name-direct"), + pytest.param('MODEL = "claude"', "gpt-4o", id="absent"), + pytest.param('model="gpt-4o-mini"', "gpt-4o", id="prefix-of-longer"), + pytest.param('model="gpt-4o"', "o", id="inside-longer"), + pytest.param( + 'MODEL = "llama3:8b-instruct-q4_K_M"', "llama3:8b-instruct-q4", id="underscore-suffix" + ), + pytest.param('model="GPT-4o"', "gpt-4o", id="case"), + pytest.param("gpt-4o is the model", "gpt-4o", id="not-quoted"), + # Dotted with a version, so not refused as a setting's name: refused as not written there. + pytest.param("llm = ChatOpenAI(model=cfg.gpt4o)", "cfg.gpt4o", id="attribute"), + pytest.param('model = "gpt-4o" # was gpt-4o-mini', "gpt-4o-mini", id="in-a-comment"), + pytest.param("«gpt-4o»", "gpt-4o", id="non-ascii-marks"), + pytest.param("model = gpt_4o", "gpt_4o", id="bare-in-python"), + ], +) +def test_model_name_not_on_the_cited_line_is_refused(root, written, name): + if not name: + # On a thread that can't hold the run open: a search that never moves on fails here. + answer: list[bool] = [] + searching = threading.Thread( + target=lambda: answer.append(named_on(written, name, config=False)), daemon=True + ) + searching.start() + searching.join(timeout=2) + assert answer == [False] + return + (root / "named.py").write_text(f"# the model\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": "named.py", "line": 2}) + + assert problems == [not_named(name, 2, "named.py")] + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("named.py", 'model="gpt-4o"', "gpt-4o", id="quoted"), + pytest.param("named.py", "model='gpt-4o'", "gpt-4o", id="single-quoted"), + pytest.param("named.ts", "const model = `gpt-4o`", "gpt-4o", id="backticks"), + pytest.param("named.yaml", "model: gpt-4o", "gpt-4o", id="line-end-in-yaml"), + pytest.param(".env", "OPENAI_MODEL=gpt-4o", "gpt-4o", id="env-file"), + pytest.param("named.toml", 'model = "gpt-4o"', "gpt-4o", id="toml-quoted"), + pytest.param( + "named.py", 'model = "openai/gpt-4o"', "openai/gpt-4o", id="with-provider-prefix" + ), + pytest.param( + "named.py", 'x = "gpt-4o-mini" or "gpt-4o"', "gpt-4o", id="second-occurrence" + ), + pytest.param( + "named.py", 'os.getenv("OPENAI_MODEL", "gpt-4o")', "gpt-4o", id="getenv-default" + ), + ], +) +def test_model_whole_name_is_found(root, file, written, name): + (root / file).write_text(f"# the model\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +# What is saved + + +def test_model_on_the_cited_line_is_saved_stripped(tmp_path, root): + destination = tmp_path / "out" + + result = accept_contract( + payload( + model={ + "provider": " openai ", + "name": " gpt-4o ", + "path": f" {root / 'agent.py'} ", + "line": 2, + } + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["model"] == { + "provider": "openai", + "name": "gpt-4o", + "path": "agent.py", + "line": 2, + } + + +# One numbering of a file's lines, for the prompt and the model + + +@pytest.mark.parametrize( + "written, line", + [ + # A form feed is no line break to the reading's Read, nor to the save. + pytest.param(b'# head\nA = 1\x0cB = 2\nMODEL = "gpt-4o"\n', 3, id="form-feed"), + # Read as latin-1, \x85 is a line break and the model's line moves down one. + pytest.param(b'# coding: latin-1\nA = "\x85"\nMODEL = "gpt-4o"\n', 3, id="latin-1-c1"), + ], +) +def test_model_line_numbering_matches_the_prompt_copy(tmp_path, root, written, line): + (root / "numbered.py").write_bytes(written) + destination = tmp_path / "out" + + result = accept_contract( + payload( + system_prompt_from=[{"path": "numbered.py", "start_line": line, "end_line": line}], + model={"name": "gpt-4o", "path": "numbered.py", "line": line}, + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + contract = saved(destination) + assert contract["system_prompt_excerpt"] == 'MODEL = "gpt-4o"' + assert contract["model"]["line"] == line + + +# Crafted files + + +def _ace_label() -> bytes: + written = "".join(chr(0x4E00 + index * 37) for index in range(40)) + while written: + try: + label = written.encode("idna") + except UnicodeError: + label = b"" + if label and len(label) <= 63: + return label + written = written[:-1] + raise AssertionError("no ACE label") + + +def _filled(head: bytes, unit: bytes) -> bytes: + return head + unit * ((MIB - len(head)) // len(unit)) + + +def _one_line() -> bytes: + head = b'MODEL = "gpt-4o" #' + return head + b"x" * (MIB - len(head)) + + +def _idna_ace() -> bytes: + return _filled(b"# coding: idna\nMODEL = 'gpt-4o'\n", _ace_label() + b".") + + +def _punycode() -> bytes: + return _filled(b"# coding: punycode\nMODEL = 'gpt-4o'\n", b"a" * 60 + b"-" + b"kva" + b"\n") + + +def _line_of_m() -> bytes: + return b"m" * MIB + + +def _name_at_the_end() -> bytes: + tail = b'="' + b"m" * 255 + b'"' + return b"m" * (MIB - len(tail)) + tail + + +@pytest.mark.parametrize( + "written, line, name, refused", + [ + pytest.param(_one_line, 1, "gpt-4o", False, id="one-line-1-mib"), + pytest.param(_idna_ace, 2, "gpt-4o", False, id="idna-ace-1-mib"), + pytest.param(_punycode, 2, "gpt-4o", False, id="punycode-1-mib"), + pytest.param(_line_of_m, 1, "m" * 255, True, id="long-name-on-a-line-of-it"), + pytest.param(_name_at_the_end, 1, "m" * 255, False, id="name-at-the-end-of-a-line-of-it"), + ], +) +def test_a_crafted_model_file_does_not_hold_up_the_save(root, written, line, name, refused): + content = written() + assert len(content) <= MIB + (root / "crafted.py").write_bytes(content) + + problems, _, took = checked(root, {"name": name, "path": "crafted.py", "line": line}) + + assert problems == ([not_named(name, line, "crafted.py")] if refused else []) + assert took < 0.5 + + +# Asking once for the prompt's lines + + +def test_a_quote_without_lines_is_asked_for_lines_once(tmp_path, root): + destination = tmp_path / "out" + submit = submitter(destination, root) + quoted = payload( + hard_constraints=["Never share a card number."], + system_prompt_excerpt="You take orders for the cafe.", + ) + + first = submit(quoted) + + assert first.get("is_error") + assert text_of(first) == ASKED_FOR_LINES + assert not (destination / "contract.json").exists() + + second = submit(quoted) + + assert not second.get("is_error"), second + assert PROMPT_LINES not in text_of(second) + assert saved(destination)["system_prompt_excerpt"] == "You take orders for the cafe." + + +@pytest.mark.parametrize("alias", ["system_prompt", "prompt", "instructions"]) +def test_quote_under_an_alias_is_asked_too(tmp_path, root, alias): + destination = tmp_path / "out" + submit = submitter(destination, root) + quoted = payload( + hard_constraints=["Never share a card number."], + **{alias: "You take orders for the cafe."}, + ) + + first = submit(quoted) + + assert text_of(first) == ASKED_FOR_LINES + assert not (destination / "contract.json").exists() + second = submit(quoted) + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "You take orders for the cafe." + + +def test_lines_given_are_not_asked(tmp_path, root): + destination = tmp_path / "out" + submit = submitter(destination, root) + + first = submit( + payload( + hard_constraints=["Never share a card number."], + system_prompt_excerpt='AGENT = "pair"', + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 2}], + ) + ) + + assert not first.get("is_error"), first + assert PROMPT_LINES not in text_of(first) + assert saved(destination)["system_prompt_excerpt"] == 'AGENT = "pair"\nMODEL = "gpt-4o"' + + +def test_no_source_is_not_asked(tmp_path): + destination = tmp_path / "out" + submit = submitter(destination) + + first = submit( + payload( + hard_constraints=["Never share a card number."], + system_prompt_excerpt="You take orders for the cafe.", + ) + ) + + assert not first.get("is_error"), first + assert PROMPT_LINES not in text_of(first) + assert saved(destination)["system_prompt_excerpt"] == "You take orders for the cafe." + + +# The schema the model writes against + + +def test_submit_contract_schema_describes_model(tmp_path): + server = contract_tools(tmp_path) + schema = next(one for one in server.tools if one.name == "submit_contract").input_schema + + model = schema["properties"]["model"] + + assert set(model["properties"]) == {"provider", "name", "path", "line"} + assert "required" not in model + assert "model" not in schema["required"] + + +def test_saved_model_and_whole_prompt_reach_the_document(tmp_path): + # The whole path the platform sees: the save copies the prompt's lines over the quote and + # records the model, the build reads contract.json back, and the document carries both. + from fi.alk.providers.agent import build_authored_agent + + source = tmp_path / "source" + source.mkdir() + (source / "agent.py").write_text( + "import openai\n" + "PROMPT = '''\n" + "You book rides for riders.\n" + "Never share a rider's PIN.\n" + "Confirm the pickup before booking.\n" + "'''\n" + "\n" + "client = openai.OpenAI()\n" + 'model="gpt-4o"\n' + ) + payload = json.loads((FIXTURES / "chat_source.json").read_text())["contract"] + payload.update( + # A quote of the prompt's start; the named lines then send the whole of it. + system_prompt_excerpt="You book rides for riders.", + system_prompt_from=[{"path": "agent.py", "start_line": 3, "end_line": 5}], + model={"provider": "openai", "name": "gpt-4o", "path": "agent.py", "line": 9}, + ) + out = tmp_path / "out" + + result = accept_contract(payload, out, source_root=source) + + assert not result.get("is_error"), result + contract = AgentContract.model_validate( + json.loads((out / "contract.json").read_text(encoding="utf-8")) + ).model_dump(mode="json") + document = build_authored_agent(contract) + assert document["behaviour"][0]["instructions"] == ( + "You book rides for riders.\n" + "Never share a rider's PIN.\n" + "Confirm the pickup before booking." + ) + assert document["model"] == { + "provider": "openai", + "name": "gpt-4o", + "temperature": None, + "max_tokens": None, + } + assert document["extension"]["contract"]["model"]["path"] == "agent.py" + + +@pytest.mark.parametrize( + "model, said", + [ + pytest.param( + {"name": "gpt-5", "path": "agent.py", "line": 2}, + lambda: not_named("gpt-5", 2, "agent.py"), + id="not-on-line", + ), + pytest.param( + {"name": "gpt-4o", "path": "../outside.py", "line": 2}, + lambda: not_a_file("../outside.py"), + id="parent", + ), + ], +) +def test_a_refused_model_is_not_saved(tmp_path, root, model, said): + # The check's answer has to reach the save: a model it refuses is never written. + (tmp_path / "outside.py").write_text('AGENT = "pair"\nMODEL = "gpt-4o"\n', encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract(payload(model=model), destination, source_root=root) + + assert result.get("is_error") + assert text_of(result) == REFUSED + said() + assert not (destination / "contract.json").exists() + + +def test_a_quote_under_an_alias_is_not_called_missing(tmp_path, root): + # A quote under another name is a quote: the save asks for its lines, and never says in the + # same answer that there is no prompt at all. + destination = tmp_path / "out" + submit = submitter(destination, root) + + first = submit(payload(prompt="You take orders for the cafe.")) + + assert text_of(first) == ASKED_FOR_LINES + + +# The name must be the value the model is set from, through the whole save + + +@pytest.mark.parametrize( + "written, name, said", + [ + pytest.param( + 'MODEL = os.getenv("OPENAI_MODEL")', + "OPENAI_MODEL", + "model: OPENAI_MODEL reads as the name of a setting, not a model; cite the line its " + "value is written on, or leave model out", + id="getenv-setting-name", + ), + pytest.param( + "llm = ChatOpenAI(model=settings.llm_model)", + "settings.llm_model", + "model: settings.llm_model reads as the name of a setting, not a model; cite the line " + "its value is written on, or leave model out", + id="attribute", + ), + pytest.param('model = "gpt-4o" # was gpt-4o-mini', "gpt-4o-mini", None, id="in-a-comment"), + ], +) +def test_a_name_that_is_not_the_model_is_never_sent(tmp_path, root, written, name, said): + from fi.alk.providers.agent import build_authored_agent + + (root / "llm.py").write_text(f"import os\n{written}\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload(model={"provider": "openai", "name": name, "path": "llm.py", "line": 2}), + destination, + source_root=root, + ) + + assert text_of(result) == REFUSED + (said or not_named(name, 2, "llm.py")) + assert not (destination / "contract.json").exists() + # Left out, the model is unknown in the document rather than wrong. + saved_without = accept_contract(payload(), destination, source_root=root) + assert not saved_without.get("is_error"), saved_without + contract = AgentContract.model_validate(saved(destination)).model_dump(mode="json") + assert build_authored_agent(contract)["model"] is None + + +# The lines named for the prompt must hold the quote they replace + +def prose(path: str) -> str: + return ( + f"model: {path} is text (for people, or a prompt), not code or configuration; cite the line " + "in the agent's code or configuration the model is set on, or leave model out" + ) + + +def lines_alone(named: str, quote: str, *, short: bool = False) -> str: + outcome = ( + "the lines are copied as the prompt and recorded as unconfirmed, since that quote is too " + "short to stand for it" + if short + else "the quote is kept and the lines are recorded as unconfirmed" + ) + return ( + f"system_prompt_from: the lines named ({named}) don't hold the quote you gave before " + f"({' '.join(quote.split())[:80]}); if these lines are the prompt, quote them word for " + f"word in system_prompt_excerpt, with ... where you leave text out. If they are refused " + f"again against it, {outcome}" + ) + + +def not_the_quote(named: str) -> str: + return ( + f"system_prompt_from: the lines named ({named}) don't hold system_prompt_excerpt; quote " + "the prompt word for word, with ... where you leave text out, and name the lines it is " + "written on" + ) + + +@pytest.mark.parametrize( + "quote, lines, refused, kept", + [ + pytest.param( + "You are pair, a helpful assistant.", (1, 2), True, None, id="wrong-lines" + ), + pytest.param( + "You are pair,\n a helpful assistant who books", + (3, 3), + False, + 'PROMPT = """You are pair, a helpful assistant who books tables."""', + id="same-text-other-spacing", + ), + pytest.param( + "", + (1, 2), + False, + 'import os\nDB = "x"', + id="no-quote", + ), + ], +) +def test_named_lines_must_contain_the_quote(tmp_path, root, quote, lines, refused, kept): + (root / "prompt.py").write_text( + 'import os\nDB = "x"\nPROMPT = """You are pair, a helpful assistant who books tables."""\n', + encoding="utf-8", + ) + destination = tmp_path / "out" + fields = { + "hard_constraints": ["Be polite."], + "system_prompt_from": [{"path": "prompt.py", "start_line": lines[0], "end_line": lines[1]}], + } + if quote: + fields["system_prompt_excerpt"] = quote + + result = accept_contract(payload(**fields), destination, source_root=root) + + if refused: + assert text_of(result) == REFUSED + not_the_quote(f"prompt.py {lines[0]}-{lines[1]}") + assert not (destination / "contract.json").exists() + else: + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == kept + + +def test_an_unreadable_prompt_file_is_refused_not_raised(tmp_path, root): + locked = root / "locked.py" + locked.write_text('PROMPT = "Be polite."\n', encoding="utf-8") + locked.chmod(0) + try: + if os.access(locked, os.R_OK): + pytest.skip("this user can read a file with no permissions") + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_from=[{"path": "locked.py", "start_line": 1, "end_line": 1}], + ), + tmp_path / "out", + source_root=root, + ) + finally: + locked.chmod(0o644) + + assert text_of(result) == ( + REFUSED + "system_prompt_from[0]: locked.py is not a file in the source" + ) + + +# Nudges come one at a time, and a wrong citation is refused once + + +def test_one_prompt_nudge_at_a_time(tmp_path, root): + destination = tmp_path / "out" + submit = submitter(destination, root) + + first = submit(payload()) + assert first.get("is_error") + assert "no hard_constraints and no system_prompt_excerpt" in text_of(first) + assert "name its lines in system_prompt_from" in text_of(first) + + second = submit(payload(system_prompt_excerpt="You take orders for the cafe.")) + + assert not second.get("is_error"), second + assert PROMPT_LINES not in text_of(second) + assert saved(destination)["system_prompt_excerpt"] == "You take orders for the cafe." + + +_LEFT_OUT_NEXT_TIME = "model: if the model is refused again, it is left out" + + +def test_a_wrong_citation_is_refused_once_then_dropped(tmp_path, root): + destination = tmp_path / "out" + submit = submitter(destination, root) + wrong = payload( + hard_constraints=["Be polite."], + model={"provider": "openai", "name": "gpt-5", "path": "agent.py", "line": 2}, + ) + + first = submit(wrong) + + assert text_of(first) == ( + REFUSED + not_named("gpt-5", 2, "agent.py") + "\n - " + _LEFT_OUT_NEXT_TIME + ) + assert not (destination / "contract.json").exists() + + second = submit(wrong) + + assert not second.get("is_error"), second + assert "model" not in saved(destination) + + +def test_accept_contract_refuses_a_wrong_citation_every_time(tmp_path, root): + wrong = payload(model={"provider": "openai", "name": "gpt-5", "path": "agent.py", "line": 2}) + + for _ in range(2): + result = accept_contract(wrong, tmp_path / "out", source_root=root) + assert text_of(result) == REFUSED + not_named("gpt-5", 2, "agent.py") + assert not (tmp_path / "out" / "contract.json").exists() + + +@pytest.mark.parametrize( + "written, name", + [ + pytest.param("model: gpt-4o-mini", "gpt-4o", id="longer-on-the-right"), + pytest.param("model: ollama:llama3", "llama3", id="longer-on-the-left"), + pytest.param("model: azure-gpt-4o", "gpt-4o", id="longer-on-the-left-after-a-dash"), + ], +) +def test_a_bare_value_in_a_configuration_file_must_be_whole(root, written, name): + # Written bare after a key, the value has no quotes to end it: the name's own edges do. + (root / "llm.yaml").write_text(f"provider: openai\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": "llm.yaml", "line": 2}) + + assert problems == [not_named(name, 2, "llm.yaml")] + + +# A configuration file's bare value is found in about one pass over the line + + +def test_a_crafted_configuration_line_does_not_hold_up_the_check(root): + (root / "crafted.yaml").write_bytes(b"a " * (MIB // 2)) + + problems, _, took = checked(root, {"name": "a", "path": "crafted.yaml", "line": 1}) + + assert problems == [not_named("a", 1, "crafted.yaml")] + assert took < 0.5 + + +# Names a setting goes by are not models, written in any case + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("llm.py", 'MODEL = os.getenv("openai_model")', "openai_model", id="lowercase-env"), + pytest.param("llm.py", 'MODEL = config["model_name"]', "model_name", id="subscript"), + pytest.param("llm.py", 'MODEL = cfg.get("model")', "model", id="generic-word"), + pytest.param("llm.json", '{"llm_model": "gpt-4o"}', "llm_model", id="json-key"), + pytest.param( + "llm.py", 'MODEL = os.getenv("OPENAI_MODEL_V2")', "OPENAI_MODEL_V2", id="capitals-with-a-digit" + ), + ], +) +def test_a_setting_name_is_not_a_model(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [ + f"model: {name} reads as the name of a setting, not a model; cite the line its value is " + "written on, or leave model out" + ] + + +def test_a_quoted_key_is_not_a_value(root): + (root / "llm.json").write_text('{"gpt-4o": true}\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.json", "line": 1}) + + assert problems == [not_named("gpt-4o", 1, "llm.json")] + + +# A value in a comment is not the model + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("llm.py", '# model = "gpt-3.5-turbo"', "gpt-3.5-turbo", id="commented-out"), + pytest.param("llm.yaml", "# model: gpt-3.5-turbo", "gpt-3.5-turbo", id="yaml-comment"), + pytest.param("llm.ts", '// const model = "gpt-3.5-turbo"', "gpt-3.5-turbo", id="slashes"), + pytest.param( + "llm.py", 'model = "gpt-4o" # was "gpt-4o-mini"', "gpt-4o-mini", id="trailing-comment" + ), + ], +) +def test_a_value_in_a_comment_is_not_the_model(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("Dockerfile", "ENV MODEL=gpt-4o", "gpt-4o", id="dockerfile-env"), + pytest.param("run.sh", "export MODEL=gpt-4o", "gpt-4o", id="shell-export"), + pytest.param("env.example", "MODEL=gpt-4o", "gpt-4o", id="env-example"), + pytest.param("models.yaml", " - gpt-4o", "gpt-4o", id="yaml-list-item"), + pytest.param("llm.py", 'model = "2024"', "2024", id="digits-only"), + pytest.param("llm.py", 'model = "gpt-4o" # the default', "gpt-4o", id="before-a-comment"), + pytest.param("llm.ts", 'const model = prod ? "gpt-4o" : "gpt-4o-mini"', "gpt-4o", id="ternary"), + pytest.param("run.sh", " --model=gpt-4o \\", "gpt-4o", id="shell-flag"), + pytest.param("llm.py", 'client(**{"model": "gpt-4o"})', "gpt-4o", id="splatted-dict"), + pytest.param("llm.py", ' *rest, model="gpt-4o")', "gpt-4o", id="star-args"), + ], +) +def test_more_honest_lines_are_found(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +# Model and deployment ids are not settings; settings in any spelling still are + + +@pytest.mark.parametrize( + "written, name", + [ + pytest.param('DEPLOYMENT = "GPT4"', "GPT4", id="azure-gpt4"), + pytest.param('DEPLOYMENT = "GPT4O"', "GPT4O", id="azure-gpt4o"), + pytest.param('DEPLOYMENT = "GPT35TURBO"', "GPT35TURBO", id="azure-gpt35turbo"), + pytest.param('MODEL = "O1"', "O1", id="capital-o1"), + pytest.param('MODEL = "mistral_large"', "mistral_large", id="mistral-large"), + pytest.param('MODEL = "command_r_plus"', "command_r_plus", id="command-r-plus"), + pytest.param('MODEL = "nova_pro"', "nova_pro", id="nova-pro"), + pytest.param('MODEL = "text_bison"', "text_bison", id="text-bison"), + pytest.param('DEPLOYMENT = "gpt4o-deployment"', "gpt4o-deployment", id="named-deployment"), + # The model is a first argument here, as for any call that isn't a lookup. + pytest.param('llm = genai.GenerativeModel("gemini-pro")', "gemini-pro", id="first-argument"), + pytest.param("llm = GenerativeModel('gemini-1.5-flash')", "gemini-1.5-flash", id="first-argument-single-quotes"), + ], +) +def test_real_model_ids_are_not_settings(root, written, name): + (root / "llm.py").write_text(f"# the model\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": "llm.py", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "written, name", + [ + pytest.param('MODEL = os.getenv("OPENAI_MODEL_V2")', "OPENAI_MODEL_V2", id="capitals"), + pytest.param('m = Field(default="gpt-4o", env="AZURE_OPENAI_DEPLOYMENT")', "AZURE_OPENAI_DEPLOYMENT", id="quoted-env-var-value"), + pytest.param('m = Field(env="AGENT_BRAIN")', "AGENT_BRAIN", id="capitals-without-a-setting-word"), + pytest.param('VAR = "llm_model"', "llm_model", id="snake-case"), + pytest.param('VAR = "openai_model_v2"', "openai_model_v2", id="snake-case-with-a-digit"), + pytest.param('VAR = "api_key"', "api_key", id="a-key"), + pytest.param('VAR = "agent_name"', "agent_name", id="a-name"), + pytest.param('VAR = "use_default"', "use_default", id="a-default"), + pytest.param('VAR = "search_engine"', "search_engine", id="an-engine"), + pytest.param('VAR = "cloud_provider"', "cloud_provider", id="a-provider"), + pytest.param('VAR = "prod_env"', "prod_env", id="an-env"), + pytest.param('VAR = "deployment_id"', "deployment_id", id="an-id"), + pytest.param('VAR = "openai.chat"', "openai.chat", id="dotted-without-a-setting-word"), + pytest.param('VAR = "model-name"', "model-name", id="kebab-case"), + pytest.param('model = this.config.get("openai.model");', "openai.model", id="dotted-get-generic"), + pytest.param('model = configService.getOrThrow("llm.model");', "llm.model", id="dotted-get-or-throw"), + pytest.param("$model = config('services.openai.model');", "services.openai.model", id="dotted-config"), + pytest.param('MODEL = env.str("llm.model")', "llm.model", id="dotted-env-str"), + pytest.param('model = props.getProperty("llm.model");', "llm.model", id="dotted-get-property"), + pytest.param('MODEL = os.environ.get ("llm.model")', "llm.model", id="dotted-space-before-the-bracket"), + ], +) +def test_settings_in_any_spelling_are_not_models(root, written, name): + (root / "llm.py").write_text(f"# the model\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": "llm.py", "line": 2}) + + assert problems == [ + f"model: {name} reads as the name of a setting, not a model; cite the line its value is " + "written on, or leave model out" + ] + + +# Comments are found outside strings only, with or without spaces + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("llm.py", '{"a": "Agent #1", "model": "gpt-4o"}', "gpt-4o", id="hash-in-a-string"), + pytest.param("llm.js", 'url = "http://x //y"; m = "gpt-4o"', "gpt-4o", id="slashes-in-a-string"), + pytest.param("llm.js", 'note = "a /* b"; m = "gpt-4o"', "gpt-4o", id="block-start-in-a-string"), + pytest.param("llm.py", 'm = "it\\"s #1"; model = "gpt-4o"', "gpt-4o", id="escaped-quote-in-a-string"), + pytest.param("run.sh", 'python run.py --model "gpt-4o"', "gpt-4o", id="a-flag-is-not-sql"), + pytest.param("run.sh", 'echo $#; MODEL="gpt-4o"', "gpt-4o", id="shell-count"), + pytest.param("index.html", 'Tom & Jerry ', "gpt-4o", id="html-character"), + ], +) +def test_comment_markers_inside_strings_do_not_hide_the_model(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("llm.js", 'x = 1;// "gpt-4o"', "gpt-4o", id="slashes-without-a-space"), + pytest.param("llm.py", 'x=1#"gpt-4o"', "gpt-4o", id="hash-without-a-space"), + pytest.param("llm.sql", "select 1 -- 'gpt-4o'", "gpt-4o", id="sql"), + pytest.param("llm.sql", "-- model 'gpt-4o'", "gpt-4o", id="sql-line"), + pytest.param("llm.sql", "select 1--'gpt-4o'", "gpt-4o", id="sql-without-spaces"), + pytest.param("init.lua", "x = a--'gpt-4o'", "gpt-4o", id="lua"), + pytest.param("llm.ts", ' * model: "gpt-4o"', "gpt-4o", id="block-comment-middle"), + pytest.param("llm.ts", 'x = 1; /* "gpt-4o" */', "gpt-4o", id="block-comment-after-code"), + pytest.param("index.html", '', "gpt-4o", id="markup"), + ], +) +def test_comments_without_spaces_are_comments(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +# Prose names models, it doesn't set them + + +@pytest.mark.parametrize( + "file, written", + [ + pytest.param("README.md", "This agent uses `gpt-4o` (we tried `claude-3-opus`)", id="readme"), + pytest.param("README.md", "| fallback | `claude-3-opus` |", id="table-cell"), + pytest.param("README.md", 'This agent uses "gpt-4o" (we tried "claude-3-opus")', id="straight-quotes"), + pytest.param("README.md", '* default model: "claude-3-opus"', id="list-item"), + pytest.param("notes.txt", "tried `claude-3-opus`", id="text"), + pytest.param("index.rst", "uses `claude-3-opus` here", id="rst"), + pytest.param("guide.markdown", 'model: "claude-3-opus"', id="markdown"), + pytest.param("guide.adoc", 'model: "claude-3-opus"', id="asciidoc"), + pytest.param("page.mdx", 'model: "claude-3-opus"', id="mdx"), + pytest.param("README", 'model: "claude-3-opus"', id="readme-without-a-suffix"), + ], +) +def test_prose_files_hold_no_model(root, file, written): + (root / file).write_text(f"# notes\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "claude-3-opus", "path": file, "line": 2}) + + assert problems == [prose(file)] + + +def test_backticks_in_code_are_values(root): + (root / "llm.ts").write_text("// the model\nconst model = `claude-3-opus`;\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "claude-3-opus", "path": "llm.ts", "line": 2}) + + assert problems == [] + + +@pytest.mark.parametrize( + "file, written", + [ + pytest.param("Makefile", "MODEL ?= gpt-4o", id="makefile"), + pytest.param("rules.mk", "MODEL := gpt-4o", id="make-include"), + pytest.param(".flaskenv", "MODEL=gpt-4o", id="flaskenv"), + pytest.param("run.sh", 'python run.py --model "${MODEL:-gpt-4o}"', id="shell-default"), + pytest.param(".env", "MODEL=${MODEL:-gpt-4o}", id="env-default"), + ], +) +def test_more_configuration_forms(root, file, written): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": "gpt-4o", "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == "gpt-4o" + + +def test_a_shell_default_is_only_found_in_configuration(root): + (root / "llm.py").write_text('# the model\nx = ${MODEL:-gpt-4o}\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "llm.py")] + + +# The reader never waits on a pipe, and leaves a trace when it can't read + + +def test_a_pipe_is_not_waited_on(tmp_path): + from fi.alk.harness.model_citation import read_source + + if not hasattr(os, "mkfifo"): + pytest.skip("no named pipes here") + pipe = tmp_path / "pipe" + os.mkfifo(pipe) + answer: list[bytes | None] = [] + reading = threading.Thread(target=lambda: answer.append(read_source(pipe, 10)), daemon=True) + reading.start() + reading.join(timeout=2) + + assert answer == [None] + + +def test_a_file_read_in_short_pieces_is_read_whole(tmp_path, monkeypatch): + from fi.alk.harness import model_citation + + (tmp_path / "agent.py").write_bytes(b"x" * 100) + real_read = os.read + monkeypatch.setattr(model_citation.os, "read", lambda fd, n: real_read(fd, min(n, 7))) + + assert model_citation.read_source(tmp_path / "agent.py", 50) == b"x" * 51 + assert model_citation.read_source(tmp_path / "agent.py", 200) == b"x" * 100 + + +def test_an_unreadable_file_is_logged_without_its_content(tmp_path, caplog): + from fi.alk.harness.model_citation import read_source + + locked = tmp_path / "locked.py" + locked.write_text('SECRET = "do-not-log"\n', encoding="utf-8") + locked.chmod(0) + try: + if os.access(locked, os.R_OK): + pytest.skip("this user can read a file with no permissions") + with caplog.at_level("INFO", logger="fi.alk.harness.model_citation"): + assert read_source(locked, 100) is None + finally: + locked.chmod(0o600) + + assert [record.getMessage() for record in caplog.records] == [ + "source file not read (PermissionError)" + ] + assert str(locked) not in caplog.text + assert "do-not-log" not in caplog.text + + +def test_a_link_put_in_place_of_the_file_is_not_followed(tmp_path): + from fi.alk.harness.model_citation import read_source + + (tmp_path / "outside.py").write_text("SECRET\n", encoding="utf-8") + link("outside.py", tmp_path / "agent.py") + + assert read_source(tmp_path / "agent.py", 100) is None + + +def test_mismatched_quotes_are_not_a_value(root): + (root / "llm.py").write_text("# settings\nmodel = \"gpt-4o'\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "llm.py")] + + +# The quote is compared by its letters, so honest ways of writing a prompt fit + + +@pytest.mark.parametrize( + "file, written", + [ + pytest.param( + "prompt.py", + 'PROMPT = ("You are pair, "\n "a helpful assistant who books tables.")', + id="string-pieces", + ), + pytest.param( + "prompt.py", + 'PROMPT = "You are pair,\\na helpful assistant who books tables."', + id="escaped-newline", + ), + pytest.param( + "prompt.json", + '{"prompt": "You are pair,\\na helpful assistant who books tables."}', + id="json", + ), + ], +) +def test_honest_prompt_forms_hold_the_quote(tmp_path, root, file, written): + (root / file).write_text(written + "\n", encoding="utf-8") + count = len(written.splitlines()) + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant who books tables.", + system_prompt_from=[{"path": file, "start_line": 1, "end_line": count}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written + + +_LONG_PROMPT = "You are pair, a helpful assistant who books tables for guests. " * 4 + + +@pytest.mark.parametrize( + "quote, refused", + [ + # A quote that says where it trails off fits. + pytest.param(_LONG_PROMPT[:120] + " ...", False, id="trails-off-with-dots"), + # The start matches and the rest differs: every part of the quote must be in the lines. + pytest.param(_LONG_PROMPT[:120] + " and something else entirely", True, id="differs-late"), + # The quote differs from the lines near its start. + pytest.param("You are pair, a helpful assistant who books rooms for guests.", True, id="differs-early"), + # The lines hold the quote's start but are shorter than the quote. + pytest.param(_LONG_PROMPT * 3, True, id="lines-shorter-than-the-quote"), + ], +) +def test_the_quote_window(tmp_path, root, quote, refused): + (root / "prompt.py").write_text(f'PROMPT = """{_LONG_PROMPT}"""\n', encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + if refused: + assert text_of(result) == REFUSED + not_the_quote("prompt.py 1-1") + else: + assert not result.get("is_error"), result + + +def test_lines_refused_twice_leave_the_quote(tmp_path, root): + (root / "prompt.py").write_text('import os\nDB = "x"\n', encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + wrong = payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant.", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 2}], + ) + + first = submit(wrong) + + assert text_of(first) == ( + REFUSED + + not_the_quote("prompt.py 1-2") + + "\n - system_prompt_from: lines named later are checked against this quote until you " + "give another; a later mismatch records the lines as unconfirmed and keeps the quote, or the " + "lines if the quote has fewer than 16 letters" + ) + second = submit(wrong) + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "You are pair, a helpful assistant." + assert saved(destination)["system_prompt_from"] == [] + assert saved(destination)["unconfirmed_prompt_from"] == wrong["system_prompt_from"] + + +# Every piece of the quote must be in the lines named, in order + +_BOOKING = ( + "import os\n" + "import json\n" + "from acme.tools import book\n" + "from acme.tools import cancel\n" + "\n" + "PROMPT = (\n" + ' "You are Acme\'s booking agent. "\n' + ' "Help guests book tables. "\n' + ' "Never share other guests\' bookings. "\n' + ' "Confirm every booking before saving it. "\n' + ' "Be warm and brief."\n' + ")\n" +) +_BOOKING_QUOTE = ( + "You are Acme's booking agent. Help guests book tables. Never share other guests' bookings. " + "Confirm every booking before saving it. Be warm and brief." +) + + +@pytest.mark.parametrize( + "quote, lines, refused", + [ + # Starting early pads the range, but its last line is still missing. + pytest.param(_BOOKING_QUOTE, (1, 10), True, id="starts-early-stops-short"), + pytest.param(_BOOKING_QUOTE, (6, 12), False, id="the-prompt"), + pytest.param("You are Acme's booking agent. ... Be warm and brief.", (6, 12), False, id="elided"), + pytest.param("You are Acme's booking agent. ... Be warm and brief.", (1, 10), True, id="elided-stops-short"), + pytest.param("Be warm and brief. ... You are Acme's booking agent.", (6, 12), True, id="out-of-order"), + pytest.param("You are Acme's booking agent. [...] Be warm and brief.", (6, 12), False, id="bracketed-dots"), + pytest.param("You are Acme's booking agent. … Be warm and brief.", (6, 12), False, id="ellipsis-character"), + pytest.param("YOU ARE ACME'S BOOKING AGENT. ... BE WARM AND BRIEF.", (6, 12), False, id="case"), + pytest.param("... Confirm every booking before saving it. Be warm and brief.", (6, 12), False, id="starts-with-dots"), + pytest.param("...", (1, 4), False, id="only-dots"), + # One line can't hold a piece the quote has twice. + pytest.param("Be warm and brief. ... Be warm and brief.", (6, 12), True, id="a-piece-twice"), + ], +) +def test_a_range_that_stops_short_is_refused(tmp_path, root, quote, lines, refused): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "agent.py", "start_line": lines[0], "end_line": lines[1]}], + ), + destination, + source_root=root, + ) + + if refused: + assert text_of(result) == REFUSED + not_the_quote(f"agent.py {lines[0]}-{lines[1]}") + else: + assert not result.get("is_error"), result + copied = "\n".join(_BOOKING.splitlines()[lines[0] - 1 : lines[1]]) + assert saved(destination)["system_prompt_excerpt"] == copied + + +@pytest.mark.parametrize( + "written, quote", + [ + pytest.param('PROMPT = "Book at Tom & Jerry\'s diner tonight"', "Book at Tom & Jerry's diner tonight", id="entity-in-the-source"), + pytest.param('PROMPT = "Book at Tom & Jerry\'s diner tonight"', "Book at Tom & Jerry's diner tonight", id="entity-in-the-quote"), + pytest.param('PROMPT = "Order a caf\u00e9 and wait for the bill"', "Order a cafe\u0301 and wait for the bill", id="decomposed-accent"), + ], +) +def test_an_honest_quote_in_another_form_fits(tmp_path, root, written, quote): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written + + +def test_lines_sent_without_the_refused_quote_are_still_checked(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=imports, + ) + ) + assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 1-4")) + + # The quote left out: the imports are still checked against it, asked about once more... + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) + assert not (destination / "contract.json").exists() + + # ... and then the quote is what's sent, with the lines on record as unconfirmed. + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["system_prompt_from"] == [] + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +def test_the_right_lines_after_a_refusal_are_copied(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + ) + assert first.get("is_error"), first + + second = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}], + ) + ) + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + + +def test_a_new_quote_replaces_the_refused_one(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant.", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + ) + assert first.get("is_error"), first + + second = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}], + ) + ) + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + + +def test_a_quote_of_many_pieces_does_not_hold_up_the_check(root): + from fi.alk.harness.model_citation import quote_problem + + copied = "a" * MIB + started = time.perf_counter() + answer = quote_problem("a ... " * 50_000 + "b", copied, "agent.py 1-1") + took = time.perf_counter() - started + + assert answer is not None + assert took < 0.5 + + +# A prompt written with Python string prefixes holds its quote + + +@pytest.mark.parametrize( + "written", + [ + pytest.param('PROMPT = ("You are pair, " f"a helpful assistant who books tables.")', id="f-string-piece"), + pytest.param('PROMPT = ("You are pair, " r"a helpful assistant who books tables.")', id="raw-piece"), + pytest.param("PROMPT = ('You are pair, ' rb'a helpful assistant who books tables.')", id="two-letter-prefix"), + pytest.param('PROMPT = ("You are pair, " F"a helpful assistant who books tables.")', id="capital-prefix"), + pytest.param('PROMPT = f"You are pair, " "a helpful assistant who books tables."', id="prefix-on-the-first-piece"), + pytest.param('PROMPT = (u"You are pair, " fr"a helpful assistant who books tables.")', id="u-and-fr"), + ], +) +def test_a_prompt_with_string_prefixes_holds_the_quote(tmp_path, root, written): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant who books tables.", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written + + +@pytest.mark.parametrize( + "written, quote, refused", + [ + # A word ending in a prefix letter keeps that letter, on both sides. + pytest.param('PROMPT = "He said" " hello and goodbye."', 'He said hello and goodbye.', False, id="word-before-a-quote"), + pytest.param('PROMPT = "Pick one of" "f the options."', "Pick one off the options.", False, id="prefix-letter-inside-a-string"), + ], +) +def test_a_word_before_a_quote_is_not_a_prefix(tmp_path, root, written, quote, refused): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + if refused: + assert text_of(result) == REFUSED + not_the_quote("prompt.py 1-1") + else: + assert not result.get("is_error"), result + + +# A value inside a docstring or a multi-line string is not the model + +_DOCSTRING = ( + "def build():\n" + ' """Build the agent.\n' + "\n" + ' Example: model = "gpt-4o"\n' + ' """\n' + ' return Agent(model="claude-3-opus")\n' +) + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("llm.py", _DOCSTRING, 4, "gpt-4o", id="docstring-line"), + pytest.param("llm.py", "X = '''\nmodel = \"gpt-4o\"\n'''\n", 2, "gpt-4o", id="single-quote-block"), + pytest.param("llm.py", 'PROMPT = """You are pair.\nUse "gpt-4o" for hard cases.\n"""\n', 2, "gpt-4o", id="prompt-line"), + pytest.param("llm.py", 'PROMPT = """Use "gpt-4o" for hard cases."""\n', 1, "gpt-4o", id="opened-on-the-line"), + pytest.param("llm.toml", 'notes = """\nmodel = "gpt-4o"\n"""\n', 2, "gpt-4o", id="toml-multiline"), + pytest.param("llm.py", '"""Docs with \'\'\' inside.\nmodel = "gpt-4o"\n"""\n', 2, "gpt-4o", id="other-kind-does-not-close"), + pytest.param("llm.py", 'X = """docs \'\'\' and "gpt-4o"\n"""\n', 1, "gpt-4o", id="other-kind-on-the-line-does-not-close"), + pytest.param("llm.py", 'X = """gpt-4o" is the default"""\n', 1, "gpt-4o", id="a-block-that-opens-at-the-name"), + ], +) +def test_a_value_inside_a_triple_quoted_block_is_not_the_model(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [not_named(name, line, file)] + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("llm.py", _DOCSTRING, 6, "claude-3-opus", id="closed-before"), + pytest.param("llm.py", 'DOC = """notes"""; MODEL = "gpt-4o"\n', 1, "gpt-4o", id="closed-on-the-line"), + pytest.param("llm.py", 'MODEL = """gpt-4o"""\n', 1, "gpt-4o", id="triple-quoted-value"), + pytest.param("llm.py", "MODEL = '''gpt-4o'''\n", 1, "gpt-4o", id="triple-single-quoted-value"), + pytest.param("llm.py", 'X = """a\nb"""\nMODEL = "gpt-4o"\n', 3, "gpt-4o", id="block-closed-on-an-earlier-line"), + ], +) +def test_a_value_after_a_closed_block_is_the_model(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "written", + [ + pytest.param('"""' * (MIB // 3 - 20) + '\nMODEL = "gpt-4o"\n', id="a-file-of-triple-quotes"), + pytest.param('"""' + '""" "gpt-4o" """' * (MIB // 20), id="a-line-of-blocks"), + ], +) +def test_a_crafted_file_of_triple_quotes_does_not_hold_up_the_check(root, written): + (root / "crafted.py").write_text(written, encoding="utf-8") + line = written.count("\n") + (0 if written.endswith("\n") else 1) + + _, _, took = checked(root, {"name": "gpt-4o", "path": "crafted.py", "line": line}) + + assert took < 0.5 + + +def test_leaving_the_quote_out_is_not_asked_about(tmp_path, root): + # Following the advice to leave the quote out must not set off the "no prompt" question. + (root / "prompt.py").write_text('PROMPT = "You are pair."\n', encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + result = submit( + payload(system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}]) + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == 'PROMPT = "You are pair."' + + +def test_a_prompt_path_through_a_link_outside_is_refused(tmp_path, root): + (tmp_path / "outside.py").write_text('PROMPT = "elsewhere"\n', encoding="utf-8") + link("../outside.py", root / "inside.py") + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_from=[{"path": "inside.py", "start_line": 1, "end_line": 1}], + ), + tmp_path / "out", + source_root=root, + ) + + assert text_of(result) == ( + REFUSED + "system_prompt_from[0]: inside.py is not a file in the source" + ) + + +# What is looked up is not the model, however it is spelled + + +@pytest.mark.parametrize( + "written, name", + [ + # Keys shaped like model names, so the setting-shape rule passes them and the lookup is + # what refuses them. + pytest.param('MODEL = os.getenv("fast-4")', "fast-4", id="getenv-hyphenated"), + pytest.param('MODEL = cfg.get("chat.v2")', "chat.v2", id="get-dotted"), + pytest.param('MODEL = os.getenv("gpt_4o")', "gpt_4o", id="getenv-with-a-digit"), + pytest.param('MODEL = os.environ["gpt-4o"]', "gpt-4o", id="subscript"), + pytest.param('model = this.config.get("gpt-4o");', "gpt-4o", id="get-generic"), + pytest.param('model = configService.getOrThrow("gpt-4o");', "gpt-4o", id="get-or-throw"), + pytest.param('model = props.getProperty("gpt-4o");', "gpt-4o", id="get-property"), + pytest.param('MODEL = os.environ.get ("gpt-4o")', "gpt-4o", id="space-before-the-bracket"), + pytest.param('MODEL = env.str("gpt-4o")', "gpt-4o", id="env-str"), + pytest.param("$model = config('gpt-4o');", "gpt-4o", id="config-call"), + pytest.param('model = env.getOrDefault("gpt-4o", x);', "gpt-4o", id="get-or-default"), + pytest.param('model = bundle.getString("gpt-4o");', "gpt-4o", id="get-string"), + pytest.param('model = config.getValue("gpt-4o");', "gpt-4o", id="get-value"), + pytest.param("$model = env('gpt-4o');", "gpt-4o", id="env-call"), + ], +) +def test_a_lookup_key_is_not_the_model(root, written, name): + (root / "llm.py").write_text(f"import os\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": "llm.py", "line": 2}) + + assert problems == [not_named(name, 2, "llm.py")] + + +@pytest.mark.parametrize( + "written, name", + [ + pytest.param('MODELS = ["gpt-4o", "gpt-4o-mini"]', "gpt-4o", id="list-literal"), + pytest.param('model = genai.GenerativeModel("gemini-1.5-pro")', "gemini-1.5-pro", id="constructor"), + pytest.param('MODEL = os.getenv("OPENAI_MODEL", "gpt-4o")', "gpt-4o", id="getenv-default"), + pytest.param('llm = target("gpt-4o")', "gpt-4o", id="a-call-ending-in-get"), + pytest.param('llm = my_get("gpt-4o")', "gpt-4o", id="a-call-ending-in-_get"), + pytest.param('["gpt-4o", "gpt-4o-mini"]', "gpt-4o", id="a-list-at-the-line-start"), + ], +) +def test_a_value_passed_to_a_call_is_the_model(root, written, name): + (root / "llm.py").write_text(f"import os\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": "llm.py", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("llm.ts", 'const model = "gpt-4o"; /* was "gpt-4o-mini" */', "gpt-4o-mini", id="block-comment"), + pytest.param("llm.html", '', "gpt-4o-mini", id="markup-comment"), + pytest.param("llm.html", 'x ', "gpt-4o-mini", id="trailing-markup-comment"), + ], +) +def test_a_value_in_a_block_comment_is_not_the_model(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +def test_a_prompt_with_escaped_characters_holds_the_quote(tmp_path, root): + # json.dump writes every non-ASCII character as an escape; the quote has the character. + written = '{"prompt": "You are pair \\u2014 a helpful assistant who books tables."}' + (root / "prompt.json").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair — a helpful assistant who books tables.", + system_prompt_from=[{"path": "prompt.json", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written + + +# Code between the string pieces of a prompt is passed over, a little at a time + +_LONG_OPENING = ( + "You are Acme's booking agent. Help guests book tables, change bookings and cancel them. " +) + + +@pytest.mark.parametrize( + "written, quote", + [ + pytest.param( + f'PROMPT = (\n "{_LONG_OPENING}"\n f"Today is {{today}}. "\n "Be warm and brief."\n)', + _LONG_OPENING + "Today is {today}. Be warm and brief.", + id="f-string-piece-late", + ), + pytest.param( + f'PROMPT = (\n "{_LONG_OPENING}" # the rules\n "Be warm and brief."\n)', + _LONG_OPENING + "Be warm and brief.", + id="comment-between-pieces", + ), + pytest.param( + f'PROMPT = (\n "{_LONG_OPENING}"\n r"Never share \\d+ ids. "\n "Be warm and brief."\n)', + _LONG_OPENING + "Never share \\d+ ids. Be warm and brief.", + id="raw-piece-late", + ), + pytest.param( + f'PROMPT = "{_LONG_OPENING}" + NAME + " Be warm and brief."', + _LONG_OPENING + "Be warm and brief.", + id="a-variable-left-out", + ), + pytest.param( + f'PROMPT = "{_LONG_OPENING}Be warm and brief."', + _LONG_OPENING + "Be warm and brief.", + id="markup-left-out", + ), + pytest.param( + f'PROMPT = "{_LONG_OPENING}{{{{ guest_name }}}}, be warm and brief."', + _LONG_OPENING + "Be warm and brief.", + id="a-placeholder-left-out", + ), + ], +) +def test_code_between_string_pieces_is_passed_over(tmp_path, root, written, quote): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + count = len(written.splitlines()) + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": count}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written + + +def test_a_long_gap_between_words_is_not_passed_over(tmp_path, root): + written = ( + f'PROMPT = (\n "{_LONG_OPENING}" # twenty-five letters of a comment here\n' + ' "Be warm and brief."\n)' + ) + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_LONG_OPENING + "Be warm and brief.", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 4}], + ), + tmp_path / "out", + source_root=root, + ) + + assert text_of(result) == REFUSED + not_the_quote("prompt.py 1-4") + + +def test_a_quote_that_passed_replaces_the_held_one(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + prompt = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant.", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + ) + assert first.get("is_error"), first + second = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=prompt, + ) + ) + assert not second.get("is_error"), second + + # The lines again, without a quote: checked against the quote that passed, not the old one. + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=prompt)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + assert saved(destination)["system_prompt_from"] == prompt + + +@pytest.mark.parametrize("placeholder", ["...", "…", "[...]", "-", "—", "?"]) +def test_a_quote_without_words_does_not_skip_the_held_one(tmp_path, root, placeholder): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=imports, + ) + ) + assert first.get("is_error"), first + + again = payload( + hard_constraints=["Be polite."], system_prompt_excerpt=placeholder, system_prompt_from=imports + ) + second = submit(again) + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) + + third = submit(again) + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["system_prompt_from"] == [] + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +@pytest.mark.parametrize( + "written", + [ + pytest.param('PROMPT = "Order a caf\\u00e9 here, then wait for the bill."', id="short-unicode-escape"), + pytest.param('PROMPT = "Order a caf\\xe9 here, then wait for the bill."', id="hex-escape"), + pytest.param('PROMPT = "Order a caf\\U000000e9 here, then wait for the bill."', id="long-unicode-escape"), + pytest.param('PROMPT = "Order a caf\\N{LATIN SMALL LETTER E WITH ACUTE} here, then wait for the bill."', id="named-escape"), + pytest.param('PROMPT = "Order a caf\\351 here, then wait for the bill."', id="octal-escape"), + ], +) +def test_more_escapes_hold_the_quote(tmp_path, root, written): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="Order a café here, then wait for the bill.", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + + +def test_a_quote_that_grows_when_normalised_costs_little(root): + from fi.alk.harness.model_citation import quote_problem + + import tracemalloc + + # U+FDFA grows eighteen times under NFKC; NFC leaves it as it is. + copied = "\ufdfa" * (MIB // 3) + tracemalloc.start() + try: + started = time.perf_counter() + quote_problem("\ufdfa" * 1000 + " b", copied, "agent.py 1-1") + took = time.perf_counter() - started + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert took < 0.5 + assert peak < 32 * MIB + + +# An apostrophe in bare text doesn't hide the comment after it + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("cfg.yaml", "description: Pair's booking agent # model: gpt-4o-mini", "gpt-4o-mini", id="yaml"), + pytest.param(".env", "AGENT_NAME=Bob's agent # MODEL=gpt-4o-mini", "gpt-4o-mini", id="env"), + pytest.param("llm.sh", "echo it\\'s # 'gpt-4o-mini'", "gpt-4o-mini", id="shell-escaped"), + pytest.param("lib.rs", "fn f<'a>(x: &'a str) -> &'a str { x } // \"gpt-4o-mini\"", "gpt-4o-mini", id="rust-lifetimes"), + pytest.param("cfg.yaml", "note: Pair's agent # it's \"gpt-4o-mini\"", "gpt-4o-mini", id="apostrophes-around-the-marker"), + pytest.param("llm.ini", "; model = gpt-4o-mini", "gpt-4o-mini", id="ini-comment"), + ], +) +def test_an_apostrophe_does_not_hide_a_comment(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("run.sh", "curl -s http://localhost:8000/v1/chat/completions -d '{\"model\": \"gpt-4o\"}'", "gpt-4o", id="curl"), + pytest.param("run.sh", "python app.py --base-url=http://localhost:4000 --model=gpt-4o", "gpt-4o", id="url-flag"), + pytest.param("Dockerfile", "CMD python app.py --base-url=http://ollama:11434 --model=llama3", "llama3", id="dockerfile-cmd"), + pytest.param("cfg.yaml", "llm: {base_url: http://localhost:11434, model: llama3}", "llama3", id="yaml-flow-map"), + pytest.param("run.sh", 'exec uv run -- python -m app --model "gpt-4o"', "gpt-4o", id="uv-run"), + pytest.param("Dockerfile", "CMD tini -- python app.py --model=gpt-4o", "gpt-4o", id="tini"), + pytest.param("run.sh", "for f in data/*; do MODEL=gpt-4o; done", "gpt-4o", id="glob"), + pytest.param("run.sh", "[ ${#MODELS[@]} -eq 0 ] && MODEL=gpt-4o", "gpt-4o", id="shell-length"), + pytest.param("llm.py", "n = len(x)//2; model = 'gpt-4o'", "gpt-4o", id="floor-division"), + pytest.param("llm.ts", ' */ const model = "gpt-4o";', "gpt-4o", id="after-a-block-comment-closes"), + ], +) +def test_urls_globs_and_dashes_are_not_comments(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "written, name", + [ + pytest.param('DEPLOYMENT = "gpt-4o_deployment"', "gpt-4o_deployment", id="underscore-deployment"), + pytest.param('DEPLOYMENT = "chat-deployment"', "chat-deployment", id="deployment-without-a-digit"), + pytest.param('MODEL = "azure/gpt4o_deployment"', "azure/gpt4o_deployment", id="provider-prefixed-deployment"), + # A setting's word with a version and no underscore: a deployment's own name. + pytest.param('DEPLOYMENT = "gpt4o-default"', "gpt4o-default", id="versioned-with-a-setting-word"), + ], +) +def test_deployment_ids_are_models(root, written, name): + (root / "llm.py").write_text(f"# the model\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": "llm.py", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +def test_a_crafted_line_of_generic_calls_does_not_hold_up_the_check(root): + # Each occurrence looks back for a generic's < only a few characters, not to the first get<. + (root / "crafted.ts").write_text('get("a") ' + '>("a") ' * 100_000, encoding="utf-8") + + _, _, took = checked(root, {"name": "a", "path": "crafted.ts", "line": 1}) + + assert took < 0.5 + + +def test_a_crafted_line_of_apostrophes_does_not_hold_up_the_check(root): + # Once one apostrophe finds no partner, none after it is paired again. + # Small enough that pairing each apostrophe again would show as seconds, not hang. + (root / "crafted.py").write_text("'" + "\\'" * 40_000 + "\n", encoding="utf-8") + + problems, _, took = checked(root, {"name": "a", "path": "crafted.py", "line": 1}) + + assert problems == [not_named("a", 1, "crafted.py")] + assert took < 0.5 + + +# Lines sent alone after a refusal are asked about once more before either side is given up + + +def test_right_lines_sent_alone_after_a_refused_paraphrase_are_kept(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + prompt = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + paraphrase = "Acme booking agent: helps guests reserve tables and never shares bookings." + + first = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=paraphrase, system_prompt_from=prompt) + ) + assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 6-12")) + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=prompt)) + assert text_of(second) == REFUSED + lines_alone("agent.py 6-12", paraphrase) + + # Quoted word for word from the lines, as asked: the lines are copied, nothing unconfirmed. + third = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, system_prompt_from=prompt) + ) + + assert not third.get("is_error"), third + contract = saved(destination) + assert contract["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + assert contract["system_prompt_from"] == prompt + assert "unconfirmed_prompt_from" not in contract + + +def test_unconfirmed_lines_reach_the_document(tmp_path, root): + from fi.alk.providers.agent import build_authored_agent + + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + quoted = payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, system_prompt_from=imports) + + assert submit(quoted).get("is_error") + assert submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)).get("is_error") + assert not submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)).get("is_error") + + contract = AgentContract.model_validate(saved(destination)).model_dump(mode="json") + document = build_authored_agent(contract) + assert document["behaviour"][0]["instructions"] == _BOOKING_QUOTE + assert document["extension"]["contract"]["unconfirmed_prompt_from"] == imports + + +def test_only_the_save_records_unconfirmed_lines(tmp_path, root): + destination = tmp_path / "out" + claimed = [{"path": "agent.py", "start_line": 1, "end_line": 1}] + + result = accept_contract( + payload(hard_constraints=["Be polite."], unconfirmed_prompt_from=claimed), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert "unconfirmed_prompt_from" not in saved(destination) + + +def test_a_quote_whose_start_matches_everywhere_does_not_hold_up_the_check(root): + from fi.alk.harness.model_citation import quote_problem + + # Every start matches a thousand words before failing: the attempts are counted and stop. + started = time.perf_counter() + answer = quote_problem("a " * 1000 + "b", "a" * MIB, "agent.py 1-1") + took = time.perf_counter() - started + + assert answer is not None + assert took < 0.5 + + +def test_a_first_word_found_early_is_tried_again(tmp_path, root): + # "you" first appears in the comment, too far from "are"; the prompt's own "You are" follows. + written = "# you should keep this comment long enough to break the match\n" + _BOOKING + (root / "agent.py").write_text(written, encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 13}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == "\n".join(written.splitlines()) + + +def test_wrong_lines_alone_after_a_passed_quote_are_asked_about_then_left_out(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}], + ) + ) + assert not first.get("is_error"), first + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) + + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +# A quote is held from the moment it is sent, whatever sends the call back + + +@pytest.mark.parametrize( + "first", + [ + pytest.param({}, id="asked-for-lines"), + pytest.param({"modality": "telepathy"}, id="refused-before-the-checks"), + ], +) +def test_a_quote_sent_without_lines_is_held(tmp_path, root, first): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + sent_back = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, **first) + ) + assert sent_back.get("is_error"), sent_back + + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +def test_the_quote_shown_with_lines_sent_alone_is_on_one_line(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + quote = "You are Acme's booking agent.\n\nHelp guests book tables." + + submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=quote, system_prompt_from=imports)) + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + assert "(You are Acme's booking agent. Help guests book tables.)" in text_of(second) + + +@pytest.mark.parametrize("placeholder", ["n/a", "N/A", "TBD", "none", "see lines", "as above"]) +def test_a_quote_of_a_few_letters_says_nothing(tmp_path, root, placeholder): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + assert submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, system_prompt_from=imports) + ).get("is_error") + again = payload(hard_constraints=["Be polite."], system_prompt_excerpt=placeholder, system_prompt_from=imports) + + assert text_of(submit(again)) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) + assert not submit(again).get("is_error") + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +# The gap between two words of a quote is 24 letters, no more + + +@pytest.mark.parametrize("letters, refused", [(24, False), (25, True)]) +def test_the_gap_between_words_is_24_letters(letters, refused): + from fi.alk.harness.model_citation import quote_problem + + copied = "Confirm every booking " + "x" * letters + " before saving it." + + answer = quote_problem("Confirm every booking before saving it.", copied, "agent.py 1-1") + + assert (answer is not None) is refused + + +# Each search is charged what it reads + + +def test_an_elided_quote_of_a_long_prompt_holds(tmp_path, root): + from fi.alk.harness.model_citation import quote_problem + + filler = "A guest asks a question and a host answers it at a table. " * 3_000 + copied = f'PROMPT = """You are Acme\'s booking agent. {filler}A short answer is best."""' + + answer = quote_problem( + "You are Acme's booking agent. ... A short answer is best.", copied, "agent.py 1-1" + ) + + assert answer is None + + +def test_a_crafted_long_word_costs_little(root): + from fi.alk.harness.model_citation import quote_problem + + started = time.perf_counter() + answer = quote_problem("a " + "a" * MIB + "b", "a" * MIB, "agent.py 1-1") + took = time.perf_counter() - started + + assert answer is not None + assert took < 0.25 + + +# Scripts written without spaces: each character is a word + + +@pytest.mark.parametrize( + "written, quote", + [ + pytest.param('PROMPT = ("你是一个乐于助人的订餐" f"助手,今天是{today}。")', "你是一个乐于助人的订餐助手,今天是{today}。", id="chinese-prefix-mid-clause"), + pytest.param('PROMPT = ("你是一个乐于助人的订餐" # 角色\n "助手,请礼貌地回答。")', "你是一个乐于助人的订餐助手,请礼貌地回答。", id="chinese-comment-mid-clause"), + pytest.param('PROMPT = ("あなたは親切な予約" f"アシスタントです。{name}")', "あなたは親切な予約アシスタントです。", id="japanese-prefix-mid-clause"), + pytest.param('PROMPT = "あなたは予約・案内のアシスタントです。"', "あなたは予約・案内のアシスタントです。", id="japanese-middle-dot"), + pytest.param('PROMPT = ("คุณเป็นผู้ช่วย" f"จองโต๊ะที่สุภาพ")', "คุณเป็นผู้ช่วยจองโต๊ะที่สุภาพ", id="thai"), + pytest.param('PROMPT = "သင်သည် စားပွဲ ကြိုတင်မှာယူမှု အကူအညီပေးသူ ဖြစ်သည်။"', "သင်သည် စားပွဲ ကြိုတင်မှာယူမှု အကူအညီပေးသူ ဖြစ်သည်။", id="burmese"), + pytest.param('PROMPT = "អ្នកគឺជាជំនួយការកក់តុដែលគួរសម។"', "អ្នកគឺជាជំនួយការកក់តុដែលគួរសម។", id="khmer"), + ], +) +def test_an_unspaced_prompt_holds_its_quote(tmp_path, root, written, quote): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + count = len(written.splitlines()) + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": count}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + + +def test_an_escape_that_names_no_character_is_kept_as_written(tmp_path, root): + from fi.alk.harness.model_citation import quote_problem + + quote = "Order a table \\UFFFFFFFF and \\N{NOT A REAL NAME} for two." + + assert quote_problem(quote, 'PROMPT = "' + quote + '"', "prompt.py 1-1") is None + + +# The reader: what counts is limit + 1, and every descriptor is closed + + +def test_a_read_that_lands_on_the_limit_reads_one_more(tmp_path, monkeypatch): + from fi.alk.harness import model_citation + + (tmp_path / "agent.py").write_bytes(b"x" * 100) + real_read = os.read + monkeypatch.setattr(model_citation.os, "read", lambda fd, n: real_read(fd, min(n, 7))) + + assert model_citation.read_source(tmp_path / "agent.py", 49) == b"x" * 50 + + +def test_the_reader_closes_what_it_opens(tmp_path, monkeypatch): + from fi.alk.harness import model_citation + + (tmp_path / "agent.py").write_bytes(b"x" * 10) + (tmp_path / "folder").mkdir() + opened: list[int] = [] + closed: list[int] = [] + real_open, real_close = os.open, os.close + monkeypatch.setattr(model_citation.os, "open", lambda *a: opened.append(real_open(*a)) or opened[-1]) + monkeypatch.setattr(model_citation.os, "close", lambda fd: closed.append(fd) or real_close(fd)) + + model_citation.read_source(tmp_path / "agent.py", 100) + model_citation.read_source(tmp_path / "folder", 100) + + assert sorted(opened) == sorted(closed) and len(opened) == 2 + + +# Triple quotes count only in languages that have them, and never inside a string or comment + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("llm.py", "parts = r.split('\"\"\"')\nMODEL = \"gpt-4o\"\n", 2, "gpt-4o", id="in-a-one-line-string"), + pytest.param("llm.py", "# docstrings use \"\"\"\nMODEL = \"gpt-4o\"\n", 2, "gpt-4o", id="in-a-comment"), + pytest.param("llm.py", "SEP = \"'''\"; MODEL = \"gpt-4o\"\n", 1, "gpt-4o", id="in-a-string-on-the-line"), + pytest.param("llm.ts", "const prompt = 'Use \"\"\" for blocks';\nconst model = \"gpt-4o\";\n", 2, "gpt-4o", id="typescript"), + pytest.param("main.go", "var sep = `\"\"\"`\nvar model = \"gpt-4o\"\n", 2, "gpt-4o", id="go"), + pytest.param("cfg.yaml", "a: '''quoted'''\nmodel: gpt-4o\n", 2, "gpt-4o", id="yaml"), + pytest.param("llm.py", 'X = """a\nb"""; MODEL = "gpt-4o"\n', 2, "gpt-4o", id="a-block-closing-before-the-value"), + pytest.param("llm.py", "msg = f'Agent #{n}'; MODEL = \"gpt-4o\"\n", 1, "gpt-4o", id="prefixed-string-with-a-hash"), + pytest.param("llm.py", "x = a[1]//2; m = \"gpt-4o\"\n", 1, "gpt-4o", id="floor-division-after-a-subscript"), + pytest.param("llm.py", "n = a//2; model = 'gpt-4o'\n", 1, "gpt-4o", id="floor-division-after-a-name"), + ], +) +def test_stray_triple_quotes_do_not_hide_the_model(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("llm.py", '"""Uses "gpt-4o".\n"""\n', 1, "gpt-4o", id="a-docstring-opening-on-the-line"), + pytest.param("llm.py", "X = '''It's a''' # it's \"gpt-4o\"\n", 1, "gpt-4o", id="a-comment-after-a-block"), + pytest.param("llm.py", 'X = """a\nb""" # was "gpt-4o"\n', 2, "gpt-4o", id="a-comment-after-a-block-closes"), + pytest.param("lib.hs", "x' = 1 -- \"gpt-4o\"\n", 1, "gpt-4o", id="haskell-prime"), + pytest.param("llm.ts", '\t*\tmodel: "gpt-4o"\n', 1, "gpt-4o", id="a-tabbed-block-comment-line"), + pytest.param("llm.cfg", "; model = gpt-4o\n", 1, "gpt-4o", id="cfg-comment"), + pytest.param("llm.conf", "; model = gpt-4o\n", 1, "gpt-4o", id="conf-comment"), + ], +) +def test_more_comments_and_docstrings_are_not_the_model(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [not_named(name, line, file)] + + +def test_a_crafted_file_of_one_byte_lines_and_triple_quotes_costs_little(root): + # Lines with no triple quote, and lines inside a block without its closing quote, are passed + # over without a scan. + (root / "crafted.py").write_text( + '"""\n' + "x\n" * (MIB // 4) + '"""\n' + "y\n" * (MIB // 4) + 'MODEL = "gpt-4o"\n', + encoding="utf-8", + ) + line = 2 * (MIB // 4) + 3 + + _, _, took = checked(root, {"name": "gpt-4o", "path": "crafted.py", "line": line}) + + assert took < 0.25 + + +@pytest.mark.parametrize( + "file", + ["notes.org", "rules.mdc", "CHANGELOG", "CHANGELOG.md", "LICENSE", "CONTRIBUTING", "README"], +) +def test_more_prose_files_hold_no_model(root, file): + (root / file).write_text('# notes\nmodel: "claude-3-opus"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "claude-3-opus", "path": file, "line": 2}) + + assert problems == [prose(file)] + + +def test_a_file_named_after_readme_is_code(root): + (root / "readme_writer.py").write_text('# writes READMEs\nMODEL = "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "readme_writer.py", "line": 2}) + + assert problems == [] + + +@pytest.mark.parametrize( + "quote, lines, refused", + [ + pytest.param("You are pair.", (1, 4), True, id="short-and-not-in-the-lines"), + pytest.param("Be warm and brief.", (6, 12), False, id="short-and-in-the-lines"), + # Too few letters to say anything: not looked for, like no quote at all. + pytest.param("TBD", (1, 4), False, id="a-placeholder-with-nothing-held"), + ], +) +def test_a_short_quote_is_looked_for_whole(tmp_path, root, quote, lines, refused): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "agent.py", "start_line": lines[0], "end_line": lines[1]}], + ), + tmp_path / "out", + source_root=root, + ) + + if refused: + assert text_of(result) == REFUSED + not_the_quote(f"agent.py {lines[0]}-{lines[1]}") + else: + assert not result.get("is_error"), result + + +def test_full_width_digits_hold_their_quote(tmp_path, root): + (root / "prompt.py").write_text('PROMPT = "Seat at most 8 guests per table tonight."\n', encoding="utf-8") + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="Seat at most 8 guests per table tonight.", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + tmp_path / "out", + source_root=root, + ) + + assert not result.get("is_error"), result + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("agent.ts", 'class A { #model = "gpt-4o"; }', "gpt-4o", id="js-private-field"), + pytest.param("agent.c", '#define MODEL "gpt-4o"', "gpt-4o", id="c-define"), + pytest.param("main.rs", '#[default = "gpt-4o"] model: String,', "gpt-4o", id="rust-attribute"), + pytest.param("cfg.yaml", "llm: {url: http://x/#top, model: gpt-4o}", "gpt-4o", id="yaml-url-fragment"), + pytest.param("cfg.yaml", "llm: {a: 1,#b, model: gpt-4o}", "gpt-4o", id="yaml-hash-without-a-space"), + pytest.param("agent.ts", '/* the default */ const model = "gpt-4o";', "gpt-4o", id="a-closed-block-comment-before"), + pytest.param("index.html", ' ', "gpt-4o", id="a-closed-markup-comment-before"), + pytest.param("llm.py", 'm = str("gpt-4o")', "gpt-4o", id="str-as-a-function"), + pytest.param("GNUmakefile", "MODEL ?= gpt-4o", "gpt-4o", id="gnu-makefile"), + pytest.param("chat.prompt", "model: googleai/gemini-1.5-pro", "googleai/gemini-1.5-pro", id="dotprompt-frontmatter"), + ], +) +def test_more_honest_forms_by_language(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("agent.ts", 'x = 1; /* "gpt-4o" */ y = 2;', "gpt-4o", id="inside-a-closed-block-comment"), + pytest.param("agent.py", "x = 1 # it's \"gpt-4o\"", "gpt-4o", id="python-hash"), + pytest.param("cfg.yaml", 'model: gpt-4o-mini # was "gpt-4o"', "gpt-4o", id="yaml-spaced-hash"), + pytest.param("agent.ts", 'const m = config?.["gpt-4o"];', "gpt-4o", id="optional-chained-lookup"), + pytest.param("agent.py", 'm = env.str("gpt-4o")', "gpt-4o", id="str-as-a-method"), + ], +) +def test_more_comments_and_lookups_by_language(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +@pytest.mark.parametrize("file", ["prompt.j2", "prompt.jinja", "prompt.jinja2", "prompt.hbs", "prompt.mustache"]) +def test_a_prompt_template_holds_no_model(root, file): + (root / file).write_text('You are an assistant.\nYou run on "gpt-4o".\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": file, "line": 2}) + + assert problems == [prose(file)] + + +def test_a_short_quote_with_nothing_held_is_looked_for_whole(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + + result = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair.", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + ) + + assert text_of(result).startswith(REFUSED + not_the_quote("agent.py 1-4")) + + +def test_a_long_first_word_is_charged_on_every_try(root): + from fi.alk.harness.model_citation import quote_problem + + started = time.perf_counter() + # Without the charge, each of 200,000 tries would compare 200,000 letters. + answer = quote_problem("a" * 200_000 + " b", "a" * 400_000, "agent.py 1-1") + took = time.perf_counter() - started + + assert answer is not None + assert took < 0.25 + + +def test_a_short_unspaced_quote_is_still_searched_with_gaps(tmp_path, root): + # Six characters, each counted as three letters: long enough to be searched with gaps. + (root / "prompt.py").write_text('PROMPT = ("你是订餐" f"助手")\n', encoding="utf-8") + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="你是订餐助手", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + tmp_path / "out", + source_root=root, + ) + + assert not result.get("is_error"), result + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("cfg.yaml", 'note: x """ y\nmodel: gpt-4o\n', 2, "gpt-4o", id="yaml-bare-triple"), + pytest.param("main.go", 'var s = x + """\nvar model = "gpt-4o"\n', 2, "gpt-4o", id="go-bare-triple"), + ], +) +def test_triple_quotes_are_tracked_only_where_they_are_strings(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [] + + +def test_a_value_before_a_block_closes_is_text(root): + (root / "llm.py").write_text('X = """a\nuse "gpt-4o" here"""\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "llm.py")] + + +def test_lines_without_a_triple_quote_are_not_scanned(root, monkeypatch): + from fi.alk.harness import model_citation + + written = '"""\n' + "x\n" * 499 + '"""\n' + "y\n" * 499 + 'MODEL = "gpt-4o"\n' + (root / "llm.py").write_text(written, encoding="utf-8") + scanned: list[str] = [] + real_scan = model_citation._line_scan + monkeypatch.setattr( + model_citation, "_line_scan", lambda line, *a, **k: scanned.append(line) or real_scan(line, *a, **k) + ) + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 1001}) + + assert problems == [] + # The block's opening and closing lines, and the cited line: nothing between. + assert scanned == ['"""', '"""', 'MODEL = "gpt-4o"'] + + +# Round 3: a short or list quote is held like any other + + +def test_a_short_quote_is_held_and_its_lines_asked_about(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt="Be warm and brief.", system_prompt_from=imports) + ) + assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 1-4")) + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", "Be warm and brief.", short=True) + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + # A quote this short can't stand for the prompt: the lines go out, on record as unconfirmed, + # and the reply says so. + assert not third.get("is_error"), third + assert text_of(third).endswith( + "\nsystem_prompt_from: the lines named (agent.py 1-4) are recorded as unconfirmed, since " + "they don't hold the quote; the lines are the prompt." + ) + assert saved(destination)["system_prompt_from"] == imports + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +@pytest.mark.parametrize( + "fields", + [ + pytest.param({"system_prompt_excerpt": _BOOKING_QUOTE.split(". ")}, id="a-list"), + pytest.param({"prompt": _BOOKING_QUOTE.split(". ")}, id="a-list-under-an-alias"), + pytest.param({"system_prompt": _BOOKING_QUOTE}, id="an-alias"), + ], +) +def test_a_quote_in_another_shape_is_held(tmp_path, root, fields): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports, **fields)) + assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 1-4")) + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + assert text_of(second).startswith(REFUSED + "system_prompt_from: the lines named (agent.py 1-4) don't hold the quote you gave before") + + +def test_a_list_quote_that_passes_replaces_the_held_one(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + prompt = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + paraphrase = "Acme booking agent: helps guests reserve tables and never shares bookings." + + assert submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=paraphrase, system_prompt_from=prompt)).get("is_error") + assert not submit( + payload(hard_constraints=["Be polite."], prompt=_BOOKING_QUOTE.split(". "), system_prompt_from=prompt) + ).get("is_error") + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=prompt)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_from"] == prompt + assert "unconfirmed_prompt_from" not in saved(destination) + + +def test_a_placeholder_with_the_right_lines_keeps_the_lines(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + prompt = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + again = payload(hard_constraints=["Be polite."], system_prompt_excerpt="see lines", system_prompt_from=prompt) + + assert submit(again).get("is_error") + second = submit(again) + + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + assert saved(destination)["unconfirmed_prompt_from"] == prompt + + +def test_two_calls_in_one_message_are_each_asked(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + server = contract_tools(destination, source_root=root) + tool = next(one for one in server.tools if one.name == "submit_contract") + paraphrase = "Acme booking agent: helps guests reserve tables and never shares bookings." + both = payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=paraphrase, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}], + ) + + async def send_together(): + return await asyncio.gather(tool.handler(dict(both)), tool.handler(dict(both))) + + first, second = asyncio.run(send_together()) + + # Neither had seen a refusal when it was sent, so both are refused. This models a dispatcher + # that runs one message's calls side by side (ADK, the Vertex backend). The Claude backend runs + # them one after another, and there the second sees the first's refusal: a known limit. + assert first.get("is_error") and second.get("is_error") + assert not (destination / "contract.json").exists() + + +def test_a_quote_sent_alone_after_its_lines_were_refused_is_not_asked_for_lines(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + assert submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ).get("is_error") + second = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE)) + + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + + +def test_a_new_quote_with_wrong_lines_after_lines_alone_were_refused_is_kept(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + quote = "You are Acme's booking agent. Help guests book tables." + + assert not submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}]) + ).get("is_error") + assert submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)).get("is_error") + third = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=quote, system_prompt_from=imports)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == quote + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +def test_lines_set_aside_in_a_call_refused_for_something_else_are_said(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + wrong = payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + assert submit(wrong).get("is_error") + + second = submit({**wrong, "model": {"name": "gpt-4o", "path": "agent.py", "line": 1}}) + + assert "model: the name gpt-4o is not the value written on line 1" in text_of(second) + assert ( + "system_prompt_from: the lines named (agent.py 1-4) are left out and the quote is kept, " + "since they don't hold it" + ) in text_of(second) + + +# Round 3: pins for the line scan + + +@pytest.mark.parametrize("quote", ['x = "' + '\\"' * 40_000, "x = `" + "\\`" * 40_000]) +def test_an_unclosed_string_of_escapes_costs_one_pass(root, quote): + (root / "crafted.ts").write_text(quote + "\n", encoding="utf-8") + + _, _, took = checked(root, {"name": "gpt-4o", "path": "crafted.ts", "line": 1}) + + assert took < 0.25 + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("agent.ts", 'const model = "gpt-4o"; /* fast */', "gpt-4o", id="ts-comment-after"), + pytest.param("Agent.java", 'String model = "gpt-4o"; /* the default */', "gpt-4o", id="java-comment-after"), + pytest.param("llm.py", 'MODEL = "gpt-4o"; PROMPT = """', "gpt-4o", id="a-block-opening-after"), + pytest.param("llm.py", ' * 2, model="gpt-4o")', "gpt-4o", id="python-continuation-star"), + pytest.param("run.sh", "echo a#b; MODEL=gpt-4o", "gpt-4o", id="shell-hash-in-a-word"), + pytest.param("run.sh", "ls /*; MODEL=gpt-4o", "gpt-4o", id="shell-slash-star"), + pytest.param("build.groovy", 'a #b; def model = "gpt-4o"', "gpt-4o", id="groovy-hash"), + pytest.param("license.py", 'MODEL = "gpt-4o"', "gpt-4o", id="license-py-is-code"), + pytest.param("readme.py", 'MODEL = "gpt-4o"', "gpt-4o", id="readme-py-is-code"), + pytest.param("llm.py", "x = rb'Agent #1'; MODEL = \"gpt-4o\"", "gpt-4o", id="two-letter-prefix"), + pytest.param("llm.py", "x = 'a\\'b # c'; MODEL = \"gpt-4o\"", "gpt-4o", id="escaped-apostrophe"), + pytest.param("agent.ts", 'u = `${b}//v1`; m = "gpt-4o"', "gpt-4o", id="backtick-string"), + ], +) +def test_more_honest_lines_after_round_3(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "suffix", + [".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", + ".java", ".go", ".rs", ".swift", ".kt", ".kts", ".cs", ".scala", ".dart", ".m", ".mm", + ".groovy", ".gradle"], +) +def test_hash_is_no_comment_in_these_languages(root, suffix): + (root / f"agent{suffix}").write_text('\na = b #c; m = "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": f"agent{suffix}", "line": 2}) + + assert problems == [] + + +def test_a_tab_before_hash_starts_a_yaml_comment(root): + (root / "cfg.yaml").write_text('\nnote: a\t# "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "cfg.yaml", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "cfg.yaml")] + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("llm.py", 'n = a // 2; model = "gpt-4o"', "gpt-4o", id="python-floor-division-spaced"), + pytest.param("init.lua", 'local n = #t; model = "gpt-4o"', "gpt-4o", id="lua-length"), + pytest.param("llm.py", "x = t'Agent #{n}'; MODEL = \"gpt-4o\"", "gpt-4o", id="python-t-string"), + pytest.param("config.yaml.j2", "model: gpt-4o", "gpt-4o", id="a-config-template"), + ], +) +def test_more_honest_lines_after_round_3_probes(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("env.py", "model = gpt-4o", "gpt-4o", id="env-py-is-code"), + pytest.param("dockerfile_gen.py", "MODEL = gpt-4o", "gpt-4o", id="dockerfile-py-is-code"), + pytest.param("agent.js", 'call()// "gpt-4o"', "gpt-4o", id="js-comment-after-a-call"), + pytest.param("agent.js", 'x = 1// "gpt-4o"', "gpt-4o", id="js-comment-after-a-number"), + pytest.param("agent.ts", 'x = a[1]// "gpt-4o"', "gpt-4o", id="js-comment-after-a-subscript"), + ], +) +def test_more_wrong_values_after_round_3_probes(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +# The settled round: pins for rules the plants found unpinned + + +@pytest.mark.parametrize( + "written", + [ + pytest.param('curl http://x/v1; MODEL="gpt-4o"', id="a-url"), + pytest.param('cp data/* out; MODEL="gpt-4o"', id="a-glob"), + pytest.param('n=${#MODELS}; MODEL="gpt-4o"', id="a-length"), + pytest.param('echo $#; MODEL="gpt-4o"', id="a-count"), + ], +) +def test_a_script_without_a_suffix_keeps_its_model(root, written): + (root / "start").write_text(f"#!/bin/sh\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": "gpt-4o", "path": "start", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == "gpt-4o" + + +@pytest.mark.parametrize("prefix", ["rt", "tr", "RT"]) +def test_a_raw_template_string_is_a_string(root, prefix): + (root / "llm.py").write_text(f"\nx = {prefix}'Agent #{{n}}'; MODEL = \"gpt-4o\"\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [] + + +def test_a_triple_quote_after_floor_division_opens_a_block(root): + (root / "llm.py").write_text( + 'half = n // 2; DOC = """\nmodel = "gpt-4o"\n"""\n', encoding="utf-8" + ) + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "llm.py")] + + +def test_a_quote_alone_after_lines_alone_were_refused_is_not_asked_for_lines(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + assert not submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}]) + ).get("is_error") + assert submit( + payload(hard_constraints=["Be polite."], + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ).get("is_error") + third = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + + +# The settled round: lines numbered as the reading numbers them, and the reviewers' pins + + +@pytest.mark.parametrize( + "written, prompt_line, model_line", + [ + pytest.param( + b'# head\nA = 1\x0cB = 2\rC = 3\nNOTE = "a\xe2\x80\xa8b"\n' + b'PROMPT = "You are Acme\'s booking agent."\nMODEL = "gpt-4o"\n', + 4, 5, id="form-feed-lone-cr-and-u2028", + ), + pytest.param( + b'# head\r\nPROMPT = "You are Acme\'s booking agent."\r\nMODEL = "gpt-4o"\r\n', + 2, 3, id="crlf", + ), + pytest.param( + b'# head\nPROMPT = "You are Acme\'s booking agent."\nMODEL = "gpt-4o"', + 2, 3, id="no-newline-at-the-end", + ), + ], +) +def test_lines_are_numbered_at_each_newline(tmp_path, root, written, prompt_line, model_line): + (root / "agent.py").write_bytes(written) + destination = tmp_path / "out" + + result = accept_contract( + payload( + system_prompt_from=[{"path": "agent.py", "start_line": prompt_line, "end_line": prompt_line}], + model={"name": "gpt-4o", "path": "agent.py", "line": model_line}, + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == 'PROMPT = "You are Acme\'s booking agent."' + assert saved(destination)["model"]["line"] == model_line + + +def test_a_file_ending_in_a_newline_has_no_line_after_it(root): + (root / "agent.py").write_text('\nMODEL = "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "agent.py", "line": 3}) + + assert problems == ["model: line 3 is outside agent.py, which has 2 lines"] + + +@pytest.mark.parametrize( + "letter, escaped", + [ + pytest.param("\U00020000", "\\ud840\\udc00", id="cjk-extension-b"), + pytest.param("\U0001d400", "\\ud835\\udc00", id="math-bold"), + ], +) +def test_a_json_escaped_letter_past_ffff_holds_its_quote(letter, escaped): + from fi.alk.harness.model_citation import quote_problem + + quote = f"Greet each guest as {letter} and seat them at once." + written = '{"prompt": "' + quote.replace(letter, escaped) + '"}' + + assert quote_problem(quote, written, "prompt.json 1-1") is None + + +def test_an_escape_past_the_last_character_is_kept_as_written(): + from fi.alk.harness.model_citation import quote_problem + + quote = "Order a table \\U00110000 for two guests tonight." + + assert quote_problem(quote, 'PROMPT = "' + quote + '"', "prompt.py 1-1") is None + + +@pytest.mark.parametrize( + "quote, says", + [ + # 6 letters is the floor of a quote that says something, looked for whole. + pytest.param("Be kind", True, id="six-letters"), + pytest.param("Be wry", False, id="five-letters"), + ], +) +def test_six_letters_say_something(quote, says): + from fi.alk.harness.model_citation import quote_problem + + assert (quote_problem(quote, 'x = "nothing here"', "a.py 1-1") is not None) == says + + +def test_sixteen_letters_are_searched_with_gaps(): + from fi.alk.harness.model_citation import quote_pieces, quote_problem + + # 16 letters: words in order, with code between them. 15 would be looked for whole. + quote = "Be warm and kind now" + written = 'P = "Be warm " + tone + " and kind now"' + + assert quote_pieces(quote) + assert quote_problem(quote, written, "a.py 1-1") is None + assert not quote_pieces("Be warm and kind no") + + +def test_a_pipe_is_logged_as_not_a_regular_file(tmp_path, caplog): + from fi.alk.harness.model_citation import read_source + + if not hasattr(os, "mkfifo"): + pytest.skip("no named pipes here") + pipe = tmp_path / "pipe" + os.mkfifo(pipe) + answer: list[bytes | None] = [] + with caplog.at_level("INFO", logger="fi.alk.harness.model_citation"): + reading = threading.Thread(target=lambda: answer.append(read_source(pipe, 10)), daemon=True) + reading.start() + reading.join(timeout=2) + + assert answer == [None] + assert [record.getMessage() for record in caplog.records] == [ + "source file not read (not a regular file)" + ] + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("init.lua", 'local n = a // 2; model = "gpt-4o"', "gpt-4o", id="lua-floor-division"), + pytest.param("Makefile", "\techo a#b; MODEL=gpt-4o", "gpt-4o", id="makefile-hash-in-a-word"), + pytest.param(".flaskenv", "URL=http://x/#a MODEL=gpt-4o", "gpt-4o", id="flaskenv-hash-in-a-word"), + pytest.param(".env.local", "URL=http://x/#a MODEL=gpt-4o", "gpt-4o", id="env-name-hash-in-a-word"), + pytest.param("Dockerfile.dev", "ENV A=a#b MODEL=gpt-4o", "gpt-4o", id="dockerfile-name-hash-in-a-word"), + pytest.param("run.bash", "echo a#b; MODEL=gpt-4o", "gpt-4o", id="bash-hash-in-a-word"), + pytest.param("run.zsh", "echo a#b; MODEL=gpt-4o", "gpt-4o", id="zsh-hash-in-a-word"), + pytest.param("rules.mk", "\techo a#b; MODEL=gpt-4o", "gpt-4o", id="mk-hash-in-a-word"), + pytest.param("app.dockerfile", "ENV A=a#b MODEL=gpt-4o", "gpt-4o", id="dockerfile-suffix-hash-in-a-word"), + pytest.param("prod.env", "URL=http://x/#a MODEL=gpt-4o", "gpt-4o", id="env-suffix-hash-in-a-word"), + pytest.param("agent.mts", 'class A { #model = "gpt-4o"; }', "gpt-4o", id="mts-private-field"), + pytest.param("agent.cts", 'this.#model = "gpt-4o";', "gpt-4o", id="cts-private-field"), + pytest.param("agent.sc", 'val t: A#B = x; val model = "gpt-4o"', "gpt-4o", id="scala-script-type-projection"), + pytest.param(".env.j2", "MODEL=gpt-4o", "gpt-4o", id="env-template"), + pytest.param("Dockerfile.j2", "ENV MODEL=gpt-4o", "gpt-4o", id="dockerfile-template"), + pytest.param(".env.local.j2", "MODEL=gpt-4o", "gpt-4o", id="env-local-template"), + pytest.param("Dockerfile.dev.j2", "ENV MODEL=gpt-4o", "gpt-4o", id="dockerfile-dev-template"), + pytest.param("config.yaml.j2", " - gpt-4o", "gpt-4o", id="yaml-template-list-item"), + pytest.param("appsettings.json", '"ModelId": "gpt-4o", // the default', "gpt-4o", id="json-comment-after"), + pytest.param("agent.json", '"url": "http://x//y", "model": "gpt-4o"', "gpt-4o", id="json-slashes-in-a-string"), + ], +) +def test_more_honest_lines_after_the_settled_round(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("appsettings.json", '// "ModelId": "gpt-35-turbo",', "gpt-35-turbo", id="json-line-comment"), + pytest.param("appsettings.json", '"ModelId": "gpt-4o" /* was "gpt-35-turbo" */', "gpt-35-turbo", id="json-block-comment"), + pytest.param("agent.json", '"model": "gpt-4o", // "gpt-4-turbo" is slower', "gpt-4-turbo", id="json-comment-after-a-value"), + pytest.param("agent.json", '* "model": "gpt-4",', "gpt-4", id="json-block-comment-middle-line"), + pytest.param("dockerfile_gen.py", 'MODEL = DEFAULT#"gpt-4"', "gpt-4", id="python-named-like-a-dockerfile"), + pytest.param(".env_loader.py", 'MODEL = DEFAULT#"gpt-4"', "gpt-4", id="python-named-like-an-env-file"), + pytest.param("cfg.yaml", "note: the cart's total # model: 'gpt-4'", "gpt-4", id="a-word-ending-like-a-prefix"), + pytest.param("cfg.yaml", "note: it_t's ok # model: 'gpt-4'", "gpt-4", id="an-underscore-before-a-prefix"), + pytest.param("env.rb", "MODEL = gpt-4o", "gpt-4o", id="env-rb-is-code"), + ], +) +def test_more_wrong_values_after_the_settled_round(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +def test_a_python_template_is_text(root): + (root / "agent.py.j2").write_text('MODEL = "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "agent.py.j2", "line": 1}) + + assert problems == [prose("agent.py.j2")] + + +def test_a_type_projection_does_not_hide_a_triple_quote(root): + (root / "agent.scala").write_text( + 'val t: A#B = x; val p = """\nmodel = "gpt-4o"\n"""\n', encoding="utf-8" + ) + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "agent.scala", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "agent.scala")] + + +# The settled round: the flows + + +_SHORT_PROMPT = 'import os\n\nINSTRUCTIONS = "Be warm and brief."\n' +_PARAPHRASE = "The agent books tables for Acme guests, keeps other bookings private, and confirms first." + + +@pytest.mark.parametrize( + "source, lines, quote, paraphrase", + [ + pytest.param( + _SHORT_PROMPT, (3, 3), "Be warm and brief.", + "The agent is told to be warm and to keep its answers brief.", id="a-whole-short-prompt", + ), + pytest.param(_BOOKING, (6, 12), "Be warm and brief.", _PARAPHRASE, id="one-sentence-of-a-long-prompt"), + pytest.param(_BOOKING, (6, 12), "be warm and brief", _PARAPHRASE, id="another-case-no-full-stop"), + pytest.param(_BOOKING, (6, 12), "Be warm ... brief.", _PARAPHRASE, id="a-short-elided-quote"), + ], +) +@pytest.mark.parametrize("then", ["the-same-again", "the-lines-alone"]) +def test_a_short_verbatim_quote_after_a_refused_paraphrase_keeps_the_lines( + tmp_path, root, source, lines, quote, paraphrase, then +): + (root / "agent.py").write_text(source, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + named = [{"path": "agent.py", "start_line": lines[0], "end_line": lines[1]}] + + assert submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=paraphrase, system_prompt_from=named) + ).get("is_error") + second = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=quote, system_prompt_from=named) + ) + assert not second.get("is_error"), second + again = ( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=quote, system_prompt_from=named) + if then == "the-same-again" + else payload(hard_constraints=["Be polite."], system_prompt_from=named) + ) + third = submit(again) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_from"] == named + assert "unconfirmed_prompt_from" not in saved(destination) + + +def test_a_short_quote_not_in_its_lines_is_still_checked_against_the_held_one(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + assert submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_PARAPHRASE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}]) + ).get("is_error") + second = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt="Be warm and brief.", system_prompt_from=imports) + ) + + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _PARAPHRASE) + + +@pytest.mark.parametrize("lines", [(6, 12), (1, 4)], ids=["the-right-lines", "the-wrong-lines"]) +def test_a_null_in_a_list_quote_is_dropped_before_it_is_held(tmp_path, root, lines): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + named = [{"path": "agent.py", "start_line": lines[0], "end_line": lines[1]}] + + result = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=["Be warm", None, "and brief."], + system_prompt_from=named) + ) + + if lines == (6, 12): + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_from"] == named + else: + # Refused as this call's own quote, not as one given before with "None" in it. + assert text_of(result).startswith(REFUSED + not_the_quote("agent.py 1-4")) + + +def test_a_blank_quote_is_no_quote(tmp_path, root): + submit = submitter(tmp_path / "out", root) + + result = submit(payload(system_prompt_excerpt=" ")) + + assert result.get("is_error") + assert "no hard_constraints and no system_prompt_excerpt" in text_of(result) + assert PROMPT_LINES not in text_of(result) + + +def test_two_quotes_alone_in_one_message_are_each_asked_for_lines(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + server = contract_tools(destination, source_root=root) + tool = next(one for one in server.tools if one.name == "submit_contract") + both = payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE) + + async def send_together(): + return await asyncio.gather(tool.handler(dict(both)), tool.handler(dict(both))) + + first, second = asyncio.run(send_together()) + + assert PROMPT_LINES in text_of(first) and PROMPT_LINES in text_of(second) + assert not (destination / "contract.json").exists() + + +def test_short_lines_set_aside_in_a_call_refused_for_something_else_are_said(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + wrong = payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="Be warm and brief.", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + assert submit(wrong).get("is_error") + + second = submit({**wrong, "model": {"name": "gpt-4o", "path": "agent.py", "line": 1}}) + + assert "model: the name gpt-4o is not the value written on line 1" in text_of(second) + assert ( + "system_prompt_from: the lines named (agent.py 1-4) are copied unconfirmed, since they " + "don't hold the short quote" + ) in text_of(second) + + +@pytest.mark.parametrize( + "fields", + [ + pytest.param({"system_prompt_excerpt": ["Be warm", "and brief."]}, id="a-list"), + pytest.param({"prompt": ["Be warm", None, "and brief."]}, id="a-list-with-a-null-under-an-alias"), + pytest.param({"system_prompt_excerpt": 42}, id="a-number"), + ], +) +def test_the_held_quote_is_the_quote_the_contract_reads(fields): + from fi.alk.harness.contract import prompt_quote + from fi.alk.harness.tools import _without_nulls + + contract = AgentContract.model_validate(_without_nulls(payload(**fields))) + + assert prompt_quote(_without_nulls(payload(**fields))) == contract.system_prompt_excerpt + + +def test_a_saved_call_that_kept_the_quote_says_so(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + wrong = payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, system_prompt_from=imports) + + assert submit(wrong).get("is_error") + second = submit(wrong) + + assert not second.get("is_error"), second + assert text_of(second).endswith( + "\nsystem_prompt_from: the lines named (agent.py 1-4) are recorded as unconfirmed, since " + "they don't hold the quote; the quote is the prompt." + ) + + +# The settled round 2: a short quote on its own lines, and its limits + + +@pytest.mark.parametrize( + "first_lines, short, line", + [ + # The quote held from a call with no lines; then a name on the prompt's first line. + pytest.param(None, "PROMPT", 6, id="a-name-after-a-quote-alone"), + pytest.param(None, "SYSTEM_PROMPT", 6, id="a-setting-name-after-a-quote-alone"), + # The quote held from lines that held it; then a sentence repeated in a comment. + pytest.param((6, 12), "Be warm and brief.", 14, id="a-comment-after-a-confirmed-quote"), + ], +) +def test_a_short_quote_on_its_own_lines_does_not_replace_a_stronger_one( + tmp_path, root, first_lines, short, line +): + (root / "agent.py").write_text(_BOOKING + "\n# Style: Be warm and brief.\n", encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + named = [{"path": "agent.py", "start_line": line, "end_line": line}] + first = payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE) + if first_lines: + first["system_prompt_from"] = [{"path": "agent.py", "start_line": first_lines[0], "end_line": first_lines[1]}] + submit(first) + + second = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=short, system_prompt_from=named)) + assert text_of(second).startswith(REFUSED + lines_alone(f"agent.py {line}-{line}", _BOOKING_QUOTE)) + third = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=short, system_prompt_from=named)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["unconfirmed_prompt_from"] == named + + +def test_a_padded_short_quote_is_this_calls_own(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + + result = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=" Be warm and brief. ", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ) + + assert text_of(result).startswith(REFUSED + not_the_quote("agent.py 1-4")) + + +@pytest.mark.parametrize( + "quote, held", + [ + pytest.param("Be warm ... brief.", True, id="pieces-in-order"), + pytest.param("brief ... Be warm", False, id="pieces-out-of-order"), + ], +) +def test_a_short_elided_quote_is_looked_for_piece_by_piece(quote, held): + from fi.alk.harness.model_citation import quote_problem + + assert (quote_problem(quote, 'P = "Be warm and brief."', "a.py 1-1") is None) == held + + +def test_the_quote_under_its_own_name_comes_before_an_alias(): + from fi.alk.harness.contract import prompt_quote + + both = payload(system_prompt_excerpt="You are Acme's booking agent.", prompt="Something else entirely.") + + assert prompt_quote(both) == "You are Acme's booking agent." + assert AgentContract.model_validate(both).system_prompt_excerpt == "You are Acme's booking agent." + + +@pytest.mark.parametrize( + "file", + ["prompts/dockerfile_writer.j2", "prompts/env.system.j2", "prompts/system.prompt.j2", "dockerfile.md"], +) +def test_a_prompt_named_like_configuration_is_text(root, file): + (root / file).parent.mkdir(parents=True, exist_ok=True) + (root / file).write_text("You write the deployment.\nmodel: gpt-4o\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": file, "line": 2}) + + assert problems == [prose(file)] + + +def test_a_conf_file_has_no_block_comments(root): + (root / "site.conf").write_text('\nrewrite ^/(.*)//$ /$1 permanent; set $m "gpt-4o";\n', encoding="utf-8") + + problems, contract, _ = checked(root, {"name": "gpt-4o", "path": "site.conf", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == "gpt-4o" + + +def test_a_json_template_keeps_its_comments(root): + (root / "config.json.j2").write_text('\n// "model": "gpt-4-turbo",\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4-turbo", "path": "config.json.j2", "line": 2}) + + assert problems == [not_named("gpt-4-turbo", 2, "config.json.j2")] + + +@pytest.mark.parametrize("nothing", ["n / a", "- -", "a b"]) +def test_words_that_say_nothing_do_not_replace_the_held_quote(tmp_path, root, nothing): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE)) + + result = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=nothing, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ) + + assert text_of(result) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) diff --git a/tests/harness/test_contract_tools_subfolder.py b/tests/harness/test_contract_tools_subfolder.py index e4168fa0b..16e3374c9 100644 --- a/tests/harness/test_contract_tools_subfolder.py +++ b/tests/harness/test_contract_tools_subfolder.py @@ -1173,25 +1173,38 @@ def test_a_strings_length_is_charged_though_it_builds_nothing(repo, copies, refu def test_a_prompt_file_without_end_is_read_only_to_its_limit(tmp_path, repo, monkeypatch): - import io + import os - class Endless(io.RawIOBase): - def readable(self): - return True - - def read(self, size=-1): - assert size >= 0, "read to the end" - return b"#" * size + from fi.alk.harness import model_citation (repo / "agents" / "a" / "prompt.txt").write_text("") - opened = Path.open - monkeypatch.setattr( - Path, - "open", - lambda path, *args, **kwargs: Endless() - if path.name == "prompt.txt" - else opened(path, *args, **kwargs), - ) + # The prompt is read through model_citation.read_source, by descriptor. + endless: set[int] = set() + real_open, real_read, real_close = os.open, os.read, os.close + + def open_(path, flags, *args, **kwargs): + descriptor = real_open(path, flags, *args, **kwargs) + if Path(path).name == "prompt.txt": + endless.add(descriptor) + return descriptor + + given: list[int] = [] + + def read(descriptor, size): + # Like a link to /proc/self/pagemap: its size says 0 and it reads without end. + if descriptor not in endless: + return real_read(descriptor, size) + given.append(size) + assert sum(given) <= 8 * 1024 * 1024, "read to the end" + return b"#" * size + + def close(descriptor): + endless.discard(descriptor) + real_close(descriptor) + + monkeypatch.setattr(model_citation.os, "open", open_) + monkeypatch.setattr(model_citation.os, "read", read) + monkeypatch.setattr(model_citation.os, "close", close) result = accept_contract( payload( @@ -1443,10 +1456,17 @@ def test_a_prompt_file_named_again_is_not_read_again(tmp_path, repo): def test_a_prompt_file_past_the_limit_is_read_once(tmp_path, repo, monkeypatch): large = repo / "agents" / "a" / "p.txt" large.write_text("Be brief.\n" + "#" * (1024 * 1024)) + import os + + from fi.alk.harness import model_citation + opened: list[Path] = [] - real = Path.open + real = os.open + # The prompt is opened through model_citation.read_source, by descriptor. monkeypatch.setattr( - Path, "open", lambda path, *args, **kwargs: opened.append(path) or real(path, *args, **kwargs) + model_citation.os, + "open", + lambda path, *args, **kwargs: opened.append(Path(path)) or real(path, *args, **kwargs), ) result = accept_contract( @@ -1774,12 +1794,14 @@ def test_a_tool_module_through_links_is_charged_to_the_saves_budget( def test_a_prompt_path_named_again_is_resolved_once(tmp_path, repo, monkeypatch): - from fi.alk.harness import tools + from fi.alk.harness import model_citation (repo / "agents" / "a" / "prompt.txt").write_text("Be polite.\nBe brief.\n") resolved: list[Path] = [] - real = tools.charged_path - monkeypatch.setattr(tools, "charged_path", lambda path: resolved.append(path) or real(path)) + real = model_citation.charged_path + monkeypatch.setattr( + model_citation, "charged_path", lambda path: resolved.append(path) or real(path) + ) result = accept_contract( payload( diff --git a/tests/providers/test_providers_agent_authored.py b/tests/providers/test_providers_agent_authored.py index 312b9f9ff..9cf867b21 100644 --- a/tests/providers/test_providers_agent_authored.py +++ b/tests/providers/test_providers_agent_authored.py @@ -187,3 +187,86 @@ def test_no_framework_leaves_extension_and_hash() -> None: assert agent == recorded["agent"] assert set(agent["extension"]) == {"contract"} + + +_ABSENT = object() + + +def _chat_contract() -> dict: + # A raw dict, not through AgentContract, so a model of any shape reaches + # build_authored_agent unchanged. + return copy.deepcopy( + json.loads((AUTHORED / "chat_source.json").read_text())["contract"] + ) + + +@pytest.mark.parametrize( + "model", + [_ABSENT, None, "gpt-4o", {"name": " "}], + ids=["absent", "none", "raw-string", "blank-name"], +) +def test_no_model_leaves_the_document_and_hash(model: object) -> None: + recorded = json.loads((AUTHORED / "chat_source.json").read_text()) + contract = _chat_contract() + expected = copy.deepcopy(recorded["agent"]) + if model is not _ABSENT: + contract["model"] = model + # The stored contract echoes the value whatever its shape, so only that + # echo and the hash over it may differ from the recorded document. + expected["extension"]["contract"]["model"] = model + expected["setup_hash"] = compute_setup_hash(expected) + + agent = build_authored_agent(contract) + + assert agent["model"] is None + assert agent == expected + + +def test_model_goes_to_the_document() -> None: + def built(model: dict) -> dict: + return build_authored_agent({**_chat_contract(), "model": model}) + + without = build_authored_agent(_chat_contract()) + saved = {"provider": " openai ", "name": " gpt-4o ", "path": "agent.py", "line": 9} + agent = built(saved) + + assert agent["model"] == { + "provider": "openai", + "name": "gpt-4o", + "temperature": None, + "max_tokens": None, + } + assert agent["extension"]["contract"]["model"] == saved + assert set(agent["extension"]) == {"contract"} + assert agent["setup_hash"] == compute_setup_hash(agent) + assert agent["setup_hash"] != without["setup_hash"] + + for provider in ("", " ", None): + assert built({"provider": provider, "name": "gpt-4o"})["model"] == { + "provider": None, + "name": "gpt-4o", + "temperature": None, + "max_tokens": None, + } + + long = built({"provider": "p" * 300, "name": "m" * 300, "path": "a.py", "line": 1}) + assert long["model"]["provider"] == "p" * 255 + assert long["model"]["name"] == "m" * 255 + + with_url = built( + { + "provider": "openai", + "name": "https://u:p@host.example:8443/v1/m?token=abc", + "path": "agent.py", + "line": 9, + } + ) + assert with_url["model"]["name"] == "https://host.example:8443/v1/m" + assert with_url["extension"]["contract"]["model"] == { + "provider": "openai", + "name": "https://host.example:8443/v1/m", + "path": "agent.py", + "line": 9, + } + assert "token=abc" not in json.dumps(with_url) + assert "u:p@" not in json.dumps(with_url) diff --git a/tests/test_harness.py b/tests/test_harness.py index 8b9246a86..e8ca3ea5b 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -3968,7 +3968,7 @@ def visit(node): advertised = asyncio.run(advertised_fields()) # Set by the amendment tools during later stages, never by whoever submits the contract. - ours = {"amendments", "instrumentation", "source_subfolder"} + ours = {"amendments", "instrumentation", "source_subfolder", "unconfirmed_prompt_from"} def named(model: type[BaseModel]) -> list[str]: """Every field the model would have to fill, nested ones included.