diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 4e661fd16..a7767a96e 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -36,7 +36,7 @@ jobs: - name: Install test dependencies # Exercise the minimum LiveKit version supported by pyproject.toml. run: >- - python -m pip install '.[livekit]' pytest pytest-asyncio + python -m pip install '.[livekit]' pytest pytest-asyncio 'hypothesis>=6.140' 'livekit-agents==1.8.3' 'livekit-plugins-cartesia==1.8.3' 'livekit-plugins-deepgram==1.8.3' @@ -49,9 +49,10 @@ jobs: # pre-existing failures unrelated to the voice engine (CLI golden-path, # init presets, plus test_config_and_facades hangs) — track those # separately so this gate stays green and meaningful. - # Deselected below, failing on dev too: the two sandbox-server tests are TH-8978. (The - # run is one folded line, so the ticket is named here rather than beside each - # --deselect.) + # Deselected below, failing on dev too: the two sandbox-server tests are TH-8978, and the + # six in tests/test_harness.py fail on dev 70fc43bd as they do here (recorded in the + # older-findings register). (The run is one folded line, so they are named here rather + # than beside each --deselect.) run: >- python -m pytest -q tests/runtime/test_livekit_engine.py @@ -63,17 +64,58 @@ jobs: tests/test_call_originator_factory.py tests/test_retell_evidence.py tests/harness/test_process_runtime.py + tests/harness/test_source_folder.py tests/test_harness_sandbox_server.py --deselect tests/test_harness_sandbox_server.py::test_private_github_submission_becomes_hosted_job --deselect tests/test_harness_sandbox_server.py::test_public_github_branch_url_populates_job_ref + tests/test_harness_architecture.py + tests/test_harness_credentials.py + tests/test_harness.py + --deselect tests/test_harness.py::test_hosted_claude_session_uses_durable_resume_and_restricted_tools + --deselect tests/test_harness.py::test_a_skill_only_names_tools_its_stage_actually_has + --deselect tests/test_harness.py::test_connect_only_provider_world_cannot_create_shadow_state + --deselect tests/test_harness.py::test_the_system_prompt_stays_within_budget + --deselect tests/test_harness.py::test_a_save_refused_for_the_suites_shape_is_accepted_on_the_second_try + --deselect tests/test_harness.py::test_dropping_every_scenario_needs_an_explicit_everything + tests/harness/test_contract_model.py + tests/harness/test_contract_writer.py + tests/harness/test_runtime_repair.py + tests/harness/test_runtime_selection_admission.py + tests/harness/test_nudge_turn.py + tests/harness/test_seed_rows.py + tests/harness/test_builder_chat.py + tests/harness/test_hosted_conversation.py + tests/harness/test_source_tool_evidence.py + tests/harness/test_source_directory_names.py + tests/harness/test_storage_decision_admission.py + tests/harness/test_authoring_runtime_validation.py + tests/harness/test_failure_file.py + tests/harness/test_contract_tools_subfolder.py + tests/harness/test_import_roots.py + tests/harness/test_source_data_invariants_folder.py + tests/harness/test_hosted_authoring_entrypoint.py + tests/harness/test_certify_only.py + tests/harness/test_bundle_author_v2.py tests/harness/test_simulation_mark_bootstrap.py tests/harness/test_process_preflight.py tests/providers tests/harness/test_scenario_source_provision_agent.py + tests/harness/test_framework_detection.py + tests/harness/test_hosted_entrypoint.py tests/harness/test_hosted_entrypoint_provision_retry.py tests/harness/test_instrumentation_evidence.py tests/harness/test_agent_type_refusal.py + tests/harness/test_python_source_codecs.py tests/harness/test_executor_completed_counts.py + tests/harness/test_reading_entry.py + tests/harness/test_agent_reading.py + tests/harness/test_scan_walk.py + tests/harness/test_scan_folders.py + tests/harness/test_scan_document.py + tests/harness/test_scan_ai.py + tests/harness/test_agent_scan.py + tests/harness/test_authoring_entrypoint_find_agents.py + tests/harness/test_seed_contract.py tests/test_bump_deployment_values.py tests/test_release_workflow.py tests/test_release_pr_step.py diff --git a/src/fi/alk/harness/HOW-IT-WORKS.md b/src/fi/alk/harness/HOW-IT-WORKS.md index 6e40b22e8..a267bf261 100644 --- a/src/fi/alk/harness/HOW-IT-WORKS.md +++ b/src/fi/alk/harness/HOW-IT-WORKS.md @@ -83,7 +83,7 @@ computer-use tool the code check found in the source. So does a chat or voice re where the check found one. The next submission stands: a browser answer is refused to `failure.json` and the build exits 78, and a chat or voice answer is saved. The look gives a value to send back, which only tells apart calls in one model message: those run before the model reads -any result. The rule and its state are `SecondLook` in `agent_type_check.py`. A browser answer to a chat or voice look is refused at once, since that look said browser stops the build. When calls in one message run one after another, or together once the first scan is done, a later call is taken as the answer before the model has read the look. +any result. The rule and its state are `SecondLook` in `agent_type_check.py`. A browser answer to a chat or voice look is refused at once, since that look said browser stops the build. When calls in one message run one after another, or together once the first scan is done, a later call is taken as the answer before the model has read the look. Once an answer with the value has come back, a later browser answer without it gets a look again, so answers that keep switching cost turns, up to the reading's turn limit (70, or 140 for a generic job read unattended), never a refusal without a look. **How it is changed later** (`amend.py`). The contract is not frozen, but every change is recorded with a reason in `amendments[]`, so months later you can still tell what came from the diff --git a/src/fi/alk/harness/agent_reading.py b/src/fi/alk/harness/agent_reading.py new file mode 100644 index 000000000..e8aac99b2 --- /dev/null +++ b/src/fi/alk/harness/agent_reading.py @@ -0,0 +1,382 @@ +"""Read one agent the way a build reads it, and leave its description, with no build after it. + +The platform shows an agent's description before any environment exists, and bills the reading +from ``cost.json``. So the reading runs the build's own stage and gate, and every way it can end +leaves the files the platform judges it by: a saved contract is never thrown away and paid for +again, a timeout is never retried as a crash, and the cost on disk is never marked complete +while some of the spend may be missing from it. +""" + +from __future__ import annotations + +import asyncio +import functools +import logging +import time +from dataclasses import replace +from pathlib import Path +from typing import Any + +from . import session +from .agent_type_check import EXIT_REFUSED +from .backends import StageDone +from .cli import guidance_text +from .failure_file import FAILURE_FILE +from .job import FailureDomain, HarnessFailure, HarnessJob, SourceKind, provider_profile_source +from .reading_entry import ( + EXIT_RETRYABLE, + READING_JOB_INVALID, + MaskSecrets, + ReadingMode, + limit_seconds, + reading_failure, + write_reading_cost, + write_reading_failure, +) +from .reading_result import ( + CONTRACT_FILE, + classify, + document, + model_error, + out_of_turns, + saved_file, +) +from .session import Stage, StageIdleTimeout +from .source_folder import refuse_unusable_folder +from .sources import resolve as resolve_source +from .understand import SUBMIT_REMINDER, open_stage, opening, reading_settings + +logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) + +READING_SOURCE_EMPTY = "reading_source_empty" +AGENT_READING_TIMED_OUT = "agent_reading_timed_out" +AGENT_READING_CRASHED = "agent_reading_crashed" + +_LIMIT_ENV = "ALK_READING_MAX_DURATION_SECONDS" +_DEFAULT_SOFT_LIMIT_SECONDS = 1800.0 +# The hard limit outlasts the soft one by the longest stop (the interrupt, the wait after it and +# the close: 90 s) and room for the saved contract's document after it, whose framework scan can +# take well over a minute on a crafted repository, so a reading that stops itself is never cut +# off by the helper first. +_HARD_LIMIT_MARGIN_SECONDS = 300.0 +_INTERRUPT_SECONDS = 10.0 +_INTERRUPT_WAIT_SECONDS = 60.0 +# Ours, not the SDK's: its close is shielded and has no bound of its own. With it the decision +# comes by soft + 90 s, which leaves the rest of the margin to the document. +_CLOSE_SECONDS = 20.0 + +_COUNTS = ("tokens_in", "tokens_out", "tokens_cached") + + +def _soft_limit() -> float: + # The helper reads the same variable the same way for its own limit. + return limit_seconds(_LIMIT_ENV, _DEFAULT_SOFT_LIMIT_SECONDS) + + +def _check_job(job: HarnessJob) -> HarnessFailure | None: + if provider_profile_source(job.agent) is not None: + # The build adds a profile to what it reads for these agents, which a reading never loads. + return reading_failure( + FailureDomain.PLATFORM_SYNC, + READING_JOB_INVALID, + "A reading without a build reads an agent from its repository only. This job's agent " + "is also described outside its repository (by its provider, or by a prompt in the " + "job), so it needs a build.", + retryable=False, + ) + if job.source.kind in (SourceKind.GITHUB, SourceKind.ARCHIVE): + return None + return reading_failure( + FailureDomain.PLATFORM_SYNC, + READING_JOB_INVALID, + f"An agent is read from a GitHub repository or an upload, not from a " + f"{job.source.kind.value} source.", + retryable=False, + ) + + +class _ReadingStage(Stage): + """The build's reading stage, with each turn's own spend kept, and no turn ever replayed.""" + + def __init__( + self, + spec: Any, + *, + name: str, + backend: Any, + running_totals: bool, + ) -> None: + # Never replayed after a quiet stream: a replayed turn is paid for again, and a new + # session would restart Claude's running totals under the baseline the difference is + # taken from. + super().__init__(spec, name=name, backend=backend, idle_retries=0) + self.running_totals = running_totals + self.done: list[StageDone] = [] + self.in_flight = False + self.opened = False + self._last_cost = 0.0 + self._last_counts = dict.fromkeys(_COUNTS, 0) + + async def __aenter__(self) -> "_ReadingStage": + await super().__aenter__() + self.opened = True + return self + + def _own(self, reported: StageDone) -> StageDone: + # Claude reports the session's running total on every result, so adding results as + # reported would count every earlier turn again. + if not self.running_totals: + return reported + cost = reported.cost_usd + if cost is not None: + cost, self._last_cost = max(0.0, cost - self._last_cost), cost + counts: dict[str, int] = {} + for key in _COUNTS: + total = int(getattr(reported, key) or 0) + counts[key] = max(0, total - self._last_counts[key]) + self._last_counts[key] = total + return replace(reported, cost_usd=cost, **counts) + + def _events(self, received: Any, turn: Any) -> list[Any]: + if not isinstance(received, StageDone): + return super()._events(received, turn) + done = self._own(received) + events = super()._events(done, turn) + self.done.append(done) + self.in_flight = False + return events + + async def say(self, message: str, **kwargs: Any) -> Any: + self.in_flight = True + return await super().say(message, **kwargs) + + +def _source_has_files(source: Path) -> bool: + try: + return source.is_dir() and next(source.iterdir(), None) is not None + except OSError: + # Read as empty, so retryable: a source the sandbox can't list yet may be listable next time. + return False + + +def _reminder_due(stage: _ReadingStage, output: Path, stopping: asyncio.Event) -> bool: + if saved_file(output) or stopping.is_set(): + return False + opened = stage.done[-1] if stage.done else None + # An error flag with no status may have passed; one nudge costs less than a whole retry. + return opened is None or ( + opened.api_error_status is None and not out_of_turns(opened) + ) + + +async def _converse( + stage: _ReadingStage, message: str, output: Path, stopping: asyncio.Event +) -> None: + # The session lives inside this task, so cancelling the task closes it. + async with stage: + await stage.say(message) + if _reminder_due(stage, output, stopping): + await stage.say(SUBMIT_REMINDER) + + +async def _stop( + stage: _ReadingStage, task: asyncio.Task[None], stopping: asyncio.Event, wait: float +) -> None: + # Set first, so a turn ending now doesn't earn the reminder. + stopping.set() + # Its own task, waited on with a bound: wait_for would wait again for a cancel the interrupt + # swallows. A Ctrl-C here reaches `_read`, which lets it through. + interrupting = asyncio.ensure_future(stage.interrupt_response()) + bound = min(wait, _INTERRUPT_SECONDS) + await asyncio.wait({interrupting}, timeout=bound) + if not interrupting.done(): + interrupting.cancel() + logger.warning("the stop at the time limit was not delivered within %g s", bound) + elif interrupting.cancelled() or interrupting.exception() is not None: + # A stop that can't be delivered is still a timeout, not a crash. + logger.warning( + "the stop at the time limit was not delivered", + exc_info=None if interrupting.cancelled() else interrupting.exception(), + ) + await asyncio.wait({task}, timeout=wait) + # Closed before the files are read, so a late tool call is unlikely to save after the + # decision. The close is bounded, though: a session that hangs closing must not keep the + # reading from deciding; a session that outlives the bound is cancelled, with every other + # task, before the reading returns (`_stop_the_rest`). + task.cancel() + closing = min(wait, _CLOSE_SECONDS) + await asyncio.wait({task}, timeout=closing) + if not task.done(): + logger.warning("the stopped reading's session did not close within %g s", closing) + elif not task.cancelled() and task.exception() is not None: + logger.warning("the stopped reading ended with an error", exc_info=task.exception()) + + +def _stop_the_rest() -> None: + # Every task on this loop but this one is cancelled, ahead of the next loop turn, so no + # late tool call can change the files the reading decided on. asyncio.run's shutdown + # reaches the same tasks a moment later and cancels any still pending a second time, so a + # cleanup that catches the first cancel and then awaits may be cut short. It relies on `_read` running as asyncio.run's main task. + # The precondition: every tool call runs as a task on this loop and honours a cancel. + # The reading's Gemini tools are pinned to run on its loop (ADK's tool_thread_pool_config + # applies only to live mode in 2.7.1, which the reading doesn't use); its Claude tools run + # in the SDK's MCP server task on the same loop. The contract save is pinned to write + # nothing when cancelled at any of its waits. A tool that runs on another thread's loop, + # or swallows its cancel, is not covered by the guard. Shown against the locked versions. + this = asyncio.current_task() + for other in asyncio.all_tasks(): + if other is not this: + other.cancel() + + +def _timed_out(output: Path, soft_limit: float) -> int: + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_TIMED_OUT, + f"The agent reading ran past its {soft_limit:g} s limit without saving the agent's " + "description, and was stopped.", + retryable=False, + ), + output, + ) + return EXIT_REFUSED + + +async def _reading( + job: HarnessJob, source: Path, output: Path, started: float, soft_limit: float, wait: float +) -> tuple[int, bool]: + """The exit code, and whether every turn's spend is known to be in the cost file.""" + # Before the folder check, so an upload that hasn't arrived isn't blamed on the folder. + if not _source_has_files(source): + write_reading_failure( + reading_failure( + FailureDomain.PLATFORM_SYNC, + READING_SOURCE_EMPTY, + "The agent's source was missing or empty, so there was nothing to read. " + "It may not have arrived yet.", + retryable=True, + ), + output, + ) + return EXIT_RETRYABLE, True + if refuse_unusable_folder(source, job.source.subfolder, output): + return EXIT_REFUSED, True + + backend = session.resolve() + agent_source = resolve_source( + "repo", + name=str(job.metadata.get("agent_name") or source.name), + root=source, + subfolder=job.source.subfolder, + ) + # Opened as the build opens its own reading, turns, thinking and setup rule included, so a + # contract saved here is one the build's gate would have kept. + opened, _ = open_stage(agent_source, out=output, **reading_settings(job)) + stage = _ReadingStage( + opened.spec, + name=opened.name, + backend=backend, + running_totals=getattr(backend, "name", "") == "claude", + ) + stopping = asyncio.Event() + # The build's opening, the job's guidance included: a generic job's runtime advice changes + # what the contract records. + message = opening(agent_source) + guidance_text(job.authoring_guidance()) + task = asyncio.create_task(_converse(stage, message, output, stopping)) + finished, _ = await asyncio.wait( + {task}, timeout=max(0.0, started + soft_limit - time.monotonic()) + ) + + if not finished: + await _stop(stage, task, stopping, wait) + # A request cut off by the stop may be billed without being in the last total. + saved = saved_file(output) + if saved == FAILURE_FILE: + return EXIT_REFUSED, False + if saved == CONTRACT_FILE: + return document(job, source, output), False + return _timed_out(output, soft_limit), False + + error = task.exception() + if error is not None and not stage.opened: + # Nothing reached the provider, so the fault is the reading's own: a crash, retried once. + raise error + if isinstance(error, StageIdleTimeout): + # The quiet turn's spend never arrived; what was saved before it still stands. + saved = saved_file(output) + if saved == FAILURE_FILE: + return EXIT_REFUSED, False + if saved == CONTRACT_FILE: + return document(job, source, output), False + return ( + model_error( + output, + "The model stopped responding before the agent's description was saved " + f"({error}).", + retryable=True, + ), + False, + ) + # A request cut with no status may still be billed, and its total never reached us. + complete = ( + error is None + and not stage.in_flight + and not any( + done.is_error and done.api_error_status is None and not out_of_turns(done) + for done in stage.done + ) + ) + return classify(job, source, output, stage, error), complete + + +async def _read(job: HarnessJob, source: Path, output: Path, *, interrupt_wait: float) -> int: + # The same instant the helper's hard limit counts from, give or take a few lines. + started = time.monotonic() + output = Path(output) + code, complete = EXIT_RETRYABLE, False + try: + try: + code, complete = await _reading( + job, Path(source), output, started, _soft_limit(), interrupt_wait + ) + except (Exception, asyncio.CancelledError) as exc: + # A contract the gate saved is paid for; a crash would have the platform pay again. + # A cancel leaking out of the session is the same: the saved contract still stands. + # A cancel of this task itself (a Ctrl-C) is let through, the saved files as they are. + # (3.10 has no `cancelling`, and there a Ctrl-C is raised as itself, not as a cancel.) + current = asyncio.current_task() + if saved_file(output) != CONTRACT_FILE or ( + isinstance(exc, asyncio.CancelledError) + and getattr(current, "cancelling", lambda: 0)() + ): + raise + logger.warning( + "reading fault after a saved contract: class=%s", type(exc).__name__, exc_info=True + ) + code, complete = document(job, Path(source), output), False + finally: + _stop_the_rest() + # The work owns the final cost file, on every way out. + write_reading_cost(output, complete=complete) + return code + + +def _read_agent(*, interrupt_wait: float = _INTERRUPT_WAIT_SECONDS) -> ReadingMode: + return ReadingMode( + name="read_agent", + check_job=_check_job, + work=functools.partial(_read, interrupt_wait=interrupt_wait), + # The helper reads the variable when the reading starts, as the work does, so its + # limit is always the reading's own plus the margin. + time_limit_seconds=_DEFAULT_SOFT_LIMIT_SECONDS, + timed_out_code=AGENT_READING_TIMED_OUT, + timed_out_retryable=False, + crashed_code=AGENT_READING_CRASHED, + limit_env=_LIMIT_ENV, + limit_margin_seconds=_HARD_LIMIT_MARGIN_SECONDS, + ) + + +READ_AGENT = _read_agent() diff --git a/src/fi/alk/harness/agent_scan.py b/src/fi/alk/harness/agent_scan.py new file mode 100644 index 000000000..2a192cd67 --- /dev/null +++ b/src/fi/alk/harness/agent_scan.py @@ -0,0 +1,600 @@ +"""Finds the agents in a repository at one commit, or in an upload, and lists them for the platform. + +The platform shows this list before it creates an agent, and bills the scan from ``cost.json``. +A plain code check runs first and costs nothing. The model is asked only when that check finds +nothing outside test and example folders, or only files that call a model directly beside +TypeScript or JavaScript it can't read, because the customer pays for every turn of it. So +once the model has answered, nothing it was paid for is thrown away: a crash after it still +writes what was kept, and never ends in a retryable failure that would pay for the same answer +again. Every return leaves the final ``cost.json``, marked incomplete whenever the model's turn +may be missing from it. +""" + +from __future__ import annotations + +import logging +import os +import stat +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field, replace +from pathlib import Path + +from .job import FailureDomain, HarnessFailure, HarnessJob, SourceKind +from .agent_type_check import EXIT_REFUSED +from .reading_entry import ( + EXIT_DONE, + EXIT_RETRYABLE, + MaskSecrets, + ReadingMode, + limit_seconds, + reading_failure, + write_reading_cost, + write_reading_failure, +) +from .scan_ai import AiOutcome, run_ai_step +from .scan_document import Row, ScanResultInvalid, finish, root_name, write_agents_json +from .scan_folders import ClockRanOut, FolderChecker, FolderCheckerBroken +from .scan_walk import _WEAKEST, Candidate, Hazard, WalkResult, describe_entry, walk_repo + +logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) + +# Failures, written to failure.json. +SCAN_JOB_INVALID = "scan_job_invalid" +SCAN_SOURCE_MISSING = "scan_source_missing" +SCAN_CRASHED = "scan_crashed" +SCAN_HARD_LIMIT_REACHED = "scan_hard_limit_reached" +SCAN_MODEL_ERROR = "scan_model_error" +SCAN_MODEL_FORBIDDEN = "scan_model_forbidden" +SCAN_RESULT_INVALID = "scan_result_invalid" + +# Results, written to agents.json as its error. +SCAN_LIMIT_REACHED = "scan_limit_reached" +SCAN_TIMED_OUT = "scan_timed_out" +SCAN_OUT_OF_MEMORY = "scan_out_of_memory" +SCAN_AI_NO_ANSWER = "scan_ai_no_answer" +SCAN_FAILED_AFTER_AI = "scan_failed_after_ai" +AGENT_PATH_UNSUPPORTED = "agent_path_unsupported" + +CLOCK_ENV = "ALK_SCAN_MAX_SECONDS" +# The scan's clock when CLOCK_ENV doesn't name one. The helper reads the same variable for its +# hard stop, both when the reading starts. +CLOCK = 1200.0 +AI_ENV = "ALK_SCAN_AI_MAX_SECONDS" +# The model step's limit when AI_ENV doesn't name one; read when the model step is asked. +AI_SECONDS = 600.0 +# The helper's hard stop comes this long after the scan's own clock, so a scan that stops itself +# always writes its own file first. +HARD_LIMIT_MARGIN = 600.0 +# The model is asked only with this much left: its own time, then what follows it. +AI_LEAST_LEFT = 360.0 +# Kept back from the model for the stop sequence, the AI rows' descriptions and their checks. +AI_RESERVE = 180.0 + +_ERROR_MESSAGES = { + SCAN_LIMIT_REACHED: "The repository is larger than the scan reads, and no agent was found " + "before the limit.", + SCAN_TIMED_OUT: "The scan ran out of time before it found an agent.", + SCAN_OUT_OF_MEMORY: "The scan ran out of memory before it found an agent.", + SCAN_AI_NO_ANSWER: "The model looked for agents and gave no answer.", + SCAN_FAILED_AFTER_AI: "The scan failed after the model looked for agents, and nothing it " + "found could be kept.", + AGENT_PATH_UNSUPPORTED: "Every agent found sits at a path the platform cannot use.", +} +_STOPPED = {"limit": SCAN_LIMIT_REACHED, "clock": SCAN_TIMED_OUT, "memory": SCAN_OUT_OF_MEMORY} + +# A walk that ran out of memory is dropped whole, so nothing it decided is half kept. +_NO_WALK = WalkResult( + agents=(), + more_than_fifty=False, + stopped="memory", + unread_files=0, + source_outside_side=False, + candidate_outside_side=False, + script_outside_side=False, + only_weak_outside_side=False, + paths_left_out=0, + upload_others=0, + upload_others_side=0, +) + + +@dataclass +class _Scan: + """What one scan has so far, kept where the after-model catch can still reach it.""" + + upload: bool + walk_rows: list[Row] = field(default_factory=list) + ai_found: list[Candidate] = field(default_factory=list) + ai_rows: list[Row] = field(default_factory=list) + truncated: bool = False + cut_by_clock: bool = False + complete: bool = True + # One look per folder: many agents can share one, and the look walks skipped folders. + hazards: dict[str, Hazard] = field(default_factory=dict) + # The model was asked past walk rows that only call a model, so the cap must not drop + # what it found in their favour. + models_first: bool = False + + +def _fail( + output: Path, domain: FailureDomain, code: str, message: str, *, retryable: bool +) -> int: + write_reading_failure(reading_failure(domain, code, message, retryable=retryable), output) + logger.warning("agent scan: failed, code=%s retryable=%s", code, retryable) + return EXIT_RETRYABLE if retryable else EXIT_REFUSED + + +def _check_job(job: HarnessJob) -> HarnessFailure | None: + source = job.source + if source.kind in (SourceKind.REMOTE, SourceKind.PROVIDER) or source.subfolder: + return reading_failure( + FailureDomain.PLATFORM_SYNC, + SCAN_JOB_INVALID, + "The agent scan reads a whole GitHub repository or an upload; this job names a " + "remote or provider agent, or a folder.", + retryable=False, + ) + return None + + +# The most the source look's list of folders still to visit may hold, by its count (TH-9525). +_LOOK_HELD = 67_108_864 +# Inside one folder, the clock is read again after this many entries, so a huge folder can't keep +# the look past its deadline. +_LOOK_CLOCK_EVERY = 4_096 + + +def _path_cost(path: str) -> int: + """At least what one folder costs on the look's list: its full path as a str, at four bytes a + character unless it is all ASCII, plus 120 for the str's fixed part (at most 60 bytes on 3.12), + the allocator's rounding and the list's slot.""" + return (1 if path.isascii() else 4) * len(path) + 120 + + +def _look( + source: Path, *, deadline: float, clock: Callable[[], float] +) -> tuple[str, int, int]: + """Whether ``source`` holds a regular file, stopping at the folder where one is found: + ``"file"``, ``"none"`` (none in the whole tree), ``"limit"`` (the folders still to visit passed + `_LOOK_HELD` first) or ``"clock"`` (the deadline passed first), with the folders read and what + the list held. Each folder is read once, entry by entry: its subfolders are taken on until the + bound, and the read goes on past the bound for a regular file, which wins wherever it is in the + folder. A folder or entry that can't be read is passed over, as os.walk does. The clock is read + before each folder after the source's own, and every `_LOOK_CLOCK_EVERY` entries inside any + folder; an empty source never reaches either, so it is "none" whatever the time.""" + try: + if not source.is_dir(): + return "none", 0, 0 + except OSError: + return "none", 0, 0 + top = os.fspath(source) + stack = [top] + held = _path_cost(top) + folders = 0 + while stack: + # Never before the source's own folder: an empty source is "missing" whatever the time. + if folders and clock() > deadline: + return "clock", folders, held + folder = stack.pop() + held -= _path_cost(folder) + folders += 1 + start = len(stack) + over = False + try: + with os.scandir(folder) as entries: + for index, entry in enumerate(entries, 1): + if not index % _LOOK_CLOCK_EVERY and clock() > deadline: + return "clock", folders, held + try: + if entry.is_dir(follow_symlinks=False): + if not over: + stack.append(entry.path) + held += _path_cost(entry.path) + over = held > _LOOK_HELD + elif stat.S_ISREG(os.lstat(entry.path).st_mode): + return "file", folders, held + except OSError: + continue + except OSError: + # A folder that can't be listed, or stops partway: what it gave is kept. + pass + if over: + return "limit", folders, held + # The first in name order is popped next, so the look goes depth first in name order. + stack[start:] = sorted(stack[start:], reverse=True) + return "none", folders, held + + +def _hints(walk: WalkResult) -> str: + """What the walk counted, for the model's opening. Counts only, never a path: a path invited + the model to submit again what the walk already listed.""" + side = sum(candidate.side for candidate in walk.agents) + walk.upload_others_side + # The model is asked past candidates outside side folders only when every one of them + # only calls a model, so each unlisted one outside them is one of those. + weak = sum( + not candidate.side and candidate.sign in _WEAKEST for candidate in walk.agents + ) + (walk.upload_others - walk.upload_others_side) + return ( + "source files outside those folders: yes; " + f"agent-like files inside them: {side}; " + f"files outside them that only call a model: {weak}; " + f"files the check could not read: {walk.unread_files}." + ) + + +def _unchecked(candidate: Candidate, *, found_by_ai: bool = True) -> Row: + return Row( + candidate=candidate, + framework={"name": "unknown", "version": None, "label": None}, + finished=False, + found_by_ai=found_by_ai, + ) + + +def _check_into( + rows: list[Row], + checker: FolderChecker, + source: Path, + candidates: Sequence[Candidate], + scan: _Scan, + *, + found_by_ai: bool, + deadline: float, + clock: Callable[[], float], +) -> None: + """Checks each candidate's folder in order, appending each row as soon as it is known.""" + for index, candidate in enumerate(candidates): + try: + word = checker.check(source, candidate.subfolder, deadline=deadline, clock=clock) + except ClockRanOut: + scan.truncated = scan.cut_by_clock = True + # Found, so listed without its folder's word rather than lost: the person can still + # pick it, and a build of it names the framework. + rows.extend( + _unchecked(left, found_by_ai=found_by_ai) for left in candidates[index:] + ) + return + rows.append(Row(candidate, word.framework, word.finished, found_by_ai)) + + +def _one_row(walk_rows: Sequence[Row], ai_rows: Sequence[Row]) -> tuple[list[Row], bool]: + """An upload's one row, at the root, and whether any other was left out.""" + every = [*walk_rows, *ai_rows] + if not every: + return [], False + outside = next((row for row in walk_rows if not row.candidate.side), None) + if outside is not None and outside.candidate.sign in _WEAKEST: + # The model was asked past it, so what the model found outside side folders is + # likelier the agent than a file that only calls a model. + outside = next((row for row in ai_rows if not row.candidate.side), outside) + chosen = ( + outside + or next((row for row in ai_rows if not row.candidate.side), None) + or every[0] + ) + if chosen.candidate.subfolder: + chosen = replace(chosen, candidate=replace(chosen.candidate, subfolder="")) + return [chosen], len(every) > 1 + + +def _write( + job: HarnessJob, + output: Path, + rows: Sequence[Row], + *, + upload: bool, + truncated: bool, + error: str | None, + models_first: bool, +) -> int: + try: + body, _ = finish( + rows=rows, + root_name=root_name(job), + commit_sha=None if upload else job.source.commit_sha, + truncated=truncated, + error=None if error is None else (error, _ERROR_MESSAGES[error]), + models_first=models_first, + ) + except ScanResultInvalid as invalid: + logger.error("agent scan: result invalid, problem=%s", invalid.problem) + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_RESULT_INVALID, + "The agent list broke the platform's rules and was not written; this is a fault " + "in the scan, not in the repository.", + retryable=False, + ) + write_agents_json(output, body) + logger.info( + "agent scan: written, rows=%d truncated=%s error=%s", + len(rows), + truncated, + error or "none", + ) + return EXIT_DONE + + +def _result( + job: HarnessJob, + output: Path, + scan: _Scan, + walk: WalkResult, + outcome: AiOutcome | None, +) -> int: + """Turn the rows and the model's outcome into the file: an upload keeps one row, the + outcome decides the status and ``truncated``, and the walk's flags add to ``truncated``.""" + if scan.upload: + rows, others = _one_row(scan.walk_rows, scan.ai_rows) + else: + rows, others = [*scan.walk_rows, *scan.ai_rows], False + truncated = scan.truncated or others + error: str | None = None + if outcome is not None: + if outcome.kind == "model_error": + logger.warning("agent scan: model step failed, %s", outcome.message) + if not rows: + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_MODEL_ERROR, + "The model failed while looking for agents; a retry may get through." + if outcome.retryable + else "The model refused the request to look for agents; a retry would be " + "refused the same way.", + retryable=outcome.retryable, + ) + # The walk's rows are already checked and cost nothing. A retry would pay for the + # model again to add at most what it missed, so the rows are kept, marked partial. + truncated = True + elif outcome.kind == "forbidden": + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_MODEL_FORBIDDEN, + "The provider billed a model this kit may not spend on, so the scan stopped.", + retryable=False, + ) + elif outcome.kind in ("no_answer", "timed_out"): + if rows: + truncated = True + else: + error = SCAN_AI_NO_ANSWER if outcome.kind == "no_answer" else SCAN_TIMED_OUT + truncated = truncated or outcome.refused > 0 or outcome.uncarried > 0 + truncated = ( + truncated + or walk.more_than_fifty + or bool(walk.stopped and rows) + or walk.paths_left_out > 0 + or walk.unread_files > 0 + or scan.cut_by_clock + or walk.upload_others > 0 + or any(row.candidate.partial or not row.finished for row in rows) + ) + if not rows and error is None: + if walk.stopped: + error = _STOPPED[walk.stopped] + elif walk.paths_left_out or (outcome is not None and outcome.uncarried): + error = AGENT_PATH_UNSUPPORTED + return _write( + job, + output, + rows, + upload=scan.upload, + truncated=truncated, + error=error, + models_first=scan.models_first, + ) + + +def _after_ai( + job: HarnessJob, + source: Path, + output: Path, + scan: _Scan, + walk: WalkResult, + outcome: AiOutcome, + checker: FolderChecker, + *, + deadline: float, + clock: Callable[[], float], +) -> int: + for folder, entry in outcome.pairs: + candidate = describe_entry( + source, + entry, + folder, + deadline=deadline, + upload=scan.upload, + clock=clock, + hazards=scan.hazards, + ) + if candidate is None: + scan.truncated = True + continue + scan.ai_found.append(candidate) + # Walk rows first, so a walk row always beats a model row for the same entry, and a + # duplicate is never checked twice. + taken = {row.candidate.entry for row in scan.walk_rows} + fresh: list[Candidate] = [] + for candidate in scan.ai_found: + if candidate.entry not in taken: + taken.add(candidate.entry) + fresh.append(candidate) + scan.ai_found = fresh + _check_into( + scan.ai_rows, + checker, + source, + fresh, + scan, + found_by_ai=True, + deadline=deadline, + clock=clock, + ) + return _result(job, output, scan, walk, outcome) + + +def _keep_what_was_paid_for( + job: HarnessJob, output: Path, scan: _Scan, crash: Exception +) -> int: + """After the model was paid for, a crash still ends in a file, never in a paid retry.""" + # Logged with its trace, as the reading helper logs a crash. Only the platform's secret + # values are masked; a repo path in the message is not. + logger.error("agent scan: failed after the model step", exc_info=crash) + checked = {row.candidate.entry for row in scan.ai_rows} + ai_rows = [ + *scan.ai_rows, + *(_unchecked(candidate) for candidate in scan.ai_found if candidate.entry not in checked), + ] + if scan.upload: + rows, _ = _one_row(scan.walk_rows, ai_rows) + else: + rows = [*scan.walk_rows, *ai_rows] + try: + body, _ = finish( + rows=rows, + root_name=root_name(job), + commit_sha=None if scan.upload else job.source.commit_sha, + truncated=True, + error=None + if rows + else (SCAN_FAILED_AFTER_AI, _ERROR_MESSAGES[SCAN_FAILED_AFTER_AI]), + models_first=scan.models_first, + ) + write_agents_json(output, body) + return EXIT_DONE + except Exception: + logger.error("agent scan: the kept rows could not be written", exc_info=True) + try: + body, _ = finish( + rows=[], + root_name=root_name(job), + commit_sha=None if scan.upload else job.source.commit_sha, + truncated=True, + error=(SCAN_FAILED_AFTER_AI, _ERROR_MESSAGES[SCAN_FAILED_AFTER_AI]), + ) + write_agents_json(output, body) + return EXIT_DONE + except Exception: + logger.error("agent scan: the fixed error could not be written", exc_info=True) + raise crash + + +async def _scan(job: HarnessJob, source: Path, output: Path, scan: _Scan) -> int: + clock = time.monotonic + # Taken first, so the source check counts against the clock too. + deadline = clock() + limit_seconds(CLOCK_ENV, CLOCK) + looked, folders, held = _look(source, deadline=deadline, clock=clock) + if looked in ("limit", "clock"): + # No regular file was found before the look's own bound or the clock: the same end as a + # walk stopped with nothing found, and no model is asked. + logger.info( + "agent scan: the source look stopped, stopped=%s folders=%d held=%d", + looked, + folders, + held, + ) + return _result(job, output, scan, replace(_NO_WALK, stopped=looked), None) + if looked == "none": + return _fail( + output, + FailureDomain.PLATFORM_SYNC, + SCAN_SOURCE_MISSING, + "The source to scan is missing, is not a folder, or holds no file.", + retryable=True, + ) + try: + walk = walk_repo( + source, + deadline=deadline, + upload=scan.upload, + clock=clock, + hazards=scan.hazards, + ) + except MemoryError: + walk = _NO_WALK + logger.warning("agent scan: the walk ran out of memory and was dropped") + with FolderChecker() as checker: + try: + _check_into( + scan.walk_rows, + checker, + source, + walk.agents, + scan, + found_by_ai=False, + deadline=deadline, + clock=clock, + ) + except FolderCheckerBroken: + # No model has run yet, so a retry costs nothing that was paid for. + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_CRASHED, + "The folder check could not run, before any model was asked.", + retryable=True, + ) + # A Python file that only calls a model says nothing of an agent written in TypeScript + # or JavaScript beside it, which the walk can't list. + asks_the_model = ( + ( + not walk.candidate_outside_side + or (walk.only_weak_outside_side and walk.script_outside_side) + ) + and walk.source_outside_side + and walk.stopped == "" + and deadline - clock() >= AI_LEAST_LEFT + ) + if not asks_the_model: + return _result(job, output, scan, walk, None) + scan.models_first = walk.candidate_outside_side + outcome = await run_ai_step( + source, + hints=_hints(walk), + seconds=min(limit_seconds(AI_ENV, AI_SECONDS), deadline - clock() - AI_RESERVE), + on_turn_end=lambda: write_reading_cost(output, complete=False), + ) + if not outcome.complete: + scan.complete = False + # `_write` has already turned a ScanResultInvalid into its 78, so it never reaches here. + try: + return _after_ai( + job, + source, + output, + scan, + walk, + outcome, + checker, + deadline=deadline, + clock=clock, + ) + except Exception as crash: + return _keep_what_was_paid_for(job, output, scan, crash) + + +async def _work(job: HarnessJob, source: Path, output: Path) -> int: + scan = _Scan(upload=job.source.kind is SourceKind.ARCHIVE) + try: + return await _scan(job, Path(source), Path(output), scan) + finally: + # The one place the final cost is written, so no return can leave the helper's zero. + write_reading_cost(output, complete=scan.complete) + + +FIND_AGENTS = ReadingMode( + name="find_agents", + check_job=_check_job, + work=_work, + time_limit_seconds=CLOCK, + limit_env=CLOCK_ENV, + limit_margin_seconds=HARD_LIMIT_MARGIN, + timed_out_code=SCAN_HARD_LIMIT_REACHED, + timed_out_retryable=False, + crashed_code=SCAN_CRASHED, +) diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index 1c4afb0dd..e6e467d6c 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -7,8 +7,6 @@ from __future__ import annotations -import ast -import codecs import csv import fnmatch import io @@ -17,14 +15,24 @@ import os import re import tokenize -from collections.abc import Iterator, Sequence +from collections.abc import Iterable, Iterator, Sequence from functools import lru_cache from pathlib import Path from typing import Any, Literal +from .credentials import imported_by_agent from .failure_file import FAILURE_FILE from .instrumentation_evidence import _SKIPPED_DIRECTORIES, _imports from .job import FailureDomain, HarnessFailure, HarnessStage +from .python_source import ( + LARGEST_PARSED_MODULE, + EscapeBudget, + ParseBudget, + TokenBudget, + parsed, + read_module, +) +from .source_folder import agent_folder, is_within logger = logging.getLogger(__name__) @@ -105,15 +113,57 @@ # What the world registry already treats as a browser. BROWSER_MODALITIES = frozenset({"browser", "computer_use", "cua"}) +# Every match of TOOL_MARKER starts with one of these, inside one name or string, so a file +# without one has no marker, and isn't tokenized to strip its comments. +_MARKER_START = re.compile( + r"computer[_-]?[uU]se|ComputerUse|computer_\d{8}|@playwright/mcp|mcp-server-playwright" +) _TOOLKIT = "PlayWrightBrowserToolkit" +# A whole name: no letter, digit or "_" before or after it. In a unicode-escape file read as bytes, +# an escape such as \t, \x20, \040 or \u0020 just before it counts as a boundary too. +_WHOLE_NAMES = { + name: re.compile( + # The name looked for first, so only the places it starts at are tried: eight lookbehinds + # at every byte held the save 2.8 s. + rf"(?={re.escape(name)})(?:(? bool: return any(fnmatch.fnmatchcase(lowered, pattern) for pattern in _TEST_FILES) -def _unpacked_at_top(root: Path, left: list[int] | None = None) -> set[str]: +def _unpacked_at_top( + root: Path, + left: list[int] | None = None, + read: dict[tuple[tuple[str, int], ...], frozenset[str]] | None = None, +) -> set[str] | frozenset[str]: """The files ``pip install -t .`` put beside the agent's code, by each distribution's record. Exact paths: one distribution shipping ``tools/x.py`` says nothing about the customer's own ``tools/``. ``left`` is the record bytes still to read, shared across one walk's folders. + ``read`` holds, for one walk, the rows of each folder read whole, keyed by its records' names + and sizes: a folder holding exactly those records again, as in one Lambda folder after + another, shares them, unread, for an eighth of each record's size, at least 4 KiB each. + Shared, so never changed. Every ``_agent_files`` walk shares, the agent folder's own walk in + ``browser_tools_in_agent`` too; only its ``rows_of``, the records at and above the agent, + doesn't. """ names: set[str] = set() try: @@ -245,7 +310,15 @@ def _unpacked_at_top(root: Path, left: list[int] | None = None) -> set[str]: return names if left is None: left = [_LARGEST_RECORDS] + sized: list[tuple[str, Path, int]] = [] + whole = True for entry in records: + # Each record costs at least a block, read or shared, so a folder of more than are left + # can't be read whole, and its key could never be stored or matched. A spent budget stops + # here, before any record is looked at. + if len(sized) * _SMALLEST_RECORD > left[0]: + whole = False + break try: record = entry / "RECORD" # pip writes neither as a link, and following them reads one record once per link. @@ -254,14 +327,30 @@ def _unpacked_at_top(root: Path, left: list[int] | None = None) -> set[str]: # A link to a device or a named pipe reports no size, and reading it never ends. if not record.is_file() or record.stat().st_size > _LARGEST_RECORD: continue - left[0] -= max(record.stat().st_size, _SMALLEST_RECORD) + sized.append((entry.name, record, record.stat().st_size)) + except OSError: + continue + key = tuple((name, size) for name, _, size in sized) + shared = read.get(key) if read is not None and whole and key else None + if shared is not None: + left[0] -= sum(max(size // 8, _SMALLEST_RECORD) for _, _, size in sized) + return shared if left[0] >= 0 else names + for _, record, size in sized: + try: + left[0] -= max(size, _SMALLEST_RECORD) if left[0] < 0: + whole = False break names.update( row[0] for row in csv.reader(io.StringIO(record.read_text(encoding="utf-8"))) if row ) except (OSError, ValueError, csv.Error): + # Rows left out: another folder with this key must read its own. + whole = False continue + if read is not None and whole and key: + read[key] = frozenset(names) + return read[key] return names @@ -275,10 +364,11 @@ def _agent_files( # records hold them, as pip wrote them, so what is kept is the rows the budget lets in and that # folder's path once: how deep the folder sits doesn't multiply what each row costs. # ``at_top`` gets the rows of the records at the top, for a caller that would read them again. - unpacked: dict[str, set[str]] = {} + unpacked: dict[str, set[str] | frozenset[str]] = {} # The folders above this one whose records gave rows, nearest last. The walk goes top down, so # leaving a folder is seeing a directory that isn't under it. holding: list[str] = [] + read: dict[tuple[tuple[str, int], ...], frozenset[str]] = {} if left is None: left = [_LARGEST_RECORDS] top = os.fspath(root) @@ -286,7 +376,7 @@ def _agent_files( while holding and not directory.startswith(holding[-1] + os.sep): del unpacked[holding.pop()] if any(name.endswith(".dist-info") for name in subdirectories): - rows = _unpacked_at_top(Path(directory), left) + rows = _unpacked_at_top(Path(directory), left, read) if rows: unpacked[directory] = rows holding.append(directory) @@ -333,71 +423,24 @@ def _packages_used(path: Path) -> set[str]: return found -def _unread_codec(source: bytes) -> bool: - """Whether the coding line names a codec the scan won't decode ``source`` in. - - A codec such as rot13 isn't a text encoding, so it can't be read as source. Three text codecs - are refused by name: punycode's decode costs the square of its size, about 2 s for 128 KB; - idna's runs in Python for each label; undefined decodes nothing. Python runs an idna file, so - the readers then look for the names in its bytes instead (``_named_raw``). unicode-escape - decodes in linear time and its escapes can spell a name, so it is decoded like any other. - """ - try: - # A non-ASCII byte on the coding line makes detection fail while the parser still decodes - # the file in that codec, so it is read here as ASCII: only the codec's name matters. - encoding, _ = tokenize.detect_encoding(io.BytesIO(re.sub(rb"[\x80-\xff]", b"?", source)).readline) - info = codecs.lookup(encoding) - except (SyntaxError, UnicodeError, LookupError): - # The reader meets the same error and refuses the file itself. - return False - return info.name in _UNREAD_CODECS or not getattr(info, "_is_text_encoding", True) - - -def _lines(source: bytes) -> bytes: - # Python reads a lone CR as a line end, and detect_encoding and the tokenizer don't: to them a - # file of only CRs is one line, so a "coding:" anywhere after a first "#" would be its cookie. - return source.replace(b"\r\n", b"\n").replace(b"\r", b"\n") - - -def _named_raw(source: bytes) -> tuple[set[str], set[str]]: - """The browser imports and tool markers whose bytes appear in ``source``, before any decode. - - For a file in a refused codec: its names are counted rather than lost, which can only bring a - second look, never a refusal. One search over bytes already read. - """ - # Whole names only: "steelers" in a string is no import of steel. - imports = { - name - for name in BROWSER_IMPORTS - if re.search(rb"(? set[str]: +def _imports_in(text: str, path: Path) -> set[str]: # One file at a time: a customer's odd file can run the parser out of memory, and that mustn't # take the rest of the scan, or the reading it serves, down with it. - try: - if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: - return set() - source = path.read_bytes() - if _unread_codec(_lines(source)): - return _named_raw(source)[0] - # From bytes, so a BOM or a coding line is honoured as Python itself would. - return _imports(ast.parse(source)) - except (OSError, ValueError, SyntaxError, RecursionError, MemoryError): - logger.debug("agent type scan read no imports from %s", path, exc_info=True) + tree = parsed(text) + if tree is None: + logger.debug("agent type scan read no imports from %s", path) return set() + return _imports(tree) -def _modules_at_top(root: Path, unpacked: set[str] | None = None) -> set[str]: +def _largest_read(path: Path) -> int: + # Python is parsed, which costs a crafted file the square of its size. + if path.suffix == ".py": + return min(_LARGEST_CODE_FILE, LARGEST_PARSED_MODULE) + return _LARGEST_CODE_FILE + + +def _modules_at_top(root: Path, unpacked: set[str] | frozenset[str] | None = None) -> set[str]: """What sits ahead of an installed package on the import path, so isn't that package. ``unpacked`` is the folder's record rows, when the caller has read them already. @@ -416,14 +459,8 @@ def _modules_at_top(root: Path, unpacked: set[str] | None = None) -> set[str]: return set() -def _python_code(source: bytes) -> str: - # From bytes, as imports are read, so a coding line or a BOM is honoured as Python itself would, - # and on the same lines Python reads. The caller has refused a codec it won't decode. - return " ".join( - token.string - for token in tokenize.tokenize(io.BytesIO(_lines(source)).readline) - if token.type not in (tokenize.COMMENT, tokenize.ENCODING) - ) +def _python_code(tokens: list[tokenize.TokenInfo]) -> str: + return " ".join(token.string for token in tokens if token.type != tokenize.COMMENT) def _script_kept(hit: re.Match[str]) -> str: @@ -434,28 +471,44 @@ def _script_kept(hit: re.Match[str]) -> str: return kept -def _markers_in(path: Path) -> set[str]: +def _python_markers(text: str, tokens: TokenBudget, path: Path) -> set[str]: + if not _MARKER_START.search(text): + return set() + if not tokens.spend(text): + # Past the total, comments can't be told from code, and a marker in one would refuse a + # browser reading without its second look; so the file's markers aren't read, and the + # model's reading stands. + return set() try: - if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: - return set() - if path.suffix == ".py": - source = path.read_bytes() - if _unread_codec(_lines(source)): - return _named_raw(source)[1] - code = _python_code(source) - else: - # As Node reads it: a stray byte that isn't UTF-8 becomes a replacement character. - text = path.read_text(encoding="utf-8", errors="replace") - code = _SCRIPT_COMMENT_OR_STRING.sub(_script_kept, text) - except (OSError, ValueError, SyntaxError, tokenize.TokenError): - logger.debug("agent type scan read no markers from %s", path, exc_info=True) + found = tokens.tokens(text) + except (ValueError, SyntaxError, SystemError, tokenize.TokenError) as error: + # The type only: the error can quote a line of the customer's file. + logger.debug("agent type scan read no markers from %s: %s", path, type(error).__name__) return set() + # Past the work, too, the file's markers aren't read. + return set() if found is None else _markers(_python_code(found)) + + +def _markers(code: str) -> set[str]: return { "Playwright MCP" if "playwright" in hit.group() else "computer-use tool" for hit in TOOL_MARKER.finditer(code) } +def _markers_in(path: Path) -> set[str]: + """A script's markers; Python's are read by ``_python_markers``.""" + try: + if not path.is_file() or path.stat().st_size > _LARGEST_CODE_FILE: + return set() + # As Node reads it: a stray byte that isn't UTF-8 becomes a replacement character. + text = path.read_text(encoding="utf-8", errors="replace") + except (OSError, ValueError) as error: + logger.debug("agent type scan read no markers from %s: %s", path, type(error).__name__) + return set() + return _markers(_SCRIPT_COMMENT_OR_STRING.sub(_script_kept, text)) + + def _file_size(path: Path) -> int: try: return path.stat().st_size @@ -490,8 +543,10 @@ def _signature(root: Path) -> tuple[int, int, int]: Every reopen asks, so records are looked at, not read: a record that starts or stops naming a file changes its own size or time. Files a record names count too, which only rescans more. + The times of every file read are summed, so one file dated ahead can't pin the key; the change + time is in the sum because a copy that puts a modified time back still moves it. """ - latest = count = size = 0 + times = count = size = 0 for directory, subdirectories, files in os.walk(root): subdirectories[:] = [ name for name in subdirectories if not _not_the_agent_dir(name, directory) @@ -504,47 +559,100 @@ def _signature(root: Path) -> tuple[int, int, int]: info = path.stat() except OSError: continue - latest = max(latest, info.st_mtime_ns) + times += info.st_mtime_ns + info.st_ctime_ns count += 1 size += info.st_size - return latest, count, size + return times, count, size @lru_cache(maxsize=8) def _scan(root: Path, _files: tuple[int, int, int]) -> tuple[str, ...]: """One scan per state of the files: every builder reopen in a process asks again.""" - found: set[str] = set() - imported: set[str] = set() - scanned = 0 + # Each record read once: the walk keeps the top's rows for the modules there. left = [_LARGEST_RECORDS] at_top: set[str] = set() - for path in sorted(filter(_is_read, _agent_files(root, left, at_top)), key=_reading_order): + files = list(_agent_files(root, left, at_top)) + # The record the budget left unread may be the one naming a package unpacked at the top, + # so no folder there shows the agent wrote it: every import counts as the agent's. + local = set() if left[0] < 0 else _modules_at_top(root, at_top) + return _tools_in(files, local) + + +def _tools_in( + paths: Iterable[Path], local: set[str], first: Path | None = None +) -> tuple[str, ...]: + """The browser tools ``paths`` import or name, the files in ``first`` read before the rest: + the totals below cut the files read last, and the agent's own mustn't be those.""" + found: set[str] = set() + imported: set[str] = set() + scanned = scripts = python = 0 + budget = ParseBudget() + tokens = TokenBudget() + escapes = EscapeBudget() + searched = ParseBudget(_LARGEST_RAW_SEARCH) + skipped = 0 + for path in sorted( + filter(_is_read, paths), + key=lambda path: (first is not None and not is_within(path, first), *_reading_order(path)), + ): size = _file_size(path) # Never read, so it mustn't use up the limit before the code that is. - if size > _LARGEST_CODE_FILE: + if size > _largest_read(path): continue scanned += size if scanned > _LARGEST_SCAN: break - # One file at a time: the tokenizer raises more than it documents (SystemError on a NUL - # byte in 3.12), and one odd file mustn't stop every reading from being checked. + if path.suffix in _SCRIPT_SUFFIXES: + scripts += size + if scripts > _LARGEST_SCRIPT_SCAN: + continue + elif path.suffix == ".py": + python += size + if python > _LARGEST_PYTHON_SCAN: + continue + # The tokenizer's known faults are caught in `_python_markers`; this catch is for the ones + # nobody foresaw. try: if path.name == "package.json": found |= _packages_used(path) - else: - markers = _markers_in(path) - names = _imports_in(path) if path.suffix == ".py" else set() + elif path.suffix != ".py": + found |= _markers_in(path) + elif ( + text := read_module( + path, _largest_read(path), _PYTHON_NAMES, undecoded := [], escapes + ) + ) is not None: + markers = _python_markers(text, tokens, path) + names = _imports_in(text, path) if budget.spend(text) else set() found |= markers imported |= names - except Exception: - # A skipped file may be the agent's main one, and the reading then finds nothing. - logger.warning("agent type scan skipped %s", path, exc_info=True) + elif undecoded: + # A codec refused by name, which Python still runs the file in: the browser + # packages and tool markers it names, as bytes, count. That leads to a second + # look, never a refusal on its own. #224 does the same. + raw = undecoded[0].decode("latin-1") + named = {name for name in BROWSER_IMPORTS if name in raw} + if _TOOLKIT in raw: + found.add(_TOOLKIT) + # The regular expressions try each place in the text, so only while the scan's + # total lasts: 1,023 crafted files held the save 9 s. Past it, a name counts + # wherever it appears, which can only bring a second look. + if searched.spend(raw): + # Whole names only: "steelers" in a string is no import of steel. + named = {name for name in named if _WHOLE_NAMES[name].search(raw)} + found |= _markers(raw) + imported |= named + except Exception as error: + # A skipped file may be the agent's main one, and the reading then finds nothing. No + # trace at any level, since a trace can quote a line of the customer's code; the type + # only at WARNING, once a scan, since a path can name their business. + skipped += 1 + if skipped == 1: + logger.warning("agent type scan skipped a file: %s", type(error).__name__) + logger.debug("agent type scan skipped %s: %s", path, type(error).__name__) continue - local = _modules_at_top(root, at_top) - if left[0] < 0: - # The record the budget left unread may be the one naming a package unpacked at the top, - # so no folder there shows the agent wrote it: every import counts as the agent's. - local = set() + if skipped > 1: + logger.warning("agent type scan skipped %d files", skipped) for name in imported: top = name.split(".")[0] if top in local: @@ -556,6 +664,151 @@ def _scan(root: Path, _files: tuple[int, int, int]) -> tuple[str, ...]: return tuple(sorted({name.replace("_", "-") for name in found})) +def _files_in(folder: Path) -> list[Path]: + try: + return [ + path for path in sorted(folder.iterdir()) if path.is_file() and not path.is_symlink() + ] + except OSError: + return [] + + +def _below_shared(path: Path, folder: Path) -> tuple[str, ...]: + """The folders holding ``path`` below the deepest one it shares with the agent's folder.""" + held, named = path.parent.parts, folder.parts + common = 0 + while common < min(len(held), len(named)) and held[common] == named[common]: + common += 1 + return held[common:] + + +def _walked(root: Path, subfolder: str) -> frozenset[Path]: + """What the agent's folder imports, followed with the save's limits. A walk a limit stopped + is logged: what it didn't reach, the save's checks don't see, and bundling's walk, with no + limit, may.""" + stopped: list[bool] = [] + found = imported_by_agent( + root, + subfolder, + parse_limit=LARGEST_PARSED_MODULE, + parse_budget=ParseBudget(_WALKED_PARSED), + script_budget=ParseBudget(_WALKED_SCRIPTS), + import_limit=_WALKED_IMPORTS, + lookup_limit=_WALKED_LOOKUPS, + stopped=stopped, + ) + if stopped and stopped[0]: + logger.warning( + "the save's import walk for %s stopped at its limits or left code unread; " + "%d files reached", + subfolder, + len(found), + ) + return found + + +def code_the_agent_reaches(source_root: Path, subfolder: str) -> list[Path]: + """The files outside the agent's folder that its code runs: the files directly above it, + less test helpers, and everything its imports lead to, as the credential scan reads them + (test-named and unpacked packages included, which the browser-tool scan leaves out). It runs + that scan; the save reuses the scan it already ran. + + Raises ``SubfolderNotFound`` for a folder that can't be used. + """ + reached: list[Path] = [] + browser_tools_in_agent(source_root, subfolder, reached) + return reached + + +def browser_tools_in_agent( + source_root: Path, subfolder: str, reached: list[Path] | None = None +) -> list[str]: + """What one agent of a repository uses: its folder, what it imports, and the files directly + in the folders above it. A sibling's browser tool says nothing about this agent. + + Given ``reached``, it's filled with the files outside the folder the agent reaches: those + above it, and every file its imports lead to, filtered or not, so another scan can read them + without walking the imports again. + + Raises ``SubfolderNotFound`` for a folder that can't be used. + """ + root = Path(source_root).resolve() + folder = agent_folder(root, subfolder) + above = [ + path + for parent in folder.parents + if is_within(parent, root) + for path in _files_in(parent) + if not _not_the_agent_file(path.name) + ] + # An import the agent wrote says this code is its own, whatever the folder is called, unless + # it's a test helper or someone else's package, unpacked beside the agent or above it. One + # record budget for every folder, each folder's records read once, and rows kept as pip + # wrote them, relative to that folder: a Path for each row cost ten times as much, and a + # budget for each folder let a deep folder read many times the limit. + left = [_RECORDS_AT_AND_ABOVE] + # Nearest first. + rows_of = { + parent: _unpacked_at_top(parent, left) + for parent in (folder, *(parent for parent in folder.parents if is_within(parent, root))) + } + holding = [(parent, rows) for parent, rows in rows_of.items() if rows] + + def unpacked(path: Path) -> bool: + # The nearest few folders holding the file, as _agent_files checks: a crafted chain of + # them would cost its depth for each of thousands of files. + nearest = 0 + for parent, rows in holding: + if is_within(path, parent): + below = path.parts[len(parent.parts) :] + # A record names files, so the file's own row is the one to find: one join per + # folder, where every prefix cost a join each. + if "/".join(below) in rows: + return True + nearest += 1 + if nearest == _NEAREST_RECORD_FOLDERS: + break + return False + + walked = sorted(_walked(root, subfolder)) + shared = [ + path + for path in walked + if not _not_the_agent_file(path.name) + and not unpacked(path) + # Only the folders the agent reaches it through: a whole repository under `e2e/` is + # still the agent's own code. + and not any(_TEST_FOLDER.fullmatch(part.lower()) for part in _below_shared(path, folder)) + ] + local = _modules_at_top(folder, rows_of[folder]) | _modules_at_top(root, rows_of[root]) + if left[0] < 0: + # Records left unread can't say a package there is installed, not the customer's, and no + # read order keeps both the folder's and the root's whole: so every import counts, as in + # the scan with no folder. A browser one is found, which leads to a second look at the + # model's reading, not a refusal. + local = set() + if reached is not None: + # Everything the walk followed, as the credential scan reads it: a test-named package the + # agent imports still runs. The files above stay filtered: a root + # conftest.py isn't the agent's code. + reached[:] = [*above, *walked] + # A shared script's dependencies are declared in its package, at or above it. + manifests = [ + parent / "package.json" + for path in shared + if path.suffix != ".py" + for parent in path.parents + if is_within(parent, root) and (parent / "package.json").is_file() + ] + return list( + _tools_in( + dict.fromkeys((*_agent_files(folder), *above, *shared, *manifests)), + local, + folder, + ) + ) + + def modality_text(value: Any) -> str: """The modality as text to compare, with a list's items one per line.""" if isinstance(value, list): diff --git a/src/fi/alk/harness/amend.py b/src/fi/alk/harness/amend.py index 841568394..42e1e4302 100644 --- a/src/fi/alk/harness/amend.py +++ b/src/fi/alk/harness/amend.py @@ -19,12 +19,18 @@ from __future__ import annotations -import json from pathlib import Path -from .contract import AgentContract, known_modalities, validate_contract +from ..providers.agent import _redact +from .contract import ( + CONTRACT_FILE, + AgentContract, + known_modalities, + validate_contract, + write_contract, +) -CONTRACT = "contract.json" +CONTRACT = CONTRACT_FILE def widen( @@ -78,12 +84,7 @@ def widen( contract.amendments.pop() return False, "the amended contract would not be valid: " + "; ".join(problems) - destination = Path(destination) - destination.mkdir(parents=True, exist_ok=True) - (destination / CONTRACT).write_text( - json.dumps(contract.model_dump(), indent=2, ensure_ascii=False), - encoding="utf-8", - ) + _persist(contract, destination) return True, ( f"{tool_name}.{argument} now accepts {', '.join(fresh)}. " f"{len(contract.amendments)} amendment(s) recorded on the contract." @@ -104,8 +105,12 @@ def add_rule( return False, "no rule given" if not why.strip(): return False, "say why: an unexplained rule is indistinguishable from a guess" - if any(rule.lower() == existing.lower() for existing in contract.hard_constraints): - return False, f"the agent already has that rule: {rule}" + # Both sides are compared as they are saved: a contract held from an older save can still + # hold its rules raw. One call, so a refused URL is logged once. The reply names the rule as + # saved too, since the chat sends it on to the platform. + cleaned, *existing = _redact([rule, *contract.hard_constraints]) + if cleaned.lower() in (one.lower() for one in existing): + return False, f"the agent already has that rule: {cleaned}" contract.hard_constraints.append(rule) contract.amendments.append(f"rule added — {rule}: {why.strip()}") @@ -115,12 +120,7 @@ def add_rule( contract.amendments.pop() return False, "the amended contract would not be valid: " + "; ".join(problems) - destination = Path(destination) - destination.mkdir(parents=True, exist_ok=True) - (destination / CONTRACT).write_text( - json.dumps(contract.model_dump(), indent=2, ensure_ascii=False), - encoding="utf-8", - ) + _persist(contract, destination) return True, ( f"added. The agent now has {len(contract.hard_constraints)} rules, and this one is " "graded from here on." @@ -166,12 +166,7 @@ def set_modality( contract.amendments.pop() return False, "the amended contract would not be valid: " + "; ".join(problems) - destination = Path(destination) - destination.mkdir(parents=True, exist_ok=True) - (destination / CONTRACT).write_text( - json.dumps(contract.model_dump(), indent=2, ensure_ascii=False), - encoding="utf-8", - ) + _persist(contract, destination) reached = { "voice": "a call placed to the agent where it is hosted, its own tools answered over a " "webhook", @@ -191,25 +186,23 @@ def drop_rule( """ if not why.strip(): return False, "say why: removing a rule changes what is being graded" + # Both sides are compared as they are saved: a contract held from an older save can still + # hold its rules raw. One call, so a refused URL is logged once. + wanted, *saved_rules = _redact([rule.strip(), *contract.hard_constraints]) + wanted = wanted.lower() + held = list(zip(contract.hard_constraints, (one.lower() for one in saved_rules))) match = next( - ( - existing - for existing in contract.hard_constraints - if existing.lower() == rule.strip().lower() - ), + (existing for existing, as_saved in held if as_saved == wanted), None, ) or next( - ( - existing - for existing in contract.hard_constraints - if rule.strip().lower() in existing.lower() - ), + (existing for existing, as_saved in held if wanted in as_saved), None, ) if match is None: + # The rules as saved, since the chat sends this reply on to the platform. return False, ( "no rule like that. It has:\n - " - + "\n - ".join(contract.hard_constraints) + + "\n - ".join(saved_rules) ) contract.hard_constraints.remove(match) contract.amendments.append(f"rule removed — {match}: {why.strip()}") @@ -322,12 +315,9 @@ def drop_tool_references(contract: AgentContract) -> list[str]: def _persist(contract: AgentContract, destination: Path) -> None: - destination = Path(destination) - destination.mkdir(parents=True, exist_ok=True) - (destination / CONTRACT).write_text( - json.dumps(contract.model_dump(), indent=2, ensure_ascii=False), - encoding="utf-8", - ) + """Save the contract the stage is holding. Its free text is cleaned in place as it is saved, + so the stage goes on working from what was saved.""" + write_contract(contract, Path(destination)) def not_offered(contract: AgentContract, candidates: dict[str, set[str]]) -> list[str]: diff --git a/src/fi/alk/harness/authoring_entrypoint.py b/src/fi/alk/harness/authoring_entrypoint.py index 402e82ce1..f861c578f 100644 --- a/src/fi/alk/harness/authoring_entrypoint.py +++ b/src/fi/alk/harness/authoring_entrypoint.py @@ -25,14 +25,19 @@ from .agent_type_check import EXIT_REFUSED, failure_message from .cli import _auto from .failure_file import FAILURE_FILE -from .job import HarnessJob, ProviderExecutionMode, SourceKind +from .job import ( + PROVIDER_MANAGED_CONNECTORS as PROVIDER_MANAGED_CONNECTORS, + HarnessJob, + ProviderExecutionMode, + SourceKind, + provider_profile_source, +) from .hosted_compose_bootstrap import initialize_hosted_compose from .provider_import import inspect_provider_target from .scenarios import load as load_written +from .source_folder import refuse_unusable_folder, saved_runtime from .understand import PROVIDER_IMPORT_PROFILE_PATH_ENV -PROVIDER_MANAGED_CONNECTORS = frozenset({"vapi", "retell", "retell_chat"}) - def _persist_authored_scenario_count( job_path: Path, job: HarnessJob, output: Path @@ -373,6 +378,10 @@ def _certify_only(args: argparse.Namespace) -> int: raise RuntimeValidationError( "environment", f"Certify-only: invalid job document: {_one_line(exc)}" ) from exc + if refuse_unusable_folder( + source, job.source.subfolder, output, saved_runtime(output) + ): + return EXIT_REFUSED if job.metadata.get("generic_harness_v1") is not True: raise RuntimeValidationError( "environment", "Certify-only requires a generic_harness_v1 job" @@ -463,10 +472,8 @@ def _load_provider_import_profile( secrets_path: Path | None, profile_cache_path: Path | None = None, ) -> dict[str, object] | None: - if ( - job.agent.mode is ProviderExecutionMode.CONNECT_ONLY - and job.agent.connector.strip().lower() == "phone" - ): + source = provider_profile_source(job.agent) + if source == "phone": # A dial-in number has no provider API to inspect. The supplied prompt is the only # behavioral source of truth; never pretend we discovered tools or implementation code. return { @@ -476,10 +483,7 @@ def _load_provider_import_profile( "system_prompt": str(job.agent.config["target_system_prompt"]), "tools": [], } - if ( - job.agent.mode is ProviderExecutionMode.CONNECT_ONLY - and job.agent.connector.strip().lower() == "livekit" - ): + if source == "livekit": if secrets_path is not None: secrets_path.unlink(missing_ok=True) return { @@ -489,14 +493,7 @@ def _load_provider_import_profile( "system_prompt": str(job.agent.config["target_system_prompt"]), "tools": [], } - inspect_connect_only_provider = ( - job.agent.mode is ProviderExecutionMode.CONNECT_ONLY - and job.agent.connector.strip().lower() in PROVIDER_MANAGED_CONNECTORS - ) - if ( - job.agent.mode is not ProviderExecutionMode.PROVIDER_IMPORT - and not inspect_connect_only_provider - ): + if source is None: return None if profile_cache_path is not None and profile_cache_path.is_file(): cached = json.loads(profile_cache_path.read_text(encoding="utf-8")) @@ -674,13 +671,83 @@ def main(argv: list[str] | None = None, *, validate_runtime: bool = False) -> in type=Path, help="Directory the sealed bundle is written to in --certify-only mode", ) + reading = parser.add_mutually_exclusive_group() + reading.add_argument( + "--read-agent-only", + action="store_true", + help="Read the agent and leave its description in --output; build nothing", + ) + reading.add_argument( + "--find-agents", + action="store_true", + help="Find the agents in the source and leave their list in --output; build nothing", + ) + parser.add_argument( + "--seed-contract", + type=Path, + help="A reading's raw contract.json at /work/seed/contract.json; the build reuses it when it matches the job", + ) args = parser.parse_args(argv) + if args.seed_contract is not None and ( + args.certify_only or args.read_agent_only or args.find_agents + ): + parser.error( + "--seed-contract cannot be combined with --certify-only, --read-agent-only or --find-agents" + ) if args.certify_only: if args.bundle_output is None: parser.error("--certify-only requires --bundle-output") return _certify_only(args) + if args.read_agent_only: + import logging + + from .agent_reading import READ_AGENT + from .reading_entry import run_reading_mode + + # The reading's start line is how its start-up is timed, and nothing else in this process + # sets up logging, so its INFO lines would otherwise be dropped. + reading_log = logging.getLogger("fi.alk.harness.reading_entry") + reading_log.setLevel(logging.INFO) + if not logging.getLogger().handlers: + reading_log.addHandler(logging.StreamHandler()) + return run_reading_mode( + READ_AGENT, + job_path=args.job, + source=args.source.resolve(), + output=args.output.resolve(), + ) + if args.find_agents: + from .agent_scan import FIND_AGENTS + from .reading_entry import run_reading_mode + + return run_reading_mode( + FIND_AGENTS, + job_path=args.job, + source=args.source.resolve(), + output=args.output.resolve(), + ) job = HarnessJob.model_validate(json.loads(args.job.read_text(encoding="utf-8"))) + if refuse_unusable_folder( + args.source.resolve(), job.source.subfolder, args.output.resolve() + ): + return EXIT_REFUSED + from .seed_contract import SEED_ACCEPTED + + # Cleared before the seed checks: a mark an earlier command left must not pass for this one's yes. + (args.output / SEED_ACCEPTED).unlink(missing_ok=True) + if args.seed_contract is not None: + if args.await_harness: + from .seed_contract import accept_seed + + refused = accept_seed(args.seed_contract, args.output) + if refused is not None: + return refused + else: + print( + "seed contract: not used; only a build the conversation's harness runs takes one", + file=sys.stderr, + ) broker_socket = initialize_hosted_compose(job, args.source.resolve()) if broker_socket is not None: os.environ["ALK_COMPOSE_BROKER_SOCKET"] = str(broker_socket) diff --git a/src/fi/alk/harness/authoring_runtime_validation.py b/src/fi/alk/harness/authoring_runtime_validation.py index 9b2d17137..354749683 100644 --- a/src/fi/alk/harness/authoring_runtime_validation.py +++ b/src/fi/alk/harness/authoring_runtime_validation.py @@ -938,15 +938,20 @@ async def validate_once( world_isolation = None if generic: from .action_certification import certify_actions + from .bundle_author_v2 import _runtime_component + from .source_folder import contract_folder contract = AgentContract.model_validate_json( (bundle / "contract.json").read_text(encoding="utf-8") ) + folder = contract_folder(contract) tool_report = certify_tool_inventory( contract, bundle, external_provider=provider_owned_tools, - source=source, + # What the agent's install can import: its folder, or the project above + # it, shared code included. A tool outside that is not this agent's. + source=_runtime_component(source.resolve(), folder) if folder else source, ) rejected = { item.tool: item.reason @@ -1422,6 +1427,27 @@ def _invariants_the_ir_still_breaks(world, authoring: Path) -> list[str]: return [] +def _promote_data_store(contract, authoring: Path, candidate_dir: Path) -> int: + """Copy only ``data_store`` from a candidate reading onto the saved contract, and save it. + + Returns 0 when it was promoted, 1 when the candidate has none to give. + """ + from .contract import AgentContract, write_contract + + candidate_path = candidate_dir / "contract.json" + if not candidate_path.is_file(): + return 1 + candidate = AgentContract.model_validate_json( + candidate_path.read_text(encoding="utf-8") + ) + if candidate.data_store is None or not candidate.data_store.kind.strip(): + return 1 + payload = contract.model_dump() + payload["data_store"] = candidate.data_store.model_dump() + write_contract(AgentContract.model_validate(payload), authoring) + return 0 + + async def validate_and_repair( job, source: Path, @@ -1504,24 +1530,11 @@ async def repair(phase, guidance): ], ) ) - candidate_path = Path(candidate_dir) / "contract.json" - if status != 0 or not candidate_path.is_file(): - return 1 - candidate = AgentContract.model_validate_json( - candidate_path.read_text(encoding="utf-8") - ) - if ( - candidate.data_store is None - or not candidate.data_store.kind.strip() - ): + if status != 0: return 1 - payload = contract.model_dump() - payload["data_store"] = candidate.data_store.model_dump() - repaired = AgentContract.model_validate(payload) - contract_path.write_text( - repaired.model_dump_json(indent=2) + "\n", encoding="utf-8" + return _promote_data_store( + contract, authoring, Path(candidate_dir) ) - return 0 if generic and _needs_runtime_plan_repair(_active_repair_diagnostics): from .contract import AgentContract from .runtime_repair import request_runtime_plan_patch diff --git a/src/fi/alk/harness/builder.py b/src/fi/alk/harness/builder.py index 56461f651..55d579355 100644 --- a/src/fi/alk/harness/builder.py +++ b/src/fi/alk/harness/builder.py @@ -11,7 +11,7 @@ import json from collections.abc import Callable -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any from .backends import ( @@ -27,6 +27,7 @@ from .config import SKILLS_ROOT, chosen_model, discovered_skills, load_skill from .contract import AgentContract from .scenario import Scenario, voicemail_enabled +from .source_folder import contract_folder from .scenario_tools import SCENARIO_SERVER, load_scenarios, scenario_tools, world_summary from .scenarios import writer_worker from .session import Stage @@ -279,6 +280,7 @@ async def change_world(args: dict[str, Any]) -> dict[str, Any]: async def declare_tool(args: dict[str, Any]) -> dict[str, Any]: from .amend import _persist from .contract import ToolSpec, validate_contract + from .source_folder import agent_scope, in_agent_scope if contract is None: return _err("there is no contract loaded to add a tool to") @@ -290,6 +292,15 @@ async def declare_tool(args: dict[str, Any]) -> dict[str, Any]: target = (root / str(args.get("evidence_file") or "")).resolve() if source_root is None or root.resolve() not in target.parents or not target.is_file(): return _err("evidence_file must be a file inside the agent's source") + folder = contract_folder(contract) + scope = agent_scope(root, folder) + if scope is not None and not in_agent_scope( + PurePosixPath(target.relative_to(root.resolve()).as_posix()), scope + ): + return _err( + f"evidence_file must be in the agent's folder {folder} or in a folder above it, " + "not in another agent's" + ) if not quote or quote not in target.read_text(encoding="utf-8", errors="replace"): return _err(f"the quoted line is not in {args.get('evidence_file')}; quote it exactly") if not str(args.get("why") or "").strip(): @@ -660,10 +671,17 @@ def open_builder( "tell the person what you found and fixed:\n- " + "\n- ".join(health[:40]) ) ) + folder = contract_folder(contract) where = ( f"\n\nThe environment's files are in {out}: contract.json, sub_goals.json, coverage.json, " "simulator_prompt.md, scenarios/ and the sealed environment-bundle/." + (f" The agent's own source is in {source_root}." if source_root else "") + + ( + f" The agent lives in its folder {folder}; other folders there belong to other " + "agents." + if source_root and folder + else "" + ) ) spec = SessionSpec( system_prompt=( diff --git a/src/fi/alk/harness/bundle_author_v2.py b/src/fi/alk/harness/bundle_author_v2.py index fc4d276af..45aabcb48 100644 --- a/src/fi/alk/harness/bundle_author_v2.py +++ b/src/fi/alk/harness/bundle_author_v2.py @@ -71,7 +71,17 @@ ) from .credentials import assess_configuration, discover_credentials from .secrets import process_secret_environment_names -from .runtime_selection import ComposeSelection, select_runtime +from .runtime_selection import ComposeSelection, bounded_yaml_load, select_runtime +from .agent_type_check import EXIT_REFUSED +from .source_folder import ( + SubfolderNotFound, + agent_folder, + bind_runtime, + recorded_folder, + recorded_root, + refuse_unusable_folder, + saved_runtime, +) from .job import HarnessJob from .job import ProviderExecutionMode, SourceKind from .process_preflight import preflight_bundle @@ -795,7 +805,9 @@ def _documented_ddl_statements(sql: str) -> list[str]: return statements -def _documented_source_schemas(source: Path) -> list[_SourceSchemaArtifact]: +def _documented_source_schemas( + source: Path, folder: str = "" +) -> list[_SourceSchemaArtifact]: """Use the first README's SQL fences without adopting duplicate alternatives. This is source-owned executable evidence, not model-authored schema. The first declaration @@ -803,7 +815,7 @@ def _documented_source_schemas(source: Path) -> list[_SourceSchemaArtifact]: alternative local and cloud recipes that redeclare the same table. """ - for readme in sorted(source.glob("README*.md")): + for readme in sorted((source / folder if folder else source).glob("README*.md")): text = readme.read_text(encoding="utf-8", errors="replace") artifacts: list[_SourceSchemaArtifact] = [] declared_tables: set[str] = set() @@ -854,7 +866,9 @@ def _compose_source_schema_paths( selected = select_runtime( source, runtime or {}, - public_environment=_declared_runtime_environment(source), + public_environment=_declared_runtime_environment( + recorded_root(source, runtime) + ), ) except ValueError as exc: raise BundleAuthorError(str(exc)) from exc @@ -904,8 +918,11 @@ def _source_schema_paths( """ candidates = _compose_source_schema_paths(source, (contract or {}).get("runtime")) + folder = recorded_folder((contract or {}).get("runtime")) for conventional in ("db/schema.sql", "schema.sql"): - path = _contained_source_path(source, conventional) + path = _contained_source_path( + source, f"{folder}/{conventional}" if folder else conventional + ) if path is not None and path.is_file(): candidates.append(path) @@ -946,7 +963,9 @@ def _source_schema_artifacts( ) for path in paths ] - return _documented_source_schemas(source) + return _documented_source_schemas( + source, recorded_folder((contract or {}).get("runtime")) + ) def _only_this_database(action: int, *_args: object) -> int: @@ -1147,6 +1166,9 @@ def _generic_postgres_seed_artifacts( ) +_LARGEST_RUNTIME_MANIFEST = 1024 * 1024 + + def _declared_runtime_environment(source: Path) -> dict[str, str]: """Load public, non-secret process defaults declared by the repository. @@ -1160,7 +1182,12 @@ def _declared_runtime_environment(source: Path) -> dict[str, str]: if not path.is_file(): return {} try: - body = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + # Read, not stat: a link to a file such as /proc/self/pagemap has size 0 and no end. + with path.open("rb") as manifest: + raw = manifest.read(_LARGEST_RUNTIME_MANIFEST + 1) + if len(raw) > _LARGEST_RUNTIME_MANIFEST: + raise BundleAuthorError("runtime_manifest_invalid: alk.yaml is larger than 1 MiB") + body = bounded_yaml_load(raw.decode("utf-8")) or {} except (OSError, yaml.YAMLError) as exc: raise BundleAuthorError(f"runtime_manifest_invalid: {exc}") from exc if not isinstance(body, dict): @@ -1258,6 +1285,7 @@ def _plan_python( version: str = "", requirements_files: list[str] | None = None, context_excludes: tuple[str, ...] = (), + run_folder: Path | None = None, ) -> SourceProcess: if ( run_override @@ -1271,7 +1299,11 @@ def _plan_python( raise BundleAuthorError( f"runtime_language_unsupported: {name} requires a non-Python installer" ) - if not run_override and entry is not None and not (root / entry).is_file(): + if ( + not run_override + and entry is not None + and not ((run_folder or root) / entry).is_file() + ): raise BundleAuthorError(f"runtime_entrypoint_missing: {name}:{entry}") source, component = source.resolve(), root.resolve() root = python_workspace_root(source, component) @@ -1369,10 +1401,31 @@ def _plan_python( run = [".venv/bin/python", *([entry] if entry is not None else [])] commands.append(supplemental_requirements_command(".venv/bin/python", requirements)) commands.append(["uv", "pip", "check", "--python", ".venv/bin/python"]) + run = run_override or run + run_component = member + if run_folder is not None: + run_component = run_folder.resolve().relative_to(root) + # The run command starts in this folder, which has no environment of its own; the + # workspace root's is linked in unless that is where it already starts. Only a build + # command creates that environment. + if commands and run_component not in {Path("."), member}: + commands.append( + [ + "ln", + "-s", + "../" * len(run_component.parts) + ".venv", + (run_component / ".venv").as_posix(), + ] + ) return process.model_copy( - update={"build_commands": commands, "run_command": run_override or run, "source_component": member.as_posix()} + update={ + "build_commands": commands, + "run_command": run, + "source_component": run_component.as_posix(), + } ) + def _python_component_command( *, source: Path, component: Path, process: SourceProcess ) -> SourceProcess: @@ -1578,7 +1631,10 @@ def _docker_python(root: Path) -> str: return f"python{candidate}" -def _dockerfile_run(root: Path) -> list[str] | None: +def _dockerfile_run(root: Path, project: Path | None = None) -> list[str] | None: + # ``project`` is where dependencies are installed when that isn't the folder holding the + # Dockerfile, because the launcher differs between a requirements file and a uv project. + project = project or root dockerfile = root / "Dockerfile" if not dockerfile.is_file(): return None @@ -1605,12 +1661,12 @@ def _dockerfile_run(root: Path) -> list[str] | None: ): raise BundleAuthorError(f"dockerfile_command_invalid: {dockerfile}") if argv[0] == "python": - argv[0] = ".venv/bin/python" if (root / "requirements.txt").is_file() else "uv" + argv[0] = ".venv/bin/python" if (project / "requirements.txt").is_file() else "uv" if argv[0] == "uv": argv[1:1] = ["run", "--no-sync", "python"] elif ( argv[0] in {"uvicorn", "gunicorn", "flask"} - and (root / "requirements.txt").is_file() + and (project / "requirements.txt").is_file() ): argv[0] = f".venv/bin/{argv[0]}" return argv @@ -1784,17 +1840,24 @@ def _subprocess_adapter_source() -> str: ) -def _discovered_python_component(root: Path) -> tuple[Path, str]: +def _discovered_python_component( + root: Path, search: Path | None = None +) -> tuple[Path, str]: + """The project holding the one ``agent.py`` under ``search`` (default ``root``) and the + entry. Under a folder named for the agent, the entry is relative to that folder.""" + base = search or root candidates = sorted( path - for path in root.glob("**/agent.py") + for path in base.glob("**/agent.py") if not any( part in _IGNORED_ARTIFACT_PARTS - for part in path.relative_to(root).parts[:-1] + for part in path.relative_to(base).parts[:-1] ) ) if len(candidates) != 1: raise BundleAuthorError("component_ambiguous: expected exactly one agent.py") + if search is not None: + return _install_root_for(root, search, candidates[0]) component = candidates[0].parent # An entrypoint directory is not necessarily its Python project root. Preserve # the nearest enclosing manifest and its sibling packages instead of flattening @@ -1810,6 +1873,25 @@ def _discovered_python_component(root: Path) -> tuple[Path, str]: return component, candidates[0].relative_to(component).as_posix() +def _install_root_for(root: Path, run_folder: Path, found: Path) -> tuple[Path, str]: + """Where a file found under the run folder installs from, and the file relative to the folder. + + A process can't start above its own project, so a project below the run folder is refused. + """ + for parent in found.parents: + if not parent.is_relative_to(root): + break + if any((parent / name).is_file() for name in ("pyproject.toml", "requirements.txt")): + if parent != run_folder and parent.is_relative_to(run_folder): + where = parent.relative_to(root).as_posix() + raise BundleAuthorError( + f"runtime_workdir_nested_project: {found.relative_to(root).as_posix()} has " + f"its own project at {where}; set runtime.workdir to {where}" + ) + return parent, found.relative_to(run_folder).as_posix() + return run_folder, found.relative_to(run_folder).as_posix() + + def _runtime_component(root: Path, configured_workdir: str) -> Path: """Resolve a source-declared workdir to its nearest installable project root.""" @@ -2194,6 +2276,22 @@ def _compose_environment(service: dict[str, Any]) -> dict[str, str]: return {} +def _agent_scope( + root: Path, subfolder: str | None, runtime: dict[str, Any] | None +) -> tuple[dict[str, Any] | None, Path, Path]: + """The runtime bound to the agent's folder, the folder, and where the process starts. + + With no subfolder nothing is bound and both folders are the repository root. + """ + if not subfolder: + return runtime, root, root + try: + bound = dict(bind_runtime(runtime, root, subfolder) or {}) + return bound, agent_folder(root, subfolder), (root / bound["workdir"]).resolve() + except (ValueError, SubfolderNotFound) as exc: + raise BundleAuthorError(str(exc)) from exc + + def resolve_environment_plan( source: str | Path, job: HarnessJob, @@ -2208,6 +2306,10 @@ def resolve_environment_plan( root = Path(source).resolve() if not root.is_dir(): raise BundleAuthorError(f"source_unavailable: {root}") + subfolder = job.source.subfolder + contract_runtime, agent_root, run_folder = _agent_scope( + root, subfolder, contract_runtime + ) connector = job.agent.connector.lower() if job.agent.mode is ProviderExecutionMode.ENVIRONMENT_BACKED: declaration = load_provider_manifest( @@ -2231,7 +2333,7 @@ def resolve_environment_plan( reference.purpose == SecretPurpose.TARGET_PROVIDER.value for reference in job.agent.secret_refs.values() ) - declared_runtime_environment = _declared_runtime_environment(root) + declared_runtime_environment = _declared_runtime_environment(agent_root) try: runtime_model = Runtime.model_validate( { @@ -2363,7 +2465,13 @@ def resolve_environment_plan( # The repository's exec-form Dockerfile CMD is an explicit source-owned declaration; use it # before falling back to a guessed agent.py entrypoint. This works for any HTTP framework. if not runtime_command and interface.get("kind") == "http" and not compose: - runtime_command = _dockerfile_run(root) or [] + runtime_command = ( + _dockerfile_run( + run_folder, + project=_runtime_component(root, runtime_workdir) if subfolder else None, + ) + or [] + ) interface_port = interface.get("port") interface_health_path = str(interface.get("health_path") or "/health") @@ -2669,10 +2777,10 @@ def resolve_environment_plan( discovered_callback = ( None if is_livekit or (runtime_model.execution == "native" and runtime_command) - else _discover_callback_entrypoint(root) + else _discover_callback_entrypoint(run_folder) ) graph_entrypoint = ( - _langgraph_entrypoint(root) + _langgraph_entrypoint(run_folder) if not is_livekit and contract_modality == "chat" and not runtime_command else None ) @@ -2708,7 +2816,8 @@ def resolve_environment_plan( ( item for item in runtime_command - if item.endswith(".py") and (component / item).is_file() + if item.endswith(".py") + and ((run_folder if subfolder else component) / item).is_file() ), "agent.py", ) @@ -2718,16 +2827,20 @@ def resolve_environment_plan( and not runtime_command and contract_interface_kind == "http" and runtime_workdir.strip() + # The folder the agent is named for is the default workdir, not a chosen one. + and not (subfolder and run_folder == agent_root) ): raise BundleAuthorError( "runtime_http_command_missing: set runtime.command to the source-declared " "native HTTP server command and runtime.workdir to its working directory; " "an internal agent.py is not evidence of an HTTP server" ) - elif not is_callback and not (root / entry).is_file(): - component, entry = _discovered_python_component(root) + elif not is_callback and not (run_folder / entry).is_file(): + component, entry = _discovered_python_component( + root, search=run_folder if subfolder else None + ) else: - component = root + component = _runtime_component(root, runtime_workdir) if subfolder else root control_name = "agent" port = None if is_livekit else int(interface_port or 8080) environment = ( @@ -2746,8 +2859,13 @@ def resolve_environment_plan( if port is not None and not is_livekit: environment["PORT"] = f"{{{{PORT_{control_name}}}}}" callback_entrypoint = ( - discovered_callback or _callback_entrypoint(root) if is_callback else None + discovered_callback or _callback_entrypoint(run_folder) + if is_callback + else None ) + if subfolder and callback_entrypoint: + module = callback_entrypoint.partition(":")[0].replace(".", "/") + _install_root_for(root, run_folder, run_folder / f"{module}.py") if is_graph: environment.update( { @@ -2768,7 +2886,12 @@ def resolve_environment_plan( root, language=runtime_model.language, name=control_name, - root=root if is_callback else component, + # Bun walks up to its own package root, so it starts from the run folder. + root=( + (run_folder if is_bun_runtime else component) + if subfolder + else root if is_callback else component + ), entry=None if is_callback or is_command_adapter or is_graph else entry, control=True, needs_secrets=needs_target_secrets, @@ -2783,8 +2906,14 @@ def resolve_environment_plan( run_override=( None if is_callback or is_command_adapter or is_graph - else runtime_command or _dockerfile_run(component) + else runtime_command + or ( + _dockerfile_run(run_folder, project=component) + if subfolder + else _dockerfile_run(component) + ) ), + run_folder=run_folder if subfolder and not is_bun_runtime else None, ) if is_graph: python_command = process.run_command @@ -2845,7 +2974,7 @@ def resolve_environment_plan( } # Only a Dockerfile CMD carries the subcommand today, so a repository without one # starts `agent.py` bare and never reaches the registration this check waits for. - if _hands_off_to_livekit_cli(component, entry) and not ( + if _hands_off_to_livekit_cli(run_folder if subfolder else component, entry) and not ( set(process.run_command) & _LIVEKIT_WORKER_SUBCOMMANDS ): update["run_command"] = [*process.run_command, "start"] @@ -2857,7 +2986,9 @@ def resolve_environment_plan( if not is_bun_runtime: process = _python_component_command( source=root, - component=root if is_callback else component, + component=( + run_folder if subfolder else root if is_callback else component + ), process=process, ) resources = {} @@ -2919,13 +3050,13 @@ def resolve_environment_plan( ) packaging = ( "dockerfile" - if (root / "Dockerfile").is_file() + if (run_folder / "Dockerfile").is_file() else "generated_bun" if is_bun_runtime else "generated_python" ) - toolbox = _mcp_toolbox_definition(root) + toolbox = _mcp_toolbox_definition(agent_root) if toolbox is not None and control_name: processes.append(_mcp_toolbox_process()) rewritten: list[ManagedProcess | SourceProcess] = [] @@ -3167,6 +3298,9 @@ def exists_from(root: Path, path: str) -> bool: workspace_required = [ (prefix / path).as_posix() for path in map(Path, required) ] + if exclusion_runtime.source_subfolder: + # The prefix is the run folder, not the folder the dependencies install from. + workspace_required.append("requirements.txt") exclusions = _context_excludes( component, workspace_runtime, @@ -3427,6 +3561,8 @@ def author_bundle_v2( contract_interface_kind: str | None = None contract_body: dict[str, Any] = {} runtime: dict[str, Any] = {} + subfolder = job.source.subfolder + agent_root = run_folder = source_root contract_path = authoring_root / "contract.json" if contract_path.is_file(): try: @@ -3440,6 +3576,11 @@ def author_bundle_v2( contract_modality = str(contract_body.get("modality") or "").strip().lower() runtime = contract_body.get("runtime") runtime = runtime if isinstance(runtime, dict) else {} + if subfolder: + runtime, agent_root, run_folder = _agent_scope( + source_root, subfolder, runtime + ) + contract_body = {**contract_body, "runtime": runtime} interface = runtime.get("interface") if isinstance(runtime, dict) else None if isinstance(interface, dict): contract_interface_kind = str(interface.get("kind") or "").strip().lower() @@ -3457,7 +3598,7 @@ def author_bundle_v2( contract_interface_kind = "http" elif ( contract_modality == "chat" - and _discover_callback_entrypoint(source_root) is not None + and _discover_callback_entrypoint(run_folder) is not None ): # The callback is a deterministic source property. Do not let a stochastic # authoring omission make the compiled adapter unreachable at call time: the @@ -3477,7 +3618,7 @@ def author_bundle_v2( contract_modality == "chat" and not isinstance(interface, dict) and not runtime.get("command") - and _langgraph_entrypoint(source_root) is not None + and _langgraph_entrypoint(run_folder) is not None ): runtime = dict(runtime) runtime["interface"] = { @@ -3514,7 +3655,7 @@ def author_bundle_v2( contract_modality == "chat" and contract_interface_kind == "callable" and runtime.get("command") - and _discover_callback_entrypoint(source_root) is None + and _discover_callback_entrypoint(run_folder) is None ): raise BundleAuthorError( "generated_runtime_entrypoint_ambiguous: callable interface requires a " @@ -3559,7 +3700,7 @@ def author_bundle_v2( str(name).upper() for name in (job.metadata.get("environment_value_names", []) or []) } - provided_environment.update(_declared_runtime_environment(source_root)) + provided_environment.update(_declared_runtime_environment(agent_root)) for process in plan.processes: provided_environment.update( str(name).upper() for name in (getattr(process, "environment", None) or {}) @@ -3568,14 +3709,26 @@ def author_bundle_v2( source_root, secret_refs=job.agent.secret_refs, provided_environment=provided_environment, + # What ships: each process's working directory, as with no folder (a uv workspace member + # ships from the workspace root, with the shared code it imports), and with a folder its + # install root too. agent_subfolder still says which keys are the agent's own. scan_paths={ - str(getattr(process, "working_directory", ".") or ".") - for process in plan.processes - if isinstance(process, SourceProcess) + *( + str(getattr(process, "working_directory", ".") or ".") + for process in plan.processes + if isinstance(process, SourceProcess) + ), + *( + (_runtime_component(source_root, subfolder).relative_to(source_root).as_posix(),) + if subfolder + else () + ), }, # Match preflight: a template documents possible integrations, not every # credential needed by the selected generic-runtime path. template_secrets_required=job.metadata.get("generic_harness_v1") is not True, + agent_subfolder=subfolder or None, + agent_compose_file=str(runtime.get("compose_file") or "") or None, ) output_root.parent.mkdir(parents=True, exist_ok=True) temporary = Path( @@ -3611,7 +3764,7 @@ def author_bundle_v2( generated / "proxy.py", ) if any(process.name == "mcp-toolbox" for process in plan.processes): - toolbox = _mcp_toolbox_definition(source_root) + toolbox = _mcp_toolbox_definition(agent_root) if toolbox is None: raise BundleAuthorError("mcp_toolbox_definition_missing") _, definition = toolbox @@ -4304,9 +4457,17 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--authoring", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args(argv) + job = _load_job(args.job) + if refuse_unusable_folder( + args.source.resolve(), + job.source.subfolder, + args.authoring.resolve(), + saved_runtime(args.authoring), + ): + return EXIT_REFUSED author_bundle_v2( source=args.source, - job=_load_job(args.job), + job=job, authoring=args.authoring, output=args.output, ) diff --git a/src/fi/alk/harness/chat_call_runner.py b/src/fi/alk/harness/chat_call_runner.py index 81afc9515..cd3945eae 100644 --- a/src/fi/alk/harness/chat_call_runner.py +++ b/src/fi/alk/harness/chat_call_runner.py @@ -29,6 +29,7 @@ from .process_runtime import EnvironmentRuntime from .run.conversation import TargetConversationEnded, Transcript, converse from .scenario import Scenario as ConversationScenario +from .source_folder import contract_folder from .usage import ( UsageDenied, UsageUnavailable, @@ -203,7 +204,10 @@ def _tool_world( world.handlers[tool.name] = path.read_text(encoding="utf-8") world.refusal_signature = contract.refusal_signature if source_directory is not None: - world.reach(str(source_directory)) + world.reach( + str(source_directory), + contract_folder(contract), + ) return world diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index 957a215fd..9c3ba928c 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -15,9 +15,11 @@ import sys import time import uuid +from collections.abc import Iterable from pathlib import Path from typing import Any +from ..providers.agent import _redact from . import spend from . import observability from .build import open_stage as build_stage @@ -39,7 +41,7 @@ from .session import TEXT, Event from .sessions import Session, new_id, save as save_session from .sources import resolve, supported -from .understand import load, open_stage, opening +from .understand import SUBMIT_REMINDER, load, open_stage, opening, reading_settings from .world.snapshot import saved as world_saved @@ -61,7 +63,9 @@ def _source_root(destination: Path, explicit: str = "") -> str: def _render(event: Event) -> None: - line = event.line() + # The platform keeps the tail of this output, so a URL's password, query and fragment are + # dropped first, in one call per event, as emit_model_event drops them from the job's events. + line = _redact(event.line()) if event.kind == TEXT: print(line, end="", flush=True) else: @@ -108,7 +112,11 @@ def receive(event: Event) -> None: emitted.append(event.text) await stage.say(content, on_event=receive) - text = "".join(emitted).strip() or "I’m still working on the active authoring stage." + # Sent to the platform's chat as the hosted builder's replies are, so cleaned the same way. + text = ( + _redact("".join(emitted).strip()) + or "I’m still working on the active authoring stage." + ) await client.emit( "assistant_message", message_id=str(uuid.uuid4()), @@ -168,11 +176,12 @@ async def _ask_operator(_tool_name: str, payload: dict[str, Any], _context: Any) def _guidance(args: argparse.Namespace) -> str: - instructions = [ - str(item).strip() - for item in (getattr(args, "guidance", None) or []) - if str(item).strip() - ] + return guidance_text(getattr(args, "guidance", None) or []) + + +def guidance_text(guidance: Iterable[Any]) -> str: + """A job's guidance as every stage that reads it is told it; empty when there is none.""" + instructions = [str(item).strip() for item in guidance if str(item).strip()] if not instructions: return "" return ( @@ -272,6 +281,12 @@ def _report_failure_file(output: Path) -> None: async def _understand(args: argparse.Namespace) -> int: + job = getattr(args, "job", None) + subfolder = ( + job.source.subfolder + if job is not None and getattr(job, "source", None) is not None + else None + ) if args.kind == "provider": source = resolve( args.kind, @@ -280,25 +295,14 @@ async def _understand(args: argparse.Namespace) -> int: scratch=args.path, ) else: - source = resolve(args.kind, name=args.name, root=args.path) - job = getattr(args, "job", None) - offered = ( - (getattr(job, "metadata", None) or {}).get("available_evals") if job else None - ) - generic = bool( - job is not None - and isinstance(getattr(job, "metadata", None), dict) - and job.metadata.get("generic_harness_v1") is True - ) + source = resolve(args.kind, name=args.name, root=args.path, subfolder=subfolder) stage, destination = open_stage( source, out=Path(args.out) if args.out else None, # Unattended, there is nobody to answer, so the model records what it could not # resolve in open_questions rather than blocking on a prompt nobody will see. ask=permission_gate(_ask_operator) if args.interactive else None, - available_evals=offered if isinstance(offered, list) else None, - require_setup_contract=generic and not args.interactive, - max_turns=140 if generic and not args.interactive else 70, + **reading_settings(job, interactive=args.interactive), ) print(f"agent: {source.name} ({source.kind})") @@ -315,10 +319,7 @@ async def _understand(args: argparse.Namespace) -> int: interactive=args.interactive, until=lambda: load(destination) is not None or (destination / FAILURE_FILE).is_file(), - nudge=( - "Nothing was saved: you finished without calling submit_contract. Call it now " - "with the contract you worked out." - ), + nudge=SUBMIT_REMINDER, ) if (destination / FAILURE_FILE).is_file(): @@ -1021,12 +1022,15 @@ def emit_model_event(event: Event) -> None: for key in ("target", "label", "call_id", "is_error", "path", "outcome", "cost_usd") if key in detail } + # The job response carries these, so a URL's password or query is dropped first, in + # one call, before the text is cut. + text, safe_detail = _redact([event.text or None, safe_detail]) emit( "harness.activity", str(detail.get("stage") or "authoring"), event_kind=event.kind, tool=event.tool or None, - text=event.text[:2000] if event.text else None, + text=text[:2000] if text else None, detail=safe_detail, ) diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index f126173d7..0b12ea2b4 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -11,19 +11,25 @@ from __future__ import annotations +import collections.abc import json +import os import re import shlex +import threading +import typing from pathlib import Path from typing import Any, Literal -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, Field, field_validator, model_serializer, model_validator +from pydantic.json_schema import SkipJsonSchema from fi.simulate.agent.wrappers.http_events import ( CompletedToolEvents, CompletedToolRows, ) +from ..providers.agent import _redact from .observations import HttpObservation # How a person reaches an agent. This decides how it is later run — voice goes out as a live @@ -207,6 +213,21 @@ class PromptSource(BaseModel): end_line: int +class ModelSource(BaseModel): + """The model the agent talks to the person with, and the line its name is written on.""" + + provider: str = "" + name: str = "" + path: str = "" + line: int = 0 + + @model_validator(mode="before") + @classmethod + def _bare_name(cls, value: Any) -> Any: + # A bare string is the name alone; the save then asks for path and line. + return {"name": value} if isinstance(value, str) else value + + class DataStore(BaseModel): """What the agent's tools read and write, and how to be there instead of it. @@ -823,6 +844,18 @@ class Runtime(BaseModel): # repository. interface: RuntimeInterface | None = None + # The folder of the repository this agent lives in. The harness records it when the + # contract is saved, so it is kept out of the schema the model writes against and out of + # the dump when empty. + source_subfolder: SkipJsonSchema[str] = "" + + @model_serializer(mode="wrap") + def _omit_empty_subfolder(self, handler: Any) -> dict[str, Any]: + data = handler(self) + if not data.get("source_subfolder"): + data.pop("source_subfolder", None) + return data + @model_validator(mode="after") def _validate_selection(self) -> "Runtime": from .world.files import validate_bindings @@ -1056,6 +1089,20 @@ def _reached(one: Dependency) -> str: return "; ".join(said) +def prompt_quote(payload: Any) -> str: + """The quote of the prompt as AgentContract reads it from a payload: under its own name or + the first alias that has one, a list joined by lines. The save holds it before validating.""" + if not isinstance(payload, dict): + return "" + for key in ("system_prompt_excerpt", *_ALIASES["system_prompt_excerpt"]): + value = payload.get(key) + if value: + if isinstance(value, list): + return "\n".join(str(item) for item in value) + return value if isinstance(value, str) else str(value) + return "" + + class AgentContract(BaseModel): """What the agent verifiably is. Nothing downstream may contradict this.""" @@ -1114,6 +1161,12 @@ def _normalize_shapes(cls, payload: Any) -> Any: system_prompt_excerpt: str = "" # When given, the excerpt is replaced by these lines copied from the source as written. system_prompt_from: list[PromptSource] = Field(default_factory=list) + # Set by the save, never by the reading: lines named for the prompt that twice didn't hold + # the quote given for it. The quote went out instead, or, when the quote was too short to + # stand for the prompt, the lines did, unconfirmed. Empty for nearly every contract. + unconfirmed_prompt_from: list[PromptSource] = Field(default_factory=list) + # None is unknown. Kept only when its name was found on the line cited in the source. + model: ModelSource | None = None hard_constraints: list[str] = Field(default_factory=list) tools: list[ToolSpec] = Field(default_factory=list) data_schema: dict[str, Any] = Field(default_factory=dict) @@ -1148,6 +1201,18 @@ def _normalize_shapes(cls, payload: Any) -> Any: amendments: list[str] = Field(default_factory=list) instrumentation: str = "none" + @model_serializer(mode="wrap") + def _omit_absent_model(self, handler: Any) -> dict[str, Any]: + # A contract with no model writes the same bytes as before the field existed, so its + # setup fingerprint is unchanged. + data = handler(self) + if data.get("model") is None: + data.pop("model", None) + # The same for the record of unconfirmed prompt lines, kept only when there are some. + if not data.get("unconfirmed_prompt_from"): + data.pop("unconfirmed_prompt_from", None) + return data + def tool_names(self) -> set[str]: return {tool.name for tool in self.tools} @@ -1332,6 +1397,222 @@ def adoptable(self, tool: str) -> bool: return bool(found and found.mode in ("import", "construct", "service")) +# Which parts of the contract are prose, written by the reader in its own words, and which are +# read as written by the build, the world or the runtime. Prose is cleaned of URL credentials +# (userinfo, query and fragment) whenever the contract is saved; the rest is never changed. +# A test walks every model the contract reaches and fails on a field in neither tuple. +FREE_TEXT = ( + "AgentContract.one_liner", + "AgentContract.system_prompt_excerpt", + "AgentContract.hard_constraints", + "AgentContract.implementation", + "AgentContract.real_use_cases", + "AgentContract.caller_identity", + "AgentContract.notes", + "AgentContract.open_questions", + "AgentContract.amendments", + "ToolSpec.description", + "ToolSpec.refuses", + "ToolSpec.effects", + "ToolEntry.notes", + "Dependency.what", +) +# A field that holds models is kept for itself, and its models' fields are classified in their +# own right. A name without a dot is a whole technical model, every field of it kept. +KEPT_AS_WRITTEN = ( + "AgentContract.agent", + "AgentContract.modality", + "AgentContract.call_direction", + "AgentContract.agent_languages", + "AgentContract.conversational", + "AgentContract.system_prompt_from", + "AgentContract.unconfirmed_prompt_from", + # The model name the code cites, with its file and line: read as written, like the prompt lines. + "AgentContract.model", + "AgentContract.tools", + "AgentContract.data_schema", + "AgentContract.base_environment", + "AgentContract.dependencies", + "AgentContract.runtime_dependencies", + "AgentContract.tool_entrypoints", + # world/runtime.py matches its quoted markers against tool results, so cleaning it would + # change what counts as a refusal. + "AgentContract.refusal_signature", + "AgentContract.data_store", + "AgentContract.runtime", + "AgentContract.chosen_evals", + "AgentContract.instrumentation", + "ToolSpec.name", + "ToolSpec.args", + "ToolSpec.arg_types", + # The world answers with these as written. + "ToolSpec.arg_values", + "ToolSpec.requires", + "ToolSpec.inputs_from", + "ToolEntry.tool", + "ToolEntry.mode", + "ToolEntry.module", + "ToolEntry.callable", + "ToolEntry.factory", + "ToolEntry.first_arg", + "ToolEntry.service", + # The tool proxy calls it as written. + "ToolEntry.endpoint", + "ToolEntry.method", + "Dependency.name", + "Dependency.kind", + "Dependency.used_by", + "Dependency.engine", + "Dependency.version", + "Dependency.reached", + "PromptSource", + "ModelSource", + "Reached", + "DataStore", + "Runtime", + "HttpObservation", + "ObservationParameter", + "RuntimeSecretFile", + "RuntimeConfigFile", + "RuntimeInterface", + "CompletedToolEvents", + "CompletedToolFailureEvent", + "CompletedToolRows", + "LocalHttpAuth", + "RuntimeHttpFailureRule", + "RuntimeHttpSetupRequest", +) +CONTRACT_FILE = "contract.json" +_SEQUENCES = ( + list, + tuple, + set, + frozenset, + collections.abc.Sequence, + collections.abc.MutableSequence, +) +_MAPPINGS = (dict, collections.abc.Mapping, collections.abc.MutableMapping) + + +def _walks_as_sequence(origin: Any) -> bool: + """Whether _holders walks a field of this origin as a sequence: any subclass of those above + (an abc registration counts), but never text.""" + return ( + isinstance(origin, type) + and issubclass(origin, _SEQUENCES) + and not issubclass(origin, (str, bytes, bytearray)) + ) + + +def _walks_as_mapping(origin: Any) -> bool: + """Whether _holders walks a field of this origin as a mapping: any subclass of those above, + such as an OrderedDict.""" + return isinstance(origin, type) and issubclass(origin, _MAPPINGS) + + +def _holders(contract: AgentContract, body: dict[str, Any], model: str) -> list[tuple[Any, Any]]: + """Each (saved form, held object) of one free-text model, wherever the contract holds it. + + The walk follows each field's type, so a model is found in a field of any shape: a list, a + tuple or a Sequence, a dict's or a Mapping's values (a subclass of each, such as an + OrderedDict, too), an Optional or a Union, or inside another model, and an empty one is + passed over. A model kept whole is not looked inside. The contract itself is + ``AgentContract``. + """ + if model == "AgentContract": + return [(body, contract)] + pairs: list[tuple[Any, Any]] = [] + seen: set[int] = set() + + def walk(annotation: Any, saved: Any, held: Any) -> None: + if saved is None or held is None: + return + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + if isinstance(held, BaseModel): + if not isinstance(held, annotation): + return + annotation = type(held) + if annotation.__name__ in KEPT_AS_WRITTEN or not isinstance(saved, dict): + return + if annotation.__name__ == model and id(saved) not in seen: + seen.add(id(saved)) + pairs.append((saved, held)) + for key, field in annotation.model_fields.items(): + inner = held.get(key) if isinstance(held, dict) else getattr(held, key, None) + walk(field.annotation, saved.get(key), inner) + return + origin, arguments = typing.get_origin(annotation), typing.get_args(annotation) + if _walks_as_sequence(origin): + # model_dump keeps a tuple a tuple. A tuple[A, B] has a type for each place. + if arguments and isinstance(saved, (list, tuple)) and isinstance(held, (list, tuple)): + fixed = origin is tuple and not (len(arguments) == 2 and arguments[1] is Ellipsis) + for index, (saved_item, held_item) in enumerate(zip(saved, held)): + if not fixed or index < len(arguments): + walk(arguments[index if fixed else 0], saved_item, held_item) + elif _walks_as_mapping(origin): + if len(arguments) == 2 and isinstance(saved, dict) and isinstance(held, dict): + for key, held_item in held.items(): + walk(arguments[1], saved.get(key), held_item) + else: + # Optional, Union and Annotated: each type it may be. + for argument in arguments: + walk(argument, saved, held) + + walk(type(contract), body, contract) + return pairs + + +def write_contract(contract: AgentContract, destination: Path) -> AgentContract: + """Save the contract to ``destination/contract.json``, its free text cleaned of URL credentials. + + Every writer of the contract goes through here, so no saved contract holds a URL's password, + query or fragment in its prose. The file is written beside its place, synced and moved in, + so a write that fails partway leaves the earlier contract as it was. Only then are the + cleaned values copied back onto ``contract`` itself, in place: the stage and its tools hold + that object, and a fresh one would leave them working from the raw text. It is returned too. + """ + body = contract.model_dump() + places: list[tuple[Any, Any, Any, str]] = [] + for name in FREE_TEXT: + model, field = name.split(".") + for saved, held in _holders(contract, body, model): + if isinstance(saved, dict) and isinstance(saved.get(field), (str, list)): + places.append((saved, held, saved[field], field)) + # One call for the whole save, over a list with no keys: only the URL rule applies, and a + # refused URL is logged once per save rather than once per string. + values = [value for _, _, value, _ in places] + cleaned = _redact(values) + for (saved, _, _, field), value in zip(places, cleaned): + saved[field] = value + + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=True) + path = destination / CONTRACT_FILE + partial = destination / f".{CONTRACT_FILE}.{os.getpid()}.{threading.get_ident()}" + try: + partial.write_text(json.dumps(body, indent=2, ensure_ascii=False), encoding="utf-8") + # Synced before the move, so the move never publishes a file the disk does not hold. + descriptor = os.open(partial, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(partial, path) + finally: + partial.unlink(missing_ok=True) + + for saved, held, _, field in places: + value = saved[field] + current = held.get(field) if isinstance(held, dict) else getattr(held, field, None) + if isinstance(current, list) and isinstance(value, list): + current[:] = value + elif isinstance(held, dict): + held[field] = value + else: + setattr(held, field, value) + return contract + + def runtime_owns_tools(contract: AgentContract) -> bool: """Whether a submitted voice worker is itself the runnable seam for every tool. diff --git a/src/fi/alk/harness/credentials.py b/src/fi/alk/harness/credentials.py index 4529aac28..1c11305d3 100644 --- a/src/fi/alk/harness/credentials.py +++ b/src/fi/alk/harness/credentials.py @@ -8,19 +8,29 @@ from __future__ import annotations import ast +import fnmatch import hashlib import json +import logging +import math +import os import re +import stat +from collections import deque from enum import Enum -from pathlib import Path -from typing import AbstractSet, Iterable, Mapping, TYPE_CHECKING, Literal +from pathlib import Path, PurePosixPath +from typing import AbstractSet, Callable, Iterable, Iterator, Mapping, TYPE_CHECKING, Literal from pydantic import BaseModel, Field import yaml from fi.simulate.runtime.spec import SecretRef +from .python_source import EscapeBudget, ParseBudget, decoded_first, parsed, utf8_size from .secrets import process_secret_environment_names +from .source_folder import SubfolderNotFound, agent_folder, agent_scope, in_agent_scope, is_within + +_LOG = logging.getLogger(__name__) if TYPE_CHECKING: from .contract import Runtime @@ -136,6 +146,30 @@ def ready(self) -> bool: } _MAX_FILES = 2_000 _MAX_FILE_BYTES = 1_000_000 +# The import walk's whole budget: a file limit alone still allows minutes of reading. +_MAX_WALK_BYTES = 32_000_000 +_MAX_WALK_IMPORTS = 20_000 +# How many folders below the repository the import walk follows. Honest repositories nest a +# dozen or so; resolving a path costs a lookup for each of its folders, and Linux allows 2,000. +_DEEPEST_FOLLOWED = 64 +# Linux's PATH_MAX: an import name longer than this can't name a file. +_LONGEST_NAME = 4096 +_SCRIPT_SUFFIXES = (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs") +_SCRIPT_IMPORT = re.compile( + r"""(?:\bfrom\s*|\bimport\s*(?:\(\s*)?|\brequire\s*\(\s*)["']([^"'\n]+)["']""" +) +# An import, or a whole string: matched from the left, a string holding import text is taken +# from its opening quote, before the words in it can be read as an import. +_SCRIPT_IMPORT_OR_STRING = re.compile( + _SCRIPT_IMPORT.pattern + + r"""|"(?:\\.|[^"\\\n])*"?|'(?:\\.|[^'\\\n])*'?|`(?:\\.|[^`\\])*`?""" +) +# A type-only import runs nothing. The keyword is matched alone, and the quotes after it found +# by `_without_type_imports`, so the text is scanned once, however many keywords it holds. +_SCRIPT_TYPE_KEYWORD = re.compile(r"\b(?:import|export)\s+type\b") +_SCRIPT_QUOTE_OR_LINE_END = re.compile(r"""["'\n]""") +# How far after the keyword the module's quote may start. +_SCRIPT_TYPE_REACH = 1000 _ENV_DECLARATION = re.compile( r"(?m)^(?:export[ \t]+)?([A-Z][A-Z0-9_]{2,})[ \t]*=[ \t]*([^\r\n#]*)" @@ -411,8 +445,16 @@ def discover_credentials( provided_environment: Iterable[str] = (), scan_paths: Iterable[str | Path] | None = None, template_secrets_required: bool = True, + agent_subfolder: str | None = None, + agent_compose_file: str | None = None, ) -> CredentialManifest: - """Inspect declarations and environment reads without executing submitted code.""" + """Inspect declarations and environment reads without executing submitted code. + + With ``agent_subfolder``, only that agent's own files can require a value: its folder, the + folders above it, and the code it imports. Anything else shipped alongside it is another + agent's, so what it reads is listed as optional. A Compose file above the folder belongs to + the whole repository, so it counts only when ``agent_compose_file`` names it. + """ root = Path(root).expanduser().resolve() if not root.is_dir(): raise ValueError(f"credential_source_missing: {root}") @@ -428,8 +470,29 @@ def discover_credentials( scanned = 0 truncated = False digest = hashlib.sha256() - - for path in _candidate_files(root, scan_paths=scan_paths): + walk_stopped = [False] + imported = ( + imported_by_agent(root, agent_subfolder, stopped=walk_stopped) + if agent_subfolder + else frozenset() + ) + if walk_stopped[0]: + # Keys in the files it didn't reach count as optional, so say the scan was cut short. + truncated = True + _LOG.warning("credential import walk stopped at its limits in %s", agent_subfolder) + scope = agent_scope(root, agent_subfolder) if agent_subfolder else None + named_compose = (root / agent_compose_file).resolve() if agent_compose_file else None + + candidates = _candidate_files(root, scan_paths=scan_paths) + if scope is not None: + # The file limit must not leave the agent's own files unread. + candidates.sort( + key=lambda path: not ( + path in imported + or in_agent_scope(PurePosixPath(path.relative_to(root).as_posix()), scope) + ) + ) + for path in candidates: if scanned >= _MAX_FILES: truncated = True break @@ -448,8 +511,15 @@ def discover_credentials( digest.update(relative.encode()) digest.update(hashlib.sha256(content.encode()).digest()) lowered = content.lower() + agents_own = ( + scope is None + or path in imported + or in_agent_scope(PurePosixPath(relative), scope) + ) + if scope is not None and path.name.lower() in _COMPOSE_NAMES: + agents_own = path == named_compose or PurePosixPath(relative).is_relative_to(scope) for connector, signals in _CONNECTOR_SIGNALS.items(): - if any(signal in lowered for signal in signals): + if agents_own and any(signal in lowered for signal in signals): connector_hits.add(connector) def record( @@ -469,9 +539,9 @@ def record( "detected_from": set(), }, ) - item["required"] = bool(item["required"]) or required - item["declared_default"] = ( - bool(item["declared_default"]) or declared_default + item["required"] = bool(item["required"]) or (required and agents_own) + item["declared_default"] = bool(item["declared_default"]) or ( + declared_default and agents_own ) detected = item["detected_from"] assert isinstance(detected, set) @@ -501,7 +571,8 @@ def record( declared_default=usable_default, ) if path.name.lower() in _COMPOSE_NAMES: - configured.update(_literal_compose_environment(content)) + if agents_own: + configured.update(_literal_compose_environment(content)) for match in _COMPOSE_VARIABLE.finditer(content): operator, fallback = match.group(2), (match.group(3) or "").strip() # Compose substitutes an unset plain ${NAME} with an empty string. Only its @@ -600,6 +671,997 @@ def record( ) +def _file_inside( + candidate: Path, + root: Path, + deepest: int, + left: list[float] | None = None, + unread: list[bool] | None = None, +) -> Path | None: + """``candidate`` resolved, if it's a plain file in ``root`` at most ``deepest`` parts long. + + ``left`` is the lookups on disk still allowed: one for the file, then one for each part of + its path, which resolving it looks up in turn. Given ``unread``, its one item is set when a + candidate is refused for its depth, or as a code file too big to read or a link to one: code + may be there that the walk doesn't follow. + """ + if len(candidate.parts) > deepest: + _mark(unread) + return None + if left is not None: + left[0] -= 1 + if not _plain_file(candidate, unread): + return None + if left is not None: + left[0] -= len(candidate.parts) + resolved = candidate.resolve() + if len(resolved.parts) > deepest: + _mark(unread) + return None + return resolved if is_within(resolved, root) else None + + +def _mark(unread: list[bool] | None) -> None: + if unread is not None: + unread[0] = True + + +# A uv workspace's members read for their names, at most. +_WORKSPACE_MEMBERS = 256 +# The members the agent depends on, at most: each is two more places every Python import is +# looked for. +_WORKSPACE_DEPENDENCIES = 32 +# The largest pyproject.toml read, and what one walk reads of them in all: a crafted one parses +# in about 0.4 s a MB. +_LARGEST_PYPROJECT = 256 * 1024 +_PYPROJECTS_READ = 1024 * 1024 +# The folder entries a member pattern's globs look at, at most. +_WORKSPACE_ENTRIES = 20_000 +# The places the member walk reads for what they install, at most: a member reached again with +# more extras is read again. +_WORKSPACE_READS = 4 * _WORKSPACE_DEPENDENCIES +# The longest folder name a file system holds, in bytes. +_LONGEST_FOLDER_NAME = 255 +_PROJECT_NAME = re.compile(r"\s*([A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?)") + + +def _project_name(value: object) -> str | None: + """A distribution's name as uv compares it, from a name or a requirement string.""" + found = _PROJECT_NAME.match(value) if isinstance(value, str) else None + return _normal_name(found.group(1)) if found else None + + +def _normal_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +class _Workspace: + """What one walk reads of a uv workspace: every lookup charged to ``left``, the + pyproject.toml files read under one total, and its folders found through ``known``.""" + + def __init__( + self, root: Path, left: list[float], known: dict[tuple[str, ...], Path | None] + ) -> None: + self.root, self.left, self.known = root, left, known + self.read = 0 + self.cut = False + + def spent(self) -> bool: + if self.left[0] <= 0: + self.cut = True + return self.left[0] <= 0 + + def table(self, folder: Path) -> dict | None: + """``folder``'s pyproject.toml, parsed, or None.""" + if self.spent(): + return None + self.left[0] -= 1 + document = folder / "pyproject.toml" + try: + mode = document.lstat().st_mode + if stat.S_ISLNK(mode): + # One link, to a file inside the root, found by the charged walk: the kernel would + # follow it anywhere, uncharged. + self.left[0] -= 1 + target = folder / os.readlink(document) + place = _real_folder(target.parent, self.root, self.known, self.left) + if place is None: + return None + document = place / target.name + self.left[0] -= 1 + mode = document.lstat().st_mode + if not stat.S_ISREG(mode): + return None + with document.open("rb") as handle: + raw = handle.read(_LARGEST_PYPROJECT + 1) + except (OSError, ValueError): + return None + if len(raw) > _LARGEST_PYPROJECT or self.read + len(raw) > _PYPROJECTS_READ: + self.cut = True + return None + self.read += len(raw) + try: + import tomllib + + table = tomllib.loads(raw.decode("utf-8-sig")) + except (ValueError, UnicodeError, RecursionError, ImportError): + return None + return table if isinstance(table, dict) else None + + def expand(self, workspace: Path, patterns: object) -> list[Path]: + """The folders ``patterns`` match, as uv globs them, one level at a time.""" + matched: list[Path] = [] + entries = 0 + for pattern in patterns if isinstance(patterns, list) else []: + # An absolute pattern or one with "..", out of the root, is found nowhere by + # _real_folder, which looks nothing up there. + parts = PurePosixPath(pattern).parts if isinstance(pattern, str) else () + if not parts: + continue + if "**" in parts: + self.cut = True + continue + if any(len(part) > _LONGEST_FOLDER_NAME for part in parts): + # Not compiled: fnmatch takes a second for 250 KB. A literal part that long names + # no folder; a pattern may still match one, so the walk says it was cut. + self.cut = self.cut or any( + len(part) > _LONGEST_FOLDER_NAME and any(mark in part for mark in "*?[") + for part in parts + ) + continue + level = [workspace] + for part in parts: + below: list[Path] = [] + for base in level: + if self.left[0] <= 0 or entries >= _WORKSPACE_ENTRIES: + self.cut = True + break + # Each base looked at counts, a memoised one too: a link back to its own + # folder makes a long pattern visit the same bases again and again. + entries += 1 + if not any(mark in part for mark in "*?["): + if (found := _real_folder(base / part, self.root, self.known, self.left)): + below.append(found) + continue + try: + with os.scandir(base) as listing: + names = [] + for entry in listing: + entries += 1 + self.left[0] -= 1 + if self.left[0] <= 0 or entries >= _WORKSPACE_ENTRIES: + self.cut = True + break + if fnmatch.fnmatchcase(entry.name, part): + names.append(entry.name) + except OSError: + continue + # A member may be a link, as uv follows it. + for name in sorted(names): + if self.spent(): + break + if (found := _real_folder(base / name, self.root, self.known, self.left)): + below.append(found) + level = sorted(set(below)) + matched.extend(level) + return matched + + +# PEP 508: a requirement's one extras bracket comes straight after its name. +_REQUESTED_EXTRAS = re.compile(r"\s*\[([^\]]*)\]") + + +def _groups(groups: dict, chosen: list) -> list[object]: + """The requirements of the dependency groups ``chosen``, each ``{include-group = ...}`` in + them taken in turn, a group once.""" + found: list[object] = [] + seen: set[str] = set() + pending = [group for group in reversed(chosen) if group != "dev"] + while pending: + group = pending.pop() + if group in seen or not isinstance(groups.get(group), list): + continue + seen.add(group) + for item in groups[group]: + if isinstance(item, dict) and isinstance(item.get("include-group"), str): + pending.append(item["include-group"]) + else: + found.append(item) + return found + + +class _Requirements: + """What ``table``'s project installs with it under the build's ``uv sync --no-dev --package``, + indexed once: its dependencies and, for the agent's own project, its default groups other than + ``dev`` (``plain``), and each extra's requirements by its name (``by_extra``). Reading it again + for more extras reads only theirs: a member reached again with one more extra each time would + otherwise read its whole list each time.""" + + def __init__(self, table: dict, own: bool) -> None: + project = table.get("project") if isinstance(table.get("project"), dict) else {} + dependencies = project.get("dependencies") + self.plain: list[object] = list(dependencies) if isinstance(dependencies, list) else [] + self.by_extra: dict[str, list[object]] = {} + optional = project.get("optional-dependencies") + for key, values in optional.items() if isinstance(optional, dict) else (): + if (name := _project_name(key)) and isinstance(values, list): + self.by_extra.setdefault(name, []).extend(values) + if own: + uv = _uv_table(table) + groups = table.get("dependency-groups") + groups = groups if isinstance(groups, dict) else {} + default = uv.get("default-groups", ["dev"]) + chosen = ( + list(groups) if default == "all" else default if isinstance(default, list) else [] + ) + names = [group for group in chosen if isinstance(group, str)] + self.plain.extend(_groups(groups, names)) + + def asked(self, plain: bool, extras: Iterable[str]) -> dict[str, set[str]]: + """The distributions these requirements install, by name, each with the extras asked of + it: the plain ones if ``plain``, and those of ``extras``.""" + found = list(self.plain) if plain else [] + for extra in extras: + found.extend(self.by_extra.get(extra, ())) + asked: dict[str, set[str]] = {} + for value in found: + if not isinstance(value, str) or not (named := _PROJECT_NAME.match(value)): + continue + wanted = asked.setdefault(_normal_name(named.group(1)), set()) + # Matched once, where the name ends: a search tried every "[" in turn, the square of + # the requirement's length (150 KB held the save 51 s). + if listed := _REQUESTED_EXTRAS.match(value, named.end()): + wanted |= { + extra for part in listed.group(1).split(",") if (extra := _project_name(part)) + } + return asked + + +def _uv_table(table: dict) -> dict: + tool = table.get("tool") if isinstance(table.get("tool"), dict) else {} + return tool.get("uv") if isinstance(tool.get("uv"), dict) else {} + + +def _sources(table: dict) -> dict[str, list[dict]]: + """``table``'s ``[tool.uv.sources]``, by normalised name, each a list of its source tables + (uv also takes a list of them, each with its own marker).""" + sources = _uv_table(table).get("sources") + found: dict[str, list[dict]] = {} + for key, value in sources.items() if isinstance(sources, dict) else (): + items = value if isinstance(value, list) else [value] + kept = [item for item in items if isinstance(item, dict)] + if (name := _project_name(key)) and kept: + found[name] = kept + return found + + +def _member_name(table: dict) -> str | None: + """The project name a pyproject.toml declares; None when it declares none, or not as a + table.""" + project = table.get("project") + return _project_name(project.get("name")) if isinstance(project, dict) else None + + +def _workspace_of(table: dict) -> dict | None: + workspace = _uv_table(table).get("workspace") + return workspace if isinstance(workspace, dict) else None + + +def _workspace_dependencies( + root: Path, folder: Path, left: list[float], known: dict[tuple[str, ...], Path | None] +) -> tuple[Path | None, list[Path], bool]: + """The agent's project folder, the folders of the local packages installed with it, and + whether some weren't looked for. + + The project is the nearest pyproject.toml at or above ``folder``. The build installs it with + what it asks for (``_Requirements``); a requirement whose source is the uv workspace + (``{workspace = true}``, in the project or the workspace root, the nearest pyproject.toml + above the project that names it a member) is that member's folder, one whose source is a + ``path`` inside the repository is that folder, and what each of those asks for is followed + in turn. Another member, a sibling agent among them, isn't installed, so it isn't looked in. + A ``**`` member pattern isn't expanded, and past the limits the rest isn't read: both say so. + A member past ``_WORKSPACE_MEMBERS`` is still read when a requirement names its folder. + """ + reader = _Workspace(root, left, known) + project: tuple[Path, dict] | None = None + workspace: tuple[Path, dict] | None = None + for place in (folder, *folder.parents): + if not is_within(place, root) or reader.cut: + break + if (table := reader.table(place)) is None: + continue + if project is None: + project = (place, table) + if _workspace_of(table) is not None: + workspace = (place, table) + break + if project is None: + return None, [], reader.cut + tables: dict[Path, dict] = {project[0]: project[1]} + by_name: dict[str, Path] = {} + shared: dict[str, list[dict]] = {} + # The members past the cut, by their folder's name: one a requirement names is read then. + late: dict[str, list[Path]] = {} + if workspace is not None: + home, top = workspace + shape = _workspace_of(top) or {} + excluded = set(reader.expand(home, shape.get("exclude", []))) + members = [ + member for member in dict.fromkeys(reader.expand(home, shape.get("members", []))) + if member not in excluded + ] + listed = set(members) + if len(members) > _WORKSPACE_MEMBERS: + for member in members[_WORKSPACE_MEMBERS:]: + if name := _project_name(member.name): + late.setdefault(name, []).append(member) + members, reader.cut = members[:_WORKSPACE_MEMBERS], True + # The project must be the workspace's root or one of its members: uv runs any other alone. + if project[0] == home or project[0] in listed: + tables[home] = top + shared = _sources(top) + for member in (home, *members): + if member not in tables and (table := reader.table(member)) is not None: + tables[member] = table + if name := _member_name(tables.get(member, {})): + by_name.setdefault(name, member) + + # Each place's sources, worked out once, each with the folder that declares it: a requirement + # list and a sources table of thousands each would otherwise cost their product. + merged: dict[Path, dict[str, tuple[Path, list[dict]]]] = {} + members_by_place = set(by_name.values()) + + late_read = [0] + + def member_named(name: str) -> Path | None: + """The member called ``name``: one already read, or one past the cut whose folder has + that name, read until one is found. A member whose folder is named otherwise stays + unread, as the cut says, and so does any past ``_WORKSPACE_DEPENDENCIES`` such reads.""" + if name in by_name: + return by_name[name] + for member in late.pop(name, []): + if late_read[0] == _WORKSPACE_DEPENDENCIES: + reader.cut = True + break + late_read[0] += 1 + if member not in tables and (table := reader.table(member)) is not None: + tables[member] = table + if _member_name(tables.get(member, {})) == name: + by_name[name] = member + members_by_place.add(member) + break + return by_name.get(name) + + def installed(place: Path, table: dict, name: str) -> Path | None: + """The folder of the local package ``name``, as ``place``'s project gets it, or None.""" + sources = merged.get(place) + if sources is None: + sources = {} + if workspace is not None and (place == project[0] or place in members_by_place): + # The workspace root's sources apply to every member, under the member's own; a + # path in one is the root's, as uv reads it. + sources = {key: (workspace[0], items) for key, items in shared.items()} + sources.update({key: (place, items) for key, items in _sources(table).items()}) + merged[place] = sources + declared, items = sources.get(name, (place, [])) + for source in items: + if source.get("workspace") is True: + return member_named(name) + if isinstance(source.get("path"), str): + target = declared / source["path"] + found = _real_folder(target, root, known, left) + if found is not None and found not in tables: + if (read := reader.table(found)) is not None: + tables[found] = read + return found if found in tables else None + return None + + followed: list[Path] = [] + # The extras asked of each member so far: one reached again with more is read again for them + # alone. + asked: dict[Path, set[str]] = {} + indexed: dict[Path, _Requirements] = {} + # The agent's own project installs all its extras: which the build asks for isn't known here. + own = _Requirements(project[1], own=True) + indexed[project[0]] = own + queue: list[tuple[Path, bool, set[str]]] = [(project[0], True, set(own.by_extra))] + reads = 0 + while queue and left[0] > 0: + if reads == _WORKSPACE_READS: + reader.cut = True + break + reads += 1 + place, plain, extras = queue.pop(0) + table = tables.get(place, {}) + if place not in indexed: + indexed[place] = _Requirements(table, own=False) + requested = indexed[place].asked(plain, extras) + for name in sorted(requested): + member = installed(place, table, name) + if member is None or member == project[0]: + continue + wanted = requested[name] + if member in asked: + if more := wanted - asked[member]: + asked[member] |= more + queue.append((member, False, more)) + continue + if len(followed) == _WORKSPACE_DEPENDENCIES: + reader.cut = True + break + followed.append(member) + asked[member] = set(wanted) + queue.append((member, True, set(wanted))) + return project[0], followed, reader.cut + + +def _top_names(base: Path, left: list[float], limit: int) -> tuple[set[str], bool]: + """The names an import can start with in ``base``: its folders and its ``.py`` files' stems, + each entry charged to ``left``; past ``limit`` entries, whether it was cut.""" + names: set[str] = set() + try: + with os.scandir(base) as listing: + for count, entry in enumerate(listing): + left[0] -= 1 + if count >= limit or left[0] <= 0: + return names, True + stem, suffix = os.path.splitext(entry.name) + if suffix == ".py" and stem.isidentifier(): + names.add(stem) + elif not suffix and entry.name.isidentifier(): + names.add(entry.name) + except OSError: + pass + return names, False + + +# Links one chain may lead through, as the kernel allows about as many in one path. +_LINKS_FOLLOWED = 40 + + +def _real_folder( + folder: Path, + root: Path, + known: dict[tuple[str, ...], Path | None], + left: list[float], + unread: list[bool] | None = None, +) -> Path | None: + """``folder`` with each link in it followed, if it's a folder inside ``root``; else None. + + Each name below ``root`` is looked up once a walk, charged one lookup to ``left``; a link is + charged one more, and one for each part of its target, which is walked a part at a time from + the real folder it's in (``_followed``): the kernel follows the links of a path uncharged, + and a chain of them held the save 11.6 s. A ``..`` is taken from the folder reached so far, + which has no link in it, so it costs nothing; above ``root`` nothing is looked up, and a path + that stays out gives None. So do a missing folder, a loop and a chain of links past + ``_LINKS_FOLLOWED``; what that limit cut isn't kept, since the same link reached by a shorter + chain may be followed. A link whose target is past ``_LONGEST_TARGET`` parts isn't followed + either, though the kernel would follow it: given ``unread``, its one item says so. + """ + # Each name here is reached first, so what the chain limit cuts below it is cut for any way to + # it, and can be kept; a link's target is walked by _step, which keeps only what it didn't cut. + # The second item: a link target refused for its length. + capped = [False, False] + parts, top = folder.parts, len(root.parts) + # A path longer than a link target may be (a uv ``path`` source is written by the customer) + # isn't walked: each of its prefixes is a key, so its length would cost its square. + if len(parts) > top + _LONGEST_TARGET or parts[:top] != root.parts: + return None + # From the deepest folder already known (most candidates share their folder with the last), + # each one below it looked up in turn, stopping at the first that isn't there: a crafted name + # of dozens of parts costs the folders up to that one. + known_end = len(parts) + while known_end > top and parts[:known_end] not in known: + known_end -= 1 + real: Path | None = root if known_end == top else known[parts[:known_end]] + for end in range(known_end + 1, len(parts) + 1): + if real is None: + # Kept for the whole path too: each candidate in this folder (a name has 13, in two + # folders) would otherwise search back down to the missing one again, uncharged. + known[parts] = None + break + name = parts[end - 1] + if name == "..": + real = real.parent + elif not is_within(real, root): + # Above the root, as written: the way back in is by the same names. + real = real / name + else: + real = _step(real / name, root, known, left, 0, capped) + known[parts[:end]] = real + if capped[1]: + _mark(unread) + return real if real is not None and is_within(real, root) else None + + +def _step( + name: Path, + root: Path, + known: dict[tuple[str, ...], Path | None], + left: list[float], + depth: int, + capped: list[bool], +) -> Path | None: + """``name``, in a folder inside ``root`` with no link in it, as a folder, kept by its real + path for the walk unless the chain limit cut it.""" + key = name.parts + if key in known: + return known[key] + # Marked first, so a link back to it reads as a loop. + known[key] = None + found = _followed(name, root, known, left, depth, capped) + if capped[0]: + del known[key] + else: + known[key] = found + return found + + +# The longest link target followed, in parts: an honest one is a few parts, and each part of a +# crafted one is a step to look up. +_LONGEST_TARGET = 2 * _DEEPEST_FOLLOWED + + +def _followed( + place: Path, + root: Path, + known: dict[tuple[str, ...], Path | None], + left: list[float], + depth: int, + capped: list[bool], +) -> Path | None: + """``place``, a name in a folder with no link in it, as a folder: itself, or its link's + target followed a part at a time, if that's a folder inside ``root``.""" + left[0] -= 1 + try: + mode = place.lstat().st_mode + except (OSError, ValueError): + return None + if stat.S_ISDIR(mode): + return place + if not stat.S_ISLNK(mode): + return None + if depth >= _LINKS_FOLLOWED: + capped[0] = True + return None + left[0] -= 1 + try: + target = Path(os.readlink(place)) + except (OSError, ValueError): + return None + if len(target.parts) > _LONGEST_TARGET: + capped[1] = True + return None + # Each part is a step on the way, charged as a lookup is. + left[0] -= len(target.parts) + real = Path(target.anchor) if target.is_absolute() else place.parent + for part in target.parts[1:] if target.is_absolute() else target.parts: + if part == "..": + real = real.parent + elif not is_within(real, root): + # Above the root: nothing is looked up there. + real = real / part + else: + found = _step(real / part, root, known, left, depth + 1, capped) + if found is None: + return None + real = found + # Out of the root, it's refused where it's used (_real_folder). + return real + + +# The folder links in the agent's folder resolved to see where they lead, at most. +_FOLDER_LINKS_RESOLVED = 32 + + +def _code_in( + folder: Path, root: Path | None = None, unread: list[bool] | None = None +) -> Iterator[Path]: + """The Python and script files in ``folder``, outside the folders no one's code is in. + + Those folders aren't entered, and each file costs its name: building a parent for each part, + as 3.12 does to make a path relative, cost a crafted folder of 31,000 nested packages 2.8 s. + A folder link isn't entered either; given ``root`` and ``unread``, one that leads inside + ``root`` sets ``unread``, since code the agent runs may be there, unless it leads back into + ``folder`` where this walk reads its target by its own path. Each link resolved costs its + chain uncharged, so past ``_FOLDER_LINKS_RESOLVED`` of them ``unread`` is set without looking. + """ + suffixes = {".py", *_SCRIPT_SUFFIXES} + resolved = 0 + for directory, subdirectories, files in os.walk(folder): + subdirectories[:] = [name for name in subdirectories if name not in _IGNORED_PARTS] + if unread is not None and root is not None and not unread[0]: + for name in subdirectories: + place = os.path.join(directory, name) + if not os.path.islink(place): + continue + resolved += 1 + if resolved > _FOLDER_LINKS_RESOLVED: + unread[0] = True + break + real = Path(os.path.realpath(place)) + if is_within(real, root) and not _read_in_place(real, folder): + unread[0] = True + break + for name in files: + if name not in _IGNORED_PARTS and os.path.splitext(name)[1] in suffixes: + yield Path(directory, name) + + +def _read_in_place(real: Path, folder: Path) -> bool: + """Whether ``_code_in`` reads ``real`` by its own path: inside ``folder``, with no part + below it that the walk skips. Parts are compared case-folded: on a case-insensitive disk a + link can spell a skipped folder in other letters.""" + return is_within(real, folder) and not any( + part.casefold() in _IGNORED_PARTS for part in real.parts[len(folder.parts) :] + ) + + +def _plain_file(path: Path, unread: list[bool] | None = None) -> bool: + """A regular file small enough to read: a committed link can point at anything, so it isn't + followed. Given ``unread``, its one item is set when a code file is refused for its size or + for being a link, since Python and Node run what one holds or points at. A dangling link is + marked too: telling it apart would follow its chain uncharged.""" + try: + details = path.lstat() + except (OSError, ValueError): + return False + if stat.S_ISREG(details.st_mode) and details.st_size <= _MAX_FILE_BYTES: + return True + if (stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode)) and ( + path.suffix == ".py" or path.suffix in _SCRIPT_SUFFIXES + ): + _mark(unread) + return False + + +def _python_imports( + content: str, limit: int = _MAX_FILE_BYTES, budget: ParseBudget | None = None +) -> list[tuple[int, str]] | None: + """The names ``content`` imports, each with its level; None when it isn't parsed: over + ``limit``, past ``budget``, or not Python the parser reads.""" + # In bytes, as the file's size is: the parser's cost follows them, not the characters. + if utf8_size(content) > limit or (budget is not None and not budget.spend(content)): + return None + tree = parsed(content) + if tree is None: + return None + names: list[tuple[int, str]] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names.extend((0, alias.name) for alias in node.names) + elif isinstance(node, ast.ImportFrom): + module = node.module or "" + names.append((node.level, module)) + names.extend( + (node.level, f"{module}.{alias.name}".lstrip(".")) for alias in node.names + ) + return [(level, name) for level, name in names if name] + + +def _script_imports(content: str) -> list[str]: + # Here because agent_type_check imports this module. + from .agent_type_check import _SCRIPT_COMMENT_OR_STRING, _script_kept + + code = _without_type_imports(_SCRIPT_COMMENT_OR_STRING.sub(_script_kept, content)) + return [hit.group(1) for hit in _SCRIPT_IMPORT_OR_STRING.finditer(code) if hit.group(1)] + + +def _without_type_imports(code: str) -> str: + """``code`` without its type-only imports: the keyword, then within 1,000 characters on its + line a quoted module name. + + One regular expression doing this rescans the reach after every keyword, which a file of + nothing but keywords turns into seconds a megabyte, holding the interpreter throughout. Here + the searches for opening quotes and those for closing ones each start where the last of + their kind did or later, and each kind keeps its own answer, so the code is scanned at most + twice over. + """ + kept: list[str] = [] + done = 0 + opening_after = _next_quote_or_line_end(code) + closing_after = _next_quote_or_line_end(code) + for keyword in _SCRIPT_TYPE_KEYWORD.finditer(code): + if keyword.start() < done: + continue + opening = opening_after(keyword.end()) + if opening == len(code) or code[opening] == "\n": + continue + if opening - keyword.end() > _SCRIPT_TYPE_REACH: + continue + closing = closing_after(opening + 1) + if closing == len(code) or code[closing] == "\n": + continue + kept.append(code[done : keyword.start()]) + done = closing + 1 + kept.append(code[done:]) + return "".join(kept) + + +def _next_quote_or_line_end(code: str) -> Callable[[int], int]: + """Where the first quote or line end at or after a position is, or the code's end. + + Asked from positions that only grow, it scans each stretch of ``code`` once. + """ + searched_from = stop = -1 + + def after(position: int) -> int: + # No quote or line end lies between searched_from and stop, so the answer stands. + nonlocal searched_from, stop + if not searched_from <= position <= stop: + hit = _SCRIPT_QUOTE_OR_LINE_END.search(code, position) + searched_from, stop = position, hit.start() if hit else len(code) + return stop + + return after + + +def imported_by_agent( + root: Path, + subfolder: str, + *, + parse_limit: int = _MAX_FILE_BYTES, + parse_budget: ParseBudget | None = None, + script_budget: ParseBudget | None = None, + import_limit: int | None = None, + lookup_limit: int | None = None, + stopped: list[bool] | None = None, +) -> frozenset[Path]: + """The files the agent's folder imports, followed as far as they go. + + Python imports, and JavaScript or TypeScript imports written as a path. A package name that + only a workspace or a path alias can resolve isn't followed. A conversation waiting on the + answer passes tighter parse limits than bundling's: a Python file is followed only if it's + at most ``parse_limit`` bytes and the walk has parsed under ``parse_budget``, and a script + only while the walk has matched under ``script_budget``. It tries at most ``import_limit`` + import names in all (bundling's 20,000 when not given), and makes at most ``lookup_limit`` + lookups on disk (no limit when not given): a file found costs one for each part of its path, + and one name can resolve a package for each of its parts. Nothing more than + ``_DEEPEST_FOLLOWED`` folders below the repository is followed. Given ``stopped``, its one + item says whether the walk left code it reached unfollowed: a limit stopped it with files or + names left, a name was dropped for its length or depth, or a code file was too big to read, + or a link. A reached ``.py`` holding ``import`` that the parser doesn't read in the reading + Python runs counts too, whatever the reason: a syntax error, newer syntax, or a template, since + its imports can't be known. + """ + root = Path(root).resolve() + try: + folder = agent_folder(root, subfolder) + except SubfolderNotFound: + return frozenset() + deepest = len(root.parts) + _DEEPEST_FOLLOWED + # Each package and each file found is looked up once in the whole walk: resolving one costs a + # lookup for each folder in it, and a chain of packages names the same ones from every file. + # Other misses aren't kept: a script name tries a few dozen places, which 20,000 names would + # fill gigabytes with. A package is looked up only above a file found, so they stay few. + looked_up: dict[Path, Path | None] = {} + # A name is tried once, but a crafted chain of packages makes it resolve dozens of paths, each + # dozens of parts deep: 1,000 names took the save 12 s. What is looked up again costs nothing. + lookups_left = [math.inf if lookup_limit is None else float(lookup_limit)] + # The uv workspace members the agent's project depends on are installed with it, so an import + # names one by its package: each one's folder and its src/ are looked + # in too. + project, members, members_cut = _workspace_dependencies(root, folder, lookups_left, {}) + # The agent's own project, when the folder is below it, is installed too, src/ layout or not. + own = () if project in (None, folder) else (project, project / "src") + python_bases = (folder, folder / "src", root, root / "src", *own) + # Each member is looked in only for a name that starts as one of its packages or modules: + # every place is a few lookups for every name, and a name from the standard library or an + # index is in none of them. + member_bases: dict[str, list[Path]] = {} + for member in members: + for base in (member / "src", member): + names, cut_short = _top_names(base, lookups_left, _WORKSPACE_ENTRIES) + members_cut |= cut_short + for top in names: + member_bases.setdefault(top, []).append(base) + # At the save, a candidate's folders are looked up one by one, each once a walk, and a folder + # link is followed to its target inside the repository, charged: the kernel and resolve() + # follow every link in a path uncharged, and a chain of them took 2,300 imports 11.6 s. So the + # save reaches what bundling, which resolves as the kernel does, reaches. + real_folders: dict[tuple[str, ...], Path | None] | None = None if lookup_limit is None else {} + # Whether the walk left code it reached unfollowed: a limit, a name dropped for its length or + # depth, or a file it couldn't read or parse. + unread = [members_cut] + + def found_at(candidates: Iterable[Path]) -> Path | None: + for candidate in candidates: + if candidate in looked_up: + hit = looked_up[candidate] + elif lookups_left[0] <= 0: + # A name left untried: the walk didn't reach all it could. + unread[0] = True + return None + else: + place = candidate + if ( + real_folders is not None + # The depth cut first: a script's import can name about 2,000 folders. + and len(candidate.parts) <= deepest + ): + # Through its folders' links, each followed once and charged. + folder_found = _real_folder( + candidate.parent, root, real_folders, lookups_left, unread + ) + place = None if folder_found is None else folder_found / candidate.name + if place is None: + lookups_left[0] -= 1 + hit = None + else: + hit = _file_inside(place, root, deepest, lookups_left, unread) + if hit is not None or candidate.name == "__init__.py": + looked_up[candidate] = hit + if hit is not None: + return hit + return None + + def python_modules(path: Path, level: int, name: str) -> list[Path]: + module = name.split(".") + # A path past the system's limit can't be opened, so it isn't built. + if len(module) > _DEEPEST_FOLLOWED or len(name) > _LONGEST_NAME: + unread[0] = True + return [] + starts = ( + (path.parents[level - 1],) + if level + else (*python_bases, *member_bases.get(module[0], ())) + ) + for start in starts: + if lookups_left[0] <= 0: + # Spent: each place left would be refused uncharged. + unread[0] = True + return [] + if len(start.parts) + len(module) > deepest: + # Past the depth cut wherever it's looked for: not built at all. + unread[0] = True + continue + target = start.joinpath(*module) + if hit := found_at((target.with_suffix(".py"), target / "__init__.py")): + # Importing a module runs every package above it first. + packages: list[Path] = [] + directory = start + for part in module[:-1]: + directory = directory / part + if package := found_at((directory / "__init__.py",)): + packages.append(package) + return [hit, *packages] + return [] + + def script_module(path: Path, name: str) -> Path | None: + if len(name) > _LONGEST_NAME: + unread[0] = True + return None + for start in (path.parent,) if name.startswith(".") else (folder, root): + # Joined as written, as Node joins it before it reads the disk: a ".." after a folder + # link leaves the link, not its target. + target = Path(os.path.normpath(start / name)) + written = [target.with_suffix(suffix) for suffix in (".ts", ".tsx")] + if hit := found_at( + ( + target, + *(written if target.suffix == ".js" else ()), + *(Path(f"{target}{suffix}") for suffix in _SCRIPT_SUFFIXES), + *(target / f"index{suffix}" for suffix in _SCRIPT_SUFFIXES), + ) + ): + return hit + return None + + # One total for the whole walk: bundling has no lookup limit and no parse total, so this is + # all that bounds what decoding first costs there, a unicode-escape file's second reading + # included (at most this much more parsing per walk). + escapes = EscapeBudget() + seen: set[Path] = set() + pending = deque(sorted(_code_in(folder, root, unread))) + own = set(pending) # Read once already, so importing one doesn't queue it again. + read = bytes_read = imports_tried = 0 + import_limit = _MAX_WALK_IMPORTS if import_limit is None else import_limit + while ( + pending + and read < _MAX_FILES + and bytes_read < _MAX_WALK_BYTES + and imports_tried < import_limit + and lookups_left[0] > 0 + ): + # The folder's own files first, so the budgets can't spend themselves on a chain. + path = pending.popleft() + if len(path.parts) > deepest: + unread[0] = True + continue + if not _plain_file(path, unread): + continue + read += 1 + try: + with path.open("rb") as handle: + raw = handle.read(_MAX_FILE_BYTES + 1) + except OSError: + continue + if len(raw) > _MAX_FILE_BYTES: + unread[0] = True + continue # Grew since it was looked at. + bytes_read += len(raw) + # A .py in a codec that can spell a name in other bytes is read as Python reads it while + # the walk's total lasts: decoded after its line ends, so hz's `~` before a CRLF joins. + taken, decoded, encoding = ( + decoded_first(raw, escapes) if path.suffix == ".py" else (False, None, None) + ) + read_before = None + if taken: + if decoded is None: + # Python refuses the file, so it imports nothing. + continue + content = decoded.replace("\r\n", "\n").replace("\r", "\n") + if encoding is None and "\r" in decoded: + # unicode-escape is also read as UTF-8, where `\r` is two characters: decoded, it's + # a CR the parser ends a comment or a string at where Python reads on, so what that + # reading imports is followed too. + read_before = raw.decode("utf-8-sig", errors="replace") + read_before = read_before.replace("\r\n", "\n").replace("\r", "\n") + else: + # As read_text reads it, line ends included; a .py in the encoding its coding line + # names, as Python reads it. + encoding = encoding or "utf-8-sig" + try: + # Strictly: replacing runs Python's own error handler for each byte in most + # codecs, 60 ms a megabyte, where UTF-8's is in C. Python itself refuses such a + # file. + text = raw.decode(encoding) + except (UnicodeError, LookupError): + # Read as UTF-8, as is a codec refused by name (and unicode-escape past the + # total): a crafted file Python reads in one of those can then give fewer imports + # here than it does to Python. + text = raw.decode("utf-8-sig", errors="replace") + content = text.replace("\r\n", "\n").replace("\r", "\n") + hits: list[Path] = [] + if path.suffix == ".py": + # A file with no import in it can't lead anywhere, so it doesn't use up the parse total + # before the files that do; a file the total cuts says so. + if "import" not in content: + continue + readings = [content] if read_before is None else [content, read_before] + # Each reading is charged as it's parsed: a file parsed twice and charged once let a + # crafted pair cost the walk twice its total. + results = [_python_imports(reading, parse_limit, parse_budget) for reading in readings] + # The decoded reading is the one Python runs: refused, the walk left code unfollowed, + # whatever the other reading gives. A refusal of the other alone says nothing. + unread[0] |= results[0] is None + unread[0] |= parse_budget is not None and parse_budget.left < 0 + names = list(dict.fromkeys(name for found in results if found for name in found)) + # One file can hold more names than the walk may try; the rest aren't tried. + unread[0] |= len(names) > import_limit - imports_tried + names = names[: import_limit - imports_tried] + imports_tried += len(names) + for level, name in names: + try: + hits.extend(python_modules(path, level, name)) + except (IndexError, ValueError, OSError): + continue # An import that can't be a path in this repository. + elif script_budget is not None and not script_budget.spend(content): + unread[0] = True + continue + else: + names = list(dict.fromkeys(_script_imports(content))) + unread[0] |= len(names) > import_limit - imports_tried + names = names[: import_limit - imports_tried] + imports_tried += len(names) + for name in names: + try: + hits.extend(filter(None, (script_module(path, name),))) + except (ValueError, OSError): + continue + for hit in hits: + if hit not in seen: + seen.add(hit) + if hit not in own: + pending.append(hit) + # A file in a codec decoded first, read without decoding once the total ran out. + unread[0] |= escapes.left < 0 + if stopped is not None: + # The loop ends with files left only when a limit ended it. + stopped[:] = [unread[0] or bool(pending)] + return frozenset(seen) + + def _candidate_files( root: Path, *, scan_paths: Iterable[str | Path] | None = None ) -> list[Path]: diff --git a/src/fi/alk/harness/framework_detection.py b/src/fi/alk/harness/framework_detection.py new file mode 100644 index 000000000..9273a8e9b --- /dev/null +++ b/src/fi/alk/harness/framework_detection.py @@ -0,0 +1,1073 @@ +"""Names the framework an agent's source is built on, from its files alone. + +The answer is a label: a wrong name costs little, a lost document or a stuck run +costs a lot. So nothing here runs or imports customer code, and any failure ends +in `unknown`. Files are read with `tokenize`, because it stays bounded on files a +parser would choke on and still reads what follows a syntax error. +""" + +from __future__ import annotations + +import glob +import io +import json +import logging +import os +import re +import stat +import tokenize +import warnings +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterator, Literal + +from .instrumentation_evidence import _SKIPPED_DIRECTORIES +from .source_folder import agent_folder + +logger = logging.getLogger(__name__) + +_MAX_LINE_BYTES = 65_536 +_MAX_TOTAL_BYTES = 67_108_864 +_MAX_FILES = 5_000 +_METADATA_BYTES = 65_536 +_VERSION_MAX = 64 +_MAX_WORK = 100_000_000_000 +_WORK_PER_TARGET = 2_000 +_MAX_ALIASES = 10_000 +# A backslash-continued dotted name has no length limit of its own, and no real +# name comes near this. +_MAX_DOTTED_PARTS = 64 +# What os.walk holds for the folders it has yet to visit grows with a tree that holds no Python, +# which none of the limits above sees. Each folder kept to walk into is counted, never refunded, at +# what its full path costs as a str: past this total the walk stops (TH-9511). +_MAX_FOLDER_BYTES = 67_108_864 +# The folder being checked counts as at least this many characters, so that while its own path is +# ASCII and no longer, every caller counts the same tree the same, wherever it sits on disk. +_TOP_FLOOR = 184 + +_FILE_SKIPPED = "file_skipped" +_FILE_UNREADABLE = "file_unreadable" +_AGENTS_AMBIGUOUS = "agents_ambiguous" +_SIDE_FOLDER_FRAMEWORK = "side_folder_framework" +_OUTSIDE_IMPORT = "outside_import" +_OTHER_SOURCE = "other_source" +_NO_PYTHON = "no_python" + +# These three are common names for a customer's own package below the top. +_TOP_ONLY_SKIPS = frozenset({"build", "dist", "env"}) +_ALWAYS_SKIPPED = frozenset(_SKIPPED_DIRECTORIES) - _TOP_ONLY_SKIPS +_SIDE_FOLDERS = frozenset( + { + "tests", + "test", + "examples", + "example", + "samples", + "sample", + "demos", + "demo", + "docs", + "doc", + "evals", + "eval", + "scripts", + "legacy", + "notebooks", + "vendor", + "vendored", + "third_party", + "external", + "contrib", + "experiments", + "benchmarks", + "archive", + } +) +_OTHER_SOURCE_SUFFIXES = ( + ".ts", + ".tsx", + ".mts", + ".cts", + ".js", + ".jsx", + ".mjs", + ".cjs", + ".ipynb", +) +_OTHER_SOURCE_NAME = "root_agent.yaml" + +_VERSION_PATTERN = re.compile( + r"\d+(\.\d+)*((a|b|rc)\d+)?(\.post\d+)?(\.dev\d+)?(\+[A-Za-z0-9.]+)?", + re.ASCII | re.IGNORECASE, +) + +# 3.11 has neither constant: an f-string is one STRING token there. +_FSTRING_START = getattr(tokenize, "FSTRING_START", None) +_FSTRING_END = getattr(tokenize, "FSTRING_END", None) +_HIDDEN_TOKENS = frozenset({tokenize.COMMENT, tokenize.NL, tokenize.ENCODING}) +_OPENING = frozenset("([{") +_CLOSING = frozenset(")]}") + + +@dataclass(frozen=True) +class Finding: + framework: str + strength: Literal["builds", "model", "import"] + file: str + line: int + + +@dataclass(frozen=True) +class FolderFindings: + findings: tuple[Finding, ...] + blockers: frozenset[str] + over_limit: bool + + +@dataclass(frozen=True) +class FrameworkFolders: + folder: Path + repo_root: Path | None = None + package_dir: Path | None = None + + +@dataclass(frozen=True) +class _Row: + result: str + imports: tuple[str, ...] + builders: frozenset[str] + packages: tuple[str, ...] + label: str | None = None + + @property + def framework(self) -> str: + return self.label or self.result + + +def _named( + result: str, + imports: tuple[str, ...], + builders: tuple[str, ...], + packages: tuple[str, ...], +) -> _Row: + return _Row(result, imports, frozenset(builders), packages) + + +def _other(label: str, imports: tuple[str, ...], packages: tuple[str, ...]) -> _Row: + return _Row("other", imports, frozenset(), packages, label) + + +_ROWS = ( + _named( + "deepagents", + ("deepagents",), + ("create_deep_agent", "middleware"), + ("deepagents",), + ), + _named( + "langgraph", + ("langgraph", "langgraph_*"), + ( + "StateGraph", + "MessageGraph", + "create_react_agent", + "entrypoint", + "create_supervisor", + "create_swarm", + ), + ("langgraph",), + ), + _named( + "langchain", + ("langchain", "langchain_*"), + ( + "create_agent", + "create_react_agent", + "create_tool_calling_agent", + "create_openai_tools_agent", + "create_openai_functions_agent", + "create_structured_chat_agent", + "AgentExecutor", + "initialize_agent", + ), + ("langchain", "langchain-core"), + ), + _named( + "livekit", + ("livekit.agents",), + ("AgentSession", "Agent", "WorkerOptions", "cli"), + ("livekit-agents",), + ), + _named("pipecat", ("pipecat",), ("Pipeline", "PipelineTask"), ("pipecat-ai",)), + _named( + "claude_agent_sdk", + ("claude_agent_sdk", "claude_code_sdk"), + ("query", "ClaudeSDKClient"), + ("claude-agent-sdk", "claude-code-sdk"), + ), + _named( + "openai_agents", + ("agents",), + ("Agent", "Runner", "RealtimeAgent", "RealtimeRunner"), + ("openai-agents",), + ), + _named( + "pydantic_ai", ("pydantic_ai",), ("Agent",), ("pydantic-ai", "pydantic-ai-slim") + ), + _named( + "google_adk", + ("google.adk",), + ( + "Agent", + "LlmAgent", + "SequentialAgent", + "ParallelAgent", + "LoopAgent", + "BaseAgent", + "Runner", + "InMemoryRunner", + ), + ("google-adk",), + ), + _named("crewai", ("crewai",), ("Crew", "Agent", "CrewBase", "Flow"), ("crewai",)), + _other("llama_index", ("llama_index",), ("llama-index-core", "llama-index")), + _other( + "autogen", + ("autogen", "autogen_agentchat", "autogen_core", "autogen_ext"), + ("autogen-agentchat", "pyautogen", "autogen"), + ), + _other("semantic_kernel", ("semantic_kernel",), ("semantic-kernel",)), + _other("haystack", ("haystack",), ("haystack-ai",)), + _other("agno", ("agno",), ("agno",)), + _other("beeai_framework", ("beeai_framework",), ("beeai-framework",)), + _other("dspy", ("dspy",), ("dspy", "dspy-ai")), + _other("smolagents", ("smolagents",), ("smolagents",)), + _other("strands", ("strands",), ("strands-agents",)), +) +_NAMED_FRAMEWORKS = frozenset(row.result for row in _ROWS if row.label is None) +_ROW_BY_FRAMEWORK = {row.framework: row for row in _ROWS} + + +def _index_rows() -> tuple[ + dict[str, list[tuple[_Row, tuple[str, ...]]]], tuple[tuple[str, _Row], ...] +]: + by_top: dict[str, list[tuple[_Row, tuple[str, ...]]]] = {} + wildcards: list[tuple[str, _Row]] = [] + for row in _ROWS: + for prefix in row.imports: + if prefix.endswith("*"): + wildcards.append((prefix[:-1], row)) + else: + parts = tuple(prefix.split(".")) + by_top.setdefault(parts[0], []).append((row, parts)) + return by_top, tuple(wildcards) + + +_PREFIXES_BY_TOP, _WILDCARD_TOPS = _index_rows() + + +def _row_match(parts: list[str]) -> tuple[_Row, int] | None: + """The row a dotted name falls under, and how many of its parts that prefix took.""" + top = parts[0] + for row, prefix in _PREFIXES_BY_TOP.get(top, ()): + if tuple(parts[: len(prefix)]) == prefix: + return row, len(prefix) + for start, row in _WILDCARD_TOPS: + if top.startswith(start): + return row, 1 + return None + + +def _is_row_top(top: str) -> bool: + return top in _PREFIXES_BY_TOP or any( + top.startswith(start) for start, _ in _WILDCARD_TOPS + ) + + +def _is_chat_model(name: str) -> bool: + if name == "init_chat_model": + return True + return name.startswith(("Chat", "AzureChat")) and not name.startswith( + ("ChatMessage", "ChatPrompt", "ChatGeneration", "ChatResult") + ) + + +class _OverBudget(Exception): + pass + + +class _Work: + """What one walk has spent. A count and not a clock, so the answer is the same + every time. Python 3.12.3 re-decodes a token's whole line for each token, and one + name can have thousands of targets: neither shows in the size of a file.""" + + def __init__(self, limit: int | None = None) -> None: + self._spent = 0 + # None follows _MAX_WORK; the agent scan gives each file its own, smaller limit. + self._limit = limit + + def charge(self, amount: int) -> None: + self._spent += amount + if self._spent > (_MAX_WORK if self._limit is None else self._limit): + raise _OverBudget() + + +def _visible_tokens(data: bytes, work: _Work) -> Iterator[tokenize.TokenInfo]: + fstring_depth = 0 + for token in tokenize.tokenize(io.BytesIO(data).readline): + work.charge(len(token.line)) + if token.type == _FSTRING_START: + fstring_depth += 1 + elif token.type == _FSTRING_END: + fstring_depth -= 1 + elif not fstring_depth and token.type not in _HIDDEN_TOKENS: + yield token + + +def _is_type_checking_header(words: list[str]) -> bool: + return words == ["if", "TYPE_CHECKING", ":"] or ( + len(words) == 5 + and words[0] == "if" + and words[2:] == [".", "TYPE_CHECKING", ":"] + ) + + +def _outside_type_checking( + tokens: Iterator[tokenize.TokenInfo], +) -> Iterator[tokenize.TokenInfo]: + """Drops an indented block under `if TYPE_CHECKING:`, because it never runs. + + Its `else` runs, and arrives here as an ordinary statement after the block's + DEDENT. A one-line `if TYPE_CHECKING: import x` is left in: that costs an + `unknown`, never a wrong name. + """ + skipping = 0 + line: list[str] = [] + header: list[str] = [] + for token in tokens: + if skipping: + if token.type == tokenize.INDENT: + skipping += 1 + elif token.type == tokenize.DEDENT: + skipping -= 1 + continue + if token.type == tokenize.INDENT and _is_type_checking_header(header): + skipping = 1 + continue + if token.type == tokenize.NEWLINE: + header, line = line, [] + elif token.type not in (tokenize.INDENT, tokenize.DEDENT) and len(line) < 6: + line.append(token.string) + yield token + + +def _ends_statement(token: tokenize.TokenInfo | None) -> bool: + return ( + token is None + or token.type in (tokenize.NEWLINE, tokenize.ENDMARKER) + or token.string == ";" + ) + + +def _skip_statement( + token: tokenize.TokenInfo | None, tokens: Iterator[tokenize.TokenInfo] +) -> None: + while not _ends_statement(token): + token = next(tokens, None) + + +def _dotted( + token: tokenize.TokenInfo | None, tokens: Iterator[tokenize.TokenInfo] +) -> tuple[list[str], tokenize.TokenInfo | None]: + """Reads `n(.n)*` from `token` on; returns its parts and the first token after it.""" + parts: list[str] = [] + while token is not None and token.type == tokenize.NAME: + if len(parts) < _MAX_DOTTED_PARTS: + parts.append(token.string) + token = next(tokens, None) + if token is None or token.string != ".": + break + token = next(tokens, None) + return parts, token + + +def _after_subscript( + tokens: Iterator[tokenize.TokenInfo], +) -> tokenize.TokenInfo | None: + """Called just past a `[`; returns the first token after its closing `]`.""" + depth = 1 + token = next(tokens, None) + while token is not None and depth: + if token.string == "[": + depth += 1 + elif token.string == "]": + depth -= 1 + token = next(tokens, None) + return token + + +class _FileScan: + """One file's two reads. Uses are resolved as they are met and never kept, so + what this holds grows with the file's imports, not with its length.""" + + def __init__( + self, shadowed: frozenset[str], outside_names: frozenset[str], work: _Work + ) -> None: + self.work = work + self._shadowed = shadowed + self._outside_names = outside_names + # Targets stay as parts: a name can be megabytes long, and a joined copy per + # target would hold far more than the source does. + self.aliases: dict[str, set[tuple[str, ...]]] = {} + self._alias_count = 0 + self.found: dict[tuple[str, str], int] = {} + self.has_row_import = False + self.imports_outside = False + + def read_imports(self, tokens: Iterator[tokenize.TokenInfo]) -> None: + # Both are keywords: `import` only ever starts an import, and `_from_import` + # skips a `yield from` or `raise ... from`, since no `import` follows them. + for token in tokens: + if token.type == tokenize.NAME and token.string == "import": + self._plain_import(token.start[0], tokens) + elif token.type == tokenize.NAME and token.string == "from": + self._from_import(token.start[0], tokens) + + def _plain_import(self, line: int, tokens: Iterator[tokenize.TokenInfo]) -> None: + while True: + parts, token = _dotted(next(tokens, None), tokens) + if parts: + if token is not None and token.string == "as": + named = next(tokens, None) + token = next(tokens, None) + if named is not None and named.type == tokenize.NAME: + self._note(parts, named.string, tuple(parts), line) + else: + self._note(parts, parts[0], (parts[0],), line) + if token is None or token.string != ",": + _skip_statement(token, tokens) + return + + def _from_import(self, line: int, tokens: Iterator[tokenize.TokenInfo]) -> None: + # A relative import starts with a `.` or `...` token, which `_dotted` stops at + # with no parts. + module, token = _dotted(next(tokens, None), tokens) + if token is None or token.string != "import": + _skip_statement(token, tokens) + return + token = next(tokens, None) + while token is not None and not _ends_statement(token): + if token.string == "*": + self._note(module, None, None, line) + elif token.type == tokenize.NAME: + imported = [*module, token.string] + alias = token.string + token = next(tokens, None) + if token is not None and token.string == "as": + named = next(tokens, None) + token = next(tokens, None) + if named is None or named.type != tokenize.NAME: + continue + alias = named.string + self._note(imported, alias, tuple(imported), line) + continue + token = next(tokens, None) + + def _note( + self, + imported: list[str], + alias: str | None, + target: tuple[str, ...] | None, + line: int, + ) -> None: + top = imported[0] + if top in self._outside_names: + self.imports_outside = True + if top in self._shadowed: + return + match = _row_match(imported) + if match is not None: + self.has_row_import = True + self.found.setdefault((match[0].framework, "import"), line) + if alias is None or target is None or not _is_row_top(top): + return + targets = self.aliases.setdefault(alias, set()) + if target not in targets: + targets.add(target) + self._alias_count += 1 + if self._alias_count > _MAX_ALIASES: + raise ValueError("too many aliases") + + def read_uses(self, tokens: Iterator[tokenize.TokenInfo]) -> None: + depth = 0 + class_depth: int | None = None + class_header = False + at_start = True + decorates = False + prev = "" + token = next(tokens, None) + while token is not None: + text = token.string + if ( + token.type == tokenize.NAME + and text in self.aliases + and prev not in (".", "def", "class") + ): + line = token.start[0] + decorator = prev == "@" and decorates + base_position = class_depth == depth and prev in ("(", ",") + parts, token = _dotted(token, tokens) + if token is not None and token.string == "[": + token = _after_subscript(tokens) + following = token.string if token is not None else "" + if decorator: + self._resolve(parts, "decorator", line) + elif following == "(": + self._resolve(parts, "call", line) + elif base_position and following in (",", ")"): + self._resolve(parts, "base", line) + else: + self._resolve(parts, "mention", line) + prev = parts[-1] + at_start = False + continue + if token.type == tokenize.NEWLINE: + depth = 0 + class_depth = None + class_header = False + at_start = True + prev = "" + elif token.type not in (tokenize.INDENT, tokenize.DEDENT): + if text in _OPENING: + depth += 1 + # Depth 1, because `class F[T: (A, B)](Base):` has brackets ahead. + if text == "(" and class_header and depth == 1: + class_depth = depth + class_header = False + elif text in _CLOSING: + depth = max(depth - 1, 0) + if class_depth is not None and depth < class_depth: + class_depth = None + elif text == "class" and token.type == tokenize.NAME: + class_header = True + elif text == ":" and depth == 0: + class_header = False + if text == "@": + decorates = at_start + at_start = text == ";" + prev = text + token = next(tokens, None) + + def _resolve(self, parts: list[str], kind: str, line: int) -> None: + targets = self.aliases[parts[0]] + self.work.charge(_WORK_PER_TARGET * len(targets)) + for target in targets: + full = [*target, *parts[1:]] + match = _row_match(full) + if match is None: + continue + row, taken = match + # `import livekit` isn't under the row, so `livekit.agents.x` is the first + # sign of it: without this the folder would say `none`. + self.found.setdefault((row.framework, "import"), line) + if kind == "mention": + continue + if row.label is not None: + # `other` means "called something from it": a base class or a + # decorator alone says too little about what the agent runs on. + if kind == "call": + self.found.setdefault((row.framework, "builds"), line) + elif any(part in row.builders for part in full[taken:]): + self.found.setdefault((row.framework, "builds"), line) + if ( + row.result == "langchain" + and full[0] != "langchain_core" + and _is_chat_model(full[-1]) + ): + self.found.setdefault((row.framework, "model"), line) + + +def _scan_file(data: bytes, scan: _FileScan, *, read_uses: bool) -> bool: + """Runs the reads; True when one failed. What was found before a failure stands.""" + failed = False + with warnings.catch_warnings(): + # On 3.12 `tokenize` warns once per line on a `\{`, which floods stderr. + warnings.simplefilter("ignore") + try: + scan.read_imports(_outside_type_checking(_visible_tokens(data, scan.work))) + except _OverBudget: + raise + except Exception: + failed = True + if read_uses and scan.aliases: + try: + scan.read_uses(_outside_type_checking(_visible_tokens(data, scan.work))) + except _OverBudget: + raise + except Exception: + failed = True + return failed + + +def _read_file(path: str, limit: int) -> bytes: + # O_NONBLOCK so that opening a named pipe can't hang the build. + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise OSError("not a regular file") + chunks: list[bytes] = [] + remaining = limit + while remaining > 0: + chunk = os.read(descriptor, min(remaining, 1_048_576)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + finally: + os.close(descriptor) + + +def _has_long_line(data: bytes) -> bool: + # A very long line is what makes `tokenize` slow. `find`, because `split` + # costs hundreds of MiB on a file at the limit. + start = 0 + while True: + newline = data.find(b"\n", start) + end = len(data) if newline < 0 else newline + if end - start > _MAX_LINE_BYTES: + return True + if newline < 0: + return False + start = newline + 1 + + +def _module_names(directory: Path) -> set[str]: + names: set[str] = set() + with os.scandir(directory) as entries: + for entry in entries: + if entry.is_dir(): + names.add(entry.name) + elif entry.name.endswith(".py"): + names.add(entry.name[: -len(".py")]) + return names + + +def _outside_names( + folder: Path, repo_root: Path | None, blockers: set[str] +) -> frozenset[str]: + """Names that, imported from the folder, are shared code sitting outside it.""" + if repo_root is None: + return frozenset() + try: + here = folder.resolve() + top = repo_root.resolve() + try: + on_path = set(here.relative_to(top).parts) + except ValueError: + # The usual packaged build: the folder is a copy outside the repo. + on_path = set() + return frozenset(_module_names(top) - _module_names(here) - on_path) + except (OSError, RuntimeError): + blockers.add(_FILE_SKIPPED) + return frozenset() + + +def _own_modules(folder: Path, blockers: set[str]) -> frozenset[str]: + """Top names that are the customer's, because they sit ahead of the installed one.""" + own: set[str] = set() + installed_here: set[str] = set() + try: + with os.scandir(folder) as entries: + for entry in entries: + name = entry.name + if name.endswith(".py"): + own.add(name[: -len(".py")]) + elif name.endswith(".dist-info") and "-" in name and entry.is_dir(): + # `pip install -t .`: the package beside it is the real one. + blockers.add(_FILE_SKIPPED) + installed_here.add( + name[: -len(".dist-info")].rpartition("-")[0].replace("-", "_") + ) + elif entry.is_dir() and os.path.isfile( + os.path.join(entry.path, "__init__.py") + ): + own.add(name) + except OSError: + blockers.add(_FILE_SKIPPED) + # `agents` is too common a name for its presence to decide; see `_agents_rule`. + return frozenset(own - installed_here - {"agents"}) + + +def _is_side_part(part: str) -> bool: + return part in _SIDE_FOLDERS or part.startswith(".") + + +def _is_side_file(name: str) -> bool: + return ( + name == "conftest.py" + or (name.startswith("test_") and name.endswith(".py")) + or name.endswith("_test.py") + or name.startswith(".") + ) + + +def _folder_bytes(top: str, below: str) -> int: + """At least what one folder costs in ``os.walk``'s list of folders to visit: the whole path, + ``top``, a separator and ``below``, at the width of its widest character (one byte if all of it is + ASCII, else counted at four), plus 120: the str's fixed part (at most 60 bytes on 3.12), the + allocator's rounding (at most 23 under glibc), the list's slot (8, and about 1 of its spare room), + and pymalloc's own share of each block (pool headers and alignment, up to about 15), with about + 13 to spare.""" + width = 1 if top.isascii() and below.isascii() else 4 + return width * (max(len(top), _TOP_FLOOR) + 1 + len(below)) + 120 + + +def _directories_to_enter( + directory: str, names: list[str], *, at_top: bool, blockers: set[str] +) -> list[str]: + kept = [] + for name in names: + if name in _ALWAYS_SKIPPED or (at_top and name in _TOP_ONLY_SKIPS): + continue + if os.path.islink(os.path.join(directory, name)): + blockers.add(_FILE_SKIPPED) + continue + kept.append(name) + return kept + + +class _FolderWalk: + def __init__( + self, + shadowed: frozenset[str], + outside_names: frozenset[str], + blockers: set[str], + ) -> None: + self._shadowed = shadowed + self._outside_names = outside_names + self.blockers = blockers + self.found: dict[tuple[str, str, str], int] = {} + self.python_seen = 0 + self.bytes_read = 0 + self.has_counted_python = False + self.agents_in_counted_path = False + self.over_limit = False + self._work = _Work() + + def take(self, directory: str, relative: tuple[str, ...], name: str) -> None: + counted = not ( + any(_is_side_part(part) for part in relative) or _is_side_file(name) + ) + if not name.endswith(".py"): + if counted and ( + name.endswith(_OTHER_SOURCE_SUFFIXES) or name == _OTHER_SOURCE_NAME + ): + self.blockers.add(_OTHER_SOURCE) + return + self.python_seen += 1 + if self.python_seen > _MAX_FILES: + self.over_limit = True + return + if counted: + self.has_counted_python = True + if "agents" in relative or name == "agents.py": + self.agents_in_counted_path = True + try: + data = _read_file( + os.path.join(directory, name), _MAX_TOTAL_BYTES - self.bytes_read + 1 + ) + except Exception: + if counted: + self.blockers.add(_FILE_SKIPPED) + return + self.bytes_read += len(data) + if self.bytes_read > _MAX_TOTAL_BYTES: + self.over_limit = True + return + if _has_long_line(data): + if counted: + self.blockers.add(_FILE_SKIPPED) + return + scan = _FileScan(self._shadowed, self._outside_names, self._work) + try: + failed = _scan_file(data, scan, read_uses=counted) + except _OverBudget: + self.over_limit = True + return + if not counted: + if scan.has_row_import: + self.blockers.add(_SIDE_FOLDER_FRAMEWORK) + return + if failed: + self.blockers.add(_FILE_UNREADABLE) + if scan.imports_outside: + self.blockers.add(_OUTSIDE_IMPORT) + file = "/".join((*relative, name)) + for (framework, strength), line in scan.found.items(): + self.found[(file, framework, strength)] = line + + +def _site_packages(package_dir: Path | None) -> list[str]: + if package_dir is None: + return [] + pattern = os.path.join( + glob.escape(os.fspath(package_dir)), ".venv", "lib", "python*", "site-packages" + ) + return sorted(path for path in glob.glob(pattern) if os.path.isdir(path)) + + +def _normalised(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def _installed(sites: list[str], package: str) -> list[str]: + """The `*.dist-info` directories of `package`, whatever their version.""" + wanted = _normalised(package) + matches = [] + for site in sites: + try: + names = sorted(os.listdir(site)) + except OSError: + continue + for name in names: + if ( + name.endswith(".dist-info") + and _normalised(name[: -len(".dist-info")].rpartition("-")[0]) == wanted + ): + matches.append(os.path.join(site, name)) + return matches + + +def _agents_rule( + walk: _FolderWalk, repo_root: Path | None, package_dir: Path | None +) -> None: + """`agents` is a common name for a customer's own package, so an import of it + only counts with evidence that the real one is meant.""" + if not any(framework == "openai_agents" for _, framework, _ in walk.found): + return + sites = _site_packages(package_dir) + if sites: + keep = bool(_installed(sites, "openai-agents")) + else: + keep = not walk.agents_in_counted_path and not ( + repo_root is not None + and ( + os.path.lexists(repo_root / "agents") + or os.path.lexists(repo_root / "agents.py") + ) + ) + if not keep: + walk.blockers.add(_AGENTS_AMBIGUOUS) + if not keep: + walk.found = { + key: line for key, line in walk.found.items() if key[1] != "openai_agents" + } + + +def folder_findings( + folder: Path, *, repo_root: Path | None = None, package_dir: Path | None = None +) -> FolderFindings: + folder = Path(folder) + blockers: set[str] = set() + walk = _FolderWalk( + _own_modules(folder, blockers), + _outside_names(folder, repo_root, blockers), + blockers, + ) + + def on_walk_error(_: OSError) -> None: + blockers.add(_FILE_SKIPPED) + + top = os.fspath(folder) + listed = 0 + try: + for directory, directories, files in os.walk( + top, followlinks=False, onerror=on_walk_error + ): + # Sorted, so that a walk stopped by a limit stops at the same file. + directories.sort() + files.sort() + relative = tuple(directory[len(top) :].split(os.sep)[1:]) + if relative and any(name.endswith(".dist-info") for name in directories): + # An installed tree, not the customer's code. + directories[:] = [] + continue + directories[:] = _directories_to_enter( + directory, directories, at_top=not relative, blockers=blockers + ) + here = "/".join(relative) + listed += sum( + _folder_bytes(top, f"{here}/{name}" if here else name) for name in directories + ) + if listed > _MAX_FOLDER_BYTES: + # Before this folder's files, so os.walk takes none of its folders on. + walk.over_limit = True + break + for name in files: + walk.take(directory, relative, name) + if walk.over_limit: + break + if walk.over_limit: + break + except (OSError, RecursionError): + # A tree deep enough to fail outside `onerror`. + blockers.add(_FILE_SKIPPED) + if not walk.has_counted_python: + blockers.add(_NO_PYTHON) + _agents_rule(walk, repo_root, package_dir) + findings = sorted( + ( + Finding(framework, strength, file, line) # type: ignore[arg-type] + for (file, framework, strength), line in walk.found.items() + ), + key=lambda f: (f.file, f.line, f.framework, f.strength), + ) + return FolderFindings(tuple(findings), frozenset(blockers), walk.over_limit) + + +def _decide(found: FolderFindings) -> tuple[str, str | None]: + """The result word, and the framework whose row it was read from.""" + if found.over_limit: + return "unknown", None + builds = {f.framework for f in found.findings if f.strength == "builds"} + named = builds & _NAMED_FRAMEWORKS + # A voice agent often drives a LangChain graph inside: the voice side is the agent. + if len(named & {"livekit", "pipecat"}) == 1: + named -= {"langchain", "langgraph", "deepagents"} + # Each is built on the next, so the higher one is what was chosen. + if "deepagents" in named: + named -= {"langgraph", "langchain"} + if "langgraph" in named: + named -= {"langchain"} + if len(named) == 1: + name = next(iter(named)) + return name, name + if named: + return "unknown", None + labels = builds - _NAMED_FRAMEWORKS + if len(labels) == 1: + return "other", next(iter(labels)) + if labels: + return "unknown", None + # A chat model names LangChain only when nothing unseen could be building the agent. + if found.blockers: + return "unknown", None + if any(f.strength == "model" for f in found.findings): + return "langchain", "langchain" + if any(f.strength == "import" for f in found.findings): + return "unknown", None + return "none", None + + +def _version_from_metadata(dist_info: str) -> str | None: + data = _read_file(os.path.join(dist_info, "METADATA"), _METADATA_BYTES) + for line in data.split(b"\n"): + if line.startswith(b"Version:"): + value = line[len(b"Version:") :].strip().decode("ascii") + if len(value) <= _VERSION_MAX and _VERSION_PATTERN.fullmatch(value): + return value + return None + return None + + +def _read_version(row: _Row, package_dir: Path | None) -> str | None: + # A bad version is a missing detail, never a reason to doubt the name. + try: + sites = _site_packages(package_dir) + for package in row.packages: + matches = _installed(sites, package) + if matches: + return _version_from_metadata(matches[0]) if len(matches) == 1 else None + except Exception: + return None + return None + + +def detect_framework( + folder: Path | None, + *, + repo_root: Path | None = None, + package_dir: Path | None = None, + raise_faults: bool = False, +) -> dict[str, Any]: + """The folder's framework word. A fault in the check is ``unknown`` for a build; the agent + scan passes ``raise_faults=True`` to see it, so it can log the trace and mark the row + unfinished instead of reporting a finished ``unknown``.""" + try: + if folder is None or not Path(folder).is_dir(): + logger.info("framework detection: name=%s blockers=%s", "unknown", []) + return {"name": "unknown", "version": None} + found = folder_findings(folder, repo_root=repo_root, package_dir=package_dir) + name, framework = _decide(found) + result: dict[str, Any] = {"name": name, "version": None} + if framework is not None: + row = _ROW_BY_FRAMEWORK[framework] + result["version"] = _read_version(row, package_dir) + if name == "other": + result["label"] = framework + logger.info( + "framework detection: name=%s blockers=%s", name, sorted(found.blockers) + ) + return result + except Exception as exc: + if raise_faults: + raise + logger.warning("framework detection failed: class=%s", type(exc).__name__) + return {"name": "unknown", "version": None} + + +def _contract_workdir( + source: Path, bundle_dir: Path, contract_text: str | None = None +) -> Path | None: + """`runtime.workdir` from the sealed contract, when it names a real inner folder.""" + try: + contract = json.loads( + contract_text + if contract_text is not None + else (bundle_dir / "contract.json").read_text(encoding="utf-8") + ) + runtime = contract.get("runtime") if isinstance(contract, dict) else None + workdir = runtime.get("workdir") if isinstance(runtime, dict) else None + if not isinstance(workdir, str) or not workdir or workdir == ".": + return None + candidate = source / workdir + resolved = candidate.resolve() + if resolved.is_relative_to(source.resolve()) and resolved.is_dir(): + return candidate + except (OSError, ValueError, RuntimeError): + return None + return None + + +def named_framework_folder( + source: Path, bundle_dir: Path, subfolder: str | None, *, contract_text: str | None = None +) -> Path | None: + """The folder the job names, else the sealed contract's own; ``None`` when neither does. + + ``contract_text`` is the contract already read, when the caller has read it. + """ + if subfolder: + return agent_folder(source, subfolder) + return _contract_workdir(source, bundle_dir, contract_text) + + +def framework_folders_for( + job: Any, manifest: Any, *, source: Path, bundle_dir: Path, work_directory: Path +) -> FrameworkFolders: + runtime = manifest.runtime + control_service = runtime.control_service if runtime.kind == "process" else None + folder = named_framework_folder(source, bundle_dir, job.source.subfolder) + if folder is None: + folder = source + if control_service: + for process in manifest.processes: + if process.name == control_service: + # The same base the runtime builds the process from. + base = bundle_dir if process.source_origin == "bundle" else source + folder = base / process.working_directory + break + package_dir = ( + work_directory / "build" / control_service if control_service else None + ) + return FrameworkFolders(folder, source, package_dir) diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 681e166f6..c1a300d22 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -6,7 +6,10 @@ import traceback import json import os +import re import signal +import sys +import threading import time import uuid from collections.abc import Awaitable @@ -36,6 +39,14 @@ ) from .tools import schema from .session import ARTIFACT, DONE, RESULT, STAGE_IDLE_TIMEOUT_SECONDS, TEXT, TOOL, Event, Stage +from ..providers.agent import ( + _LETTER, + _QUERY_OR_FRAGMENT, + _REDACTED, + _START, + _URL_BREAKS, + _redact, +) _NUMBERS = {"type": "array", "items": {"type": "integer"}} _RANGES = { @@ -57,6 +68,8 @@ _PUBLISHED = ".futureagi-published.json" _LEDGER = ".futureagi-ledger.json" _LEDGER_NOTES = 40 +# One install at a time: an opening cancelled mid-install leaves its thread running. +_SEED_LOCK = threading.Lock() # Tools whose effect the person confirms before it happens. _NEEDS_CONFIRMATION = frozenset( { @@ -132,6 +145,161 @@ } ) +# The platform stores every chat event as sent, so a URL's password, query and fragment are +# dropped before each emit. Streamed text comes in pieces, and a URL can be split between two, +# so _DeltaCleaner holds back the text after the last cut and cleans each piece it lets out. +# +# The whole message is the reply as stored: the model's text with its leading and trailing +# whitespace stripped. Whitespace before the first character is sent as it comes. +# +# Why a cut is safe: the cleaner never lets a URL cross whitespace, with one exception. A value +# that begins with a scheme and :// and has no whitespace but tabs and line breaks is one URL: +# those are removed and the rest is cleaned as one token. Until the message is known not to be +# such a value, it is held, and every piece let out (by a shed or by the flush) has its tabs and +# line breaks removed before it is cleaned. The message has had no space yet, so each piece is +# one span of that one token, and is cleaned as the whole would clean it. If the message turns +# out to be prose, joining those pieces only removed more: line breaks, and anything a joined +# query or password then owns. +# +# It is known to be prose once its opening (after leading whitespace) is not a scheme's run and +# ://, or once a space has come with more text after it. A space with nothing after it yet +# doesn't decide it: the stripped reply may still be one URL. In prose the whole message is +# cleaned token by token, split at every whitespace; any whitespace is then a cut, and each +# piece is cleaned the prose way too, so it splits at the same tokens and keeps its whitespace. +# +# The holdback is bounded. Past _HOLD_SOFT characters with no cut, what is held is shed: +# - with a ? or # after its first URL start, the rest of the token is that URL's query. The held +# text is sent cleaned (the URL with its host and path), and the rest of the token is dropped +# up to the next cut, which is what cleaning the whole would drop; +# - otherwise everything before its last URL is sent cleaned, and the URL is kept. Each URL +# starts where _redact_token starts it (its scheme's first letter, or its :// when the run +# has no letter), so the pieces are the whole's own pieces; +# - with no URL start, it is sent, keeping only a trailing ":" or ":/" that the next piece could +# turn into "://"; +# - held text that is a whole token and a space after it, before the message is known to be +# prose, is sent cleaned as it is: nothing after the space can join that token. +# A URL still held past _HOLD_HARD characters is sent as [redacted], and the rest of its token is +# dropped. Nothing of a dropped token is ever sent. +_HOLD_SOFT = 4_096 +_HOLD_HARD = 16_384 +_CUT = re.compile(r"[^\S\t\r\n]") +_ANY_CUT = re.compile(r"\s") +_SCHEME_RUN = re.compile(r"[A-Za-z][A-Za-z0-9+.\-]*") +_SCHEME_TAIL = re.compile(r":/?\Z") + + +class _DeltaCleaner: + """The cleaned text of one streamed message, let out piece by piece.""" + + def __init__(self) -> None: + self.held = "" + self.dropping = False + # Whether the whole message is cleaned the prose way, learned from its opening. + self.prose = False + self._opening = "" + # Whether the message's first character has come: whitespace before it is no part of it. + self._begun = False + # Before prose is known: a space has come after the opening began, with only whitespace + # after it so far. + self._spaced = False + + def feed(self, text: str) -> str: + out: list[str] = [] + if not self._begun: + body = text.lstrip() + out.append(text[: len(text) - len(body)]) + text = body + if not text: + return "".join(out) + self._begun = True + was_prose = self.prose + self._learn(text) + if self.prose and not was_prose: + # What was held before may hold tabs, line breaks and a space: it is cut with the rest. + text, self.held = self.held + text, "" + if self.dropping: + cut = (_ANY_CUT if self.prose else _CUT).search(text) + if cut is None: + return "".join(out) + text = text[cut.start() :] + self.dropping = False + last = _ANY_CUT.search(text[::-1]) if self.prose else None + if last is None: + self.held += text + else: + end = len(text) - last.start() + out.append(self._clean(self.held + text[:end])) + self.held = text[end:] + if len(self.held) > _HOLD_SOFT: + out.append(self._shed()) + return "".join(out) + + def flush(self) -> str: + held, self.held, self.dropping = self.held, "", False + return self._clean(held) if held else "" + + def _learn(self, text: str) -> None: + if self.prose or not text: + return + if self._spaced: + if not text.isspace(): + self.prose = True + return + cut = _CUT.search(text) + if cut is not None: + if text[cut.end() :].strip(): + self.prose = True + return + self._spaced = True + if len(self._opening) >= _HOLD_SOFT: + return + self._opening += text[: _HOLD_SOFT - len(self._opening)] + run = _SCHEME_RUN.match(self._opening) + rest = self._opening[run.end() :] if run else self._opening + # Not a letter first, or a scheme's run followed by anything but "://". + if run is None or (rest and not "://".startswith(rest[:3])): + self.prose = True + + def _clean(self, piece: str) -> str: + if not self.prose: + # Joined as cleaning the whole joins a message that is one URL. Whitespace at the end + # is after the message's last character (no part of the stripped reply), so it is + # kept as it came. + body = piece.rstrip() + return _redact(body.translate(_URL_BREAKS)) + piece[len(body) :] + # The space in front only keeps the cleaner from reading the piece as one URL, which + # would drop its tabs and line breaks; the prose way keeps them, and the space comes off. + return _redact(" " + piece)[1:] + + def _shed(self) -> str: + held = self.held + if not self.prose: + if self._spaced: + self.held = "" + return self._clean(held) + self.held = held = held.translate(_URL_BREAKS) + starts = [] + for found in _START.finditer(held): + letter = _LETTER.search(held, found.start(), found.end() - 3) + starts.append(letter.start() if letter else found.end() - 3) + if not starts: + tail = _SCHEME_TAIL.search(held) + keep = tail.start() if tail else len(held) + self.held = held[keep:] + return _redact(held[:keep]) + if _QUERY_OR_FRAGMENT.search(held, starts[0]): + self.held, self.dropping = "", True + return _redact(held) + out = "" + if starts[-1] > 0: + out = _redact(held[: starts[-1]]) + self.held = held = held[starts[-1] :] + if len(held) > _HOLD_HARD: + self.held, self.dropping = "", True + out += _REDACTED + return out + + # Both the foreground coordinator and resumed stage conversations need this boundary: # authoring tools can edit harness artifacts, but cannot update the target configuration # used by the platform or deploy changes to an externally hosted agent. @@ -283,6 +451,10 @@ async def close(self) -> None: class HostedChatRuntime: + # Whether this process's first opening has decided on the seed, and what it decided. + _seed_taken = False + _seed_outcome: tuple[str, list[str]] | None = None + def __init__( self, *, @@ -360,9 +532,24 @@ async def _open_builder(self, *, resume: bool = True) -> Stage: from .understand import load as load_contract policy_job = guest_booking_policy_job(self.job) + # A reading the platform handed this build is used before anything opens. + await asyncio.to_thread(self._take_seed) contract = load_contract(self.workspace) self._opened_with = self._what_exists() if not self._opened_with[1]: + if contract is not None: + try: + # What follows a saved contract is done before the world stage opens. + await asyncio.to_thread(self._after_saved) + except Exception as exc: # noqa: BLE001 - the world stage opens without it; it is tried again after a turn, at finish_build and at the next opening + from .seed_contract import print_trace + + print( + f"source model not written before the world stage; class: {type(exc).__name__}", + file=sys.stderr, + flush=True, + ) + print_trace(exc) stage = self._open_for_building(contract) else: try: @@ -455,6 +642,7 @@ def _nudge_tools(self, stage: Stage, catalogue: Any) -> None: """Only the rebuild stage's tools, with report_update; a suite turn never renames.""" from . import nudge from .scenario_tools import SCENARIO_SERVER + from .source_folder import saved_folder from .world.tools import WORLD_SERVER command = self._nudge or {} @@ -465,7 +653,10 @@ def _nudge_tools(self, stage: Stage, catalogue: Any) -> None: if WORLD_SERVER in servers: world = {} servers[WORLD_SERVER] = nudge.track_world( - servers[WORLD_SERVER], world, self.source + servers[WORLD_SERVER], + world, + self.source, + saved_folder(self.workspace), ) else: allowed = nudge.SUITE_TOOLS @@ -501,7 +692,12 @@ def _open_for_building(self, contract: Any) -> Stage: if job is not None and agent_source_kind(job, profile) == "provider": source = resolve("provider", name=name, profile=profile or {}, scratch=str(self.source)) else: - source = resolve(kind if kind in {"repo", "spec"} else "repo", name=name, root=str(self.source)) + source = resolve( + kind if kind in {"repo", "spec"} else "repo", + name=name, + root=str(self.source), + subfolder=(self.job.get("source") or {}).get("subfolder"), + ) return open_for_building( contract, out=self.workspace, @@ -1280,6 +1476,87 @@ def _after_saved(self, *, world_changed: bool = False) -> None: if has_world and (world_changed or not (self.workspace / "environment-bundle" / "manifest.json").is_file()): after_world(self.source if self.source.is_dir() else None, self.workspace, load_contract(self.workspace)) + def _take_seed(self) -> None: + """Use the reading the platform handed this build, at the first opening only, and leave + what was decided for the platform.""" + from .seed_contract import SEED_RESULT, write_result + + with _SEED_LOCK: + if self._seed_taken: + # Decided at this process's first opening; a record that couldn't be written is + # tried again, but nothing is decided twice. + if self._seed_outcome is not None and not os.path.lexists(self.workspace / SEED_RESULT): + write_result(self.workspace, *self._seed_outcome) + return + self._seed_taken = True + self._seed_outcome = self._decide_seed() + if self._seed_outcome is not None: + write_result(self.workspace, *self._seed_outcome) + + def _decide_seed(self) -> tuple[str, list[str]] | None: + """What the first opening does with the seed, or None when there is nothing to say.""" + from .seed_contract import ( + SEED_ACCEPTED, + SEED_INSTALLED, + SEED_RESULT, + holds_seed, + print_trace, + seed_beside, + ) + + seed = seed_beside(self.workspace) + # The platform placed no seed: nothing to decide, and nothing to say. + if not os.path.lexists(seed): + return None + # Decided by an earlier process that could say so: once the file exists, nothing installs. + if os.path.lexists(self.workspace / SEED_RESULT): + return None + outcome: tuple[str, list[str]] + contract = self.workspace / "contract.json" + if contract.exists() and ((self.workspace / SEED_INSTALLED).is_file() or holds_seed(contract, seed)): + # Installed by an earlier process that stopped before it could say so. The record + # outlasts a contract edit; before it is written no stage has opened, so the bytes + # are still the seed's, which no reading makes, as only the paid one is recorded. + outcome = ("used", []) + elif ( + contract.exists() + or (self.workspace / FAILURE_FILE).exists() + or self._what_exists()[1] + or self._build_is_over() + ): + # Not a fresh seeded build: the build command refused, the workspace was restored, + # or a reading of the builder's own has saved. + return None + elif not (self.workspace / SEED_ACCEPTED).is_file(): + # Opened before the build command accepted it, so this build reads the agent itself. + return "not_accepted", [] + else: + outcome = self._use_seed(seed) + try: + # Consumed whatever the outcome, so nothing but this opening ever acts on it. + (self.workspace / SEED_ACCEPTED).unlink(missing_ok=True) + except OSError as exc: + print_trace(exc) + return outcome + + def _use_seed(self, seed: Path) -> tuple[str, list[str]]: + from pydantic import ValidationError + + from .job import HarnessJob + from .seed_contract import print_trace, use_seed + + try: + job = HarnessJob.model_validate(self.job) + except ValidationError: + # Only the class: its text repeats values from the job. + print("seed contract: error; class: ValidationError", file=sys.stderr, flush=True) + return "error", [] + except Exception as exc: # noqa: BLE001 - a job the builder can't read reads the agent itself + print(f"seed contract: error; class: {type(exc).__name__}", file=sys.stderr, flush=True) + print_trace(exc) + return "error", [] + return use_seed(seed, job, self.source, self.workspace) + async def _handle(self, command: dict[str, Any]) -> None: if str(command["kind"]) != "nudge": await self._handle_command(command) @@ -1541,7 +1818,7 @@ def receive(event: Event) -> None: stage=self.conversation.stage_name, invocation_id=invocation_id, payload={ - "text": "".join(emitted_text).strip(), + "text": _redact("".join(emitted_text).strip()), "interrupted": True, }, ) @@ -1580,19 +1857,24 @@ def receive(event: Event) -> None: return if turn_failures: detail = turn_failures[-1] or "the model turn failed" + # Cleaned whole, stripped as the reply is, before the cut: a cut first could end a URL right + # after a password, and leading whitespace would stop a lone URL being read as one. await self._fail_turn( journal, message_id, assistant_message_id, invocation_id, - f"I couldn't complete that turn: {detail[:400]}", + f"I couldn't complete that turn: {_redact(detail.strip())[:400]}", "stage_failed", ) return - final_text = "".join(emitted_text).strip() + # The model's text is cleaned alone, as the stream cleaned it, and only then are the notes + # added: added first, they would change how it is read (a reply that is one URL broken by + # a line break is cleaned whole). + final_text = _redact("".join(emitted_text).strip()) saved, unsealed = await self._publish_if_changed() if saved: - final_text = f"{final_text}\n\n{saved}".strip() + final_text = f"{final_text}\n\n{_redact(saved)}".strip() if unsealed: # Changes no run could replay are not promoted with this turn. await self._fail_turn( @@ -1610,7 +1892,7 @@ def receive(event: Event) -> None: problems = [line for line in self._last_health.splitlines() if line.strip()] final_text = ( f"{final_text}\n\nStill open from the last health check ({len(problems)}): " - + "; ".join(problems[:3]) + + _redact("; ".join(problems[:3])) + ("; and more" if len(problems) > 3 else "") ).strip() self._last_health = "" @@ -1655,7 +1937,10 @@ async def _fail_turn( text: str, error: str, ) -> None: - """Say why and end the turn failed: its checkpoint is not promoted.""" + """Say why and end the turn failed: its checkpoint is not promoted. + + ``text`` is sent as given, so each caller cleans the model's or the provider's part of it + before composing it.""" await self.client.emit( "assistant_message", message_id=assistant_message_id, @@ -1737,10 +2022,25 @@ async def _pump_events( turn_failures: list[str], ) -> None: visible_stage = self.conversation.stage_name + deltas = _DeltaCleaner() + delta_stage, delta_invocation_id = visible_stage, fallback_invocation_id while True: event = await queue.get() try: if event is None: + # The message ends here, before its assistant_message: what is held goes out. + rest = deltas.flush() + if rest: + try: + await self.client.emit( + "assistant_delta", + message_id=assistant_message_id, + stage=delta_stage, + invocation_id=delta_invocation_id, + payload={"text": rest, "partial": True}, + ) + except Exception: # noqa: BLE001 - the pump still ends. + traceback.print_exc() return if event.kind == DONE and event.detail.get("outcome") == "failed": turn_failures.append( @@ -1762,16 +2062,19 @@ async def _pump_events( visible_stage = stage if event.kind == TEXT and event.text: emitted_text.append(event.text) - await self.client.emit( - "assistant_delta", - message_id=assistant_message_id, - stage=stage, - invocation_id=invocation_id, - payload={ - "text": event.text, - "partial": bool(event.detail.get("partial")), - }, - ) + shown = deltas.feed(event.text) + delta_stage, delta_invocation_id = stage, invocation_id + if shown: + await self.client.emit( + "assistant_delta", + message_id=assistant_message_id, + stage=stage, + invocation_id=invocation_id, + payload={ + "text": shown, + "partial": bool(event.detail.get("partial")), + }, + ) elif event.kind == TOOL: call_id = str(event.detail.get("call_id") or "") tool = event.tool.rsplit("__", 1)[-1] @@ -1811,14 +2114,19 @@ async def _pump_events( stage=stage, invocation_id=invocation_id, function_call_id=call_id or None, - payload={"tool": tool, "text": event.text, "is_error": failed}, + payload={ + "tool": tool, + "text": _redact(event.text), + "is_error": failed, + }, ) elif event.kind == ARTIFACT: await self.client.emit( "tool_result", stage=stage, invocation_id=invocation_id, - payload={"artifact": event.detail.get("path")}, + # Taken from the tool's own words, so it could be a URL. + payload={"artifact": _redact(event.detail.get("path"))}, ) except Exception: # noqa: BLE001 - a lost event or scratch checkpoint must not end the turn. # What the turn says and whether it failed are recorded before any call here. @@ -1905,11 +2213,16 @@ async def _ask( ) temporary.replace(pending_path) await self._checkpoint() + # The model wrote the question and its options. The file above keeps them raw, as asked, so + # a resumed turn still matches them; only the event below is cleaned. The raw file still + # reaches the platform: the workspace checkpoint archives it (TH-9521), or, in + # control-only mode, _checkpoint sends it as coordinator_state instead (TH-9532). + shown_question, *shown_options = _redact([question, *options]) await self.client.emit( "question_requested", message_id=message_id, stage=self.conversation.stage_name, - payload={"prompt": question, "options": options}, + payload={"prompt": shown_question, "options": shown_options}, ) while not self.stopping.is_set(): commands = await self.client.commands() diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index 7039b8d50..6ccbc8a23 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -35,12 +35,15 @@ from pathlib import Path from typing import Any, Callable, Protocol, Sequence +from ..providers.agent import _redact from . import observability from . import outbound as ob from .backends import GOOGLE_ADK_APPROVAL_ENV from .bundle_v2 import BundleV2Error, EnvironmentBundleV2, RuntimeKindV2, load_bundle_v2 from .compose_broker import BrokerError, ComposeBrokerClient +from .framework_detection import FrameworkFolders, framework_folders_for from .hosted_compose_bootstrap import initialize_hosted_compose +from .source_folder import folder_failure from .world.observation import ObservationWorldFactory from .call_runner import CallRunnerContext, CallRunnerImpl from .hosted_scheduler import ( @@ -573,6 +576,29 @@ def _find_postgres_endpoint(runtime: EnvironmentRuntime) -> RuntimeEndpoint: ) +def _framework_folders( + job: HarnessJob, + manifest: EnvironmentBundleV2, + *, + source: Path, + bundle_dir: Path, + work_directory: Path, +) -> FrameworkFolders: + # Called outside the `try` around `build`: an exception here would end the run with no + # terminal event, and all that is at stake is a label. + try: + return framework_folders_for( + job, + manifest, + source=source, + bundle_dir=bundle_dir, + work_directory=work_directory, + ) + except Exception as exc: + logger.warning("framework folders failed: class=%s", type(exc).__name__) + return FrameworkFolders(source, source, None) + + def load_build_output(work_directory: Path) -> dict[str, Any]: """`write_build_output` (process_runtime.py) writes `/artifacts/build.json`. Read fresh each call — cheap, and the row counts are immutable after baseline freeze, so @@ -757,7 +783,8 @@ def _agent_instructions(job: Any, bundle_dir: Path) -> str: parts.append(f"Summary: {summary}") if rules: parts.append("Rules:\n- " + "\n- ".join(rules)) - return "\n\n".join(parts) + # The whole text, once: the job's own prompt, the summary and the rules may each hold a URL. + return _redact("\n\n".join(parts)) def _bundle_contract_value(bundle_dir: Path, key: str) -> str | None: @@ -2444,6 +2471,14 @@ async def _fail( logger.error("job.json invalid: %s", exc) return EXIT_CRASHED + if (unusable := folder_failure(source, job.source.subfolder)) is not None: + return await _fail( + domain=unusable.domain, + fail_stage=unusable.stage, + code=unusable.code, + message=unusable.message, + ) + try: broker_socket = await asyncio.to_thread(initialize_hosted_compose, job, source) except BrokerError as exc: @@ -2724,7 +2759,15 @@ async def _fail( if isinstance(scenario_source, NotWiredScenarioSource) and bundle_has_scenarios( bundle_dir ): - scenario_source = BundleScenarioSource() + scenario_source = BundleScenarioSource( + framework_folders=_framework_folders( + job, + manifest, + source=source, + bundle_dir=bundle_dir, + work_directory=work_directory, + ) + ) try: scenarios = await scenario_source.build( job, diff --git a/src/fi/alk/harness/instrumentation_evidence.py b/src/fi/alk/harness/instrumentation_evidence.py index 9706cc425..ba0e2c3ce 100644 --- a/src/fi/alk/harness/instrumentation_evidence.py +++ b/src/fi/alk/harness/instrumentation_evidence.py @@ -3,6 +3,10 @@ import ast import os from pathlib import Path +from typing import Iterable + +from .python_source import EscapeBudget, ParseBudget, parsed, read_module +from .source_folder import is_within _SKIPPED_DIRECTORIES = { ".git", @@ -19,15 +23,20 @@ "site-packages", "venv", } +# A file that names none of these can't import them, so it isn't parsed. +_NAMES = (b"fi_instrumentation", b"traceai_", b"opentelemetry") _OTEL_SDK_TRACE = "opentelemetry.sdk.trace" _OTEL_OTLP_EXPORTER = "opentelemetry.exporter.otlp" _OTEL_GRPC_EXPORTER = "opentelemetry.exporter.otlp.proto.grpc" -def detect_instrumentation(source_root: Path | None) -> str: +def detect_instrumentation(source_root: Path | None, reached: Iterable[Path] = ()) -> str: + """How the agent is traced, from its imports: the files in ``source_root``, and the Python + files in ``reached``, the code it imports from elsewhere in the repository, under one budget. + """ if source_root is None: return "none" - imported = _python_imports(source_root.resolve()) + imported = _python_imports(source_root.resolve(), reached) if any( name.split(".")[0] == "fi_instrumentation" or name.split(".")[0].startswith("traceai_") @@ -49,39 +58,50 @@ def detect_instrumentation(source_root: Path | None) -> str: return "opentelemetry" -def _python_imports(root: Path) -> set[str]: +def _python_imports(root: Path, reached: Iterable[Path] = ()) -> set[str]: imported: set[str] = set() local: set[str] = set() + budget = ParseBudget() + escapes = EscapeBudget() for directory, subdirectories, files in os.walk(root): if Path(directory) != root and any( name.endswith(".dist-info") for name in subdirectories ): subdirectories[:] = [] continue - subdirectories[:] = [ + # In name order: past the total, what's read mustn't depend on how the disk lists it. + subdirectories[:] = sorted( name for name in subdirectories if name not in _SKIPPED_DIRECTORIES - ] + ) if Path(directory).parent == root and "__init__.py" in files: # Only a package at the top of the repository sits ahead of the # installed one; app/opentelemetry is imported as app.opentelemetry. local.add(Path(directory).name) - for file_name in files: + for file_name in sorted(files): if not file_name.endswith(".py"): continue if Path(directory) == root: # Only a module at the top of the repository sits ahead of the # installed package on the import path. local.add(file_name[: -len(".py")]) - try: - tree = ast.parse( - (Path(directory) / file_name).read_text(encoding="utf-8") - ) - imported.update(_imports(tree)) - except (OSError, UnicodeError, SyntaxError, ValueError, RecursionError): - continue + _read_imports(Path(directory) / file_name, budget, escapes, imported) + for path in sorted(reached): + # The folder's own files were read above. + if path.suffix == ".py" and not is_within(path, root): + _read_imports(path, budget, escapes, imported) return {name for name in imported if name.split(".")[0] not in local} +def _read_imports( + path: Path, budget: ParseBudget, escapes: EscapeBudget, imported: set[str] +) -> None: + text = read_module(path, needles=_NAMES, escapes=escapes) + if text is None or not budget.spend(text): + return + if (tree := parsed(text)) is not None: + imported.update(_imports(tree)) + + def _imports(tree: ast.AST) -> set[str]: found: set[str] = set() # An explicit stack, because a generated file can nest deeper than the diff --git a/src/fi/alk/harness/job.py b/src/fi/alk/harness/job.py index 294156150..f35121a71 100644 --- a/src/fi/alk/harness/job.py +++ b/src/fi/alk/harness/job.py @@ -56,9 +56,21 @@ class RepositorySource(BaseModel): archive_artifact_id: str | None = None image: str | None = None endpoint: str | None = None + subfolder: str | None = None + + @model_validator(mode="before") + @classmethod + def _empty_subfolder_is_none(cls, data): + # The platform sends "" for "no folder"; the echo compares the value exactly, + # so anything else is kept as sent. + if isinstance(data, dict) and data.get("subfolder") == "": + return {**data, "subfolder": None} + return data @model_validator(mode="after") def _required_locator(self) -> "RepositorySource": + if self.subfolder and self.kind is not SourceKind.GITHUB: + raise ValueError("subfolder_requires_github_source") # A connected provider agent is itself the source of truth. Its ID lives # in AgentConnection.config and its definition is fetched with the # run-scoped provider credential during authoring, so no repository @@ -192,6 +204,27 @@ class ProviderExecutionMode(str, Enum): PROVIDER_IMPORT = "provider_import" +PROVIDER_MANAGED_CONNECTORS = frozenset({"vapi", "retell", "retell_chat"}) + + +def provider_profile_source(agent: AgentConnection) -> str | None: + """Where the build loads this agent's profile from, if it loads one. + + A profile from the provider's API or from the job's own config: ``"phone"`` or + ``"livekit"`` for a connect-only job on those connectors, ``"provider"`` for an import or a + connect-only job on a provider-managed connector, and ``None`` when there is none. + """ + connector = agent.connector.strip().lower() + if agent.mode is ProviderExecutionMode.CONNECT_ONLY and connector in {"phone", "livekit"}: + return connector + if agent.mode is ProviderExecutionMode.PROVIDER_IMPORT or ( + agent.mode is ProviderExecutionMode.CONNECT_ONLY + and connector in PROVIDER_MANAGED_CONNECTORS + ): + return "provider" + return None + + class ArtifactLevel(str, Enum): METADATA_ONLY = "metadata-only" TRACES = "traces" diff --git a/src/fi/alk/harness/model_citation.py b/src/fi/alk/harness/model_citation.py new file mode 100644 index 000000000..755e5315e --- /dev/null +++ b/src/fi/alk/harness/model_citation.py @@ -0,0 +1,808 @@ +"""Reading the model an agent talks to the person with, and the prompt it is given, from the +agent's own files, while a conversation waits on the answer. + +A model is kept only when its name is the value written on the line the reader cites: the +Agents page shows it as fact, and an unknown model is better left empty than guessed. The files +can be built to stall whoever reads them, so every path is resolved within the save's one lookup +budget, every read is bounded in size and decoded as UTF-8 whatever coding line the file carries, +and every check of a line costs at most a few passes over it. +""" + +from __future__ import annotations + +import functools +import html +import logging +import os +import re +import stat +import string +import unicodedata +from pathlib import Path + +from .contract import AgentContract +from .runtime_selection import charged_path +from .source_folder import is_within + +_LOG = logging.getLogger(__name__) +# The platform refuses the whole agent document past this, for the model's name and provider. +LONGEST_MODEL_FIELD = 255 +# One path as written: charged_path doesn't refuse a long path itself. +LONGEST_MODEL_PATH = 4096 +# The cited file, as for a prompt file. +LARGEST_MODEL_FILE = 1024 * 1024 +# What a model name is made of. A match must have none of these right before or after it, so +# "gpt-4o" is not found inside "gpt-4o-mini". +MODEL_NAME_CHARS = frozenset(string.ascii_letters + string.digits + "._:/@-") +# Files whose values are written bare after a key: `model: gpt-4o`, `MODEL=gpt-4o`, +# `ENV MODEL=gpt-4o`, `export MODEL=gpt-4o`. +CONFIG_SUFFIXES = frozenset( + { + ".yaml", ".yml", ".toml", ".json", ".ini", ".cfg", ".conf", ".env", ".properties", + ".sh", ".bash", ".zsh", ".dockerfile", ".mk", ".prompt", + } +) +_CONFIG_NAMES = frozenset({"makefile", "gnumakefile", ".flaskenv"}) +# Languages where # isn't a comment: C and C++ (#define), JavaScript and TypeScript (#field), +# Java, Go, Rust (#[attr]), Swift (#if), Kotlin, C#, Scala (A#B), Dart, Objective-C, Groovy, +# Gradle, Lua (#t is a length). +_NO_HASH_COMMENT_SUFFIXES = frozenset( + { + ".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", + ".java", ".go", ".rs", ".swift", ".kt", ".kts", ".cs", ".scala", ".dart", ".m", ".mm", + ".groovy", ".gradle", ".lua", ".mts", ".cts", ".sc", + } +) +# Files where # starts a comment only at a line's start or after a space: YAML, shell, Makefile +# and Dockerfile (a#b is a word there), and .env files. +_SPACED_HASH_SUFFIXES = frozenset( + {".yaml", ".yml", ".sh", ".bash", ".zsh", ".mk", ".dockerfile", ".env"} +) +# Files with no block comments, nor //: there `* ` starts no comment (a Python continuation +# line), and /* and // are text (ls /*, a // 2). +_NO_BLOCK_COMMENT_SUFFIXES = frozenset( + {".py", ".pyi", ".pyw", ".rb", ".pl", ".r", ".jl", ".ex", ".exs", ".nim", ".lua"} +) +_YAML_SUFFIXES = frozenset({".yaml", ".yml"}) +# INI files: a line starting with ; is a comment. +_INI_SUFFIXES = frozenset({".ini", ".cfg", ".conf"}) +# Files where -- starts a comment: SQL, Lua, Haskell. +_DASH_COMMENT_SUFFIXES = frozenset({".sql", ".lua", ".hs"}) +# Files written for people: they say which model is used; they don't set it. +# And prompt templates (.j2, .hbs): their text is the agent's prompt, which may name a model +# without setting it. +PROSE_SUFFIXES = frozenset( + { + ".md", ".markdown", ".mdx", ".mdc", ".rst", ".txt", ".adoc", ".org", ".j2", ".jinja", + ".jinja2", ".hbs", ".mustache", + } +) +# Files written for people, by name with no suffix: README, CHANGELOG, LICENSE, CONTRIBUTING +# (README.md is prose by its suffix; readme.py is code). +_PROSE_STEMS = frozenset({"readme", "changelog", "license", "contributing"}) +# Words a setting is named with, never a model: `cfg.get("model")`, OPENAI_MODEL, llm.model. +# Not "deployment": an Azure deployment's own name often has it (gpt-4o_deployment). +_SETTING_WORDS = frozenset({"model", "llm", "name", "default", "engine", "provider", "key", "env", "id"}) +_QUOTES = frozenset("\"'`") +_SETTING_NAME_CHARS = frozenset(string.ascii_uppercase + string.digits + "_") +_SETTING_PARTS = re.compile(r"[_\-.:/@]") +# An escape in a string: \uXXXX and \xXX (as json.dump writes any non-ASCII character), +# \UXXXXXXXX, \N{NAME} and octal. +_CHAR_ESCAPE = re.compile( + r"\\(?:u([0-9a-fA-F]{4})|x([0-9a-fA-F]{2})|U([0-9a-fA-F]{8})|N\{([A-Za-z0-9 \-]{1,88})\}" + r"|([0-7]{1,3}))" +) +_NOT_A_LETTER = re.compile(r"[\W_]+") +_SURROGATE = re.compile("[\ud800-\udfff]") +# Full-width letters and digits (3, A) as their ASCII forms: what NFKC would do for them, +# without NFKC's growth elsewhere. +_FULL_WIDTH = {code: code - 0xFEE0 for code in range(0xFF01, 0xFF5F)} +# Scripts written without spaces between words (Thai, Lao, Burmese, Khmer, Japanese kana, +# Chinese and Japanese ideographs): each character is a word, so the gap below applies between +# characters there too. +_UNSPACED = "\u0e00-\u0eff\u1000-\u109f\u1780-\u17ff\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff" +_UNSPACED_CHAR = re.compile(f"[{_UNSPACED}]") +# Only a letter of those scripts is a word: their vowel signs and tone marks, and ・, are not. +_WORD = re.compile(f"(?![\\W_])[{_UNSPACED}]|[^\\W_{_UNSPACED}]+") +# A quote of fewer letters than this says nothing to check lines against ("n/a", "TBD", "see +# lines"): a few short words are found in almost any lines. A character of an unspaced script +# counts as three. +_SHORTEST_QUOTE = 16 +# Fewer letters than this and a quote says nothing at all ("n/a", "TBD", "none"): it isn't held +# and isn't checked. +_SAYS_NOTHING = 6 +# Languages with triple-quoted strings ("""...""" or '''...'''): Python, TOML, Kotlin, Scala, +# Swift, Groovy and Gradle, Java and C# text blocks, Julia, Dart, Elixir. +_TRIPLE_SUFFIXES = frozenset( + { + ".py", ".pyi", ".pyw", ".toml", ".kt", ".kts", ".scala", ".sc", ".swift", ".groovy", + ".gradle", ".java", ".cs", ".jl", ".dart", ".ex", ".exs", + } +) +# Code languages where . +_NO_MARKUP_COMMENT_SUFFIXES = ( + _NO_HASH_COMMENT_SUFFIXES | _NO_BLOCK_COMMENT_SUFFIXES | _DASH_COMMENT_SUFFIXES +) +# Where a quote leaves text out: ... or …, also inside brackets, since only letters are compared. +_ELISION = re.compile(r"\.{3,}|…") +# How many letters of code may sit between two words of a quote in the lines: a string prefix +# (f"), a short comment, `+ NAME +`, a markup tag or a placeholder the quote leaves out. +_WORD_GAP = 24 + + +# How # starts a comment: anywhere (Python, shell, Ruby, TOML...), only at a line's start or +# after a space (YAML, where `url: http://x/#top` and `{a: 1,#b}` are values), or not at all. +_HASH_ANYWHERE, _HASH_SPACED, _HASH_NONE = "anywhere", "spaced", "none" + + +@functools.cache +def _comment_tokens( + *, dashes: bool, triple: bool, hash_comments: str, block_comments: bool +) -> re.Pattern[str]: + """What the line scan stops at: a triple quote (in languages that have them), a + double-quoted or backticked string, matched whole (to the line's end if it isn't closed); an + apostrophe, which the scan pairs itself; or a comment marker. # not in $# (a shell count), + &# (an HTML character) or ${# (a shell length).""" + alternatives = ['"""', "'''"] if triple else [] + alternatives += [r'"(?:[^"\\]|\\.)*"?', r"`(?:[^`\\]|\\.)*`?", "'"] + if hash_comments == _HASH_ANYWHERE: + alternatives.append(r"(? bool: + """Whether the apostrophe at ``at`` can open a string: not inside a word (Pair's, it's, x' + in Haskell) and not after a backslash (it\\'s), unless the word is a string prefix (r'...', + rb'...'). Looks at three characters at most: a prefix is one or two letters, so three word + characters before the apostrophe (letters, digits, _) are a word, whatever comes before.""" + start = at + while start > 0 and at - start < 3 and (line[start - 1].isalnum() or line[start - 1] == "_"): + start -= 1 + if start == at: + return at == 0 or line[at - 1] != "\\" + return line[start:at].lower() in _STRING_PREFIXES + + +# What closes a block comment left open at a line's end. +_COMMENT_CLOSERS = frozenset({"*/", "-->"}) + + +def _line_scan( + line: str, + block: str | None, + *, + dashes: bool = False, + ini: bool = False, + triple: bool = False, + hash_comments: str = _HASH_ANYWHERE, + block_comments: bool = True, + markup: bool = False, +) -> tuple[int, list[tuple[int, int]], str | None]: + """One pass over a line: where its comment starts (or its length), the stretches of it that + are text (inside a triple-quoted string, or a block comment closed on the line), and the + block still open at its end (a triple quote, or the */ or --> a comment ends at; --> only in + ``markup``, where " + close = line.find(closer, at) + if close < 0: + if token == " of a + block comment), the lines before it read with the same scan as the cited one. A comment is + carried to later lines only when its closer is on a later line of the file: an opener that + never closes is text (a regex, a template), not a comment over the rest of the file. A line + that could open none, outside a block, is passed over without a scan; inside one, only its + closer is looked for.""" + openers = ['"""', "'''"] if triple else [] + if block_comments: + openers.append("/*") + if markup: + openers.append(", + open where the line starts), and + between matching quotes (not a key, not a lookup), or in a configuration file written bare + after ``:`` or ``=`` or as a shell default, ``${MODEL:-gpt-4o}``. ``name`` is made only of + MODEL_NAME_CHARS, so an occurrence starting inside one that failed has a name character just + before it, and the search resumes after the failed one. The line is scanned once, its + triple-quoted stretches walked alongside the occurrences, and each step looks at a few + characters beside one, so a line costs about one pass.""" + if not name: + # find("") matches at every position without moving on. + return False + comment, inside, _ = _line_scan( + line, + block, + dashes=dashes, + ini=ini, + triple=triple, + hash_comments=hash_comments, + block_comments=block_comments, + ) + stretches = iter(inside) + stretch = next(stretches, None) + at = line.find(name, 0, comment) + while at >= 0: + end = at + len(name) + while stretch is not None and stretch[1] <= at: + stretch = next(stretches, None) + before = line[at - 1] if at else "" + after = line[end] if end < len(line) else "" + # Inside a triple-quoted string or a block comment the line is text, unless the string is + # the value: MODEL = """gpt-4o""". + inside_text = stretch is not None and stretch[0] <= at and stretch != (at, end) + if not inside_text and after not in MODEL_NAME_CHARS: + if before not in MODEL_NAME_CHARS: + if before and before in _QUOTES and after == before: + if not _is_a_key(line, at, end) and not _is_a_lookup(line, at): + return True + elif config and _bare_after_key(line, at, yaml=yaml): + return True + elif config and after == "}" and line[max(0, at - 2) : at] == ":-": + # The default of a shell variable: ${MODEL:-gpt-4o}. + return True + at = line.find(name, end, comment) + return False + + +# Prompt and page templates: what they hold is text, unless the name says they render a +# configuration file (config.yaml.j2), which is then read as that file. +_TEMPLATE_SUFFIXES = frozenset({".j2", ".jinja", ".jinja2", ".hbs", ".mustache"}) + + +def kind_suffix(path: Path) -> str: + """The suffix that says what a file is: config.yaml.j2 is YAML.""" + suffix = path.suffix.lower() + if suffix in _TEMPLATE_SUFFIXES: + inner = Path(path.stem).suffix.lower() + # Not .prompt: system.prompt.j2 is a Jinja prompt, not a Dotprompt file. + if inner in CONFIG_SUFFIXES and inner != ".prompt": + return inner + return suffix + + +def is_config_file(path: Path) -> bool: + name = path.name.lower() + suffix = kind_suffix(path) + # env.py and dockerfile_gen.py are code, whatever their names start with. + code = suffix in _TRIPLE_SUFFIXES | _NO_HASH_COMMENT_SUFFIXES | _NO_BLOCK_COMMENT_SUFFIXES + return ( + suffix in CONFIG_SUFFIXES + or name in _CONFIG_NAMES + or (not code and name.startswith((".env", "env.", "dockerfile"))) + ) + + +def _names_configuration(name: str) -> bool: + """A file name that is configuration by itself: Makefile, .env, .env.local, Dockerfile, + Dockerfile.dev.""" + return ( + name in _CONFIG_NAMES + or name in (".env", "dockerfile") + or name.startswith((".env.", "dockerfile.")) + ) + + +def is_prose_file(path: Path) -> bool: + if path.suffix.lower() in _TEMPLATE_SUFFIXES and _names_configuration(Path(path.stem).name.lower()): + # A template that renders configuration by its name: .env.j2, Dockerfile.j2 (not + # dockerfile_writer.j2 or env.system.j2, which are prompts). + return False + return kind_suffix(path) in PROSE_SUFFIXES or ( + not path.suffix and path.name.lower() in _PROSE_STEMS + ) + + +def _reads_as_a_setting(name: str) -> bool: + """A name shaped like where a model is read from rather than which: OPENAI_MODEL, + openai_model, model-name, llm.model. Model and deployment ids such as GPT4O, O1, + mistral_large, gpt4o-deployment and gemini-1.5-pro are not.""" + # An environment variable: capitals, digits and underscores, with an underscore. + if "_" in name and _SETTING_NAME_CHARS.issuperset(name): + return True + has_digit = any(char.isdigit() for char in name) + parts = _SETTING_PARTS.split(name.casefold()) + # A setting's words, alone, in snake case or without a version number: model, llm_model, + # model-name. + if ("_" in name or not has_digit) and _SETTING_WORDS.intersection(parts): + return True + # A dotted key with no version in it: services.openai.model, llm.provider. + return "." in name and not has_digit + + +def model_problems(contract: AgentContract, source_root: Path) -> list[str]: + """Problems with contract.model, checked against the cited line. On success, strips name and + provider and stores path as the resolved file relative to the root, in POSIX form. Resolves + one path (charged to the save's budget) and reads at most one file.""" + model = contract.model + if model is None: + return [] + name, provider, path = model.name.strip(), model.provider.strip(), model.path.strip() + shown_name, shown_path = name[:100], path[:200] + problems: list[str] = [] + if not name: + problems.append( + "model: name is empty; give the model's name as written in the source, or leave " + "model out" + ) + elif not MODEL_NAME_CHARS.issuperset(name): + problems.append( + "model: a model name is letters, digits and . _ : / @ -; write it as it appears in " + "the source, or leave model out" + ) + elif _reads_as_a_setting(name): + # OPENAI_MODEL in os.getenv("OPENAI_MODEL") is where the model is read from, not which. + problems.append( + f"model: {shown_name} reads as the name of a setting, not a model; cite the line its " + "value is written on, or leave model out" + ) + if not MODEL_NAME_CHARS.issuperset(provider): + problems.append( + "model: a provider is letters, digits and . _ : / @ -; write it as the source names " + "it, or leave it out" + ) + if len(name) > LONGEST_MODEL_FIELD or len(provider) > LONGEST_MODEL_FIELD: + problems.append("model: name and provider are at most 255 characters") + if not path or model.line < 1: + problems.append( + "model: give path and line, the file and the line the model's name is written on" + ) + if len(path) > LONGEST_MODEL_PATH: + problems.append("model: path is too long") + # All of these at once, and before the disk: a citation wrong in its shape costs no lookup. + if problems: + return problems + root = source_root.resolve() + resolved = resolved_in_source(root, path) + if resolved is None: + return [f"model: {shown_path} is not a file in the source"] + if is_prose_file(resolved): + # A README, a text file or a prompt template says which model is used; the code or its + # configuration sets it. + return [ + f"model: {shown_path} is text (for people, or a prompt), not code or configuration; " + "cite the line in the agent's code or configuration the model is set on, or leave " + "model out" + ] + raw = read_source(resolved, LARGEST_MODEL_FILE) + if raw is None or len(raw) > LARGEST_MODEL_FILE: + return [ + f"model: {shown_path} can't be read as a file of at most 1 MiB; check the path, and " + "cite the line in the file the name is written in" + ] + lines = source_lines(raw) + if model.line > len(lines): + return [f"model: line {model.line} is outside {shown_path}, which has {len(lines)} lines"] + suffix = kind_suffix(resolved) + dashes, ini = suffix in _DASH_COMMENT_SUFFIXES, suffix in _INI_SUFFIXES + triple = suffix in _TRIPLE_SUFFIXES + config = is_config_file(resolved) + name_lower = resolved.name.lower() + if suffix in _NO_HASH_COMMENT_SUFFIXES: + hash_comments = _HASH_NONE + elif ( + suffix in _SPACED_HASH_SUFFIXES + or name_lower in _CONFIG_NAMES + # dockerfile_gen.py and .env_loader.py are Python: # anywhere there. + or (config and name_lower.startswith((".env", "dockerfile"))) + ): + hash_comments = _HASH_SPACED + else: + hash_comments = _HASH_ANYWHERE + # Configuration files have no block comments, but JSON read as JSONC (appsettings.json, + # tsconfig.json) has: outside a string, // and /* can only be comments there. + block_comments = suffix == ".json" or not (config or suffix in _NO_BLOCK_COMMENT_SUFFIXES) + if not named_on( + lines[model.line - 1], + name, + config=config, + yaml=suffix in _YAML_SUFFIXES, + dashes=dashes, + ini=ini, + triple=triple, + hash_comments=hash_comments, + block_comments=block_comments, + # A block open from an earlier line: a triple quote, /* where block comments are, ', "gpt-4o", id="markup"), + ], +) +def test_comments_without_spaces_are_comments(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +# Prose names models, it doesn't set them + + +@pytest.mark.parametrize( + "file, written", + [ + pytest.param("README.md", "This agent uses `gpt-4o` (we tried `claude-3-opus`)", id="readme"), + pytest.param("README.md", "| fallback | `claude-3-opus` |", id="table-cell"), + pytest.param("README.md", 'This agent uses "gpt-4o" (we tried "claude-3-opus")', id="straight-quotes"), + pytest.param("README.md", '* default model: "claude-3-opus"', id="list-item"), + pytest.param("notes.txt", "tried `claude-3-opus`", id="text"), + pytest.param("index.rst", "uses `claude-3-opus` here", id="rst"), + pytest.param("guide.markdown", 'model: "claude-3-opus"', id="markdown"), + pytest.param("guide.adoc", 'model: "claude-3-opus"', id="asciidoc"), + pytest.param("page.mdx", 'model: "claude-3-opus"', id="mdx"), + pytest.param("README", 'model: "claude-3-opus"', id="readme-without-a-suffix"), + ], +) +def test_prose_files_hold_no_model(root, file, written): + (root / file).write_text(f"# notes\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "claude-3-opus", "path": file, "line": 2}) + + assert problems == [prose(file)] + + +def test_backticks_in_code_are_values(root): + (root / "llm.ts").write_text("// the model\nconst model = `claude-3-opus`;\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "claude-3-opus", "path": "llm.ts", "line": 2}) + + assert problems == [] + + +@pytest.mark.parametrize( + "file, written", + [ + pytest.param("Makefile", "MODEL ?= gpt-4o", id="makefile"), + pytest.param("rules.mk", "MODEL := gpt-4o", id="make-include"), + pytest.param(".flaskenv", "MODEL=gpt-4o", id="flaskenv"), + pytest.param("run.sh", 'python run.py --model "${MODEL:-gpt-4o}"', id="shell-default"), + pytest.param(".env", "MODEL=${MODEL:-gpt-4o}", id="env-default"), + ], +) +def test_more_configuration_forms(root, file, written): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": "gpt-4o", "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == "gpt-4o" + + +def test_a_shell_default_is_only_found_in_configuration(root): + (root / "llm.py").write_text('# the model\nx = ${MODEL:-gpt-4o}\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "llm.py")] + + +# The reader never waits on a pipe, and leaves a trace when it can't read + + +def test_a_pipe_is_not_waited_on(tmp_path): + from fi.alk.harness.model_citation import read_source + + if not hasattr(os, "mkfifo"): + pytest.skip("no named pipes here") + pipe = tmp_path / "pipe" + os.mkfifo(pipe) + answer: list[bytes | None] = [] + reading = threading.Thread(target=lambda: answer.append(read_source(pipe, 10)), daemon=True) + reading.start() + reading.join(timeout=2) + + assert answer == [None] + + +def test_a_file_read_in_short_pieces_is_read_whole(tmp_path, monkeypatch): + from fi.alk.harness import model_citation + + (tmp_path / "agent.py").write_bytes(b"x" * 100) + real_read = os.read + monkeypatch.setattr(model_citation.os, "read", lambda fd, n: real_read(fd, min(n, 7))) + + assert model_citation.read_source(tmp_path / "agent.py", 50) == b"x" * 51 + assert model_citation.read_source(tmp_path / "agent.py", 200) == b"x" * 100 + + +def test_an_unreadable_file_is_logged_without_its_content(tmp_path, caplog): + from fi.alk.harness.model_citation import read_source + + locked = tmp_path / "locked.py" + locked.write_text('SECRET = "do-not-log"\n', encoding="utf-8") + locked.chmod(0) + try: + if os.access(locked, os.R_OK): + pytest.skip("this user can read a file with no permissions") + with caplog.at_level("INFO", logger="fi.alk.harness.model_citation"): + assert read_source(locked, 100) is None + finally: + locked.chmod(0o600) + + assert [record.getMessage() for record in caplog.records] == [ + "source file not read (PermissionError)" + ] + assert str(locked) not in caplog.text + assert "do-not-log" not in caplog.text + + +def test_a_link_put_in_place_of_the_file_is_not_followed(tmp_path): + from fi.alk.harness.model_citation import read_source + + (tmp_path / "outside.py").write_text("SECRET\n", encoding="utf-8") + link("outside.py", tmp_path / "agent.py") + + assert read_source(tmp_path / "agent.py", 100) is None + + +def test_mismatched_quotes_are_not_a_value(root): + (root / "llm.py").write_text("# settings\nmodel = \"gpt-4o'\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "llm.py")] + + +# The quote is compared by its letters, so honest ways of writing a prompt fit + + +@pytest.mark.parametrize( + "file, written", + [ + pytest.param( + "prompt.py", + 'PROMPT = ("You are pair, "\n "a helpful assistant who books tables.")', + id="string-pieces", + ), + pytest.param( + "prompt.py", + 'PROMPT = "You are pair,\\na helpful assistant who books tables."', + id="escaped-newline", + ), + pytest.param( + "prompt.json", + '{"prompt": "You are pair,\\na helpful assistant who books tables."}', + id="json", + ), + ], +) +def test_honest_prompt_forms_hold_the_quote(tmp_path, root, file, written): + (root / file).write_text(written + "\n", encoding="utf-8") + count = len(written.splitlines()) + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant who books tables.", + system_prompt_from=[{"path": file, "start_line": 1, "end_line": count}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written + + +_LONG_PROMPT = "You are pair, a helpful assistant who books tables for guests. " * 4 + + +@pytest.mark.parametrize( + "quote, refused", + [ + # A quote that says where it trails off fits. + pytest.param(_LONG_PROMPT[:120] + " ...", False, id="trails-off-with-dots"), + # The start matches and the rest differs: every part of the quote must be in the lines. + pytest.param(_LONG_PROMPT[:120] + " and something else entirely", True, id="differs-late"), + # The quote differs from the lines near its start. + pytest.param("You are pair, a helpful assistant who books rooms for guests.", True, id="differs-early"), + # The lines hold the quote's start but are shorter than the quote. + pytest.param(_LONG_PROMPT * 3, True, id="lines-shorter-than-the-quote"), + ], +) +def test_the_quote_window(tmp_path, root, quote, refused): + (root / "prompt.py").write_text(f'PROMPT = """{_LONG_PROMPT}"""\n', encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + if refused: + assert text_of(result) == REFUSED + not_the_quote("prompt.py 1-1") + else: + assert not result.get("is_error"), result + + +def test_lines_refused_twice_leave_the_quote(tmp_path, root): + (root / "prompt.py").write_text('import os\nDB = "x"\n', encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + wrong = payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant.", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 2}], + ) + + first = submit(wrong) + + assert text_of(first) == ( + REFUSED + + not_the_quote("prompt.py 1-2") + + "\n - system_prompt_from: lines named later are checked against this quote until you " + "give another; a later mismatch records the lines as unconfirmed and keeps the quote, " + "or saves no prompt if the quote has fewer than 16 letters" + ) + second = submit(wrong) + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "You are pair, a helpful assistant." + assert saved(destination)["system_prompt_from"] == [] + assert saved(destination)["unconfirmed_prompt_from"] == wrong["system_prompt_from"] + + +# Every piece of the quote must be in the lines named, in order + +_BOOKING = ( + "import os\n" + "import json\n" + "from acme.tools import book\n" + "from acme.tools import cancel\n" + "\n" + "PROMPT = (\n" + ' "You are Acme\'s booking agent. "\n' + ' "Help guests book tables. "\n' + ' "Never share other guests\' bookings. "\n' + ' "Confirm every booking before saving it. "\n' + ' "Be warm and brief."\n' + ")\n" +) +_BOOKING_QUOTE = ( + "You are Acme's booking agent. Help guests book tables. Never share other guests' bookings. " + "Confirm every booking before saving it. Be warm and brief." +) + + +@pytest.mark.parametrize( + "quote, lines, refused", + [ + # Starting early pads the range, but its last line is still missing. + pytest.param(_BOOKING_QUOTE, (1, 10), True, id="starts-early-stops-short"), + pytest.param(_BOOKING_QUOTE, (6, 12), False, id="the-prompt"), + pytest.param("You are Acme's booking agent. ... Be warm and brief.", (6, 12), False, id="elided"), + pytest.param("You are Acme's booking agent. ... Be warm and brief.", (1, 10), True, id="elided-stops-short"), + pytest.param("Be warm and brief. ... You are Acme's booking agent.", (6, 12), True, id="out-of-order"), + pytest.param("You are Acme's booking agent. [...] Be warm and brief.", (6, 12), False, id="bracketed-dots"), + pytest.param("You are Acme's booking agent. … Be warm and brief.", (6, 12), False, id="ellipsis-character"), + pytest.param("YOU ARE ACME'S BOOKING AGENT. ... BE WARM AND BRIEF.", (6, 12), False, id="case"), + pytest.param("... Confirm every booking before saving it. Be warm and brief.", (6, 12), False, id="starts-with-dots"), + pytest.param("...", (1, 4), False, id="only-dots"), + # One line can't hold a piece the quote has twice. + pytest.param("Be warm and brief. ... Be warm and brief.", (6, 12), True, id="a-piece-twice"), + ], +) +def test_a_range_that_stops_short_is_refused(tmp_path, root, quote, lines, refused): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "agent.py", "start_line": lines[0], "end_line": lines[1]}], + ), + destination, + source_root=root, + ) + + if refused: + assert text_of(result) == REFUSED + not_the_quote(f"agent.py {lines[0]}-{lines[1]}") + else: + assert not result.get("is_error"), result + copied = "\n".join(_BOOKING.splitlines()[lines[0] - 1 : lines[1]]) + assert saved(destination)["system_prompt_excerpt"] == copied + + +@pytest.mark.parametrize( + "written, quote", + [ + pytest.param('PROMPT = "Book at Tom & Jerry\'s diner tonight"', "Book at Tom & Jerry's diner tonight", id="entity-in-the-source"), + pytest.param('PROMPT = "Book at Tom & Jerry\'s diner tonight"', "Book at Tom & Jerry's diner tonight", id="entity-in-the-quote"), + pytest.param('PROMPT = "Order a caf\u00e9 and wait for the bill"', "Order a cafe\u0301 and wait for the bill", id="decomposed-accent"), + ], +) +def test_an_honest_quote_in_another_form_fits(tmp_path, root, written, quote): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written + + +def test_lines_sent_without_the_refused_quote_are_still_checked(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=imports, + ) + ) + assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 1-4")) + + # The quote left out: the imports are still checked against it, asked about once more... + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) + assert not (destination / "contract.json").exists() + + # ... and then the quote is what's sent, with the lines on record as unconfirmed. + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["system_prompt_from"] == [] + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +def test_the_right_lines_after_a_refusal_are_copied(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + ) + assert first.get("is_error"), first + + second = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}], + ) + ) + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + + +def test_a_new_quote_replaces_the_refused_one(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + first = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant.", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + ) + assert first.get("is_error"), first + + second = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}], + ) + ) + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "\n".join(_BOOKING.splitlines()[5:12]) + + +def test_a_quote_of_many_pieces_does_not_hold_up_the_check(root): + from fi.alk.harness.model_citation import quote_problem + + copied = "a" * MIB + started = time.perf_counter() + answer = quote_problem("a ... " * 50_000 + "b", copied, "agent.py 1-1") + took = time.perf_counter() - started + + assert answer is not None + assert took < 0.5 + + +# A prompt written with Python string prefixes holds its quote + + +@pytest.mark.parametrize( + "written", + [ + pytest.param('PROMPT = ("You are pair, " f"a helpful assistant who books tables.")', id="f-string-piece"), + pytest.param('PROMPT = ("You are pair, " r"a helpful assistant who books tables.")', id="raw-piece"), + pytest.param("PROMPT = ('You are pair, ' rb'a helpful assistant who books tables.')", id="two-letter-prefix"), + pytest.param('PROMPT = ("You are pair, " F"a helpful assistant who books tables.")', id="capital-prefix"), + pytest.param('PROMPT = f"You are pair, " "a helpful assistant who books tables."', id="prefix-on-the-first-piece"), + pytest.param('PROMPT = (u"You are pair, " fr"a helpful assistant who books tables.")', id="u-and-fr"), + ], +) +def test_a_prompt_with_string_prefixes_holds_the_quote(tmp_path, root, written): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair, a helpful assistant who books tables.", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == written + + +@pytest.mark.parametrize( + "written, quote, refused", + [ + # A word ending in a prefix letter keeps that letter, on both sides. + pytest.param('PROMPT = "He said" " hello and goodbye."', 'He said hello and goodbye.', False, id="word-before-a-quote"), + pytest.param('PROMPT = "Pick one of" "f the options."', "Pick one off the options.", False, id="prefix-letter-inside-a-string"), + ], +) +def test_a_word_before_a_quote_is_not_a_prefix(tmp_path, root, written, quote, refused): + (root / "prompt.py").write_text(written + "\n", encoding="utf-8") + destination = tmp_path / "out" + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=quote, + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + destination, + source_root=root, + ) + + if refused: + assert text_of(result) == REFUSED + not_the_quote("prompt.py 1-1") + else: + assert not result.get("is_error"), result + + +# A value inside a docstring or a multi-line string is not the model + +_DOCSTRING = ( + "def build():\n" + ' """Build the agent.\n' + "\n" + ' Example: model = "gpt-4o"\n' + ' """\n' + ' return Agent(model="claude-3-opus")\n' +) + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("llm.py", _DOCSTRING, 4, "gpt-4o", id="docstring-line"), + pytest.param("llm.py", "X = '''\nmodel = \"gpt-4o\"\n'''\n", 2, "gpt-4o", id="single-quote-block"), + pytest.param("llm.py", 'PROMPT = """You are pair.\nUse "gpt-4o" for hard cases.\n"""\n', 2, "gpt-4o", id="prompt-line"), + pytest.param("llm.py", 'PROMPT = """Use "gpt-4o" for hard cases."""\n', 1, "gpt-4o", id="opened-on-the-line"), + pytest.param("llm.toml", 'notes = """\nmodel = "gpt-4o"\n"""\n', 2, "gpt-4o", id="toml-multiline"), + pytest.param("llm.py", '"""Docs with \'\'\' inside.\nmodel = "gpt-4o"\n"""\n', 2, "gpt-4o", id="other-kind-does-not-close"), + pytest.param("llm.py", 'X = """docs \'\'\' and "gpt-4o"\n"""\n', 1, "gpt-4o", id="other-kind-on-the-line-does-not-close"), + pytest.param("llm.py", 'X = """gpt-4o" is the default"""\n', 1, "gpt-4o", id="a-block-that-opens-at-the-name"), + ], +) +def test_a_value_inside_a_triple_quoted_block_is_not_the_model(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [not_named(name, line, file)] + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("llm.py", _DOCSTRING, 6, "claude-3-opus", id="closed-before"), + pytest.param("llm.py", 'DOC = """notes"""; MODEL = "gpt-4o"\n', 1, "gpt-4o", id="closed-on-the-line"), + pytest.param("llm.py", 'MODEL = """gpt-4o"""\n', 1, "gpt-4o", id="triple-quoted-value"), + pytest.param("llm.py", "MODEL = '''gpt-4o'''\n", 1, "gpt-4o", id="triple-single-quoted-value"), + pytest.param("llm.py", 'X = """a\nb"""\nMODEL = "gpt-4o"\n', 3, "gpt-4o", id="block-closed-on-an-earlier-line"), + ], +) +def test_a_value_after_a_closed_block_is_the_model(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "written", + [ + pytest.param('"""' * (MIB // 3 - 20) + '\nMODEL = "gpt-4o"\n', id="a-file-of-triple-quotes"), + pytest.param('"""' + '""" "gpt-4o" """' * (MIB // 20), id="a-line-of-blocks"), + ], +) +def test_a_crafted_file_of_triple_quotes_does_not_hold_up_the_check(root, written): + (root / "crafted.py").write_text(written, encoding="utf-8") + line = written.count("\n") + (0 if written.endswith("\n") else 1) + + _, _, took = checked(root, {"name": "gpt-4o", "path": "crafted.py", "line": line}) + + assert took < 0.5 + + +def test_leaving_the_quote_out_is_not_asked_about(tmp_path, root): + # Following the advice to leave the quote out must not set off the "no prompt" question. + (root / "prompt.py").write_text('PROMPT = "You are pair."\n', encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + result = submit( + payload(system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}]) + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == 'PROMPT = "You are pair."' + + +def test_a_prompt_path_through_a_link_outside_is_refused(tmp_path, root): + (tmp_path / "outside.py").write_text('PROMPT = "elsewhere"\n', encoding="utf-8") + link("../outside.py", root / "inside.py") + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_from=[{"path": "inside.py", "start_line": 1, "end_line": 1}], + ), + tmp_path / "out", + source_root=root, + ) + + assert text_of(result) == ( + REFUSED + "system_prompt_from[0]: inside.py is not a file in the source" + ) + + +# What is looked up is not the model, however it is spelled + + +@pytest.mark.parametrize( + "written, name", + [ + # Keys shaped like model names, so the setting-shape rule passes them and the lookup is + # what refuses them. + pytest.param('MODEL = os.getenv("fast-4")', "fast-4", id="getenv-hyphenated"), + pytest.param('MODEL = cfg.get("chat.v2")', "chat.v2", id="get-dotted"), + pytest.param('MODEL = os.getenv("gpt_4o")', "gpt_4o", id="getenv-with-a-digit"), + pytest.param('MODEL = os.environ["gpt-4o"]', "gpt-4o", id="subscript"), + pytest.param('model = this.config.get("gpt-4o");', "gpt-4o", id="get-generic"), + pytest.param('model = configService.getOrThrow("gpt-4o");', "gpt-4o", id="get-or-throw"), + pytest.param('model = props.getProperty("gpt-4o");', "gpt-4o", id="get-property"), + pytest.param('MODEL = os.environ.get ("gpt-4o")', "gpt-4o", id="space-before-the-bracket"), + pytest.param('MODEL = env.str("gpt-4o")', "gpt-4o", id="env-str"), + pytest.param("$model = config('gpt-4o');", "gpt-4o", id="config-call"), + pytest.param('model = env.getOrDefault("gpt-4o", x);', "gpt-4o", id="get-or-default"), + pytest.param('model = bundle.getString("gpt-4o");', "gpt-4o", id="get-string"), + pytest.param('model = config.getValue("gpt-4o");', "gpt-4o", id="get-value"), + pytest.param("$model = env('gpt-4o');", "gpt-4o", id="env-call"), + ], +) +def test_a_lookup_key_is_not_the_model(root, written, name): + (root / "llm.py").write_text(f"import os\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": "llm.py", "line": 2}) + + assert problems == [not_named(name, 2, "llm.py")] + + +@pytest.mark.parametrize( + "written, name", + [ + pytest.param('MODELS = ["gpt-4o", "gpt-4o-mini"]', "gpt-4o", id="list-literal"), + pytest.param('model = genai.GenerativeModel("gemini-1.5-pro")', "gemini-1.5-pro", id="constructor"), + pytest.param('MODEL = os.getenv("OPENAI_MODEL", "gpt-4o")', "gpt-4o", id="getenv-default"), + pytest.param('llm = target("gpt-4o")', "gpt-4o", id="a-call-ending-in-get"), + pytest.param('llm = my_get("gpt-4o")', "gpt-4o", id="a-call-ending-in-_get"), + pytest.param('["gpt-4o", "gpt-4o-mini"]', "gpt-4o", id="a-list-at-the-line-start"), + ], +) +def test_a_value_passed_to_a_call_is_the_model(root, written, name): + (root / "llm.py").write_text(f"import os\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": "llm.py", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("llm.ts", 'const model = "gpt-4o"; /* was "gpt-4o-mini" */', "gpt-4o-mini", id="block-comment"), + pytest.param("llm.html", '', "gpt-4o-mini", id="markup-comment"), + pytest.param("llm.html", 'x ', "gpt-4o-mini", id="trailing-markup-comment"), + ], +) +def test_a_value_in_a_block_comment_is_not_the_model(root, file, written, name): + (root / file).write_text(f"# settings\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +@pytest.mark.parametrize( + "file, written, line", + [ + pytest.param("a.js", '/*\nmodel: "gpt-4o"\n*/\n', 2, id="a-block-over-lines"), + pytest.param( + "a.js", + 'const a = 1; /* was\nnotes\nconst model = "gpt-4o";\n*/\n', + 3, + id="opened-after-code", + ), + pytest.param( + "a.js", '/* was\nconst model = "gpt-4o"; */\n', 2, id="before-the-closer" + ), + pytest.param( + "a.ts", + '/** Settings.\n @default "gpt-4o"\n */\n', + 2, + id="a-doc-comment-without-stars", + ), + pytest.param( + "page.html", + '\n', + 2, + id="a-markup-block-over-lines", + ), + pytest.param( + "page.xml", + '\n', + 2, + id="a-markup-block-before-its-closer", + ), + pytest.param( + "page.vue", + '\n', + 3, + id="a-markup-block-in-a-component", + ), + pytest.param( + "a.js", + '/* was\nconst model = "gpt-4o";\nmore\n*/\n', + 2, + id="a-closer-after-the-cited-line", + ), + pytest.param( + "a.js", + '/* a */\nx\n/* real\nconst model = "gpt-4o";\n*/\n', + 4, + id="a-comment-after-an-earlier-closed-one", + ), + # \n', + 2, + id="a-markup-block-in-dotnet-config", + ), + pytest.param( + "a.csproj", + '\n', + 2, + id="a-markup-block-in-a-project-file", + ), + pytest.param( + "page.php", + '\n', + 2, + id="a-markup-block-in-php", + ), + pytest.param( + "env.xml", + '\n', + 2, + id="a-markup-block-in-markup-named-like-configuration", + ), + ], +) +def test_a_value_inside_a_block_comment_over_lines_is_not_the_model( + root, file, written, line +): + (root / file).write_text(written, encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": file, "line": line}) + + assert problems == [not_named("gpt-4o", line, file)] + + +@pytest.mark.parametrize( + "file, written, line", + [ + pytest.param( + "a.js", + '/*\nold\n*/\nconst model = "gpt-4o";\n', + 4, + id="after-the-closing-line", + ), + pytest.param( + "a.js", + '/*\nold */ const model = "gpt-4o";\n', + 2, + id="after-the-closer-on-its-line", + ), + pytest.param( + "a.js", + '/* old */\nconst model = "gpt-4o";\n', + 2, + id="closed-on-an-earlier-line", + ), + pytest.param( + "a.js", + '/* old */ const model = "gpt-4o";\n', + 1, + id="closed-on-the-same-line", + ), + pytest.param( + "page.html", + '\n\n', + 4, + id="after-a-markup-block", + ), + pytest.param( + "a.js", + 'const open = "/*";\nconst model = "gpt-4o";\n', + 2, + id="an-opener-in-a-string", + ), + pytest.param( + "a.js", + "const open = '/;\n', + 2, + id="a-markup-opener-outside-markup-with-a-closer-later", + ), + pytest.param( + "c.yaml", + "banner: \n", + 2, + id="a-markup-opener-in-configuration-with-a-closer-later", + ), + pytest.param( + ".env", + "BANNER=\n", + 2, + id="a-markup-opener-in-configuration-by-name-with-a-closer-later", + ), + ], +) +def test_a_value_outside_a_block_comment_is_the_model(root, file, written, line): + (root / file).write_text(written, encoding="utf-8") + + problems, contract, _ = checked( + root, {"name": "gpt-4o", "path": file, "line": line} + ) + + assert problems == [] + assert contract.model is not None and contract.model.name == "gpt-4o" + + +@pytest.mark.parametrize( + "written", + [ + pytest.param( + "/*\n" * (MIB // 3 - 20) + 'const model = "gpt-4o";\n', + id="a-file-of-openers", + ), + pytest.param( + "*/ /*\n" * (MIB // 6 - 20) + 'const model = "gpt-4o";\n', + id="a-file-of-closers-and-openers", + ), + pytest.param( + "/* */ " * (MIB // 6 - 20) + '"gpt-4o"\n', id="a-line-of-closed-blocks" + ), + pytest.param( + " ', "gpt-4o", id="a-closed-markup-comment-before"), + pytest.param("llm.py", 'm = str("gpt-4o")', "gpt-4o", id="str-as-a-function"), + pytest.param("GNUmakefile", "MODEL ?= gpt-4o", "gpt-4o", id="gnu-makefile"), + pytest.param("chat.prompt", "model: googleai/gemini-1.5-pro", "googleai/gemini-1.5-pro", id="dotprompt-frontmatter"), + ], +) +def test_more_honest_forms_by_language(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("agent.ts", 'x = 1; /* "gpt-4o" */ y = 2;', "gpt-4o", id="inside-a-closed-block-comment"), + pytest.param("agent.py", "x = 1 # it's \"gpt-4o\"", "gpt-4o", id="python-hash"), + pytest.param("cfg.yaml", 'model: gpt-4o-mini # was "gpt-4o"', "gpt-4o", id="yaml-spaced-hash"), + pytest.param("agent.ts", 'const m = config?.["gpt-4o"];', "gpt-4o", id="optional-chained-lookup"), + pytest.param("agent.py", 'm = env.str("gpt-4o")', "gpt-4o", id="str-as-a-method"), + ], +) +def test_more_comments_and_lookups_by_language(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +@pytest.mark.parametrize("file", ["prompt.j2", "prompt.jinja", "prompt.jinja2", "prompt.hbs", "prompt.mustache"]) +def test_a_prompt_template_holds_no_model(root, file): + (root / file).write_text('You are an assistant.\nYou run on "gpt-4o".\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": file, "line": 2}) + + assert problems == [prose(file)] + + +def test_a_short_quote_with_nothing_held_is_looked_for_whole(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + + result = submit( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="You are pair.", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + ) + + assert text_of(result).startswith(REFUSED + not_the_quote("agent.py 1-4")) + + +def test_a_long_first_word_is_charged_on_every_try(root): + from fi.alk.harness.model_citation import quote_problem + + started = time.perf_counter() + # Without the charge, each of 200,000 tries would compare 200,000 letters. + answer = quote_problem("a" * 200_000 + " b", "a" * 400_000, "agent.py 1-1") + took = time.perf_counter() - started + + assert answer is not None + assert took < 0.25 + + +def test_a_short_unspaced_quote_is_still_searched_with_gaps(tmp_path, root): + # Six characters, each counted as three letters: long enough to be searched with gaps. + (root / "prompt.py").write_text('PROMPT = ("你是订餐" f"助手")\n', encoding="utf-8") + + result = accept_contract( + payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="你是订餐助手", + system_prompt_from=[{"path": "prompt.py", "start_line": 1, "end_line": 1}], + ), + tmp_path / "out", + source_root=root, + ) + + assert not result.get("is_error"), result + + +@pytest.mark.parametrize( + "file, written, line, name", + [ + pytest.param("cfg.yaml", 'note: x """ y\nmodel: gpt-4o\n', 2, "gpt-4o", id="yaml-bare-triple"), + pytest.param("main.go", 'var s = x + """\nvar model = "gpt-4o"\n', 2, "gpt-4o", id="go-bare-triple"), + ], +) +def test_triple_quotes_are_tracked_only_where_they_are_strings(root, file, written, line, name): + (root / file).write_text(written, encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": line}) + + assert problems == [] + + +def test_a_value_before_a_block_closes_is_text(root): + (root / "llm.py").write_text('X = """a\nuse "gpt-4o" here"""\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "llm.py")] + + +def test_lines_without_a_triple_quote_are_not_scanned(root, monkeypatch): + from fi.alk.harness import model_citation + + written = '"""\n' + "x\n" * 499 + '"""\n' + "y\n" * 499 + 'MODEL = "gpt-4o"\n' + (root / "llm.py").write_text(written, encoding="utf-8") + scanned: list[str] = [] + real_scan = model_citation._line_scan + monkeypatch.setattr( + model_citation, "_line_scan", lambda line, *a, **k: scanned.append(line) or real_scan(line, *a, **k) + ) + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 1001}) + + assert problems == [] + # The block's opening and closing lines, and the cited line: nothing between. + assert scanned == ['"""', '"""', 'MODEL = "gpt-4o"'] + + +# A short or list quote is held like any other + + +def test_a_short_quote_is_held_and_its_lines_asked_about(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt="Be warm and brief.", system_prompt_from=imports) + ) + assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 1-4")) + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", "Be warm and brief.", short=True) + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + # A quote this short can't stand for the prompt, and lines that twice didn't hold it would be + # shown as it: no prompt is saved, the lines are on record as unconfirmed, and the reply says + # so. + assert not third.get("is_error"), third + assert text_of(third).endswith( + "\nsystem_prompt_from: the lines named (agent.py 1-4) are recorded as unconfirmed, since " + "they don't hold the quote; no prompt is saved, since the quote is too short to stand " + "for it." + ) + assert saved(destination)["system_prompt_excerpt"] == "" + assert saved(destination)["system_prompt_from"] == [] + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +@pytest.mark.parametrize( + "fields", + [ + pytest.param({"system_prompt_excerpt": _BOOKING_QUOTE.split(". ")}, id="a-list"), + pytest.param({"prompt": _BOOKING_QUOTE.split(". ")}, id="a-list-under-an-alias"), + pytest.param({"system_prompt": _BOOKING_QUOTE}, id="an-alias"), + ], +) +def test_a_quote_in_another_shape_is_held(tmp_path, root, fields): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + first = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports, **fields)) + assert text_of(first).startswith(REFUSED + not_the_quote("agent.py 1-4")) + second = submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)) + + assert text_of(second).startswith(REFUSED + "system_prompt_from: the lines named (agent.py 1-4) don't hold the quote you gave before") + + +def test_a_list_quote_that_passes_replaces_the_held_one(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + prompt = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + paraphrase = "Acme booking agent: helps guests reserve tables and never shares bookings." + + assert submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=paraphrase, system_prompt_from=prompt)).get("is_error") + assert not submit( + payload(hard_constraints=["Be polite."], prompt=_BOOKING_QUOTE.split(". "), system_prompt_from=prompt) + ).get("is_error") + third = submit(payload(hard_constraints=["Be polite."], system_prompt_from=prompt)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_from"] == prompt + assert "unconfirmed_prompt_from" not in saved(destination) + + +def test_a_placeholder_with_the_right_lines_saves_no_prompt(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + prompt = [{"path": "agent.py", "start_line": 6, "end_line": 12}] + again = payload(hard_constraints=["Be polite."], system_prompt_excerpt="see lines", system_prompt_from=prompt) + + assert submit(again).get("is_error") + second = submit(again) + + # Lines that don't hold even a placeholder are unconfirmed, however right they are. + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == "" + assert saved(destination)["system_prompt_from"] == [] + assert saved(destination)["unconfirmed_prompt_from"] == prompt + + +def test_two_calls_in_one_message_are_each_asked(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + server = contract_tools(destination, source_root=root) + tool = next(one for one in server.tools if one.name == "submit_contract") + paraphrase = "Acme booking agent: helps guests reserve tables and never shares bookings." + both = payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=paraphrase, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}], + ) + + async def send_together(): + return await asyncio.gather(tool.handler(dict(both)), tool.handler(dict(both))) + + first, second = asyncio.run(send_together()) + + # Neither had seen a refusal when it was sent, so both are refused. This models a dispatcher + # that runs one message's calls side by side (ADK, the Vertex backend). The Claude backend runs + # them one after another, and there the second sees the first's refusal: a known limit. + assert first.get("is_error") and second.get("is_error") + assert not (destination / "contract.json").exists() + + +def test_a_quote_sent_alone_after_its_lines_were_refused_is_not_asked_for_lines(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + assert submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ).get("is_error") + second = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE)) + + assert not second.get("is_error"), second + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + + +def test_a_new_quote_with_wrong_lines_after_lines_alone_were_refused_is_kept(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + quote = "You are Acme's booking agent. Help guests book tables." + + assert not submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}]) + ).get("is_error") + assert submit(payload(hard_constraints=["Be polite."], system_prompt_from=imports)).get("is_error") + third = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=quote, system_prompt_from=imports)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == quote + assert saved(destination)["unconfirmed_prompt_from"] == imports + + +def test_lines_set_aside_in_a_call_refused_for_something_else_are_said(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + wrong = payload( + hard_constraints=["Be polite."], + system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + assert submit(wrong).get("is_error") + + second = submit({**wrong, "model": {"name": "gpt-4o", "path": "agent.py", "line": 1}}) + + assert "model: the name gpt-4o is not the value written on line 1" in text_of(second) + assert ( + "system_prompt_from: the lines named (agent.py 1-4) are left out and the quote is kept, " + "since they don't hold it" + ) in text_of(second) + + +# Pins for the line scan + + +@pytest.mark.parametrize("quote", ['x = "' + '\\"' * 40_000, "x = `" + "\\`" * 40_000]) +def test_an_unclosed_string_of_escapes_costs_one_pass(root, quote): + (root / "crafted.ts").write_text(quote + "\n", encoding="utf-8") + + _, _, took = checked(root, {"name": "gpt-4o", "path": "crafted.ts", "line": 1}) + + assert took < 0.25 + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("agent.ts", 'const model = "gpt-4o"; /* fast */', "gpt-4o", id="ts-comment-after"), + pytest.param("Agent.java", 'String model = "gpt-4o"; /* the default */', "gpt-4o", id="java-comment-after"), + pytest.param("llm.py", 'MODEL = "gpt-4o"; PROMPT = """', "gpt-4o", id="a-block-opening-after"), + pytest.param("llm.py", ' * 2, model="gpt-4o")', "gpt-4o", id="python-continuation-star"), + pytest.param("run.sh", "echo a#b; MODEL=gpt-4o", "gpt-4o", id="shell-hash-in-a-word"), + pytest.param("run.sh", "ls /*; MODEL=gpt-4o", "gpt-4o", id="shell-slash-star"), + pytest.param("build.groovy", 'a #b; def model = "gpt-4o"', "gpt-4o", id="groovy-hash"), + pytest.param("license.py", 'MODEL = "gpt-4o"', "gpt-4o", id="license-py-is-code"), + pytest.param("readme.py", 'MODEL = "gpt-4o"', "gpt-4o", id="readme-py-is-code"), + pytest.param("llm.py", "x = rb'Agent #1'; MODEL = \"gpt-4o\"", "gpt-4o", id="two-letter-prefix"), + pytest.param("llm.py", "x = 'a\\'b # c'; MODEL = \"gpt-4o\"", "gpt-4o", id="escaped-apostrophe"), + pytest.param("agent.ts", 'u = `${b}//v1`; m = "gpt-4o"', "gpt-4o", id="backtick-string"), + ], +) +def test_more_honest_lines_after_round_3(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "suffix", + [".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", + ".java", ".go", ".rs", ".swift", ".kt", ".kts", ".cs", ".scala", ".dart", ".m", ".mm", + ".groovy", ".gradle"], +) +def test_hash_is_no_comment_in_these_languages(root, suffix): + (root / f"agent{suffix}").write_text('\na = b #c; m = "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": f"agent{suffix}", "line": 2}) + + assert problems == [] + + +def test_a_tab_before_hash_starts_a_yaml_comment(root): + (root / "cfg.yaml").write_text('\nnote: a\t# "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "cfg.yaml", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "cfg.yaml")] + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("llm.py", 'n = a // 2; model = "gpt-4o"', "gpt-4o", id="python-floor-division-spaced"), + pytest.param("init.lua", 'local n = #t; model = "gpt-4o"', "gpt-4o", id="lua-length"), + pytest.param("llm.py", "x = t'Agent #{n}'; MODEL = \"gpt-4o\"", "gpt-4o", id="python-t-string"), + pytest.param("config.yaml.j2", "model: gpt-4o", "gpt-4o", id="a-config-template"), + ], +) +def test_more_honest_lines_after_round_3_probes(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("env.py", "model = gpt-4o", "gpt-4o", id="env-py-is-code"), + pytest.param("dockerfile_gen.py", "MODEL = gpt-4o", "gpt-4o", id="dockerfile-py-is-code"), + pytest.param("agent.js", 'call()// "gpt-4o"', "gpt-4o", id="js-comment-after-a-call"), + pytest.param("agent.js", 'x = 1// "gpt-4o"', "gpt-4o", id="js-comment-after-a-number"), + pytest.param("agent.ts", 'x = a[1]// "gpt-4o"', "gpt-4o", id="js-comment-after-a-subscript"), + ], +) +def test_more_wrong_values_after_round_3_probes(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +# Pins for rules the plants found unpinned + + +@pytest.mark.parametrize( + "written", + [ + pytest.param('curl http://x/v1; MODEL="gpt-4o"', id="a-url"), + pytest.param('cp data/* out; MODEL="gpt-4o"', id="a-glob"), + pytest.param('n=${#MODELS}; MODEL="gpt-4o"', id="a-length"), + pytest.param('echo $#; MODEL="gpt-4o"', id="a-count"), + ], +) +def test_a_script_without_a_suffix_keeps_its_model(root, written): + (root / "start").write_text(f"#!/bin/sh\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": "gpt-4o", "path": "start", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == "gpt-4o" + + +@pytest.mark.parametrize("prefix", ["rt", "tr", "RT"]) +def test_a_raw_template_string_is_a_string(root, prefix): + (root / "llm.py").write_text(f"\nx = {prefix}'Agent #{{n}}'; MODEL = \"gpt-4o\"\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [] + + +def test_a_triple_quote_after_floor_division_opens_a_block(root): + (root / "llm.py").write_text( + 'half = n // 2; DOC = """\nmodel = "gpt-4o"\n"""\n', encoding="utf-8" + ) + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "llm.py", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "llm.py")] + + +def test_a_quote_alone_after_lines_alone_were_refused_is_not_asked_for_lines(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + + assert not submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}]) + ).get("is_error") + assert submit( + payload(hard_constraints=["Be polite."], + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ).get("is_error") + third = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + + +# Lines numbered as the reading numbers them + + +@pytest.mark.parametrize( + "written, prompt_line, model_line", + [ + pytest.param( + b'# head\nA = 1\x0cB = 2\rC = 3\nNOTE = "a\xe2\x80\xa8b"\n' + b'PROMPT = "You are Acme\'s booking agent."\nMODEL = "gpt-4o"\n', + 4, 5, id="form-feed-lone-cr-and-u2028", + ), + pytest.param( + b'# head\r\nPROMPT = "You are Acme\'s booking agent."\r\nMODEL = "gpt-4o"\r\n', + 2, 3, id="crlf", + ), + pytest.param( + b'# head\nPROMPT = "You are Acme\'s booking agent."\nMODEL = "gpt-4o"', + 2, 3, id="no-newline-at-the-end", + ), + ], +) +def test_lines_are_numbered_at_each_newline(tmp_path, root, written, prompt_line, model_line): + (root / "agent.py").write_bytes(written) + destination = tmp_path / "out" + + result = accept_contract( + payload( + system_prompt_from=[{"path": "agent.py", "start_line": prompt_line, "end_line": prompt_line}], + model={"name": "gpt-4o", "path": "agent.py", "line": model_line}, + ), + destination, + source_root=root, + ) + + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_excerpt"] == 'PROMPT = "You are Acme\'s booking agent."' + assert saved(destination)["model"]["line"] == model_line + + +def test_a_file_ending_in_a_newline_has_no_line_after_it(root): + (root / "agent.py").write_text('\nMODEL = "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "agent.py", "line": 3}) + + assert problems == ["model: line 3 is outside agent.py, which has 2 lines"] + + +@pytest.mark.parametrize( + "letter, escaped", + [ + pytest.param("\U00020000", "\\ud840\\udc00", id="cjk-extension-b"), + pytest.param("\U0001d400", "\\ud835\\udc00", id="math-bold"), + ], +) +def test_a_json_escaped_letter_past_ffff_holds_its_quote(letter, escaped): + from fi.alk.harness.model_citation import quote_problem + + quote = f"Greet each guest as {letter} and seat them at once." + written = '{"prompt": "' + quote.replace(letter, escaped) + '"}' + + assert quote_problem(quote, written, "prompt.json 1-1") is None + + +def test_an_escape_past_the_last_character_is_kept_as_written(): + from fi.alk.harness.model_citation import quote_problem + + quote = "Order a table \\U00110000 for two guests tonight." + + assert quote_problem(quote, 'PROMPT = "' + quote + '"', "prompt.py 1-1") is None + + +@pytest.mark.parametrize( + "quote, says", + [ + # 6 letters is the floor of a quote that says something, looked for whole. + pytest.param("Be kind", True, id="six-letters"), + pytest.param("Be wry", False, id="five-letters"), + ], +) +def test_six_letters_say_something(quote, says): + from fi.alk.harness.model_citation import quote_problem + + assert (quote_problem(quote, 'x = "nothing here"', "a.py 1-1") is not None) == says + + +def test_sixteen_letters_are_searched_with_gaps(): + from fi.alk.harness.model_citation import quote_pieces, quote_problem + + # 16 letters: words in order, with code between them. 15 would be looked for whole. + quote = "Be warm and kind now" + written = 'P = "Be warm " + tone + " and kind now"' + + assert quote_pieces(quote) + assert quote_problem(quote, written, "a.py 1-1") is None + assert not quote_pieces("Be warm and kind no") + + +def test_a_pipe_is_logged_as_not_a_regular_file(tmp_path, caplog): + from fi.alk.harness.model_citation import read_source + + if not hasattr(os, "mkfifo"): + pytest.skip("no named pipes here") + pipe = tmp_path / "pipe" + os.mkfifo(pipe) + answer: list[bytes | None] = [] + with caplog.at_level("INFO", logger="fi.alk.harness.model_citation"): + reading = threading.Thread(target=lambda: answer.append(read_source(pipe, 10)), daemon=True) + reading.start() + reading.join(timeout=2) + + assert answer == [None] + assert [record.getMessage() for record in caplog.records] == [ + "source file not read (not a regular file)" + ] + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("init.lua", 'local n = a // 2; model = "gpt-4o"', "gpt-4o", id="lua-floor-division"), + pytest.param("Makefile", "\techo a#b; MODEL=gpt-4o", "gpt-4o", id="makefile-hash-in-a-word"), + pytest.param(".flaskenv", "URL=http://x/#a MODEL=gpt-4o", "gpt-4o", id="flaskenv-hash-in-a-word"), + pytest.param(".env.local", "URL=http://x/#a MODEL=gpt-4o", "gpt-4o", id="env-name-hash-in-a-word"), + pytest.param("Dockerfile.dev", "ENV A=a#b MODEL=gpt-4o", "gpt-4o", id="dockerfile-name-hash-in-a-word"), + pytest.param("run.bash", "echo a#b; MODEL=gpt-4o", "gpt-4o", id="bash-hash-in-a-word"), + pytest.param("run.zsh", "echo a#b; MODEL=gpt-4o", "gpt-4o", id="zsh-hash-in-a-word"), + pytest.param("rules.mk", "\techo a#b; MODEL=gpt-4o", "gpt-4o", id="mk-hash-in-a-word"), + pytest.param("app.dockerfile", "ENV A=a#b MODEL=gpt-4o", "gpt-4o", id="dockerfile-suffix-hash-in-a-word"), + pytest.param("prod.env", "URL=http://x/#a MODEL=gpt-4o", "gpt-4o", id="env-suffix-hash-in-a-word"), + pytest.param("agent.mts", 'class A { #model = "gpt-4o"; }', "gpt-4o", id="mts-private-field"), + pytest.param("agent.cts", 'this.#model = "gpt-4o";', "gpt-4o", id="cts-private-field"), + pytest.param("agent.sc", 'val t: A#B = x; val model = "gpt-4o"', "gpt-4o", id="scala-script-type-projection"), + pytest.param(".env.j2", "MODEL=gpt-4o", "gpt-4o", id="env-template"), + pytest.param("Dockerfile.j2", "ENV MODEL=gpt-4o", "gpt-4o", id="dockerfile-template"), + pytest.param(".env.local.j2", "MODEL=gpt-4o", "gpt-4o", id="env-local-template"), + pytest.param("Dockerfile.dev.j2", "ENV MODEL=gpt-4o", "gpt-4o", id="dockerfile-dev-template"), + pytest.param("config.yaml.j2", " - gpt-4o", "gpt-4o", id="yaml-template-list-item"), + pytest.param("appsettings.json", '"ModelId": "gpt-4o", // the default', "gpt-4o", id="json-comment-after"), + pytest.param("agent.json", '"url": "http://x//y", "model": "gpt-4o"', "gpt-4o", id="json-slashes-in-a-string"), + ], +) +def test_more_honest_lines_after_the_settled_round(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, contract, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == name + + +@pytest.mark.parametrize( + "file, written, name", + [ + pytest.param("appsettings.json", '// "ModelId": "gpt-35-turbo",', "gpt-35-turbo", id="json-line-comment"), + pytest.param("appsettings.json", '"ModelId": "gpt-4o" /* was "gpt-35-turbo" */', "gpt-35-turbo", id="json-block-comment"), + pytest.param("agent.json", '"model": "gpt-4o", // "gpt-4-turbo" is slower', "gpt-4-turbo", id="json-comment-after-a-value"), + pytest.param("agent.json", '* "model": "gpt-4",', "gpt-4", id="json-block-comment-middle-line"), + pytest.param("dockerfile_gen.py", 'MODEL = DEFAULT#"gpt-4"', "gpt-4", id="python-named-like-a-dockerfile"), + pytest.param(".env_loader.py", 'MODEL = DEFAULT#"gpt-4"', "gpt-4", id="python-named-like-an-env-file"), + pytest.param("cfg.yaml", "note: the cart's total # model: 'gpt-4'", "gpt-4", id="a-word-ending-like-a-prefix"), + pytest.param("cfg.yaml", "note: it_t's ok # model: 'gpt-4'", "gpt-4", id="an-underscore-before-a-prefix"), + pytest.param("env.rb", "MODEL = gpt-4o", "gpt-4o", id="env-rb-is-code"), + ], +) +def test_more_wrong_values_after_the_settled_round(root, file, written, name): + (root / file).write_text(f"\n{written}\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": name, "path": file, "line": 2}) + + assert problems == [not_named(name, 2, file)] + + +def test_a_python_template_is_text(root): + (root / "agent.py.j2").write_text('MODEL = "gpt-4o"\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "agent.py.j2", "line": 1}) + + assert problems == [prose("agent.py.j2")] + + +def test_a_type_projection_does_not_hide_a_triple_quote(root): + (root / "agent.scala").write_text( + 'val t: A#B = x; val p = """\nmodel = "gpt-4o"\n"""\n', encoding="utf-8" + ) + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": "agent.scala", "line": 2}) + + assert problems == [not_named("gpt-4o", 2, "agent.scala")] + + +# The flows + + +_SHORT_PROMPT = 'import os\n\nINSTRUCTIONS = "Be warm and brief."\n' +_PARAPHRASE = "The agent books tables for Acme guests, keeps other bookings private, and confirms first." + + +@pytest.mark.parametrize( + "source, lines, quote, paraphrase", + [ + pytest.param( + _SHORT_PROMPT, (3, 3), "Be warm and brief.", + "The agent is told to be warm and to keep its answers brief.", id="a-whole-short-prompt", + ), + pytest.param(_BOOKING, (6, 12), "Be warm and brief.", _PARAPHRASE, id="one-sentence-of-a-long-prompt"), + pytest.param(_BOOKING, (6, 12), "be warm and brief", _PARAPHRASE, id="another-case-no-full-stop"), + pytest.param(_BOOKING, (6, 12), "Be warm ... brief.", _PARAPHRASE, id="a-short-elided-quote"), + ], +) +@pytest.mark.parametrize("then", ["the-same-again", "the-lines-alone"]) +def test_a_short_verbatim_quote_after_a_refused_paraphrase_keeps_the_lines( + tmp_path, root, source, lines, quote, paraphrase, then +): + (root / "agent.py").write_text(source, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + named = [{"path": "agent.py", "start_line": lines[0], "end_line": lines[1]}] + + assert submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=paraphrase, system_prompt_from=named) + ).get("is_error") + second = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=quote, system_prompt_from=named) + ) + assert not second.get("is_error"), second + again = ( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=quote, system_prompt_from=named) + if then == "the-same-again" + else payload(hard_constraints=["Be polite."], system_prompt_from=named) + ) + third = submit(again) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_from"] == named + assert "unconfirmed_prompt_from" not in saved(destination) + + +def test_a_short_quote_not_in_its_lines_is_still_checked_against_the_held_one(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + + assert submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=_PARAPHRASE, + system_prompt_from=[{"path": "agent.py", "start_line": 6, "end_line": 12}]) + ).get("is_error") + second = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt="Be warm and brief.", system_prompt_from=imports) + ) + + assert text_of(second) == REFUSED + lines_alone("agent.py 1-4", _PARAPHRASE) + + +@pytest.mark.parametrize("lines", [(6, 12), (1, 4)], ids=["the-right-lines", "the-wrong-lines"]) +def test_a_null_in_a_list_quote_is_dropped_before_it_is_held(tmp_path, root, lines): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + named = [{"path": "agent.py", "start_line": lines[0], "end_line": lines[1]}] + + result = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=["Be warm", None, "and brief."], + system_prompt_from=named) + ) + + if lines == (6, 12): + assert not result.get("is_error"), result + assert saved(destination)["system_prompt_from"] == named + else: + # Refused as this call's own quote, not as one given before with "None" in it. + assert text_of(result).startswith(REFUSED + not_the_quote("agent.py 1-4")) + + +def test_a_blank_quote_is_no_quote(tmp_path, root): + submit = submitter(tmp_path / "out", root) + + result = submit(payload(system_prompt_excerpt=" ")) + + assert result.get("is_error") + assert "no hard_constraints and no system_prompt_excerpt" in text_of(result) + assert PROMPT_LINES not in text_of(result) + + +def test_two_quotes_alone_in_one_message_are_each_asked_for_lines(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + server = contract_tools(destination, source_root=root) + tool = next(one for one in server.tools if one.name == "submit_contract") + both = payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE) + + async def send_together(): + return await asyncio.gather(tool.handler(dict(both)), tool.handler(dict(both))) + + first, second = asyncio.run(send_together()) + + assert PROMPT_LINES in text_of(first) and PROMPT_LINES in text_of(second) + assert not (destination / "contract.json").exists() + + +def test_short_lines_set_aside_in_a_call_refused_for_something_else_are_said(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + wrong = payload( + hard_constraints=["Be polite."], + system_prompt_excerpt="Be warm and brief.", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}], + ) + assert submit(wrong).get("is_error") + + second = submit({**wrong, "model": {"name": "gpt-4o", "path": "agent.py", "line": 1}}) + + assert "model: the name gpt-4o is not the value written on line 1" in text_of(second) + assert ( + "system_prompt_from: the lines named (agent.py 1-4) are recorded as unconfirmed and no " + "prompt is saved, since they don't hold the quote and it is too short to stand for the " + "prompt" + ) in text_of(second) + + +@pytest.mark.parametrize( + "fields", + [ + pytest.param({"system_prompt_excerpt": ["Be warm", "and brief."]}, id="a-list"), + pytest.param({"prompt": ["Be warm", None, "and brief."]}, id="a-list-with-a-null-under-an-alias"), + pytest.param({"system_prompt_excerpt": 42}, id="a-number"), + ], +) +def test_the_held_quote_is_the_quote_the_contract_reads(fields): + from fi.alk.harness.contract import prompt_quote + from fi.alk.harness.tools import _without_nulls + + contract = AgentContract.model_validate(_without_nulls(payload(**fields))) + + assert prompt_quote(_without_nulls(payload(**fields))) == contract.system_prompt_excerpt + + +def test_a_saved_call_that_kept_the_quote_says_so(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + imports = [{"path": "agent.py", "start_line": 1, "end_line": 4}] + wrong = payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE, system_prompt_from=imports) + + assert submit(wrong).get("is_error") + second = submit(wrong) + + assert not second.get("is_error"), second + assert text_of(second).endswith( + "\nsystem_prompt_from: the lines named (agent.py 1-4) are recorded as unconfirmed, since " + "they don't hold the quote; the quote is the prompt." + ) + + +# A short quote on its own lines, and its limits + + +@pytest.mark.parametrize( + "first_lines, short, line", + [ + # The quote held from a call with no lines; then a name on the prompt's first line. + pytest.param(None, "PROMPT", 6, id="a-name-after-a-quote-alone"), + pytest.param(None, "SYSTEM_PROMPT", 6, id="a-setting-name-after-a-quote-alone"), + # The quote held from lines that held it; then a sentence repeated in a comment. + pytest.param((6, 12), "Be warm and brief.", 14, id="a-comment-after-a-confirmed-quote"), + ], +) +def test_a_short_quote_on_its_own_lines_does_not_replace_a_stronger_one( + tmp_path, root, first_lines, short, line +): + (root / "agent.py").write_text(_BOOKING + "\n# Style: Be warm and brief.\n", encoding="utf-8") + destination = tmp_path / "out" + submit = submitter(destination, root) + named = [{"path": "agent.py", "start_line": line, "end_line": line}] + first = payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE) + if first_lines: + first["system_prompt_from"] = [{"path": "agent.py", "start_line": first_lines[0], "end_line": first_lines[1]}] + submit(first) + + second = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=short, system_prompt_from=named)) + assert text_of(second).startswith(REFUSED + lines_alone(f"agent.py {line}-{line}", _BOOKING_QUOTE)) + third = submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=short, system_prompt_from=named)) + + assert not third.get("is_error"), third + assert saved(destination)["system_prompt_excerpt"] == _BOOKING_QUOTE + assert saved(destination)["unconfirmed_prompt_from"] == named + + +def test_a_padded_short_quote_is_this_calls_own(tmp_path, root): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + + result = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=" Be warm and brief. ", + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ) + + assert text_of(result).startswith(REFUSED + not_the_quote("agent.py 1-4")) + + +@pytest.mark.parametrize( + "quote, held", + [ + pytest.param("Be warm ... brief.", True, id="pieces-in-order"), + pytest.param("brief ... Be warm", False, id="pieces-out-of-order"), + ], +) +def test_a_short_elided_quote_is_looked_for_piece_by_piece(quote, held): + from fi.alk.harness.model_citation import quote_problem + + assert (quote_problem(quote, 'P = "Be warm and brief."', "a.py 1-1") is None) == held + + +def test_the_quote_under_its_own_name_comes_before_an_alias(): + from fi.alk.harness.contract import prompt_quote + + both = payload(system_prompt_excerpt="You are Acme's booking agent.", prompt="Something else entirely.") + + assert prompt_quote(both) == "You are Acme's booking agent." + assert AgentContract.model_validate(both).system_prompt_excerpt == "You are Acme's booking agent." + + +@pytest.mark.parametrize( + "file", + ["prompts/dockerfile_writer.j2", "prompts/env.system.j2", "prompts/system.prompt.j2", "dockerfile.md"], +) +def test_a_prompt_named_like_configuration_is_text(root, file): + (root / file).parent.mkdir(parents=True, exist_ok=True) + (root / file).write_text("You write the deployment.\nmodel: gpt-4o\n", encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4o", "path": file, "line": 2}) + + assert problems == [prose(file)] + + +def test_a_conf_file_has_no_block_comments(root): + (root / "site.conf").write_text('\nrewrite ^/(.*)//$ /$1 permanent; set $m "gpt-4o";\n', encoding="utf-8") + + problems, contract, _ = checked(root, {"name": "gpt-4o", "path": "site.conf", "line": 2}) + + assert problems == [] + assert contract.model is not None and contract.model.name == "gpt-4o" + + +def test_a_json_template_keeps_its_comments(root): + (root / "config.json.j2").write_text('\n// "model": "gpt-4-turbo",\n', encoding="utf-8") + + problems, _, _ = checked(root, {"name": "gpt-4-turbo", "path": "config.json.j2", "line": 2}) + + assert problems == [not_named("gpt-4-turbo", 2, "config.json.j2")] + + +@pytest.mark.parametrize("nothing", ["n / a", "- -", "a b"]) +def test_words_that_say_nothing_do_not_replace_the_held_quote(tmp_path, root, nothing): + (root / "agent.py").write_text(_BOOKING, encoding="utf-8") + submit = submitter(tmp_path / "out", root) + submit(payload(hard_constraints=["Be polite."], system_prompt_excerpt=_BOOKING_QUOTE)) + + result = submit( + payload(hard_constraints=["Be polite."], system_prompt_excerpt=nothing, + system_prompt_from=[{"path": "agent.py", "start_line": 1, "end_line": 4}]) + ) + + assert text_of(result) == REFUSED + lines_alone("agent.py 1-4", _BOOKING_QUOTE) diff --git a/tests/harness/test_contract_tools_subfolder.py b/tests/harness/test_contract_tools_subfolder.py new file mode 100644 index 000000000..60e1df84a --- /dev/null +++ b/tests/harness/test_contract_tools_subfolder.py @@ -0,0 +1,2135 @@ +from __future__ import annotations + +import asyncio +import gc +import json +import sys +from pathlib import Path + +import pytest + +from fi.alk.harness.amend import add_rule +from fi.alk.harness.contract import AgentContract, Runtime +from fi.alk.harness.tools import accept_contract, contract_tools +from fi.alk.harness.understand import load, open_stage +from fi.alk.harness.sources import resolve + + +def payload(**fields): + return { + "agent": "pair", + "modality": "chat", + "real_use_cases": ["order"], + "tools": [{"name": "add", "args": ["item_id"]}], + **fields, + } + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + root = tmp_path / "repo" + for name in ("a", "b"): + (root / "agents" / name).mkdir(parents=True) + (root / "agents" / name / "agent.py").write_text("x = 1\n") + (root / "agents" / "b" / "traced.py").write_text("import traceai_openai\n") + return root + + +def _saved(destination: Path) -> dict: + return json.loads((destination / "contract.json").read_text()) + + +def _submit(server, arguments: dict, destination: Path) -> dict: + """What the model's call does; a first thin submission is sent back once, then accepted.""" + result = {} + for _ in range(2): + result = asyncio.run(server.tools[0].handler(arguments)) + if (destination / "contract.json").exists(): + break + return result + + +def test_save_defaults_workdir_and_records_folder(tmp_path, repo): + result = accept_contract( + payload(), tmp_path / "out", source_root=repo, subfolder="agents/a" + ) + + assert not result.get("is_error") + runtime = _saved(tmp_path / "out")["runtime"] + assert runtime["source_subfolder"] == "agents/a" + assert runtime["workdir"] == "agents/a" + + +def test_save_refuses_workdir_outside(tmp_path, repo): + result = accept_contract( + payload(runtime={"language": "python", "workdir": "agents/b"}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert result.get("is_error") + assert "runtime_workdir_outside_subfolder" in result["content"][0]["text"] + assert not (tmp_path / "out" / "contract.json").exists() + + +def test_no_subfolder_contract_dump_unchanged(tmp_path, repo): + assert "source_subfolder" not in Runtime().model_dump() + assert "source_subfolder" not in AgentContract(agent="x", runtime=Runtime()).model_dump()["runtime"] + assert "source_subfolder" not in json.dumps(Runtime.model_json_schema()) + + for subfolder in (None, ""): + destination = tmp_path / f"out-{subfolder!r}" + accept_contract( + payload(runtime={"language": "python", "source_subfolder": "agents/z"}), + destination, + source_root=repo, + subfolder=subfolder, + ) + runtime = _saved(destination)["runtime"] + assert "source_subfolder" not in runtime + assert runtime["workdir"] == "" + + +def test_amend_keeps_recorded_folder(tmp_path, repo): + accept_contract(payload(), tmp_path, source_root=repo, subfolder="agents/a") + contract = load(tmp_path) + + added, _ = add_rule(contract, tmp_path, rule="Never share secrets", why="policy") + + assert added + assert _saved(tmp_path)["runtime"]["source_subfolder"] == "agents/a" + + +def test_instrumentation_reads_agent_folder(tmp_path, repo): + own = tmp_path / "own" + accept_contract(payload(), own, source_root=repo, subfolder="agents/a") + assert _saved(own)["instrumentation"] == "none" + + whole = tmp_path / "whole" + accept_contract(payload(), whole, source_root=repo) + assert _saved(whole)["instrumentation"] == "traceai" + + +def test_save_on_missing_folder_saves_nothing(tmp_path, repo): + destination = tmp_path / "out" + server = contract_tools(destination, source_root=repo, subfolder="agents/zzz") + + result = _submit(server, payload(), destination) + + assert result.get("is_error") + assert "subfolder_not_found" in result["content"][0]["text"] + assert not (destination / "contract.json").exists() + + +def test_save_with_no_source_root_works(tmp_path): + result = accept_contract(payload(), tmp_path, subfolder="agents/a") + + assert not result.get("is_error") + assert "source_subfolder" not in (_saved(tmp_path).get("runtime") or {}) + + +def test_no_subfolder_null_runtime_stays_null(tmp_path, repo): + source = resolve("repo", name="demo", root=str(repo), subfolder="") + stage, destination = open_stage(source, out=tmp_path / "out") + + _submit(stage.spec.servers["contract"], payload(), destination) + + assert _saved(destination).get("runtime") is None + + +@pytest.mark.parametrize("subfolder", ["agents/a", ""]) +def test_open_stage_saves_the_folder(tmp_path, repo, subfolder): + source = resolve("repo", name="demo", root=str(repo), subfolder=subfolder) + stage, destination = open_stage(source, out=tmp_path / "out") + + _submit( + stage.spec.servers["contract"], + payload(runtime={"language": "python"}), + destination, + ) + + runtime = _saved(destination)["runtime"] + if subfolder: + assert runtime["source_subfolder"] == "agents/a" + assert runtime["workdir"] == "agents/a" + else: + assert "source_subfolder" not in runtime + assert runtime["workdir"] == "" + + +def _compose_stack(root: Path, location: str, builds: str) -> dict: + (root / location).parent.mkdir(parents=True, exist_ok=True) + (root / location).write_text( + f"services:\n agent:\n build: {builds}\n command: [python, agent.py]\n" + ) + return { + "language": "python", + "execution": "compose", + "compose_file": location, + "service": "agent", + } + + +@pytest.mark.parametrize( + ("location", "builds"), + [("compose.yaml", "agents/b"), ("agents/a/compose.yaml", "../b")], +) +def test_save_refuses_a_control_service_building_a_sibling(tmp_path, repo, location, builds): + runtime = _compose_stack(repo, location, builds) + + result = accept_contract( + payload(runtime=runtime), tmp_path / "out", source_root=repo, subfolder="agents/a" + ) + + assert result.get("is_error") + assert "runtime_compose_service_outside_subfolder" in result["content"][0]["text"] + assert not (tmp_path / "out" / "contract.json").exists() + + +def test_save_keeps_a_control_service_building_the_folder(tmp_path, repo): + runtime = _compose_stack(repo, "compose.yaml", "agents/a") + + result = accept_contract( + payload(runtime=runtime), tmp_path / "out", source_root=repo, subfolder="agents/a" + ) + + assert not result.get("is_error") + assert _saved(tmp_path / "out")["runtime"]["source_subfolder"] == "agents/a" + + +def _compose_with_declared_port(root: Path, location: str, builds: str) -> dict: + (root / location).parent.mkdir(parents=True, exist_ok=True) + (root / location).write_text( + f"services:\n agent:\n build: {builds}\n command: [python, agent.py]\n" + ' ports: ["${AGENT_PORT}:8080"]\n' + ) + (root / "agents" / "a" / "alk.yaml").write_text( + 'runtime:\n environment:\n AGENT_PORT: "9000"\n' + ) + return { + "language": "python", + "execution": "compose", + "compose_file": location, + "service": "agent", + } + + +def test_save_reads_a_siblings_build_through_a_declared_value(tmp_path, repo): + (repo / "compose.yaml").write_text( + "services:\n agent:\n build: ${AGENT_DIR}\n command: [python, agent.py]\n" + ) + (repo / "agents" / "a" / "alk.yaml").write_text( + 'runtime:\n environment:\n AGENT_DIR: "agents/b"\n' + ) + runtime = { + "language": "python", + "execution": "compose", + "compose_file": "compose.yaml", + "service": "agent", + } + + result = accept_contract( + payload(runtime=runtime), tmp_path / "out", source_root=repo, subfolder="agents/a" + ) + + assert "runtime_compose_service_outside_subfolder" in result["content"][0]["text"] + + +def test_save_still_refuses_a_sibling_with_declared_values(tmp_path, repo): + runtime = _compose_with_declared_port(repo, "compose.yaml", "agents/b") + + result = accept_contract( + payload(runtime=runtime), tmp_path / "out", source_root=repo, subfolder="agents/a" + ) + + assert "runtime_compose_service_outside_subfolder" in result["content"][0]["text"] + + +def test_save_leaves_a_bad_alk_yaml_to_bundling(tmp_path, repo): + (repo / "agents" / "a" / "alk.yaml").write_text("- not an object\n") + + result = accept_contract( + payload(runtime={"language": "python"}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert not result.get("is_error"), result + assert (tmp_path / "out" / "contract.json").exists() + + +_GOOD_SERVICE = "services:\n agent:\n build: .\n" + + +@pytest.mark.parametrize( + ("files", "links", "runtime"), + [ + ({"docker-compose.yml": "x: " + "[" * 1000 + "]" * 1000 + "\n"}, [], {}), + ({"docker-compose.yml": "x: [\n"}, [], {}), + ( + {"docker-compose.yml": "include:\n - loop.yml\n" + _GOOD_SERVICE}, + [("loop.yml", "loop.yml")], + {}, + ), + ({"docker-compose.yml": "include:\n - " + "a" * 5000 + ".yml\n" + _GOOD_SERVICE}, [], {}), + ({}, [], {"compose_file": "agents/a/" + "a" * 5000 + ".yml"}), + ({"docker-compose.yml": 'include:\n - "a\\0b.yml"\n' + _GOOD_SERVICE}, [], {}), + ({}, [], {"compose_file": "agents/a/missing.yml"}), + ], + ids=["deep", "invalid", "include-loop", "include-long", "named-long", "include-nul", "named-missing"], +) +def test_a_compose_file_nobody_can_read_does_not_stop_the_save( + tmp_path, repo, files, links, runtime +): + folder = repo / "agents" / "a" + for name, body in files.items(): + (folder / name).write_text(body) + for name, target in links: + (folder / name).symlink_to(target) + + result = accept_contract( + payload(runtime={"language": "python", **runtime}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert not result.get("is_error"), result + assert (tmp_path / "out" / "contract.json").exists() + + +def test_save_checks_a_tool_in_the_folder_the_run_imports_from(tmp_path, repo): + (repo / "agent.py").write_text("def lookup():\n return 1\n") + (repo / "agents" / "a" / "agent.py").write_text("# def lookup():\n# return 1\n") + entry = {"tool": "lookup", "mode": "import", "module": "agent", "callable": "lookup"} + + result = accept_contract( + payload(tools=[{"name": "lookup"}], tool_entrypoints=[entry]), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert result.get("is_error"), result + assert "lookup" in result["content"][0]["text"] + assert not (tmp_path / "out" / "contract.json").exists() + + +def _alias_bomb(levels: int) -> str: + compose = "x-0: &a0 [v,v,v,v,v,v,v,v,v]\n" + "".join( + f"x-{level}: &a{level} [{','.join([f'*a{level - 1}'] * 9)}]\n" for level in range(1, levels) + ) + return compose + f"services:\n agent:\n build: {{context: ., args: *a{levels - 1}}}\n" + + +def _anchors(levels: int, leaf: str = "v") -> str: + return f"x-0: &a0 [{','.join([leaf] * 9)}]\n" + "".join( + f"x-{level}: &a{level} [{','.join([f'*a{level - 1}'] * 9)}]\n" for level in range(1, levels) + ) + + +def _many_services(services: int, levels: int) -> str: + top = f"*a{levels - 1}" + return _anchors(levels) + "services:\n" + "".join( + f" s{index}: {{build: {{context: ., args: {top}}}, image: {top}, ports: {top}, " + f"deploy: {top}, volumes: {top}}}\n" + for index in range(services) + ) + + +def _long_string(length: int, levels: int) -> str: + return ( + f'x-s: &s "{"v" * length}"\n' + + _anchors(levels, leaf="*s") + + f"services:\n agent:\n build: {{context: ., args: *a{levels - 1}}}\n" + ) + + +def _dict_bomb(levels: int) -> str: + def keys(ref): + return "{" + ", ".join(f"k{index}: {ref}" for index in range(9)) + "}" + + return ( + f"x-0: &a0 {keys('v')}\n" + + "".join(f"x-{level}: &a{level} {keys(f'*a{level - 1}')}\n" for level in range(1, levels)) + + f"services:\n agent:\n build: {{context: ., args: *a{levels - 1}}}\n" + ) + + +def _deep_groups(depth: int, size: int) -> str: + group = "[" * depth + "]" * depth + return "x-a: [" + ",".join([group] * (size // (len(group) + 1))) + "]\n" + + +def _merge_chain(keys: int, levels: int) -> str: + # Each level is a new mapping holding every key above it. + return ( + "x-0: &m0 {" + ", ".join(f"k{index}: 1" for index in range(keys)) + "}\n" + + "".join(f"x-{level}: &m{level} {{<<: *m{level - 1}, l{level}: 1}}\n" for level in range(1, levels)) + ) + + +@pytest.mark.parametrize( + "name, text", + [ + pytest.param("docker-compose.yml", _alias_bomb(9), id="nested-aliases"), + pytest.param("docker-compose.yml", _many_services(200, 5), id="many-services"), + pytest.param("docker-compose.yml", _long_string(60_000, 4), id="long-string"), + pytest.param("docker-compose.yml", _dict_bomb(8), id="dict-aliases"), + pytest.param( + "docker-compose.yml", + _merge_chain(20_000, 2_000) + "services:\n agent:\n build: .\n", + id="compose-merge-keys", + ), + pytest.param("alk.yaml", _merge_chain(20_000, 2_000), id="alk-merge-keys"), + ], +) +def test_a_crafted_yaml_file_does_not_hold_up_the_save(tmp_path, repo, name, text): + import time + + (repo / "agents" / "a" / name).write_text(text) + started = time.monotonic() + + result = accept_contract( + payload(runtime={"language": "python"}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert time.monotonic() - started < 5 + assert not result.get("is_error"), result + + +def test_selection_refuses_a_compose_file_of_nested_aliases(repo): + from fi.alk.harness.runtime_selection import RuntimeSelectionError, select_runtime + + (repo / "agents" / "a" / "docker-compose.yml").write_text(_alias_bomb(9)) + + with pytest.raises(RuntimeSelectionError, match="compose_structural_interpolation_limit"): + select_runtime( + repo, + {"language": "python", "source_subfolder": "agents/a", "workdir": "agents/a"}, + validate_environment_dependencies=False, + ) + + +def _select_in_folder(repo): + from fi.alk.harness.runtime_selection import select_runtime + + return select_runtime( + repo, + {"language": "python", "source_subfolder": "agents/a", "workdir": "agents/a"}, + validate_environment_dependencies=False, + ) + + +@pytest.mark.parametrize( + "text", + [ + pytest.param(_alias_bomb(4), id="nested-aliases"), + # Each value is under the budget alone; together they're over it. + pytest.param(_many_services(20, 2), id="many-services"), + # Few nodes, many characters. + pytest.param(_long_string(5_000, 2), id="long-string"), + pytest.param(_dict_bomb(3), id="dict-aliases"), + ], +) +def test_one_budget_covers_the_whole_selection(repo, monkeypatch, text): + from fi.alk.harness import runtime_selection + + monkeypatch.setattr(runtime_selection, "_STRUCTURE_NODES", 1_000) + (repo / "agents" / "a" / "docker-compose.yml").write_text(text) + + with pytest.raises( + runtime_selection.RuntimeSelectionError, match="compose_structural_interpolation_limit" + ): + _select_in_folder(repo) + + +@pytest.mark.parametrize( + "text, entries, code", + [ + pytest.param(_merge_chain(20, 10), 100, "compose_yaml_too_large", id="wide"), + # Under the entry limit, but deeper than any real file nests its merges. + pytest.param(_merge_chain(1, 40), None, "compose_yaml_merge_cycle_or_depth", id="deep"), + pytest.param(_deep_groups(17, 100), None, "compose_yaml_too_deep", id="deep-flow"), + ], +) +def test_selection_refuses_merge_keys_that_build_too_much(repo, monkeypatch, text, entries, code): + from fi.alk.harness import runtime_selection + + if entries: + monkeypatch.setattr(runtime_selection, "_YAML_ENTRIES", entries) + (repo / "agents" / "a" / "docker-compose.yml").write_text( + text + "services:\n agent:\n build: .\n" + ) + + with pytest.raises(runtime_selection.RuntimeSelectionError, match=f"^{code}$"): + _select_in_folder(repo) + + +def test_bounded_loading_reads_an_honest_file_as_yaml_does(repo): + import yaml + + from fi.alk.harness.runtime_selection import bounded_yaml_load + + text = ( + "x-common: &common {restart: always, environment: {A: '1'}}\n" + "x-more: &more {<<: *common, init: true}\n" + "services:\n" + " agent: {<<: [*more], build: ., ports: ['8080:8080']}\n" + " worker: {<<: *common, build: ., depends_on: [agent]}\n" + ) + (repo / "agents" / "a" / "docker-compose.yml").write_text(text) + + assert bounded_yaml_load(text) == yaml.safe_load(text) + assert bounded_yaml_load("") is None + _select_in_folder(repo) + + +def test_runtime_manifest_refuses_merge_keys_that_build_too_much(repo, monkeypatch): + from fi.alk.harness import runtime_selection + from fi.alk.harness.bundle_author_v2 import BundleAuthorError, _declared_runtime_environment + + monkeypatch.setattr(runtime_selection, "_YAML_ENTRIES", 100) + (repo / "agents" / "a" / "alk.yaml").write_text(_merge_chain(20, 10)) + + with pytest.raises(BundleAuthorError, match="runtime_manifest_invalid: yaml_too_large"): + _declared_runtime_environment(repo / "agents" / "a") + + +def test_runtime_manifest_refuses_a_file_past_its_size(repo, monkeypatch): + from fi.alk.harness import bundle_author_v2 + + monkeypatch.setattr(bundle_author_v2, "_LARGEST_RUNTIME_MANIFEST", 10) + (repo / "agents" / "a" / "alk.yaml").write_text("runtime: {environment: {}}\n") + + with pytest.raises(bundle_author_v2.BundleAuthorError, match="larger than"): + bundle_author_v2._declared_runtime_environment(repo / "agents" / "a") + + +def test_loading_refuses_a_base_60_number_too_long_to_build(repo): + import time + + from fi.alk.harness.bundle_author_v2 import BundleAuthorError, _declared_runtime_environment + from fi.alk.harness.runtime_selection import YamlTooLarge, bounded_yaml_load + + assert bounded_yaml_load("t: 1:30:00\nu: " + ":".join(["1"] * 17) + "\n")["t"] == 5400 + # Only numbers: text with more colons, such as a PATH, still loads. + many = ":".join(["1"] * 30) + assert bounded_yaml_load(f'p: "{many}"\nq: a:{many}\n') == {"p": many, "q": f"a:{many}"} + started = time.monotonic() + with pytest.raises(YamlTooLarge, match="yaml_scalar_too_large"): + bounded_yaml_load("x-a: " + ":".join(["1"] * 200_000) + "\n") + assert time.monotonic() - started < 2 + + (repo / "agents" / "a" / "alk.yaml").write_text("x-a: " + ":".join(["1"] * 18) + "\n") + with pytest.raises(BundleAuthorError, match="runtime_manifest_invalid: yaml_scalar_too_large"): + _declared_runtime_environment(repo / "agents" / "a") + + +def test_loading_refuses_flow_nesting_past_16(repo): + import time + + from fi.alk.harness.bundle_author_v2 import BundleAuthorError, _declared_runtime_environment + from fi.alk.harness.runtime_selection import YamlTooLarge, bounded_yaml_load + + assert bounded_yaml_load("x: " + "[" * 16 + "]" * 16 + "\n") is not None + with pytest.raises(YamlTooLarge, match="yaml_too_deep"): + bounded_yaml_load("x: " + "{a: " * 16 + "[]" + "}" * 16 + "\n") + # About 4 s to read under the save's 128 KB before the limit. + started = time.monotonic() + with pytest.raises(YamlTooLarge, match="yaml_too_deep"): + bounded_yaml_load(_deep_groups(400, 120_000)) + assert time.monotonic() - started < 0.5 + + (repo / "agents" / "a" / "alk.yaml").write_text(_deep_groups(17, 100)) + with pytest.raises(BundleAuthorError, match="runtime_manifest_invalid: yaml_too_deep"): + _declared_runtime_environment(repo / "agents" / "a") + + +@pytest.mark.parametrize("name", ["docker-compose.yml", "alk.yaml"]) +def test_deep_flow_nesting_does_not_hold_up_the_save(tmp_path, repo, name, monkeypatch): + from fi.alk.harness import runtime_selection + + # About 4 s to read before the limit. The bound is shown by the loader's work, not the runner's + # clock: the scanner stops at the sixteenth open group, a handful of tokens in, however long + # the file is. Without the limit, a sixth of this file takes about 19,000 token fetches. + (repo / "agents" / "a" / name).write_text(_deep_groups(400, 120_000)) + loader = runtime_selection._BoundedLoader + fetch, opening = loader.fetch_more_tokens, loader.fetch_flow_collection_start + fetched, refused = [0], [] + + def counted(self): + fetched[0] += 1 + return fetch(self) + + def opened(self, token_class): + try: + return opening(self, token_class) + except runtime_selection.YamlTooLarge: + refused.append(self.flow_level) + raise + + monkeypatch.setattr(loader, "fetch_more_tokens", counted) + monkeypatch.setattr(loader, "fetch_flow_collection_start", opened) + + result = accept_contract( + payload(runtime={"language": "python"}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert not result.get("is_error"), result + assert refused == [16] + assert fetched[0] < 100 + + +def test_loading_refuses_a_long_tag_prefix(): + from fi.alk.harness.runtime_selection import YamlTooLarge, bounded_yaml_load + + assert bounded_yaml_load("%TAG !e! tag:yaml.org,2002:\n---\na: !e!str 5\n") == {"a": "5"} + # Each `!a` gets its own copy of the prefix: 131 KB built about 1 GB. + with pytest.raises(YamlTooLarge, match="yaml_tag_too_long"): + bounded_yaml_load("%TAG !a! tag:" + "x" * 1025 + "\n--- !a!x\na: 1\n") + + +def test_the_check_runs_off_the_loop_and_a_refusal_meanwhile_wins(tmp_path, repo, monkeypatch): + import time + + from fi.alk.harness import tools + from fi.alk.harness.failure_file import FAILURE_FILE + + destination = tmp_path / "out" + server = contract_tools(destination, source_root=repo, scan_root=repo, subfolder="agents/a") + arguments = payload( + runtime={"language": "python"}, hard_constraints=["be polite"], data_store={"kind": "none"} + ) + + def slow(*args): + time.sleep(0.5) + + async def save(during): + worst = [0.0] + done = asyncio.Event() + + async def beat(): + last = time.perf_counter() + while not done.is_set(): + await asyncio.sleep(0.01) + now = time.perf_counter() + worst[0], last = max(worst[0], now - last), now + + beating = asyncio.create_task(beat()) + monkeypatch.setattr(tools, "_builds_a_sibling", during) + result = await server.tools[0].handler(arguments) + done.set() + await beating + return result, worst[0] + + # The heap the tests before this one built is frozen, so a full collection of it can't land + # inside the save. The save's own collections still count. + gc.collect() + gc.freeze() + try: + result, worst = asyncio.run(save(slow)) + finally: + gc.unfreeze() + assert (destination / "contract.json").exists(), result + assert worst < 0.25 + + (destination / "contract.json").unlink() + + def refused_meanwhile(*args): + (destination / FAILURE_FILE).write_text("{}") + + asyncio.run(save(refused_meanwhile)) + assert not (destination / "contract.json").exists() + + +def test_an_alk_yaml_without_end_is_read_only_to_its_limit(tmp_path, repo, monkeypatch, capsys): + import io + + from fi.alk.harness.bundle_author_v2 import BundleAuthorError, _declared_runtime_environment + + class Endless(io.RawIOBase): + def readable(self): + return True + + def read(self, size=-1): + assert size >= 0, "read to the end" + return b"#" * size + + folder = repo / "agents" / "a" + (folder / "alk.yaml").write_text("") + (folder / "compose.yaml").write_text("services:\n agent:\n build: ../b\n") + opened = Path.open + # Like a link to /proc/self/pagemap: its size says 0 and it reads without end. + monkeypatch.setattr( + Path, + "open", + lambda path, *args, **kwargs: Endless() + if path.name == "alk.yaml" + else opened(path, *args, **kwargs), + ) + with pytest.raises(BundleAuthorError, match="larger than 1 MiB"): + _declared_runtime_environment(folder) + result = accept_contract( + payload(runtime=_SIBLING), tmp_path / "out", source_root=repo, subfolder="agents/a" + ) + + assert not result.get("is_error"), result + assert capsys.readouterr().err == "" + + +def test_one_count_covers_every_included_file(repo, monkeypatch): + from fi.alk.harness import runtime_selection + + included = "x-b: [" + ", ".join(["1"] * 30) + "]\n" + main = "include:\n - more.yml\nx-a: [" + ", ".join(["1"] * 30) + "]\n" + _GOOD_SERVICE + counts = [] + for text in (main, included): + budget = [10**6] + runtime_selection.bounded_yaml_load(text, budget) + counts.append(10**6 - budget[0]) + # Each file is under the count alone; together they're over it. + monkeypatch.setattr(runtime_selection, "_YAML_ENTRIES", max(counts) + 1) + (repo / "agents" / "a" / "docker-compose.yml").write_text(main) + (repo / "agents" / "a" / "more.yml").write_text(included) + + with pytest.raises(runtime_selection.RuntimeSelectionError, match="compose_yaml_too_large"): + _select_in_folder(repo) + + +def _plain_list(items: int) -> str: + return "x-a: [" + ",".join(["1"] * items) + "]\n" + + +_SIBLING = { + "language": "python", + "execution": "compose", + "compose_file": "agents/a/compose.yaml", + "service": "agent", +} + + +@pytest.mark.parametrize( + "files", + [ + # Each took the save 5-10 s to read on the event loop. + pytest.param({"compose.yaml": _plain_list(1_000_000)}, id="compose"), + pytest.param({"alk.yaml": _plain_list(500_000)}, id="alk"), + # Under the save's size alone; together over it. + pytest.param( + {"compose.yaml": "#" * 70_000 + "\n", "alk.yaml": "#" * 70_000 + "\n"}, + id="together", + ), + ], +) +def test_save_leaves_files_past_real_sizes_to_bundling(tmp_path, repo, capsys, files): + import time + + from fi.alk.harness.runtime_selection import COMPOSE_SERVICE_OUTSIDE_FOLDER, select_runtime + + folder = repo / "agents" / "a" + sibling = "services:\n agent:\n build: ../b\n" + (folder / "compose.yaml").write_text(files.get("compose.yaml", "") + sibling) + if "alk.yaml" in files: + (folder / "alk.yaml").write_text(files["alk.yaml"]) + started = time.monotonic() + + result = accept_contract( + payload(runtime=_SIBLING), tmp_path / "out", source_root=repo, subfolder="agents/a" + ) + + assert time.monotonic() - started < 2 + assert not result.get("is_error"), result + assert capsys.readouterr().err == "" + if len(files) == 2: + # Bundling reads up to 2 MB and refuses the same service; this row is quick to read. + with pytest.raises(ValueError, match=COMPOSE_SERVICE_OUTSIDE_FOLDER): + select_runtime( + repo, + {**_SIBLING, "source_subfolder": "agents/a", "workdir": "agents/a"}, + validate_environment_dependencies=False, + ) + + +def test_save_leaves_compose_overlays_to_bundling(tmp_path, repo, capsys, monkeypatch): + from fi.alk.harness import compose_merge + + merged = [] + monkeypatch.setattr(compose_merge, "merge_compose", lambda *args, **kwargs: merged.append(args)) + folder = repo / "agents" / "a" + (folder / "compose.yaml").write_text("services:\n agent:\n build: ../b\n") + (folder / "override.yaml").write_text("services:\n agent:\n init: true\n") + + result = accept_contract( + payload(runtime={**_SIBLING, "compose_overrides": ["agents/a/override.yaml"]}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert not result.get("is_error"), result + assert merged == [] + assert capsys.readouterr().err == "" + + +def test_a_skipped_check_names_the_error_type_alone(tmp_path, repo, caplog): + (repo / "agents" / "a" / "alk.yaml").write_text("runtime: [customer-line-42\n") + + with caplog.at_level("WARNING", logger="fi.alk.harness.tools"): + result = accept_contract( + payload(runtime={"language": "python"}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert not result.get("is_error"), result + assert [record.getMessage() for record in caplog.records] == [ + "sibling check skipped: BundleAuthorError" + ] + assert "customer-line-42" not in caplog.text + + +def test_a_bug_in_the_sibling_check_is_not_swallowed(tmp_path, repo, monkeypatch): + from fi.alk.harness import tools + + def broken(*args, **kwargs): + raise TypeError("a bug") + + monkeypatch.setattr(tools, "select_runtime", broken) + + with pytest.raises(TypeError, match="a bug"): + accept_contract( + payload(runtime={"language": "python"}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + +@pytest.mark.parametrize( + "link, target, named", + [ + ("agents/up", "..", "agents/up"), + ("self", ".", "self"), + ("agents/a/x", "..", "agents/a/x"), + # Out of the repository and back in, to the root. + ("ext", "..", "ext/repo"), + ], +) +def test_a_folder_that_links_to_its_own_folder_or_above_is_not_a_folder(repo, link, target, named): + from fi.alk.harness.source_folder import SubfolderNotFound, agent_folder + + (repo / link).symlink_to(target) + + with pytest.raises(SubfolderNotFound): + agent_folder(repo, named) + + +def test_a_folder_that_links_to_a_sibling_is_still_a_folder(repo): + from fi.alk.harness.source_folder import agent_folder + + (repo / "agents" / "current").symlink_to("b") + + assert agent_folder(repo, "agents/current") == (repo / "agents" / "b").resolve() + + +@pytest.mark.parametrize("shell", ["sh", "/bin/sh"]) +def test_save_refuses_a_shell_command_only_under_a_folder(tmp_path, repo, shell): + runtime = {"language": "python", "command": [shell, "-c", "cd ../b && python agent.py"]} + + refused = accept_contract( + payload(runtime=runtime), tmp_path / "in", source_root=repo, subfolder="agents/a" + ) + kept = accept_contract( + payload(runtime=runtime), tmp_path / "root", source_root=repo + ) + + assert refused.get("is_error") + assert "runtime_command_shell_forbidden" in refused["content"][0]["text"] + assert not (tmp_path / "in" / "contract.json").exists() + assert not kept.get("is_error") + assert _saved(tmp_path / "root")["runtime"]["command"] == runtime["command"] + + +def _deploy(folder: Path, value: str, copies: int = 1, build: str = ".") -> None: + (folder / "compose.yaml").write_text( + f"x-s: &s '{value}'\nservices:\n agent:\n build: {build}\n deploy: [" + + ", ".join(["*s"] * copies) + + "]\n" + ) + + +def _select_compose(repo: Path, environment: dict[str, str]): + from fi.alk.harness.runtime_selection import select_runtime + + return select_runtime( + repo, + { + "language": "python", + "execution": "compose", + "compose_file": "agents/a/compose.yaml", + "service": "agent", + "source_subfolder": "agents/a", + "workdir": "agents/a", + }, + public_environment=environment, + validate_environment_dependencies=False, + ) + + +@pytest.mark.parametrize("reference", ["$A", "${A}"]) +@pytest.mark.parametrize("times, refused", [(65, False), (66, True)]) +def test_selection_refuses_a_value_repeated_past_the_string_limit( + repo, reference, times, refused +): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + _deploy(repo / "agents" / "a", reference * times) + if refused: + with pytest.raises(RuntimeSelectionError, match="^compose_structural_interpolation_limit$"): + _select_compose(repo, {"A": "x" * 1000}) + else: + _select_compose(repo, {"A": "x" * 1000}) + + +@pytest.mark.parametrize("copies, refused", [(100, False), (200, True)]) +def test_what_a_string_builds_is_charged_for_every_alias(repo, copies, refused): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + _deploy(repo / "agents" / "a", "$A", copies) + if refused: + with pytest.raises(RuntimeSelectionError, match="^compose_structural_interpolation_limit$"): + _select_compose(repo, {"A": "x" * 60_000}) + else: + _select_compose(repo, {"A": "x" * 60_000}) + + +@pytest.mark.parametrize("times, raised", [(65, None), (66, "compose_structural_interpolation_limit")]) +def test_the_save_caps_a_value_from_alk_yaml_repeated_in_compose( + tmp_path, repo, capsys, monkeypatch, times, raised +): + from fi.alk.harness import tools + + folder = repo / "agents" / "a" + (folder / "alk.yaml").write_text("runtime:\n environment:\n A: '" + "x" * 1000 + "'\n") + _deploy(folder, "$A" * times) + seen = [] + real = tools.select_runtime + + def recorded(*args, **kwargs): + try: + return real(*args, **kwargs) + except Exception as exc: + seen.append(str(exc)) + raise + + monkeypatch.setattr(tools, "select_runtime", recorded) + + result = accept_contract( + payload(runtime={**_SIBLING, "service": "agent"}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert not result.get("is_error"), result + assert seen == ([raised] if raised else []) + assert capsys.readouterr().err == "" + + +def test_loading_counts_mapping_keys(repo): + from fi.alk.harness.runtime_selection import YamlTooLarge, bounded_yaml_load + + text = "{" + ", ".join(f"k{i}: {i}" for i in range(10)) + "}" + + # The mapping, what it merges, ten keys and ten values. + assert len(bounded_yaml_load(text, [31])) == 10 + with pytest.raises(YamlTooLarge): + bounded_yaml_load(text, [30]) + + +def _racing_server(tmp_path, repo): + destination = tmp_path / "out" + server = contract_tools(destination, source_root=repo, scan_root=repo, subfolder="agents/a") + return destination, server.tools[0].handler + + +def _plain(notes: str) -> dict: + return payload( + runtime={"language": "python"}, + hard_constraints=["be polite"], + data_store={"kind": "none"}, + notes=notes, + ) + + +def test_the_contract_sent_last_is_saved_when_an_earlier_one_finishes_later( + tmp_path, repo, monkeypatch +): + import threading + + from fi.alk.harness import tools + + destination, submit = _racing_server(tmp_path, repo) + second_checked = threading.Event() + real = tools._checked_contract + + def checked(arguments, **kwargs): + result = real(arguments, **kwargs) + if arguments.get("notes") == "sent-first": + assert second_checked.wait(10) + else: + second_checked.set() + return result + + monkeypatch.setattr(tools, "_checked_contract", checked) + + async def both(): + return await asyncio.gather(submit(_plain("sent-first")), submit(_plain("sent-second"))) + + first, second = asyncio.run(both()) + + assert _saved(destination)["notes"] == "sent-second" + assert first["content"][0]["text"].startswith("Not saved: a contract submitted after this one") + assert not second.get("is_error"), second + + +def test_a_refusal_during_the_check_wins_though_its_failure_file_is_not_written( + tmp_path, repo, monkeypatch +): + import threading + + from fi.alk.harness import tools + + destination, submit = _racing_server(tmp_path, repo) + checking, refused = threading.Event(), threading.Event() + real = tools._checked_contract + + def checked(arguments, **kwargs): + checking.set() + assert refused.wait(10) + return real(arguments, **kwargs) + + def unwritable(*args): + refused.set() + raise OSError("disk full") + + monkeypatch.setattr(tools, "_checked_contract", checked) + from fi.alk.harness import agent_type_check + + # The reading's verdict, as SecondLook asks for it. + monkeypatch.setattr(agent_type_check, "decide", lambda claimed, *args, **kwargs: "refuse" if claimed == "browser" else "keep") + monkeypatch.setattr(tools, "write_failure_file", unwritable) + + async def both(): + saving = asyncio.ensure_future(submit(_plain("saving"))) + # The refusal comes once the check is under way, so only the check after it can see it. + assert await asyncio.to_thread(checking.wait, 10) + with pytest.raises(OSError): + await submit({**_plain("refusing"), "modality": "browser"}) + return await saving + + saving = asyncio.run(both()) + + assert not (destination / "contract.json").exists() + assert "Don't submit again" in saving["content"][0]["text"] + + +def test_a_crafted_python_file_does_not_hold_up_the_conversation(tmp_path, repo): + import time + + from fi.alk.harness.python_source import LARGEST_PARSED_MODULE + + # Parsing this costs the square of its size: about 7 s on 3.12 and 0.7 s on 3.11 at 1 MiB. + line = "f'{a}'\n" + lines = (1024 * 1024) // len(line) + (repo / "agents" / "a" / "crafted.py").write_text("import traceai_openai\n" + line * lines) + # Just under the limit it's parsed, and that is quick. + small = "import traceai_openai\n" + line * ((LARGEST_PARSED_MODULE // 2) // len(line)) + (repo / "agents" / "a" / "small.py").write_text(small) + destination, submit = _racing_server(tmp_path, repo) + + async def save(): + worst = [0.0] + done = asyncio.Event() + + async def beat(): + last = time.perf_counter() + while not done.is_set(): + await asyncio.sleep(0.01) + now = time.perf_counter() + worst[0], last = max(worst[0], now - last), now + + beating = asyncio.create_task(beat()) + result = await submit(_plain("crafted")) + done.set() + await beating + return result, worst[0] + + # The heap the tests before this one built is frozen, so a full collection of it can't land + # inside the save (one took 0.3 s inside a parse on CI). The save's own collections still count. + gc.collect() + gc.freeze() + try: + result, worst = asyncio.run(save()) + finally: + gc.unfreeze() + + assert not result.get("is_error"), result + assert _saved(destination)["instrumentation"] == "traceai" + assert worst < 0.25 + + +@pytest.mark.parametrize("copies, refused", [(3, False), (4, True)]) +def test_each_reference_is_charged_however_little_it_builds(repo, copies, refused): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + # An empty value builds nothing, but each `$A` is still worked: 30,000 a copy. + _deploy(repo / "agents" / "a", "$A" * 30_000, copies) + if refused: + with pytest.raises(RuntimeSelectionError, match="^compose_structural_interpolation_limit$"): + _select_compose(repo, {"A": ""}) + else: + _select_compose(repo, {"A": ""}) + + +@pytest.mark.parametrize( + "text, refused", + [ + ("x" * 536 + "$A", False), + ("x" * 537 + "$A", True), + # What it keeps after the last reference, too. + ("${A}" + "x" * 536, False), + ("${A}" + "x" * 537, True), + ], +) +def test_what_a_string_keeps_counts_towards_what_it_builds(repo, text, refused): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + _deploy(repo / "agents" / "a", text) + if refused: + with pytest.raises(RuntimeSelectionError, match="^compose_structural_interpolation_limit$"): + _select_compose(repo, {"A": "y" * 65_000}) + else: + _select_compose(repo, {"A": "y" * 65_000}) + + +@pytest.mark.parametrize("copies, refused", [(3, False), (4, True)]) +def test_each_brace_is_charged(repo, copies, refused): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + # 16,000 `${A}` a copy: 32,000 with its closing brace charged, 16,000 with the `$` alone, so + # a fourth copy passes 100,000 only when the braces count. + _deploy(repo / "agents" / "a", "${A}" * 16_000, copies) + if refused: + with pytest.raises(RuntimeSelectionError, match="^compose_structural_interpolation_limit$"): + _select_compose(repo, {"A": ""}) + else: + _select_compose(repo, {"A": ""}) + + +def test_an_earlier_contract_is_saved_when_the_later_ones_write_fails(tmp_path, repo, monkeypatch): + import threading + + from fi.alk.harness import tools + + destination, submit = _racing_server(tmp_path, repo) + second_written = threading.Event() + real_check, real_save = tools._checked_contract, tools._save_contract + + def checked(arguments, **kwargs): + result = real_check(arguments, **kwargs) + if arguments.get("notes") == "sent-first": + assert second_written.wait(10) + return result + + def save(contract, where): + if contract.notes == "sent-second": + second_written.set() + raise OSError("disk full") + return real_save(contract, where) + + monkeypatch.setattr(tools, "_checked_contract", checked) + monkeypatch.setattr(tools, "_save_contract", save) + + async def both(): + return await asyncio.gather( + submit(_plain("sent-first")), submit(_plain("sent-second")), return_exceptions=True + ) + + first, second = asyncio.run(both()) + + assert isinstance(second, OSError) + assert not first.get("is_error"), first + assert _saved(destination)["notes"] == "sent-first" + + +@pytest.mark.parametrize("over, refused", [(0, False), (1, True)]) +def test_the_prompt_is_copied_only_from_a_file_up_to_1_mib(tmp_path, repo, over, refused): + folder = repo / "agents" / "a" + line = "Be brief.\n" + (folder / "prompt.txt").write_text(line + "#" * (1024 * 1024 + over - len(line))) + + result = accept_contract( + payload( + runtime={"language": "python"}, + system_prompt_from=[{"path": "agents/a/prompt.txt", "start_line": 1, "end_line": 1}], + ), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert bool(result.get("is_error")) is refused, result + if refused: + assert "larger than 1 MiB" in result["content"][0]["text"] + + +@pytest.mark.parametrize("copies, refused", [(100, False), (200, True)]) +def test_a_strings_length_is_charged_though_it_builds_nothing(repo, copies, refused): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + # A unset, so `:+` builds nothing; the 60,000 characters are still read for every alias. + _deploy(repo / "agents" / "a", "${A:+" + "x" * 60_000 + "}", copies) + if refused: + with pytest.raises(RuntimeSelectionError, match="^compose_structural_interpolation_limit$"): + _select_compose(repo, {}) + else: + _select_compose(repo, {}) + + +def test_a_prompt_file_without_end_is_read_only_to_its_limit(tmp_path, repo, monkeypatch): + import os + + from fi.alk.harness import model_citation + + (repo / "agents" / "a" / "prompt.txt").write_text("") + # The prompt is read through model_citation.read_source, by descriptor. + endless: set[int] = set() + real_open, real_read, real_close = os.open, os.read, os.close + + def open_(path, flags, *args, **kwargs): + descriptor = real_open(path, flags, *args, **kwargs) + if Path(path).name == "prompt.txt": + endless.add(descriptor) + return descriptor + + given: list[int] = [] + + def read(descriptor, size): + # Like a link to /proc/self/pagemap: its size says 0 and it reads without end. + if descriptor not in endless: + return real_read(descriptor, size) + given.append(size) + assert sum(given) <= 8 * 1024 * 1024, "read to the end" + return b"#" * size + + def close(descriptor): + endless.discard(descriptor) + real_close(descriptor) + + monkeypatch.setattr(model_citation.os, "open", open_) + monkeypatch.setattr(model_citation.os, "read", read) + monkeypatch.setattr(model_citation.os, "close", close) + + result = accept_contract( + payload( + runtime={"language": "python"}, + system_prompt_from=[{"path": "agents/a/prompt.txt", "start_line": 1, "end_line": 1}], + ), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert "larger than 1 MiB" in result["content"][0]["text"], result + + +@pytest.mark.parametrize( + "helper, image_only, refused", + [("../b", False, True), (".", False, False), ("../b", True, False)], + ids=["builds-a-sibling", "builds-the-folder", "pulled-as-an-image"], +) +def test_every_service_the_stack_runs_is_checked_for_a_sibling(repo, helper, image_only, refused): + from fi.alk.harness.runtime_selection import COMPOSE_SERVICE_OUTSIDE_FOLDER, RuntimeSelectionError, select_runtime + + folder = repo / "agents" / "a" + for name in ("a", "b"): + (repo / "agents" / name / "Dockerfile").write_text("FROM python\n") + (folder / "compose.yaml").write_text( + "services:\n agent:\n build: .\n depends_on: [helper]\n" + f" helper:\n build: {helper}\n image: helper:1\n" + ) + runtime = { + "language": "python", + "execution": "compose", + "compose_file": "agents/a/compose.yaml", + "service": "agent", + "source_subfolder": "agents/a", + "workdir": "agents/a", + "compose_image_services": ["helper"] if image_only else [], + } + if refused: + with pytest.raises(RuntimeSelectionError, match=f"^{COMPOSE_SERVICE_OUTSIDE_FOLDER}: helper$"): + select_runtime(repo, runtime, validate_environment_dependencies=False) + else: + select_runtime(repo, runtime, validate_environment_dependencies=False) + + +def _helpers(count: int, build: str) -> str: + names = [f"h{n}" for n in range(count)] + return ( + f"x-b: &b {build}\nservices:\n agent:\n build: .\n" + f" depends_on: [{', '.join(names)}]\n" + + "".join(f" {name}:\n build: *b\n" for name in names) + ) + + +def test_services_sharing_a_build_are_checked_once(repo, monkeypatch): + from fi.alk.harness import runtime_selection + + # An alias gives every helper the same build at no cost in the file. + (repo / "agents" / "a" / "Dockerfile").write_text("FROM python\n") + (repo / "agents" / "a" / "compose.yaml").write_text( + _helpers(20, "{context: ., dockerfile: Dockerfile}") + ) + resolved: list[str] = [] + real = runtime_selection._resolved + monkeypatch.setattr( + runtime_selection, + "_resolved", + lambda base, value: resolved.append(value) or real(base, value), + ) + + _select_compose(repo, {}) + + assert resolved.count("Dockerfile") == 1 + + +def test_a_build_path_past_the_systems_limit_is_refused(repo): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + (repo / "agents" / "a" / "compose.yaml").write_text( + "services:\n agent:\n build: ." + "/x" * 2048 + "\n" + ) + + with pytest.raises(RuntimeSelectionError, match="^compose_source_path_invalid$"): + _select_compose(repo, {}) + + +def test_each_part_of_a_build_path_is_charged(repo): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + # 60 builds of 2,000 parts each: each resolve looks up every part, 120,000 lookups. + names = [f"h{n}" for n in range(60)] + (repo / "agents" / "a" / "compose.yaml").write_text( + "services:\n agent:\n build: .\n" + f" depends_on: [{', '.join(names)}]\n" + + "".join(f" {name}:\n build: ./{name}" + "/x" * 2000 + "\n" for name in names) + ) + + with pytest.raises(RuntimeSelectionError, match="^compose_structural_interpolation_limit$"): + _select_compose(repo, {}) + + +@pytest.mark.parametrize("key", ["network_mode", "ipc", "pid"]) +def test_a_mode_that_is_not_a_string_is_refused(repo, key): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + # Ten levels of such a list expand to gigabytes when written out. + (repo / "agents" / "a" / "compose.yaml").write_text( + f"x-l: &l [a, b]\nservices:\n agent:\n build: .\n {key}: *l\n" + ) + + with pytest.raises(RuntimeSelectionError, match=f"^compose_invalid: agent {key} must be a string$"): + _select_compose(repo, {}) + + +@pytest.mark.parametrize("environment", ["{A: [x, y]}", "[{A: x}]"]) +def test_an_environment_value_that_is_not_plain_is_refused(repo, environment): + from fi.alk.harness.runtime_selection import RuntimeSelectionError, select_runtime + + (repo / "agents" / "a" / "compose.yaml").write_text( + f"services:\n agent:\n build: .\n environment: {environment}\n" + ) + + with pytest.raises(RuntimeSelectionError, match="^compose_invalid: agent environment values must be plain$"): + select_runtime( + repo, + { + "language": "python", + "execution": "compose", + "compose_file": "agents/a/compose.yaml", + "service": "agent", + "source_subfolder": "agents/a", + "workdir": "agents/a", + }, + validate_environment_dependencies=True, + ) + + +@pytest.mark.parametrize("profiles, refused", [("", True), (" profiles: [debug]\n", False)]) +def test_with_no_control_service_a_service_behind_a_profile_is_not_checked(repo, profiles, refused): + from fi.alk.harness.runtime_selection import COMPOSE_SERVICE_OUTSIDE_FOLDER, RuntimeSelectionError, select_runtime + + (repo / "agents" / "b" / "Dockerfile").write_text("FROM python\n") + (repo / "agents" / "a" / "compose.yaml").write_text( + "services:\n agent:\n build: .\n helper:\n build: ../b\n" + profiles + ) + # Found by looking in the folder, with no service named: the whole file runs. + runtime = { + "language": "python", + "execution": "auto", + "source_subfolder": "agents/a", + "workdir": "agents/a", + } + if refused: + with pytest.raises(RuntimeSelectionError, match=f"^{COMPOSE_SERVICE_OUTSIDE_FOLDER}: helper$"): + select_runtime(repo, runtime, validate_environment_dependencies=False) + else: + select_runtime(repo, runtime, validate_environment_dependencies=False) + + +@pytest.mark.parametrize("entries, refused", [(1, False), (2, True)]) +def test_the_prompt_copies_at_most_1_mib_in_all(tmp_path, repo, entries, refused): + # Named twice, 600 KB of lines copy twice: a model naming one file a hundred times wrote + # 100 MB into contract.json. + lines = 6000 + (repo / "agents" / "a" / "prompt.txt").write_text(("#" * 99 + "\n") * lines) + + result = accept_contract( + payload( + runtime={"language": "python"}, + system_prompt_from=[ + {"path": "agents/a/prompt.txt", "start_line": 1, "end_line": lines} + ] + * entries, + ), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert bool(result.get("is_error")) is refused, result + if refused: + assert "more than 1 MiB" in result["content"][0]["text"] + + +def test_a_write_that_fails_partway_leaves_the_saved_contract(tmp_path, repo, monkeypatch): + from fi.alk.harness import tools + from fi.alk.harness.contract import AgentContract + + destination = tmp_path / "out" + accept_contract(payload(runtime={"language": "python"}), destination, source_root=repo, subfolder="agents/a") + saved = (destination / "contract.json").read_text() + contract = AgentContract.model_validate_json(saved) + write_text = Path.write_text + + def fails_partway(path, text, *args, **kwargs): + write_text(path, text[: len(text) // 2], *args, **kwargs) + raise OSError("disk full") + + monkeypatch.setattr(Path, "write_text", fails_partway) + + with pytest.raises(OSError): + tools._save_contract(contract.model_copy(update={"notes": "later"}), destination) + + assert (destination / "contract.json").read_text() == saved + assert sorted(path.name for path in destination.iterdir()) == ["contract.json"] + + +@pytest.mark.parametrize("files, refused", [(4, False), (5, True)]) +def test_the_prompt_reads_at_most_4_mib_of_files_in_all(tmp_path, repo, files, refused): + # Line 1 of each, but each file is read whole to find its lines. + for n in range(files): + (repo / "agents" / "a" / f"p{n}.txt").write_text("Be brief.\n" + "#" * 1_000_000) + + result = accept_contract( + payload( + runtime={"language": "python"}, + system_prompt_from=[ + {"path": f"agents/a/p{n}.txt", "start_line": 1, "end_line": 1} + for n in range(files) + ], + ), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert bool(result.get("is_error")) is refused, result + if refused: + assert "more than 4 MiB" in result["content"][0]["text"] + + +def test_a_prompt_file_named_again_is_not_read_again(tmp_path, repo): + # Read again for each entry, these five would come to more than 4 MiB. + (repo / "agents" / "a" / "p.txt").write_text("Be brief.\n" + "#" * 1_000_000) + + result = accept_contract( + payload( + runtime={"language": "python"}, + system_prompt_from=[{"path": "agents/a/p.txt", "start_line": 1, "end_line": 1}] * 5, + ), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert not result.get("is_error"), result + + +def test_a_prompt_file_past_the_limit_is_read_once(tmp_path, repo, monkeypatch): + large = repo / "agents" / "a" / "p.txt" + large.write_text("Be brief.\n" + "#" * (1024 * 1024)) + import os + + from fi.alk.harness import model_citation + + opened: list[Path] = [] + real = os.open + # The prompt is opened through model_citation.read_source, by descriptor. + monkeypatch.setattr( + model_citation.os, + "open", + lambda path, *args, **kwargs: opened.append(Path(path)) or real(path, *args, **kwargs), + ) + + result = accept_contract( + payload( + runtime={"language": "python"}, + system_prompt_from=[{"path": "agents/a/p.txt", "start_line": 1, "end_line": 1}] * 5, + ), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert result.get("is_error"), result + assert result["content"][0]["text"].count("larger than 1 MiB") == 5 + assert opened.count(large.resolve()) == 1 + + +def _charged(path: Path) -> int: + from fi.alk.harness import runtime_selection + + budget = [10**9] + token = runtime_selection._SELECTION_BUDGET.set(budget) + try: + resolved = runtime_selection._charged_resolve(path) + finally: + runtime_selection._SELECTION_BUDGET.reset(token) + assert resolved == path.resolve() + return 10**9 - budget[0] + + +def test_each_lookup_is_charged_more_the_deeper_it_is(tmp_path): + real = tmp_path.resolve() + shallow, deep = real / "x", real.joinpath(*["x"] * 600) + + # A lookup for each part, and the stat that names a link loop; past 256 parts, two each. + def cost(path: Path) -> int: + prefixes = ["/".join(path.parts[: n + 1])[1:] for n in range(1, len(path.parts))] + return sum(1 + prefix.count("/") // 256 for prefix in prefixes) + 1 + str(path).count("/") // 256 + + assert _charged(shallow) == cost(shallow) + assert _charged(deep) == cost(deep) + assert _charged(deep) > len(deep.parts) + 300 + + +def test_a_link_is_charged_for_the_parts_it_leads_to(tmp_path): + real = tmp_path.resolve() + deep = real.joinpath(*["d"] * 300) + deep.mkdir(parents=True) + (real / "L").symlink_to(deep) + # Down and back up again: as short resolved as written, but each part of the way is looked + # up. A link's target is at most 1,024 bytes on macOS. + (real / "C").symlink_to("d/" * 150 + "../" * 150) + + assert _charged(real / "L" / "s") > 300 + assert _charged(real / "C" / "s") > 150 + assert _charged(real / "x" / "s") < len(real.parts) + 4 + + +def test_a_link_loop_is_refused_as_resolve_refuses_it(tmp_path): + from fi.alk.harness import runtime_selection + + (tmp_path / "one").symlink_to("two") + (tmp_path / "two").symlink_to("one") + + if sys.version_info < (3, 13): + # 3.13's non-strict resolve no longer raises on a loop; the charged one still does. + with pytest.raises(RuntimeError, match="Symlink loop"): + (tmp_path / "one" / "x").resolve() + with pytest.raises(RuntimeError, match="Symlink loop"): + runtime_selection._charged_resolve(tmp_path / "one" / "x") + + +@pytest.mark.parametrize("linked, refused", [(False, False), (True, True)]) +def test_helpers_building_through_a_link_to_a_deep_folder_are_charged(repo, linked, refused): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + folder = repo / "agents" / "a" + deep = folder.joinpath("d", *["x"] * 400) + deep.mkdir(parents=True) + if linked: + (folder / "L").symlink_to(deep) + else: + (folder / "L").mkdir() + names = [f"h{n}" for n in range(300)] + (folder / "compose.yaml").write_text( + "services:\n agent:\n build: .\n" + f" depends_on: [{', '.join(names)}]\n" + + "".join(f" {name}:\n build: L/{name}\n" for name in names) + ) + + if refused: + with pytest.raises(RuntimeSelectionError, match="^compose_structural_interpolation_limit$"): + _select_compose(repo, {}) + else: + _select_compose(repo, {}) + + +def test_a_shared_context_with_a_siblings_dockerfile_is_checked_apart(repo): + from fi.alk.harness.runtime_selection import COMPOSE_SERVICE_OUTSIDE_FOLDER, RuntimeSelectionError + + (repo / "agents" / "a" / "Dockerfile").write_text("FROM python\n") + (repo / "agents" / "b" / "Dockerfile").write_text("FROM python\n") + # `inner` is checked first; the same context with another Dockerfile is a new check. + (repo / "agents" / "a" / "compose.yaml").write_text( + "services:\n" + " agent:\n build: {context: ., dockerfile: ../b/Dockerfile}\n depends_on: [inner]\n" + " inner:\n build: {context: ., dockerfile: Dockerfile}\n" + ) + + with pytest.raises(RuntimeSelectionError, match=f"^{COMPOSE_SERVICE_OUTSIDE_FOLDER}: agent$"): + _select_compose(repo, {}) + + +def test_with_a_control_service_a_helper_behind_a_profile_is_checked(repo): + from fi.alk.harness.runtime_selection import COMPOSE_SERVICE_OUTSIDE_FOLDER, RuntimeSelectionError + + (repo / "agents" / "b" / "Dockerfile").write_text("FROM python\n") + # The control service depends on it, so it runs whatever its profile. + (repo / "agents" / "a" / "compose.yaml").write_text( + "services:\n agent:\n build: .\n depends_on: [helper]\n" + " helper:\n build: ../b\n profiles: [debug]\n" + ) + + with pytest.raises(RuntimeSelectionError, match=f"^{COMPOSE_SERVICE_OUTSIDE_FOLDER}: helper$"): + _select_compose(repo, {}) + + +@pytest.mark.parametrize("asked, refused", [("debug", True), ("*", True), ("other,debug", True), ("other", False)]) +def test_with_no_control_service_a_profile_asked_for_is_checked(repo, asked, refused): + from fi.alk.harness.runtime_selection import COMPOSE_SERVICE_OUTSIDE_FOLDER, RuntimeSelectionError, select_runtime + + (repo / "agents" / "b" / "Dockerfile").write_text("FROM python\n") + (repo / "agents" / "a" / "compose.yaml").write_text( + "services:\n agent:\n build: .\n helper:\n build: ../b\n profiles: [debug]\n" + ) + runtime = { + "language": "python", + "execution": "auto", + "source_subfolder": "agents/a", + "workdir": "agents/a", + } + environment = {"COMPOSE_PROFILES": asked} + if refused: + with pytest.raises(RuntimeSelectionError, match=f"^{COMPOSE_SERVICE_OUTSIDE_FOLDER}: helper$"): + select_runtime(repo, runtime, public_environment=environment, validate_environment_dependencies=False) + else: + select_runtime(repo, runtime, public_environment=environment, validate_environment_dependencies=False) + + +def test_an_include_is_resolved_with_its_lookups_charged(repo, monkeypatch): + from fi.alk.harness import runtime_selection + + (repo / "agents" / "a" / "more.yaml").write_text("services:\n helper:\n image: redis\n") + (repo / "agents" / "a" / "compose.yaml").write_text( + "include: [more.yaml]\nservices:\n agent:\n build: .\n depends_on: [helper]\n" + ) + resolved: list[str] = [] + real = runtime_selection._charged_resolve + monkeypatch.setattr( + runtime_selection, "_charged_resolve", lambda path: resolved.append(path.name) or real(path) + ) + + _select_compose(repo, {}) + + assert {"compose.yaml", "more.yaml"} <= set(resolved) + + +def test_the_parts_a_link_steps_over_are_charged(tmp_path): + real = tmp_path.resolve() + # Empty, `.` and `..` parts need no lookup, but each is walked, on every resolve. + (real / "E").symlink_to("." + "/" * 1000) + (real / "D").symlink_to("./" * 500) + (real / "U").symlink_to("x/../" * 200) + + assert _charged(real / "E" / "s") > 1000 + assert _charged(real / "D" / "s") > 500 + assert _charged(real / "U" / "s") > 400 + + +@pytest.mark.parametrize("padded, refused", [(False, False), (True, True)]) +def test_helpers_building_through_padded_links_are_charged(repo, padded, refused): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + folder = repo / "agents" / "a" + # A chain of links, each to the next through a thousand empty parts: short as written and + # resolved, inside the folder. + pad = "/" * 1000 if padded else "/" + for n in range(1, 5): + (folder / f"L{n}").symlink_to(f".{pad}L{n + 1}") + (folder / "L5").symlink_to(f".{pad}.") + names = [f"h{n}" for n in range(30)] + (folder / "compose.yaml").write_text( + "services:\n agent:\n build: .\n" + f" depends_on: [{', '.join(names)}]\n" + + "".join(f" {name}:\n build: L1/{name}\n" for name in names) + ) + + if refused: + with pytest.raises(RuntimeSelectionError, match="^compose_structural_interpolation_limit$"): + _select_compose(repo, {}) + else: + _select_compose(repo, {}) + + +@pytest.mark.parametrize("links, refused", [(3, False), (20, True)]) +def test_links_whose_leftovers_unwind_past_the_limit_are_refused(repo, links, refused): + from fi.alk.harness.runtime_selection import RuntimeSelectionError + + folder = repo / "agents" / "a" + # Each link leaves 1,000 characters behind as it unwinds, so the path grows a level at a time. + for n in range(1, links + 1): + (folder / f"L{n}").symlink_to(f"L{n + 1}/" + "A" * 1000) + (folder / f"L{links + 1}").symlink_to(".") + (folder / "compose.yaml").write_text( + "services:\n agent:\n build: .\n depends_on: [h0]\n h0:\n build: L1/h0\n" + ) + + if refused: + with pytest.raises(RuntimeSelectionError, match="^compose_source_path_invalid$"): + _select_compose(repo, {}) + else: + _select_compose(repo, {}) + + +def _link_fan(root: Path, heads: int = 4, chain: int = 3, pad: int = 50) -> str: + """Hawk's shape, small: each head is a chain of links whose targets are "." and many slashes + and the next link, the last one back to the root. A path through them names the root, and + resolving it costs a lookup for each part of every target. Under the kernel's link limit, so + the system resolves it too. Returns the heads, as a path.""" + for head in range(heads): + for link in range(chain + 1): + name = f"H{head}" if link == 0 else f"H{head}_{link}" + after = f"H{head}_{link + 1}" if link < chain else "." + (root / name).symlink_to("." + "/" * pad + after) + return "/".join(f"H{head}" for head in range(heads)) + + +def _lookups(monkeypatch, refused: bool) -> None: + from fi.alk.harness import runtime_selection + + # The fan above costs about 900 lookups. + monkeypatch.setattr(runtime_selection, "_SAVE_PATH_LOOKUPS", 300 if refused else 20_000) + + +@pytest.mark.parametrize("subfolder", ["agents/a", None]) +@pytest.mark.parametrize("refused", [False, True]) +def test_a_prompt_path_through_links_is_charged_to_the_saves_budget( + tmp_path, repo, monkeypatch, refused, subfolder +): + _lookups(monkeypatch, refused) + (repo / "agents" / "a" / "prompt.txt").write_text("Be polite.\n") + fan = _link_fan(repo) + + result = accept_contract( + payload( + runtime={"language": "python"}, + system_prompt_from=[ + {"path": f"{fan}/agents/a/prompt.txt", "start_line": 1, "end_line": 1} + ], + ), + tmp_path / "out", + source_root=repo, + subfolder=subfolder, + ) + + assert bool(result.get("is_error")) is refused, result + if refused: + assert "path_lookup_limit:" in result["content"][0]["text"] + else: + assert _saved(tmp_path / "out")["system_prompt_excerpt"] == "Be polite." + + +@pytest.mark.parametrize("refused", [False, True]) +@pytest.mark.parametrize("where", ["compose_file", "command", "workdir"]) +def test_a_runtime_path_through_links_is_charged_to_the_saves_budget( + tmp_path, repo, monkeypatch, where, refused +): + _lookups(monkeypatch, refused) + (repo / "agents" / "a" / "compose.yaml").write_text("services:\n agent:\n build: .\n") + fan = _link_fan(repo) + runtime = { + "compose_file": {"language": "python", "compose_file": f"{fan}/agents/a/compose.yaml"}, + # From the folder, out through the links and back into it: past the budget the check + # can't say where it leads, so it mustn't pass. + "command": {"language": "python", "command": ["python", f"../../{fan}/agents/a/agent.py"]}, + "workdir": {"language": "python", "workdir": f"{fan}/agents/a"}, + }[where] + + result = accept_contract( + payload(runtime=runtime), tmp_path / "out", source_root=repo, subfolder="agents/a" + ) + + assert bool(result.get("is_error")) is refused, result + if refused: + assert "path_lookup_limit:" in result["content"][0]["text"] + + +@pytest.mark.parametrize("subfolder", ["agents/a", None]) +@pytest.mark.parametrize("refused", [False, True]) +def test_a_tool_module_through_links_is_charged_to_the_saves_budget( + tmp_path, repo, monkeypatch, refused, subfolder +): + _lookups(monkeypatch, refused) + (repo / "agents" / "a" / "tools.py").write_text("def add(item_id):\n return item_id\n") + fan = _link_fan(repo) + + result = accept_contract( + payload( + runtime={"language": "python"}, + tool_entrypoints=[ + { + "tool": "add", + "mode": "import", + "module": fan.replace("/", ".") + ".agents.a.tools", + "callable": "add", + } + ], + ), + tmp_path / "out", + source_root=repo, + subfolder=subfolder, + ) + + assert bool(result.get("is_error")) is refused, result + if refused: + assert "path_lookup_limit:" in result["content"][0]["text"] + + +def test_a_prompt_path_named_again_is_resolved_once(tmp_path, repo, monkeypatch): + from fi.alk.harness import model_citation + + (repo / "agents" / "a" / "prompt.txt").write_text("Be polite.\nBe brief.\n") + resolved: list[Path] = [] + real = model_citation.charged_path + monkeypatch.setattr( + model_citation, "charged_path", lambda path: resolved.append(path) or real(path) + ) + + result = accept_contract( + payload( + runtime={"language": "python"}, + system_prompt_from=[ + {"path": "agents/a/prompt.txt", "start_line": line, "end_line": line} + for line in (1, 2, 1, 2) + ], + ), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert not result.get("is_error"), result + assert len(resolved) == 1 + + +@pytest.mark.parametrize( + "named, refused", + [ + ({"system_prompt_from": [{"path": "x/" * 2500 + "p.txt", "start_line": 1, "end_line": 1}]}, "is not a file in the source"), + ({"system_prompt_from": [{"path": "agents/a/p\x00.txt", "start_line": 1, "end_line": 1}]}, "is not a file in the source"), + # Not found, so the tool's evidence is unproven, not refused. + ({"tool_entrypoints": [{"tool": "add", "mode": "import", "module": "x." * 2500 + "t", "callable": "add"}]}, None), + ({"tool_entrypoints": [{"tool": "add", "mode": "import", "module": "x\x00.t", "callable": "add"}]}, None), + ({"runtime": {"language": "python", "compose_file": "x/" * 2500 + "compose.yaml"}}, "runtime_compose_file_outside_subfolder"), + # A program no system can open can't leave the folder; bundling refuses it. + ({"runtime": {"language": "python", "command": ["python", "x/" * 2500 + "a.py"]}}, None), + ({"runtime": {"language": "python", "command": ["python", "a\x00.py"]}}, None), + ], + ids=["prompt-long", "prompt-nul", "module-long", "module-nul", "compose-long", "command-long", "command-nul"], +) +def test_a_path_no_system_can_open_is_answered_not_raised(tmp_path, repo, named, refused): + # A path past the system's limit, or holding a NUL, raised out of the save before. + result = accept_contract( + payload(**{"runtime": {"language": "python"}, **named}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert bool(result.get("is_error")) is (refused is not None), result + if refused: + assert refused in result["content"][0]["text"] + + +@pytest.mark.parametrize( + "helper, refused", + [ + (' volumes: ["../b:/app"]\n', True), + (" volumes:\n - type: bind\n source: ../b\n target: /app\n", True), + (' env_file: ["../b/.env"]\n', True), + (" env_file:\n - path: ../b/.env\n", True), + (' volumes: ["./data:/data"]\n', False), + (' volumes: ["../..:/repo"]\n', False), + (' volumes: ["/var/run/docker.sock:/var/run/docker.sock"]\n', False), + (' volumes: ["cache:/cache", "~/.aws:/root/.aws"]\n', False), + (' env_file: [".env"]\n', False), + (" env_file: ../b/.env\n", True), + (' volumes: ["../b/agent.py:/app/main.py:ro"]\n', True), + (' volumes: ["${SRC:-../b}:/app"]\n', True), + (" env_file: ../../.env\n", False), + (' env_file: ["../shared.env"]\n', False), + (' volumes: ["../../config.yaml:/app/config.yaml:ro"]\n', False), + (" env_file:\n - path: .env.local\n required: false\n", False), + (" env_file: ${ENV_FILE}\n", False), + ], + ids=["bind-sibling", "long-bind-sibling", "env-file-sibling", "env-path-sibling", "bind-own", + "bind-root", "bind-host", "named-and-home", "env-own", "env-string-sibling", + "file-bind-sibling", "interpolated-bind-sibling", "env-root-file", "env-above-file", + "file-bind-root", "env-optional-missing", "env-unset-variable"], +) +def test_a_service_that_binds_or_reads_a_siblings_files_is_refused(repo, helper, refused): + from fi.alk.harness.runtime_selection import COMPOSE_SERVICE_OUTSIDE_FOLDER, RuntimeSelectionError, select_runtime + + (repo / "agents" / "b" / ".env").write_text("SIBLING_KEY=x\n") + # A root .env or config file, and one beside the agents, are the agent's to read, as a + # Dockerfile's folder is what a build check counts. + (repo / ".env").write_text("ROOT_KEY=x\n") + (repo / "config.yaml").write_text("a: 1\n") + (repo / "agents" / "shared.env").write_text("SHARED=x\n") + (repo / "agents" / "a" / "compose.yaml").write_text( + "services:\n agent:\n build: .\n" + " helper:\n image: python:3.12\n command: [python, /app/main.py]\n" + helper + ) + runtime = { + "language": "python", + "execution": "auto", + "source_subfolder": "agents/a", + "workdir": "agents/a", + } + + if refused: + with pytest.raises(RuntimeSelectionError, match=f"^{COMPOSE_SERVICE_OUTSIDE_FOLDER}: helper$"): + select_runtime(repo, runtime, validate_environment_dependencies=False) + else: + select_runtime(repo, runtime, validate_environment_dependencies=False) + + +@pytest.mark.parametrize("imports, expected", [("from common import tracing\n", "traceai"), ("x = 1\n", "none")]) +def test_the_folders_instrumentation_counts_the_shared_code_it_imports(tmp_path, repo, imports, expected): + # agents/b's own traced.py says nothing about agents/a. + (repo / "agents" / "a" / "agent.py").write_text(imports) + (repo / "common").mkdir() + (repo / "common" / "__init__.py").write_text("") + (repo / "common" / "tracing.py").write_text("import traceai_openai\n") + + result = accept_contract( + payload(runtime={"language": "python"}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert not result.get("is_error"), result + assert _saved(tmp_path / "out")["instrumentation"] == expected + + +@pytest.mark.parametrize("package", ["testing_utils", "tests_shared", "e2e"]) +def test_the_folders_instrumentation_counts_a_test_named_package_it_imports(tmp_path, repo, package): + # The walk follows it and the credential scan reads it, so instrumentation does too: the root + # build says traceai. + (repo / "agents" / "a" / "agent.py").write_text(f"from {package} import tracing\n") + (repo / package).mkdir() + (repo / package / "__init__.py").write_text("") + (repo / package / "tracing.py").write_text("import traceai_openai\n") + + result = accept_contract( + payload(runtime={"language": "python"}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert not result.get("is_error"), result + assert _saved(tmp_path / "out")["instrumentation"] == "traceai" + + +def test_the_folders_instrumentation_counts_another_workspace_members_code(tmp_path, repo): + (repo / "pyproject.toml").write_text( + '[tool.uv.workspace]\nmembers=["agents/*", "libs/*"]\n[tool.uv.sources]\ncommon={workspace=true}\n' + ) + (repo / "agents" / "a" / "pyproject.toml").write_text( + "[project]\nname='agent-a'\nversion='1'\ndependencies=['common']\n" + ) + (repo / "agents" / "a" / "agent.py").write_text("from common import tracing\n") + shared = repo / "libs" / "common" / "src" / "common" + shared.mkdir(parents=True) + (repo / "libs" / "common" / "pyproject.toml").write_text("[project]\nname='common'\nversion='1'\n") + (shared / "__init__.py").write_text("") + (shared / "tracing.py").write_text("import traceai_openai\n") + + result = accept_contract( + payload(runtime={"language": "python"}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert not result.get("is_error"), result + assert _saved(tmp_path / "out")["instrumentation"] == "traceai" + + +def test_the_save_follows_a_committed_folder_link_to_the_shared_code(tmp_path, repo): + # common -> libs/common: bundling follows it, so the save does too, and instrumentation and the + # browser check read what's behind it. + (repo / "agents" / "a" / "agent.py").write_text("from common import tracing, browse\n") + shared = repo / "libs" / "common" + shared.mkdir(parents=True) + (shared / "__init__.py").write_text("") + (shared / "tracing.py").write_text("import traceai_openai\n") + (shared / "browse.py").write_text("import playwright\n") + (repo / "common").symlink_to("libs/common") + + from fi.alk.harness.agent_type_check import browser_tools_in_agent + + assert browser_tools_in_agent(repo, "agents/a") == ["playwright"] + result = accept_contract( + payload(runtime={"language": "python"}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + assert not result.get("is_error"), result + assert _saved(tmp_path / "out")["instrumentation"] == "traceai" + + +def test_the_save_walks_the_agents_imports_once(tmp_path, repo, monkeypatch): + from fi.alk.harness import agent_type_check + + (repo / "agents" / "a" / "agent.py").write_text("from common import tracing\n") + (repo / "common").mkdir() + (repo / "common" / "__init__.py").write_text("") + (repo / "common" / "tracing.py").write_text("import traceai_openai\n") + walks: list[str] = [] + real = agent_type_check.imported_by_agent + monkeypatch.setattr( + agent_type_check, + "imported_by_agent", + lambda root, subfolder, **limits: walks.append(subfolder) or real(root, subfolder, **limits), + ) + destination = tmp_path / "out" + server = contract_tools(destination, source_root=repo, scan_root=repo, subfolder="agents/a") + + _submit(server, payload(runtime={"language": "python"}), destination) + + # The browser scan's walk also tells the instrumentation check what the folder reaches. + assert walks == ["agents/a"] + assert _saved(destination)["instrumentation"] == "traceai" + + +def test_a_root_compose_file_may_read_the_roots_env_file_and_name_a_volume(repo): + from fi.alk.harness.runtime_selection import select_runtime + + (repo / ".env").write_text("ROOT_KEY=x\n") + (repo / "compose.yaml").write_text( + "services:\n agent:\n build: ./agents/a\n env_file: .env\n" + " volumes: [\"data:/data\", \"./config.yaml:/app/config.yaml:ro\"]\n" + ) + (repo / "config.yaml").write_text("a: 1\n") + # A bare name is a named volume, not the root folder of that name. + (repo / "data").mkdir() + + select_runtime( + repo, + { + "language": "python", + "execution": "auto", + "source_subfolder": "agents/a", + "workdir": "agents/a", + "compose_file": "compose.yaml", + }, + validate_environment_dependencies=False, + ) + + +@pytest.mark.parametrize( + "files, runtime", + [ + ({"agents/a/compose.yaml": "services:\n agent:\n build: .\ninclude: [more.yaml]\n", + "agents/a/more.yaml": "services:\n extra:\n image: redis\n1: x\n"}, {}), + ({"agents/a/compose.yaml": "services:\n agent:\n build: .\ninclude: [sub/more.yaml]\n", + "agents/a/sub/more.yaml": "services:\n extra:\n build: 5\n"}, {}), + ({"agents/a/compose.yaml": "services:\n agent:\n build: .\ninclude: [sub/more.yaml]\n", + "agents/a/sub/more.yaml": "services:\n extra:\n image: redis\n volumes: ./data:/data\n"}, {}), + ({"agents/a/compose.yaml": "services:\n agent:\n build: .\ninclude: [sub/more.yaml]\n", + "agents/a/sub/more.yaml": "services:\n extra:\n image: redis\n volumes:\n - type: bind\n target: /x\n"}, {}), + ({"agents/a/compose.yaml": "services:\n agent:\n build: .\ninclude: [sub/more.yaml]\n", + "agents/a/sub/more.yaml": "services:\n extra:\n image: redis\n env_file: 5\n"}, {}), + ], + ids=["int-key", "build-int", "volumes-string", "bind-without-source", "env-file-int"], +) +def test_a_compose_file_docker_refuses_is_left_to_bundling_not_raised(tmp_path, repo, files, runtime): + # The sibling check catches only what the customer's files can cause; reading a malformed + # file must raise the selection's own error, not a Python one out of the save. + for name, text in files.items(): + (repo / name).parent.mkdir(parents=True, exist_ok=True) + (repo / name).write_text(text) + + result = accept_contract( + payload(runtime={"language": "python", "compose_file": "agents/a/compose.yaml", **runtime}), + tmp_path / "out", + source_root=repo, + subfolder="agents/a", + ) + + assert "content" in result + + +def test_a_test_helper_above_the_folder_says_nothing_of_its_instrumentation(tmp_path, repo): + # As the browser scan counts it: a root conftest.py is the tests', not the agent's. + (repo / "conftest.py").write_text( + "from opentelemetry.sdk.trace import TracerProvider\n" + "from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter\n" + ) + + result = accept_contract( + payload(runtime={"language": "python"}), tmp_path / "out", source_root=repo, subfolder="agents/a" + ) + + assert not result.get("is_error"), result + assert _saved(tmp_path / "out")["instrumentation"] == "none" + + +def test_a_path_past_twice_the_system_limit_is_refused_before_any_lookup(monkeypatch): + from fi.alk.harness import runtime_selection + + looked: list[Path] = [] + monkeypatch.setattr(runtime_selection, "_charged_resolve", lambda path: looked.append(path) or path) + + with pytest.raises(ValueError): + runtime_selection.charged_path(Path("/" + "../" * 3000 + "x")) + assert looked == [] diff --git a/tests/harness/test_contract_writer.py b/tests/harness/test_contract_writer.py new file mode 100644 index 000000000..2bcfa3e9a --- /dev/null +++ b/tests/harness/test_contract_writer.py @@ -0,0 +1,1884 @@ +"""The one contract writer (TH-9491, plan v5.1 §2, §3, §V5-V7, §W6-W7). + +Every save of the contract goes through ``write_contract``. It cleans the prose of URL +credentials and keeps the technical fields as written, and it cleans the caller's own object in +place, so the stage that holds it goes on from what was saved. +""" + +from __future__ import annotations + +import ast +import asyncio +import collections +import copy +import inspect +import json +import logging +import os +import re +import subprocess +import sys +import types +import typing +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest +from pydantic import BaseModel, Field + +from fi.alk.harness import ( + amend, + authoring_entrypoint, + authoring_runtime_validation, + builder, + contract as contract_module, + nudge, + runtime_repair, + tools, +) +from fi.alk.harness.contract import ( + FREE_TEXT, + KEPT_AS_WRITTEN, + AgentContract, + Dependency, + Runtime, + RuntimeInterface, + ToolSpec, + write_contract, +) +from fi.alk.providers.agent import _redact + +SOURCE_TREE = Path(contract_module.__file__).resolve().parents[1] +PASSWORD = "hunter2" +KEY = "sk-1" +URL = f"https://bot:{PASSWORD}@hooks.example/v1?key={KEY}#frag" +CLEAN = "https://hooks.example/v1" +PROSE = f"Post to {URL} after each." +CLEANED = f"Post to {CLEAN} after each." + + +def _secret_free(value: Any) -> bool: + text = json.dumps(value, ensure_ascii=False) + return PASSWORD not in text and KEY not in text + + +# -- §3, V7: which fields are cleaned --------------------------------------------------------------- + +# Pinned by name: moving one of these into KEPT_AS_WRITTEN fails here (P16b). +PINNED_FREE_TEXT = ( + "AgentContract.one_liner", + "AgentContract.system_prompt_excerpt", + "AgentContract.hard_constraints", + "AgentContract.implementation", + "AgentContract.real_use_cases", + "AgentContract.caller_identity", + "AgentContract.notes", + "AgentContract.open_questions", + "AgentContract.amendments", + "ToolSpec.description", + "ToolSpec.refuses", + "ToolSpec.effects", + "ToolEntry.notes", + "Dependency.what", +) + + +def _models_in(annotation: Any) -> list[type[BaseModel]]: + """The models a field holds, through list, dict, Optional and Union (W10).""" + found: list[type[BaseModel]] = [] + if inspect.isclass(annotation) and issubclass(annotation, BaseModel): + found.append(annotation) + for argument in typing.get_args(annotation): + found.extend(_models_in(argument)) + return found + + +def _reachable_models() -> dict[str, type[BaseModel]]: + seen: dict[str, type[BaseModel]] = {} + stack: list[type[BaseModel]] = [AgentContract] + while stack: + model = stack.pop() + if model.__name__ in seen: + continue + seen[model.__name__] = model + for field in model.model_fields.values(): + stack.extend(_models_in(field.annotation)) + return seen + + +def _containers_of_models(annotation: Any) -> list[Any]: + """Each generic part of a field's type that holds a model, such as list[ToolSpec].""" + found: list[Any] = [] + if typing.get_origin(annotation) is not None and _models_in(annotation): + found.append(annotation) + for argument in typing.get_args(annotation): + found.extend(_containers_of_models(argument)) + return found + + +def _walked(annotation: Any) -> bool: + """Whether write_contract's walk (contract._holders) follows a type of this shape.""" + origin = typing.get_origin(annotation) + return ( + origin in (typing.Union, types.UnionType, typing.Annotated) + or contract_module._walks_as_sequence(origin) + or contract_module._walks_as_mapping(origin) + ) + + +def test_the_free_text_fields_are_the_fourteen_named() -> None: + assert FREE_TEXT == PINNED_FREE_TEXT + + +def test_every_field_the_contract_reaches_is_classified() -> None: + models = _reachable_models() + # 19 models, three of them from fi.simulate (CompletedToolEvents, CompletedToolFailureEvent, + # CompletedToolRows): a field added there must be classified here too. + assert {"CompletedToolEvents", "CompletedToolFailureEvent", "CompletedToolRows"} <= set(models) + classified = set(FREE_TEXT) | set(KEPT_AS_WRITTEN) + unclassified = [ + f"{name}.{field}" + for name, model in models.items() + if name not in KEPT_AS_WRITTEN + for field in model.model_fields + if f"{name}.{field}" not in classified + ] + assert unclassified == [] + assert not set(FREE_TEXT) & set(KEPT_AS_WRITTEN) + for entry in (*FREE_TEXT, *KEPT_AS_WRITTEN): + model, _, field = entry.partition(".") + assert model in models, entry + assert not field or field in models[model].model_fields, entry + # A model with free text in it is never kept whole. + assert not {entry.split(".")[0] for entry in FREE_TEXT} & set(KEPT_AS_WRITTEN) + # Every field that holds a model holds it in a shape the walk follows, so a new container + # type can't save a model's free text raw (change review r3, B-L1). + unwalked = [ + f"{name}.{field}: {shape}" + for name, model in models.items() + for field, info in model.model_fields.items() + for shape in _containers_of_models(info.annotation) + if not _walked(shape) + ] + assert unwalked == [] + + +@pytest.mark.parametrize( + "order", + [ + ("fi.alk.harness.contract", "fi.alk.providers.agent"), + ("fi.alk.providers.agent", "fi.alk.harness.contract"), + ], +) +def test_the_contract_and_the_cleaner_import_in_either_order(order: tuple[str, str]) -> None: + code = "import importlib\n" + "".join(f"importlib.import_module({name!r})\n" for name in order) + subprocess.run([sys.executable, "-c", code], check=True, timeout=120) + + +# -- B12: what write_contract writes ---------------------------------------------------------------- + +# The KEPT_AS_WRITTEN fields that take a free string, each given a URL with credentials so that +# cleaning one is seen (P10). Literal and pattern-checked fields can't hold one (W10). +KEPT_WITH_A_URL = ( + "agent", + "refusal_signature", + "instrumentation", + "agent_languages", + "chosen_evals", + "data_schema", + "base_environment", + "tools[].arg_values", + "tool_entrypoints[].endpoint", + "tool_entrypoints[].module", + "dependencies[].name", + "dependencies[].engine", + "runtime_dependencies[].name", + "data_store.host", + "data_store.configured_by", +) + + +def _full_contract() -> AgentContract: + """A contract with a URL holding a password and a key in every free-text field, and in each + kept field that takes a free string.""" + body: dict[str, Any] = { + "agent": URL, + "modality": "chat", + "refusal_signature": URL, + "instrumentation": URL, + "agent_languages": [URL], + "chosen_evals": [URL], + "data_schema": {"hook": URL}, + "base_environment": {"hook": [URL]}, + "tools": [ + {"name": "post", "args": ["kind"], "arg_values": {"kind": [URL]}}, + {"name": "book", "args": []}, + ], + "tool_entrypoints": [ + {"tool": "post", "mode": "service", "endpoint": URL, "module": URL}, + ], + "dependencies": [{"name": URL, "engine": URL}], + "runtime_dependencies": [{"name": URL}], + "data_store": {"kind": "database", "host": URL, "configured_by": URL}, + } + for name in FREE_TEXT: + model, field = name.split(".") + holders = ( + [body] + if model == "AgentContract" + else [ + item + for key, kind in ( + ("tools", "ToolSpec"), + ("tool_entrypoints", "ToolEntry"), + ("dependencies", "Dependency"), + ("runtime_dependencies", "Dependency"), + ) + if kind == model + for item in body[key] + ] + ) + for holder in holders: + annotation = AgentContract.model_fields.get(field) + is_list = annotation is not None and typing.get_origin(annotation.annotation) is list + holder[field] = [PROSE, "kept as written"] if is_list else PROSE + return AgentContract.model_validate(body) + + +def _free_values(body: dict[str, Any], name: str) -> list[Any]: + """Every value of one FREE_TEXT name in a saved body.""" + model, field = name.split(".") + if model == "AgentContract": + return [body[field]] + keys = [ + key + for key, info in AgentContract.model_fields.items() + if any(found.__name__ == model for found in _models_in(info.annotation)) + and typing.get_origin(info.annotation) is list + ] + return [item[field] for key in keys for item in body[key]] + + +def _without_free_text(body: dict[str, Any]) -> dict[str, Any]: + body = copy.deepcopy(body) + for name in FREE_TEXT: + model, field = name.split(".") + if model == "AgentContract": + body.pop(field) + continue + for key, info in AgentContract.model_fields.items(): + if any(found.__name__ == model for found in _models_in(info.annotation)): + for item in body.get(key) or []: + item.pop(field) + return body + + +def test_write_contract_cleans_every_free_text_field_and_keeps_every_other_field( + tmp_path: Path, +) -> None: + held = _full_contract() + raw = held.model_dump() + + write_contract(held, tmp_path) + + saved = json.loads((tmp_path / "contract.json").read_text(encoding="utf-8")) + for name in FREE_TEXT: + before, after = _free_values(raw, name), _free_values(saved, name) + assert before and len(before) == len(after), name + for given, cleaned in zip(before, after): + assert cleaned == _redact(given), name + assert _secret_free(cleaned), name + # Every field outside FREE_TEXT is exactly as it was given, URL and all. + assert _without_free_text(saved) == _without_free_text(raw) + kept = json.dumps(_without_free_text(saved)) + assert kept.count(URL) >= len(KEPT_WITH_A_URL) + + +def test_writing_a_written_contract_again_changes_nothing(tmp_path: Path) -> None: + write_contract(_full_contract(), tmp_path / "first") + first = (tmp_path / "first" / "contract.json").read_bytes() + + write_contract(AgentContract.model_validate_json(first), tmp_path / "second") + + assert (tmp_path / "second" / "contract.json").read_bytes() == first + + +def test_write_contract_cleans_the_callers_object_in_place_and_returns_it(tmp_path: Path) -> None: + held = _full_contract() + rules, tool = held.hard_constraints, held.tools[0] + + returned = write_contract(held, tmp_path) + + assert returned is held + assert held.hard_constraints is rules and held.tools[0] is tool + assert held.hard_constraints == [CLEANED, "kept as written"] + assert tool.description == CLEANED + assert held.model_dump() == AgentContract.model_validate_json( + (tmp_path / "contract.json").read_text(encoding="utf-8") + ).model_dump() + + +def test_a_failed_move_leaves_the_saved_contract_and_the_held_one_as_they_were( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_contract(AgentContract(agent="desk", one_liner="books desks"), tmp_path) + saved = (tmp_path / "contract.json").read_bytes() + held = AgentContract(agent="desk", one_liner=PROSE) + + def refuse(*_args: Any, **_kwargs: Any) -> None: + raise OSError("no space left on the device") + + monkeypatch.setattr(contract_module.os, "replace", refuse) + with pytest.raises(OSError): + write_contract(held, tmp_path) + + assert (tmp_path / "contract.json").read_bytes() == saved + assert sorted(path.name for path in tmp_path.iterdir()) == ["contract.json"] + # The held object is cleaned only after the move succeeds. + assert held.one_liner == PROSE + + +def test_write_contract_syncs_the_written_file_before_moving_it_in( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Change review r1, A-L2 (plan §2.2): the file moved in is the one synced, synced first.""" + calls: list[tuple[str, int, str]] = [] + real_fsync, real_replace = os.fsync, os.replace + + def fsync(descriptor: int) -> None: + calls.append(("fsync", os.fstat(descriptor).st_ino, "")) + real_fsync(descriptor) + + def replace(source: Any, target: Any) -> None: + calls.append(("replace", os.stat(source).st_ino, Path(target).name)) + real_replace(source, target) + + monkeypatch.setattr(contract_module.os, "fsync", fsync) + monkeypatch.setattr(contract_module.os, "replace", replace) + + write_contract(AgentContract(agent="desk", one_liner=PROSE), tmp_path) + + assert [(kind, name) for kind, _, name in calls] == [("fsync", ""), ("replace", "contract.json")] + assert calls[0][1] == calls[1][1] + assert json.loads((tmp_path / "contract.json").read_text(encoding="utf-8"))["one_liner"] == ( + CLEANED + ) + + +class _Wider(AgentContract): + """A contract with free-text models in field shapes AgentContract doesn't use yet (A-L3).""" + + fallback_tool: ToolSpec | None = None + tools_by_name: dict[str, ToolSpec] = Field(default_factory=dict) + tool_groups: list[list[ToolSpec]] = Field(default_factory=list) + dependency_by_use: dict[str, list[Dependency | None]] = Field(default_factory=dict) + + +def test_write_contract_finds_free_text_models_in_any_field_shape(tmp_path: Path) -> None: + """Change review r1, A-L3: an Optional, a dict's values, nested lists, all cleaned in place.""" + fallback = ToolSpec(name="fallback", description=PROSE, refuses=PROSE) + named = ToolSpec(name="post", description=PROSE, effects=PROSE) + grouped = ToolSpec(name="book", effects=PROSE) + dependency = Dependency(name="hooks", what=PROSE) + held = _Wider( + agent="desk", + fallback_tool=fallback, + tools_by_name={"post": named}, + tool_groups=[[grouped]], + dependency_by_use={"post": [None, dependency]}, + ) + + write_contract(held, tmp_path) + + saved = json.loads((tmp_path / "contract.json").read_text(encoding="utf-8")) + assert _secret_free(saved) + assert saved["fallback_tool"]["description"] == CLEANED + assert saved["fallback_tool"]["refuses"] == CLEANED + assert saved["tools_by_name"]["post"]["description"] == CLEANED + assert saved["tools_by_name"]["post"]["effects"] == CLEANED + assert saved["tool_groups"][0][0]["effects"] == CLEANED + empty, given = saved["dependency_by_use"]["post"] + assert empty is None and given["what"] == CLEANED and given["name"] == "hooks" + # The caller's own objects, in place. + assert held.fallback_tool is fallback and fallback.description == CLEANED + assert held.tools_by_name["post"] is named and named.effects == CLEANED + assert grouped.effects == CLEANED and dependency.what == CLEANED + + +class _Shaped(AgentContract): + """Change review r2, A-L1: tuples, a Sequence and a Mapping, which a first walk missed.""" + + tool_tuple: tuple[ToolSpec, ...] = () + tool_pair: tuple[ToolSpec, Dependency] | None = None + tool_seq: typing.Sequence[ToolSpec] = Field(default_factory=list) + tool_map: typing.Mapping[str, ToolSpec] = Field(default_factory=dict) + tool_list: typing.List[ToolSpec] = Field(default_factory=list) # noqa: UP006 + + +def test_write_contract_finds_free_text_models_in_tuples_sequences_and_mappings( + tmp_path: Path, +) -> None: + """Change review r2, A-L1: each shape is cleaned on disk and on the caller's own objects.""" + tools = {name: ToolSpec(name=name, description=PROSE) for name in ("t", "p", "s", "m", "l")} + dependency = Dependency(name="hooks", what=PROSE) + held = _Shaped( + agent="desk", + tool_tuple=(tools["t"],), + tool_pair=(tools["p"], dependency), + tool_seq=[tools["s"]], + tool_map={"m": tools["m"]}, + tool_list=[tools["l"]], + ) + + write_contract(held, tmp_path) + + saved = json.loads((tmp_path / "contract.json").read_text(encoding="utf-8")) + assert _secret_free(saved) + assert saved["tool_tuple"][0]["description"] == CLEANED + assert saved["tool_pair"][0]["description"] == CLEANED + assert saved["tool_pair"][1]["what"] == CLEANED and saved["tool_pair"][1]["name"] == "hooks" + assert saved["tool_seq"][0]["description"] == CLEANED + assert saved["tool_map"]["m"]["description"] == CLEANED + assert saved["tool_list"][0]["description"] == CLEANED + assert all(tool.description == CLEANED for tool in tools.values()) + assert dependency.what == CLEANED and _secret_free(held.model_dump()) + + +class _Ordered(AgentContract): + """Change review r3, B-L1: a subclass of a mapping, which a membership test missed.""" + + tools_in_order: collections.OrderedDict[str, ToolSpec] = Field( + default_factory=collections.OrderedDict + ) + + +def test_write_contract_finds_free_text_models_in_a_mapping_subclass(tmp_path: Path) -> None: + tool = ToolSpec(name="o", description=PROSE) + held = _Ordered(agent="desk", tools_in_order=collections.OrderedDict(o=tool)) + + write_contract(held, tmp_path) + + saved = json.loads((tmp_path / "contract.json").read_text(encoding="utf-8")) + assert _secret_free(saved) + assert saved["tools_in_order"]["o"]["description"] == CLEANED + assert tool.description == CLEANED + assert _walked(_Ordered.model_fields["tools_in_order"].annotation) + + +def test_write_contract_passes_over_an_empty_optional_model(tmp_path: Path) -> None: + """Change review r1, A-L3: a free-text model field left None saves as None. + + A regression guard only: no plant on the current walk makes it fail, since every branch + already passes a None over. It pins the behaviour for a rewrite of the walk.""" + held = _Wider(agent="desk", one_liner=PROSE) + + write_contract(held, tmp_path) + + saved = json.loads((tmp_path / "contract.json").read_text(encoding="utf-8")) + assert saved["fallback_tool"] is None and saved["one_liner"] == CLEANED + assert held.fallback_tool is None and held.one_liner == CLEANED + + +def test_one_save_logs_one_line_however_many_urls_are_refused( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + refused = "https://admin:hunter2#x@db.example/" + held = AgentContract( + agent="desk", one_liner=refused, notes=refused, hard_constraints=[refused] * 5 + ) + + with caplog.at_level(logging.DEBUG, logger="fi.alk.providers.agent"): + write_contract(held, tmp_path) + + logged = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] + assert [record.getMessage() for record in logged] == [ + "a URL was redacted whole: class=ValueError" + ] + assert PASSWORD not in caplog.text + assert PASSWORD not in (tmp_path / "contract.json").read_text(encoding="utf-8") + + +# -- B13: every writer goes through write_contract -------------------------------------------------- + + +class _Refused(Exception): + pass + + +@pytest.fixture +def refusing_writer(monkeypatch: pytest.MonkeyPatch) -> None: + def refuse(*_args: Any, **_kwargs: Any) -> None: + raise _Refused + + # Every module that holds the name, so no caller can reach the real one. + for module in (contract_module, tools, amend, runtime_repair): + monkeypatch.setattr(module, "write_contract", refuse) + + +def _amendable() -> AgentContract: + return AgentContract( + agent="desk", + one_liner="books desks", + real_use_cases=["book a desk"], + hard_constraints=["Confirm before booking"], + tools=[{"name": "post", "args": ["kind"], "arg_values": {"kind": ["a"]}}], + tool_entrypoints=[{"tool": "gone", "mode": "unreachable"}], + ) + + +def _declare_tool(tmp_path: Path, contract: AgentContract, description: str = "cancel") -> Any: + source = tmp_path / "source" + source.mkdir(exist_ok=True) + (source / "agent.py").write_text("def cancel():\n return 'ok'\n", encoding="utf-8") + server = builder.suite_server(tmp_path / "out", [], contract, source) + declare = next(one for one in server.tools if one.name == "declare_tool") + return asyncio.run( + declare.handler( + { + "name": "cancel", + "description": description, + "evidence_file": "agent.py", + "evidence_quote": "def cancel():", + "why": "the agent has it", + } + ) + ) + + +def _runtime_repair(tmp_path: Path) -> None: + source = tmp_path / "source" + service = source / "service" + service.mkdir(parents=True) + (service / "pyproject.toml").write_text("[project]\nname='agent'\nversion='1'\n") + (service / "app.py").write_text("# entrypoint\n") + contract = AgentContract(agent="support-agent", runtime=Runtime(language="python")) + contract_path = tmp_path / "contract.json" + contract_path.write_text(contract.model_dump_json(indent=2) + "\n") + runtime_repair.apply_runtime_plan_patch( + source, + contract_path, + runtime_repair.RuntimePlanPatch( + base_contract_hash=runtime_repair._contract_hash(contract), + runtime=Runtime( + language="python", + workdir="service", + command=["python", "app.py"], + interface=RuntimeInterface(kind="http", port=8080, path="/chat"), + ), + evidence_paths=("service/pyproject.toml", "service/app.py"), + diagnostic_codes=("generated_runtime_plan_invalid",), + summary="Run the HTTP service declared by the submitted component.", + ), + allowed_diagnostic_codes={"generated_runtime_plan_invalid"}, + ) + + +def _data_store_repair(tmp_path: Path) -> int: + """V8 row 5: the data-store repair's helper, called directly with a fake candidate folder.""" + authoring = tmp_path / "authoring" + authoring.mkdir() + contract = AgentContract(agent="desk", real_use_cases=["book"]) + (authoring / "contract.json").write_text(contract.model_dump_json(), encoding="utf-8") + candidate = tmp_path / "candidate" + candidate.mkdir() + (candidate / "contract.json").write_text( + AgentContract(agent="desk", data_store={"kind": "in_memory"}).model_dump_json(), + encoding="utf-8", + ) + return authoring_runtime_validation._promote_data_store(contract, authoring, candidate) + + +def _write_stale_contract(tmp_path: Path) -> Path: + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "contract.json").write_text(_amendable().model_dump_json(), encoding="utf-8") + return workspace + + +_WRITERS = { + "_save_contract": lambda tmp: tools._save_contract(_amendable(), tmp), + "widen": lambda tmp: amend.widen( + _amendable(), tmp, tool_name="post", argument="kind", values=["b"], why="w" + ), + "add_rule": lambda tmp: amend.add_rule(_amendable(), tmp, rule="Be brief", why="w"), + "drop_rule": lambda tmp: amend.drop_rule( + _amendable(), tmp, rule="Confirm before booking", why="w" + ), + "fix_tool": lambda tmp: amend.fix_tool( + _amendable(), tmp, tool_name="post", why="w", description="posts" + ), + "set_modality": lambda tmp: amend.set_modality(_amendable(), tmp, modality="voice", why="w"), + "builder.declare_tool": lambda tmp: _declare_tool(tmp, _amendable()), + "nudge": lambda tmp: nudge.drop_stale_tool_references(_write_stale_contract(tmp)), + "authoring_entrypoint": lambda tmp: authoring_entrypoint._refresh_source_facts( + None, tmp, _write_stale_contract(tmp) + ), + "runtime_plan_repair": _runtime_repair, + "data_store_repair": _data_store_repair, +} + + +@pytest.mark.parametrize("writer", list(_WRITERS)) +def test_every_contract_writer_goes_through_write_contract( + writer: str, tmp_path: Path, refusing_writer: None +) -> None: + with pytest.raises(_Refused): + _WRITERS[writer](tmp_path) + + +# The static half of B13 (W7). A function is flagged when it names the contract file and calls a +# write verb; the allow-list holds every such function by (module, function, verb, count), so a +# new write in any of them, or a new function, fails the test. The scan reads the source, so it +# is a heuristic: the raising patch above and the canaries are what prove each writer goes +# through write_contract. Change review r1 (A-M1) widened it to the nine shapes it missed: +# a path held on self, helpers to any depth, methods and nested defs as helpers, lambdas, an +# aliased CONTRACT_FILE, functions imported from shutil and os, the file named at the end of a +# path, an open mode held in a variable, and os.open, os.fdopen and os.link. Change review r2 +# (A-M1) added three more: a module-level name bound to the file, imported into another module +# under any name or read as module.NAME; a helper method called on an instance of a class +# defined in the same module (Keeper().save(...), or keeper = Keeper() then keeper.save(...)); +# and zipfile's writestr and tarfile's addfile. Change review r3 (B-M1) added three more: a static +# or class method called on a class defined in the same module (Store.keep(...)); a class +# constant bound to the file in a class body, read as Names.CONTRACT_JSON; and a name bound to a +# marker by an import (from .contract import CONTRACT_FILE as CONTRACT_JSON), which is a marker +# in every module that imports it from there. +# +# What it still can't see, each a way to write the file on purpose rather than by accident: +# - a getter that returns the path (contract_path(out).write_text(...)); +# - a path handed to a constructor and written by a method of the object +# (JsonFile(out / "contract.json").save(body), B's plant R3a); +# - a copy run as a process (subprocess and cp), or through functools.partial, getattr or +# io.FileIO; +# - a name built at run time ("contract" + ".json"), the file named as bytes (b"contract.json"), +# or a module global set in one function and written in another; +# - a helper kept in a module-level lambda (_keep = lambda path, body: path.write_text(body)), +# or a bound method kept in a name (write = (out / "contract.json").write_text; write(body)); +# - a def that hands its own parameter to an instance method called on its class with the +# instance passed by hand (Keeper.save(keeper, path)) is not made a helper, since the slots +# are read one place off; the call itself is flagged where the file is named; +# - a class constant read on a class from another module, or on an instance other than self; +# - ZipFile.write and TarFile.add with the file as the archive name, since write and add are +# too common to be verbs; +# - a helper method inherited from a class in another module, a method called on an instance +# of an imported class, and a helper function re-exported through a package __init__; +# - a changed target: the allow-list keys each function by (module, function, verb, count), so +# a write already counted in an allow-listed function can be pointed at contract.json without +# changing its count, and the scan still passes. +MARKER_NAMES = frozenset({"CONTRACT", "CONTRACT_FILE"}) +MARKER_TEXT = "contract.json" +# The file by itself, or as the last part of a path: "authoring/contract.json". +_MARKER_PATH = re.compile(r"(?:^|[/\\])" + re.escape(MARKER_TEXT) + r"$") +_DOTTED_VERBS = frozenset( + { + "os.replace", + "os.rename", + "os.renames", + "os.link", + "os.symlink", + "shutil.copy", + "shutil.copy2", + "shutil.copyfile", + "shutil.copytree", + "shutil.move", + "json.dump", + } +) +# A call into one of these modules is a verb only when it is listed above or is an open that +# writes, never by the attribute rules below (os.open is not Path.open). +_KNOWN_MODULES = frozenset({"os", "shutil", "json", "io", "codecs"}) +_WRITE_FLAGS = frozenset({"O_WRONLY", "O_RDWR", "O_CREAT", "O_APPEND", "O_TRUNC", "O_EXCL"}) +_READ_FLAGS = frozenset( + {"os", "O_RDONLY", "O_CLOEXEC", "O_NOFOLLOW", "O_BINARY", "O_NONBLOCK", "O_DIRECTORY"} +) +_SELF = frozenset({"self", "cls"}) +_DEFS = (ast.FunctionDef, ast.AsyncFunctionDef) +# A lambda is not a scope of its own: its body counts in the def around it. +_SCOPES = (*_DEFS, ast.ClassDef) + + +@dataclass(frozen=True) +class _Module: + """What a module's names resolve to, for telling which def a call reaches.""" + + name: str + # Local names bound to a module: the stdlib's, or one of this package's. + modules: dict[str, str] + # Local names bound to a function of _KNOWN_MODULES, as "module.function". + functions: dict[str, str] + # Local names imported from one of this package's modules, as (module, name). + names: dict[str, tuple[str, str]] + # The functions defined here, at any depth (methods aside). + defined: frozenset[str] + # The classes defined here, at any depth, and the names bound anywhere here to an instance + # of one: keeper = Keeper(...). + classes: frozenset[str] = frozenset() + instances: frozenset[str] = frozenset() + + +def _module_name(path: Path, root: Path) -> str: + parts = path.relative_to(root.parent.parent).with_suffix("").parts + return ".".join(parts[:-1] if parts[-1] == "__init__" else parts) + + +def _context(tree: ast.Module, path: Path, root: Path, package: frozenset[str]) -> _Module: + name = _module_name(path, root) + here = name.split(".") if path.name == "__init__.py" else name.split(".")[:-1] + modules = {one: one for one in _KNOWN_MODULES} + functions: dict[str, str] = {} + names: dict[str, tuple[str, str]] = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.asname: + modules[alias.asname] = alias.name + else: + modules[alias.name.split(".")[0]] = alias.name.split(".")[0] + elif isinstance(node, ast.ImportFrom): + base = here[: len(here) - node.level + 1] if node.level else [] + source = ".".join([*base, *([node.module] if node.module else [])]) + for alias in node.names: + local = alias.asname or alias.name + if f"{source}.{alias.name}" in package: + modules[local] = f"{source}.{alias.name}" + elif source in _KNOWN_MODULES: + functions[local] = f"{source}.{alias.name}" + else: + names[local] = (source, alias.name) + defined = frozenset(node.name for node, method in _defs(tree) if not method) + classes = frozenset(node.name for node in ast.walk(tree) if isinstance(node, ast.ClassDef)) + instances: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + value, targets = node.value, node.targets + elif isinstance(node, (ast.AnnAssign, ast.NamedExpr)) and node.value is not None: + value, targets = node.value, [node.target] + elif isinstance(node, ast.withitem) and node.optional_vars is not None: + value, targets = node.context_expr, [node.optional_vars] + else: + continue + if _instance_of(value, classes): + instances |= {one.id for one in targets if isinstance(one, ast.Name)} + return _Module(name, modules, functions, names, defined, classes, frozenset(instances)) + + +def _instance_of(node: ast.AST, classes: frozenset[str]) -> bool: + """Whether a node makes an instance of one of these classes: Keeper(...).""" + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in classes + ) + + +def _marker_aliases(tree: ast.Module) -> frozenset[str]: + """Names CONTRACT or CONTRACT_FILE is imported under: ``import CONTRACT_FILE as _NAME``.""" + return frozenset( + alias.asname + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + for alias in node.names + if alias.name in MARKER_NAMES and alias.asname + ) + + +def _docstrings(tree: ast.AST) -> set[int]: + found = set() + for node in ast.walk(tree): + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + body = node.body + if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant): + found.add(id(body[0].value)) + return found + + +def _own_nodes(scope: ast.AST) -> list[ast.AST]: + """A scope's own nodes: a nested def or class is scanned as its own scope, a lambda is not.""" + nodes: list[ast.AST] = [] + stack = list(ast.iter_child_nodes(scope)) + while stack: + node = stack.pop() + nodes.append(node) + if not isinstance(node, _SCOPES): + stack.extend(ast.iter_child_nodes(node)) + return nodes + + +def _argument(call: ast.Call, position: int, keyword: str) -> ast.AST | None: + for one in call.keywords: + if one.arg == keyword: + return one.value + return call.args[position] if len(call.args) > position else None + + +def _mode_writes(mode: ast.AST | None) -> bool: + """Whether an open mode may write. One held in a variable may.""" + if mode is None: + return False + if isinstance(mode, ast.Constant): + return isinstance(mode.value, str) and any(flag in mode.value for flag in "wax+") + return True + + +def _flags_write(call: ast.Call) -> bool: + """Whether os.open's flags may write: a write flag, or flags this can't read.""" + flags = _argument(call, 1, "flags") + if flags is None: + return False + if isinstance(flags, ast.Constant): + return flags.value != 0 + names = {sub.attr for sub in ast.walk(flags) if isinstance(sub, ast.Attribute)} + names |= _names_in(flags) + return bool(names & _WRITE_FLAGS or names - _READ_FLAGS - _WRITE_FLAGS) + + +def _dotted(func: ast.AST, imports: _Module) -> str | None: + """``module.function`` for a call into one of _KNOWN_MODULES, through any import alias.""" + if isinstance(func, ast.Name): + return imports.functions.get(func.id) + if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name): + module = imports.modules.get(func.value.id) + if module in _KNOWN_MODULES: + return f"{module}.{func.attr}" + return None + + +def _written(call: ast.Call, slots: frozenset[int | str]) -> list[ast.AST]: + """The arguments a call to a helper writes to: those in its slots, and any * or ** ones.""" + targets: list[ast.AST] = [ + argument + for index, argument in enumerate(call.args) + if index in slots or isinstance(argument, ast.Starred) + ] + targets += [one.value for one in call.keywords if one.arg is None or one.arg in slots] + return targets + + +# A helper is keyed by ("function" or "method", module, name). A call reaches one through the +# module's own names: a bare name defined here or imported from where it is defined, a function +# on an imported module, or a method of this module on self or cls or on an instance of a class +# defined here. A call on any other object is never a helper call, so a common name such as get +# or append can't spread through the tree. +_Helpers = dict[tuple[str, str, str], frozenset[int | str]] + + +def _helper(func: ast.AST, helpers: _Helpers, imports: _Module) -> frozenset[int | str] | None: + if isinstance(func, ast.Name): + if func.id in imports.defined: + return helpers.get(("function", imports.name, func.id)) + if func.id in imports.names: + return helpers.get(("function", *imports.names[func.id])) + return None + if isinstance(func, ast.Attribute) and _instance_of(func.value, imports.classes): + return helpers.get(("method", imports.name, func.attr)) + if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name): + if func.value.id in _SELF or func.value.id in imports.instances: + return helpers.get(("method", imports.name, func.attr)) + if func.value.id in imports.classes: + # A static or class method called on its class: Store.keep(...) (change review r3). + return helpers.get(("method", imports.name, func.attr)) + module = imports.modules.get(func.value.id) + if module is not None: + return helpers.get(("function", module, func.attr)) + return None + + +def _verb(call: ast.Call, helpers: _Helpers, imports: _Module) -> tuple[str, list[ast.AST]] | None: + """The write verb a call is, with the expressions it writes to.""" + func = call.func + dotted = _dotted(func, imports) + if dotted in _DOTTED_VERBS: + return dotted, list(call.args[:2]) + if dotted == "os.open": + return (dotted, list(call.args[:1])) if _flags_write(call) else None + if dotted in ("os.fdopen", "io.open", "codecs.open"): + return (dotted, list(call.args[:1])) if _mode_writes(_argument(call, 1, "mode")) else None + if dotted is not None: + return None + if isinstance(func, ast.Attribute): + if func.attr in ("write_text", "write_bytes"): + return func.attr, [func.value] + # Path.replace(target) and Path.rename(target) take one argument; str.replace two. + if func.attr in ("replace", "rename") and len(call.args) == 1 and not call.keywords: + return f"Path.{func.attr}", [func.value, call.args[0]] + if func.attr in ("hardlink_to", "symlink_to") and len(call.args) == 1: + return f"Path.{func.attr}", [func.value, call.args[0]] + if func.attr == "open": + # tarfile.open(name, mode) takes the mode second; Path.open(mode) first. + on_a_module = isinstance(func.value, ast.Name) and func.value.id in imports.modules + mode = _argument(call, 1 if on_a_module else 0, "mode") + return ("Path.open", [func.value]) if _mode_writes(mode) else None + if func.attr == "_write_whole": + return func.attr, list(call.args[:1]) + if func.attr in ("writestr", "addfile"): + # ZipFile.writestr(arcname, data) and TarFile.addfile(tarinfo, fileobj). + return func.attr, list(call.args[:1]) + slots = _helper(func, helpers, imports) + return (func.attr, _written(call, slots)) if slots is not None else None + if isinstance(func, ast.Name): + if func.id == "open": + return ("open", list(call.args[:1])) if _mode_writes(_argument(call, 1, "mode")) else None + if func.id == "_write_whole": + return func.id, list(call.args[:1]) + slots = _helper(func, helpers, imports) + return (func.id, _written(call, slots)) if slots is not None else None + return None + + +def _names_in(node: ast.AST) -> set[str]: + return {sub.id for sub in ast.walk(node) if isinstance(sub, ast.Name)} + + +def _bound(nodes: list[ast.AST], seeds: set[str], is_seed: Any = None) -> set[str]: + """The seeds, and every name bound here from an expression holding one, to a fixed point.""" + marked = set(seeds) + pairs: list[tuple[set[str], bool, set[str]]] = [] + for node in nodes: + if isinstance(node, ast.Assign): + value, targets = node.value, node.targets + elif isinstance(node, (ast.AnnAssign, ast.AugAssign, ast.NamedExpr)) and node.value: + value, targets = node.value, [node.target] + elif isinstance(node, ast.withitem) and node.optional_vars is not None: + value, targets = node.context_expr, [node.optional_vars] + elif isinstance(node, (ast.For, ast.AsyncFor)): + value, targets = node.iter, [node.target] + else: + continue + seeded = is_seed is not None and any(is_seed(sub) for sub in ast.walk(value)) + pairs.append((_names_in(value), seeded, set().union(*map(_names_in, targets)))) + changed = True + while changed: + changed = False + for names, seeded, bound in pairs: + if (seeded or names & marked) and bound - marked: + marked |= bound + changed = True + return marked + + +def _defs(tree: ast.Module) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, bool]]: + """Every def in a module, at any depth, and whether it is a method (a def nested in a method + is a function: it is called by its name).""" + found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, bool]] = [] + + def collect(node: ast.AST, in_class: bool) -> None: + for child in ast.iter_child_nodes(node): + if isinstance(child, _DEFS): + found.append((child, in_class)) + collect(child, False) + elif isinstance(child, ast.ClassDef): + collect(child, True) + else: + collect(child, in_class) + + collect(tree, False) + return found + + +def _parameters(node: ast.FunctionDef | ast.AsyncFunctionDef, method: bool) -> list[tuple[int | None, str]]: + """A def's parameters by (position, name); a method's self or cls is not one of them.""" + arguments = node.args + positional = [one.arg for one in (*arguments.posonlyargs, *arguments.args)] + static = any( + isinstance(decorator, ast.Name) and decorator.id == "staticmethod" + for decorator in node.decorator_list + ) + if method and not static and positional and positional[0] in _SELF: + positional = positional[1:] + named: list[tuple[int | None, str]] = list(enumerate(positional)) + named += [(None, one.arg) for one in arguments.kwonlyargs] + return named + + +def _writer_helpers(trees: dict[Path, ast.Module], imports: dict[Path, _Module]) -> _Helpers: + """Functions, methods and nested defs that write to one of their parameters, each with the + parameters it writes to (by position and by name). Found to a fixed point: a def that hands + a parameter to a known helper is a helper too, however deep. + + write_contract is the sanctioned writer, so a call to it is never a write verb here.""" + prepared = [] + for path, tree in trees.items(): + for node, method in _defs(tree): + if node.name == "write_contract": + continue + own = _own_nodes(node) + calls = [sub for sub in own if isinstance(sub, ast.Call)] + if not calls: + continue + derived = [ + (slot, name, _bound(own, {name})) for slot, name in _parameters(node, method) + ] + key = ("method" if method else "function", imports[path].name, node.name) + prepared.append((key, calls, derived, imports[path])) + helpers: dict[tuple[str, str, str], set[int | str]] = {} + changed = True + while changed: + changed = False + known = {key: frozenset(slots) for key, slots in helpers.items()} + for key, calls, derived, module_imports in prepared: + for call in calls: + found = _verb(call, known, module_imports) + if not found: + continue + written = set().union(*map(_names_in, found[1])) + for slot, parameter, names in derived: + if written & names: + fresh = {parameter} if slot is None else {parameter, slot} + slots = helpers.setdefault(key, set()) + if not fresh <= slots: + slots |= fresh + changed = True + return {key: frozenset(slots) for key, slots in helpers.items()} + + +def _self_attrs(node: ast.ClassDef, marked: set[str], is_marker: Any) -> frozenset[str]: + """The attributes of self or cls that hold the contract file in a class, set in any of its + methods or in its body, to a fixed point.""" + scopes = [_own_nodes(node)] + [_own_nodes(sub) for sub in ast.walk(node) if isinstance(sub, _DEFS)] + attrs: set[str] = set() + while True: + known = frozenset(attrs) + + def seed(sub: ast.AST, known: frozenset[str] = known) -> bool: + return is_marker(sub, known) + + for index, own in enumerate(scopes): + names = _bound(own, marked, seed) - _SELF + for sub in own: + if isinstance(sub, ast.Assign): + value, targets = sub.value, sub.targets + elif isinstance(sub, (ast.AnnAssign, ast.AugAssign)) and sub.value is not None: + value, targets = sub.value, [sub.target] + else: + continue + if not (_names_in(value) & names or any(seed(part) for part in ast.walk(value))): + continue + for target in targets: + for part in ast.walk(target): + if isinstance(part, ast.Attribute) and isinstance(part.value, ast.Name): + if part.value.id in _SELF: + attrs.add(part.attr) + elif isinstance(part, ast.Name) and index == 0: + # A class constant, read as self.NAME or cls.NAME. + attrs.add(part.id) + if attrs == known: + return known + + +def _names_the_file( + sub: ast.AST, + docs: set[int], + marker_names: frozenset[str], + imports: _Module, + exported: dict[str, frozenset[str]], +) -> bool: + """Whether a node names the contract file: the text, a marker name, or module.NAME for a + name another module binds to the file.""" + if isinstance(sub, ast.Constant): + return isinstance(sub.value, str) and bool(_MARKER_PATH.search(sub.value)) and id(sub) not in docs + if isinstance(sub, ast.Name): + return sub.id in marker_names + if isinstance(sub, ast.Attribute): + if sub.attr in MARKER_NAMES: + return True + if isinstance(sub.value, ast.Name) and sub.value.id in imports.modules: + return sub.attr in exported.get(imports.modules[sub.value.id], frozenset()) + return False + + +def _marker_names( + tree: ast.Module, imports: _Module, exported: dict[str, frozenset[str]] +) -> frozenset[str]: + """The names a module reads the contract file by: CONTRACT and CONTRACT_FILE under any name, + and any name imported from a module that binds it to the file.""" + imported = { + local + for local, (source, name) in imports.names.items() + if name in exported.get(source, frozenset()) + } + return MARKER_NAMES | _marker_aliases(tree) | imported + + +def _exported_markers( + trees: dict[Path, ast.Module], imports: dict[Path, _Module] +) -> dict[str, frozenset[str]]: + """Each module's module-level names bound to the contract file, or to an expression holding + it, found to a fixed point across modules: a name bound to one imported from elsewhere is + one too (change review r2, plant N1).""" + exported: dict[str, frozenset[str]] = {} + docs = {path: _docstrings(tree) for path, tree in trees.items()} + owns = {path: _own_nodes(tree) for path, tree in trees.items()} + while True: + found: dict[str, frozenset[str]] = {} + for path, tree in trees.items(): + module_imports = imports[path] + names = _marker_names(tree, module_imports, exported) + + def seed( + sub: ast.AST, + path: Path = path, + names: frozenset[str] = names, + module_imports: _Module = module_imports, + ) -> bool: + return _names_the_file(sub, docs[path], names, module_imports, exported) + + # A name bound to a marker by an import (from .contract import CONTRACT_FILE as + # CONTRACT_JSON) is one too, as W7 says (change review r3, B-M1). + found[module_imports.name] = frozenset( + _bound(owns[path], set(), seed) | (names - MARKER_NAMES) + ) + if found == exported: + return exported + exported = found + + +def scan_contract_writers(root: Path) -> dict[tuple[str, str, str], int]: + trees = {path: ast.parse(path.read_text(encoding="utf-8")) for path in sorted(root.rglob("*.py"))} + package = frozenset(_module_name(path, root) for path in trees) + imports = {path: _context(tree, path, root, package) for path, tree in trees.items()} + helpers = _writer_helpers(trees, imports) + exported = _exported_markers(trees, imports) + hits: dict[tuple[str, str, str], int] = {} + for path, tree in trees.items(): + docs = _docstrings(tree) + module = ".".join(path.relative_to(root.parent.parent).with_suffix("").parts) + module_imports = imports[path] + marker_names = _marker_names(tree, module_imports, exported) + + def names_it( + sub: ast.AST, + docs: set[int] = docs, + marker_names: frozenset[str] = marker_names, + module_imports: _Module = module_imports, + ) -> bool: + return _names_the_file(sub, docs, marker_names, module_imports, exported) + + # The constants each class of this module binds to the file in its body, read as + # Names.CONTRACT_JSON (change review r3). + constants = { + node.name: frozenset(_bound(_own_nodes(node), set(), names_it)) + for node in ast.walk(tree) + if isinstance(node, ast.ClassDef) + } + + def is_marker( + sub: ast.AST, + attrs: frozenset[str] = frozenset(), + names_it: Any = names_it, + constants: dict[str, frozenset[str]] = constants, + ) -> bool: + if isinstance(sub, ast.Attribute) and isinstance(sub.value, ast.Name): + if sub.value.id in _SELF and sub.attr in attrs: + return True + if sub.attr in constants.get(sub.value.id, frozenset()): + return True + return names_it(sub) + + def record( + qualified: str, + own: list[ast.AST], + names: set[str], + attrs: frozenset[str], + module: str = module, + module_imports: _Module = module_imports, + is_marker: Any = is_marker, + ) -> None: + mentions = any( + is_marker(sub, attrs) or (isinstance(sub, ast.Name) and sub.id in names) + for sub in own + ) + if not mentions: + return + verbs: Counter[str] = Counter() + for sub in own: + found = _verb(sub, helpers, module_imports) if isinstance(sub, ast.Call) else None + if found: + verbs[found[0]] += 1 + for verb, count in verbs.items(): + hits[(module, qualified, verb)] = count + + def visit( + node: ast.AST, + marked: set[str], + prefix: str, + attrs: frozenset[str], + is_marker: Any = is_marker, + record: Any = record, + ) -> None: + for child in ast.iter_child_nodes(node): + if isinstance(child, _DEFS): + own = _own_nodes(child) + names = _bound(own, marked, lambda sub, attrs=attrs: is_marker(sub, attrs)) + qualified = prefix + child.name + record(qualified, own, names, attrs) + visit(child, names, qualified + ".", attrs) + elif isinstance(child, ast.ClassDef): + held = _self_attrs(child, marked, is_marker) + own = _own_nodes(child) + names = _bound(own, marked, lambda sub, held=held: is_marker(sub, held)) + record(prefix + child.name + ".", own, names, held) + visit(child, marked, prefix + child.name + ".", held) + else: + visit(child, marked, prefix, attrs) + + own = _own_nodes(tree) + marked = _bound(own, set(), is_marker) + record("", own, marked, frozenset()) + visit(tree, marked, "", frozenset()) + return hits + + +_COPIES = "copies the contract.json write_contract wrote into a bundle, as written" +_READS = "reads contract.json; the write is another file" +_FAILURE = "writes the reading's failure file (failure_file.write_failure_file)" +_TEMP_COMPOSE = ( + "reaches runtime_selection.select_runtime, whose compose_merge.merge_compose writes compose " + "configs into a temporary directory" +) +_TRACE = "run.conversation.converse writes http-failure.json beside the run's trace" +# (module, function, verb): (count, reason). Taken from reviewer A's probe +# (plan-v5-r1/copy-A/probes-v5A/scan_v6_strict.py), run again on the fixed tree with W7's rule. +ALLOWED: dict[tuple[str, str, str], tuple[int, str]] = { + ("fi.alk.harness.contract", "write_contract", "write_text"): (1, "the writer itself"), + ("fi.alk.harness.contract", "write_contract", "os.replace"): (1, "the writer itself"), + # Readers: each names contract.json to read it, and writes something else. + ("fi.alk.harness.agent_reading", "_read", "document"): ( + 1, "compares the saved file's name; reading_result.document writes agent.json"), + ("fi.alk.harness.agent_reading", "_reading", "document"): ( + 2, "compares the saved file's name; reading_result.document writes agent.json"), + ("fi.alk.harness.reading_result", "classify", "document"): ( + 1, "compares the saved file's name; document writes agent.json"), + ("fi.alk.harness.reading_result", "document", "_write_whole"): ( + 1, "reads contract.json; writes agent.json through build_authored_agent's _redact"), + # Copies: each writes contract.json from bytes an earlier save already wrote. + ("fi.alk.harness.reading_result", "document", "replace_if_unchanged"): ( + 1, "rewrites the saved contract's own bytes with the reading's record (a technical key) " + "spliced in; the free text is the saved, cleaned text"), + ("fi.alk.harness.seed_contract", "use_seed", "_write_whole"): ( + 2, "installs the platform's seed, a reading's saved contract, as written; a reading from " + "before the cleaner stays as written (a named limit), and every send cleans again"), + ("fi.alk.harness.authoring_entrypoint", "_prepare_source_data_invariants", + "prune_invariant_scopes"): (1, _READS + " (the invariants file)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_and_repair", + "_write_generic_certificate"): (1, _READS + " (the certificate)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_and_repair", "write_text"): ( + 3, _READS + " (diagnostics and runtime-validation records)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_once", + "_freeze_certified_bundle"): (2, _READS + " (the frozen bundle)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_once", + "_probe_runtime_readiness"): (1, _READS + " (the readiness record)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_once", "author_invariants"): ( + 1, _READS + " (the invariants file)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_once", "probe_chat_ingress"): ( + 1, _READS + " (the ingress probe record)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_once", "probe_chat_workflow"): ( + 1, _READS + " (the workflow probe record)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_once", "validate_native"): ( + 1, _READS + " (the native validation record)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_once", "write_bytes"): ( + 1, _READS + " (a validation record)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_once", "write_text"): ( + 2, _READS + " (validation records)"), + ("fi.alk.harness.bundle_author_v2", "_reuse_certified_bundle", "Path.rename"): ( + 3, _READS + " (its digest); moves bundle folders"), + ("fi.alk.harness.bundle_author_v2", "_reuse_certified_bundle", "shutil.copy2"): ( + 1, _READS + " (its digest); copies another artifact"), + ("fi.alk.harness.bundle_author_v2", "_reuse_certified_bundle", "shutil.copytree"): ( + 1, _READS + " (its digest); copies a bundle folder"), + ("fi.alk.harness.cli", "_auto", "_write_adjustment_status"): (2, _READS + " (the status)"), + ("fi.alk.harness.cli", "_auto", "seal_artifacts"): (1, _READS + " (the seal)"), + ("fi.alk.harness.cli", "_auto", "write_text"): (1, _READS + " (job.json)"), + ("fi.alk.harness.compose_integration", "validate_native", "_freeze_certified_bundle"): ( + 1, _READS + " (the frozen bundle)"), + ("fi.alk.harness.compose_integration", "validate_native", "_probe_runtime_readiness"): ( + 1, _READS + " (the readiness record)"), + ("fi.alk.harness.compose_integration", "validate_native", "probe_chat_ingress"): ( + 1, _READS + " (the ingress probe record)"), + ("fi.alk.harness.compose_integration", "validate_native", "write_text"): ( + 1, _READS + " (the native validation record)"), + ("fi.alk.harness.sandbox_server", "LocalSandbox.rerun", "_write_json"): ( + 1, "checks contract.json is there; writes the job state"), + ("fi.alk.harness.source_data_invariants", "author_invariants", "write_text"): ( + 2, _READS + " (the invariants file)"), + # Copies: each carries the authoring contract.json, already cleaned, into a bundle. + ("fi.alk.harness.bundle", "export_session_bundle", "Path.replace"): (1, _COPIES), + ("fi.alk.harness.bundle", "export_session_bundle", "_copy_generated_runtime_context"): ( + 1, _COPIES), + ("fi.alk.harness.bundle", "export_session_bundle", "seal_bundle"): (1, _COPIES), + ("fi.alk.harness.bundle", "export_session_bundle", "shutil.copy2"): (2, _COPIES), + ("fi.alk.harness.bundle", "export_session_bundle", "write_environment_plan"): (1, _COPIES), + ("fi.alk.harness.bundle", "export_session_bundle", "write_text"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "_carry_contract", "_compile_source_tool_handlers"): ( + 1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "_carry_contract", "shutil.copy2"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "_carry_contract", "shutil.copytree"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "_carry_contract", "write_text"): ( + 1, _COPIES + ", with only runtime.interface (kept) set from the build"), + ("fi.alk.harness.bundle_author_v2", "_copy_chat_authoring", "shutil.copy2"): (2, _COPIES), + ("fi.alk.harness.bundle_author_v2", "_copy_chat_authoring", "shutil.copytree"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "Path.rename"): (3, _COPIES), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "_compile_source_tool_handlers"): ( + 1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "_copy_chat_authoring"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "_copy_scenarios"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "_copy_sub_goal_catalogue"): ( + 1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "_generic_postgres_seed_artifacts"): ( + 1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "_reuse_certified_bundle"): ( + 1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "compile_bundle"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "shutil.copy2"): (3, _COPIES), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "write_text"): ( + 5, _COPIES + " (:3752 writes the parsed authoring body back as written)"), + ("fi.alk.harness.bundle_author_v2", "reseal_scenarios", "Path.rename"): (2, _COPIES), + ("fi.alk.harness.bundle_author_v2", "reseal_scenarios", "_carry_contract"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "reseal_scenarios", "_copy_scenarios"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "reseal_scenarios", "_recertify_scenarios"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "reseal_scenarios", "shutil.copy2"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "reseal_scenarios", "shutil.copytree"): (1, _COPIES), + ("fi.alk.harness.bundle_author_v2", "reseal_scenarios", "write_text"): (1, _COPIES), + ("fi.alk.harness.compose_integration", "compile_bundle", "Path.rename"): (1, _COPIES), + ("fi.alk.harness.compose_integration", "compile_bundle", "_compile_source_tool_handlers"): ( + 1, _COPIES), + ("fi.alk.harness.compose_integration", "compile_bundle", "_copy_chat_authoring"): (1, _COPIES), + ("fi.alk.harness.compose_integration", "compile_bundle", "_copy_scenarios"): (1, _COPIES), + ("fi.alk.harness.compose_integration", "compile_bundle", "_copy_sub_goal_catalogue"): ( + 1, _COPIES), + ("fi.alk.harness.compose_integration", "compile_bundle", "write_text"): (4, _COPIES), + # Found by the widened scan (change review r1, A-M1): calls to a helper that writes to what + # it is given, found to a fixed point. None of these helpers names contract.json (one that + # did would be listed under its own name), so each reason says what the helper writes. + ("fi.alk.harness.agent_reading", "_read", "_reading"): (1, "calls _reading, listed below"), + ("fi.alk.harness.agent_reading", "_reading", "_timed_out"): (1, _FAILURE), + ("fi.alk.harness.agent_reading", "_reading", "classify"): ( + 1, _FAILURE + "; classify's document writes agent.json"), + ("fi.alk.harness.agent_reading", "_reading", "model_error"): (1, _FAILURE), + ("fi.alk.harness.agent_reading", "_reading", "refuse_unusable_folder"): (1, _FAILURE), + ("fi.alk.harness.agent_reading", "_reading", "write_reading_failure"): (1, _FAILURE), + ("fi.alk.harness.authoring_runtime_validation", "validate_and_repair", + "apply_runtime_plan_patch"): ( + 1, "the runtime-plan repair, which saves through write_contract (B13); " + _TEMP_COMPOSE), + ("fi.alk.harness.authoring_runtime_validation", "validate_and_repair", "repair"): ( + 3, "calls its nested repair, scanned as validate_and_repair.repair"), + ("fi.alk.harness.authoring_runtime_validation", "validate_and_repair", "validate"): ( + 2, "calls its nested validate, which runs validate_once (listed above)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_and_repair", "write_scenarios"): ( + 1, "writes the scenario folders (folder.write_folder and write_index)"), + ("fi.alk.harness.authoring_runtime_validation", "validate_and_repair.repair", "_build"): ( + 1, "the build stage (cli._build), which saves the contract through tools._save_contract " + "and amend, both write_contract (B13); it writes the world besides"), + ("fi.alk.harness.authoring_runtime_validation", "validate_and_repair.repair", "_scenarios"): ( + 1, "the scenario stage (cli._scenarios), which writes the scenario folders"), + ("fi.alk.harness.authoring_runtime_validation", "validate_and_repair.repair", + "request_runtime_plan_patch"): (1, "builds a runtime patch; " + _TEMP_COMPOSE), + ("fi.alk.harness.bundle_author_v2", "", "main"): ( + 1, "the __main__ guard; main runs author_bundle_v2 (listed) and refuse_unusable_folder " + "(the failure file)"), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "_adopted_seed_sql"): ( + 1, "reads the agent's schema files; " + _TEMP_COMPOSE), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "_package_process_sources"): ( + 1, "copies the agent's own source components into the bundle " + "(generated_runtime._copy_sanitized_context)"), + ("fi.alk.harness.bundle_author_v2", "author_bundle_v2", "resolve_environment_plan"): ( + 1, _TEMP_COMPOSE), + ("fi.alk.harness.bundle_author_v2", "reseal_scenarios", "_reseeded"): ( + 1, "_carried_world writes the world seed (world.sqlite, world-ir.json)"), + ("fi.alk.harness.chat_call_runner", "HostedChatCallRunner.run", "_drive_conversation"): ( + 1, _TRACE), + ("fi.alk.harness.cli", "_auto", "save_session"): ( + 1, "sessions.save writes the session's own record (META)"), + ("fi.alk.harness.compose_integration", "compile_bundle", "select_runtime"): ( + 1, _TEMP_COMPOSE), + ("fi.alk.harness.reading_result", "classify", "model_error"): (2, _FAILURE), + ("fi.alk.harness.reading_result", "classify", "write_reading_failure"): (1, _FAILURE), + ("fi.alk.harness.reading_result", "document", "write_reading_failure"): (2, _FAILURE), + ("fi.alk.harness.retell_chat_call_runner", "RetellChatCallRunner.run", + "_drive_conversation"): (1, _TRACE), + ("fi.alk.harness.sandbox_server", "LocalSandbox.rerun", "_claim_secret_files"): ( + 1, "moves the job's pending secret files into the job's folder"), + ("fi.alk.harness.sandbox_server", "LocalSandbox.rerun", "_execute_rerun"): ( + 1, "writes the job state and worker.log, and copies the environment bundle as written"), + ("fi.alk.harness.tools", "contract_tools.submit_contract", "write_failure_file"): ( + 1, "writes the stage's failure file, not the contract"), +} + + +def test_no_contract_write_outside_write_contract_and_the_listed_readers_and_copies() -> None: + found = scan_contract_writers(SOURCE_TREE) + assert found == {key: count for key, (count, _) in ALLOWED.items()} + assert all(reason for _, reason in ALLOWED.values()) + + +def test_the_scan_flags_each_writer_shape_the_plan_names(tmp_path: Path) -> None: + """P12b and P12c as fixtures, plus the W7 verbs, so the scan is seen to catch each.""" + package = tmp_path / "fi" / "alk" + package.mkdir(parents=True) + (package / "shapes.py").write_text( + ''' +import json, os, shutil +CONTRACT_FILE = "contract.json" + + +def _write_json(path, payload): + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(payload)) + temporary.replace(path) + + +def like_save_contract(contract, destination): + path = destination / "contract.json" + partial = destination / ".contract.json.partial" + partial.write_text(contract.model_dump_json()) + os.replace(partial, path) + + +def through_a_helper(run_root, body): + _write_json(run_root / "contract.json", body) + + +def opened(destination, body): + with (destination / CONTRACT_FILE).open("w") as stream: + json.dump(body, stream) + + +def moved(staging, destination): + shutil.move(staging / "contract.json", destination) + + +def renamed(staging, destination): + contract = staging / "contract.json" + contract.rename(destination) + + +def copied(staging, destination): + shutil.copytree(staging, destination / "bundle") + return staging / CONTRACT_FILE + + +def a_reader(destination): + """Writes nothing, though its docstring says contract.json and it calls replace.""" + return (destination / "contract.json").read_text().replace("a", "b") +''', + encoding="utf-8", + ) + + found = scan_contract_writers(package) + + module = "fi.alk.shapes" + assert found == { + (module, "like_save_contract", "write_text"): 1, + (module, "like_save_contract", "os.replace"): 1, + (module, "through_a_helper", "_write_json"): 1, + (module, "opened", "Path.open"): 1, + (module, "opened", "json.dump"): 1, + (module, "moved", "shutil.move"): 1, + (module, "renamed", "Path.rename"): 1, + (module, "copied", "shutil.copytree"): 1, + } + + +def test_the_scan_flags_each_shape_the_change_review_found_it_missed(tmp_path: Path) -> None: + """Change review r1, A-M1: the nine writer shapes the one-level scan missed, one each.""" + package = tmp_path / "fi" / "alk" + package.mkdir(parents=True) + (package / "missed.py").write_text( + ''' +import os +from shutil import copyfile +from .contract import CONTRACT_FILE as _NAME + + +class Held: + """1. The path is held on self, and written in another method.""" + + def __init__(self, root): + self.path = root / "contract.json" + + def save(self, body): + self.path.write_text(body) + + +def _write_json(path, body): + path.write_text(body) + + +def _save_state_file(path, body): + _write_json(path, body) + + +def two_levels(run_root, body): + """2. A helper two levels deep.""" + _save_state_file(run_root / "contract.json", body) + + +def in_a_lambda(destination, body): + """3. The write is inside a lambda.""" + target = destination / "contract.json" + put = lambda: target.write_text(body) + put() + + +def aliased(destination, body): + """4. CONTRACT_FILE imported under another name.""" + (destination / _NAME).write_text(body) + + +def imported_copy(staging, destination): + """5. copyfile imported from shutil.""" + copyfile(staging / "contract.json", destination) + + +def one_constant(root, body): + """6. The file named at the end of a path.""" + (root / "authoring/contract.json").write_text(body) + + +def mode_in_a_variable(destination, body, mode): + """7. The open mode is held in a variable.""" + with open(destination / "contract.json", mode) as stream: + stream.write(body) + + +class Store: + def _write(self, path, body): + path.write_text(body) + + def keep(self, root, body): + """8. A method helper on self.""" + self._write(root / "contract.json", body) + + +def linked(destination, body): + """9. os.open and os.fdopen, then os.link into place.""" + partial = destination / ".partial" + descriptor = os.open(partial, os.O_WRONLY | os.O_CREAT) + with os.fdopen(descriptor, "w") as stream: + stream.write(body) + os.link(partial, destination / "contract.json") + + +def read_only(destination): + """Not a writer: it reads the file, through os.open and an open with a read mode.""" + descriptor = os.open(destination / "contract.json", os.O_RDONLY) + os.close(descriptor) + with open(destination / "contract.json", "r") as stream: + return stream.read() +''', + encoding="utf-8", + ) + + found = scan_contract_writers(package) + + module = "fi.alk.missed" + assert found == { + (module, "Held.save", "write_text"): 1, + (module, "two_levels", "_save_state_file"): 1, + (module, "in_a_lambda", "write_text"): 1, + (module, "aliased", "write_text"): 1, + (module, "imported_copy", "shutil.copyfile"): 1, + (module, "one_constant", "write_text"): 1, + (module, "mode_in_a_variable", "open"): 1, + (module, "Store.keep", "_write"): 1, + (module, "linked", "os.open"): 1, + (module, "linked", "os.fdopen"): 1, + (module, "linked", "os.link"): 1, + } + + +def test_the_scan_flags_each_shape_the_second_change_review_found_it_missed(tmp_path: Path) -> None: + """Change review r2, A-M1: plants N1 and N2 as fixtures, and the archive writes.""" + package = tmp_path / "fi" / "alk" + package.mkdir(parents=True) + (package / "names.py").write_text( + 'CONTRACT_NAME = "contract.json"\nKEPT_AS = CONTRACT_NAME\n', encoding="utf-8" + ) + (package / "user.py").write_text( + ''' +import tarfile +import zipfile +from . import names +from .names import CONTRACT_NAME as _OTHER, KEPT_AS + + +def imported_under_another_name(run_root, body): + """N1. A module constant bound to the file, imported under another name.""" + (run_root / _OTHER).write_text(body) + + +def through_a_second_binding(run_root, body): + (run_root / KEPT_AS).write_text(body) + + +def read_as_module_name(run_root, body): + (run_root / names.CONTRACT_NAME).write_text(body) + + +class _RawKeeper: + def save(self, path, body): + path.write_text(body) + + +def on_an_instance(run_root, body): + """N2. A helper method called on an instance of a class defined here.""" + _RawKeeper().save(run_root / "contract.json", body) + + +def on_a_bound_instance(run_root, body): + keeper = _RawKeeper() + keeper.save(run_root / "contract.json", body) + + +def zipped(bundle, body): + with zipfile.ZipFile(bundle, "w") as archive: + archive.writestr("contract.json", body) + + +def tarred(bundle, stream): + with tarfile.open(bundle, "w") as archive: + archive.addfile(tarfile.TarInfo("contract.json"), stream) +''', + encoding="utf-8", + ) + + found = scan_contract_writers(package) + + module = "fi.alk.user" + assert found == { + (module, "imported_under_another_name", "write_text"): 1, + (module, "through_a_second_binding", "write_text"): 1, + (module, "read_as_module_name", "write_text"): 1, + (module, "on_an_instance", "save"): 1, + (module, "on_a_bound_instance", "save"): 1, + (module, "zipped", "writestr"): 1, + (module, "tarred", "Path.open"): 1, + (module, "tarred", "addfile"): 1, + } + + + +# Reviewer B's three real-tree plants of change review r3, as B wrote them, each a new module. +_R3_PLANTS = { + "r3a_constructor": ( + '"""planted"""\nimport json\n\n\nclass JsonFile:\n def __init__(self, path):\n' + " self.path = path\n\n def save(self, body):\n" + " self.path.write_text(json.dumps(body))\n\n\n" + "def keep_raw_contract(run_root, body):\n" + ' JsonFile(run_root / "contract.json").save(body)\n' + ), + "r3b_constant": ( + '"""planted"""\nimport json\n\n\nclass Names:\n CONTRACT_JSON = "contract.json"\n\n\n' + "def keep_raw_contract(run_root, body):\n" + " (run_root / Names.CONTRACT_JSON).write_text(json.dumps(body))\n" + ), + "r3c_static": ( + '"""planted"""\nimport json\n\n\nclass Store:\n @staticmethod\n' + " def keep(path, body):\n path.write_text(json.dumps(body))\n\n\n" + "def keep_raw_contract(run_root, body):\n" + ' Store.keep(run_root / "contract.json", body)\n' + ), +} + + +def test_the_scan_flags_each_shape_the_third_change_review_found_it_missed(tmp_path: Path) -> None: + """Change review r3, B-M1: B's plants R3b and R3c, a class method on its class, and a marker + re-exported through a module by ``import ... as`` (W7's "a name bound to one of them"). + + R3a, a path handed to a constructor and written by a method, is a named limit (the comment + above MARKER_NAMES): the exact result pins that it is still missed.""" + package = tmp_path / "fi" / "alk" + package.mkdir(parents=True) + for name, code in _R3_PLANTS.items(): + (package / f"{name}.py").write_text(code, encoding="utf-8") + (package / "contract.py").write_text('CONTRACT_FILE = "contract.json"\n', encoding="utf-8") + (package / "relay.py").write_text( + "from .contract import CONTRACT_FILE as CONTRACT_JSON\n", encoding="utf-8" + ) + (package / "user.py").write_text( + ''' +from .relay import CONTRACT_JSON + + +def through_a_re_export(out, body): + (out / CONTRACT_JSON).write_text(body) + + +class Store: + @classmethod + def keep(cls, path, body): + path.write_text(body) + + +def through_a_class_method(out, body): + Store.keep(out / "contract.json", body) +''', + encoding="utf-8", + ) + + found = scan_contract_writers(package) + + assert found == { + ("fi.alk.r3b_constant", "keep_raw_contract", "write_text"): 1, + ("fi.alk.r3c_static", "keep_raw_contract", "keep"): 1, + ("fi.alk.user", "through_a_re_export", "write_text"): 1, + ("fi.alk.user", "through_a_class_method", "keep"): 1, + } + +# -- B14, W6: the rule checks compare cleaned text --------------------------------------------------- + +RULE = f"Always post to {URL}" + + +def test_add_rule_refuses_a_rule_whose_cleaned_form_is_saved(tmp_path: Path) -> None: + held = _amendable() + assert amend.add_rule(held, tmp_path, rule=RULE, why="asked")[0] + + added, said = amend.add_rule(held, tmp_path, rule=RULE, why="asked again") + + assert not added and "already has that rule" in said + assert held.hard_constraints.count(f"Always post to {CLEAN}") == 1 + + +def test_drop_rule_finds_a_rule_given_as_the_operator_wrote_it(tmp_path: Path) -> None: + held = _amendable() + amend.add_rule(held, tmp_path, rule=RULE, why="asked") + + dropped, said = amend.drop_rule(held, tmp_path, rule=RULE, why="not the agent's") + + assert dropped, said + assert f"Always post to {CLEAN}" not in held.hard_constraints + + +def _held_raw() -> AgentContract: + """A contract held with its rule raw, as one saved before this fix is loaded.""" + held = _amendable() + held.hard_constraints.append(RULE) + return held + + +@pytest.mark.parametrize("given", [RULE, f"Always post to {CLEAN}"], ids=["raw", "cleaned"]) +def test_add_rule_refuses_a_rule_a_raw_held_contract_already_has( + tmp_path: Path, given: str +) -> None: + """Change review r1, A-L1.""" + held = _held_raw() + + added, said = amend.add_rule(held, tmp_path, rule=given, why="asked again") + + assert not added and "already has that rule" in said + assert held.hard_constraints == ["Confirm before booking", RULE] + assert not (tmp_path / "contract.json").exists() + + +@pytest.mark.parametrize( + "given", [RULE, f"Always post to {CLEAN}", "post to https://hooks.example"], + ids=["raw", "cleaned", "part"], +) +def test_drop_rule_finds_a_rule_a_raw_held_contract_holds(tmp_path: Path, given: str) -> None: + """Change review r1, A-L1: the raw rule is the one removed, and the save holds no password.""" + held = _held_raw() + + dropped, said = amend.drop_rule(held, tmp_path, rule=given, why="not the agent's") + + assert dropped, said + assert held.hard_constraints == ["Confirm before booking"] + saved = (tmp_path / "contract.json").read_text(encoding="utf-8") + assert PASSWORD not in saved and KEY not in saved + assert json.loads(saved)["hard_constraints"] == ["Confirm before booking"] + + +def test_add_rule_names_the_rule_it_refuses_as_saved(tmp_path: Path) -> None: + """Change review r2, A-H1: the hosted chat sends this reply on to the platform.""" + held = _held_raw() + + added, said = amend.add_rule(held, tmp_path, rule=RULE, why="asked again") + + assert not added + assert said == f"the agent already has that rule: Always post to {CLEAN}" + + +def test_drop_rule_lists_the_rules_it_has_as_saved(tmp_path: Path) -> None: + """Change review r2, A-H1: the reply lists the held rules cleaned, in their own case.""" + held = _held_raw() + + dropped, said = amend.drop_rule(held, tmp_path, rule="never rings", why="not the agent's") + + assert not dropped + assert said == ( + "no rule like that. It has:\n - Confirm before booking\n" + f" - Always post to {CLEAN}" + ) + assert held.hard_constraints == ["Confirm before booking", RULE] + + +# -- B15, W6: the caller's own object is cleaned ----------------------------------------------------- + + +def test_add_rule_cleans_the_contract_the_stage_holds(tmp_path: Path) -> None: + held = _amendable() + rules = held.hard_constraints + + assert amend.add_rule(held, tmp_path, rule=RULE, why=f"see {URL}")[0] + + assert held.hard_constraints is rules + assert _secret_free(held.model_dump()) and _secret_free(held.brief()) + assert f"Always post to {CLEAN}" in held.brief() + + +def test_fix_tool_cleans_the_contract_the_stage_holds(tmp_path: Path) -> None: + held = _amendable() + tool = held.tools[0] + + assert amend.fix_tool(held, tmp_path, tool_name="post", why="w", description=PROSE)[0] + + assert held.tools[0] is tool and tool.description == CLEANED + assert _secret_free(held.model_dump()) and _secret_free(held.brief()) + + +def test_declare_tool_cleans_the_contract_the_builder_holds(tmp_path: Path) -> None: + held = _amendable() + + result = _declare_tool(tmp_path, held, description=PROSE) + + assert not result.get("is_error"), result + assert held.tools[-1].description == CLEANED + assert _secret_free(held.model_dump()) and _secret_free(held.brief()) + saved = (tmp_path / "out" / "contract.json").read_text(encoding="utf-8") + assert PASSWORD not in saved and KEY not in saved + + +# -- W7's canary: the amend chain leaves no password on disk ----------------------------------------- + + +def test_no_file_the_amend_chain_writes_holds_the_password(tmp_path: Path) -> None: + output = tmp_path / "authoring" + accepted = tools.accept_contract( + { + "agent": "desk", + "modality": "chat", + "one_liner": f"Books desks via {URL}", + "system_prompt_excerpt": PROSE, + "hard_constraints": [RULE], + "real_use_cases": ["book a desk"], + "notes": PROSE, + "tools": [ + {"name": "post", "args": ["kind"], "arg_values": {"kind": ["a"]}, + "description": PROSE}, + ], + }, + output, + ) + assert not accepted.get("is_error"), accepted + held = AgentContract.model_validate_json((output / "contract.json").read_text("utf-8")) + amend.add_rule(held, output, rule=f"Never call {URL} twice", why=f"from {URL}") + amend.fix_tool(held, output, tool_name="post", why=URL, refuses=PROSE) + amend.widen(held, output, tool_name="post", argument="kind", values=["b"], why=URL) + amend.set_modality(held, output, modality="voice", why=URL) + _declare_tool(tmp_path, held, description=PROSE) + + written = [path for path in tmp_path.rglob("*") if path.is_file() and path.name != "agent.py"] + assert {path.name for path in written} == {"contract.json"} + for path in written: + text = path.read_text(encoding="utf-8") + assert PASSWORD not in text and KEY not in text, path diff --git a/tests/harness/test_failure_file.py b/tests/harness/test_failure_file.py new file mode 100644 index 000000000..882e1ccbc --- /dev/null +++ b/tests/harness/test_failure_file.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import json +import stat +from pathlib import Path + +from fi.alk.harness.failure_file import FAILURE_FILE, write_failure_file +from fi.alk.harness.job import FailureDomain, HarnessFailure, HarnessStage + + +def _failure(code: str = "subfolder_not_found") -> HarnessFailure: + return HarnessFailure( + domain=FailureDomain.AGENT, + stage=HarnessStage.ACQUIRING_SOURCE, + code=code, + message="The folder is not there.", + retryable=False, + details={"subfolder": "agents/x"}, + ) + + +def test_writer_creates_folder(tmp_path: Path) -> None: + output = tmp_path / "work" / "authoring" + path = write_failure_file(_failure(), output) + assert path == output / FAILURE_FILE + assert path.is_file() + + +def test_writer_writes_full_shape(tmp_path: Path) -> None: + path = write_failure_file(_failure(), tmp_path) + assert json.loads(path.read_text()) == { + "domain": "agent", + "stage": "acquiring_source", + "code": "subfolder_not_found", + "message": "The folder is not there.", + "retryable": False, + "details": {"subfolder": "agents/x"}, + } + assert not list(tmp_path.glob(".failure-*")) + + +def test_first_failure_stands(tmp_path: Path) -> None: + write_failure_file(_failure("first"), tmp_path) + write_failure_file(_failure("second"), tmp_path) + assert json.loads((tmp_path / FAILURE_FILE).read_text())["code"] == "first" + assert not list(tmp_path.glob(".failure-*")) + + +def test_file_is_world_readable(tmp_path: Path) -> None: + path = write_failure_file(_failure(), tmp_path) + assert stat.S_IMODE(path.stat().st_mode) == 0o644 diff --git a/tests/harness/test_framework_detection.py b/tests/harness/test_framework_detection.py new file mode 100644 index 000000000..c834526c3 --- /dev/null +++ b/tests/harness/test_framework_detection.py @@ -0,0 +1,1868 @@ +from __future__ import annotations + +import io +import json +import logging +import os +import shutil +import stat +import sys +import threading +import tokenize +import tracemalloc +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from fi.alk.harness import framework_detection as fd +from fi.alk.harness.framework_detection import ( + Finding, + FrameworkFolders, + detect_framework, + folder_findings, + framework_folders_for, +) +from fi.alk.harness.job import RepositorySource, SourceKind, SourceVisibility +from fi.alk.harness.source_folder import SubfolderNotFound + +EXAMPLES = Path(__file__).resolve().parents[2] / "examples" / "frameworks" +CREW = "from crewai import Crew\nCrew()\n" +PLAIN = "print('hello')\n" +LONG_LINE = "x = '" + "a" * 70_000 + "'\n" +ROT13 = "# coding: rot13\n" +UNKNOWN = {"name": "unknown", "version": None} +NONE = {"name": "none", "version": None} + + +def _write(folder: Path, files: dict[str, str]) -> None: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _named(name: str, label: str | None = None) -> dict[str, Any]: + result: dict[str, Any] = {"name": name, "version": None} + if label is not None: + result["label"] = label + return result + + +def _dist(package_dir: Path, dist: str, metadata: bytes | None = None) -> Path: + folder = ( + package_dir + / ".venv" + / "lib" + / "python3.12" + / "site-packages" + / f"{dist}.dist-info" + ) + folder.mkdir(parents=True) + if metadata is not None: + (folder / "METADATA").write_bytes(metadata) + return folder + + +def _metadata(version: str) -> bytes: + return f"Metadata-Version: 2.1\nName: x\nVersion: {version}\n".encode() + + +@pytest.mark.parametrize( + ("example", "expected"), + [ + ("langchain_agent.py", _named("langchain")), + ("langgraph_agent.py", _named("langgraph")), + ("crewai_agent.py", _named("crewai")), + ("google_adk_agent.py", _named("google_adk")), + ("claude_agent_sdk_agent.py", _named("claude_agent_sdk")), + ("openai_agents_agent.py", _named("openai_agents")), + ("pydantic_ai_agent.py", _named("pydantic_ai")), + ("llamaindex_agent.py", _named("other", "llama_index")), + ("autogen_agent.py", _named("other", "autogen")), + ("semantic_kernel_agent.py", _named("other", "semantic_kernel")), + ("haystack_agent.py", _named("other", "haystack")), + ("agno_agent.py", _named("other", "agno")), + ("beeai_agent.py", _named("other", "beeai_framework")), + ("dspy_agent.py", _named("other", "dspy")), + ("smolagents_agent.py", _named("other", "smolagents")), + ("strands_agent.py", _named("other", "strands")), + ("acme_thirdparty_agent.py", NONE), + ], + ids=lambda value: value if isinstance(value, str) else "", +) +def test_examples_report_their_framework( + tmp_path: Path, example: str, expected: dict[str, Any] +) -> None: + shutil.copy(EXAMPLES / example, tmp_path / example) + + assert detect_framework(tmp_path) == expected + + +@pytest.mark.parametrize( + ("source", "name"), + [ + ( + "from livekit.agents import AgentSession, cli\n" + "cli.run_app(AgentSession())\n", + "livekit", + ), + ( + "from pipecat.pipeline.pipeline import Pipeline\nPipeline([])\n", + "pipecat", + ), + ( + "from deepagents import create_deep_agent\ncreate_deep_agent()\n", + "deepagents", + ), + ( + "from langgraph_supervisor import create_supervisor\ncreate_supervisor()\n", + "langgraph", + ), + ("from claude_code_sdk import query\nquery()\n", "claude_agent_sdk"), + ], + ids=["livekit", "pipecat", "deepagents", "langgraph_star", "claude_code_sdk"], +) +def test_every_framework_is_detected(tmp_path: Path, source: str, name: str) -> None: + _write(tmp_path, {"agent.py": source}) + + assert detect_framework(tmp_path) == _named(name) + + +LIVEKIT = "from livekit.agents import AgentSession\nAgentSession()\n" +PIPECAT = "from pipecat.pipeline.pipeline import Pipeline\nPipeline([])\n" +LANGGRAPH = "from langgraph.graph import StateGraph\nStateGraph(int)\n" +DEEP = "from deepagents import create_deep_agent\ncreate_deep_agent()\n" +LANGCHAIN = "from langchain.agents import create_agent\ncreate_agent()\n" +CHAT_OPENAI = "from langchain_openai import ChatOpenAI\nChatOpenAI()\n" +OPENAI_AGENT = "from agents import Agent\nAgent()\n" +AUTOGEN = "from autogen_agentchat.agents import AssistantAgent\nAssistantAgent()\n" +DSPY = "import dspy\ndspy.Predict('q')\n" +CREW_AGENT = "from crewai import Agent\n" +PYDANTIC_AGENT = "from pydantic_ai import Agent\n" + + +@pytest.mark.parametrize( + ("sources", "expected"), + [ + ((CREW, CHAT_OPENAI), _named("crewai")), + ((LIVEKIT, LANGGRAPH), _named("livekit")), + ((LIVEKIT, DEEP), _named("livekit")), + ((LIVEKIT, LANGCHAIN), _named("livekit")), + ((PIPECAT, LANGGRAPH), _named("pipecat")), + ((LANGGRAPH, LANGCHAIN), _named("langgraph")), + ((DEEP, LANGCHAIN), _named("deepagents")), + ((LIVEKIT, PIPECAT), UNKNOWN), + ((CREW, OPENAI_AGENT), UNKNOWN), + ((DEEP, LANGGRAPH), _named("deepagents")), + ((LANGGRAPH, CHAT_OPENAI), _named("langgraph")), + ((AUTOGEN, CHAT_OPENAI), _named("other", "autogen")), + ((AUTOGEN, DSPY), UNKNOWN), + (("from langchain_core.messages import HumanMessage\n",), UNKNOWN), + ( + ( + "from openai import OpenAI\nfrom anthropic import Anthropic\n" + "OpenAI()\nAnthropic()\n", + ), + NONE, + ), + ( + ( + "from langchain.schema import ChatMessage\n" + "from langchain.prompts import ChatPromptTemplate\n" + "from openai import OpenAI\n" + "ChatMessage()\nChatPromptTemplate()\nOpenAI()\n", + ), + UNKNOWN, + ), + ( + ("from langchain_openai import ChatOpenAI\nChatOpenAI().bind_tools([])\n",), + _named("langchain"), + ), + (("from crewai import *\nCrew()\n",), UNKNOWN), + ( + ("from importlib import import_module\nimport_module('crewai')\n",), + NONE, + ), + ((CREW_AGENT + PYDANTIC_AGENT + "Agent()\n",), UNKNOWN), + ((PYDANTIC_AGENT + CREW_AGENT + "Agent()\n",), UNKNOWN), + ], + ids=[ + "crew_and_model", + "livekit_and_langgraph", + "livekit_and_deepagents", + "livekit_and_langchain", + "pipecat_and_langgraph", + "langgraph_and_langchain", + "deepagents_and_langchain", + "livekit_and_pipecat", + "crew_and_openai_agents", + "deepagents_and_langgraph", + "langgraph_and_model", + "other_and_model", + "two_others", + "unused_import", + "direct_model_calls", + "chat_message_and_prompt", + "model_alone", + "star_import", + "dynamic_import", + "one_name_two_frameworks", + "one_name_two_frameworks_reversed", + ], +) +def test_decision_table( + tmp_path: Path, sources: tuple[str, ...], expected: dict[str, Any] +) -> None: + _write(tmp_path, {f"m{index}.py": s for index, s in enumerate(sources)}) + + assert detect_framework(tmp_path) == expected + + +CREW_IMPORT = ("crewai", "import") +CREW_BUILDS = ("crewai", "builds") +TYPE_ONLY_IMPORT = "from typing import TYPE_CHECKING\n{} from crewai import Crew\n" +LANGCHAIN_IMPORT = ("langchain", "import") +TYPE_CHECKING_LINE = "from typing import TYPE_CHECKING\n" + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ( + "from langgraph.graph import StateGraph as SG\nSG(int)\n", + {("langgraph", "import"), ("langgraph", "builds")}, + ), + ( + 'from pydantic_ai import Agent\nAgent[int]("m")\n', + {("pydantic_ai", "import"), ("pydantic_ai", "builds")}, + ), + ( + "def f():\n from crewai import Crew\n Crew()\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ( + "try: import crewai\nexcept ImportError: crewai = None\ncrewai.Crew()\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ( + "from crewai import Crew\nclass C(Crew):\n pass\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ( + "from crewai import CrewBase\n@CrewBase\nclass C:\n pass\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ( + "from langgraph.func import entrypoint\n" + "@entrypoint()\ndef f(x):\n return x\n", + {("langgraph", "import"), ("langgraph", "builds")}, + ), + ( + TYPE_CHECKING_LINE + "if TYPE_CHECKING:\n from crewai import Crew\n", + set(), + ), + ( + "import typing\nif typing.TYPE_CHECKING:\n from crewai import Crew\n", + set(), + ), + ( + TYPE_CHECKING_LINE + + "if TYPE_CHECKING:\n import os\nelse:\n from crewai import Crew\n" + + "Crew()\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ("from .crewai import Crew\nCrew()\n", set()), + ("from . import crewai\ncrewai.Crew()\n", set()), + ("from ... import crewai\ncrewai.Crew()\n", set()), + ("from ...crewai import Crew\nCrew()\n", set()), + ( + "from crewai import (\n Agent,\n Crew as C,\n)\nC()\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ("import os, crewai\ncrewai.Crew()\n", {CREW_IMPORT, CREW_BUILDS}), + ("import os; import crewai\ncrewai.Crew()\n", {CREW_IMPORT, CREW_BUILDS}), + ( + "import google.adk\ngoogle.adk.agents.Agent()\n", + {("google_adk", "import"), ("google_adk", "builds")}, + ), + ("from crewai import Crew\ndef Crew(x):\n return x\n", {CREW_IMPORT}), + ("from crewai import Crew\nclass Crew:\n pass\n", {CREW_IMPORT}), + ('from crewai import Crew\n# Crew()\nx = "Crew()"\n', {CREW_IMPORT}), + ("from crewai import Crew\nf(Crew=1)\n", {CREW_IMPORT}), + ("from crewai import Crew\nx = a @ Crew\n", {CREW_IMPORT}), + ( + "from crewai import Crew\nclass C(Base, metaclass=Crew):\n pass\n", + {CREW_IMPORT}, + ), + ("import dspy\nclass Q(dspy.Module):\n pass\n", {("dspy", "import")}), + ( + "from langchain_openai import AzureChatOpenAI\nAzureChatOpenAI()\n", + {("langchain", "import"), ("langchain", "model")}, + ), + ( + 'from langchain.chat_models import init_chat_model\ninit_chat_model("m")\n', + {("langchain", "import"), ("langchain", "model")}, + ), + ('from crewai import Crew\nx = f"{Crew()}"\n', {CREW_IMPORT}), + ( + "print(f'{x}')\nfrom crewai import Crew\nCrew()\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ("from crewai import Crew\nself.Crew()\n", {CREW_IMPORT}), + ( + "import google.adk.agents as g\ng.LlmAgent()\n", + {("google_adk", "import"), ("google_adk", "builds")}, + ), + ("from langchain_core.x import ChatFoo\nChatFoo()\n", {LANGCHAIN_IMPORT}), + ( + "from langchain_core.outputs import ChatGeneration\n" + "from langchain.x import ChatGeneration as C\nC()\n", + {LANGCHAIN_IMPORT}, + ), + ("from langchain.x import ChatResult\nChatResult()\n", {LANGCHAIN_IMPORT}), + ("def f():\n yield from crewai\n", set()), + ("raise E from crewai\n", set()), + ( + "from crewai import Crew\nclass C[T](Crew):\n pass\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ( + "from crewai import Crew\nclass C[T: (A, B)](Crew):\n pass\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ("from crewai import Crew\nclass C: x = (Crew)\n", {CREW_IMPORT}), + ( + TYPE_ONLY_IMPORT.format("if TYPE_CHECKING: # pragma: no cover\n"), + set(), + ), + ( + TYPE_ONLY_IMPORT.format( + "try:\n import os\nexcept E:\n pass\nif TYPE_CHECKING:\n" + ), + set(), + ), + ( + TYPE_ONLY_IMPORT.format( + "if TYPE_CHECKING:\n if x:\n import os\n" + ), + set(), + ), + ( + "from crewai import Crew\nclass C(\n Crew,\n):\n pass\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ("from crewai import Crew\nclass C(A): x = (Crew)\n", {CREW_IMPORT}), + ("from crewai import Crew\nclass C(make(Crew)):\n pass\n", {CREW_IMPORT}), + ("from crewai import Crew\nx = f\"{f'{a}'} {Crew()}\"\n", {CREW_IMPORT}), + ], + ids=[ + "alias", + "subscript_then_call", + "import_inside_function", + "import_after_a_block_colon", + "class_base", + "decorator", + "decorator_with_call", + "type_checking_block", + "dotted_type_checking_block", + "type_checking_else", + "relative_module", + "relative_from_dot", + "relative_from_ellipsis", + "relative_ellipsis_module", + "parenthesised", + "comma_import", + "semicolon_import", + "dotted_use", + "def_name", + "class_name", + "comment_and_string", + "keyword_name", + "matrix_multiply", + "metaclass_value", + "other_row_base_alone", + "azure_model", + "init_chat_model", + "inside_fstring", + "after_fstring", + "method_of_another_object", + "dotted_alias", + "langchain_core_model_excluded", + "chat_generation_excluded", + "chat_result_excluded", + "yield_from", + "raise_from", + "generic_class_base", + "generic_class_constraint_brackets", + "class_body_on_one_line", + "type_checking_with_a_comment", + "type_checking_after_a_block", + "type_checking_with_a_nested_block", + "base_on_its_own_line", + "bracket_in_a_one_line_class_body", + "call_inside_the_bases", + "after_nested_fstring", + ], +) +def test_findings_rules( + tmp_path: Path, source: str, expected: set[tuple[str, str]] +) -> None: + _write(tmp_path, {"agent.py": source}) + + found = folder_findings(tmp_path) + + assert {(f.framework, f.strength) for f in found.findings} == expected + + +def test_one_finding_per_file_framework_strength(tmp_path: Path) -> None: + _write(tmp_path, {"agent.py": "from crewai import Crew\nCrew()\nCrew()\nCrew()\n"}) + + assert folder_findings(tmp_path).findings == ( + Finding("crewai", "import", "agent.py", 1), + Finding("crewai", "builds", "agent.py", 2), + ) + + +def test_findings_same_when_a_limit_stops_the_walk( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write(tmp_path, {"a.py": CREW, "b.py": PIPECAT, "z/c.py": LANGGRAPH}) + monkeypatch.setattr(fd, "_MAX_FILES", 1) + in_order = folder_findings(tmp_path) + real_walk = os.walk + + def reversed_walk(*args: Any, **kwargs: Any) -> Any: + for directory, directories, files in real_walk(*args, **kwargs): + directories.reverse() + files.reverse() + yield directory, directories, files + + monkeypatch.setattr(fd.os, "walk", reversed_walk) + + assert in_order.over_limit + assert folder_findings(tmp_path) == in_order + assert [f.file for f in in_order.findings] == ["a.py", "a.py"] + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + ({"main.py": PLAIN, "build/x.py": CREW}, NONE), + ({"main.py": PLAIN, "dist/x.py": CREW}, NONE), + ({"main.py": PLAIN, "env/x.py": CREW}, NONE), + ({"pkg/build/x.py": CREW}, _named("crewai")), + ({"pkg/dist/x.py": CREW}, _named("crewai")), + ({"pkg/env/x.py": CREW}, _named("crewai")), + ({"main.py": PLAIN, "pkg/node_modules/x.py": CREW}, NONE), + ({"main.py": PLAIN, "pkg/site-packages/x.py": CREW}, NONE), + ( + { + "main.py": PLAIN, + "lib/x.py": CREW, + "lib/crewai-1.0.dist-info/METADATA": "", + }, + NONE, + ), + ( + { + "main.py": CREW, + "crewai/__init__.py": "", + "crewai-1.0.dist-info/METADATA": "", + }, + _named("crewai"), + ), + ( + {"main.py": PLAIN, "crewai-1.0.dist-info/METADATA": ""}, + UNKNOWN, + ), + ], + ids=[ + "build_at_top", + "dist_at_top", + "env_at_top", + "build_deeper", + "dist_deeper", + "env_deeper", + "node_modules_at_depth", + "site_packages_at_depth", + "dist_info_child_stops_descent", + "pip_install_target_not_shadowed", + "dist_info_at_top_blocks_none", + ], +) +def test_walk_rules( + tmp_path: Path, files: dict[str, str], expected: dict[str, Any] +) -> None: + _write(tmp_path, files) + + assert detect_framework(tmp_path) == expected + + +def test_walk_rules_unreadable_directory_blocks_none(tmp_path: Path) -> None: + if os.geteuid() == 0: + pytest.skip("root reads a mode-000 directory") + _write(tmp_path, {"main.py": PLAIN, "locked/x.py": CREW}) + locked = tmp_path / "locked" + locked.chmod(0) + try: + found = folder_findings(tmp_path) + result = detect_framework(tmp_path) + finally: + locked.chmod(stat.S_IRWXU) + + assert fd._FILE_SKIPPED in found.blockers + assert result == UNKNOWN + + +def test_walk_rules_symlinked_directory_blocks_none(tmp_path: Path) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": PLAIN}) + _write(tmp_path, {"elsewhere/x.py": CREW}) + (folder / "link").symlink_to(tmp_path / "elsewhere", target_is_directory=True) + + assert fd._FILE_SKIPPED in folder_findings(folder).blockers + assert detect_framework(folder) == UNKNOWN + + +def test_byte_order_mark_and_latin1_files_read(tmp_path: Path) -> None: + (tmp_path / "bom.py").write_bytes(b"\xef\xbb\xbf" + CREW.encode()) + assert detect_framework(tmp_path) == _named("crewai") + + (tmp_path / "bom.py").unlink() + (tmp_path / "latin.py").write_bytes( + b"# -*- coding: latin-1 -*-\n# caf\xe9\n" + CREW.encode() + ) + assert detect_framework(tmp_path) == _named("crewai") + + +def test_long_line_in_a_side_file_is_not_scanned(tmp_path: Path) -> None: + _write( + tmp_path, + {"main.py": PLAIN, "tests/x.py": "from crewai import Crew\n" + LONG_LINE}, + ) + + assert detect_framework(tmp_path) == NONE + + +def test_long_line_blocks_none_only(tmp_path: Path) -> None: + _write(tmp_path, {"main.py": PLAIN, "long.py": LONG_LINE}) + assert detect_framework(tmp_path) == UNKNOWN + + _write(tmp_path, {"crew.py": CREW}) + assert detect_framework(tmp_path) == _named("crewai") + + +@pytest.mark.parametrize( + ("content", "skipped"), + [ + (CREW + LONG_LINE, True), + ("#" * fd._MAX_LINE_BYTES + "\n", False), + ("#" * (fd._MAX_LINE_BYTES + 1) + "\n", True), + (PLAIN + "#" * (fd._MAX_LINE_BYTES + 1), True), + ], + ids=[ + "after_a_short_line", + "exactly_at_the_limit", + "one_over", + "last_with_no_newline", + ], +) +def test_long_line_on_any_line(tmp_path: Path, content: str, skipped: bool) -> None: + _write(tmp_path, {"main.py": content}) + + found = folder_findings(tmp_path) + + assert (fd._FILE_SKIPPED in found.blockers) is skipped + assert detect_framework(tmp_path) == (UNKNOWN if skipped else NONE) + + +def _in_thread(call: Any) -> Any: + results: list[Any] = [] + thread = threading.Thread(target=lambda: results.append(call()), daemon=True) + thread.start() + thread.join(10) + assert not thread.is_alive(), "detection blocked on a named pipe" + return results[0] + + +def test_named_pipes_never_block(tmp_path: Path) -> None: + plain = tmp_path / "plain" + _write(plain, {"main.py": PLAIN}) + os.mkfifo(plain / "x.py") + assert _in_thread(lambda: detect_framework(plain)) == UNKNOWN + + crew = tmp_path / "crew" + _write(crew, {"main.py": CREW}) + package_dir = tmp_path / "package" + os.mkfifo(_dist(package_dir, "crewai-1.0") / "METADATA") + assert _in_thread( + lambda: detect_framework(crew, package_dir=package_dir) + ) == _named("crewai") + + +def test_symlinked_python_is_not_followed(tmp_path: Path) -> None: + folder = tmp_path / "agent" + folder.mkdir() + _write(tmp_path, {"elsewhere.py": CREW}) + (folder / "x.py").symlink_to(tmp_path / "elsewhere.py") + + assert detect_framework(folder) == UNKNOWN + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + ({"crew.py": CREW, "bad.py": ROT13}, _named("crewai")), + ({"main.py": PLAIN, "bad.py": ROT13}, UNKNOWN), + ({"agent.py": CREW + "x = '''\n"}, _named("crewai")), + ({"agent.py": "x = (\n" + CREW}, _named("crewai")), + ], + ids=[ + "bad_neighbour_keeps_crew", + "bad_neighbour_blocks_none", + "error_after_use", + "import_inside_an_open_bracket", + ], +) +def test_failure_scanning_a_file_keeps_what_was_found( + tmp_path: Path, files: dict[str, str], expected: dict[str, Any] +) -> None: + _write(tmp_path, files) + + assert fd._FILE_UNREADABLE in folder_findings(tmp_path).blockers + assert detect_framework(tmp_path) == expected + + +def test_failure_scanning_a_file_keeps_what_was_found_any_exception( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write(tmp_path, {"agent.py": CREW}) + real = fd._visible_tokens + + def fails_after_the_use(data: bytes, work: Any) -> Any: + yield from real(data, work) + raise SystemError("from the tokenizer") + + monkeypatch.setattr(fd, "_visible_tokens", fails_after_the_use) + + found = folder_findings(tmp_path) + + assert fd._FILE_UNREADABLE in found.blockers + assert CREW_BUILDS in {(f.framework, f.strength) for f in found.findings} + + +def test_too_many_aliases_is_unreadable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(fd, "_MAX_ALIASES", 1) + _write(tmp_path, {"agent.py": "from crewai import Crew, Agent\n"}) + + assert fd._FILE_UNREADABLE in folder_findings(tmp_path).blockers + + +@pytest.mark.parametrize( + ("constant", "value"), + [("_MAX_FILES", 1), ("_MAX_TOTAL_BYTES", 10)], +) +def test_total_limits_give_unknown( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, constant: str, value: int +) -> None: + _write(tmp_path, {"a.py": CREW, "b.py": CREW}) + monkeypatch.setattr(fd, constant, value) + + assert folder_findings(tmp_path).over_limit + assert detect_framework(tmp_path) == UNKNOWN + + +@pytest.mark.parametrize( + ("constant", "value", "side_file"), + [("_MAX_FILES", 1, PLAIN), ("_MAX_TOTAL_BYTES", 20, "#" * 50 + "\n")], + ids=["files", "bytes"], +) +def test_total_limits_count_side_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + constant: str, + value: int, + side_file: str, +) -> None: + _write(tmp_path, {"main.py": PLAIN, "tests/a.py": side_file}) + monkeypatch.setattr(fd, constant, value) + + assert folder_findings(tmp_path).over_limit + + +def test_limits_have_their_real_values() -> None: + assert ( + fd._MAX_LINE_BYTES, + fd._MAX_TOTAL_BYTES, + fd._MAX_FILES, + fd._MAX_WORK, + fd._WORK_PER_TARGET, + fd._MAX_ALIASES, + fd._MAX_DOTTED_PARTS, + fd._METADATA_BYTES, + fd._VERSION_MAX, + fd._MAX_FOLDER_BYTES, + fd._TOP_FLOOR, + ) == ( + 65_536, + 67_108_864, + 5_000, + 100_000_000_000, + 2_000, + 10_000, + 64, + 65_536, + 64, + 67_108_864, + 184, + ) + + +def test_read_stops_at_the_bytes_left( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write(tmp_path, {"a.py": "x=1\n", "b.py": "y=2\n"}) + limits: list[int] = [] + real = fd._read_file + + def spy(path: str, limit: int) -> bytes: + limits.append(limit) + return real(path, limit) + + monkeypatch.setattr(fd, "_read_file", spy) + + folder_findings(tmp_path) + + assert limits == [fd._MAX_TOTAL_BYTES + 1, fd._MAX_TOTAL_BYTES - 4 + 1] + + +def test_tokenize_warnings_are_silenced( + tmp_path: Path, capfd: pytest.CaptureFixture[str], recwarn: pytest.WarningsRecorder +) -> None: + _write(tmp_path, {"a.py": "x = f'\\{a}'\n" * 3}) + + detect_framework(tmp_path) + + assert not recwarn.list + assert "Warning" not in capfd.readouterr().err + + +def _token_work(source: str) -> int: + tokens = tokenize.tokenize(io.BytesIO(source.encode()).readline) + return sum(len(token.line) for token in tokens) + + +# Two reads, and two walks of its one target: the name in the import, then the call. +CREW_WORK = 2 * _token_work(CREW) + 2 * fd._WORK_PER_TARGET +# Comments and what's inside an f-string are never seen by the scan, but cost the same. +UNSEEN = "# note\nx = f'{a}'\n" + CREW +UNSEEN_WORK = 2 * _token_work(UNSEEN) + 2 * fd._WORK_PER_TARGET + + +@pytest.mark.parametrize( + ("files", "work"), + [ + ({"a.py": CREW}, CREW_WORK), + ({"a.py": CREW, "b.py": CREW}, 2 * CREW_WORK), + ({"main.py": CREW, "tests/a.py": PLAIN}, CREW_WORK + _token_work(PLAIN)), + ({"a.py": UNSEEN}, UNSEEN_WORK), + ], + ids=[ + "one_target", + "shared_across_files", + "side_file_read_once", + "tokens_the_scan_never_sees", + ], +) +def test_work_limit_edge( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + files: dict[str, str], + work: int, +) -> None: + _write(tmp_path, files) + + monkeypatch.setattr(fd, "_MAX_WORK", work) + assert detect_framework(tmp_path) == _named("crewai") + + monkeypatch.setattr(fd, "_MAX_WORK", work - 1) + assert folder_findings(tmp_path).over_limit + assert detect_framework(tmp_path) == UNKNOWN + + +def test_work_limit_counts_every_target_of_a_name( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + imports = "".join(f"import crewai.m{index} as C\n" for index in range(10_000)) + source = imports + "C()\n" * 100 + limit = 1_000_000_000 + assert 2 * _token_work(source) < limit + _write(tmp_path, {"agent.py": source}) + monkeypatch.setattr(fd, "_MAX_WORK", limit) + + found = folder_findings(tmp_path) + + assert found.over_limit + assert fd._FILE_UNREADABLE not in found.blockers + assert detect_framework(tmp_path) == UNKNOWN + + +def test_work_limit_counts_long_lines( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + long_line = "x = [" + "a," * 30_000 + "]\n" + assert len(long_line) <= fd._MAX_LINE_BYTES + _write(tmp_path, {"agent.py": CREW + long_line * 2}) + monkeypatch.setattr(fd, "_MAX_WORK", 10_000_000) + + found = folder_findings(tmp_path) + + assert found.over_limit + assert detect_framework(tmp_path) == UNKNOWN + + +def test_bytes_exactly_at_the_limit_are_not_over( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write(tmp_path, {"a.py": CREW}) + monkeypatch.setattr(fd, "_MAX_TOTAL_BYTES", len(CREW)) + + assert not folder_findings(tmp_path).over_limit + + +LONG_PART = b"p" * 65_000 + + +def test_names_under_one_long_module_share_its_memory(tmp_path: Path) -> None: + module = b"from crewai.\\\n" + (LONG_PART + b".\\\n") * 62 + LONG_PART + names = b"".join(b"a%d,\n" % index for index in range(100)) + (tmp_path / "agent.py").write_bytes(module + b" import (\n" + names + b")\n") + + tracemalloc.start() + try: + found = folder_findings(tmp_path) + peak = tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + assert peak < 50 * 1024 * 1024 + assert {(f.framework, f.strength) for f in found.findings} == {CREW_IMPORT} + + +def _aliases(source: str) -> dict[str, set[tuple[str, ...]]]: + scan = fd._FileScan(frozenset(), frozenset(), fd._Work()) + scan.read_imports( + fd._outside_type_checking(fd._visible_tokens(source.encode(), scan.work)) + ) + return scan.aliases + + +def test_a_target_keeps_at_most_the_part_limit() -> None: + aliases = _aliases("import crewai." + "a." * 200 + "z as C\n") + + assert [len(target) for target in aliases["C"]] == [fd._MAX_DOTTED_PARTS] + + +def test_plain_dotted_import_maps_the_top_name_to_the_top_name() -> None: + assert _aliases("import crewai.tools\n") == {"crewai": {("crewai",)}} + + +def test_use_inside_a_type_checking_block_does_not_count(tmp_path: Path) -> None: + _write( + tmp_path, + {"agent.py": "from crewai import Crew\nif TYPE_CHECKING:\n Crew()\n"}, + ) + + found = folder_findings(tmp_path) + + assert {(f.framework, f.strength) for f in found.findings} == {CREW_IMPORT} + + +def test_a_use_of_a_long_target_allocates_nothing_of_its_size() -> None: + data = b"import crewai.\\\n" + (LONG_PART + b".\\\n") * 63 + LONG_PART + b" as C\n" + scan = fd._FileScan(frozenset(), frozenset(), fd._Work()) + scan.read_imports(fd._outside_type_checking(fd._visible_tokens(data, scan.work))) + + tracemalloc.start() + try: + scan._resolve(["C"], "call", 1) + peak = tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + assert peak < 64 * 1024 + + +BLOCKER_CASES: list[tuple[str, dict[str, str], str, str]] = [ + ( + fd._FILE_SKIPPED, + {"agent/main.py": PLAIN, "agent/long.py": LONG_LINE}, + "agent", + "", + ), + (fd._FILE_UNREADABLE, {"agent/main.py": PLAIN, "agent/bad.py": ROT13}, "agent", ""), + ( + fd._AGENTS_AMBIGUOUS, + {"agent/main.py": OPENAI_AGENT, "agent/src/agents/x.py": PLAIN}, + "agent", + "", + ), + ( + fd._SIDE_FOLDER_FRAMEWORK, + {"agent/main.py": PLAIN, "agent/tests/x.py": CREW}, + "agent", + "", + ), + ( + fd._OUTSIDE_IMPORT, + {"agent/main.py": "import shared\n", "shared.py": PLAIN}, + "agent", + ".", + ), + ( + fd._OUTSIDE_IMPORT, + {"agent/main.py": "from shared import x\nimport os, tools\n", "tools/y.py": ""}, + "agent", + ".", + ), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/ui.ts": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/ui.tsx": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.mts": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.cts": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.js": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.jsx": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.mjs": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.cjs": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/n.ipynb": ""}, "agent", ""), + ( + fd._OTHER_SOURCE, + {"agent/main.py": PLAIN, "agent/root_agent.yaml": ""}, + "agent", + "", + ), + (fd._NO_PYTHON, {"agent/README.md": ""}, "agent", ""), +] + + +@pytest.mark.parametrize( + ("blocker", "files", "folder", "repo_root"), + BLOCKER_CASES, + ids=[ + "file_skipped", + "file_unreadable", + "agents_ambiguous", + "side_folder_framework", + "outside_import", + "outside_import_from_and_comma", + "other_source_ts", + "other_source_tsx", + "other_source_mts", + "other_source_cts", + "other_source_js", + "other_source_jsx", + "other_source_mjs", + "other_source_cjs", + "other_source_ipynb", + "other_source_root_agent", + "no_python", + ], +) +def test_blockers_stop_none( + tmp_path: Path, blocker: str, files: dict[str, str], folder: str, repo_root: str +) -> None: + _write(tmp_path, files) + root = tmp_path if repo_root else None + + assert blocker in folder_findings(tmp_path / folder, repo_root=root).blockers + assert detect_framework(tmp_path / folder, repo_root=root) == UNKNOWN + + # Anything that blocks may hide what builds the agent, so a chat model names nothing. + if blocker != fd._NO_PYTHON: + _write(tmp_path, {f"{folder}/model.py": CHAT_OPENAI}) + found = folder_findings(tmp_path / folder, repo_root=root) + assert any(f.strength == "model" for f in found.findings) + assert detect_framework(tmp_path / folder, repo_root=root) == UNKNOWN + + # What the scan saw being built still names the framework. + _write(tmp_path, {f"{folder}/crew.py": CREW}) + assert detect_framework(tmp_path / folder, repo_root=root) == _named("crewai") + + +@pytest.mark.parametrize( + ("folder", "files", "blocked"), + [ + ("repo/agent", {"repo/agent/main.py": "import agent\n"}, False), + ("repo/a/b", {"repo/a/b/main.py": "import a\n"}, False), + ("repo/agent", {"repo/agent/main.py": "from . import shared\n"}, False), + ( + "repo/agent", + {"repo/agent/main.py": "import shared\n", "repo/agent/shared.py": ""}, + False, + ), + ("copy", {"copy/main.py": "import agent\n", "repo/agent/x.py": ""}, True), + ], + ids=[ + "folder_itself", + "folder_on_the_path", + "relative", + "under_the_folder", + "folder_outside_the_root_excludes_nothing", + ], +) +def test_blockers_stop_none_outside_import_scope( + tmp_path: Path, folder: str, files: dict[str, str], blocked: bool +) -> None: + _write(tmp_path, files) + (tmp_path / "repo" / "shared.py").write_text(PLAIN, encoding="utf-8") + (tmp_path / "repo" / "a").mkdir(exist_ok=True) + (tmp_path / "repo" / "agent").mkdir(exist_ok=True) + + found = folder_findings(tmp_path / folder, repo_root=tmp_path / "repo") + + assert (fd._OUTSIDE_IMPORT in found.blockers) is blocked + + +def test_blockers_stop_none_no_outside_import_when_folder_is_the_root( + tmp_path: Path, +) -> None: + _write(tmp_path, {"main.py": "import shared\n", "shared.py": PLAIN}) + + assert detect_framework(tmp_path, repo_root=tmp_path) == NONE + + +AGENT_JS = ( + "import { Agent, run } from '@openai/agents';\n" + "const agent = new Agent({ name: 'x' });\n" + "await run(agent, 'hi');\n" +) +HEALTHCHECK = "import urllib.request\n" + + +@pytest.mark.parametrize( + "files", + [ + {"tests/index.js": AGENT_JS, "healthcheck.py": HEALTHCHECK}, + {"tests/x.jsx": AGENT_JS, "healthcheck.py": HEALTHCHECK}, + {"examples/node/x.mjs": AGENT_JS, "healthcheck.py": HEALTHCHECK}, + ], + ids=["js_in_tests", "jsx_in_tests", "mjs_in_examples"], +) +def test_javascript_in_test_and_example_folders_does_not_block( + tmp_path: Path, files: dict[str, str] +) -> None: + _write(tmp_path, files) + + assert fd._OTHER_SOURCE not in folder_findings(tmp_path).blockers + assert detect_framework(tmp_path) == NONE + + +def test_a_label_beside_a_blocker_stays_other(tmp_path: Path) -> None: + _write(tmp_path, {"main.py": AUTOGEN, "ui.js": ""}) + + assert fd._OTHER_SOURCE in folder_findings(tmp_path).blockers + assert detect_framework(tmp_path) == _named("other", "autogen") + + +SIDE_WORDS = [ + "tests", + "test", + "examples", + "example", + "samples", + "sample", + "demos", + "demo", + "docs", + "doc", + "evals", + "eval", + "scripts", + "legacy", + "notebooks", + "vendor", + "vendored", + "third_party", + "external", + "contrib", + "experiments", + "benchmarks", + "archive", + ".hidden", +] +SIDE_PATHS = [f"{word}/crew.py" for word in SIDE_WORDS] + [ + "test_x.py", + "x_test.py", + "conftest.py", + "pkg/tests/crew.py", + ".x.py", +] + + +@pytest.mark.parametrize("path", SIDE_PATHS) +def test_side_folders_do_not_decide(tmp_path: Path, path: str) -> None: + _write(tmp_path, {"main.py": PLAIN, path: CREW}) + + found = folder_findings(tmp_path) + + assert found.findings == () + assert found.blockers == {fd._SIDE_FOLDER_FRAMEWORK} + assert detect_framework(tmp_path) == UNKNOWN + + +def test_side_folder_as_the_agent_folder_counts(tmp_path: Path) -> None: + _write(tmp_path, {"examples/x/main.py": CREW}) + + assert detect_framework(tmp_path / "examples" / "x") == _named("crewai") + + +@pytest.mark.parametrize("content", [LONG_LINE, ROT13], ids=["long_line", "rot13"]) +def test_problem_in_a_side_file_is_no_blocker(tmp_path: Path, content: str) -> None: + _write(tmp_path, {"main.py": PLAIN, "tests/bad.py": content}) + + assert detect_framework(tmp_path) == NONE + + +def test_agents_folder_beside_plain_code_is_none(tmp_path: Path) -> None: + _write(tmp_path, {"main.py": PLAIN, "agents/x.py": PLAIN}) + + assert detect_framework(tmp_path) == NONE + + +@pytest.mark.parametrize( + ("files", "installed", "repo_root", "expected"), + [ + ({"main.py": OPENAI_AGENT}, "openai_agents-0.1", None, "openai_agents"), + ({"main.py": OPENAI_AGENT}, "requests-2.0", None, "none"), + ({"main.py": OPENAI_AGENT}, None, None, "openai_agents"), + ({"main.py": OPENAI_AGENT, "src/agents/x.py": PLAIN}, None, None, "unknown"), + ({"main.py": OPENAI_AGENT, "agents.py": PLAIN}, None, None, "unknown"), + ( + {"main.py": OPENAI_AGENT, "agents.py": PLAIN}, + "openai_agents-0.1", + None, + "openai_agents", + ), + ({"main.py": OPENAI_AGENT}, None, "with_agents", "unknown"), + ], + ids=[ + "installed", + "other_packages_installed", + "no_tree_no_folder", + "no_tree_own_folder", + "no_tree_own_module", + "own_module_with_package_installed", + "no_tree_folder_under_repo_root", + ], +) +def test_agents_import_rules( + tmp_path: Path, + files: dict[str, str], + installed: str | None, + repo_root: str | None, + expected: str, +) -> None: + folder = tmp_path / "repo" / "agent" + _write(folder, files) + package_dir = None + if installed is not None: + package_dir = tmp_path / "package" + _dist(package_dir, installed) + root = None + if repo_root is not None: + root = tmp_path / "repo" + (root / "agents").mkdir() + + result = detect_framework(folder, repo_root=root, package_dir=package_dir) + + assert result["name"] == expected + + +def test_agents_module_under_repo_root_is_ambiguous(tmp_path: Path) -> None: + _write(tmp_path, {"agent/main.py": OPENAI_AGENT, "agents.py": PLAIN}) + + assert detect_framework(tmp_path / "agent", repo_root=tmp_path) == UNKNOWN + + +def test_folder_with_no_init_does_not_shadow(tmp_path: Path) -> None: + _write(tmp_path, {"main.py": CREW, "crewai/x.py": PLAIN}) + + assert detect_framework(tmp_path) == _named("crewai") + + +@pytest.mark.parametrize( + "files", + [ + {"main.py": CREW, "crewai.py": PLAIN}, + { + "main.py": "from langchain.agents import create_agent\ncreate_agent()\n", + "langchain/__init__.py": "", + }, + ], + ids=["module", "package"], +) +def test_own_module_shadows_framework(tmp_path: Path, files: dict[str, str]) -> None: + _write(tmp_path, files) + + assert detect_framework(tmp_path) == NONE + + +def test_missing_folder_gives_unknown(tmp_path: Path) -> None: + assert detect_framework(None) == UNKNOWN + assert detect_framework(tmp_path / "missing") == UNKNOWN + + +def test_detector_failure_gives_unknown( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + secret = "/customer/private/path" + + def _raise(*_: Any, **__: Any) -> Any: + raise RuntimeError(secret) + + monkeypatch.setattr(fd, "folder_findings", _raise) + + with caplog.at_level(logging.INFO, logger=fd.__name__): + result = detect_framework(tmp_path) + + assert result == UNKNOWN + assert "RuntimeError" in caplog.text + assert secret not in caplog.text + + +@pytest.mark.parametrize( + "failure", [MemoryError(), RecursionError()], ids=["memory", "recursion"] +) +def test_detector_failure_gives_unknown_on_resource_errors( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: Exception +) -> None: + def _raise(*_: Any, **__: Any) -> Any: + raise failure + + monkeypatch.setattr(fd, "folder_findings", _raise) + + assert detect_framework(tmp_path) == UNKNOWN + + +def test_never_imports_or_runs_customer_code(tmp_path: Path) -> None: + marker = tmp_path / "ran" + source = ( + f"from pathlib import Path\nPath({str(marker)!r}).write_text('x')\n" + "from crewai import Crew\nCrew()\n" + ) + folder = tmp_path / "agent" + _write(folder, {"customer_module_never_imported.py": source}) + + assert detect_framework(folder) == _named("crewai") + + assert not marker.exists() + assert "customer_module_never_imported" not in sys.modules + assert "crewai" not in sys.modules + + +LAYERED = DEEP + LANGGRAPH +PYDANTIC_BUILDS = "from pydantic_ai import Agent\nAgent()\n" + + +@pytest.mark.parametrize( + ("source", "dists", "expected"), + [ + ( + "from pydantic_ai import Agent\nAgent()\n", + [("pydantic_ai_slim-1.2.3", "1.2.3")], + "1.2.3", + ), + ( + "from pydantic_ai import Agent\nAgent()\n", + [("pydantic_ai-2.0", "2.0"), ("pydantic_ai_slim-1.2.3", "1.2.3")], + "2.0", + ), + (LIVEKIT, [("livekit_agents-1.8.8", "1.8.8")], "1.8.8"), + (LIVEKIT, [("livekit-1.1.20", "1.1.20")], None), + (CREW, [("crewai-1.0", "1.0"), ("crewai-2.0", "2.0")], None), + (LAYERED, [("deepagents-0.2", "0.2"), ("langgraph-1.0", "1.0")], "0.2"), + ( + AUTOGEN, + [("autogen_agentchat-0.4.1", "0.4.1"), ("pyautogen-0.2", "0.2")], + "0.4.1", + ), + ( + CREW, + [("crewai-1.0rc1.post2.dev3+local.1", "1.0rc1.post2.dev3+local.1")], + "1.0rc1.post2.dev3+local.1", + ), + (PYDANTIC_BUILDS, [("PyDantic_AI-1.0", "1.0")], "1.0"), + (PYDANTIC_BUILDS, [("pydantic.ai-1.0", "1.0")], "1.0"), + (PYDANTIC_BUILDS, [("pydantic_ai-1.0", "1.0RC1")], "1.0RC1"), + ], + ids=[ + "second_package_of_the_row", + "first_package_wins", + "livekit_agents", + "livekit_alone_is_not_the_row", + "two_versions", + "layered_reads_the_chosen_row", + "other_row", + "full_version_shape", + "name_case_ignored", + "name_dots_normalised", + "version_case_ignored", + ], +) +def test_version_read( + tmp_path: Path, source: str, dists: list[tuple[str, str]], expected: str | None +) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": source}) + package_dir = tmp_path / "package" + for dist, version in dists: + _dist(package_dir, dist, _metadata(version)) + + assert detect_framework(folder, package_dir=package_dir)["version"] == expected + + +@pytest.mark.parametrize( + ("python", "metadata"), + [("python3.13", b"Version: 1.0\n"), ("python3.12", b"Version: 1.0\r\n")], + ids=["any_python_folder", "crlf_line"], +) +def test_version_read_real_shapes(tmp_path: Path, python: str, metadata: bytes) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": CREW}) + dist_info = ( + tmp_path / "pkg/.venv/lib" / python / "site-packages/crewai-1.0.dist-info" + ) + dist_info.mkdir(parents=True) + (dist_info / "METADATA").write_bytes(metadata) + + found = detect_framework(folder, package_dir=tmp_path / "pkg") + + assert found["version"] == "1.0" + + +def test_version_read_is_null_for_a_result_with_no_row(tmp_path: Path) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": "from crewai import Crew\n"}) + package_dir = tmp_path / "package" + _dist(package_dir, "crewai-1.0", _metadata("1.0")) + + assert detect_framework(folder, package_dir=package_dir) == UNKNOWN + + +@pytest.mark.parametrize( + "metadata", + [ + b"Metadata-Version: 2.1\nVersion: 1.0\x00\n", + b"Metadata-Version: 2.1\nVersion: sk_live_x\n", + b"Metadata-Version: 2.1\nVersion: 1" + b".0" * 32 + b"\n", + b"Metadata-Version: 2.1\nName: crewai\n", + b"Metadata-Version: 2.1\nX-Version: 1.0\nNotVersion: 2.0\n", + b"Metadata-Version: 2.1\nVersion: caf\xc3\xa9\n", + b"Metadata-Version: 2.1\nVersion: bad\nVersion: 1.0\n", + ], + ids=[ + "nul", + "secret_like", + "too_long", + "no_version_line", + "not_at_line_start", + "non_ascii", + "later_line_not_read", + ], +) +def test_bad_metadata_gives_null_version(tmp_path: Path, metadata: bytes) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": CREW}) + package_dir = tmp_path / "package" + _dist(package_dir, "crewai-1.0", metadata) + + assert detect_framework(folder, package_dir=package_dir) == _named("crewai") + + +def test_bad_metadata_gives_null_version_symlink(tmp_path: Path) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": CREW}) + package_dir = tmp_path / "package" + elsewhere = tmp_path / "METADATA" + elsewhere.write_bytes(_metadata("1.0")) + (_dist(package_dir, "crewai-1.0") / "METADATA").symlink_to(elsewhere) + + assert detect_framework(folder, package_dir=package_dir) == _named("crewai") + + +def test_metadata_read_stops_at_64_kib(tmp_path: Path) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": CREW}) + package_dir = tmp_path / "package" + padding = b"Metadata-Version: 2.1\n" + b"X: y\n" * 14_000 + assert len(padding) > fd._METADATA_BYTES + _dist(package_dir, "crewai-1.0", padding + b"Version: 1.0\n") + + assert detect_framework(folder, package_dir=package_dir) == _named("crewai") + + +def _process_manifest(source_origin: str = "bundle") -> SimpleNamespace: + return SimpleNamespace( + runtime=SimpleNamespace(kind="process", control_service="agent"), + processes=[ + SimpleNamespace(name="postgres", working_directory=None), + SimpleNamespace( + name="agent", + working_directory="generated/source/agent", + source_origin=source_origin, + ), + ], + ) + + +def _compose_manifest() -> SimpleNamespace: + return SimpleNamespace( + runtime=SimpleNamespace(kind="compose", control_service=None), processes=[] + ) + + +@pytest.mark.parametrize( + ("subfolder", "contract", "manifest", "expected"), + [ + ("sub", {"runtime": {"workdir": "inner"}}, _process_manifest(), "source/sub"), + (None, {"runtime": {"workdir": "inner"}}, _process_manifest(), "source/inner"), + (None, {"runtime": {"workdir": "inner"}}, _compose_manifest(), "source/inner"), + ( + None, + {"runtime": {"workdir": "."}}, + _process_manifest(), + "bundle/generated/source/agent", + ), + ( + None, + {"runtime": {"workdir": ""}}, + _process_manifest(), + "bundle/generated/source/agent", + ), + ( + None, + {"runtime": {"workdir": 3}}, + _process_manifest(), + "bundle/generated/source/agent", + ), + ( + None, + {"runtime": {"workdir": "../outside"}}, + _process_manifest(), + "bundle/generated/source/agent", + ), + ( + None, + {"runtime": {"workdir": "a_file"}}, + _process_manifest(), + "bundle/generated/source/agent", + ), + (None, None, _process_manifest(), "bundle/generated/source/agent"), + (None, "{not json", _process_manifest(), "bundle/generated/source/agent"), + (None, ["a", "list"], _process_manifest(), "bundle/generated/source/agent"), + ( + None, + None, + _process_manifest("repository"), + "source/generated/source/agent", + ), + (None, None, _compose_manifest(), "source"), + ], + ids=[ + "subfolder_first", + "workdir_before_process", + "workdir_on_compose", + "dot_workdir", + "empty_workdir", + "non_string_workdir", + "workdir_escaping_source", + "workdir_is_a_file", + "missing_contract", + "corrupt_contract", + "contract_not_an_object", + "repository_origin_process", + "compose_falls_to_source", + ], +) +def test_folder_order( + tmp_path: Path, + subfolder: str | None, + contract: Any, + manifest: SimpleNamespace, + expected: str, +) -> None: + source = tmp_path / "source" + bundle_dir = tmp_path / "bundle" + work = tmp_path / "work" + _write( + tmp_path, + { + "source/a_file": "", + "source/inner/x.py": "", + "source/sub/x.py": "", + "outside/x.py": "", + }, + ) + bundle_dir.mkdir() + if isinstance(contract, str): + (bundle_dir / "contract.json").write_text(contract, encoding="utf-8") + elif contract is not None: + (bundle_dir / "contract.json").write_text( + json.dumps(contract), encoding="utf-8" + ) + job = SimpleNamespace(source=SimpleNamespace(subfolder=subfolder)) + + folders = framework_folders_for( + job, manifest, source=source, bundle_dir=bundle_dir, work_directory=work + ) + + is_process = manifest.runtime.kind == "process" + assert folders == FrameworkFolders( + tmp_path / expected, source, work / "build" / "agent" if is_process else None + ) + + +def _github_job(subfolder: str | None) -> SimpleNamespace: + return SimpleNamespace( + source=RepositorySource( + kind=SourceKind.GITHUB, + repository="org/repo", + visibility=SourceVisibility.PUBLIC, + commit_sha="a" * 40, + subfolder=subfolder, + ) + ) + + +@pytest.mark.parametrize( + ("subfolder", "expected"), + [(None, "source"), ("inner", "source/inner")], + ids=["no_subfolder", "subfolder"], +) +def test_folder_order_on_a_real_job_source( + tmp_path: Path, subfolder: str | None, expected: str +) -> None: + source = tmp_path / "source" + _write(tmp_path, {"source/inner/x.py": ""}) + + folders = framework_folders_for( + _github_job(subfolder), + _compose_manifest(), + source=source, + bundle_dir=tmp_path / "bundle", + work_directory=tmp_path / "work", + ) + + assert folders == FrameworkFolders(tmp_path / expected, source, None) + + +def test_folder_order_refuses_a_subfolder_outside_the_source(tmp_path: Path) -> None: + _write(tmp_path, {"source/x.py": "", "outside/x.py": ""}) + + with pytest.raises(SubfolderNotFound): + framework_folders_for( + _github_job("../outside"), + _compose_manifest(), + source=tmp_path / "source", + bundle_dir=tmp_path / "bundle", + work_directory=tmp_path / "work", + ) + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ("import livekit\nlivekit.agents.AgentSession()\n", _named("livekit")), + ("import google\ngoogle.adk.agents.Agent()\n", _named("google_adk")), + ("import google\ngoogle.genai.Client()\n", NONE), + ("import livekit\nlivekit.agents.JobContext()\n", UNKNOWN), + ("import livekit\nagents = livekit.agents\n", UNKNOWN), + ("import livekit\nprint(livekit.agents.__version__)\n", UNKNOWN), + ("import google\ngoogle.adk.tools.FunctionTool(f)\n", UNKNOWN), + ("import google\nadk = google.adk\n", UNKNOWN), + ("from livekit import api\napi.AccessToken()\n", NONE), + ], + ids=[ + "livekit", + "google_adk", + "google_not_adk", + "livekit_no_builder", + "livekit_assigned", + "livekit_read_only", + "google_adk_no_builder", + "google_adk_assigned", + "livekit_not_the_row", + ], +) +def test_top_name_import_then_dotted_use( + tmp_path: Path, source: str, expected: dict[str, Any] +) -> None: + _write(tmp_path, {"main.py": source}) + + assert detect_framework(tmp_path) == expected + + +# The walk's bound on folders listed to walk into (TH-9511, contract 1.9 P6). Every tree below is sized +# from fixed numbers, and every expected stop is worked out here from the contract's formula, never by +# calling the code under test. + + +def _broom(top: Path, depth: int, siblings: int, *, chain: str, sibling: str, crew_in_chain: bool) -> None: + """A chain ``depth`` folders deep, with ``siblings`` folders beside each chain folder, each holding + one file that isn't Python. Every name is 10 characters, and the first character of ``chain`` + and of ``sibling`` decides which sorts first. A CREW file sits at the top, and in each chain + folder when ``crew_in_chain``.""" + _write(top, {"agent.py": CREW}) + folder = top + for level in range(depth): + for index in range(siblings): + beside = folder / f"{sibling[:1]}{level:02}{index:07}" + beside.mkdir(parents=True) + (beside / "x").write_text("", encoding="utf-8") + folder = folder / f"{chain[:1]}{level:02}_chain_" + folder.mkdir(parents=True) + if crew_in_chain: + (folder / "agent.py").write_text(CREW, encoding="utf-8") + + +def _level_cost(level: int, siblings: int, width: int = 1) -> int: + """By hand, contract 1.9 P6, for a top of at most 184 characters: each folder at depth ``level`` + (1 for the top's own) has a 10-character name, so its path below the top is ``11 * level - 1`` + characters, and costs ``width * (184 + 1 + 11 * level - 1) + 120``.""" + return (siblings + 1) * (width * (184 + 11 * level) + 120) + + +def _stop_level(bound: int, siblings: int, width: int = 1) -> int: + """The level whose listing first takes the running total past ``bound``.""" + total, level = 0, 0 + while total <= bound: + level += 1 + total += _level_cost(level, siblings, width) + return level + + +def _short_top(path: Path) -> Path: + if len(os.fspath(path)) > 184: + pytest.skip("this machine's tmp_path is past the 184-character floor") + return path + + +def _padded(base: Path, length: int) -> Path: + """A top exactly ``length`` characters long, in parts the file system accepts.""" + left = length - len(os.fspath(base)) - 1 + if left < 1: + pytest.skip("this machine's tmp_path is too long to pad") + parts = [] + while left > 201: + parts.append("p" * 200) + left -= 201 + parts.append("p" * left) + top = base.joinpath(*parts) + assert len(os.fspath(top)) == length, len(os.fspath(top)) + top.mkdir(parents=True) + return top + + +def _crew_files(found: fd.FolderFindings) -> set[str]: + return {finding.file for finding in found.findings if finding.framework == "crewai"} + + +@pytest.mark.parametrize("chain", ["a_chain", "z_chain"], ids=["first", "last"]) +def test_a_broom_of_folders_past_the_bound_gives_unknown( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, chain: str +) -> None: + top = _short_top(tmp_path) + _broom(top, 30, 40, chain=chain, sibling="s_sibli", crew_in_chain=True) + assert detect_framework(top)["name"] == "crewai" + # Below the broom's total, above any one level's (the deepest is about 26 KB). + bound = 300_000 + assert max(_level_cost(level, 40) for level in range(1, 31)) < bound + monkeypatch.setattr(fd, "_MAX_FOLDER_BYTES", bound) + + found = folder_findings(top) + + assert found.over_limit + assert detect_framework(top) == UNKNOWN + # The folder whose listing takes the total past the bound reads none of its files: the CREW + # files found are the top's and those of the chain folders above that one. A last chain is + # entered after its siblings, which list nothing, so it stops at the same folder. + stop = _stop_level(bound, 40) + assert len(_crew_files(found)) == stop - 1 + + +def test_the_folder_bound_counts_only_folders_it_will_enter( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + top = _short_top(tmp_path) + folder = top + kept = 0 + for level in range(1, 31): + for name in sorted(fd._ALWAYS_SKIPPED): + (folder / name).mkdir(parents=True) + folder = folder / "k" + folder.mkdir() + # The kept folder "k" at depth ``level``: its path below the top is ``2 * level - 1`` long. + kept += (184 + 1 + 2 * level - 1) + 120 + _write(folder, {"agent.py": CREW}) + monkeypatch.setattr(fd, "_MAX_FOLDER_BYTES", 3 * kept) + + found = folder_findings(top) + + assert not found.over_limit + assert _crew_files(found) == {"/".join(["k"] * 30 + ["agent.py"])} + + +@pytest.mark.parametrize( + ("chain", "sibling", "width", "bound"), + [("a_chain", "s_sibli", 1, 300_000), ("\u00e0_chain", "\u00e9_sibli", 4, 800_000)], + ids=["ascii", "latin1"], +) +def test_the_folder_bound_stops_at_the_same_folder_under_any_short_top( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, chain: str, sibling: str, width: int, bound: int +) -> None: + """Two tops of at most 184 ASCII characters count the same tree the same, whatever its names.""" + short = _short_top(tmp_path / "a") + long = _padded(tmp_path / "b", 180) + for top in (short, long): + _broom(top, 30, 40, chain=chain, sibling=sibling, crew_in_chain=True) + monkeypatch.setattr(fd, "_MAX_FOLDER_BYTES", bound) + + found = [folder_findings(top) for top in (short, long)] + + assert found[0].over_limit and found[1].over_limit + assert _crew_files(found[0]) == _crew_files(found[1]) + assert len(_crew_files(found[0])) == _stop_level(bound, 40, width) - 1 + + +def _allocated(size: int) -> int: + """The block a request of ``size`` gets on the Linux sandbox: pymalloc's 16-byte classes up to + 512, then glibc's malloc (an 8-byte header, 16-byte alignment, 32 at least). Kept apart from the + code's own count, so a fault there can't hide here (as in test_scan_walk.py).""" + if size <= 512: + return max(16, (size + 15) // 16 * 16) + return max(32, (size + 8 + 15) // 16 * 16) + + +def _watch_the_stack(monkeypatch: pytest.MonkeyPatch) -> list[int]: + """What os.walk's own list of folders still to visit holds, at each folder it yields.""" + real_walk = os.walk + held: list[int] = [] + + def watching(*args: Any, **kwargs: Any) -> Any: + walk = real_walk(*args, **kwargs) + for item in walk: + # A missing `stack` fails loudly rather than reading nothing. + stack = walk.gi_frame.f_locals["stack"] + held.append(sum(_allocated(sys.getsizeof(path)) + 8 for path in stack if isinstance(path, str))) + yield item + + monkeypatch.setattr(fd.os, "walk", watching) + return held + + +# os.walk keeps its folders still to visit in a `stack` local only from 3.12; before, it recurses. +_STACK_WALK = pytest.mark.skipif( + sys.version_info < (3, 12), reason="os.walk has a `stack` list to watch only from Python 3.12" +) + + +_STACK_CASES = { + # top length (None: tmp_path itself), depth, siblings, chain stem, sibling stem + # Sized so the whole list passes 2 MiB even under a Linux tmp_path of about 50 characters, and + # so each broom passes the bound under the plant aimed at it. + "short": (None, 30, 300, "a_chain", "s_sibli"), + "long": (400, 30, 150, "a_chain", "s_sibli"), + "wide": (160, 30, 150, "\U0001d400chain", "\U0001d41asibli"), + # A top with one wide character in it, and ASCII names below: the top alone makes every path wide. + "wide_top": (-160, 30, 150, "a_chain", "s_sibli"), + "deep": (None, 60, 115, "a_chain", "s_sibli"), +} + + +@_STACK_WALK +@pytest.mark.parametrize("case", list(_STACK_CASES)) +def test_the_walks_own_list_never_passes_the_folder_bound( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, case: str +) -> None: + length, depth, siblings, chain, sibling = _STACK_CASES[case] + if length is None: + top = _short_top(tmp_path / "t") + elif length < 0: + top = _padded(tmp_path / "\U0001d400", -length) + assert not os.fspath(top).isascii() + else: + top = _padded(tmp_path / "t", length) + _broom(top, depth, siblings, chain=chain, sibling=sibling, crew_in_chain=False) + bound = 2_097_152 + monkeypatch.setattr(fd, "_MAX_FOLDER_BYTES", bound) + held = _watch_the_stack(monkeypatch) + + found = folder_findings(top) + + assert found.over_limit + assert max(held) <= bound, (max(held), bound) + + +@_STACK_WALK +def test_the_walks_own_list_grows_past_the_test_bound_without_it( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The control: with the real bound, the deep broom's list passes the 2 MiB the test above holds + it to, so that test can see a count that is too low.""" + top = _short_top(tmp_path / "t") + _broom(top, 60, 115, chain="a_chain", sibling="s_sibli", crew_in_chain=False) + held = _watch_the_stack(monkeypatch) + + assert not folder_findings(top).over_limit + assert max(held) > 2_097_152 + + +def test_a_folder_total_exactly_at_the_bound_is_not_over( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + top = _short_top(tmp_path) + _write(top, {"agent.py": CREW, **{f"folder_{index:03}/x": "" for index in range(3)}}) + # Three 10-character folders below a top of at most 184 ASCII characters. + exact = 3 * ((184 + 1 + 10) + 120) + + monkeypatch.setattr(fd, "_MAX_FOLDER_BYTES", exact) + assert not folder_findings(top).over_limit + monkeypatch.setattr(fd, "_MAX_FOLDER_BYTES", exact - 1) + assert folder_findings(top).over_limit + + +def test_a_folder_total_below_the_top_is_exact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Below depth 1 the path below the top is joined with a "/" at each level: a 10-character folder + whose three 10-character subfolders sit at depth 2, each 21 characters below the top.""" + top = _short_top(tmp_path) + _write(top, {"agent.py": CREW, **{f"d000000001/folder_{index:03}/x": "" for index in range(3)}}) + exact = ((184 + 1 + 10) + 120) + 3 * ((184 + 1 + 21) + 120) + + monkeypatch.setattr(fd, "_MAX_FOLDER_BYTES", exact) + assert not folder_findings(top).over_limit + monkeypatch.setattr(fd, "_MAX_FOLDER_BYTES", exact - 1) + assert folder_findings(top).over_limit + + +def test_an_installed_tree_adds_none_of_its_folders(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A folder holding a .dist-info is an installed tree: the walk enters none of its folders, so they + add nothing. What is counted: the top's listing, "pkg" and "zzz", then "zzz"'s one folder, whose + listing comes after "pkg" and is where a total that counted "pkg"'s folders would be seen.""" + top = _short_top(tmp_path) + _write( + top, + { + "agent.py": CREW, + **{f"pkg/{name}/x": "" for name in ("x-1.0.dist-info", "alpha", "beta", "gamma")}, + "zzz/c/x": "", + }, + ) + exact = 2 * ((184 + 1 + 3) + 120) + ((184 + 1 + 5) + 120) + monkeypatch.setattr(fd, "_MAX_FOLDER_BYTES", exact) + + assert not folder_findings(top).over_limit diff --git a/tests/harness/test_hosted_authoring_entrypoint.py b/tests/harness/test_hosted_authoring_entrypoint.py index f4a7490da..f89a12245 100644 --- a/tests/harness/test_hosted_authoring_entrypoint.py +++ b/tests/harness/test_hosted_authoring_entrypoint.py @@ -239,3 +239,102 @@ async def forbidden(*args, **kwargs): worker.join(timeout=5) assert not worker.is_alive() assert not (output / "generic-harness").exists() + + +def _subfolder_job(tmp_path: Path, subfolder: str) -> Path: + from fi.alk.harness.job import HarnessJob + + job = HarnessJob( + job_id="job", + run_id="run", + execution="hosted", + source={ + "kind": "github", + "visibility": "public", + "repository": "acme/agent", + "commit_sha": "a" * 40, + "subfolder": subfolder, + }, + runtime={"isolation": "dedicated_vm"}, + agent={"connector": "auto"}, + ) + path = tmp_path / "job.json" + path.write_text(job.model_dump_json(), encoding="utf-8") + return path + + +def _watch_authoring(monkeypatch): + from fi.alk.harness import authoring_entrypoint as authoring + + calls: list[str] = [] + + def compose(*_args, **_kwargs): + calls.append("compose") + + async def auto(_namespace) -> int: + calls.append("auto") + return 0 + + def await_harness(_output) -> int: + calls.append("await") + return 0 + + monkeypatch.setattr(authoring, "initialize_hosted_compose", compose) + monkeypatch.setattr(authoring, "_auto", auto) + monkeypatch.setattr(authoring, "_await_harness", await_harness) + monkeypatch.setattr(authoring, "_persist_authored_scenario_count", lambda *a: None) + return authoring, calls + + +def _run_authoring(authoring, tmp_path: Path, subfolder: str, *flags: str) -> int: + source = tmp_path / "source" + source.mkdir(exist_ok=True) + (source / "agents" / "a").mkdir(parents=True, exist_ok=True) + (source / "agents" / "a" / "agent.py").write_text("x = 1\n") + return authoring.main( + [ + str(_subfolder_job(tmp_path, subfolder)), + "--source", + str(source), + "--output", + str(tmp_path / "authoring"), + *flags, + ] + ) + + +def _written_failure(tmp_path: Path) -> dict: + return json.loads((tmp_path / "authoring" / "failure.json").read_text()) + + +def test_bad_subfolder_writes_failure_and_exits_78_await_mode(tmp_path, monkeypatch): + authoring, calls = _watch_authoring(monkeypatch) + + assert _run_authoring(authoring, tmp_path, "agents/missing", "--await-harness") == 78 + + failure = _written_failure(tmp_path) + assert {key: failure[key] for key in ("domain", "stage", "code", "retryable")} == { + "domain": "agent", + "stage": "acquiring_source", + "code": "subfolder_not_found", + "retryable": False, + } + assert calls == [] + + +def test_bad_subfolder_writes_failure_and_exits_78_auto_mode(tmp_path, monkeypatch): + authoring, calls = _watch_authoring(monkeypatch) + + assert _run_authoring(authoring, tmp_path, "agents/missing") == 78 + + assert _written_failure(tmp_path)["code"] == "subfolder_not_found" + assert calls == [] + + +def test_good_subfolder_continues(tmp_path, monkeypatch): + authoring, calls = _watch_authoring(monkeypatch) + + assert _run_authoring(authoring, tmp_path, "agents/a") == 0 + + assert calls == ["compose", "auto"] + assert not (tmp_path / "authoring" / "failure.json").exists() diff --git a/tests/harness/test_hosted_conversation.py b/tests/harness/test_hosted_conversation.py index 16ef4da67..a7e3a6a6d 100644 --- a/tests/harness/test_hosted_conversation.py +++ b/tests/harness/test_hosted_conversation.py @@ -2,6 +2,7 @@ import asyncio import json +from types import SimpleNamespace from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any @@ -679,6 +680,108 @@ async def handle(command): assert "call finish_build" in handled[0] +def test_open_for_building_passes_subfolder(tmp_path, monkeypatch): + policy_path = tmp_path / "policy.yaml" + policy_path.write_text( + "schema_version: 1\nharness_chat:\n enabled: true\n", encoding="utf-8" + ) + source = tmp_path / "source" + (source / "agents" / "a").mkdir(parents=True) + (source / "agents" / "a" / "agent.py").write_text("x = 1\n") + (source / "agents" / "b").mkdir() + (source / "agents" / "b" / "agent.py").write_text("x = 2\n") + opened: dict[str, Any] = {} + monkeypatch.setattr( + "fi.alk.harness.builder.open_for_building", + lambda contract, **kwargs: opened.update(kwargs), + ) + monkeypatch.setattr(HostedChatRuntime, "_builder_platform_server", lambda self: None) + monkeypatch.setattr(HostedChatRuntime, "_hosted_prompt", lambda self: "") + runtime = HostedChatRuntime( + client=SimpleNamespace(turn_context={}), # type: ignore[arg-type] + policy=load_chat_policy(policy_path), + workspace=tmp_path, + source=source, + # A job that doesn't validate, as when the platform sends a field this build doesn't know. + job={ + "metadata": {"agent_name": "demo"}, + "scenario_count": 1, + "source": {"kind": "github", "subfolder": "agents/a"}, + }, + ) + + runtime._open_for_building(None) + + assert opened["source"].subfolder == "agents/a" + briefing = opened["source"].briefing() + assert "agents/a/agent.py" in briefing and "agents/b/agent.py" not in briefing + + +def test_nudge_world_checks_cite_only_the_recorded_folder(tmp_path): + from fi.alk.harness.backends import SessionSpec, tool, tool_server + from fi.alk.harness.world.tools import WORLD_SERVER + + policy_path = tmp_path / "policy.yaml" + policy_path.write_text( + "schema_version: 1\nharness_chat:\n enabled: true\n", encoding="utf-8" + ) + source = tmp_path / "source" + for name in ("a", "b"): + (source / "agents" / name).mkdir(parents=True) + (source / "agents" / name / "seed.sql").write_text( + "INSERT INTO menu VALUES ('fries');\n" + ) + workspace = tmp_path / "work" + workspace.mkdir() + (workspace / "contract.json").write_text( + json.dumps( + {"agent": "demo", "runtime": {"source_subfolder": "agents/a", "workdir": "agents/a"}} + ), + encoding="utf-8", + ) + runtime = HostedChatRuntime( + client=SimpleNamespace(turn_context={}), # type: ignore[arg-type] + policy=load_chat_policy(policy_path), + workspace=workspace, + source=source, + job={"metadata": {"agent_name": "demo"}, "scenario_count": 1}, + ) + runtime._nudge = {"nudge_id": "n1", "stage": "environment"} + + @tool("add_world_check", "Add one check.", {}) + async def add_world_check(args: dict) -> dict: + return {"content": []} + + configured: list[SessionSpec] = [] + stage = SimpleNamespace( + spec=SessionSpec( + system_prompt="", + servers={WORLD_SERVER: tool_server(name=WORLD_SERVER, tools=[add_world_check])}, + ), + kept=None, + configure=configured.append, + ) + + runtime._nudge_tools(stage, None) # type: ignore[arg-type] + + check = next(one for one in configured[0].servers[WORLD_SERVER].tools) + + def cite(path: str) -> dict: + return asyncio.run( + check.handler( + { + "name": "menu", + "code": "def check(world): return None", + "source_path": path, + "source_quote": "INSERT INTO menu VALUES ('fries');", + } + ) + ) + + assert cite("agents/b/seed.sql").get("is_error") + assert not cite("agents/a/seed.sql").get("is_error") + + def test_a_file_that_shrinks_while_the_workspace_is_packed_does_not_fail_the_pack( tmp_path, monkeypatch ): diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py index 1d3170301..ff45917a2 100644 --- a/tests/harness/test_hosted_entrypoint.py +++ b/tests/harness/test_hosted_entrypoint.py @@ -3578,9 +3578,15 @@ def test_post_terminal_wire_block_is_bounded_by_the_remaining_flush_window() -> # `emit_skipped_receipts`/`receipt()` push with `deadline=None` -- a degraded-but-alive # events channel could previously retry every skipped scenario's receipt for the full # `RetryPolicy` budget regardless of how much of the flush window was already spent, well past - # the point the gateway tears the sandbox down. A tiny window plus a slow - # `emit_skipped_receipts` reproduces the shape without a real multi-second stall dominating the - # suite: `asyncio.wait_for`'s own cancellation cuts the sleep short well before it would run out. + # the point the gateway tears the sandbox down. Here `emit_skipped_receipts` never finishes on + # its own (the stall stands in for the receipt push), so the run can end only when the flush + # window's `wait_for` cancels it. The test checks the bound itself, not elapsed time: a spy on + # `wait_for` records the `timeout` the code passes with `emit_skipped_receipts`, and that must + # be the REMAINING window -- `deadline - now`, where the deadline is the arm time plus the + # window and the terminal's own flush has already run since arming. So the tightest true bound + # is `0 < timeout < window`: `None`, a padded timeout, or the full window (which would not + # shrink as the window is spent) all fail, whatever the runner's speed. The outer guard turns + # a missing bound into a failure rather than a hang. async def scenario() -> None: scenarios = [ FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), @@ -3589,27 +3595,47 @@ async def scenario() -> None: harness = _build_harness( scenarios=scenarios, cancel_on_scenario="first", instances=1 ) - harness.deps.flush_window_seconds = 0.2 + harness.deps.flush_window_seconds = 2.0 original_emit = HostedScheduler.emit_skipped_receipts - - async def slow_emit(self: HostedScheduler, result: RunResult) -> None: - await asyncio.sleep(2.0) - await original_emit(self, result) - - HostedScheduler.emit_skipped_receipts = slow_emit # type: ignore[method-assign] - started = time.monotonic() + never = asyncio.Event() + stall: dict[str, float] = {} + bounds: list[float | None] = [] + real_wait_for = asyncio.wait_for + + async def stalled_emit(self: HostedScheduler, result: RunResult) -> None: + try: + await never.wait() + except asyncio.CancelledError: + stall["cut"] = time.monotonic() + raise # swallowing the cancel would break the bound under test + + async def spying_wait_for(aw: Any, timeout: float | None = None) -> Any: + # `he.asyncio` is the shared asyncio module, so this also sees the outer guard below; + # record only the call that wraps the stalled receipts. + code = getattr(aw, "cr_code", None) + if code is stalled_emit.__code__: + bounds.append(timeout) + return await real_wait_for(aw, timeout=timeout) + + HostedScheduler.emit_skipped_receipts = stalled_emit # type: ignore[method-assign] try: - code = await he.run_job( - harness.job_path, harness.source, harness.output, deps=harness.deps - ) + try: + with mock.patch.object(he.asyncio, "wait_for", spying_wait_for): + code = await real_wait_for( + he.run_job( + harness.job_path, + harness.source, + harness.output, + deps=harness.deps, + ), + timeout=30.0, + ) + except asyncio.TimeoutError: + pytest.fail("the post-terminal wire block was not bounded by the flush window") finally: HostedScheduler.emit_skipped_receipts = original_emit - elapsed = time.monotonic() - started - assert elapsed < 1.5, ( - f"post-terminal wire block was not bounded: {elapsed:.2f}s" - ) assert ( code == he.EXIT_OK ) # the terminal was already delivered before the window ran out @@ -3621,9 +3647,15 @@ async def slow_emit(self: HostedScheduler, result: RunResult) -> None: key[1]: body["status"] for key, body in harness.transport.receipts.items() } assert statuses.get("first") == "passed" - assert ( - "second" not in statuses - ) # window ran out before the skipped receipt could go + + assert "cut" in stall, "the stalled receipts were never cancelled" + window = harness.deps.flush_window_seconds + # Exactly one bounded wait, and its timeout is the remaining window: positive, and + # strictly under the full window because the terminal flush ran after the window armed. + assert len(bounds) == 1, bounds + bound = bounds[0] + assert bound is not None, "the receipts wait was given no timeout" + assert 0 < bound < window, (bound, window) assert ( harness.provisioner.closed is True @@ -5231,3 +5263,287 @@ def world(path): assert selected == ["broker", "observation"] assert harness.provisioner.provision_calls == 1 assert harness.provisioner.closed + + +def test_bad_subfolder_fails_terminal_before_compose(monkeypatch): + from fi.alk.harness.source_folder import subfolder_not_found + + harness = _build_harness(scenarios=[]) + job = _job().model_copy( + update={ + "source": RepositorySource( + kind=SourceKind.GITHUB, + repository="org/repo", + visibility=SourceVisibility.PUBLIC, + commit_sha="a" * 40, + subfolder="agents/missing", + ) + } + ) + _write_job(harness.job_path, job) + started: list[str] = [] + monkeypatch.setattr( + he, "initialize_hosted_compose", lambda job, source: started.append("compose") + ) + + assert _run(harness) == he.EXIT_OK + + assert started == [] + assert harness.provisioner.provision_calls == 0 + terminal = harness.transport.terminal_events() + assert len(terminal) == 1 + assert terminal[0]["payload"]["failure"] == { + "domain": "agent", + "stage": "acquiring_source", + "code": "subfolder_not_found", + "message": subfolder_not_found("agents/missing").message, + } + + +def _harness_with_a_packaged_agent( + packaged_main: bytes = b"from crewai import Crew\nCrew()\n", + source_origin: str = "bundle", +) -> Harness: + real_body = _base_manifest_body + + def packaged_body() -> dict[str, Any]: + body = real_body() + body["processes"][1]["working_directory"] = "generated/source/agent" + body["processes"][1]["source_origin"] = source_origin + return body + + files = { + **_scenario_doc_files( + "passing", + scenario_key="passing", + scenario_id="platform-passing", + sub_goals=["holds"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ), + "contract.json": json.dumps({"agent": "crew-bot", "tools": []}).encode("utf-8"), + "generated/source/agent/main.py": packaged_main, + } + with mock.patch.dict(globals(), {"_base_manifest_body": packaged_body}): + return _build_harness( + scenarios=[], + instances=1, + use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, files + ), + ) + + +def _posted_framework(harness: Harness) -> dict[str, Any]: + [document] = [ + body["agent"]["agent"] + for _, body in harness.transport.scenarios_calls + if "agent" in body + ] + return document["extension"]["framework"] + + +def _install_crewai(package_dir: Path) -> None: + dist_info = ( + package_dir + / ".venv" + / "lib" + / "python3.12" + / "site-packages" + / "crewai-0.86.0.dist-info" + ) + dist_info.mkdir(parents=True) + (dist_info / "METADATA").write_bytes( + b"Metadata-Version: 2.1\nName: crewai\nVersion: 0.86.0\n" + ) + + +def _framework_log_lines(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.getMessage().startswith("framework") + ] + + +def test_run_job_gives_bundle_source_its_folders(caplog) -> None: + async def scenario() -> None: + harness = _harness_with_a_packaged_agent() + _install_crewai(harness.work / "build" / "agent") + + with mock.patch.object( + he, "framework_folders_for", wraps=he.framework_folders_for + ) as folders_for: + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + + assert code == he.EXIT_OK + assert _posted_framework(harness) == {"name": "crewai", "version": "0.86.0"} + assert folders_for.call_args.kwargs["source"] == harness.source + lines = _framework_log_lines(caplog) + assert any("name=crewai" in line for line in lines) + assert not any("0.86.0" in line for line in lines) + + with caplog.at_level("INFO", logger="fi.alk.harness"): + asyncio.run(scenario()) + + +def test_run_job_gives_repository_source_its_folders(caplog) -> None: + async def scenario() -> None: + # The crew only in `source`: scanning the bundle's copy would find none. + harness = _harness_with_a_packaged_agent( + packaged_main=b"print('plain')\n", source_origin="repository" + ) + agent = harness.source / "generated" / "source" / "agent" + agent.mkdir(parents=True) + (agent / "main.py").write_text( + "from crewai import Crew\nCrew()\n", encoding="utf-8" + ) + _install_crewai(harness.work / "build" / "agent") + + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + + assert code == he.EXIT_OK + assert _posted_framework(harness) == {"name": "crewai", "version": "0.86.0"} + + with caplog.at_level("INFO", logger="fi.alk.harness"): + asyncio.run(scenario()) + + +def test_folder_step_failure_keeps_the_run_going(caplog) -> None: + secret = "/customer/private/path" + + async def scenario() -> None: + # The crew only in `source`, and a version where each wrong fallback would read + # one: the fallback is the source, with no build tree. + harness = _harness_with_a_packaged_agent(packaged_main=b"print('plain')\n") + (harness.source / "main.py").write_text( + "from crewai import Crew\nCrew()\n", encoding="utf-8" + ) + _install_crewai(harness.work / "build" / "agent") + _install_crewai(harness.work) + + with mock.patch.object( + he, "framework_folders_for", side_effect=RuntimeError(secret) + ): + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + + assert code == he.EXIT_OK + assert _posted_framework(harness) == {"name": "crewai", "version": None} + assert len(harness.transport.terminal_events()) == 1 + lines = _framework_log_lines(caplog) + assert any("class=RuntimeError" in line for line in lines) + assert not any(secret in line for line in lines) + + with caplog.at_level("INFO", logger="fi.alk.harness"): + asyncio.run(scenario()) + + +# -- TH-9491: URL credentials never reach the judge or the job's events ------------------------- + +_TH9491_URL = "https://bot:hunter2@hooks.example/v1?key=sk-1#frag" + + +@pytest.mark.parametrize("where", ["job_prompt", "excerpt", "summary", "rule"]) +def test_the_judge_text_loses_url_credentials_wherever_the_url_is( + tmp_path: Path, where: str +) -> None: + """B4: the job's own prompt, the excerpt, the summary or a rule.""" + said = f"Post to {_TH9491_URL} after." + body = { + "system_prompt_excerpt": said if where == "excerpt" else "Take orders.", + "one_liner": said if where == "summary" else "Takes orders", + "hard_constraints": [said if where == "rule" else "Be brief"], + } + (tmp_path / "contract.json").write_text(json.dumps(body), encoding="utf-8") + config = {"target_system_prompt": said} if where == "job_prompt" else {} + job = SimpleNamespace(agent=SimpleNamespace(config=config)) + + text = he._agent_instructions(job, tmp_path) + + assert "Post to https://hooks.example/v1 after." in text + assert "hunter2" not in text and "sk-1" not in text + + +def test_activity_events_in_the_job_response_lose_url_credentials(tmp_path, monkeypatch): + """B16 (plan v5.1 §W8a): harness.activity events, which /v1/jobs/{id} returns.""" + from fi.alk.harness import cli + from fi.alk.harness.session import Event + + source = tmp_path / "source" + source.mkdir() + output = tmp_path / "output" + + async def reading(_args): + cli._render_observable( + Event( + kind="tool", + text=f"grep found: hook = '{_TH9491_URL}'", + tool="grep", + detail={"stage": "understand", "target": _TH9491_URL, "path": "agent.py"}, + ) + ) + return 78 + + monkeypatch.setattr(cli, "_understand", reading) + monkeypatch.setattr("fi.alk.harness.provision.stop", lambda _: None) + monkeypatch.setattr(cli, "_AUTO_EVENT_SINK", None) + args = SimpleNamespace( + path=str(source), + out=str(output), + name="fixture", + kind="repo", + count=2, + model=None, + authoring_only=True, + job=HarnessJob( + job_id="events", + run_id="events", + execution="local", + source={"kind": "local_repository", "local_path": str(source)}, + agent={"connector": "auto"}, + scenario_count=2, + metadata={"generic_harness_v1": True}, + ), + runtime_readiness=None, + ) + + assert asyncio.run(cli._auto(args)) == 78 + + events = (output / "harness-events.jsonl").read_text(encoding="utf-8") + [activity] = [ + json.loads(line) + for line in events.splitlines() + if json.loads(line).get("type") == "harness.activity" + ] + payload = activity["payload"] + assert payload["text"] == "grep found: hook = 'https://hooks.example/v1" + assert payload["detail"] == {"target": "https://hooks.example/v1", "path": "agent.py"} + assert "hunter2" not in events and "sk-1" not in events + + +def test_the_activity_printed_to_stdout_loses_url_credentials(capsys, monkeypatch): + """Change review r1, A-H1: the platform keeps the tail of the guest's stdout.""" + from fi.alk.harness import cli + from fi.alk.harness.session import Event + + monkeypatch.setattr(cli, "_AUTO_EVENT_SINK", None) + cli._render_observable( + Event(kind="tool", tool="grep", detail={"stage": "understand", "target": _TH9491_URL}) + ) + cli._render_observable( + Event(kind="result", text=f"agent.py:3: hook = {_TH9491_URL}\nagent.py:4: ok") + ) + cli._render_observable(Event(kind="text", text=f"The agent posts to {_TH9491_URL} after.")) + + printed = capsys.readouterr().out + assert "hunter2" not in printed and "bot:" not in printed + assert "sk-1" not in printed and "key=" not in printed and "frag" not in printed + assert " [grep https://hooks.example/v1" in printed + assert " > agent.py:3: hook = https://hooks.example/v1\n > agent.py:4: ok" in printed + assert "The agent posts to https://hooks.example/v1 after." in printed diff --git a/tests/harness/test_import_roots.py b/tests/harness/test_import_roots.py new file mode 100644 index 000000000..e6343ea12 --- /dev/null +++ b/tests/harness/test_import_roots.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import importlib +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import fi.alk +from fi.alk.harness import chat_call_runner +from fi.alk.harness.contract import AgentContract, Runtime +from fi.alk.harness.world.runtime import GeneratedWorld +from fi.alk.harness.world.snapshot import save +from fi.alk.harness.world.tools import world_tools + + +@pytest.fixture +def repo(tmp_path: Path, monkeypatch) -> Path: + root = tmp_path / "repo" + for name, value in (("a", 1), ("b", 2)): + (root / "agents" / name).mkdir(parents=True) + (root / "agents" / name / "helper_flat.py").write_text(f"VALUE = {value}\n") + # Same flat name at the repository root: the agent's own folder has to win. + (root / "helper_flat.py").write_text("VALUE = 9\n") + monkeypatch.setattr(sys, "path", list(sys.path)) + yield root + for name in [name for name in sys.modules if name.split(".")[0] in {"helper_flat", "agents"}]: + sys.modules.pop(name, None) + + +def _contract(subfolder: str) -> AgentContract: + return AgentContract( + agent="demo", + runtime=Runtime(source_subfolder=subfolder, workdir=subfolder) if subfolder else None, + ) + + +class _Store: + key = "fake" + + def start(self) -> None: + pass + + +def test_import_roots_equal_at_build_certification_and_run(tmp_path, repo, monkeypatch): + reached: list[tuple[str, str]] = [] + real = GeneratedWorld.reach + + def spy(self, source_root, subfolder=""): + reached.append((source_root, subfolder)) + real(self, source_root, subfolder) + + monkeypatch.setattr(GeneratedWorld, "reach", spy) + monkeypatch.setattr(chat_call_runner, "_HostedToolStore", lambda dsn: _Store()) + contract = _contract("agents/a") + + world_tools(contract, tmp_path / "build", source_root=str(repo), deferred_runtime=True) + runtime = SimpleNamespace( + metadata={}, + endpoints={"world_db": SimpleNamespace(protocol="postgres", address="unused")}, + ) + chat_call_runner._tool_world(tmp_path / "bundle", contract, runtime, repo) + + assert reached == [(str(repo), "agents/a")] * 2 + + +def test_agent_module_imports_flat_and_qualified(repo): + world = GeneratedWorld(":memory:", kind="sqlite") + + world.reach(str(repo), "agents/a") + + path = [Path(item) for item in sys.path] + assert path.index(repo / "agents" / "a") < path.index(repo) + assert importlib.import_module("helper_flat").VALUE == 1 + assert importlib.import_module("agents.b.helper_flat").VALUE == 2 + + +def test_agent_package_folder_comes_before_its_parent(repo): + (repo / "agents" / "a" / "__init__.py").write_text("") + (repo / "agents" / "helper_flat.py").write_text("VALUE = 7\n") + + GeneratedWorld(":memory:", kind="sqlite").reach(str(repo), "agents/a") + + path = [Path(item) for item in sys.path] + assert path.index(repo / "agents" / "a") < path.index(repo / "agents") + assert importlib.import_module("helper_flat").VALUE == 1 + + +def test_no_subfolder_import_roots_unchanged(tmp_path, monkeypatch): + monkeypatch.setattr(sys, "path", list(sys.path)) + package = tmp_path / "repo" / "pkg" + inner = package / "inner" + inner.mkdir(parents=True) + (package / "__init__.py").write_text("") + (inner / "__init__.py").write_text("") + + GeneratedWorld(":memory:", kind="sqlite").reach(str(inner)) + + assert [Path(item) for item in sys.path[:3]] == [tmp_path / "repo", package, inner] + + +def test_new_process_after_restore_imports_flat(tmp_path, repo): + world = GeneratedWorld(":memory:", kind="sqlite") + world.name = "demo" + world.reach(str(repo), "agents/a") + snapshot = save(world, tmp_path / "snapshot") + source = Path(fi.alk.__file__).resolve().parents[2] + script = f""" +import sys +sys.path.insert(0, {str(source)!r}) +from fi.alk.harness.world.runtime import GeneratedWorld +from fi.alk.harness.world.snapshot import restore + +GeneratedWorld(":memory:", kind="sqlite").reach({str(repo)!r}) +restored = restore({str(snapshot)!r}) +restored.reach({str(repo)!r}) +import helper_flat +print(restored.source_subfolder, helper_flat.VALUE) +""" + + said = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, check=True + ) + + assert said.stdout.split() == ["agents/a", "1"] diff --git a/tests/harness/test_instrumentation_evidence.py b/tests/harness/test_instrumentation_evidence.py index 5ef250c5b..caccef2d7 100644 --- a/tests/harness/test_instrumentation_evidence.py +++ b/tests/harness/test_instrumentation_evidence.py @@ -6,7 +6,9 @@ import pytest +from fi.alk.harness import instrumentation_evidence, python_source from fi.alk.harness.instrumentation_evidence import detect_instrumentation +from fi.alk.harness.python_source import LARGEST_PARSED_MODULE from fi.alk.harness.tools import accept_contract RECORDED_CONTRACT = ( @@ -355,3 +357,159 @@ def parse(text: str, *args: object, **kwargs: object) -> ast.AST: monkeypatch.setattr(ast, "parse", parse) assert detect_instrumentation(source) == "traceai" + + +def test_a_link_to_a_file_without_end_is_read_only_to_a_limit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import io + import os + + source = _repository( + tmp_path, {"agent.py": "x = 1\n", "zero.py": "", "big.py": ""} + ) + os.mkfifo(source / "pipe.py") + + class Endless(io.RawIOBase): + def readable(self) -> bool: + return True + + def read(self, size: int = -1) -> bytes: + assert size >= 0, "read to the end" + return b"#" * size + + opened = Path.open + + def open_(path: Path, *args: object, **kwargs: object): + # Like a link to /dev/zero; and a pipe must never be opened, as that waits for a writer. + assert path.name != "pipe.py", "opened a pipe" + return Endless() if path.name == "zero.py" else opened(path, *args, **kwargs) + + monkeypatch.setattr(Path, "open", open_) + # Past the limit a file is skipped, as one that doesn't parse is. + (source / "big.py").write_text(_padded("import traceai_openai\n", LARGEST_PARSED_MODULE + 1)) + + assert detect_instrumentation(source) == "none" + + +def _padded(line: str, size: int) -> str: + return line + "#" * (size - len(line.encode())) + + +@pytest.mark.parametrize( + "size, found", [(LARGEST_PARSED_MODULE, "traceai"), (LARGEST_PARSED_MODULE + 1, "none")] +) +def test_a_file_is_parsed_up_to_the_limit_and_skipped_past_it( + tmp_path: Path, size: int, found: str +) -> None: + source = _repository(tmp_path, {"agent.py": _padded("import traceai_openai\n", size)}) + + assert detect_instrumentation(source) == found + + +def test_only_files_naming_what_the_scan_looks_for_are_parsed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = _repository( + tmp_path, + { + "a.py": "import os\n", + "b.py": "import traceai_openai\n", + "c.py": "from fi_instrumentation import register\n", + "d.py": "from opentelemetry.sdk.trace import TracerProvider\n", + }, + ) + seen: list[str] = [] + real = instrumentation_evidence.parsed + monkeypatch.setattr( + instrumentation_evidence, "parsed", lambda text: seen.append(text) or real(text) + ) + + assert detect_instrumentation(source) == "traceai" + assert sorted(seen) == [ + "from fi_instrumentation import register\n", + "from opentelemetry.sdk.trace import TracerProvider\n", + "import traceai_openai\n", + ] + + +@pytest.mark.parametrize("total, found", [(22, "traceai"), (21, "none")]) +def test_the_scan_parses_only_up_to_its_total( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, total: int, found: str +) -> None: + source = _repository(tmp_path, {"agent.py": "import traceai_openai\n"}) + monkeypatch.setattr(python_source, "LARGEST_PARSED_TOTAL", total) + + assert detect_instrumentation(source) == found + + +def test_a_file_that_runs_the_parser_out_of_memory_counts_for_nothing(tmp_path: Path) -> None: + # The parser's own stack overflows on this, as MemoryError, on 3.11 and 3.12. + source = _repository( + tmp_path, + {"agent.py": "import traceai_openai\nx = " + "-" * 7000 + "1\n", "b.py": "import os\n"}, + ) + with pytest.raises(MemoryError): + ast.parse((source / "agent.py").read_text()) + + assert detect_instrumentation(source) == "none" + + +@pytest.mark.parametrize("listed", [list, lambda names: list(reversed(names))]) +def test_past_the_total_the_answer_does_not_depend_on_how_the_disk_lists_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, listed +) -> None: + import os + + source = _repository( + tmp_path, + { + "a/agent.py": "import traceai_openai\n", + "a/b.py": "# opentelemetry, someday\n", + "z/agent.py": "# opentelemetry, someday\n", + }, + ) + walk = os.walk + + def listed_walk(root): + for directory, subdirectories, files in walk(root): + subdirectories[:] = listed(sorted(subdirectories)) + yield directory, subdirectories, listed(sorted(files)) + + monkeypatch.setattr(instrumentation_evidence.os, "walk", listed_walk) + monkeypatch.setattr(python_source, "LARGEST_PARSED_TOTAL", 30) + + assert detect_instrumentation(source) == "traceai" + + +def test_a_file_with_a_byte_order_mark_is_read_as_python_reads_it(tmp_path: Path) -> None: + source = tmp_path + (source / "agent.py").write_bytes(b"\xef\xbb\xbfimport traceai_openai\n") + + assert detect_instrumentation(source) == "traceai" + + +def test_a_file_with_a_byte_order_mark_and_a_coding_line_is_read(tmp_path: Path) -> None: + (tmp_path / "agent.py").write_bytes(b"\xef\xbb\xbf# coding: utf-8\nimport traceai_openai\n") + + assert detect_instrumentation(tmp_path) == "traceai" + + +@pytest.mark.parametrize("coding", ["rot13", "nosuch"]) +def test_a_coding_line_python_cant_read_leaves_the_file_unread(tmp_path: Path, coding) -> None: + (tmp_path / "agent.py").write_bytes(f"# coding: {coding}\nimport traceai_openai\n".encode()) + + assert detect_instrumentation(tmp_path) != "traceai" + + +def test_a_fullwidth_name_is_a_known_limit(tmp_path: Path) -> None: + # Python folds `traceai_openai` to traceai_openai; the scan matches names as written. + source = _repository(tmp_path, {"agent.py": "import traceai_openai\n"}) + + assert detect_instrumentation(source) != "traceai" + + +def test_an_import_spelled_in_escapes_is_found_under_unicode_escape(tmp_path: Path) -> None: + # Python decodes the file before it reads it, so this runs `import traceai_openai`. + (tmp_path / "agent.py").write_bytes(b"# coding: unicode_escape\nimport \\x74raceai_openai\n") + assert detect_instrumentation(tmp_path) == "traceai" diff --git a/tests/harness/test_nudge_turn.py b/tests/harness/test_nudge_turn.py index 2aed04058..f5e353a8c 100644 --- a/tests/harness/test_nudge_turn.py +++ b/tests/harness/test_nudge_turn.py @@ -797,6 +797,95 @@ def test_a_seeded_table_with_no_key_is_held_only_to_having_rows(tmp_path): assert nudge.seed_row_problems(root, source) == [] +def _two_agents(tmp_path: Path) -> Path: + """A repository of two agents, each seeding a different booking.""" + source = tmp_path / "source" + for name, ref in (("a", "A_REF"), ("b", "B_REF")): + (source / "agents" / name / "db").mkdir(parents=True) + (source / "agents" / name / "db" / "seed.sql").write_text( + f"INSERT INTO bookings (ref, status) VALUES ('{ref}', 'matched');\n", + encoding="utf-8", + ) + return source + + +def _record_folder(root: Path, folder: str) -> None: + path = root / "contract.json" + body = json.loads(path.read_text(encoding="utf-8")) + body["runtime"] = ( + {"source_subfolder": folder, "workdir": folder} if folder else None + ) + path.write_text(json.dumps(body), encoding="utf-8") + + +def test_seed_row_problems_ignore_a_siblings_seed(tmp_path): + root = _with_bookings(_cart(tmp_path / "work"), "A_REF") + source = _two_agents(tmp_path) + + [whole] = nudge.seed_row_problems(root, source) + assert whole["item"] == "bookings" and "B_REF" in whole["message"] + + _record_folder(root, "agents/a") + assert nudge.seed_row_problems(root, source) == [] + + +def test_a_world_check_cannot_cite_a_siblings_file(tmp_path): + source = _two_agents(tmp_path) + added: list[dict] = [] + + @tool("add_world_check", "Add one check.", {}) + async def add_world_check(args: dict) -> dict: + added.append(args) + return {"content": []} + + def cite(path: str) -> dict: + return { + "name": "bookings", + "code": "def check(world): return None", + "source_path": path, + "source_quote": "INSERT INTO bookings (ref, status)", + } + + server = nudge.track_world( + tool_server(name="world", tools=[add_world_check]), {}, source, "agents/a" + ) + assert _call(server, "add_world_check", cite("agents/b/db/seed.sql"))["is_error"] + assert added == [] + assert not _call(server, "add_world_check", cite("agents/a/db/seed.sql")).get( + "is_error" + ) + + +def test_an_invariant_revision_cannot_cite_a_siblings_file(tmp_path): + root, _ = _pinned_review(tmp_path) + source = _two_agents(tmp_path / "repo") + args = { + "name": "cart_starts_empty", + "violations_sql": "SELECT item_id FROM cart WHERE item_id = 'x'", + "source_path": "agents/b/db/seed.sql", + "source_quote": "INSERT INTO bookings (ref, status)", + } + _record_folder(root, "agents/a") + before = (root / ARTIFACT).read_bytes() + + assert nudge.revise_invariant(root, args, source).startswith("Not changed") + assert (root / ARTIFACT).read_bytes() == before + + # Its own file is cited, and recorded, as it is with no folder. + own = {**args, "source_path": "agents/a/db/seed.sql"} + assert nudge.revise_invariant(root, own, source) == "revised, cites agents/a/db/seed.sql" + revised = next( + check + for check in json.loads((root / ARTIFACT).read_text(encoding="utf-8"))["checks"] + if check["name"] == "cart_starts_empty" + ) + assert revised["violations_sql"] == args["violations_sql"] + assert [one["path"] for one in revised["evidence"]] == ["agents/a/db/seed.sql"] + + _record_folder(root, "") + assert nudge.revise_invariant(root, args, source) == "revised, cites agents/b/db/seed.sql" + + def test_a_contract_naming_a_removed_tool_is_cleaned_and_warned(tmp_path): from fi.alk.harness.contract import Dependency, ToolEntry diff --git a/tests/harness/test_python_source_codecs.py b/tests/harness/test_python_source_codecs.py new file mode 100644 index 000000000..e096cfa4c --- /dev/null +++ b/tests/harness/test_python_source_codecs.py @@ -0,0 +1,1353 @@ +"""A Python file whose codec can spell a name in other bytes is read as Python reads it. + +Each row that says Python runs a name checks it with ``compile``, which is what an import runs. +""" + +from __future__ import annotations + +import ast +import codecs +import collections +import encodings +import encodings.aliases +import pkgutil +import string +import sys +import types +from pathlib import Path + +import pytest + +from fi.alk.harness import agent_type_check, python_source +from fi.alk.harness.agent_type_check import ( + browser_tools_found, + browser_tools_in_agent, + code_the_agent_reaches, +) +from fi.alk.harness.credentials import imported_by_agent +from fi.alk.harness.instrumentation_evidence import detect_instrumentation +from fi.alk.harness.python_source import ( + _RAW_SAFE_CODECS, + EscapeBudget, + _coding, + _decoded_first, + read_module, +) + +# Python reads a decoded CR as a line end, and a line it starts as blank, only from 3.12. +_FROM_312 = pytest.mark.skipif( + sys.version_info < (3, 12), reason="Python reads a decoded CR this way from 3.12" +) +_ISO2022 = ( + "iso2022_jp", + "iso2022_jp_1", + "iso2022_jp_2", + "iso2022_jp_2004", + "iso2022_jp_3", + "iso2022_jp_ext", + "iso2022_kr", +) +# The codecs Python accepts on a coding line that spell a name character from other bytes. +_SPELLING_CODECS = frozenset( + { + "raw-unicode-escape", + "unicode-escape", + "utf-7", + "hz", + "mac-arabic", + "mac-farsi", + *_ISO2022, + } +) +_UTF7 = b"# coding: utf-7\n" +_ESCAPED = b"# coding: unicode-escape\n" +_RAW_ESCAPED = b"# coding: raw-unicode-escape\n" + + +def _in(codec: str, body: bytes) -> bytes: + return f"# coding: {codec}\n".encode() + body + + +def _names_python_runs(raw: bytes) -> set[str]: + """The names the code compiled from ``raw`` uses, its functions' included.""" + names: set[str] = set() + pending = [compile(raw, "", "exec", dont_inherit=True)] + while pending: + code = pending.pop() + names.update(code.co_names) + pending.extend(c for c in code.co_consts if isinstance(c, types.CodeType)) + return names + + +def _imported(tree: ast.AST) -> set[str]: + """The top-level modules ``tree`` imports.""" + found: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + found.add(node.module.split(".")[0]) + return found + + +def _write(folder: Path, raw: bytes, name: str = "agent.py") -> Path: + folder.mkdir(parents=True, exist_ok=True) + path = folder / name + path.write_bytes(raw) + return path + + +def _shared(root: Path, agent: bytes) -> Path: + """A repository whose agent, in agents/a, may import common.web, which imports the names + both scans look for.""" + root = root.resolve() + _write(root / "agents" / "a", agent) + _write(root / "common", b"", "__init__.py") + _write(root / "common", b"import playwright\nimport traceai_openai\n", "web.py") + return root + + +def _scan(folder: Path, raw: bytes, name: str) -> object: + """What the reader that looks for ``name`` makes of a file holding ``raw``: whether the walk + reaches common/web.py, the instrumentation verdict, or the browser tools.""" + if name == "common.web": + repo = _shared(folder, raw) + return repo / "common" / "web.py" in imported_by_agent(repo, "agents/a") + _write(folder, raw) + if name.startswith("traceai_"): + return detect_instrumentation(folder) + return browser_tools_found(folder) + + +def _found(name: str) -> object: + if name == "common.web": + return True + return "traceai" if name.startswith("traceai_") else [name.replace("_", "-")] + + +def _missed(name: str) -> object: + if name == "common.web": + return False + return "none" if name.startswith("traceai_") else [] + + +@pytest.mark.parametrize( + "raw, name", + [ + pytest.param( + _RAW_ESCAPED + b"import \\u0070laywright\n", + "playwright", + id="raw-unicode-escape", + ), + pytest.param( + _ESCAPED + b"import \\x70laywright\n", "playwright", id="unicode-escape" + ), + pytest.param(_UTF7 + b"import +AHA-laywright\n", "playwright", id="utf-7"), + *( + pytest.param( + _in(codec, b"import play\x1b(Bwright\n"), "playwright", id=codec + ) + for codec in _ISO2022 + ), + pytest.param(_in("hz", b"import play~{~}wright\n"), "playwright", id="hz"), + pytest.param(_in("hz", b"import play~\nwright\n"), "playwright", id="hz-lf"), + pytest.param( + b"# coding: hz\r\nimport play~\r\nwright\r\n", "playwright", id="hz-crlf" + ), + pytest.param(b"# coding: hz\rimport play~\rwright\r", "playwright", id="hz-cr"), + # 0xDF is `_` here, and `playwright` before it would be found in the bytes. + pytest.param( + _in("mac-arabic", b"import undetected\xdfchromedriver\n"), + "undetected_chromedriver", + id="mac-arabic", + ), + pytest.param( + _in("mac-farsi", b"import undetected\xdfchromedriver\n"), + "undetected_chromedriver", + id="mac-farsi", + ), + ], +) +def test_a_browser_import_spelled_in_a_codec_that_can_spell_one_is_found( + tmp_path, raw, name +): + assert name in _names_python_runs(raw) + _write(tmp_path, raw) + assert browser_tools_found(tmp_path) == [name.replace("_", "-")] + + +@pytest.mark.parametrize( + "raw", + [ + pytest.param( + _RAW_ESCAPED + b"import \\u0074raceai_openai\n", id="raw-unicode-escape" + ), + pytest.param(_UTF7 + b"import +AHQ-raceai_openai\n", id="utf-7"), + pytest.param(_in("mac-arabic", b"import traceai\xdfopenai\n"), id="mac-arabic"), + pytest.param( + _in("iso2022_jp", b"import trace\x1b(Bai_openai\n"), id="iso2022_jp" + ), + pytest.param(_in("hz", b"import trace~{~}ai_openai\n"), id="hz"), + ], +) +def test_traceai_spelled_in_another_codec_is_found(tmp_path, raw): + assert "traceai_openai" in _names_python_runs(raw) + _write(tmp_path, raw) + assert detect_instrumentation(tmp_path) == "traceai" + + +@pytest.mark.parametrize( + "agent", + [ + # hz joins a `~` and the line end after it, once Python has made the CRLF an LF. + pytest.param(b"# coding: hz\r\nimport commo~\r\nn.web\r\n", id="hz-crlf"), + pytest.param(_ESCAPED + b"import \\x63ommon.web\n", id="unicode-escape"), + # Decoded whole, the NUL refuses the parse; Python reads up to it. + pytest.param(_UTF7 + b"import +AGM-ommon.web\n+AAA-\n", id="utf-7"), + ], +) +def test_the_walk_follows_an_import_spelled_in_another_codec(tmp_path, agent): + assert "common.web" in _names_python_runs(agent) + repo = _shared(tmp_path, agent) + assert repo / "common" / "web.py" in imported_by_agent(repo, "agents/a") + assert browser_tools_in_agent(repo, "agents/a") == ["playwright"] + reached = code_the_agent_reaches(repo, "agents/a") + assert detect_instrumentation(repo / "agents" / "a", reached) == "traceai" + + +@pytest.mark.parametrize( + "raw", + [ + pytest.param( + _in("iso2022_jp", b"import play\x1b(Bwright\n#\x1b"), id="iso2022_jp" + ), + pytest.param(_in("hz", b"import play~{~}wright\n#~"), id="hz"), + pytest.param(_ESCAPED + b"import \\x70laywright\n#\\", id="unicode-escape"), + ], +) +def test_a_file_ending_in_an_open_sequence_is_read(tmp_path, raw): + assert "playwright" in _names_python_runs(raw) + _write(tmp_path, raw) + assert browser_tools_found(tmp_path) == ["playwright"] + + +@pytest.mark.parametrize( + "raw, found", + [ + pytest.param( + _UTF7 + b"import +AHA-laywright\n+AAA-\nimport os\n", + True, + id="utf-7-before", + ), + pytest.param( + _UTF7 + b"import os\n+AAA-\nimport +AHA-laywright\n", + False, + id="utf-7-after", + ), + pytest.param( + _RAW_ESCAPED + b"import \\u0070laywright\n\\u0000\nimport os\n", + True, + id="raw-unicode-escape-before", + ), + pytest.param( + _RAW_ESCAPED + b"import os\n\\u0000\nimport playwright\n", + False, + id="raw-unicode-escape-after", + ), + pytest.param( + _ESCAPED + b"import \\x70laywright\n\\x00\nimport os\n", + True, + id="unicode-escape-before", + ), + pytest.param( + _ESCAPED + b"import os\n\\x00\nimport playwright\n", + False, + id="unicode-escape-after", + ), + ], +) +def test_a_decoded_nul_ends_the_text(tmp_path, raw, found): + assert ("playwright" in _names_python_runs(raw)) is found + _write(tmp_path, raw) + assert browser_tools_found(tmp_path) == (["playwright"] if found else []) + + +@pytest.mark.parametrize( + "raw, name", + [ + # A line that starts with a decoded CR, before a token that can't start a statement. + pytest.param( + _UTF7 + b"import playwright\n+AA0-1\n", "playwright", id="cr-then-1" + ), + pytest.param( + _UTF7 + b"import playwright\n+AA0-@dec\ndef f(): pass\n", + "playwright", + id="cr-then-decorator", + ), + pytest.param( + _ESCAPED + b"import playwright\n\\r(1)\n", + "playwright", + id="ue-cr-then-call", + ), + pytest.param( + _UTF7 + b"import traceai_openai\n+AA0-1\n", + "traceai_openai", + id="cr-then-1-traceai", + ), + pytest.param( + _UTF7 + b"import common.web\n+AA0-1\n", "common.web", id="cr-then-1-walk" + ), + # A decoded CR in a comment, then code. + pytest.param( + _UTF7 + b"x = f # c+AA0-(1); import playwright\n", + "playwright", + id="comment-cr", + ), + pytest.param( + _ESCAPED + b"x = f # c\\r(1); import playwright\n", + "playwright", + id="ue-comment-cr", + ), + pytest.param( + _UTF7 + b"x = f # c+AA0-(1); import traceai_openai\n", + "traceai_openai", + id="comment-cr-traceai", + ), + # A line of blanks and a `\`, then a line that starts with a decoded CR. + pytest.param( + _UTF7 + b"import playwright\n \\\n +AA0-1\n", + "playwright", + id="continued-blank-then-cr", + ), + pytest.param( + _RAW_ESCAPED + b" \\\n\\u000d-c\nimport playwright\n", + "playwright", + id="rue-continued-blank-then-cr", + ), + # A `\` before a decoded CR and an LF: one line end, as Python 3.12 reads it. + pytest.param( + _UTF7 + b"x = 'a\\+AA0-\nb'\nimport playwright\n", + "playwright", + id="string-continued", + ), + pytest.param( + _UTF7 + b"x = 'a\\+AA0-\nb'\nimport traceai_openai\n", + "traceai_openai", + id="string-continued-traceai", + ), + pytest.param( + _UTF7 + b"import os, \\+AA0-\nplaywright\n", + "playwright", + id="line-continued", + ), + pytest.param( + _ESCAPED + b"import os, \\\\\\r\nplaywright\n", + "playwright", + id="ue-line-continued", + ), + pytest.param( + _ESCAPED + b"import \\\\\\r\n\\x70laywright\n", + "playwright", + id="ue-line-continued-escaped-name", + ), + pytest.param( + _RAW_ESCAPED + b"import os, \\u005c\\u000d\n\\u0070laywright\n", + "playwright", + id="rue-line-continued", + ), + pytest.param( + _UTF7 + b"import \\+AA0-\n+AHA-laywright\n", + "playwright", + id="line-continued-spelled-name", + ), + ], +) +@_FROM_312 +def test_a_decoded_cr_reads_as_todays_code_reads_it(tmp_path, raw, name): + assert name in _names_python_runs(raw) + assert _scan(tmp_path, raw, name) == _found(name) + + +@pytest.mark.parametrize( + "raw, name", + [ + # Python keeps the CR in the string; the parser ends the line there. + pytest.param( + _ESCAPED + b'x = "\\r"\nimport playwright\n', + "playwright", + id="cr-in-string", + ), + # Python ends the comment at the CR and reads on; the parser keeps it in the comment. + pytest.param( + _UTF7 + b"import os # hi+AA0-, playwright\n", + "playwright", + id="comment-cr", + marks=_FROM_312, + ), + pytest.param( + _ESCAPED + b"x = 1 # hi\\r; import playwright\n", + "playwright", + id="ue-comment-cr", + marks=_FROM_312, + ), + # Python reads a line a decoded CR starts as blank, inside the block; the parser as + # a line at column 0, which ends it. + pytest.param( + _UTF7 + b"def f():\n a = 1\n+AA0-(1)\n import +AHA-laywright\n", + "playwright", + id="cr-in-a-block", + marks=_FROM_312, + ), + pytest.param( + _UTF7 + + b"try:\n import +AHA-laywright\n+AA0-[0]\nexcept ImportError:\n pass\n", + "playwright", + id="cr-in-a-try", + marks=_FROM_312, + ), + pytest.param( + _UTF7 + + b"try:\n import +AHQ-raceai_openai\n+AA0-[0]\nexcept ImportError:\n pass\n", + "traceai_openai", + id="cr-in-a-try-traceai", + marks=_FROM_312, + ), + pytest.param( + _UTF7 + b"+AA0-\\+AA0-\n import playwright\n", + "playwright", + id="cr-then-continued", + marks=_FROM_312, + ), + # More failed cookie places than _coding reads before the cookie: read as a refused + # codec's file, searched as bytes. + pytest.param( + b"#" + b"coding:#" * 65 + b" coding: utf-7\nimport +AHA-laywright\n", + "playwright", + id="cookie-after-65-failed-places", + ), + ], +) +def test_the_known_misses_stay_unread(tmp_path, raw, name): + assert name in _names_python_runs(raw) + assert _scan(tmp_path, raw, name) == _missed(name) + + +@pytest.mark.parametrize( + "hidden, scan, found, missed", + [ + pytest.param( + _UTF7 + b"import +AHA-laywright\n", + browser_tools_found, + ["playwright"], + [], + id="browser", + ), + pytest.param( + _UTF7 + b"import +AHQ-raceai_openai\n", + detect_instrumentation, + "traceai", + "none", + id="instrumentation", + ), + ], +) +def test_decode_first_files_share_the_scans_total( + tmp_path, monkeypatch, hidden, scan, found, missed +): + # Names nothing either scan looks for, but is decoded first all the same. + first = _UTF7 + b"import os\n# " + b"-" * 40 + b"\n" + # No larger, so a total of its own would let it through. + assert len(hidden) <= len(first) + for total, expected in ((len(first), missed), (len(first) + len(hidden), found)): + monkeypatch.setattr(python_source, "LARGEST_ESCAPED_TOTAL", total) + folder = tmp_path / str(total) + # Both scans read in name order: a.py first. + _write(folder, first, "a.py") + _write(folder, hidden, "b.py") + assert scan(folder) == expected + + +def test_past_the_total_a_file_reads_as_today(tmp_path, monkeypatch): + monkeypatch.setattr(python_source, "LARGEST_ESCAPED_TOTAL", 0) + _write(tmp_path / "plain", _UTF7 + b"import playwright\n") + _write(tmp_path / "spelled", _UTF7 + b"import +AHA-laywright\n") + assert browser_tools_found(tmp_path / "plain") == ["playwright"] + assert browser_tools_found(tmp_path / "spelled") == [] + + +@pytest.mark.parametrize( + "codec, word", + [ + ("latin-1", "café"), + ("cp1252", "€"), + ("shift_jis", "日本"), + ("gb18030", "中文"), + ("euc_kr", "한국"), + ], +) +def test_a_raw_safe_codec_reads_as_today(tmp_path, codec, word): + # CRLF and no final LF: decoding first would end both lines with an LF. + raw = f"# coding: {codec}\r\nimport playwright\r\nNAME = '{word}'".encode(codec) + budget = EscapeBudget() + assert _decoded_first(raw, _coding(raw), budget) == (False, None) + path = _write(tmp_path, raw) + assert read_module(path, needles=(b"playwright",), escapes=budget) == raw.decode( + codec + ) + assert budget.left == python_source.LARGEST_ESCAPED_TOTAL + + +def test_without_a_total_every_codec_reads_as_today(tmp_path): + unended = _UTF7 + b"import +AHA-laywright" + assert _decoded_first(unended, _coding(unended), None) == (False, None) + assert read_module(_write(tmp_path / "utf-7", unended)) == unended.decode("utf-7") + escaped = _ESCAPED + b"import \\x70laywright\n" + assert read_module(_write(tmp_path / "unicode-escape", escaped)) is None + + +@pytest.mark.parametrize("codec", ["rot13", "base64"]) +def test_a_non_text_codec_is_never_charged(tmp_path, codec): + raw = _in(codec, b"import playwright\n") + assert _coding(raw) is not None + budget = EscapeBudget() + assert read_module(_write(tmp_path, raw), escapes=budget) is None + assert budget.left == python_source.LARGEST_ESCAPED_TOTAL + + +def test_a_text_codec_whose_decode_raises_lookup_error_gives_none(): + def decode(data, errors="strict"): + raise LookupError("no table") + + stub = codecs.CodecInfo(codecs.utf_8_encode, decode, name="lookup-failing") + + def search(name): + return stub if name == "lookup_failing" else None + + codecs.register(search) + try: + coding = ("lookup-failing", codecs.lookup("lookup-failing")) + assert _decoded_first(b"x = 1\n", coding, EscapeBudget()) == (True, None) + finally: + codecs.unregister(search) + + +def test_the_walk_keeps_the_decoded_reading_beside_an_escaped_cr(tmp_path): + # Only the decoded reading spells the import; the escaped CR makes the walk read both. + agent = _ESCAPED + b"X = 1 # \\r\nimport \\x63ommon.web\n" + assert "common.web" in _names_python_runs(agent) + repo = _shared(tmp_path, agent) + assert repo / "common" / "web.py" in imported_by_agent(repo, "agents/a") + + +def test_a_file_read_two_ways_is_charged_for_each_reading(tmp_path): + pad = b"# " + b"x" * 76 + b"\n" + agent = _ESCAPED + b'SEP = "\\r"\nimport common.web\n' + pad * 600 + repo = _shared(tmp_path, agent) + _write(repo / "agents" / "a", b"import common.tools\n" + pad * 600, "b.py") + _write(repo / "common", b"", "tools.py") + stopped: list[bool] = [] + budget = python_source.ParseBudget() + found = imported_by_agent( + repo, + "agents/a", + parse_limit=python_source.LARGEST_PARSED_MODULE, + parse_budget=budget, + stopped=stopped, + ) + # Each reading is charged as it's parsed: the decoded one, where `\\r` is one character, and + # the UTF-8 one, where it's two. + assert repo / "common" / "tools.py" in found + others = [repo / "agents" / "a" / "b.py", repo / "common" / "web.py"] + assert budget.left == python_source.LARGEST_PARSED_TOTAL - ( + 2 * len(agent) - 1 + sum(p.stat().st_size for p in others) + ) + # The decoded reading, the one Python runs, is refused: its CR splits the string. The walk + # can't be sure it followed what Python runs, so it says it stopped. + assert stopped == [True] + + +def test_the_union_is_for_unicode_escape_only(tmp_path): + # Read as UTF-8, `+ACM-` is arithmetic and the import is code; decoded, it's a `#`. + agent = _UTF7 + b'"""+AA0-"""\nx = 1 +ACM- 2; import common.web\n' + assert "common" not in _names_python_runs(agent) + repo = _shared(tmp_path, agent) + assert repo / "common" / "web.py" not in imported_by_agent(repo, "agents/a") + + +def test_a_failed_decode_is_still_charged(): + bad = _UTF7 + b"import os\n# +\xff\n" + budget = EscapeBudget() + assert _decoded_first(bad, _coding(bad), budget) == (True, None) + assert budget.left == python_source.LARGEST_ESCAPED_TOTAL - len(bad) + + +@pytest.mark.parametrize( + "agent", + [ + pytest.param(_ESCAPED + b"import common.web # see \\r, x\n", id="comment"), + pytest.param(_ESCAPED + b'SEP = "\\r"\nimport common.web\n', id="string"), + pytest.param(_ESCAPED + b"CR = b'\\r'\nimport common.web\n", id="bytes"), + pytest.param( + _ESCAPED + b'"""\\r"""\nimport \\x63ommon.web\n', id="and-an-escaped-import" + ), + pytest.param( + _ESCAPED + b'SEP = "\\r" # caf\xe9\nimport common.web\n', + id="and-a-latin-1-byte", + ), + ], +) +def test_the_walk_still_follows_a_unicode_escape_file_with_an_escaped_cr( + tmp_path, agent +): + # Decoded, the CR breaks the parse; read as UTF-8 as well, the file is followed. + assert "common.web" in _names_python_runs(agent) + repo = _shared(tmp_path, agent) + assert repo / "common" / "web.py" in imported_by_agent(repo, "agents/a") + assert browser_tools_in_agent(repo, "agents/a") == ["playwright"] + reached = code_the_agent_reaches(repo, "agents/a") + assert detect_instrumentation(repo / "agents" / "a", reached) == "traceai" + + +def test_a_cookie_with_many_coding_words_after_it_is_still_decoded_first(tmp_path): + raw = b"# coding: utf-7" + b" coding" * 65 + b"\nimport +AHA-laywright\n" + assert "playwright" in _names_python_runs(raw) + _write(tmp_path, raw) + assert browser_tools_found(tmp_path) == ["playwright"] + + +def test_the_walk_skips_a_python_file_it_cant_decode(tmp_path): + # The bad byte in a comment, so read as UTF-8 the import would still parse. + agent = _UTF7 + b"import common.web\n# +\xff\n" + with pytest.raises(SyntaxError): + compile(agent, "", "exec", dont_inherit=True) + repo = _shared(tmp_path, agent) + assert repo / "common" / "web.py" not in imported_by_agent(repo, "agents/a") + + +def test_the_walk_reads_scripts_as_before(tmp_path): + folder = tmp_path.resolve() / "agents" / "a" + _write(folder, b"", "util.js") + _write(folder, b"const u = require('./util')\n", "index.js") + assert folder / "util.js" in imported_by_agent(tmp_path, "agents/a") + # A coding line means nothing to a script: utf-7 would spell `./util` here. + _write(folder, _UTF7 + b"const u = require('./u+AHQ-il')\n", "index.js") + assert folder / "util.js" not in imported_by_agent(tmp_path, "agents/a") + + +def test_the_walks_total_is_spent_across_one_walk(tmp_path, monkeypatch): + first = _ESCAPED + b"import os\n# " + b"-" * 20 + b"\n" + # Past the total it's read as UTF-8, where the escape spells nothing. + second = _ESCAPED + b"import \\x63ommon.web\n" + assert len(second) <= len(first) + repo = _shared(tmp_path, b"") + folder = repo / "agents" / "a" + # The walk reads its folder's files in name order. + _write(folder, first, "a.py") + _write(folder, second, "b.py") + for total, followed in ((len(first), False), (len(first) + len(second), True)): + monkeypatch.setattr(python_source, "LARGEST_ESCAPED_TOTAL", total) + assert ( + repo / "common" / "web.py" in imported_by_agent(repo, "agents/a") + ) is followed + + +def _punycode_decodes(monkeypatch) -> list[int]: + import encodings.punycode + + decoded: list[int] = [] + real = encodings.punycode.punycode_decode + monkeypatch.setattr( + encodings.punycode, + "punycode_decode", + lambda *args: decoded.append(len(args[0])) or real(*args), + ) + return decoded + + +def test_the_walk_never_decodes_a_refused_codec(tmp_path, monkeypatch): + import encodings.idna + + decoded = _punycode_decodes(monkeypatch) + labels: list[bytes] = [] + real = encodings.idna.ToUnicode + monkeypatch.setattr( + encodings.idna, "ToUnicode", lambda label: labels.append(label) or real(label) + ) + repo = _shared(tmp_path, b"# coding: punycode\nimport common.web\n-" + b"9" * 64) + _write( + repo / "agents" / "a", + b"# coding: idna\nimport common.web\nHOST = 'xn--bcher-kva.example'\n", + "host.py", + ) + imported_by_agent(repo, "agents/a") + assert (decoded, labels) == ([], []) + + +def test_the_walk_reads_each_coding_line_once(tmp_path, monkeypatch): + asked: list[bytes] = [] + real = python_source._coding + monkeypatch.setattr( + python_source, "_coding", lambda raw: asked.append(raw) or real(raw) + ) + repo = _shared(tmp_path, b"import common.web\n") + _write(repo / "common", b"# a package\n", "__init__.py") + _write(repo / "agents" / "a", b"const u = require('./util')\n", "index.js") + _write(repo / "agents" / "a", b"", "util.js") + imported_by_agent(repo, "agents/a") + python = [ + repo / "agents" / "a" / "agent.py", + repo / "common" / "__init__.py", + repo / "common" / "web.py", + ] + assert sorted(asked) == sorted(path.read_bytes() for path in python) + + +def test_ci_runs_this_file(): + workflows = ( + Path(__file__).resolve().parents[2] / ".github" / "workflows" / "sdk-smoke.yml" + ) + assert "tests/harness/test_python_source_codecs.py" in workflows.read_text( + encoding="utf-8" + ) + + +@pytest.mark.parametrize( + "raw", + [ + # A fullwidth `p`, which the parser folds to `p`; the needle is in a comment's bytes. + pytest.param(_UTF7 + b"import +/1A-laywright\n# +playwrightA-\n", id="utf-7"), + pytest.param( + b"# coding: hz\nimport ~{#p#l#a#y#w#r#i#g#h#t~}\n# ~{playwright~}\n", + id="hz", + ), + ], +) +def test_a_raw_needle_still_lets_the_file_in(tmp_path, raw): + assert "playwright" in _names_python_runs(raw) + _write(tmp_path, raw) + assert browser_tools_found(tmp_path) == ["playwright"] + + +# Every byte form that could join, end or start a name when it decodes. +_NAME_OR_LINE = frozenset(string.ascii_letters + string.digits + "_." + "\0\r\n") +_INCOMPLETE = frozenset({"incomplete multibyte sequence", "unexpected end of data"}) +_DEEPEST_FORM = 4 +# Complete forms by length, for the codecs whose long forms are enumerated by structure. +_UTF8_FORMS = { + # Unicode's scalar values by length: U+0000-U+007F, U+0080-U+07FF, U+0800-U+FFFF less the + # 2,048 surrogates, and U+10000-U+10FFFF. utf-8-sig's byte-order mark is a three-byte form. + 1: 0x80, + 2: 0x800 - 0x80, + 3: 0x10000 - 0x800 - 0x800, + 4: 0x110000 - 0x10000, +} +_FORMS = { + "utf-8": _UTF8_FORMS, + "utf-8-sig": _UTF8_FORMS, + # gb18030 spells each scalar value once: ASCII in one byte, 126 leads (81-FE) by 190 trails + # (40-7E, 80-FE) in two, and in four the rest of the plane below U+10000 (less the + # surrogates) and every value above it. + "gb18030": { + 1: 0x80, + 2: 126 * 190, + 4: (0x10000 - 0x800 - 0x80 - 126 * 190) + (0x110000 - 0x10000), + }, + # CPython's KS X 1001 table (_codecs_kr.c): 2,350 Hangul, 4,888 Hanja and 987 others in two + # bytes (the filler, A4 D4, starts the eight-byte form instead; the euro and registered signs + # came in 1998), and in eight the 19 * 21 * 28 modern syllables the make-up forms compose. + "euc_kr": {1: 0x80, 2: 2350 + 4888 + 987, 8: 19 * 21 * 28}, +} + + +def _accepted_codecs() -> set[str]: + """Every codec Python accepts on a coding line, by its codec info's name.""" + names = set(encodings.aliases.aliases) | set(encodings.aliases.aliases.values()) + names |= {module.name for module in pkgutil.iter_modules(encodings.__path__)} + accepted: set[str] = set() + for name in names - {"aliases"}: + try: + codec = codecs.lookup(name).name + compile( + f"# coding: {name}\nx = 1\n".encode(), + "", + "exec", + dont_inherit=True, + ) + except (LookupError, SyntaxError): + continue + accepted.add(codec) + return accepted + + +def _read(codec: str, form: bytes) -> tuple[str, str | None]: + """Whether ``form`` is complete, incomplete or refused in ``codec``, and its text.""" + try: + return "complete", form.decode(codec) + except UnicodeDecodeError as error: + if error.reason in _INCOMPLETE and error.end == len(form): + return "incomplete", None + return "refused", None + + +class _Sweep: + """Every complete byte form of one codec, found depth-first from each first byte, and those + that could spell a name from other bytes.""" + + def __init__(self, codec: str) -> None: + self.codec = codec + self.forms: collections.Counter[int] = collections.Counter() + self.unsafe: list[str] = [] + + def walk(self, prefix: bytes = b"") -> None: + for byte in range(256): + form = prefix + bytes((byte,)) + if self._by_structure(form): + continue + state, text = _read(self.codec, form) + if state == "complete": + self._check(form, text) + elif state == "incomplete": + if len(form) == _DEEPEST_FORM: + self.unsafe.append(f"{form.hex()}: still incomplete") + else: + self.walk(form) + + def _by_structure(self, form: bytes) -> bool: + # These decoders ask for a whole long form before they read its bytes, or would cost a + # tree of every byte at each place, so their long forms are built from their ranges. + if ( + self.codec in ("utf-8", "utf-8-sig") + and len(form) == 1 + and 0xE0 <= form[0] <= 0xF4 + ): + self._utf8(form, 3 if form[0] < 0xF0 else 4) + elif self.codec == "gb18030" and len(form) == 2 and 0x30 <= form[1] <= 0x39: + for third in range(0x81, 0xFF): + for fourth in range(0x30, 0x3A): + self._leaf(form + bytes((third, fourth))) + elif self.codec == "euc_kr" and form == b"\xa4\xd4": + for initial in range(0xA1, 0xFF): + for medial in range(0xA1, 0xFF): + for final in range(0xA1, 0xFF): + self._leaf( + bytes( + (0xA4, 0xD4, 0xA4, initial, 0xA4, medial, 0xA4, final) + ) + ) + else: + return False + return True + + def _utf8(self, prefix: bytes, length: int) -> None: + # Only a continuation byte, 80-BF, can follow a lead. + for byte in range(0x80, 0xC0): + form = prefix + bytes((byte,)) + if len(form) == length: + self._leaf(form) + elif _read(self.codec, form)[0] == "incomplete": + self._utf8(form, length) + + def _leaf(self, form: bytes) -> None: + state, text = _read(self.codec, form) + if state == "complete": + self._check(form, text) + elif state == "incomplete": + self.unsafe.append(f"{form.hex()}: incomplete at its full length") + + def _check(self, form: bytes, text: str | None) -> None: + self.forms[len(form)] += 1 + if len(form) == 1 and text == chr(form[0]): + return + if not text: + # A byte-order mark is read only at the file's start, before any name. + if not (self.codec == "utf-8-sig" and form == codecs.BOM_UTF8): + self.unsafe.append(f"{form.hex()}: decodes to nothing") + elif not _NAME_OR_LINE.isdisjoint(text): + self.unsafe.append(f"{form.hex()}: decodes to {text!r}") + + +def _outside_the_structure() -> list[str]: + """Each form with a byte the structural ranges leave out that isn't refused: a decoder that + took one would have forms the sweep never tried.""" + taken: list[str] = [] + for lead in range(0x81, 0xFF): + for digit in range(0x30, 0x3A): + for third in (*range(0x00, 0x81), 0xFF): + for fourth in (0x30, 0x39): + form = bytes((lead, digit, third, fourth)) + if _read("gb18030", form)[0] != "refused": + taken.append(f"gb18030 {form.hex()}") + for fourth in (*range(0x00, 0x30), *range(0x3A, 0x100)): + form = bytes((lead, digit, 0x81, fourth)) + if _read("gb18030", form)[0] != "refused": + taken.append(f"gb18030 {form.hex()}") + # Each place of a make-up form after the filler: A4, or a jamo byte of A1-FE. + syllable = bytes((0xA4, 0xD4, 0xA4, 0xA1, 0xA4, 0xBF, 0xA4, 0xA1)) + assert _read("euc_kr", syllable)[0] == "complete" + for place in range(2, 8): + allowed = {0xA4} if place % 2 == 0 else set(range(0xA1, 0xFF)) + for byte in sorted(set(range(256)) - allowed): + form = syllable[:place] + bytes((byte,)) + syllable[place + 1 :] + if _read("euc_kr", form)[0] != "refused": + taken.append(f"euc_kr {form.hex()}") + return taken + + +def test_every_raw_safe_codec_is_proven(): + accepted = _accepted_codecs() + assert _RAW_SAFE_CODECS <= accepted + assert len(_RAW_SAFE_CODECS) == 81 + assert _RAW_SAFE_CODECS.isdisjoint(_SPELLING_CODECS) + unsafe: dict[str, list[str]] = {} + forms: dict[str, dict[int, int]] = {} + for codec in sorted(_RAW_SAFE_CODECS): + sweep = _Sweep(codec) + sweep.walk() + if sweep.unsafe: + unsafe[codec] = sweep.unsafe[:5] + if codec in _FORMS: + forms[codec] = dict(sweep.forms) + assert unsafe == {} + assert forms == _FORMS + assert _outside_the_structure() == [] + # The escape codecs refuse a lone `\` rather than call it incomplete, so the sweep can't see + # them; the disjointness above keeps them out. Every other spelling codec it must flag. + for codec in sorted(_SPELLING_CODECS - {"unicode-escape", "raw-unicode-escape"}): + sweep = _Sweep(codec) + sweep.walk() + assert sweep.unsafe, codec + + +def test_the_accepted_codecs_are_exactly_the_three_groups(): + # A codec a new Python adds fails here until it's put in one group or the other. + assert _accepted_codecs() == _RAW_SAFE_CODECS | _SPELLING_CODECS | {"idna"} + + +# Each holds a needle in bytes its codec decodes to other characters, beside an import of +# playwright Python runs: the bytes let the file in, and the text read is Python's. Sorted by +# their bytes; two need Python 3.12's reading of a decoded CR. +_DECOYS = [ + ( + b"# coding: hz\n" + b"# ~{playwright~}\n" + b"import os, \\\n" + b"~{#p#l#a#y#w#r#i#g#h#t~}\n" + b"import ~{playwright~}\n" + ), + ( + b"# coding: hz\n" + b"import os, \\\r\n" + b"~{#p#l#a#y#w#r#i#g#h#t~}\r\r\n" + b"import os\r\r\n" + b"x = 1 # ~{playwright~}\r\n" + ), + ( + b"# coding: hz\n" + b"try:\r import ~{#p#l#a#y#w#r#i#g#h#t~}\rexcept E:\r pass\rx = 1\rtry:\r import ~{playwright~}\rexcept E:\r pass" + ), + ( + b"# coding: hz\n" + b"x = f # c~{playwright~}(1); import ~{playwright~}\n" + b"import os\n" + b"def f():\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}\n" + ), + ( + b"# coding: hz\r\n" + b"def f():\r\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}\r\r\n" + b"x = f # c~{playwright~}(1); import ~{#p#l#a#y#w#r#i#g#h#t~}\r\n" + ), + ( + b"# coding: hz\r\n" + b"import os, \\\r\n" + b"~{#p#l#a#y#w#r#i#g#h#t~}\r\r\n" + b"x = f # c~{#p#l#a#y#w#r#i#g#h#t~}(1); import ~{playwright~}\r\n" + ), + ( + b"# coding: hz\r\n" + b"import ~{#p#l#a#y#w#r#i#g#h#t~}\r\r\n" + b"x = 1 # ~{playwright~}\r\r\n" + b"import ~{playwright~}\r\r\n" + b"import os, \\\r\n" + b"~{#p#l#a#y#w#r#i#g#h#t~}\r\n" + ), + ( + b"# coding: hz\r\n" + b"try:\n" + b" import ~{playwright~}\n" + b"except E:\n" + b" pass\n" + b"def g():\n" + b" ~{playwright~}\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}\n" + ), + ( + b"# coding: hz\rdef f():\r\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}\r\r\n" + b"~{playwright~}" + ), + b"# coding: hz\rdef f():\r import ~{#p#l#a#y#w#r#i#g#h#t~}\rimport ~{playwright~} # ~{playwright~}\rdef f():\r import ~{playwright~}\r", + ( + b"# coding: hz\rdef g():\n" + b" ~{playwright~}\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}\n" + ), + ( + b"# coding: hz\rimport os # ~{#p#l#a#y#w#r#i#g#h#t~}, ~{playwright~}\n" + b"# ~{#p#l#a#y#w#r#i#g#h#t~}\n" + b"try:\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}\n" + b"except E:\n" + b" pass\n" + b"x = 1 # ~{#p#l#a#y#w#r#i#g#h#t~}" + ), + b"# coding: hz\rimport ~{#p#l#a#y#w#r#i#g#h#t~} # ~{playwright~}", + ( + b"# coding: hz\rimport ~{playwright~} # ~\n" + b"\n" + b"import ~{#p#l#a#y#w#r#i#g#h#t~} # ~{playwright~}\n" + b"~\n" + b"\n" + ), + ( + b"# coding: hz\rtry:\r\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}\r\n" + b"except E:\r\n" + b" pass\r\r\n" + b" \\\r\n" + b"~{playwright~}1\r\n" + ), + b"# coding: hz\rx = '~{playwright~}'\rimport ~{#p#l#a#y#w#r#i#g#h#t~}", + ( + b"# coding: hz\rx = 1\n" + b"def f():\n" + b" import ~{playwright~}\n" + b"import ~{#p#l#a#y#w#r#i#g#h#t~}\n" + ), + b"# coding: hz\rx = 1\rx = 1\rimport os\rdef f():\r import ~{playwright~}\rtry:\r import ~{#p#l#a#y#w#r#i#g#h#t~}\rexcept E:\r pass\r", + ( + b"# coding: hz\rx = 1 # ~~\r\r\n" + b"import os\r\r\n" + b"~{playwright~}\r\r\n" + b"import ~{#p#l#a#y#w#r#i#g#h#t~} # ~{#p#l#a#y#w#r#i#g#h#t~}\r\n" + ), + ( + b"# coding: iso2022_kr\r\n" + b"x = 1\n" + b"\x1b$)C\n" + b"try:\n" + b" import \x0e#p#l#a#y#w#r#i#g#h#t\x0f\n" + b"except E:\n" + b" pass\n" + b"import \x0eplaywright\x0f" + ), + ( + b"# coding: utf-7\n" + b"def g():\r\n" + b" +AAo-\r\n" + b" import +/1A-laywright\r\r\n" + b"try:\r\n" + b" import +playwrightA-\r\n" + b"except E:\r\n" + b" pass\r\n" + ), + (b"# coding: utf-7\ndef g():\r +playwrightA-\r import +/1A-laywright\r"), + ( + b"# coding: utf-7\n" + b"if a:\n" + b" pass\n" + b"import +/1A-laywright\n" + b"x = '+-'\n" + b"import +playwrightA- # +/1A-laywright\n" + ), + (b"# coding: utf-7\nimport os, \\\r+/1A-laywright\r+-+playwrightA-"), + ( + b"# coding: utf-7\n" + b"x = 1 # +-\rtry:\r import +/1A-laywright\rexcept E:\r pass\rx = '+-'\rtry:\r import +playwrightA-\rexcept E:\r pass" + ), + (b"# coding: utf-7\r\nimport +/1A-laywright\nx = 1 # +playwrightA-"), + (b"# coding: utf-7\r\nimport +/1A-laywright # +playwrightA-"), + ( + b"# coding: utf-7\r\n" + b"import os # +-, +playwrightA-\r\r\n" + b"import os, \\\r\n" + b"+/1A-laywright\r\n" + ), + (b"# coding: utf-7\r\nimport os, \\\r+/1A-laywright\rx = '+playwrightA-'\r"), + ( + b"# coding: utf-7\r\n" + b"x = 1\rif a:\r pass\rimport +/1A-laywright # +/1A-laywright\rx = '+playwrightA-'\r" + ), + b"# coding: utf-7\r# +playwrightA-\rtry:\r import +/1A-laywright\rexcept E:\r pass+AHA", + (b"# coding: utf-7\r# -\nx = 1\nimport +/1A-laywright # +playwrightA-\n"), + b"# coding: utf-7\r+playwrightA-x = 1\rtry:\r import +/1A-laywright\rexcept E:\r pass\rx = 1 # +playwrightA-\r", + ( + b"# coding: utf-7\rif a:\n" + b" pass\n" + b"import +/1A-laywright\n" + b"'''+AHA'''\n" + b"import os, \\\n" + b"+playwrightA-+playwrightA-" + ), + b"# coding: utf-7\rimport +/1A-laywright\rtry:\r import +playwrightA-\rexcept E:\r pass\r", + ( + b"# coding: utf-7\rtry:\r\n" + b" import +/1A-laywright\r\n" + b"except E:\r\n" + b" pass\r\r\n" + b"x = 1 # +playwrightA-" + ), + ( + b"# coding: utf-7\rx = f # c+AA0ACg-(1); import +playwrightA-\r\r\n" + b"try:\r\n" + b" import +/1A-laywright\r\n" + b"except E:\r\n" + b" pass\r\r\n" + b"x = 1\r\n" + ), + ( + b"# coding=hz\n" + b"def f():\r\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}\r\r\n" + b"import os\r\r\n" + b"import os, \\\r\n" + b"~{playwright~}\r\r\n" + b"x = 1" + ), + ( + b"# coding=hz\n" + b"def g():\r ~{playwright~}\r import ~{#p#l#a#y#w#r#i#g#h#t~}\r" + ), + ( + b"# coding=hz\n" + b"import os, \\\r\n" + b"~{#p#l#a#y#w#r#i#g#h#t~}\r\r\n" + b"'''~~'''\r\r\n" + b"def f():\r\n" + b" import ~{playwright~}\r\r\n" + b"x = 1" + ), + ( + b"# coding=hz\n" + b"import os, \\\r~{playwright~}\rimport ~{playwright~}\rimport ~{#p#l#a#y#w#r#i#g#h#t~}\r" + ), + ( + b"# coding=hz\n" + b"import ~{#p#l#a#y#w#r#i#g#h#t~}\rtry:\r import ~{playwright~}\rexcept E:\r pass\r" + ), + ( + b"# coding=hz\n" + b"import ~{playwright~}\n" + b"import os, \\\n" + b"~{#p#l#a#y#w#r#i#g#h#t~}\n" + b"if a:\n" + b" pass\n" + b"if a:\n" + b" pass" + ), + ( + b"# coding=hz\n" + b"import ~{playwright~}\r\r\n" + b"import os, \\\r\n" + b"~{#p#l#a#y#w#r#i#g#h#t~}\r\n" + ), + ( + b"# coding=hz\n" + b"x = 1\n" + b"~{playwright~}\n" + b"try:\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}\n" + b"except E:\n" + b" pass\n" + ), + pytest.param( + ( + b"# coding=utf-7\n" + b"+playwrightA-\r'''+-'''\rimport os, \\\r+/1A-laywright+AA0ACg-" + ), + marks=_FROM_312, + ), + pytest.param( + ( + b"# coding=utf-7\n" + b"def f():\n" + b" import +playwrightA-\n" + b"+AA0-\n" + b"def g():\n" + b" +AA0ACg-\n" + b" import +/1A-laywright\n" + ), + marks=_FROM_312, + ), + ( + b"# coding=utf-7\n" + b"if a:\n" + b" x = 1\n" + b"+AAo-\n" + b" import +/1A-laywright\n" + b"def f():\n" + b" import +playwrightA-\n" + ), + ( + b"# coding=utf-7\n" + b"import +playwrightA- # +playwrightA-\r\r\n" + b"import os\r\r\n" + b"try:\r\n" + b" import +/1A-laywright\r\n" + b"except E:\r\n" + b" pass\r\r\n" + b"'''+AHA'''\r\n" + ), + ( + b"# coding=utf-7\n" + b"import os, \\\n" + b"+/1A-laywright\n" + b"+/1A-laywrightx = 1\n" + b"# +AAo-\n" + b"def f():\n" + b" import +/1A-laywright+playwrightA-" + ), + ( + b"# coding=utf-7\n" + b"x = 1 # +-\r\r\n" + b"+/1A-laywright\r\r\n" + b"'''+playwrightA-'''\r\r\n" + b"try:\r\n" + b" import +/1A-laywright\r\n" + b"except E:\r\n" + b" pass" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: hz -*-\n" + b"# ~{playwright~}\r\r\n" + b"try:\r\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}\r\n" + b"except E:\r\n" + b" pass\r\r\n" + b"~{#p#l#a#y#w#r#i#g#h#t~}" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: hz -*-\n" + b"# ~{playwright~}\rimport ~{#p#l#a#y#w#r#i#g#h#t~}\r~{#p#l#a#y#w#r#i#g#h#t~}" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: hz -*-\n" + b"if a:\r pass\rimport ~{#p#l#a#y#w#r#i#g#h#t~}\rimport ~{#p#l#a#y#w#r#i#g#h#t~}~{playwright~}" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: hz -*-\n" + b"import os, \\\n" + b"~{#p#l#a#y#w#r#i#g#h#t~}\n" + b"import os, \\\n" + b"~{playwright~}\n" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: hz -*-\n" + b"import ~{#p#l#a#y#w#r#i#g#h#t~}\n" + b"'''~{playwright~}'''\n" + b"x = 1\n" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: hz -*-\n" + b"import ~{#p#l#a#y#w#r#i#g#h#t~}\n" + b"def f():\n" + b" import ~{playwright~}\n" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: hz -*-\n" + b"import ~{#p#l#a#y#w#r#i#g#h#t~}\n" + b"x = 1~\n" + b"\n" + b"import os, \\\n" + b"~{#p#l#a#y#w#r#i#g#h#t~}\n" + b"x = '~\n" + b"'\n" + b"import os, \\\n" + b"~{playwright~}\n" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: hz -*-\n" + b"import ~{playwright~}\r\r\n" + b"def f():\r\n" + b" import ~{playwright~}\r\r\n" + b"def f():\r\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: hz -*-\n" + b"try:\n" + b" import ~{playwright~}\n" + b"except E:\n" + b" pass\n" + b"# ~~\n" + b"import os, \\\n" + b"~{#p#l#a#y#w#r#i#g#h#t~}" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: hz -*-\n" + b"x = f # c~{playwright~}(1); import ~{#p#l#a#y#w#r#i#g#h#t~}\r\r\n" + b"def f():\r\n" + b" import ~{#p#l#a#y#w#r#i#g#h#t~}\r\r\n" + b"x = 1" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: iso2022_kr -*-\n" + b"x = 1\x1b$)C\rtry:\r import \x0eplaywright\x0f\rexcept E:\r pass\rtry:\r import \x0eplaywright\x0f\rexcept E:\r pass\rimport \x0e#p#l#a#y#w#r#i#g#h#t\x0f # \x1b\r" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: utf-7 -*-\n" + b"+/1A-laywright\r\r\n" + b"def g():\r\n" + b" +playwrightA-\r\n" + b" import +/1A-laywright" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: utf-7 -*-\n" + b"def f():\r\n" + b" import +playwrightA-\r\r\n" + b"import +/1A-laywright # +/1A-laywright\r\n" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: utf-7 -*-\n" + b"if a:\r\n" + b" pass\r\r\n" + b"def g():\r\n" + b" +playwrightA-\r\n" + b" import +/1A-laywright\r\r\n" + b"x = 1+AAo-" + ), + ( + b"#!/usr/bin/env python\n" + b"# -*- coding: utf-7 -*-\n" + b"import +playwrightA- # +/1A-laywright\n" + b"import +/1A-laywright # +-" + ), +] + + +@pytest.mark.parametrize( + "raw", _DECOYS, ids=[f"decoy-{n:02}" for n in range(len(_DECOYS))] +) +def test_no_decoy_case_regresses(tmp_path, raw): + assert "playwright" in _imported(ast.parse(raw)) + path = _write(tmp_path, raw) + text = read_module( + path, needles=agent_type_check._PYTHON_NAMES, escapes=EscapeBudget() + ) + assert text is not None + assert "playwright" in _imported(ast.parse(text)) diff --git a/tests/harness/test_reading_entry.py b/tests/harness/test_reading_entry.py new file mode 100644 index 000000000..a5a062076 --- /dev/null +++ b/tests/harness/test_reading_entry.py @@ -0,0 +1,798 @@ +"""The shared way into a reading: what it refuses, the cost it leaves, and its time limit.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from fi.alk.harness import reading_entry, spend +from fi.alk.harness.job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage +from fi.alk.harness.reading_entry import ( + MESSAGE_LIMIT, + READING_JOB_INVALID, + READING_OUTPUT_NOT_EMPTY, + ReadingMode, + run_reading_mode, + write_reading_cost, + write_reading_failure, +) + +_DEFAULT_SECRET_FILES = reading_entry._SECRET_FILES + +READ_AGENT_VALUES = { + "name": "read_agent", + "time_limit_seconds": 2100.0, + "timed_out_code": "agent_reading_timed_out", + "timed_out_retryable": False, + "crashed_code": "agent_reading_crashed", +} +FIND_AGENTS_VALUES = { + "name": "find_agents", + "time_limit_seconds": 1800.0, + "timed_out_code": "scan_hard_limit_reached", + "timed_out_retryable": False, + "crashed_code": "scan_crashed", +} + + +class _LoudExit: + """``os`` for the helper under test: its hard limit ending this test process is a failure, + never the silent exit 0 its backstop gives a finished reading.""" + + def __getattr__(self, name): + return getattr(os, name) + + @staticmethod + def _exit(_code): + os.write(2, b"a reading's hard limit ended the test process\n") + os._exit(97) + + +@pytest.fixture(autouse=True) +def _fresh_state(monkeypatch, tmp_path): + monkeypatch.setattr(reading_entry, "os", _LoudExit()) + monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) + # The machine's own secrets files must not decide what a test sees masked. + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) + spend._stages.clear() + spend.journal_to(None) + yield + spend._stages.clear() + spend.journal_to(None) + + +@pytest.fixture(params=[READ_AGENT_VALUES, FIND_AGENTS_VALUES], ids=["read_agent", "find_agents"]) +def values(request): + return request.param + + +def _job(source: dict | None = None) -> dict: + return { + "job_id": "job", + "run_id": "run", + "execution": "hosted", + "source": source + or { + "kind": "github", + "visibility": "public", + "repository": "acme/agent", + "commit_sha": "a" * 40, + }, + "runtime": {"isolation": "dedicated_vm"}, + "agent": {"connector": "auto"}, + } + + +class _Work: + """A mode's work that remembers being called; ``action`` stands in for what it does.""" + + def __init__(self, code: int = 0, action=None) -> None: + self.code = code + self.action = action + self.calls: list[tuple] = [] + + async def __call__(self, job, source, output) -> int: + self.calls.append((job, source, output)) + if self.action is not None: + self.action(job, source, output) + return self.code + + +def _accept(_parsed) -> None: + return None + + +def _mode(values: dict, work, check_job=_accept, **overrides) -> ReadingMode: + return ReadingMode(check_job=check_job, work=work, **{**values, **overrides}) + + +def _run(mode: ReadingMode, root: Path, job_path: Path | None = None) -> int: + if job_path is None: + job_path = root / "job.json" + job_path.write_text(json.dumps(_job()), encoding="utf-8") + return run_reading_mode( + mode, job_path=job_path, source=root / "source", output=root / "reading" + ) + + +def _failure_in(output: Path) -> dict: + return json.loads((output / "failure.json").read_text(encoding="utf-8")) + + +def _cost_in(output: Path) -> dict: + return json.loads((output / "cost.json").read_text(encoding="utf-8")) + + +def _shape(failure: dict) -> dict: + return {key: failure[key] for key in ("domain", "stage", "code", "retryable")} + + +def _refusal(code: str = "scan_job_invalid") -> HarnessFailure: + return HarnessFailure( + domain=FailureDomain.PLATFORM_SYNC, + stage=HarnessStage.UNDERSTANDING_AGENT, + code=code, + message="A scan reads a whole repository, not one folder of it.", + retryable=False, + details={"subfolder": "agents/support"}, + ) + + +# The helper ends the process with os._exit at its limit, so those tests run it in a child. +_CHILD = r""" +import asyncio, json, logging, shutil, sys, time +config = json.loads(sys.argv[1]) +sys.path[:0] = config["sys_path"] +logging.basicConfig(level=logging.INFO) +from pathlib import Path +from fi.alk.harness import reading_entry, spend +from fi.alk.harness.reading_entry import ReadingMode, run_reading_mode + +reading_entry._SECRET_FILES = () +root = Path(config["root"]) + + +def broken(*_args, **_kwargs): + raise RuntimeError("this writer is broken") + + +async def work(job, source, output): + if config["work"] == "returns": + return 0 + if config["work"] == "leaves_a_thread": + (output / "agent.json").write_text('{"agent": "kept"}') + reading_entry.write_reading_cost(output, complete=True) + try: + # A scan given up on by its caller: the thread itself runs on past the limit. + await asyncio.wait_for(asyncio.to_thread(time.sleep, 30.0), 0.1) + except TimeoutError: + pass + return 0 + if config["work"] in ("completes_then_blocks", "completes_then_raises"): + spend.record("read-agent", 0.3, turns=1, models={"m"}) + reading_entry.write_reading_cost(output, complete=True) + if config["work"] == "completes_then_raises": + raise RuntimeError("the work broke after its last write") + if config["work"] == "spoils_the_output_then_blocks": + # Every write at the limit now fails, and each failure would log. + shutil.rmtree(output) + output.write_text("a file, not a folder") + if config["work"] == "records_then_blocks": + spend.record("read-agent", 0.21, turns=1, models={"m"}) + if config["work"] == "breaks_the_writers_then_blocks": + reading_entry.write_reading_failure = broken + reading_entry.write_reading_cost = broken + await asyncio.to_thread(time.sleep, config["block_seconds"]) + return 0 + + +mode = ReadingMode(check_job=lambda job: None, work=work, **config["mode"]) +(root / "started").write_text(repr(time.time())) +code = run_reading_mode( + mode, job_path=root / "job.json", source=root / "source", output=root / "reading" +) +time.sleep(config["linger_seconds"]) +sys.exit(code) +""" + +_BLOCK_SECONDS = 20.0 + + +def _run_in_a_child(root: Path, mode_values: dict, work: str, *, linger: float = 0.0): + """Returns the finished child and the seconds from the helper's start to the child's end.""" + root.mkdir(parents=True, exist_ok=True) + (root / "job.json").write_text(json.dumps(_job()), encoding="utf-8") + config = { + "sys_path": [entry for entry in sys.path if entry], + "root": str(root), + "mode": mode_values, + "work": work, + "block_seconds": _BLOCK_SECONDS, + "linger_seconds": linger, + } + env = { + **os.environ, + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:9", + "OTEL_TRACES_EXPORTER": "none", + } + env.pop(spend.JOURNAL_ALIAS, None) + finished = subprocess.run( + [sys.executable, "-c", _CHILD, json.dumps(config)], + env=env, + capture_output=True, + text=True, + timeout=_BLOCK_SECONDS + 40, + ) + ended = time.time() + started = float((root / "started").read_text(encoding="utf-8")) + return finished, ended - started + + +@pytest.mark.parametrize("case", ["missing", "not_json", "not_a_job", "folder_on_an_upload"]) +def test_an_unreadable_job_is_refused_before_any_work(tmp_path, values, case): + job_path = tmp_path / "job.json" + if case == "not_json": + job_path.write_text("{not json", encoding="utf-8") + elif case == "not_a_job": + job_path.write_text(json.dumps({"hello": "world"}), encoding="utf-8") + elif case == "folder_on_an_upload": + upload = {"kind": "archive", "archive_artifact_id": "upload-1"} + # Without the folder the same upload is a good job, so only the folder is refused. + HarnessJob.model_validate(_job(upload)) + job_path.write_text( + json.dumps(_job({**upload, "subfolder": "agents/support"})), encoding="utf-8" + ) + checked: list = [] + work = _Work() + + code = _run(_mode(values, work, check_job=checked.append), tmp_path, job_path) + + output = tmp_path / "reading" + assert code == 78 + assert _shape(_failure_in(output)) == { + "domain": "platform_sync", + "stage": "understanding_agent", + "code": READING_JOB_INVALID, + "retryable": False, + } + assert not (output / "cost.json").exists() + assert checked == [] + assert work.calls == [] + + +_TOKEN = "ghp_REVIEWSECRET1234567890abcdef" + + +@pytest.mark.parametrize( + "refused", + [ + {**_job(), "metadata": {"token": _TOKEN}}, + # pydantic's own message for this one repeats the value whole. + {**_job(), "schema_version": _TOKEN}, + # A dict's key is in the error's location. + {**_job(), "agent": {"connector": "auto", "secret_refs": {_TOKEN: "not-an-object"}}}, + ], + ids=["secret_field", "schema_version", "secret_refs_key"], +) +def test_a_refused_job_leaves_none_of_its_values_anywhere( + tmp_path, values, capfd, caplog, refused +): + token = _TOKEN + job_path = tmp_path / "job.json" + job_path.write_text(json.dumps(refused), encoding="utf-8") + + with caplog.at_level("DEBUG"): + assert _run(_mode(values, _Work()), tmp_path, job_path) == 78 + + failure = (tmp_path / "reading" / "failure.json").read_text(encoding="utf-8") + assert json.loads(failure)["code"] == READING_JOB_INVALID + captured = capfd.readouterr() + kept = failure + captured.out + captured.err + caplog.text + assert not [token[at : at + 8] for at in range(len(token) - 7) if token[at : at + 8] in kept] + + +def test_a_job_nested_past_the_parsers_depth_is_refused(tmp_path, values): + job_path = tmp_path / "job.json" + job_path.write_text("[" * 100_000 + "]" * 100_000, encoding="utf-8") + work = _Work() + + assert _run(_mode(values, work), tmp_path, job_path) == 78 + + failure = _failure_in(tmp_path / "reading") + assert (failure["code"], failure["message"]) == ( + READING_JOB_INVALID, + "The job file could not be read as a harness job (RecursionError).", + ) + assert work.calls == [] + + +def test_the_modes_own_job_check_refuses_before_any_work(tmp_path, values): + seen: list = [] + + def check_job(job): + seen.append(job) + return _refusal() + + work = _Work() + + assert _run(_mode(values, work, check_job=check_job), tmp_path) == 78 + + output = tmp_path / "reading" + assert _failure_in(output) == _refusal().model_dump(mode="json") + assert [type(job) for job in seen] == [HarnessJob] + assert seen[0].job_id == "job" + assert not (output / "cost.json").exists() + assert work.calls == [] + + +def test_a_job_check_that_raises_is_a_crash_before_any_work(tmp_path, values): + def check_job(_job): + raise KeyError("metadata") + + work = _Work() + + assert _run(_mode(values, work, check_job=check_job), tmp_path) == 1 + + output = tmp_path / "reading" + failure = _failure_in(output) + assert _shape(failure) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["crashed_code"], + "retryable": True, + } + assert "KeyError" in failure["message"] + assert not (output / "cost.json").exists() + assert work.calls == [] + + +def test_a_non_empty_output_is_refused_and_left_alone(tmp_path, values): + old = tmp_path / "old" + (old / "reading").mkdir(parents=True) + (old / "reading" / "contract.json").write_bytes(b'{"old": true}') + stale = tmp_path / "stale" + (stale / "reading").mkdir(parents=True) + stale_failure = b'{"code": "from_an_earlier_run"}' + (stale / "reading" / "failure.json").write_bytes(stale_failure) + a_file = tmp_path / "a_file" + a_file.mkdir() + (a_file / "reading").write_bytes(b"not a folder") + work = _Work() + + assert _run(_mode(values, work), old) == 78 + assert _run(_mode(values, work), stale) == 78 + assert _run(_mode(values, work), a_file) == 78 + + assert (old / "reading" / "contract.json").read_bytes() == b'{"old": true}' + assert sorted(entry.name for entry in (old / "reading").iterdir()) == [ + "contract.json", + "failure.json", + ] + assert _shape(_failure_in(old / "reading")) == { + "domain": "platform_sync", + "stage": "understanding_agent", + "code": READING_OUTPUT_NOT_EMPTY, + "retryable": False, + } + assert [entry.name for entry in (stale / "reading").iterdir()] == ["failure.json"] + assert (stale / "reading" / "failure.json").read_bytes() == stale_failure + assert (a_file / "reading").read_bytes() == b"not a folder" + assert work.calls == [] + + +def test_cost_starts_at_zero_and_incomplete_before_the_work(tmp_path, values): + seen: dict = {} + + def look(_job, _source, output): + seen.update(_cost_in(output)) + + assert _run(_mode(values, _Work(action=look)), tmp_path) == 0 + + assert seen["total_usd"] == 0 + assert seen["unpriced_turns"] == 0 + assert seen["stages"] == [] + assert seen["complete"] is False + + +def test_a_failed_first_cost_write_stops_before_the_work(tmp_path, values, monkeypatch): + blocked = tmp_path / "blocked" + blocked.write_text("a file, not a folder", encoding="utf-8") + assert write_reading_cost(blocked, complete=False) is False + + monkeypatch.setattr(reading_entry, "write_reading_cost", lambda output, *, complete: False) + work = _Work() + + assert _run(_mode(values, work), tmp_path) == 1 + + assert work.calls == [] + assert _shape(_failure_in(tmp_path / "reading")) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["crashed_code"], + "retryable": True, + } + + +@pytest.mark.parametrize("retryable, exit_code", [(False, 78), (True, 1)]) +def test_the_time_limit_writes_its_code_and_ends_the_process( + tmp_path, values, retryable, exit_code +): + limit = 1.0 + mode_values = {**values, "time_limit_seconds": limit, "timed_out_retryable": retryable} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "blocks") + + output = tmp_path / "reading" + assert finished.returncode == exit_code, finished.stderr + assert limit <= elapsed <= limit + 5 + assert _shape(_failure_in(output)) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["timed_out_code"], + "retryable": retryable, + } + assert _cost_in(output)["complete"] is False + assert "reading started at " in finished.stderr + + +def test_the_time_limit_ends_the_process_even_when_its_writes_fail(tmp_path): + limit = 1.0 + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": limit} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "breaks_the_writers_then_blocks") + + assert finished.returncode == 78, finished.stderr + assert limit <= elapsed <= limit + 5 + # Proof the writers really raised: the limit could leave no failure of its own. + assert not (tmp_path / "reading" / "failure.json").exists() + + +def test_the_time_limit_does_not_fire_after_the_work_returns(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "returns", linger=2.0) + + assert finished.returncode == 0, finished.stderr + assert elapsed >= 2.0 + assert not (tmp_path / "reading" / "failure.json").exists() + + +def test_a_limit_named_by_the_environment_is_read_when_the_reading_starts( + tmp_path, monkeypatch +): + monkeypatch.setenv("ALK_TEST_READING_LIMIT", "0.5") + mode_values = { + **READ_AGENT_VALUES, + "time_limit_seconds": 30.0, + "limit_env": "ALK_TEST_READING_LIMIT", + "limit_margin_seconds": 0.5, + } + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "blocks") + + assert finished.returncode == 78, finished.stderr + assert 1.0 <= elapsed <= 1.0 + 5 + failure = _failure_in(tmp_path / "reading") + assert failure["code"] == "agent_reading_timed_out" + assert "past its 1 s limit" in failure["message"] + + +def test_a_thread_left_after_the_work_ends_the_process_quietly_at_the_limit(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "leaves_a_thread") + + output = tmp_path / "reading" + # The work's own code, at the limit, not after the 30 s thread; and nothing written over the + # files the work left. + assert finished.returncode == 0, finished.stderr + assert 1.0 <= elapsed <= 1.0 + 5 + assert not (output / "failure.json").exists() + assert (output / "agent.json").read_text() == '{"agent": "kept"}' + assert _cost_in(output) == {**_empty_spend(), "complete": True} + + +def _empty_spend() -> dict: + return {"schema": "futureagi.harness-spend.v1", "total_usd": 0.0, "unpriced_turns": 0, "stages": []} + + +def test_a_timeout_after_a_complete_cost_marks_it_incomplete(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, _ = _run_in_a_child(tmp_path, mode_values, "completes_then_blocks") + + assert finished.returncode == 78, finished.stderr + cost = _cost_in(tmp_path / "reading") + assert (cost["total_usd"], cost["complete"]) == (0.3, False) + assert _failure_in(tmp_path / "reading")["code"] == "agent_reading_timed_out" + + +def test_a_crash_after_a_complete_cost_marks_it_incomplete(tmp_path): + finished, _ = _run_in_a_child(tmp_path, READ_AGENT_VALUES, "completes_then_raises") + + assert finished.returncode == 1, finished.stderr + cost = _cost_in(tmp_path / "reading") + assert (cost["total_usd"], cost["complete"]) == (0.3, False) + assert _failure_in(tmp_path / "reading")["code"] == "agent_reading_crashed" + + +def test_the_way_out_at_the_limit_logs_nothing(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "spoils_the_output_then_blocks") + + assert finished.returncode == 78, finished.stderr + assert elapsed <= 1.0 + 5 + # The writes did fail, and a write that fails logs everywhere else. + assert (tmp_path / "reading").is_file() + assert "reading started at " in finished.stderr + assert "not written" not in finished.stderr + + +def test_a_crash_writes_its_code_with_no_secret_and_a_capped_message(tmp_path, values): + def crash(*_args): + raise RuntimeError("could not fetch https://user:pw@host/repo.git " + "x" * 3000) + + assert _run(_mode(values, _Work(action=crash)), tmp_path) == 1 + + output = tmp_path / "reading" + failure = _failure_in(output) + assert _shape(failure) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["crashed_code"], + "retryable": True, + } + assert failure["message"].startswith( + "RuntimeError: could not fetch https://user:***@host/repo.git " + ) + assert "pw" not in failure["message"] + assert MESSAGE_LIMIT == 2000 + assert len(failure["message"]) == MESSAGE_LIMIT + assert _cost_in(output)["complete"] is False + + blocked = tmp_path / "blocked" + blocked.write_text("a file, not a folder", encoding="utf-8") + assert write_reading_failure(_refusal(), blocked) is None + + +def test_a_secret_is_masked_in_the_shapes_a_trace_shows_it(tmp_path, monkeypatch, caplog): + pem = "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEF\n-----END PRIVATE KEY-----\n" + credential = json.dumps({"type": "service_account", "private_key": pem}) + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"GOOGLE_CREDENTIALS_JSON": credential}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + + def crash(*_args): + raise RuntimeError(f"bad key {pem!r}; line MIIEvQIBADANBgkqhkiG9w0BAQEF") + + with caplog.at_level("INFO", logger=reading_entry.__name__): + assert _run(_mode(READ_AGENT_VALUES, _Work(action=crash)), tmp_path) == 1 + + kept = _failure_in(tmp_path / "reading")["message"] + caplog.text + assert "MIIEvQIBADANBgkqhkiG9w0BAQEF" not in kept + assert "BEGIN PRIVATE KEY" not in kept + # The whole value is still masked first; a value under 8 characters, or in neither file, is + # one the mask can't know. + assert reading_entry._read_secret_values((secrets,))[0] in (credential, json.dumps(credential)[1:-1]) + + +def test_each_reading_reads_the_secrets_again(tmp_path, monkeypatch): + secrets = tmp_path / "secrets.json" + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + for run, value in enumerate(("first-secret-value", "second-secret-value")): + secrets.write_text(json.dumps({"KEY": value}), encoding="utf-8") + root = tmp_path / f"run{run}" + root.mkdir() + + def crash(*_args, said=value): + raise RuntimeError(f"refused {said}") + + assert _run(_mode(READ_AGENT_VALUES, _Work(action=crash)), root) == 1 + assert _failure_in(root / "reading")["message"] == "RuntimeError: refused ***" + + +def test_a_value_from_the_secrets_file_is_masked(tmp_path, monkeypatch, values): + target = "sk-0123456789abcdefg" + simulator = "sim-0123456789abcdef" + assert len(target) == len(simulator) == 20 + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"TARGET_KEY": target}), encoding="utf-8") + simulator_secrets = tmp_path / "simulator-secrets.json" + simulator_secrets.write_text(json.dumps({"SIMULATOR_KEY": simulator}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets, simulator_secrets)) + + def crash(*_args): + raise RuntimeError(f"the provider refused {target} and {simulator}") + + assert _run(_mode(values, _Work(action=crash)), tmp_path) == 1 + + assert _failure_in(tmp_path / "reading")["message"] == ( + "RuntimeError: the provider refused *** and ***" + ) + assert _DEFAULT_SECRET_FILES == ( + Path("/run/futureagi/secrets.json"), + Path("/run/futureagi/simulator-secrets.json"), + ) + + +@pytest.mark.parametrize("exit_code", [0, 1, 78]) +def test_the_works_exit_code_is_returned(tmp_path, values, exit_code): + assert _run(_mode(values, _Work(code=exit_code)), tmp_path) == exit_code + assert not (tmp_path / "reading" / "failure.json").exists() + + +def test_a_works_own_incomplete_cost_is_kept(tmp_path, values): + written: list[int] = [] + + def account(_job, _source, output): + spend.record("read-agent", 0.42, turns=1, models={"m"}) + assert write_reading_cost(output, complete=False) + # Every write moves a new file into place, so a rewrite after the work would show. + written.append((output / "cost.json").stat().st_ino) + + assert _run(_mode(values, _Work(action=account)), tmp_path) == 0 + + output = tmp_path / "reading" + assert (output / "cost.json").stat().st_ino == written[0] + cost = _cost_in(output) + assert cost["total_usd"] == 0.42 + assert cost["complete"] is False + + +def test_each_turn_after_the_first_write_lands_in_the_cost_file(tmp_path): + output = tmp_path / "reading" + assert write_reading_cost(output, complete=False) is True + + spend.record("read-agent", 0.25, turns=1, models={"m"}) + + assert _cost_in(output) == {**spend.snapshot(), "complete": False} + assert sorted(entry.name for entry in output.iterdir()) == ["cost.json"] + + +def test_cost_is_left_after_a_timeout_and_a_crash(tmp_path): + def record_then_crash(*_args): + spend.record("read-agent", 0.21, turns=1, models={"m"}) + raise RuntimeError("the reading broke") + + crashed = tmp_path / "crash" + crashed.mkdir() + assert _run(_mode(READ_AGENT_VALUES, _Work(action=record_then_crash)), crashed) == 1 + cost = _cost_in(crashed / "reading") + assert cost["total_usd"] == 0.21 + assert cost["complete"] is False + assert _failure_in(crashed / "reading")["code"] == "agent_reading_crashed" + + timed_out = tmp_path / "timeout" + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + finished, _ = _run_in_a_child(timed_out, mode_values, "records_then_blocks") + assert finished.returncode == 78, finished.stderr + cost = _cost_in(timed_out / "reading") + assert cost["total_usd"] == 0.21 + assert cost["complete"] is False + assert _failure_in(timed_out / "reading")["code"] == "agent_reading_timed_out" + + +def test_the_checks_run_in_order(tmp_path, values): + unreadable = tmp_path / "unreadable" + (unreadable / "reading").mkdir(parents=True) + (unreadable / "reading" / "old.txt").write_text("old", encoding="utf-8") + bad_job = unreadable / "job.json" + bad_job.write_text("{not json", encoding="utf-8") + refused = tmp_path / "refused" + (refused / "reading").mkdir(parents=True) + (refused / "reading" / "old.txt").write_text("old", encoding="utf-8") + work = _Work() + + assert _run(_mode(values, work), unreadable, bad_job) == 78 + assert _run(_mode(values, work, check_job=lambda _parsed: _refusal()), refused) == 78 + + assert _failure_in(unreadable / "reading")["code"] == READING_JOB_INVALID + assert _failure_in(refused / "reading")["code"] == "scan_job_invalid" + assert not (unreadable / "reading" / "cost.json").exists() + assert not (refused / "reading" / "cost.json").exists() + assert work.calls == [] + + +def test_a_crash_keeps_its_trace_in_the_log_masked_and_out_of_the_file( + tmp_path, monkeypatch, caplog +): + secret = "sk-0123456789abcdefg" + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"TARGET_KEY": secret}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + + def crash(*_args): + raise RuntimeError(f"the provider refused {secret}") + + with caplog.at_level("INFO", logger=reading_entry.__name__): + assert _run(_mode(READ_AGENT_VALUES, _Work(action=crash)), tmp_path) == 1 + + assert "Traceback" not in _failure_in(tmp_path / "reading")["message"] + assert "reading started at " in caplog.text + assert "Traceback (most recent call last)" in caplog.text + assert "RuntimeError: the provider refused ***" in caplog.text + assert secret not in caplog.text + + +def test_each_swallowed_error_is_logged_with_its_trace(tmp_path, monkeypatch, caplog): + blocked = tmp_path / "blocked" + blocked.write_text("a file, not a folder", encoding="utf-8") + + with caplog.at_level("WARNING"): + assert write_reading_failure(_refusal(), blocked) is None + assert write_reading_cost(blocked, complete=False) is False + monkeypatch.setitem(sys.modules, "fi.alk.harness.hosted_entrypoint", None) + assert reading_entry._read_secret_values((tmp_path / "unread.json",)) == () + + traced = [ + record.getMessage() + for record in caplog.records + if record.levelname == "WARNING" and (record.exc_info or record.exc_text) + ] + for said in ("failure file not written", "spend file not written", "secret values not read"): + assert any(said in line for line in traced), (said, traced) + + +def test_cost_json_keeps_the_build_keys_and_adds_complete(tmp_path): + spend.record( + "read-agent", + 0.125, + turns=3, + models={"claude-x"}, + tokens_in=1000, + tokens_out=200, + tokens_cached=100, + tools={"submit_contract": 1}, + refusals={"agent_type": 1}, + ) + output = tmp_path / "reading" + + assert write_reading_cost(output, complete=True) is True + + snapshot = spend.snapshot() + assert "complete" not in snapshot + assert _cost_in(output) == {**snapshot, "complete": True} + assert [entry.name for entry in output.iterdir()] == ["cost.json"] + + +def test_an_unpriced_turn_marks_the_cost_incomplete(tmp_path): + spend.record("read-agent", 0.1, turns=1, models={"m"}) + spend.record("read-agent", None, turns=1, models={"m"}) + output = tmp_path / "reading" + + assert write_reading_cost(output, complete=True) is True + + cost = _cost_in(output) + assert cost["unpriced_turns"] == 1 + assert cost["total_usd"] == 0.1 + assert cost["complete"] is False + + +def test_a_failed_cost_write_leaves_no_half_file(tmp_path, monkeypatch): + output = tmp_path / "reading" + output.mkdir() + + def refuses(*_args, **_kwargs): + raise OSError("no space left on the device") + + monkeypatch.setattr(os, "replace", refuses) + + assert write_reading_cost(output, complete=False) is False + assert list(output.iterdir()) == [] + + +def test_a_builds_spend_file_gains_no_complete_key(tmp_path): + assert write_reading_cost(tmp_path / "reading", complete=True) is True + build = tmp_path / "authoring" / "cost.json" + + spend.journal_to(build) + spend.record("understand-agent", 0.1, turns=1, models={"m"}) + + assert json.loads(build.read_text(encoding="utf-8")) == spend.snapshot() diff --git a/tests/harness/test_runtime_repair.py b/tests/harness/test_runtime_repair.py index d97d8e291..313ebf3da 100644 --- a/tests/harness/test_runtime_repair.py +++ b/tests/harness/test_runtime_repair.py @@ -2432,3 +2432,214 @@ def test_ingress_openai_default_profile_also_requires_source_evidence(tmp_path, with pytest.raises(ValueError, match="profile_evidence_required"): validate_runtime_plan_patch(tmp_path, contract, patch, allowed_diagnostic_codes={"chat_ingress_failed"}, allow_ingress_repair=True) + + +def _folder_repo(tmp_path): + source = tmp_path / "source" + for name in ("a", "b"): + (source / "agents" / name).mkdir(parents=True) + (source / "agents" / name / "pyproject.toml").write_text( + f"[project]\nname='agent-{name}'\nversion='1'\n" + ) + (source / "agents" / name / "app.py").write_text("# entrypoint\n") + return source + + +def _folder_contract(**runtime) -> AgentContract: + return AgentContract( + agent="support-agent", + runtime=Runtime( + language="python", + source_subfolder="agents/a", + workdir="agents/a", + **runtime, + ), + ) + + +def _folder_patch(contract: AgentContract, *, mode="runtime_plan", codes=None, **changes): + runtime = Runtime( + language="python", + workdir="agents/a", + command=["python", "app.py"], + interface=RuntimeInterface(kind="http", port=8080, path="/chat"), + **changes, + ) + return RuntimePlanPatch( + mode=mode, + base_contract_hash=_contract_hash(contract), + runtime=runtime, + evidence_paths=("agents/a/pyproject.toml", "agents/a/app.py"), + diagnostic_codes=tuple(codes or ("generated_runtime_plan_invalid",)), + summary="Run the HTTP service declared by the agent's folder.", + ) + + +def test_dependency_root_uses_subfolder(tmp_path) -> None: + from fi.alk.harness.runtime_repair import _dependency_root + + root = _folder_repo(tmp_path).resolve() + (root / "pyproject.toml").write_text("[project]\nname='top'\nversion='1'\n") + # A sibling's callback must not decide where this agent installs from. + (root / "agents" / "b" / "agent.py").write_text("def agent_callback(m):\n return m\n") + + assert _dependency_root(root, _folder_contract().runtime) == root / "agents" / "a" + + +def test_dependency_root_keeps_compose_branch(tmp_path) -> None: + from fi.alk.harness.runtime_repair import _dependency_root + + root = _folder_repo(tmp_path).resolve() + (root / "agents" / "a" / "app").mkdir() + (root / "agents" / "a" / "compose.yaml").write_text( + "services:\n agent:\n build: ./app\n command: [python, app.py]\n" + ) + + assert _dependency_root(root, _folder_contract().runtime) == root / "agents" / "a" / "app" + + +def test_repair_patch_drops_a_folder_the_contract_never_recorded(tmp_path) -> None: + source = _folder_repo(tmp_path) + contract = AgentContract(agent="support-agent", runtime=Runtime(language="python")) + + repaired = validate_runtime_plan_patch( + source, + contract, + _folder_patch(contract, source_subfolder="agents/b"), + allowed_diagnostic_codes={"generated_runtime_plan_invalid"}, + ) + + assert repaired.runtime.source_subfolder == "" + + +def test_repair_patch_keeps_recorded_folder(tmp_path) -> None: + source = _folder_repo(tmp_path) + contract = _folder_contract() + allowed = {"generated_runtime_plan_invalid"} + + kept = validate_runtime_plan_patch( + source, contract, _folder_patch(contract), allowed_diagnostic_codes=allowed + ) + assert kept.runtime.source_subfolder == "agents/a" + + # A folder the model wrote is replaced by the one the contract recorded. + written = validate_runtime_plan_patch( + source, + contract, + _folder_patch(contract, source_subfolder="agents/b"), + allowed_diagnostic_codes=allowed, + ) + assert written.runtime.source_subfolder == "agents/a" + + # The key on disk survives a patch that never names it. + path = tmp_path / "contract.json" + path.write_text(contract.model_dump_json(indent=2) + "\n") + applied = apply_runtime_plan_patch( + source, path, _folder_patch(contract), allowed_diagnostic_codes=allowed + ) + assert applied.runtime.source_subfolder == "agents/a" + assert json.loads(path.read_text())["runtime"]["source_subfolder"] == "agents/a" + + +def test_repair_patch_keeps_recorded_folder_for_ingress_repair(tmp_path) -> None: + source = _folder_repo(tmp_path) + contract = AgentContract( + agent="support-agent", + modality="chat", + runtime=Runtime( + language="python", + source_subfolder="agents/a", + workdir="agents/a", + command=["python", "app.py"], + interface=RuntimeInterface(kind="http", port=8081, path="/chat"), + ), + ) + + repaired = validate_runtime_plan_patch( + source, + contract, + _folder_patch( + contract, mode="chat_ingress", codes=("chat_ingress_failed",) + ), + allowed_diagnostic_codes={"chat_ingress_failed"}, + allow_ingress_repair=True, + ) + + assert repaired.runtime.interface.port == 8080 + assert repaired.runtime.source_subfolder == "agents/a" + + +def test_repair_patch_refuses_climbing_command(tmp_path) -> None: + source = _folder_repo(tmp_path) + contract = _folder_contract() + patch = _folder_patch(contract) + patch = patch.model_copy( + update={ + "runtime": patch.runtime.model_copy( + update={"command": ["python", "../b/app.py"]} + ) + } + ) + + with pytest.raises(ValueError, match="runtime_command_outside_subfolder"): + validate_runtime_plan_patch( + source, + contract, + patch, + allowed_diagnostic_codes={"generated_runtime_plan_invalid"}, + ) + + +def test_repair_prompt_names_the_folder(tmp_path, monkeypatch) -> None: + import asyncio + + from fi.alk.harness import runtime_repair + from fi.alk.harness.diagnostics import HarnessDiagnostic + from fi.alk.harness.job import HarnessStage + + prompts: list[str] = [] + + class InspectingStage: + def __init__(self, spec, **kwargs): + prompts.append(spec.system_prompt) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + pass + + async def say(self, briefing, *, on_event=None): + pass + + monkeypatch.setattr(runtime_repair, "Stage", InspectingStage) + diagnostic = HarnessDiagnostic.create( + stage=HarnessStage.BUILDING_ENVIRONMENT, + component="runtime", + code="generated_runtime_plan_invalid", + message="bad runtime", + ) + for contract in (_folder_contract(), AgentContract(agent="x", runtime=Runtime())): + with pytest.raises(RuntimeError, match="typed_runtime_plan_patch_not_submitted"): + asyncio.run( + runtime_repair.request_runtime_plan_patch(tmp_path, contract, (diagnostic,)) + ) + + assert "This agent is the folder agents/a." in prompts[0] + # Every service is checked at the save, not only the control one. + assert "every service the stack runs must build this folder" in prompts[0] + assert "Compose file may sit in it or above it" in prompts[0] + assert "This agent is the folder" not in prompts[-1] + + +def test_repair_reads_the_folders_alk_yaml(tmp_path) -> None: + from fi.alk.harness.runtime_repair import _select_repair_runtime + + root = _folder_repo(tmp_path).resolve() + # The repository-level file is unusable on purpose: only the agent folder's is read. + (root / "alk.yaml").write_text("runtime: 5\n") + (root / "agents" / "a" / "alk.yaml").write_text( + "runtime:\n environment:\n MODE: test\n" + ) + + _select_repair_runtime(root, _folder_contract().runtime) diff --git a/tests/harness/test_runtime_selection_admission.py b/tests/harness/test_runtime_selection_admission.py index 861438c4b..cd6eec5e4 100644 --- a/tests/harness/test_runtime_selection_admission.py +++ b/tests/harness/test_runtime_selection_admission.py @@ -716,3 +716,173 @@ def test_process_lane_rejects_selected_dependency_image(tmp_path): "compose_image_services": ["cache"], }, ) + + +def _stacks(tmp_path, *, root_builds="agents/a"): + from pathlib import Path + + root = Path(tmp_path) + for name in ("a", "b"): + (root / "agents" / name).mkdir(parents=True) + (root / "agents" / name / "agent.py").write_text("print('agent')\n") + (root / "compose.yaml").write_text( + f"services:\n agent:\n build: {root_builds}\n command: [python, agent.py]\n" + ) + (root / "agents" / "a" / "compose.yaml").write_text( + "services:\n agent:\n build: .\n command: [python, agent.py]\n" + ) + return root + + +def test_select_runtime_reads_source_subfolder(tmp_path): + from fi.alk.harness.runtime_selection import ComposeSelection, NativeSelection, select_runtime + + root = _stacks(tmp_path) + + own = select_runtime( + root, {"source_subfolder": "agents/a", "workdir": "agents/a"} + ) + assert isinstance(own, ComposeSelection) + assert own.compose_file == (root / "agents" / "a" / "compose.yaml").resolve() + + # The root's stack belongs to the whole repository; a folder with none of its own has none. + other = select_runtime( + root, {"source_subfolder": "agents/b", "workdir": "agents/b"} + ) + assert isinstance(other, NativeSelection) + + +def test_no_subfolder_lookup_unchanged(tmp_path): + from fi.alk.harness.runtime_selection import ComposeSelection, select_runtime + + root = _stacks(tmp_path) + + selected = select_runtime(root, {}) + + assert isinstance(selected, ComposeSelection) + assert selected.compose_file == (root / "compose.yaml").resolve() + + +def _explicit_root_stack(root): + return { + "source_subfolder": "agents/a", + "workdir": "agents/a", + "execution": "compose", + "compose_file": "compose.yaml", + "service": "agent", + } + + +def test_control_service_building_a_sibling_refused(tmp_path): + from fi.alk.harness.runtime_selection import RuntimeSelectionError, select_runtime + + root = _stacks(tmp_path, root_builds="agents/b") + + with pytest.raises(RuntimeSelectionError, match="runtime_compose_service_outside_subfolder"): + select_runtime(root, _explicit_root_stack(root)) + + +def test_root_compose_building_the_folder_allowed(tmp_path): + from fi.alk.harness.runtime_selection import ComposeSelection, select_runtime + + root = _stacks(tmp_path, root_builds="agents/a") + + selected = select_runtime(root, _explicit_root_stack(root)) + + assert isinstance(selected, ComposeSelection) + assert selected.control_service == "agent" + assert selected.compose_file == (root / "compose.yaml").resolve() + + +def _compose_building(build: str) -> str: + return f"services:\n agent:\n build: {build}\n command: [python, agent.py]\n" + + +_ROOT_BUILDS = [ + ("agents/b", False), + ("{context: agents/b}", False), + ("{context: ., dockerfile: agents/b/Dockerfile}", False), + ("{context: agents, dockerfile: b/Dockerfile}", False), + (".", True), + ("{context: ., dockerfile: agents/a/Dockerfile}", True), + ("{context: agents/a, dockerfile: Dockerfile}", True), +] + + +@pytest.mark.parametrize(("build", "allowed"), _ROOT_BUILDS) +def test_root_compose_service_build_forms(tmp_path, build, allowed): + from fi.alk.harness.runtime_selection import ( + ComposeSelection, + RuntimeSelectionError, + select_runtime, + ) + + root = _stacks(tmp_path) + (root / "compose.yaml").write_text(_compose_building(build)) + + if allowed: + assert isinstance( + select_runtime(root, _explicit_root_stack(root)), ComposeSelection + ) + else: + with pytest.raises( + RuntimeSelectionError, match="runtime_compose_service_outside_subfolder" + ): + select_runtime(root, _explicit_root_stack(root)) + + +@pytest.mark.parametrize( + ("build", "allowed"), + [ + ("../b", False), + ("{context: ../b}", False), + ("./app", True), + ("{context: ./app}", True), + ], +) +def test_folder_compose_service_build_forms(tmp_path, build, allowed): + from fi.alk.harness.runtime_selection import ( + ComposeSelection, + RuntimeSelectionError, + select_runtime, + ) + + root = _stacks(tmp_path) + (root / "agents" / "a" / "app").mkdir() + (root / "agents" / "a" / "compose.yaml").write_text(_compose_building(build)) + runtime = { + **_explicit_root_stack(root), + "compose_file": "agents/a/compose.yaml", + } + + if allowed: + assert isinstance(select_runtime(root, runtime), ComposeSelection) + else: + with pytest.raises( + RuntimeSelectionError, match="runtime_compose_service_outside_subfolder" + ): + select_runtime(root, runtime) + + +@pytest.mark.parametrize(("build", "allowed"), [("./app", True), ("../b", False)]) +def test_included_service_build_is_read_from_its_own_file(tmp_path, build, allowed): + from fi.alk.harness.runtime_selection import ( + ComposeSelection, + RuntimeSelectionError, + select_runtime, + ) + + root = _stacks(tmp_path) + (root / "agents" / "a" / "app").mkdir() + (root / "compose.yaml").write_text("include:\n - agents/a/compose.yaml\n") + (root / "agents" / "a" / "compose.yaml").write_text(_compose_building(build)) + + if allowed: + assert isinstance( + select_runtime(root, _explicit_root_stack(root)), ComposeSelection + ) + else: + with pytest.raises( + RuntimeSelectionError, match="runtime_compose_service_outside_subfolder" + ): + select_runtime(root, _explicit_root_stack(root)) diff --git a/tests/harness/test_scan_ai.py b/tests/harness/test_scan_ai.py new file mode 100644 index 000000000..e53825f6a --- /dev/null +++ b/tests/harness/test_scan_ai.py @@ -0,0 +1,1052 @@ +"""The AI step of the agent scan: one paid turn, what it ends as, and the tools the model reads with. + +No test calls a model. A fake backend plays a script: what the model says, which tools it calls +(through the real handlers), and how its turn ends. Every test that waits on a stage runs under +its own bound, so a missing bound in the code fails the test instead of hanging the run. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from fi.alk.harness import scan_ai, session, spend +from fi.alk.harness.backends import Call, ModelReply, Say, StageDone, ToolReturned +from fi.alk.harness.config import load_skill + +GEMINI = "gemini-fake-model" +SUBMIT = "mcp__scan__submit_agents" +READ = "mcp__scan_files__read" +GLOB = "mcp__scan_files__glob" +GREP = "mcp__scan_files__grep" +AGENT = ("services/support", "services/support/agent.py") +KEPT = ("call", SUBMIT, {"agents": [{"folder": AGENT[0], "entry": AGENT[1]}]}) +FINISHED = ("done", StageDone(outcome="success", turns=3, cost_usd=0.01, models={GEMINI})) +# ADK's whole message, as the Vertex-Gemini backend passes it on. +CALL_LIMIT_MESSAGE = "Max number of llm calls limit of `40` exceeded" +TEST_BOUND = 15 + +# Read before any test shortens them. +_BOUNDS = { + name: getattr(scan_ai, name) + for name in ( + "OPEN_SECONDS", + "INTERRUPT_SECONDS", + "SPEND_WAIT_SECONDS", + "EXIT_SECONDS", + "TOOL_SECONDS", + "IDLE_SECONDS", + "MAX_TURNS", + ) +} + + +@pytest.fixture(autouse=True) +def _clean_ledger(monkeypatch): + retries = session.STAGE_IDLE_RETRIES + monkeypatch.delenv("ALK_SPEND_JOURNAL", raising=False) + monkeypatch.setattr(spend, "_path", None) + monkeypatch.setenv("ALK_HARNESS_MODEL", GEMINI) + yield + session.STAGE_IDLE_RETRIES = retries + spend._stages.clear() + + +@pytest.fixture +def quick(monkeypatch): + for name, seconds in ( + ("OPEN_SECONDS", 0.3), + ("INTERRUPT_SECONDS", 0.2), + ("SPEND_WAIT_SECONDS", 0.3), + ("CANCEL_SECONDS", 0.2), + ("EXIT_SECONDS", 0.2), + ): + monkeypatch.setattr(scan_ai, name, seconds) + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _repo(tmp_path: Path) -> Path: + source = tmp_path / "repo" + _write(source / AGENT[1], "from openai import OpenAI\nclient = OpenAI()\n") + _write(source / "services/support/pyproject.toml", "[project]\nname = 'support'\n") + _write(source / "main.py", "print('hello')\n") + _write(source / "tests/test_agent.py", "def test_nothing():\n pass\n") + return source + + +def _said(result: dict[str, Any]) -> str: + return "".join(part.get("text", "") for part in result.get("content", [])) + + +class _Backend: + """Sessions that play ``steps`` instead of calling a model, and count what they were asked.""" + + name = "fake" + default_model = GEMINI + + def __init__(self, *steps: tuple, start: str = "", stop: str = "", interrupt: str = ""): + self.steps = steps + self.start = start + self.stop = stop + self.interrupt = interrupt + self.specs: list[Any] = [] + self.sent: list[str] = [] + self.results: list[dict[str, Any]] = [] + self.interrupts = 0 + self.stops = 0 + + def create(self, spec): + self.specs.append(spec) + return _Session(self, spec) + + def can_drive(self, model: str) -> bool: + return True + + +class _Session: + def __init__(self, backend: _Backend, spec) -> None: + self.backend = backend + self.spec = spec + self.interrupted = asyncio.Event() + + async def start(self) -> None: + if self.backend.start == "hang": + await asyncio.Event().wait() + if self.backend.start == "raise": + raise RuntimeError("the provider refused the session") + + async def stop(self) -> None: + self.backend.stops += 1 + if self.backend.stop == "hang": + await asyncio.Event().wait() + + async def send(self, message: str) -> None: + self.backend.sent.append(message) + + async def interrupt(self) -> bool: + self.backend.interrupts += 1 + if self.backend.interrupt == "hang": + await asyncio.Event().wait() + self.interrupted.set() + return True + + async def resume(self, invocation_id: str) -> None: + raise RuntimeError("not supported") + + def _handler(self, qualified: str): + _, server, name = qualified.split("__") + return next( + spec.handler for spec in self.spec.servers[server].tools if spec.name == name + ) + + async def replies(self): + for step in self.backend.steps: + kind = step[0] + if kind == "say": + yield ModelReply(parts=[Say(text=step[1])], model=GEMINI) + elif kind == "call": + call_id = f"call-{len(self.backend.results)}" + yield ModelReply(parts=[Call(id=call_id, name=step[1], arguments=step[2])]) + result = await self._handler(step[1])(step[2]) + self.backend.results.append(result) + yield ToolReturned( + id=call_id, text=_said(result), is_error=bool(result.get("is_error")) + ) + elif kind == "done": + yield step[1] + elif kind == "quiet": + await asyncio.Event().wait() + elif kind == "until_interrupt": + await self.interrupted.wait() + elif kind == "raise": + raise step[1] + + +def _scan(monkeypatch, source: Path, backend: _Backend, *, seconds: float = 5.0, + hints: str = "3 Python files outside side folders", ends: list | None = None): + monkeypatch.setattr(session, "resolve", lambda *_args, **_kwargs: backend) + noted = ends if ends is not None else [] + return asyncio.run( + asyncio.wait_for( + scan_ai.run_ai_step( + source, hints=hints, seconds=seconds, on_turn_end=lambda: noted.append(1) + ), + TEST_BOUND, + ) + ) + + +@dataclass +class _Case: + steps: tuple + kind: str + retryable: bool = False + complete: bool = True + ends: int = 1 + seconds: float = 5.0 + interrupt: str = "" + idle: float | None = None + pairs: tuple = field(default_factory=tuple) + + +def _done(**facts: Any) -> tuple: + facts.setdefault("models", {GEMINI}) + return ("done", StageDone(**facts)) + + +_OUTCOMES = { + "pairs": _Case((KEPT, FINISHED), "pairs", pairs=(AGENT,)), + "pairs_then_a_forbidden_model": _Case( + (KEPT, _done(models={"claude-sonnet-4-6"})), "pairs", pairs=(AGENT,) + ), + "forbidden": _Case((_done(models={"claude-sonnet-4-6"}),), "forbidden"), + "timed_out_quiet": _Case((("quiet",),), "timed_out", complete=False, ends=0, seconds=0.3), + "timed_out_bill_after_the_interrupt": _Case( + (("until_interrupt",), FINISHED), "timed_out", complete=False, seconds=0.3 + ), + "interrupt_never_returns": _Case( + (("quiet",),), "timed_out", complete=False, ends=0, seconds=0.3, interrupt="hang" + ), + "idle_stream": _Case((("quiet",),), "model_error", True, complete=False, ends=0, idle=0.2), + "say_raises": _Case( + (("raise", RuntimeError("stream broke")),), "model_error", True, complete=False, ends=0 + ), + "status_400": _Case( + (_done(outcome="failed", is_error=True, api_error_status=400),), "model_error", False + ), + "status_408": _Case( + (_done(outcome="failed", is_error=True, api_error_status=408),), "model_error", True + ), + "status_409": _Case( + (_done(outcome="failed", is_error=True, api_error_status=409),), "model_error", True + ), + "status_429": _Case( + (_done(outcome="failed", is_error=True, api_error_status=429),), "model_error", True + ), + "status_500": _Case( + (_done(outcome="failed", is_error=True, api_error_status=500),), "model_error", True + ), + "is_error_without_status": _Case( + (_done(outcome="failed", is_error=True, turns=3),), "model_error", True + ), + "is_error_without_status_on_the_last_turn": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=40, + errors=["The connection was reset by the server."], + ), + ), + "model_error", + True, + ), + "call_limit_words_inside_other_text": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=7, + errors=[f"Retry failed: {CALL_LIMIT_MESSAGE} (upstream)"], + ), + ), + "model_error", + True, + ), + "call_limit_words_without_a_number": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=7, + errors=["Max number of llm calls limit of `` exceeded"], + ), + ), + "model_error", + True, + ), + "claude_turn_limit": _Case( + (_done(outcome="error_max_turns", is_error=True, turns=40),), "no_answer" + ), + "gemini_max_turns": _Case( + (_done(outcome="max_turns", is_error=True, turns=12),), "no_answer" + ), + "gemini_call_limit": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=40, + errors=[CALL_LIMIT_MESSAGE], + ), + ), + "no_answer", + ), + "gemini_call_limit_in_adks_words": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=7, + errors=[CALL_LIMIT_MESSAGE], + ), + ), + "no_answer", + ), + "empty": _Case((("call", SUBMIT, {"agents": []}), FINISHED), "empty"), + "every_pair_refused": _Case( + (("call", SUBMIT, {"agents": [{"folder": "", "entry": "missing.py"}]}), FINISHED), + "empty", + ), + "no_answer": _Case((("say", "I looked around."), FINISHED), "no_answer"), +} + + +@pytest.mark.parametrize("name", sorted(_OUTCOMES)) +def test_each_ai_outcome_gives_its_result(tmp_path, monkeypatch, quick, name): + case = _OUTCOMES[name] + if case.idle is not None: + monkeypatch.setattr(scan_ai, "IDLE_SECONDS", case.idle) + monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", 1) + backend = _Backend(*case.steps, interrupt=case.interrupt) + ends: list = [] + + outcome = _scan(monkeypatch, _repo(tmp_path), backend, seconds=case.seconds, ends=ends) + + assert outcome.kind == case.kind + assert outcome.pairs == case.pairs + assert outcome.complete is case.complete + assert outcome.retryable is case.retryable + assert len(ends) == case.ends + # The scan's stage turns replay off for itself, never for the whole process. + assert session.STAGE_IDLE_RETRIES == 1 + assert len(backend.sent) == 1 + if case.kind == "timed_out": + assert backend.interrupts == 1 + + +def test_a_gemini_call_limit_is_no_answer(tmp_path, monkeypatch): + """The real Vertex-Gemini backend and ADK, with only the model scripted: it never submits.""" + pytest.importorskip("google.adk") + from google.adk.models.base_llm import BaseLlm + from google.adk.models.llm_response import LlmResponse + from google.genai import types + + from fi.alk.harness.backends import vertex_gemini + + calls: list[int] = [] + + class LoopingLlm(BaseLlm): + async def generate_content_async(self, llm_request, stream=False): + calls.append(1) + yield LlmResponse( + content=types.Content( + role="model", + parts=[ + types.Part( + function_call=types.FunctionCall( + name=GLOB, args={"pattern": "*.py"}, id=f"c{len(calls)}" + ) + ) + ], + ), + usage_metadata=types.GenerateContentResponseUsageMetadata( + prompt_token_count=1000, candidates_token_count=10 + ), + ) + + # The backend writes these three itself; set here so they are put back afterwards. + for name, value in ( + ("GOOGLE_CLOUD_PROJECT", "fake-project"), + ("GOOGLE_CLOUD_LOCATION", "global"), + ("GOOGLE_GENAI_USE_VERTEXAI", "TRUE"), + ("ALK_HARNESS_MODEL", "gemini-2.5-flash"), + ): + monkeypatch.setenv(name, value) + monkeypatch.delenv("AGENTCC_API_KEY", raising=False) + monkeypatch.setattr(vertex_gemini, "COMPACT_ABOVE_TOKENS", 0) + monkeypatch.setattr(vertex_gemini, "_llm", lambda model: LoopingLlm(model=model)) + for name, seconds in ( + ("INTERRUPT_SECONDS", 0.2), + ("SPEND_WAIT_SECONDS", 0.3), + ("CANCEL_SECONDS", 0.2), + ("EXIT_SECONDS", 1.0), + ("OPEN_SECONDS", 5.0), + ): + monkeypatch.setattr(scan_ai, name, seconds) + ends: list = [] + + outcome = _scan( + monkeypatch, + _repo(tmp_path), + vertex_gemini.VertexGeminiBackend(), + seconds=8.0, + ends=ends, + ) + + assert outcome.kind == "no_answer" + assert outcome.complete is True + assert len(ends) == 1 + assert 1 <= len(calls) <= scan_ai.MAX_TURNS + 1 + assert "llm calls" not in outcome.message.lower() + + +def test_an_idle_ai_stream_is_not_replayed(tmp_path, monkeypatch, quick): + monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", 1) + monkeypatch.setattr(scan_ai, "IDLE_SECONDS", 0.2) + backend = _Backend(("quiet",)) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "model_error" + assert outcome.retryable is True + assert len(backend.specs) == 1 + assert len(backend.sent) == 1 + # The scan's stage turns replay off for itself, never for the whole process. + assert session.STAGE_IDLE_RETRIES == 1 + + +def test_pairs_submitted_before_an_idle_stream_are_kept(tmp_path, monkeypatch, quick): + monkeypatch.setattr(scan_ai, "IDLE_SECONDS", 0.2) + backend = _Backend(KEPT, ("quiet",)) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.complete is False + + +# A model that ends without submitting is the turn a reminder would be tempted to follow. +@pytest.mark.parametrize( + ("steps", "kind"), + [((("say", "Looking."), KEPT, FINISHED), "pairs"), ((("say", "Looking."), FINISHED), "no_answer")], + ids=["submitted", "ended_without_submitting"], +) +def test_the_stage_says_exactly_once(tmp_path, monkeypatch, quick, steps, kind): + backend = _Backend(*steps) + ends: list = [] + hints = "3 Python files outside side folders" + + outcome = _scan(monkeypatch, _repo(tmp_path), backend, hints=hints, ends=ends) + + assert outcome.kind == kind + assert len(backend.specs) == 1 + assert backend.sent == [scan_ai._opening(hints)] + assert len(ends) == 1 + + +def test_complete_is_false_after_the_stop_sequence(tmp_path, monkeypatch, quick): + assert _BOUNDS["INTERRUPT_SECONDS"] == 10 + assert _BOUNDS["SPEND_WAIT_SECONDS"] == 60 + backend = _Backend(KEPT, ("until_interrupt",), FINISHED) + ends: list = [] + + outcome = _scan(monkeypatch, _repo(tmp_path), backend, seconds=0.3, ends=ends) + + assert outcome.kind == "pairs" + assert backend.interrupts == 1 + # The bill arrived during the wait, and still the turn may be missing a call in flight. + assert len(ends) == 1 + assert outcome.complete is False + + +@pytest.mark.parametrize("start", ["hang", "raise"]) +def test_a_stage_that_will_not_open_is_a_model_error(tmp_path, monkeypatch, quick, start): + assert _BOUNDS["OPEN_SECONDS"] == 30 + backend = _Backend(KEPT, FINISHED, start=start) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "model_error" + assert outcome.retryable is True + assert outcome.complete is False + assert backend.sent == [] + + +def test_a_stage_that_will_not_stop_is_left_after_ten_seconds(tmp_path, monkeypatch, quick): + assert _BOUNDS["EXIT_SECONDS"] == 10 + backend = _Backend(KEPT, FINISHED, stop="hang") + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "pairs" + assert outcome.complete is True + assert backend.stops == 1 + + +def test_the_opening_carries_no_path_and_the_outcome_no_model_text(tmp_path, monkeypatch, quick): + source = _repo(tmp_path) + hints = "2 Python files outside side folders; 1 candidate in side folders" + model_text = "MODEL-TEXT services/support/agent.py says sk-fake-123" + assert "/" not in scan_ai._opening("") + + failed = _Backend( + ("say", model_text), + _done(outcome="failed", is_error=True, turns=2, errors=[model_text]), + ) + outcome = _scan(monkeypatch, source, failed, hints=hints) + assert outcome.kind == "model_error" + assert failed.sent == [scan_ai._opening(hints)] + assert failed.sent[0].endswith(hints) + for path in (str(source), str(tmp_path), "services/", "tests/", "agent.py"): + assert path not in failed.sent[0] + + raised = _Backend(("raise", RuntimeError(model_text)),) + raised_outcome = _scan(monkeypatch, source, raised, hints=hints) + assert raised_outcome.message == "the model step failed (RuntimeError)" + + answered = _Backend(("say", model_text), KEPT, FINISHED) + kept = _scan(monkeypatch, source, answered, hints=hints) + assert kept.pairs == (AGENT,) + + for one in (outcome, raised_outcome, kept): + assert "MODEL-TEXT" not in one.message + assert "sk-fake-123" not in one.message + + +def test_the_stage_has_no_built_in_tools(tmp_path, monkeypatch, quick): + source = _repo(tmp_path) + backend = _Backend(FINISHED) + + _scan(monkeypatch, source, backend) + + spec = backend.specs[0] + assert spec.builtins == () + assert spec.ask is None + assert spec.gated is True + assert spec.thinking is False + assert spec.max_turns == 40 + assert spec.idle_timeout_seconds == 90 + assert spec.cwd == str(source) + assert spec.model == GEMINI + assert spec.system_prompt == load_skill("find-agents") + assert set(spec.servers) == {"scan_files", "scan"} + assert sorted(spec.granted()) == sorted([READ, GLOB, GREP, SUBMIT]) + for name in (READ, GLOB, GREP, SUBMIT): + assert name in spec.system_prompt + + +def test_the_skills_description_says_when_the_model_is_asked(): + # The model reads the whole file, its description included. + skill = load_skill("find-agents", preamble=False) + description = yaml.safe_load(skill.split("---\n")[1])["description"] + + assert "came back empty" not in skill + assert "found no agent outside test and example folders" in description + assert "apart from any files that only call a model" in description + assert "TypeScript or JavaScript" in description + assert skill in load_skill("find-agents") + + +# -- the tools --------------------------------------------------------------------------------- + + +def _tools(root: Path) -> dict[str, Any]: + return {spec.name: spec.handler for spec in scan_ai.rooted_file_tools(root).tools} + + +def _call(handler, arguments: Any) -> dict[str, Any]: + return asyncio.run(asyncio.wait_for(handler(arguments), TEST_BOUND)) + + +def _listed(text: str) -> set[str]: + """The paths a glob or grep result names.""" + return {line.split(":", 1)[0] for line in text.splitlines() if line} + + +def test_the_ai_cannot_read_outside_the_source(tmp_path): + outside = _write(tmp_path / "outside/creds.txt", "outside-needle\n") + source = tmp_path / "repo" + _write(source / "inside.py", "inside-needle\n") + os.symlink(outside.parent, source / "out_link") + os.symlink(outside, source / "file_link.py") + tools = _tools(source) + + for path in ( + "../outside/creds.txt", + str(outside), + "out_link/creds.txt", + "file_link.py", + "inside.py/../../outside/creds.txt", + ): + result = _call(tools["read"], {"path": path}) + assert result.get("is_error"), path + assert "outside-needle" not in _said(result) + for path in ("../outside", "/", str(tmp_path), "out_link"): + assert _call(tools["glob"], {"pattern": "*", "path": path}).get("is_error"), path + assert _call(tools["grep"], {"text": "needle", "path": path}).get("is_error"), path + + assert _listed(_said(_call(tools["glob"], {"pattern": "*"}))) == {"inside.py"} + found = _said(_call(tools["grep"], {"text": "needle"})) + assert "outside-needle" not in found + assert _listed(found) == {"inside.py"} + assert "inside-needle" in _said(_call(tools["read"], {"path": str(source / "inside.py")})) + + +_HIDDEN = ( + ".env", + "config/service_account.json", + "config/token.json", + "prod.env", + "app.env.local", + "secrets.toml", + "credentials.yml", + "id_rsa", + "terraform.tfvars", + "secrets/keys.py", +) + + +def test_grep_and_glob_never_show_a_secret_file(tmp_path): + source = tmp_path / "repo" + _write(source / "agent.py", "needle = 1\n") + _write(source / "config/settings.py", "needle = 2\n") + for name in _HIDDEN: + _write(source / name, "needle API_KEY=sk-fake-123\n") + tools = _tools(source) + + for path in (None, ".", "config", str(source), str(source / "config")): + arguments = {} if path is None else {"path": path} + for grep in ({"text": "needle"}, {"text": "NEEDLE", "ignore_case": True}): + found = _said(_call(tools["grep"], {**grep, **arguments})) + assert "sk-fake-123" not in found + assert not _listed(found) & set(_HIDDEN), (path, found) + assert _listed(found) <= {"agent.py", "config/settings.py"} + for pattern in ("*", "**", "*.env", "*.json", "id_*", "*secret*"): + listed = _said(_call(tools["glob"], {"pattern": pattern, **arguments})) + assert not _listed(listed) & set(_HIDDEN), (path, pattern, listed) + assert "agent.py" in _listed(_said(_call(tools["grep"], {"text": "needle"}))) + for name in _HIDDEN: + result = _call(tools["read"], {"path": name}) + assert result.get("is_error"), name + assert "sk-fake-123" not in _said(result) + + +def test_the_tools_skip_fifos_and_big_files_and_hard_links(tmp_path): + source = tmp_path / "repo" + _write(source / "agent.py", "needle\n") + os.mkfifo(source / "pipe.py") + _write(source / "big.py", "needle\n" * 300_000) + _write(source / "linked.py", "needle\n") + os.link(source / "linked.py", source / "also.py") + os.link(_write(tmp_path / "outside.py", "needle\n"), source / "hard.py") + _write(source / "app.min.js", "needle\n") + _write(source / "long.py", "needle" + "x" * 70_000 + "\nneedle\n") + tools = _tools(source) + + assert _listed(_said(_call(tools["glob"], {"pattern": "*"}))) == {"agent.py", "long.py"} + found = _said(_call(tools["grep"], {"text": "needle"})) + assert found.splitlines() == ["agent.py:1:needle", "long.py:2:needle"] + for name in ("pipe.py", "big.py", "linked.py", "also.py", "hard.py"): + assert _call(tools["read"], {"path": name}).get("is_error"), name + assert _call(tools["grep"], {"text": "needle", "path": name}).get("is_error"), name + + +def test_a_slow_tool_call_ends_with_an_error(tmp_path, monkeypatch): + assert _BOUNDS["TOOL_SECONDS"] == 20 + source = _repo(tmp_path) + monkeypatch.setattr(scan_ai, "TOOL_SECONDS", 0.2) + monkeypatch.setattr(scan_ai, "_grep_files", lambda root, arguments: time.sleep(3) or "late") + tools = _tools(source) + + began = time.monotonic() + result = _call(tools["grep"], {"text": "needle"}) + + assert time.monotonic() - began < 2 + assert result.get("is_error") + assert "late" not in _said(result) + + +def test_no_tool_raises(tmp_path, monkeypatch): + source = _repo(tmp_path) + tools = _tools(source) + calls = [ + ("read", {"path": "main\x00.py"}), + ("read", {"path": "main.py", "offset": "x"}), + ("read", {"path": "main.py", "limit": [1]}), + ("read", {"path": "/".join(["a"] * 3_000)}), + ("read", {"path": "a" * 5_000}), + ("read", {"path": 7}), + ("read", {}), + ("read", None), + ("glob", {"pattern": "x" * 300}), + ("glob", {"pattern": None}), + ("glob", {"pattern": "*", "path": "\x00"}), + ("grep", {"text": ""}), + ("grep", {"text": "x" * 201}), + ("grep", {"text": "a", "ignore_case": "yes"}), + ("grep", {"text": "a", "glob": 5}), + ("grep", {"text": "a", "path": "missing/folder"}), + ] + for name, arguments in calls: + assert _call(tools[name], arguments).get("is_error"), (name, arguments) + + def broken(*_args): + raise RuntimeError("MODEL-TEXT from inside a worker") + + for worker in ("_read_lines", "_glob_files", "_grep_files", "_checked_pairs"): + monkeypatch.setattr(scan_ai, worker, broken) + for name, arguments in ( + ("read", {"path": "main.py"}), + ("glob", {"pattern": "*"}), + ("grep", {"text": "a"}), + ): + result = _call(tools[name], arguments) + assert result.get("is_error") + assert _said(result) == scan_ai._FAILED + submit = scan_ai._submit_server(source, scan_ai._Answer()).tools[0].handler + submitted = _call(submit, {"agents": [{"folder": "", "entry": "main.py"}]}) + assert submitted.get("is_error") + assert _said(submitted) == scan_ai._FAILED + + +def test_tool_paths_are_relative(tmp_path): + source = tmp_path / "repo" + _write(source / "pkg/agent.py", "needle\n") + tools = _tools(source) + + for path in (None, "pkg", str(source / "pkg")): + arguments = {} if path is None else {"path": path} + listed = _said(_call(tools["glob"], {"pattern": "*.py", **arguments})) + found = _said(_call(tools["grep"], {"text": "needle", **arguments})) + assert listed == "pkg/agent.py" + assert found == "pkg/agent.py:1:needle" + for text in (listed, found): + assert str(tmp_path) not in text + assert _said(_call(tools["read"], {"path": "pkg/agent.py"})) == "1\tneedle" + + +def test_a_capped_search_says_it_stopped_early(tmp_path): + source = tmp_path / "repo" + for index in range(2_100): + _write(source / f"many/f{index:04}.txt", "") + tools = _tools(source) + + for name, arguments in ( + ("glob", {"pattern": "*.nothing"}), + ("grep", {"text": "nothing-here"}), + ): + said = _said(_call(tools[name], arguments)) + assert said == scan_ai.STOPPED_EARLY, (name, said) + + full = tmp_path / "full" + _write(full / "hits.py", "".join(f"needle {index}\n" for index in range(2_000))) + _write(full / "wide.py", "".join(f"{'y' * 100}\n" for _ in range(400))) + full_tools = _tools(full) + for name, arguments in ( + ("grep", {"text": "needle"}), + ("read", {"path": "wide.py"}), + ): + said = _said(_call(full_tools[name], arguments)) + assert said.endswith(scan_ai.STOPPED_EARLY), name + assert len(said) <= 8_000, name + + +def test_one_long_hit_does_not_hide_the_others(tmp_path): + source = tmp_path / "repo" + _write(source / "a.py", "needle" + "x" * 20_000 + "\n") + for name in ("b.py", "c.py", "d.py", "e.py"): + _write(source / name, "needle\n") + tools = _tools(source) + + found = _said(_call(tools["grep"], {"text": "needle"})) + + assert _listed(found) == {"a.py", "b.py", "c.py", "d.py", "e.py"} + assert scan_ai.STOPPED_EARLY not in found + for line in found.splitlines(): + assert len(line.split(":", 2)[2]) <= 300 + + +# -- the answer -------------------------------------------------------------------------------- + + +def _submit(source: Path, agents: Any) -> tuple[str, Any]: + answer = scan_ai._Answer() + handler = scan_ai._submit_server(source, answer).tools[0].handler + return _said(_call(handler, {"agents": agents})), answer + + +def test_the_models_paths_are_checked(tmp_path): + source = _repo(tmp_path) + _write(source / "node_modules/pkg/agent.js", "x\n") + _write(source / "build/agent.py", "x\n") + _write(source / "services/other/run.py", "x\n") + evil = _write(tmp_path / "outside/evil.py", "x\n") + os.symlink(evil, source / "services/support/evil.py") + os.symlink(evil.parent, source / "linked") + + for folder, entry, kept in ( + ("", "main.py", ("", "main.py")), + ("", AGENT[1], ("", AGENT[1])), + (AGENT[0], AGENT[1], AGENT), + ("./" + AGENT[0], "./" + AGENT[1], AGENT), + (f"{source}/{AGENT[0]}", f"{source}/{AGENT[1]}", AGENT), + (str(source), f"{source}/main.py", ("", "main.py")), + ): + said, answer = _submit(source, [{"folder": folder, "entry": entry}]) + assert said.startswith("Recorded 1; 0 refused."), (folder, entry, said) + assert answer.pairs == (kept,) + + for agent in ( + {"folder": "services/other", "entry": AGENT[1]}, + {"folder": "services/sup", "entry": AGENT[1]}, + {"folder": "", "entry": "../outside/evil.py"}, + {"folder": "", "entry": str(evil)}, + {"folder": AGENT[0], "entry": "services/support/evil.py"}, + {"folder": "linked", "entry": "linked/evil.py"}, + {"folder": "", "entry": AGENT[0]}, + {"folder": "", "entry": "missing.py"}, + {"folder": "", "entry": "node_modules/pkg/agent.js"}, + {"folder": "", "entry": "build/agent.py"}, + {"folder": "", "entry": "main.py\n"}, + {"folder": "", "entry": ""}, + {"folder": "../", "entry": "main.py"}, + {"folder": 1, "entry": "main.py"}, + "main.py", + ): + said, answer = _submit(source, [agent]) + assert said.startswith("Recorded 0; 1 refused."), (agent, said) + assert answer.pairs == () + assert answer.called is True + + said, answer = _submit(source, [{"folder": "", "entry": "main.py"}] * 2) + assert said.startswith("Recorded 1; 0 refused.") + assert answer.pairs == (("", "main.py"),) + + said, answer = _submit(source, [{"folder": "", "entry": "main.py"}] * 51) + assert answer.pairs == () + assert answer.called is False + + +def test_the_first_kept_submit_is_final_and_the_tools_stop(tmp_path, monkeypatch, quick): + backend = _Backend( + ("call", SUBMIT, {"agents": [{"folder": "", "entry": "missing.py"}]}), + KEPT, + ("call", SUBMIT, {"agents": [{"folder": "", "entry": "main.py"}]}), + ("call", READ, {"path": "main.py"}), + ("call", GLOB, {"pattern": "*"}), + ("call", GREP, {"text": "print"}), + FINISHED, + ) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + said = [_said(result) for result in backend.results] + assert said[0].startswith("Recorded 0; 1 refused.") + assert said[1] == "Recorded 1; 0 refused. Your work is done: end your turn now." + assert said[2] == "Already submitted." + assert said[3:] == ["Already submitted: end your turn."] * 3 + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.refused == 0 + + +def test_a_refused_pair_in_the_kept_submit_is_counted(tmp_path, monkeypatch, quick): + agents = [ + {"folder": AGENT[0], "entry": AGENT[1]}, + {"folder": "", "entry": "missing.py"}, + {"folder": "services/other", "entry": AGENT[1]}, + ] + backend = _Backend(("call", SUBMIT, {"agents": agents}), FINISHED) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert _said(backend.results[0]).startswith("Recorded 1; 2 refused.") + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.refused == 2 + + +def test_an_entry_reached_through_a_link_out_of_the_source_is_refused(tmp_path): + source = _repo(tmp_path) + _write(tmp_path / "outside/other.py", "x\n") + # A link in the middle of the path: the entry itself is a regular file, and its folder is + # inside the source. + os.symlink(tmp_path / "outside", source / "services/support/ext") + + for folder in (AGENT[0], ""): + said, answer = _submit(source, [{"folder": folder, "entry": f"{AGENT[0]}/ext/other.py"}]) + assert said.startswith("Recorded 0; 1 refused."), (folder, said) + assert answer.pairs == () + + +def test_the_tools_work_when_finished_is_none(tmp_path): + source = _repo(tmp_path) + tools = { + spec.name: spec.handler + for spec in scan_ai.rooted_file_tools(source, finished=None).tools + } + + result = _call(tools["read"], {"path": "main.py"}) + + assert not result.get("is_error"), _said(result) + assert "hello" in _said(result) + assert not _call(tools["glob"], {"pattern": "*.py"}).get("is_error") + + +def test_two_submits_at_once_keep_exactly_one_set(tmp_path, monkeypatch): + source = _repo(tmp_path) + + async def checked_after_a_yield(work, *args): + # A thread's answer comes back on a later turn of the loop; yielding once here makes + # the two calls interleave the same way on every run. + await asyncio.sleep(0) + return work(*args) + + monkeypatch.setattr(scan_ai, "_in_thread", checked_after_a_yield) + answer = scan_ai._Answer() + handler = scan_ai._submit_server(source, answer).tools[0].handler + first = [{"folder": AGENT[0], "entry": AGENT[1]}] + second = [{"folder": "", "entry": "main.py"}] + + async def both() -> list[dict[str, Any]]: + return list(await asyncio.gather(handler({"agents": first}), handler({"agents": second}))) + + said = [_said(result) for result in asyncio.run(asyncio.wait_for(both(), TEST_BOUND))] + + recorded = "Recorded 1; 0 refused. Your work is done: end your turn now." + assert sorted(said) == sorted([recorded, scan_ai.SUBMIT_AFTER_SUBMIT]) + assert answer.pairs == ((AGENT,), (("", "main.py"),))[said.index(recorded)] + + +# -- a path the platform can't carry (the vercel/ai-chatbot run, 6 Nov) ----------------------- + +ROUTE = "app/(chat)/api/chat/route.ts" + + +def _route_repo(tmp_path: Path) -> Path: + source = _repo(tmp_path) + _write(source / ROUTE, "export async function POST() {}\n") + return source + + +def test_a_real_file_the_path_rule_refuses_is_counted_once_and_named( + tmp_path, monkeypatch, quick +): + backend = _Backend( + ("call", SUBMIT, {"agents": [{"folder": "", "entry": ROUTE}]}), + ("call", SUBMIT, {"agents": [{"folder": "app", "entry": "./" + ROUTE}]}), + FINISHED, + ) + + outcome = _scan(monkeypatch, _route_repo(tmp_path), backend) + + said = [_said(result) for result in backend.results] + for line in said: + assert line.startswith("Recorded 0; 1 refused. 1 name a file whose path the platform") + assert "don't submit them again" in line + assert outcome.kind == "empty" + assert outcome.uncarried == 1 + + +@pytest.mark.parametrize( + "entry", + ["app/(chat)/missing.ts", "../outside/(x).ts", "/etc/(x).ts", "node_modules/(x)/a.ts"], +) +def test_a_missing_or_outside_file_is_not_counted_as_uncarried( + tmp_path, monkeypatch, quick, entry +): + source = _route_repo(tmp_path) + _write(source / "node_modules/(x)/a.ts", "x\n") + backend = _Backend(("call", SUBMIT, {"agents": [{"folder": "", "entry": entry}]}), FINISHED) + + outcome = _scan(monkeypatch, source, backend) + + assert "can't carry" not in _said(backend.results[0]) + assert outcome.kind == "empty" + assert outcome.uncarried == 0 + + +def test_an_uncarried_file_before_a_kept_submit_is_still_counted(tmp_path, monkeypatch, quick): + backend = _Backend( + ("call", SUBMIT, {"agents": [{"folder": "", "entry": ROUTE}]}), + KEPT, + FINISHED, + ) + + outcome = _scan(monkeypatch, _route_repo(tmp_path), backend) + + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.refused == 0 + assert outcome.uncarried == 1 + + +def test_glob_reads_a_pattern_as_a_shell_would(tmp_path): + source = _route_repo(tmp_path) + _write(source / "package.json", "{}\n") + _write(source / "web/package.json", "{}\n") + _write(source / "web/src/page.tsx", "x\n") + tools = _tools(source) + + def globbed(pattern: str, **extra: str) -> set[str]: + return _listed(_said(_call(tools["glob"], {"pattern": pattern, **extra}))) + + assert globbed("**/package.json") == {"package.json", "web/package.json"} + assert globbed("**/*.{ts,tsx}") == {ROUTE, "web/src/page.tsx"} + assert globbed("web/**/page.tsx") == {"web/src/page.tsx"} + assert globbed("*.{py,json}") >= {"main.py", "package.json", AGENT[1]} + assert _said(_call(tools["glob"], {"pattern": "{"})) == "No file matches." + assert len(scan_ai._pattern_forms("{a,b}" * 20)) <= scan_ai._MAX_FORMS + found = _said(_call(tools["grep"], {"text": "POST", "glob": "**/*.{ts,tsx}"})) + assert _listed(found) == {ROUTE} + + +def test_a_failed_cost_note_keeps_the_answer_and_logs_its_trace(tmp_path, monkeypatch, quick, caplog): + """A paid answer is never turned into a retry by the cost note, and the note's fault is + logged with its trace so it can be found.""" + backend = _Backend(KEPT, FINISHED) + monkeypatch.setattr(session, "resolve", lambda *_args, **_kwargs: backend) + + def broken_note() -> None: + raise OSError("cost file not writable") + + with caplog.at_level(logging.WARNING, logger=scan_ai.logger.name): + outcome = asyncio.run( + asyncio.wait_for( + scan_ai.run_ai_step( + _repo(tmp_path), hints="x", seconds=5.0, on_turn_end=broken_note + ), + TEST_BOUND, + ) + ) + + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert "the cost note after the turn failed" in caplog.text + assert "Traceback" in caplog.text and "OSError" in caplog.text + + +def test_a_tool_fault_logs_the_workers_own_error(tmp_path, monkeypatch, caplog): + """The model is told only that the call failed; the log carries the worker's real fault.""" + + def broken(root, arguments): + raise KeyError("a fault in the glob worker") + + monkeypatch.setattr(scan_ai, "_glob_files", broken) + tools = _tools(_repo(tmp_path)) + with caplog.at_level(logging.WARNING, logger=scan_ai.logger.name): + said = _said(_call(tools["glob"], {"pattern": "*"})) + + assert said == scan_ai._FAILED + assert "a tool call failed" in caplog.text + assert "KeyError" in caplog.text and "a fault in the glob worker" in caplog.text diff --git a/tests/harness/test_scan_document.py b/tests/harness/test_scan_document.py new file mode 100644 index 000000000..f44aa263e --- /dev/null +++ b/tests/harness/test_scan_document.py @@ -0,0 +1,605 @@ +from __future__ import annotations + +import json +import logging +import os +import re +import stat +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from fi.alk.harness import scan_document as sd +from fi.alk.harness.scan_document import ( + Row, + ScanResultInvalid, + choose_rows, + finish, + root_name, + wire_problems, + write_agents_json, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "agent_scan_e07d56fcd.json" +SHA = "0123456789abcdef0123456789abcdef01234567" +PROBLEM_NAME = re.compile(r"[a-z0-9_.\[\]]+") + + +@dataclass(frozen=True) +class _Candidate: + """The walk's candidate, with the fields the document reads.""" + + entry: str + subfolder: str = "" + sign: str = "framework" + side: bool = False + builds: tuple[str, ...] = ("langgraph",) + build_may_stop: str = "" + partial: bool = False + env_names: tuple[str, ...] = ("OPENAI_API_KEY",) + tools: int = 2 + prompt_files: int = 1 + evidence: tuple[tuple[str, str, int], ...] = () + + +def _framework(name: str = "langgraph", label: str | None = None) -> dict[str, Any]: + return {"name": name, "version": None, "label": label} + + +def _row( + entry: str, + subfolder: str = "", + *, + framework: dict[str, Any] | None = None, + finished: bool = True, + found_by_ai: bool = False, + **candidate: Any, +) -> Row: + return Row( + candidate=_Candidate(entry=entry, subfolder=subfolder, **candidate), + framework=framework if framework is not None else _framework(), + finished=finished, + found_by_ai=found_by_ai, + ) + + +def _finish( + rows: list[Row], + *, + root: str = "shop", + commit_sha: str | None = SHA, + truncated: bool = False, + error: tuple[str, str] | None = None, +) -> tuple[dict[str, Any], bytes, bool]: + body, dropped = finish( + rows=rows, root_name=root, commit_sha=commit_sha, truncated=truncated, error=error + ) + document = json.loads(body) + assert wire_problems(document) == [] + return document, body, dropped + + +def _confidence(*rows: Row) -> list[str]: + document, _, _ = _finish(list(rows)) + return [agent["confidence"] for agent in document["agents"]] + + +def test_found_lists_each_agent_in_the_platform_format() -> None: + graph = _row( + "services/support/graph.py", + "services/support", + env_names=("OPENAI_API_KEY", "SUPPORT_DB_URL"), + tools=3, + evidence=(("builds langgraph", "services/support/graph.py", 12),), + ) + root = _row( + "main.py", + framework=_framework("other", "llama_index"), + builds=("llama_index",), + tools=0, + prompt_files=0, + ) + + document, _, dropped = _finish([root, graph]) + + assert dropped is False + assert list(document) == [ + "schema_version", + "status", + "commit_sha", + "agents", + "truncated", + "error", + ] + assert document == { + "schema_version": "futureagi.agent-scan.v1", + "status": "found", + "commit_sha": SHA, + "agents": [ + { + "name": "support", + "subfolder": "services/support", + "entry": "services/support/graph.py", + "framework": {"name": "langgraph", "version": None, "label": None}, + "confidence": "high", + "evidence": ["Builds with LangGraph in services/support/graph.py, line 12."], + "env_names": ["OPENAI_API_KEY", "SUPPORT_DB_URL"], + "estimates": {"tools": 3, "prompt_files": 1}, + }, + { + "name": "shop", + "subfolder": "", + "entry": "main.py", + "framework": {"name": "other", "version": None, "label": "llama_index"}, + "confidence": "medium", + "evidence": [], + "env_names": ["OPENAI_API_KEY"], + "estimates": {"tools": 0, "prompt_files": 0}, + }, + ], + "truncated": False, + "error": None, + } + assert [list(agent) for agent in document["agents"]] == [list(sd._AGENT_KEYS)] * 2 + assert sd._AGENT_KEYS == ( + "name", + "subfolder", + "entry", + "framework", + "confidence", + "evidence", + "env_names", + "estimates", + ) + + none, _, _ = _finish([]) + assert (none["status"], none["agents"], none["error"]) == ("none", [], None) + error, _, _ = _finish([], error=("scan_timed_out", "The scan ran out of time.")) + assert error["status"] == "error" + assert error["error"] == {"code": "scan_timed_out", "message": "The scan ran out of time."} + assert error["truncated"] is False + + +def test_confidence_rules() -> None: + assert _confidence(_row("a.py", "a")) == ["high"] + assert _confidence(_row("a.py", "a", builds=("langchain",))) == ["medium"] + assert _confidence(_row("a.py", "a", framework=_framework("other", "dspy"))) == ["medium"] + assert _confidence(_row("a.py", "a", framework=_framework("unknown"))) == ["low"] + assert _confidence(_row("a.py", "a", framework=_framework("none"), builds=())) == ["low"] + assert _confidence(_row("a.py", "a", found_by_ai=True)) == ["low"] + assert _confidence(_row("a.py", "a", build_may_stop="graphs")) == ["low"] + assert _confidence(_row("a.py", "a", build_may_stop="agent_py")) == ["low"] + assert _confidence(_row("a.py", "a", partial=True)) == ["low"] + assert _confidence(_row("a.py", "a", finished=False)) == ["low"] + # Two agents in one folder both build cleanly, but a build of that folder can't pick one. + assert _confidence(_row("a/one.py", "a"), _row("a/two.py", "a")) == ["low", "low"] + assert _confidence(_row("a/one.py", "a"), _row("b/two.py", "b")) == ["high", "high"] + + document, _, _ = _finish( + [ + _row("tests/t.py", "tests", side=True), + _row("c/low.py", "c", framework=_framework("unknown")), + _row("b/medium.py", "b", builds=()), + _row("z/high.py", "z"), + _row("a/high.py", "a"), + ] + ) + assert [agent["entry"] for agent in document["agents"]] == [ + "a/high.py", + "z/high.py", + "b/medium.py", + "c/low.py", + "tests/t.py", + ] + + +def test_sharing_a_folder_is_decided_over_the_rows_written() -> None: + shares = sd._TEMPLATES["shares_folder"] + + # The neighbour breaks the platform's path rule, so it isn't written and nothing is shared. + document, _, dropped = _finish([_row("svc/main.py", "svc"), _row("svc/../x.py", "svc")]) + assert dropped is True + assert [agent["entry"] for agent in document["agents"]] == ["svc/main.py"] + assert document["agents"][0]["confidence"] == "high" + assert shares not in document["agents"][0]["evidence"] + + # The neighbour is the fifty-first, cut by the cap. + rows = [_row(f"f{i:02}/main.py", f"f{i:02}") for i in range(50)] + rows.append(_row("f00/tests/test_main.py", "f00", side=True)) + document, _, dropped = _finish(rows) + assert dropped is True + assert len(document["agents"]) == 50 + assert {agent["confidence"] for agent in document["agents"]} == {"high"} + + # The neighbour repeats an entry already listed. + document, _, _ = _finish( + [_row("svc/main.py", "svc"), _row("svc/main.py", "svc", found_by_ai=True)] + ) + assert [(agent["entry"], agent["confidence"]) for agent in document["agents"]] == [ + ("svc/main.py", "high") + ] + + document, _, _ = _finish([_row("svc/main.py", "svc"), _row("svc/other.py", "svc")]) + assert all(shares in agent["evidence"] for agent in document["agents"]) + + +def test_an_uppercase_commit_is_lowercased() -> None: + assert _finish([_row("a.py")], commit_sha=SHA.upper())[0]["commit_sha"] == SHA + assert _finish([_row("a.py")], commit_sha="0123456789ABCdef" * 2 + "01234567")[0][ + "commit_sha" + ] == SHA + assert _finish([_row("a.py")], commit_sha=None)[0]["commit_sha"] is None + assert _finish([_row("a.py")], commit_sha=SHA[:39])[0]["commit_sha"] is None + assert _finish([_row("a.py")], commit_sha="g" * 40)[0]["commit_sha"] is None + assert _finish([_row("a.py")], commit_sha=SHA + "\n")[0]["commit_sha"] is None + + +def test_rows_and_an_error_together_keep_the_rows() -> None: + error = ("scan_timed_out", "The scan ran out of time.") + + document, _, dropped = _finish([_row("a/main.py", "a")], error=error) + assert document["status"] == "found" + assert document["error"] is None + assert document["truncated"] is True + assert [agent["entry"] for agent in document["agents"]] == ["a/main.py"] + assert dropped is False + + # Every row given breaks a rule: nothing is left to list, and the error was already dropped. + document, _, dropped = _finish([_row("a/../main.py", "a")], error=error) + assert (document["status"], document["agents"], document["error"]) == ("none", [], None) + assert document["truncated"] is True + assert dropped is True + + +def test_names_evidence_and_paths_always_fit_the_platform_rules() -> None: + def job(agent_name: Any = None, repository: Any = None) -> Any: + metadata = {} if agent_name is None else {"agent_name": agent_name} + return SimpleNamespace(metadata=metadata, source=SimpleNamespace(repository=repository)) + + assert root_name(job(" Support\u2003Bot\x00 ", "org/shop.git")) == "Support Bot" + assert root_name(job(3, "org/shop.git")) == "shop" + assert root_name(job("\u200b\u3000", "https://github.com/org/shop.git/")) == "shop" + assert root_name(job(None, "shop")) == "shop" + assert root_name(job(None, ".git")) == "agent" + assert root_name(job()) == "agent" + + document, _, _ = _finish( + [ + _row("a/svc/main.py", "a/svc"), + _row("b/svc/app.py", "b/svc"), + _row("c/api/main.py", "c/api"), + _row("d/api/main.py", "d/api"), + _row("main.py"), + _row("server.py"), + ], + root="shop", + ) + assert sorted(agent["name"] for agent in document["agents"]) == [ + "api (c/api/main.py)", + "api (d/api/main.py)", + "shop (main.py)", + "shop (server.py)", + "svc (app.py)", + "svc (main.py)", + ] + assert _finish([_row("main.py")], root="x" * 300)[0]["agents"][0]["name"] == "x" * 255 + assert _finish([_row("main.py")], root="\u200b\x00")[0]["agents"][0]["name"] == "agent" + + long_path = "deep/" * 90 + "graph.py" + facts = tuple(("builds langgraph", f"f{i}/graph.py", i + 1) for i in range(7)) + row = _row( + long_path, + "deep", + evidence=(("builds langgraph", long_path, 4), *facts), + env_names=tuple(f"NAME_{i}" for i in range(60)), + build_may_stop="graphs", + partial=True, + finished=False, + ) + agent = _finish([row])[0]["agents"][0] + assert len(agent["evidence"]) == 5 + assert all(0 < len(sentence) <= 200 for sentence in agent["evidence"]) + assert agent["evidence"][0].startswith("Builds with LangGraph in …/") + assert agent["evidence"][0].endswith("deep/graph.py, line 4.") + assert agent["evidence"][1:4] == [ + sd._TEMPLATES["graphs_may_stop"], + sd._TEMPLATES["not_finished"], + sd._TEMPLATES["partial"], + ] + assert agent["env_names"] == [f"NAME_{i}" for i in range(50)] + + assert sd._LONGEST_TEMPLATE == max(len(template) for template in sd._TEMPLATES.values()) + assert sd._LONGEST_TEMPLATE + sd._PATH_ROOM <= sd.EVIDENCE_MAX + assert sd._PATH_ROOM == 120 + + aside = (("openai_agents_set_aside", "", 0),) + assert sd._TEMPLATES["openai_agents_set_aside"] in _finish( + [_row("a/main.py", "a", framework=_framework("unknown"), evidence=aside)] + )[0]["agents"][0]["evidence"] + assert _finish( + [_row("a/main.py", "a", framework=_framework("openai_agents"), evidence=aside)] + )[0]["agents"][0]["evidence"] == [] + + +def test_clean_name_collapses_every_kind_of_space() -> None: + assert sd._clean_name("a\u2003\u3000b") == "a b" + assert sd._clean_name("\u3000a \t\n b\u2028c\u00a0d\u205f ") == "a b c d" + assert sd._clean_name("a\u200bb\x00c\x1bd\ue000e\ud800f\u202eg\ufeffh") == "abcdefgh" + assert sd._clean_name("\u3000\u2003\u200b") == "" + assert _finish([_row("main.py")], root="Shop\u3000\u2003Bot")[0]["agents"][0]["name"] == ( + "Shop Bot" + ) + + +def test_a_sentence_never_names_a_path_the_path_rule_refuses() -> None: + undecodable = os.fsdecode(b"agent\xff.py") + refused = [ + undecodable, + "../agent.py", + "/srv/agent.py", + "my agent.py", + "a//agent.py", + "agent.py\n", + "x" * 513, + "", + None, + ] + for kind in ("builds langgraph", "graph", "agent_py", "client openai", "project"): + for path in refused: + sentence = sd._sentence(kind, path, 3) + assert sentence is not None + assert sd._UNNAMED_PATH in sentence + if path: + assert path not in sentence + assert "\udcff" not in sentence + sentence.encode("utf-8") + assert len(sentence) <= 200 + + long_path = "d/" * 200 + "agent.py" + sentence = sd._sentence("builds langgraph", long_path, 3) + assert "…/" in sentence and sentence.endswith("/d/agent.py, line 3.") + assert len(sentence) <= 200 + + one_long_part = "a" * 300 + ".py" + sentence = sd._sentence("agent_py", one_long_part, 0) + assert "…" in sentence and len(sentence) <= 200 + + assert sd._sentence("client open ai!", "app.py", 1) == ( + "Creates a client for a model provider in app.py, line 1." + ) + assert sd._sentence("client openai", "app.py", 1) == ( + "Creates a client for openai in app.py, line 1." + ) + assert sd._sentence("builds \x00", "app.py", 1) == ( + "Builds with an agent framework in app.py, line 1." + ) + assert sd._sentence("no_such_fact", "app.py", 1) is None + + +def test_the_file_is_compact_utf8_and_never_fails_on_broken_characters() -> None: + undecodable = os.fsdecode(b"agent\xff.py") + rows = [ + _row("main.py", evidence=(("builds langgraph", undecodable, 2),)), + _row("x" + undecodable, "x" + undecodable), + ] + + document, body, dropped = _finish(rows, root="Café\ud800 Bot") + + assert body == json.dumps(document, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + body.decode("utf-8") + assert "Café Bot".encode("utf-8") in body + assert b"\\u00e9" not in body + assert b"\n" not in body + assert dropped is True + assert [agent["entry"] for agent in document["agents"]] == ["main.py"] + assert document["agents"][0]["evidence"] == [ + "Builds with LangGraph in a file in this folder, line 2." + ] + + +def test_finish_never_writes_more_than_fifty() -> None: + rows = [_row(f"f{i:02}/main.py", f"f{i:02}") for i in range(60)] + + document, _, dropped = _finish(rows) + + assert len(document["agents"]) == 50 + assert document["truncated"] is True + assert dropped is True + assert [agent["entry"] for agent in document["agents"]] == [ + f"f{i:02}/main.py" for i in range(50) + ] + + fifty, _, dropped = _finish(rows[:50]) + assert (len(fifty["agents"]), fifty["truncated"], dropped) == (50, False, False) + + first = _row("a/main.py", "a") + again = _row("a/main.py", "b", found_by_ai=True) + chosen, left_out = choose_rows([_row("z/main.py", "z"), first, again], limit=50) + assert [row.candidate.entry for row in chosen] == ["a/main.py", "z/main.py"] + assert chosen[0] is first + assert left_out is True + chosen, left_out = choose_rows(rows, limit=3) + assert (len(chosen), left_out) == (3, True) + assert choose_rows(rows[:3]) == (sorted(rows[:3], key=lambda row: row.candidate.entry), False) + + +def test_the_models_rows_are_chosen_first_only_when_asked() -> None: + graph = _row("a/graph.py", "a") + helpers = [_row(f"{name}/main.py", name, sign="client", builds=()) for name in ("b", "c")] + found = _row("z/agent.ts", "z", found_by_ai=True, builds=()) + in_tests = _row("tests/agent.ts", "tests", side=True, found_by_ai=True, builds=()) + rows = [graph, *helpers, found, in_tests] + + def entries(chosen: list[Row]) -> list[str]: + return [row.candidate.entry for row in chosen] + + plain, plain_left_out = choose_rows(rows, limit=3) + first, first_left_out = choose_rows(rows, limit=3, models_first=True) + many = [_row(f"h{i:02}/main.py", f"h{i:02}", sign="client", builds=()) for i in range(49)] + body, dropped = finish( + rows=[graph, *many, found], + root_name="shop", + commit_sha=SHA, + truncated=False, + error=None, + models_first=True, + ) + written = [agent["entry"] for agent in json.loads(body)["agents"]] + + assert _confidence(graph, *helpers, found) == ["high", "medium", "medium", "low"] + assert (entries(plain), plain_left_out) == (["a/graph.py", "b/main.py", "c/main.py"], True) + assert (entries(first), first_left_out) == (["a/graph.py", "z/agent.ts", "b/main.py"], True) + # Which rows are kept changes; the order they are written in does not. + assert dropped is True + assert written == ["a/graph.py", *(f"h{i:02}/main.py" for i in range(48)), "z/agent.ts"] + + +def test_a_row_the_platform_rules_reject_is_dropped_with_truncated(caplog: pytest.LogCaptureFixture) -> None: + good = _row("good/main.py", "good") + bad = [ + _row("bad/../escape.py", "bad"), + _row("label/main.py", "label", framework={"name": "langgraph", "version": None}), + _row("env/main.py", "env", env_names=("BAD-NAME-sk-fake-123",)), + _row("tools/main.py", "tools", tools=-1), + _row("word/main.py", "word", framework=_framework("autogen")), + ] + + with caplog.at_level(logging.WARNING, logger="fi.alk.harness.scan_document"): + document, _, dropped = _finish([good, *bad]) + + assert [agent["entry"] for agent in document["agents"]] == ["good/main.py"] + assert document["status"] == "found" + assert document["truncated"] is True + assert dropped is True + left_out = [record.getMessage() for record in caplog.records] + assert len(left_out) == len(bad) + assert all(message.startswith("agent scan: row left out, problem=") for message in left_out) + assert "sk-fake-123" not in caplog.text + assert "escape" not in caplog.text + + document, _, dropped = _finish(bad) + assert (document["status"], document["agents"], document["truncated"]) == ("none", [], True) + assert dropped is True + + +def test_a_bad_top_level_raises(monkeypatch: pytest.MonkeyPatch) -> None: + def problem(**changes: Any) -> str: + arguments: dict[str, Any] = { + "rows": [], + "root_name": "shop", + "commit_sha": SHA, + "truncated": False, + "error": None, + } + arguments.update(changes) + with pytest.raises(ScanResultInvalid) as raised: + finish(**arguments) + assert raised.value.problem in str(raised.value) + return raised.value.problem + + assert problem(error=("Scan-Failed", "The scan failed.")) == "error.code.pattern" + assert problem(error=("scan_failed", " ")) == "error.message.blank" + assert problem(error=("scan_failed", "m" * 2001)) == "error.message.too_long" + assert problem(error=("scan_failed", "sk-fake-123\x00")) == "error.message.null_byte" + assert "sk-fake-123" not in problem(error=("sk-fake-123", "The scan failed.")) + + monkeypatch.setattr(sd, "MAX_BYTES", 100) + assert problem(rows=[_row("main.py")]) == "document.too_large" + + assert wire_problems([]) == ["not_an_object"] + assert wire_problems({"schema_version": "futureagi.agent-scan.v1"}) == [ + "status.missing", + "commit_sha.missing", + "agents.missing", + "truncated.missing", + "error.missing", + ] + + +def test_the_file_matches_the_platform_serializer() -> None: + fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) + assert fixture["serializer_commit"] == "e07d56fcd" + cases = fixture["cases"] + assert len(cases) > 100 + assert {case["accepted"] for case in cases} == {True, False} + # Lone surrogates and null bytes reach the reader only as escapes in the text. + assert any("\\ud800" in case["document_text"] for case in cases) + assert any("\\u0000" in case["document_text"] for case in cases) + + disagreements = [] + for case in cases: + problems = wire_problems(json.loads(case["document_text"])) + if (problems == []) != case["accepted"]: + disagreements.append((case["name"], case["accepted"], problems)) + assert all(PROBLEM_NAME.fullmatch(problem) for problem in problems), case["name"] + assert disagreements == [] + + +def test_a_failed_write_leaves_no_agents_json( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, body, _ = _finish([_row("main.py")]) + + written = write_agents_json(tmp_path / "ok", body) + assert written == tmp_path / "ok" / "agents.json" + assert written.read_bytes() == body + assert stat.S_IMODE(written.stat().st_mode) == 0o644 + assert [path.name for path in (tmp_path / "ok").iterdir()] == ["agents.json"] + + def refuse(*args: Any, **kwargs: Any) -> None: + raise OSError("disk full") + + for name in ("replace", "fsync"): + output = tmp_path / name + output.mkdir() + with monkeypatch.context() as patch: + patch.setattr(sd.os, name, refuse) + with pytest.raises(OSError): + write_agents_json(output, body) + assert list(output.iterdir()) == [] + + earlier = tmp_path / "earlier" + earlier.mkdir() + (earlier / "agents.json").write_bytes(b"{}") + with monkeypatch.context() as patch: + patch.setattr(sd.os, "replace", refuse) + with pytest.raises(OSError): + write_agents_json(earlier, body) + assert [path.name for path in earlier.iterdir()] == ["agents.json"] + assert (earlier / "agents.json").read_bytes() == b"{}" + + +def test_a_disambiguating_suffix_survives_the_255_cut() -> None: + document, _, _ = _finish([_row("a/main.py"), _row("b/main.py")], root="x" * 300) + + names = sorted(agent["name"] for agent in document["agents"]) + assert names == [ + "x" * (255 - len(" (a/main.py)")) + " (a/main.py)", + "x" * (255 - len(" (b/main.py)")) + " (b/main.py)", + ] + + +def test_a_chat_model_row_is_low_with_its_own_sentence() -> None: + evidence = (("model langchain", "app/bot.py", 4),) + + agent = _finish( + [ + _row( + "app/bot.py", + "app", + sign="model", + builds=(), + evidence=evidence, + framework=_framework("langchain"), + ) + ] + )[0]["agents"][0] + + assert agent["confidence"] == "low" + assert agent["evidence"] == ["Calls a chat model through LangChain in app/bot.py, line 4."] + assert sd._sentence("model no such!", "app.py", 1) == ( + "Calls a chat model through an agent framework in app.py, line 1." + ) diff --git a/tests/harness/test_scan_folders.py b/tests/harness/test_scan_folders.py new file mode 100644 index 000000000..451dadcdf --- /dev/null +++ b/tests/harness/test_scan_folders.py @@ -0,0 +1,292 @@ +from __future__ import annotations + +import multiprocessing.process +import os +import signal +import time +from pathlib import Path +from typing import Any + +import pytest + +from fi.alk.harness import scan_folders +from fi.alk.harness.framework_detection import detect_framework +from fi.alk.harness.scan_folders import ( + ClockRanOut, + FolderChecker, + FolderCheckerBroken, + FolderWord, +) +from fi.alk.harness.source_folder import agent_folder + +CREW = "from crewai import Crew\nCrew()\n" +AUTOGEN = "from autogen_agentchat.agents import AssistantAgent\nAssistantAgent()\n" +OPENAI_AGENT = "from agents import Agent\nAgent()\n" +PLAIN = "print('hello')\n" +SITE = ".venv/lib/python3.12/site-packages" +# The first check in a session also starts the fork server, which imports the kit. +FIRST_CHECK_SECONDS = 120.0 +# Killing and reaping a stopped child, on a loaded machine. +KILL_MARGIN_SECONDS = 4.0 +UNKNOWN = {"name": "unknown", "version": None, "label": None} +FOLDERS = { + "": None, + "crew": {"name": "crewai", "version": None, "label": None}, + "autogen": {"name": "other", "version": None, "label": "autogen"}, + "oa": {"name": "openai_agents", "version": None, "label": None}, + "plain": {"name": "none", "version": None, "label": None}, +} + + +def _write(folder: Path, files: dict[str, str]) -> None: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _repo(tmp_path: Path) -> Path: + source = tmp_path / "repo" + _write( + source, + { + "crew/main.py": CREW, + "autogen/main.py": AUTOGEN, + "oa/main.py": OPENAI_AGENT, + "plain/main.py": PLAIN, + # Installed packages without openai-agents: a check that read them would drop + # the `agents` import, so the word shows which package folder the child used. + f"{SITE}/requests-2.0.dist-info/METADATA": "Version: 2.0\n", + f"oa/{SITE}/requests-2.0.dist-info/METADATA": "Version: 2.0\n", + }, + ) + return source + + +def _later(seconds: float) -> float: + return time.monotonic() + seconds + + +def _gone(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + return False + + +def _kill_if_alive(pid: int | None) -> None: + if pid is not None and not _gone(pid): + os.kill(pid, signal.SIGKILL) + + +def _exits_three(conn: Any) -> None: + os._exit(3) + + +def _refuses_to_start(self: Any) -> None: + raise OSError("no processes left") + + +def test_each_folder_gets_the_builds_framework_word(tmp_path: Path) -> None: + source = _repo(tmp_path) + words: dict[str, FolderWord] = {} + pids = set() + with FolderChecker() as checker: + for subfolder in FOLDERS: + words[subfolder] = checker.check( + source, subfolder, deadline=_later(FIRST_CHECK_SECONDS) + ) + pids.add(checker.pid) + + for subfolder, expected in FOLDERS.items(): + answer = detect_framework( + agent_folder(source, subfolder) if subfolder else source, + repo_root=source, + package_dir=None, + ) + assert words[subfolder] == FolderWord( + {"name": answer["name"], "version": None, "label": answer.get("label")}, + finished=True, + ) + assert set(words[subfolder].framework) == {"name", "version", "label"} + if expected is not None: + assert words[subfolder].framework == expected + assert len(pids) == 1 + assert None not in pids and os.getpid() not in pids + + +def test_the_clock_ends_a_stopped_child(tmp_path: Path) -> None: + source = _repo(tmp_path) + stopped: int | None = None + try: + with FolderChecker() as checker: + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + stopped = checker.pid + assert stopped is not None + os.kill(stopped, signal.SIGSTOP) + + started = time.monotonic() + with pytest.raises(ClockRanOut): + checker.check(source, "plain", deadline=started + 1.0) + + assert time.monotonic() - started < 1.0 + KILL_MARGIN_SECONDS + assert checker.pid is None + assert _gone(stopped) + + with pytest.raises(ClockRanOut): + checker.check(source, "autogen", deadline=time.monotonic() - 1.0) + assert checker.pid is None + + word = checker.check(source, "plain", deadline=_later(FIRST_CHECK_SECONDS)) + assert word == FolderWord(FOLDERS["plain"], finished=True) + assert checker.pid not in (None, stopped) + finally: + _kill_if_alive(stopped) + + +def test_the_child_is_restarted_after_a_kill(tmp_path: Path) -> None: + source = _repo(tmp_path) + killed: int | None = None + try: + with FolderChecker() as checker: + first = checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + assert first == FolderWord(FOLDERS["crew"], finished=True) + killed = checker.pid + assert killed is not None + os.kill(killed, signal.SIGKILL) + + word = checker.check(source, "autogen", deadline=_later(FIRST_CHECK_SECONDS)) + + assert word == FolderWord(UNKNOWN, finished=False) + assert _gone(killed) + + again = checker.check(source, "plain", deadline=_later(FIRST_CHECK_SECONDS)) + assert again == FolderWord(FOLDERS["plain"], finished=True) + assert checker.pid not in (None, killed) + finally: + _kill_if_alive(killed) + + +def test_a_child_that_cannot_start_is_broken( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = _repo(tmp_path) + + # The target is looked up when the child starts, so the child runs this one and exits 3. + monkeypatch.setattr(scan_folders, "_serve", _exits_three) + with FolderChecker() as checker: + with pytest.raises(FolderCheckerBroken, match="exit code 3"): + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + assert checker.pid is None + monkeypatch.undo() + + monkeypatch.setattr(multiprocessing.process.BaseProcess, "start", _refuses_to_start) + with FolderChecker() as checker: + with pytest.raises(FolderCheckerBroken, match="could not start"): + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + assert checker.pid is None + + +def test_exit_never_raises(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + source = _repo(tmp_path) + + assert FolderChecker().__exit__(None, None, None) is None + + with FolderChecker() as checker: + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + ended = checker.pid + assert ended is not None + assert checker.pid is None + assert _gone(ended) + assert checker.__exit__(None, None, None) is None + + stuck = FolderChecker() + stuck.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + alive = stuck.pid + assert alive is not None + real_kill = os.kill + + def refuse(pid: int, sig: int) -> None: + raise PermissionError("not allowed") + + monkeypatch.setattr(os, "kill", refuse) + try: + assert stuck.__exit__(None, None, None) is None + assert stuck.pid is None + finally: + monkeypatch.undo() + try: + real_kill(alive, signal.SIGKILL) + except ProcessLookupError: + pass + + +def test_a_fault_in_the_check_comes_back_as_a_trace(tmp_path, monkeypatch) -> None: + """Only a folder the build refuses is a plain ``unknown``; a fault inside the framework + check itself (here in ``folder_findings``, below detect_framework's own catch) is sent + back with its trace, so the parent logs it and marks the row unfinished.""" + import fi.alk.harness.framework_detection as fd + + def broken(*args, **kwargs): + raise KeyError("a fault in the check") + + monkeypatch.setattr(fd, "folder_findings", broken) + (tmp_path / "agent.py").write_text(CREW, encoding="utf-8") + assert fd.detect_framework(tmp_path) == {"name": "unknown", "version": None} + answer = scan_folders._answer(tmp_path, "") + assert set(answer) == {scan_folders._FAILED} + assert "KeyError" in answer[scan_folders._FAILED] + + assert scan_folders._answer(tmp_path, "no/such/folder") == {"name": "unknown", "version": None} + + +def test_running_out_of_memory_in_the_check_comes_back_as_a_trace(tmp_path, monkeypatch) -> None: + """A MemoryError inside one folder's check is that folder's problem: it comes back with its + trace, as any other fault does, and the child's loop goes on to the next folder. This + differs on purpose from the walk, which re-raises a MemoryError: the walk runs in the scan's + own process, while this runs in the child, where ending it would only lose every later + folder's word. `_serve` leaves its loop only on EOFError or OSError, so an answer here is + the loop going on.""" + import fi.alk.harness.framework_detection as fd + + def out_of_memory(*args, **kwargs): + raise MemoryError() + + monkeypatch.setattr(fd, "folder_findings", out_of_memory) + (tmp_path / "agent.py").write_text(CREW, encoding="utf-8") + + answer = scan_folders._answer(tmp_path, "") + + assert set(answer) == {scan_folders._FAILED} + assert "MemoryError" in answer[scan_folders._FAILED] + + +def test_a_failed_answer_marks_the_row_unfinished_and_logs_the_trace(tmp_path, caplog) -> None: + """The parent's half: a child answer carrying a trace becomes an unfinished ``unknown``, + kept for the folder, with the trace in the log.""" + + class _Conn: + def send(self, message: object) -> None: + pass + + def poll(self, timeout: float) -> bool: + return True + + def recv(self) -> object: + return {scan_folders._FAILED: "Traceback (most recent call last):\nKeyError: 'x'"} + + checker = FolderChecker() + checker._process = object() # a started child, as far as check() can tell + checker._conn = _Conn() + try: + with caplog.at_level("WARNING", logger=scan_folders.logger.name): + word = checker.check(tmp_path, "", deadline=time.monotonic() + 30) + finally: + checker._process = None + checker._conn = None + + assert word.finished is False + assert word.framework["name"] == "unknown" + assert "the check failed in the child" in caplog.text and "KeyError" in caplog.text + assert checker.check(tmp_path, "", deadline=time.monotonic() + 30).finished is False diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py new file mode 100644 index 000000000..d7095fae6 --- /dev/null +++ b/tests/harness/test_scan_walk.py @@ -0,0 +1,2419 @@ +from __future__ import annotations + +import ast +import gc +import inspect +import io +import itertools +import json +import os +import tokenize +import tracemalloc +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from fi.alk.harness import framework_detection as fd +from fi.alk.harness import scan_walk +from fi.alk.harness import source_folder +from fi.alk.harness.scan_walk import ( + Candidate, + WalkResult, + build_hazard, + describe_entry, + fits_path_rule, + walk_repo, +) + +GRAPH = "from langgraph.graph import StateGraph\nStateGraph(dict)\n" +CLIENT = "from openai import OpenAI\nOpenAI()\n" +FOREVER = float("inf") +ALLOWANCE = 1_000_000_000 + + +def _write(folder: Path, files: dict[str, str]) -> None: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _walk(source: Path, **kwargs: object) -> WalkResult: + return walk_repo(source, deadline=FOREVER, **kwargs) # type: ignore[arg-type] + + +def _entries(result: WalkResult) -> list[str]: + return [agent.entry for agent in result.agents] + + +def _agent(result: WalkResult, entry: str) -> Candidate: + return next(agent for agent in result.agents if agent.entry == entry) + + +def _snapshot(folder: Path) -> list[tuple[str, int, int]]: + seen = [] + for directory, directories, files in os.walk(folder, followlinks=False): + for name in sorted(directories + files): + path = os.path.join(directory, name) + status = os.lstat(path) + seen.append((path, status.st_size, status.st_mtime_ns)) + return sorted(seen) + + +def _token_work(source: str) -> int: + tokens = tokenize.tokenize(io.BytesIO(source.encode()).readline) + return sum(len(token.line) for token in tokens) + + +def test_the_walk_skips_what_the_build_skips_and_never_follows_a_link( + tmp_path: Path, +) -> None: + outside = tmp_path / "outside" + _write(outside, {"agent_graph.py": GRAPH}) + source = tmp_path / "source" + ran = tmp_path / "ran" + _write( + source, + { + "app/graph.py": GRAPH + f"open({str(ran)!r}, 'w').close()\n", + "app/build/graph.py": GRAPH, + "build/graph.py": GRAPH, + "env/graph.py": GRAPH, + "node_modules/pkg/graph.py": GRAPH, + ".venv/lib/graph.py": GRAPH, + "lib/site-packages/graph.py": GRAPH, + "lib/site/pkg-1.0.dist-info/METADATA": "Name: pkg\n", + "lib/site/graph.py": GRAPH, + }, + ) + (source / "linked").symlink_to(outside, target_is_directory=True) + (source / "linked_graph.py").symlink_to(outside / "agent_graph.py") + before = _snapshot(tmp_path) + + result = _walk(source) + + assert sorted(_entries(result)) == ["app/build/graph.py", "app/graph.py"] + assert result.unread_files == 1 + assert result.stopped == "" + assert not ran.exists() + assert _snapshot(tmp_path) == before + + +def test_a_heavy_file_uses_only_its_own_allowance(tmp_path: Path) -> None: + # Every mention of `C`, in its imports and its uses, walks all of its targets. + targets = 500 + imports = "".join(f"import crewai.m{index} as C\n" for index in range(targets)) + light = imports + "C.Crew()\n" * 100 + heavy = imports + "C.Crew()\n" * 1_500 + per_mention = fd._WORK_PER_TARGET * targets + light_work = (targets + 100) * per_mention + 3 * _token_work(light) + heavy_work = (targets + 1_500) * per_mention + assert scan_walk._FILE_WORK == ALLOWANCE + assert light_work < ALLOWANCE < 2 * light_work + assert ALLOWANCE < heavy_work < fd._MAX_WORK + _write(tmp_path, {"a_light.py": light, "b_light.py": light, "c_heavy.py": heavy}) + + result = _walk(tmp_path) + + assert _entries(result) == ["a_light.py", "b_light.py"] + assert result.unread_files == 1 + assert result.stopped == "" + + +def test_the_private_detector_names_it_uses_exist() -> None: + for name in ( + "_ALWAYS_SKIPPED", + "_TOP_ONLY_SKIPS", + "_FileScan", + "_has_long_line", + "_is_side_file", + "_is_side_part", + "_OverBudget", + "_own_modules", + "_read_file", + "_scan_file", + "_visible_tokens", + ): + assert hasattr(fd, name), name + assert hasattr(source_folder, "_FOLDER_FORM") + assert source_folder._FOLDER_MAX == 512 + assert list(inspect.signature(fd._FileScan).parameters) == [ + "shadowed", + "outside_names", + "work", + ] + assert "read_uses" in inspect.signature(fd._scan_file).parameters + + +@pytest.mark.parametrize( + "shape", + [ + b'x = """never closed\n', + b"if True:\n a = 1\n b = 2\n", + b"if True:\n\ta = 1\n b = 2\n", + b"# caf\xe9\nx = 1\n", + b"x = 1\x00\n", + ], + ids=["unclosed_string", "bad_dedent", "tab_and_spaces", "latin1_no_coding", "null"], +) +def test_a_file_the_tokenizer_rejects_is_unread_not_a_crash( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, shape: bytes +) -> None: + (tmp_path / "a_shape.py").write_bytes(GRAPH.encode() + shape) + _write(tmp_path, {"z_agent.py": GRAPH}) + + result = _walk(tmp_path) + + assert result.stopped == "" + assert "z_agent.py" in _entries(result) + assert ("a_shape.py" in _entries(result)) != (result.unread_files == 1) + + def rejects(data: bytes, work: object) -> object: + raise LookupError("a reader fault of any kind") + + monkeypatch.setattr(scan_walk, "_visible_tokens", rejects) + failing = _walk(tmp_path) + + assert failing.stopped == "" + assert failing.agents == () + assert failing.unread_files == 2 + + +def test_the_token_pass_writes_no_warnings( + tmp_path: Path, capfd: pytest.CaptureFixture[str], recwarn: pytest.WarningsRecorder +) -> None: + _write( + tmp_path, + {"agent.py": CLIENT + ("import os\nos.getenv('A\\{')\nx = f'\\{a}'\n" * 3)}, + ) + + _walk(tmp_path) + + assert not recwarn.list + assert "Warning" not in capfd.readouterr().err + + +def test_a_relative_import_joins_the_sub_agent(tmp_path: Path) -> None: + _write( + tmp_path, + { + "app/__init__.py": "", + "app/main.py": GRAPH + + "from . import planner\n" + + "def build():\n from .researcher import agent\n return agent\n", + "app/planner.py": GRAPH, + "app/researcher.py": GRAPH, + "deep/a/b/main.py": GRAPH + + "from ..helpers import util\nfrom .. import (shared)\n", + "deep/a/helpers/util.py": GRAPH, + "deep/a/shared.py": GRAPH, + "pkg/pyproject.toml": "", + "pkg/run.py": GRAPH + "import tools.search\n", + "pkg/src/tools/search.py": GRAPH, + }, + ) + + result = _walk(tmp_path) + + assert sorted(_entries(result)) == ["app/main.py", "deep/a/b/main.py", "pkg/run.py"] + + +def test_settings_tools_and_clients_are_read(tmp_path: Path) -> None: + source = tmp_path / "one" + _write( + source, + { + "bot.py": "import os\n" + + CLIENT + + "a = os.getenv('FIRST')\n" + + 'b = os.environ.get("SECOND")\n' + + "c = os.environ['THIRD']\n" + + "d = os.getenv(f'FOURTH')\n" + + "e = os.getenv('FIF' 'TH')\n" + + "f = os.getenv('SIX' + 'TH')\n" + + "g = os.getenv(NAME)\n" + + "h = os.environ[b'EIGHTH']\n" + + "def handler():\n return os.getenv( 'NINTH' )\n" + + "@tool\ndef one(): ...\n" + + "@function_tool\ndef two(): ...\n" + + "@mcp.tool()\ndef three(): ...\n" + + "@agents.function_tool(name='x')\ndef four(): ...\n" + + "@toolkit\ndef not_a_tool(): ...\n" + }, + ) + + [agent] = _walk(source).agents + + assert agent.sign == "client" + assert agent.env_names == ("FIRST", "NINTH", "SECOND", "THIRD") + assert agent.tools == 4 + assert [(kind, path) for kind, path, _ in agent.evidence] == [ + ("client openai", "bot.py") + ] + + forms = { + "openai.py": "OpenAI()", + "openai_async.py": "AsyncOpenAI()", + "openai_azure.py": "openai.AzureOpenAI()", + "anthropic.py": "Anthropic()", + "anthropic_async.py": "AsyncAnthropic()", + "genai.py": "genai.Client()", + "mistral.py": "Mistral(api_key=k)", + "cohere.py": "cohere.Client()", + "cohere_v2.py": "cohere.ClientV2()", + "groq.py": "Groq()", + "litellm.py": "litellm.completion(model=m)", + "litellm_async.py": "litellm.acompletion(model=m)", + } + clients = tmp_path / "clients" + _write(clients, {name: call + "\n" for name, call in forms.items()}) + _write(clients, {"not_a_client.py": "other.Client()\nOpenAI\n"}) + + assert sorted(_entries(_walk(clients))) == sorted(forms) + + +def test_a_getenv_default_is_never_an_env_name(tmp_path: Path) -> None: + _write( + tmp_path, + { + "bot.py": "import os\n" + + CLIENT + + "key = os.getenv('API_KEY', 'sk-fake-123')\n" + + "url = os.getenv('BASE_URL', 'FALLBACK_NAME')\n" + + "region = os.environ.get('REGION', 'DEFAULT_REGION')\n" + + "model = os.getenv('MODEL', default='DEFAULT_MODEL')\n" + }, + ) + + [agent] = _walk(tmp_path).agents + + assert agent.env_names == ("API_KEY", "BASE_URL", "MODEL", "REGION") + assert "sk-fake-123" not in repr(agent) + + +def test_env_files_with_values_are_never_opened( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write( + tmp_path, + { + ".env": "REAL_SECRET=sk-fake-123\n", + ".env.local": "LOCAL_SECRET=sk-fake-456\n", + "app/.env": "APP_SECRET=sk-fake-789\n", + ".env.example": "# a note\nEXAMPLE_NAME=sk-fake-000\n" + "export EXPORTED_NAME=1\n BAD-NAME=2\n", + "bot.py": CLIENT, + }, + ) + opened: list[str] = [] + read = scan_walk._read_file + + def spy(path: str, limit: int) -> bytes: + opened.append(os.path.basename(path)) + return read(path, limit) + + monkeypatch.setattr(scan_walk, "_read_file", spy) + + result = _walk(tmp_path) + + assert ".env.example" in opened + assert ".env" not in opened + assert ".env.local" not in opened + [agent] = result.agents + assert agent.env_names == ("EXAMPLE_NAME", "EXPORTED_NAME") + assert "sk-fake" not in repr(result) + + +@pytest.mark.parametrize( + "content", + [b"{not json", b"[" * 100_000 + b"]" * 100_000, b'{"graphs": "\xff"}'], + ids=["not_json", "deep", "not_utf8"], +) +def test_a_broken_langgraph_json_is_unread_not_a_crash( + tmp_path: Path, content: bytes +) -> None: + (tmp_path / "langgraph.json").write_bytes(content) + _write(tmp_path, {"graph.py": GRAPH}) + + result = _walk(tmp_path) + + assert result.stopped == "" + assert result.unread_files == 1 + assert _entries(result) == ["graph.py"] + + +def test_the_walk_limit_and_the_clock_stop_it( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert (scan_walk._MAX_JOINED, scan_walk._MAX_BYTES, scan_walk._MAX_HELD) == ( + 40_000, + 268_435_456, + 1_610_612_736, + ) + _write(tmp_path, {"a.py": GRAPH, "b.py": GRAPH, "c.py": GRAPH}) + calls: list[int] = [] + + def clock() -> float: + calls.append(1) + return 0.0 if len(calls) == 1 else 100.0 + + timed = walk_repo(tmp_path, deadline=10.0, clock=clock) + + assert timed.stopped == "clock" + assert _entries(timed) == ["a.py"] + # The walk checked before each of its two files; the join after it checks the clock too. + assert len(calls) >= 2 + + monkeypatch.setattr(scan_walk, "_MAX_JOINED", 2) + by_count = _walk(tmp_path) + assert by_count.stopped == "limit" + assert _entries(by_count) == ["a.py", "b.py"] + + monkeypatch.setattr(scan_walk, "_MAX_JOINED", 40_000) + monkeypatch.setattr(scan_walk, "_MAX_BYTES", len(GRAPH)) + by_bytes = _walk(tmp_path) + assert by_bytes.stopped == "limit" + assert _entries(by_bytes) == ["a.py"] + + +def test_a_memory_error_stops_the_walk_with_its_facts_dropped( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write(tmp_path, {"a.py": GRAPH, "b.py": GRAPH}) + walks: list[scan_walk._Walk] = [] + + class Recording(scan_walk._Walk): + def __init__(self, *args: object, **kwargs: object) -> None: + super().__init__(*args, **kwargs) # type: ignore[arg-type] + walks.append(self) + + read = scan_walk._read_file + + def runs_out(path: str, limit: int) -> bytes: + if path.endswith("b.py"): + raise MemoryError() + return read(path, limit) + + monkeypatch.setattr(scan_walk, "_Walk", Recording) + monkeypatch.setattr(scan_walk, "_read_file", runs_out) + + result = _walk(tmp_path) + + assert result.stopped == "memory" + assert result.agents == () + [walk] = walks + assert walk.facts == {} + + +def test_a_weak_candidate_does_not_swallow_a_strong_one(tmp_path: Path) -> None: + _write( + tmp_path, + { + "app/pyproject.toml": "", + "app/graph.py": GRAPH + "from llm import ask\n", + "app/llm.py": CLIENT, + "evaluation/pyproject.toml": "", + "evaluation/run.py": CLIENT + "from app.graph import graph\n", + }, + ) + + result = _walk(tmp_path) + + assert sorted(_entries(result)) == ["app/graph.py", "evaluation/run.py"] + + +def test_a_side_folder_script_does_not_swallow_the_agent(tmp_path: Path) -> None: + _write( + tmp_path, + { + "app/graph.py": GRAPH, + "evals/run.py": GRAPH + "from app.graph import graph\n", + }, + ) + + result = _walk(tmp_path) + + assert [(agent.entry, agent.side) for agent in result.agents] == [ + ("app/graph.py", False), + ("evals/run.py", True), + ] + assert result.candidate_outside_side + + +def test_an_import_cycle_is_one_agent(tmp_path: Path) -> None: + direct = tmp_path / "direct" + _write( + direct, + {"long_name.py": GRAPH + "import b\n", "b.py": GRAPH + "import long_name\n"}, + ) + through = tmp_path / "through" + _write( + through, + { + "c2.py": GRAPH + "import c1\n", + "c1.py": GRAPH + "import helper\n", + "helper.py": "import c2\n", + }, + ) + + assert _entries(_walk(direct)) == ["b.py"] + assert _entries(_walk(through)) == ["c1.py"] + + +def test_a_client_beside_a_stronger_agent_is_not_listed(tmp_path: Path) -> None: + _write( + tmp_path, + { + "pyproject.toml": "", + "app/graph.py": GRAPH, + "tools/backfill.py": CLIENT, + "other/pyproject.toml": "", + "other/bot.py": CLIENT, + }, + ) + + result = _walk(tmp_path) + + assert sorted(_entries(result)) == ["app/graph.py", "other/bot.py"] + + +def test_a_root_test_file_is_side(tmp_path: Path) -> None: + _write( + tmp_path, + {"test_agent.py": GRAPH, "agent_test.py": GRAPH, "conftest.py": GRAPH}, + ) + + result = _walk(tmp_path) + + assert {agent.entry: (agent.side, agent.subfolder) for agent in result.agents} == { + "agent_test.py": (True, ""), + "conftest.py": (True, ""), + "test_agent.py": (True, ""), + } + assert not result.candidate_outside_side + assert not result.source_outside_side + + +def test_the_project_folder_rule(tmp_path: Path) -> None: + one = tmp_path / "one" + _write( + one, + {"pyproject.toml": "", "app/graph.py": GRAPH, "scripts/run.py": GRAPH}, + ) + two = tmp_path / "two" + _write( + two, + { + "svc/pyproject.toml": "", + "svc/a/x.py": GRAPH, + "svc/b/y.py": GRAPH, + "lone/requirements.txt": "", + "lone/deep/z.py": GRAPH, + "plain/graph.py": GRAPH, + }, + ) + + first = _walk(one) + second = _walk(two) + + assert {agent.entry: agent.subfolder for agent in first.agents} == { + "app/graph.py": "", + "scripts/run.py": "scripts", + } + assert ("project pyproject.toml", "pyproject.toml", 0) in _agent( + first, "app/graph.py" + ).evidence + assert {agent.entry: agent.subfolder for agent in second.agents} == { + "svc/a/x.py": "svc/a", + "svc/b/y.py": "svc/b", + "lone/deep/z.py": "lone", + "plain/graph.py": "plain", + } + + +def test_two_graphs_in_one_langgraph_json_mark_the_folder(tmp_path: Path) -> None: + _write( + tmp_path, + { + "svc/pyproject.toml": "", + "svc/langgraph.json": json.dumps( + {"graphs": {"a": "./a.py:graph", "b": "./b.py:graph"}} + ), + "svc/a.py": "graph = None\n", + "svc/b.py": "graph = None\n", + "one/langgraph.json": json.dumps({"graphs": {"a": "./a.py:graph"}}), + "one/a.py": "graph = None\n", + }, + ) + + result = _walk(tmp_path) + + assert [ + (agent.entry, agent.sign, agent.subfolder, agent.build_may_stop) + for agent in result.agents + ] == [ + ("one/a.py", "graph", "one", ""), + ("svc/a.py", "graph", "svc", "graphs"), + ("svc/b.py", "graph", "svc", "graphs"), + ] + assert ("graph", "svc/langgraph.json", 0) in _agent(result, "svc/a.py").evidence + assert build_hazard(tmp_path, "svc") == "graphs" + assert build_hazard(tmp_path, "one") == "" + + +def test_two_agent_py_files_mark_the_folder(tmp_path: Path) -> None: + _write( + tmp_path, + { + "bot/requirements.txt": "", + "bot/agent.py": CLIENT, + "bot/tests/agent.py": "x = 1\n", + "single/agent.py": CLIENT, + "single/node_modules/pkg/agent.py": "", + "single/.venv/agent.py": "", + "single/__pycache__/agent.py": "", + "kept/agent.py": "", + "kept/venv/agent.py": "", + }, + ) + + result = _walk(tmp_path) + + assert { + agent.entry: (agent.sign, agent.subfolder, agent.build_may_stop) + for agent in result.agents + } == { + "bot/agent.py": ("agent_py", "bot", "agent_py"), + "single/agent.py": ("agent_py", "single", ""), + } + assert build_hazard(tmp_path, "kept") == "agent_py" + + +def test_more_than_fifty_agents_in_one_folder_keeps_fifty(tmp_path: Path) -> None: + _write(tmp_path, {f"bot_{index:02d}.py": CLIENT for index in range(51)}) + + over = _walk(tmp_path) + + assert _entries(over) == [f"bot_{index:02d}.py" for index in range(50)] + assert over.more_than_fifty + + (tmp_path / "bot_50.py").unlink() + exact = _walk(tmp_path) + + assert len(exact.agents) == 50 + assert not exact.more_than_fifty + + +def test_paths_the_path_rule_rejects_are_left_out(tmp_path: Path) -> None: + long_folder = "/".join(["d" * 100] * 5) + _write( + tmp_path, + { + "main.py": GRAPH, + "ok/graph.py": GRAPH, + "my agent/graph.py": GRAPH, + "svc\n/graph.py": GRAPH, + f"{long_folder}/graph.py": GRAPH, + }, + ) + + result = _walk(tmp_path) + + assert {agent.entry: agent.subfolder for agent in result.agents} == { + "main.py": "", + "ok/graph.py": "ok", + } + assert result.paths_left_out == 3 + assert fits_path_rule("ok/graph.py") + assert fits_path_rule("x" * 512) + assert not fits_path_rule("") + assert not fits_path_rule("svc\n") + assert not fits_path_rule("a/../b") + assert not fits_path_rule("x" * 513) + + +def test_an_upload_keeps_one_agent_at_the_root_counted_for_the_root( + tmp_path: Path, +) -> None: + source = tmp_path / "repo" + _write( + source, + { + "svc/pyproject.toml": "", + "svc/graph.py": GRAPH + "import os\nos.getenv('SVC_KEY')\n", + "main.py": "import os\nos.getenv('ROOT_KEY')\n@tool\ndef search(): ...\n", + "prompts/system.txt": "hello\n", + ".env.example": "TEMPLATE_KEY=x\n", + "tests/test_x.py": "import os\nos.getenv('TEST_KEY')\n", + "other/bot.py": CLIENT, + }, + ) + + uploaded = _walk(source, upload=True) + listed = _walk(source) + + [agent] = uploaded.agents + assert (agent.entry, agent.subfolder) == ("svc/graph.py", "") + assert agent.env_names == ("ROOT_KEY", "SVC_KEY", "TEMPLATE_KEY") + assert agent.tools == 1 + assert agent.prompt_files == 1 + assert agent.build_may_stop == build_hazard(source, "") + assert uploaded.upload_others == 1 + in_folder = _agent(listed, "svc/graph.py") + assert in_folder.subfolder == "svc" + assert in_folder.env_names == ("SVC_KEY", "TEMPLATE_KEY") + assert (in_folder.tools, in_folder.prompt_files) == (0, 0) + + side_only = tmp_path / "side" + _write( + side_only, + { + "scripts/run.py": CLIENT + "import os\nos.getenv('SIDE_KEY')\n", + "main.py": "import os\nos.getenv('MAIN_KEY')\n", + }, + ) + + [side] = _walk(side_only, upload=True).agents + + assert (side.entry, side.subfolder, side.side) == ("scripts/run.py", "", True) + assert side.env_names == ("MAIN_KEY", "SIDE_KEY") + + +def test_env_names_are_checked_and_ordered(tmp_path: Path) -> None: + names = [ + "Z_CODE", + "A_CODE", + "BAD NAME", + "ENDS\\n", + "1DIGIT", + "X" * 129, + "Y" * 128, + "A_CODE", + "naïve", + ] + code = "import os\n" + CLIENT + "".join(f"os.getenv('{n}')\n" for n in names) + _write( + tmp_path, + { + "svc/pyproject.toml": "", + "svc/bot.py": code, + "svc/.env.example": "B_TEMPLATE=1\nexport A_TEMPLATE=2\nA_CODE=3\n", + ".env.sample": "ROOT_TEMPLATE=1\n", + "elsewhere/.env.template": "ELSEWHERE=1\n", + }, + ) + + [agent] = _walk(tmp_path).agents + + assert agent.subfolder == "svc" + assert agent.env_names == ( + "A_CODE", + "Y" * 128, + "Z_CODE", + "A_TEMPLATE", + "B_TEMPLATE", + "ROOT_TEMPLATE", + ) + + +def test_describe_entry_reads_process_env(tmp_path: Path) -> None: + _write( + tmp_path, + { + "web/package.json": "{}\n", + "web/src/agent.ts": "const a = process.env.OPENAI_API_KEY;\n" + 'const b = process.env["OTHER_KEY"];\n' + "const c = process.env['THIRD_KEY'];\n" + "const d = process.env[name];\n" + "const e = myprocess.env.NOT_THIS;\n", + "web/src/util.mjs": "export const u = process.env.UTIL_KEY;\n", + "web/tests/agent.test.ts": "process.env.TEST_ONLY;\n", + "web/node_modules/pkg/index.js": "process.env.DEPENDENCY_KEY;\n", + "web/.env.example": "TEMPLATE_KEY=1\n", + "api/server.js": "process.env.ELSEWHERE;\n", + }, + ) + + candidate = describe_entry(tmp_path, "web/src/agent.ts", "web", deadline=FOREVER) + + assert candidate is not None + assert (candidate.sign, candidate.subfolder, candidate.side, candidate.partial) == ( + "client", + "web", + False, + False, + ) + assert candidate.env_names == ( + "OPENAI_API_KEY", + "OTHER_KEY", + "THIRD_KEY", + "UTIL_KEY", + "TEMPLATE_KEY", + ) + + +def test_describe_entry_keeps_the_models_folder_or_finds_package_json( + tmp_path: Path, +) -> None: + source = tmp_path / "repo" + _write( + source, + { + "pyproject.toml": "", + "web/package.json": "{}\n", + "web/src/agent.ts": "export {};\n", + "plain/deep/bot.py": CLIENT, + }, + ) + bare = tmp_path / "bare" + _write(bare, {"a/b/bot.py": CLIENT}) + + def folder_of(root: Path, entry: str, folder: str, upload: bool = False) -> str: + found = describe_entry(root, entry, folder, deadline=FOREVER, upload=upload) + assert found is not None + return found.subfolder + + assert folder_of(source, "web/src/agent.ts", "web/src") == "web/src" + assert folder_of(source, "web/src/agent.ts", "") == "" + assert folder_of(source, "web/src/agent.ts", "elsewhere") == "web" + assert folder_of(source, "web/src/agent.ts", "web/sr") == "web" + assert folder_of(source, "web/src/agent.ts", "web\n") == "web" + assert folder_of(source, "plain/deep/bot.py", "nowhere") == "" + assert folder_of(source, "web/src/agent.ts", "web/src", upload=True) == "" + assert folder_of(bare, "a/b/bot.py", "x") == "a/b" + assert describe_entry(source, "web/src/agent.ts\n", "web", deadline=FOREVER) is None + assert describe_entry(source, "", "", deadline=FOREVER) is None + + +def test_describe_entry_stops_at_its_deadline_with_partial( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert (scan_walk._DESCRIBE_FILES, scan_walk._DESCRIBE_BYTES) == (2_000, 33_554_432) + _write( + tmp_path, + { + "web/package.json": "{}\n", + "web/agent.ts": "process.env.FIRST;\n", + "web/b.ts": "process.env.SECOND;\n", + "web/c.ts": "process.env.THIRD;\n", + }, + ) + + late = describe_entry( + tmp_path, "web/agent.ts", "web", deadline=0.0, clock=lambda: 1.0 + ) + on_time = describe_entry( + tmp_path, "web/agent.ts", "web", deadline=10.0, clock=lambda: 1.0 + ) + monkeypatch.setattr(scan_walk, "_DESCRIBE_FILES", 1) + capped = describe_entry(tmp_path, "web/agent.ts", "web", deadline=FOREVER) + + assert late is not None and on_time is not None and capped is not None + assert (late.entry, late.partial, late.env_names) == ("web/agent.ts", True, ()) + assert not on_time.partial + assert on_time.env_names == ("FIRST", "SECOND", "THIRD") + assert (capped.partial, capped.env_names) == (True, ("FIRST",)) + + +# Every value line below carries "FAKE", so any name holding it is a piece of a value. +FAKE_KEY_LINE = "FAKEk3yAAAAAbbbbbbbbCCCCCCCCdddddddd0123456789EEEEEEEEffffffffGG" +AROUND = ("A_BEFORE", "Z_AFTER") +# Each shape: a settings template, and the names a reader of it should give, no more. +TEMPLATE_SHAPES: dict[str, tuple[str, tuple[str, ...]]] = { + "wrapped_bare_short_tail": ( + f"A_BEFORE=1\nSIGNING_KEY={FAKE_KEY_LINE}\n{FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "SIGNING_KEY"), + ), + "wrapped_once_short_tail": ( + f"A_BEFORE=1\nSIGNING_KEY={FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "SIGNING_KEY"), + ), + "bare_body_below_an_empty_setting": ( + f"A_BEFORE=1\nSIGNING_KEY=\n{FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "SIGNING_KEY"), + ), + "crlf_wrapped_short_tail": ( + f"A_BEFORE=1\r\nKEY={FAKE_KEY_LINE}\r\nFAKEcb=\r\nZ_AFTER=2\r\n", + (*AROUND, "KEY"), + ), + "base64url_tail_with_underscore": ( + f"A_BEFORE=1\nJWK_D={FAKE_KEY_LINE}\nFAKE_url_tail_AAAAbbbbCCCCddddEEEEffff0123=\nZ_AFTER=2\n", + (*AROUND, "JWK_D"), + ), + "base64url_line_below_an_empty_setting": ( + "A_BEFORE=1\nJWK_D=\nFAKE_url_tail_AAAAbbbbCCCCddddEEEEffff0123=\nZ_AFTER=2\n", + (*AROUND, "JWK_D"), + ), + "base32_tail": ( + "A_BEFORE=1\nTOTP_SEED=FAKEBASE32AAAABBBBCCCCDDDDEEEE\nFAKEBASE32TAIL234567ABCDEFGH=\nZ_AFTER=2\n", + (*AROUND, "TOTP_SEED"), + ), + "triple_double_quotes": ( + f'A_BEFORE=1\nTRIPLE="""\n{FAKE_KEY_LINE}\nFAKEtq=\n"""\nZ_AFTER=2\n', + (*AROUND, "TRIPLE"), + ), + "triple_single_quotes": ( + f"A_BEFORE=1\nTRIPLE='''\n{FAKE_KEY_LINE}\nFAKEts=\n'''\nZ_AFTER=2\n", + (*AROUND, "TRIPLE"), + ), + "escaped_single_quote": ( + f"A_BEFORE=1\nESC='it\\'s\n{FAKE_KEY_LINE}\nFAKEesc=\n'\nZ_AFTER=2\n", + (*AROUND, "ESC"), + ), + "escaped_double_quote": ('A_BEFORE=1\nESC="say \\"hi\nFAKEesd=\n"\nZ_AFTER=2\n', (*AROUND, "ESC")), + "quoted_value_left_open": ('A_BEFORE=1\nKEY="\nFAKEdq=\n"\nZ_AFTER=2\n', (*AROUND, "KEY")), + "key_block_with_one_short_line": ( + "A_BEFORE=1\nPEM=-----BEGIN PRIVATE KEY-----\nFAKEpm=\n-----END PRIVATE KEY-----\nZ_AFTER=2\n", + (*AROUND, "PEM"), + ), + "padded_tail_below_a_short_line": ( + "A_BEFORE=1\nKEY=FAKEshortline\nFAKEpad==\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "key_block_opened_in_a_comment": ( + f"A_BEFORE=1\n#PEM=-----BEGIN PRIVATE KEY-----\n{FAKE_KEY_LINE}\nFAKEpc=\n" + "-----END PRIVATE KEY-----\nZ_AFTER=2\n", + AROUND, + ), + "line_joined_by_a_backslash": ( + f"A_BEFORE=1\nexport LONG={FAKE_KEY_LINE}\\\nFAKEbs=\nZ_AFTER=2\n", + (*AROUND, "LONG"), + ), + "lines_joined_by_backslashes": ( + "A_BEFORE=1\nexport OPTS=--first \\\n --second \\\n FAKEthird=x\nZ_AFTER=2\n", + (*AROUND, "OPTS"), + ), + "key_wrapped_below_a_joined_line": ( + f"A_BEFORE=1\nexport LONG={FAKE_KEY_LINE}\\\n{FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "LONG"), + ), + "line_separator_in_a_value": ("A_BEFORE=1\nKEY=abc\u2028FAKEls=\nZ_AFTER=2\n", (*AROUND, "KEY")), + "form_feed_in_a_value": ("A_BEFORE=1\nKEY=abc\x0cFAKEff=\nZ_AFTER=2\n", (*AROUND, "KEY")), + "byte_order_mark": ("\ufeffOPENAI_API_KEY=\nZ_AFTER=2\n", ("OPENAI_API_KEY", "Z_AFTER")), + "escaped_backslash_before_the_closing_quote": ( + 'A_BEFORE=1\nWIN="C:\\\\tmp\\\\"\nMID_NAME=3\nZ_AFTER=2\n', + (*AROUND, "WIN", "MID_NAME"), + ), + "key_marker_in_an_inline_comment": ( + "A_BEFORE=1\nKEY=x # paste -----BEGIN RSA PRIVATE KEY----- here\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME"), + ), + "key_marker_in_a_comment_line": ( + "A_BEFORE=1\n# paste the key from -----BEGIN on\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "MID_NAME"), + ), + "one_line_key_block": ( + "A_BEFORE=1\nKEY=-----BEGIN FAKE KEY-----FAKEbody-----END FAKE KEY-----\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME"), + ), + "word_names": ( + "apiKey=\njwtSecret=\nOPENAIAPIKEY=\nlowercasename=\nLANGCHAIN_TRACING_V2=\n" + "NEXT_PUBLIC_SUPABASE_ANON_KEY=\n", + ( + "apiKey", + "jwtSecret", + "OPENAIAPIKEY", + "lowercasename", + "LANGCHAIN_TRACING_V2", + "NEXT_PUBLIC_SUPABASE_ANON_KEY", + ), + ), + "export_quoted": ( + f'A_BEFORE=1\nexport KEY="{FAKE_KEY_LINE}\nFAKEex=\n"\nZ_AFTER=2\n', + (*AROUND, "KEY"), + ), + "inline_comment_with_a_quote": ( + "A_BEFORE=1\nKEY=plain # don't\nMID_NAME=3\nKEY2=\"v\" # it's\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME", "KEY2"), + ), + "key_block_marked": ( + f"A_BEFORE=1\nPEM=-----BEGIN PRIVATE KEY-----\n{FAKE_KEY_LINE}\nFAKEpm=\n" + "-----END PRIVATE KEY-----\nZ_AFTER=2\n", + (*AROUND, "PEM"), + ), + "json_blob_over_lines": ( + 'A_BEFORE=1\nSA={\n "type": "service_account",\n "private_key_id": "FAKEid",\n' + ' "private_key": "-----BEGIN PRIVATE KEY-----\\nFAKEjs=\\n-----END PRIVATE KEY-----\\n"\n' + "}\nZ_AFTER=2\n", + (*AROUND, "SA"), + ), + "json_blob_on_one_line": ( + 'A_BEFORE=1\nSA={"k":"FAKEjc=","p":"FAKEjc2="}\nZ_AFTER=2\n', + (*AROUND, "SA"), + ), + "crlf_quoted": ( + f'A_BEFORE=1\r\nKEY="{FAKE_KEY_LINE}\r\nFAKEcr=\r\n"\r\nZ_AFTER=2\r\n', + (*AROUND, "KEY"), + ), + "backtick_over_two_lines": ( + f"A_BEFORE=1\nKEY=`{FAKE_KEY_LINE}\nFAKEbt=`\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "backtick_over_three_lines": ( + f"A_BEFORE=1\nKEY=`{FAKE_KEY_LINE}\n{FAKE_KEY_LINE}\nFAKEbt=`\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME"), + ), + "escaped_backtick": ("A_BEFORE=1\nESC=`say \\`hi\nFAKEebt=\n`\nZ_AFTER=2\n", (*AROUND, "ESC")), + "encoded_line_ending_in_a_blank": ( + f"A_BEFORE=1\nKEY={FAKE_KEY_LINE} \nFAKEtb=\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "encoded_line_ending_in_a_tab": ( + f"A_BEFORE=1\nKEY={FAKE_KEY_LINE}\t\nFAKEtt=\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "encoded_line_ending_in_a_comment": ( + f"A_BEFORE=1\nKEY={FAKE_KEY_LINE} # paste the key here\nFAKEtc=\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), +} + + +def _name_problems(names: object, expected: tuple[str, ...]) -> tuple[list[int], list[str]]: + """The lengths of the names that shouldn't be there, and the names missing. Lengths only, + so a failure never prints a piece of a value.""" + found = set(names) # type: ignore[call-overload] + return sorted(len(name) for name in found - set(expected)), sorted(set(expected) - found) + + +@pytest.mark.parametrize("shape", sorted(TEMPLATE_SHAPES)) +def test_a_template_shape_gives_its_names_and_no_value(shape: str) -> None: + text, expected = TEMPLATE_SHAPES[shape] + + problems = _name_problems(scan_walk._template_names(text.encode("utf-8")), expected) + + assert problems == ([], []) + + +@pytest.mark.parametrize( + "encoded", + [ + lambda text: b"\xff\xfe" + text.encode("utf-16-le"), + lambda text: b"\xfe\xff" + text.encode("utf-16-be"), + lambda text: text.encode("latin-1"), + ], + ids=["utf16_little_endian", "utf16_big_endian", "latin1"], +) +def test_a_template_in_utf16_or_latin1_gives_its_names_and_no_value( + tmp_path: Path, encoded: object +) -> None: + # Not UTF-8 either way: a byte-order mark of UTF-16, or a Latin-1 `ü`. + text = ( + f"A_BEFORE=1\nCITY=Zürich\nSIGNING_KEY={FAKE_KEY_LINE}\n{FAKE_KEY_LINE}\nFAKEt4il=\n" + "Z_AFTER=2\n" + ) + expected = ("A_BEFORE", "CITY", "SIGNING_KEY", "Z_AFTER") + data = encoded(text) # type: ignore[operator] + (tmp_path / ".env.example").write_bytes(data) + (tmp_path / "bot.py").write_text(CLIENT, encoding="utf-8") + + problems = _name_problems(scan_walk._template_names(data), expected) + walked = _walk(tmp_path) + walked_problems = _name_problems(walked.agents[0].env_names, expected) + + assert problems == ([], []) + assert walked.unread_files == 0 + assert walked_problems == ([], []) + + +def test_a_multi_line_value_in_a_template_is_never_a_name(tmp_path: Path) -> None: + # Every line of a pasted key below carries "Fake"; the last base64 line of a key ends in + # `=` padding, so it reads like a setting with an empty value. + template = ( + "BEFORE_NAME=1\n" + 'QUOTED_KEY="-----BEGIN PRIVATE KEY-----\n' + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCFake01\n" + "QuotedTailFake02=\n" + "-----END PRIVATE KEY-----\"\n" + "MIDDLE_NAME=2\n" + "RAW_KEY=-----BEGIN PRIVATE KEY-----\n" + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCFake03\n" + "RawTailFake04=\n" + "-----END PRIVATE KEY-----\n" + "SINGLE_QUOTED='first line\n" + "SingleTailFake05=\n" + "'\n" + "BarePaddedFake06==\n" + "WRAPPED_BLOB=MIIBIjANBgkqhkiG9w0BAQEFAAOCFake07\n" + "hkiG9w0BAQEFAAOCAQ8AFake08=\n" + "AFTER_NAME=3\n" + "OPENAIAPIKEY=\n" + "lowercasename=\n" + ) + expected = ( + "AFTER_NAME", + "BEFORE_NAME", + "MIDDLE_NAME", + "OPENAIAPIKEY", + "QUOTED_KEY", + "RAW_KEY", + "SINGLE_QUOTED", + "WRAPPED_BLOB", + "lowercasename", + ) + # Each shape in a repository of its own, so one shape's value can't hide another's. + repos = {"mixed": (template, expected), **TEMPLATE_SHAPES} + for shape, (text, _) in repos.items(): + (tmp_path / shape).mkdir() + (tmp_path / shape / ".env.example").write_bytes(text.encode("utf-8")) + (tmp_path / shape / "bot.py").write_text(CLIENT, encoding="utf-8") + + for shape, (_, names) in repos.items(): + [agent] = _walk(tmp_path / shape).agents + described = describe_entry(tmp_path / shape, "bot.py", "", deadline=FOREVER) + + assert described is not None + for found in (agent.env_names, described.env_names): + problems = _name_problems(found, names) + assert (shape, problems) == (shape, ([], [])) + assert found == tuple(sorted(names)) + assert not any("FAKE" in name.upper() or "MIIE" in name for name in found), shape + + +def test_the_walks_hazard_look_stops_at_the_walks_deadline( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The build counts `agent.py` in `venv/`, which the walk itself skips. + _write(tmp_path, {"bot.py": CLIENT, "venv/a/agent.py": "", "venv/b/agent.py": ""}) + [on_time] = _walk(tmp_path).agents + assert on_time.build_may_stop == "agent_py" + listed: list[str] = [] + real_walk = os.walk + + def counting(top, *args, **kwargs): + for directory, directories, files in real_walk(top, *args, **kwargs): + listed.extend(os.path.join(directory, name) for name in directories + files) + yield directory, directories, files + + monkeypatch.setattr(scan_walk.os, "walk", counting) + # On time while the walk reads its one file, past the deadline from then on. + ticks = iter([0.0]) + + [late] = walk_repo(tmp_path, deadline=10.0, clock=lambda: next(ticks, 100.0)).agents + + assert (late.entry, late.build_may_stop) == ("bot.py", "") + assert not any(path.endswith("agent.py") for path in listed) + + +def test_build_hazard_is_bounded_and_asked_once_per_folder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert scan_walk._HAZARD_ENTRIES == 200_000 + _write(tmp_path, {f"bot_{index}.py": CLIENT for index in range(5)}) + for index in range(30): + (tmp_path / "venv" / f"pkg_{index:02d}").mkdir(parents=True) + entries: list[str] = [] + real_walk = os.walk + + def counting(top, *args, **kwargs): + for directory, directories, files in real_walk(top, *args, **kwargs): + entries.extend(os.path.join(directory, name) for name in directories + files) + yield directory, directories, files + + monkeypatch.setattr(scan_walk.os, "walk", counting) + venv = os.path.join(tmp_path, "venv") + os.sep + + hazards: dict[str, str] = {} + result = walk_repo(tmp_path, deadline=FOREVER, hazards=hazards) + for _ in range(2): + describe_entry(tmp_path, "bot_0.py", "", deadline=FOREVER, hazards=hazards) + + # Five root agents and two descriptions share the root: its skipped `venv/` is looked + # through once. + assert len(result.agents) == 5 + assert hazards == {"": ""} + assert sum(path.startswith(venv) for path in entries) == 30 + + # Two `agent.py` files, each 15 folders down: finding both visits about 30 entries. + for branch in ("a", "b"): + _write(tmp_path, {"/".join(["deep", *[branch] * 15, "agent.py"]): ""}) + entries.clear() + monkeypatch.setattr(scan_walk, "_HAZARD_ENTRIES", 10) + assert build_hazard(tmp_path, "deep") == "" + assert len(entries) <= 11 + + monkeypatch.setattr(scan_walk, "_HAZARD_ENTRIES", 200_000) + assert build_hazard(tmp_path, "deep") == "agent_py" + + entries.clear() + assert build_hazard(tmp_path, "deep", deadline=0.0, clock=lambda: 1.0) == "" + # The folder's own listing and the one the clock is checked after, never the rest. + assert len(entries) <= 3 + + +def _ordered_walk(real_walk, *, descending: bool, listed: list[str]): + """``os.walk`` with each folder's subfolders handed over in one forced order, recording every + file name it lists.""" + + def walk(*args, **kwargs): + for here, directories, files in real_walk(*args, **kwargs): + directories.sort(reverse=descending) + listed.extend(files) + yield here, directories, files + + return walk + + +@pytest.mark.parametrize("below", ["", "x/"], ids=["top", "nested"]) +@pytest.mark.parametrize("case", ["found", "stopped"]) +def test_a_hazard_look_stopped_by_its_cap_gives_one_answer_in_any_listing_order( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, case: str, below: str +) -> None: + """The look goes in name order whatever order the file system lists folders in, so where its + cap stops it is the same everywhere. Both orders are forced, at the folder the look starts + from and a level below it.""" + if case == "found": + # Name order reaches both agent.py files before the big folder. + _files(tmp_path, {f"{below}a/agent.py": "", f"{below}b/agent.py": ""}) + _files(tmp_path, {f"{below}z/f{index:02}": "" for index in range(50)}) + else: + # Name order meets the big folder first and stops there, before either agent.py. + _files(tmp_path, {f"{below}a/f{index:02}": "" for index in range(50)}) + _files(tmp_path, {f"{below}b/agent.py": "", f"{below}c/agent.py": ""}) + monkeypatch.setattr(scan_walk, "_HAZARD_ENTRIES", 20) + real_walk = os.walk + answers, seen = [], [] + + for descending in (False, True): + listed: list[str] = [] + monkeypatch.setattr(scan_walk.os, "walk", _ordered_walk(real_walk, descending=descending, listed=listed)) + answers.append(scan_walk.build_hazard(tmp_path, "")) + seen.append(listed.count("agent.py")) + + if case == "found": + assert answers == ["agent_py", "agent_py"] + else: + assert answers == ["", ""] + # The "" is the cap's, not a look that missed them: neither agent.py was ever listed. + assert seen == [0, 0] + + +def test_the_prompt_files_listed_are_capped( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert scan_walk._MAX_PROMPT_PATHS == 20_000 + _write(tmp_path, {"bot.py": CLIENT, **{f"prompts/p{index}.txt": "x\n" for index in range(5)}}) + monkeypatch.setattr(scan_walk, "_MAX_PROMPT_PATHS", 3) + + [agent] = _walk(tmp_path).agents + + assert agent.prompt_files == 3 + + +def _decision_table_sources(row: str) -> tuple[str, ...]: + """One row's files from the framework check's own decision table, read from its test.""" + tree = ast.parse((Path(__file__).parent / "test_framework_detection.py").read_text()) + test = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "test_decision_table" + ) + table = test.decorator_list[0] + assert isinstance(table, ast.Call) + ids = next(keyword.value for keyword in table.keywords if keyword.arg == "ids") + index = ast.literal_eval(ids).index(row) + rows = table.args[1] + assert isinstance(rows, ast.List) + sources, _ = rows.elts[index].elts # type: ignore[attr-defined] + return ast.literal_eval(sources) + + +def test_a_chat_model_alone_is_a_candidate(tmp_path: Path) -> None: + [source] = _decision_table_sources("model_alone") + _write(tmp_path, {"bot.py": source}) + + result = _walk(tmp_path) + + [agent] = result.agents + assert (agent.entry, agent.sign, agent.builds) == ("bot.py", "model", ()) + assert result.candidate_outside_side + assert ("model langchain", "bot.py", 2) in agent.evidence + + +def test_a_chat_model_beside_a_stronger_agent_is_not_listed(tmp_path: Path) -> None: + chat_model = "from langchain_openai import ChatOpenAI\nChatOpenAI()\n" + _write( + tmp_path, + { + "svc/pyproject.toml": "", + "svc/graph.py": GRAPH, + "svc/helper.py": chat_model, + "other/pyproject.toml": "", + "other/chain.py": chat_model, + }, + ) + + result = _walk(tmp_path) + + assert {agent.entry: agent.sign for agent in result.agents} == { + "svc/graph.py": "framework", + "other/chain.py": "model", + } + + +def test_a_deeper_project_does_not_move_an_agents_folder(tmp_path: Path) -> None: + """The platform keys an agent on repo and subfolder, so adding an unrelated project with + its own agent must not move the first agent's folder.""" + project = "[project]\nname = 'x'\n" + _write(tmp_path, {"pyproject.toml": project, "src/app/agent.py": GRAPH}) + alone = _walk(tmp_path) + assert {agent.entry: agent.subfolder for agent in alone.agents} == {"src/app/agent.py": ""} + + _write(tmp_path, {"plugins/x/pyproject.toml": project, "plugins/x/agent.py": GRAPH}) + beside = _walk(tmp_path) + assert {agent.entry: agent.subfolder for agent in beside.agents} == { + "src/app/agent.py": "", + "plugins/x/agent.py": "plugins/x", + } + + +def test_side_folders_cannot_use_up_the_file_cap_before_the_agent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The customer's own code is read first; a big tests folder sorted ahead of the agent + no longer hides it behind the file cap. Each test file imports something, so it counts.""" + monkeypatch.setattr(scan_walk, "_MAX_JOINED", 10) + _write(tmp_path, {f"tests/test_{index:02}.py": "import os\n" for index in range(25)}) + _write(tmp_path, {"zz_agent/agent.py": GRAPH}) + + result = _walk(tmp_path) + + assert "zz_agent/agent.py" in _entries(result) + assert result.stopped == "limit" + + +@pytest.mark.parametrize("shape", ["linked_template", "unterminated_py", "long_line_py", "over_size_py"]) +def test_one_unreadable_file_does_not_stop_an_ai_rows_description(tmp_path: Path, shape: str) -> None: + """The row was paid for: a bad file marks it partial, and the reading goes on.""" + _write( + tmp_path, + { + "app/bot.py": GRAPH + "import os\nos.getenv('OPENAI_API_KEY')\n", + "app/z_tool.py": "import os\nos.getenv('BOT_MODEL')\n", + }, + ) + if shape == "linked_template": + (tmp_path / "outside.env").write_text("SEARCH_KEY=x\n", encoding="utf-8") + (tmp_path / "app/.env.example").symlink_to(tmp_path / "outside.env") + elif shape == "unterminated_py": + (tmp_path / "app/a_gen.py").write_text("x = '''never closed\n", encoding="utf-8") + elif shape == "long_line_py": + (tmp_path / "app/a_gen.py").write_text("x = 1 # " + "y" * 70_000 + "\n", encoding="utf-8") + else: + (tmp_path / "app/a_gen.py").write_text("x = 1\n" * 300_000, encoding="utf-8") + + candidate = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert candidate is not None + assert candidate.partial is True + assert {"OPENAI_API_KEY", "BOT_MODEL"} <= set(candidate.env_names) + assert "langgraph" in candidate.builds + + +def test_every_file_is_read_in_exactly_one_pass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The two passes split the files: together they give each file once.""" + _write( + tmp_path, + { + "agent.py": GRAPH, + "app/.env.example": "A=1\n", + "app/test_helpers.py": "x = 1\n", + "app/conftest.py": "x = 1\n", + "tests/test_a.py": "x = 1\n", + "tests/.env.example": "B=1\n", + "examples/demo/agent.py": GRAPH, + "pkg/sub/mod.py": "x = 1\n", + "pkg/sub/tests/test_b.py": "x = 1\n", + "zz_app/.env.example": "C=1\n", + }, + ) + seen: list[str] = [] + take = scan_walk._Walk._take + + def counting(self, directory, folder, name): + seen.append(scan_walk._join(folder, name)) + return take(self, directory, folder, name) + + monkeypatch.setattr(scan_walk._Walk, "_take", counting) + _walk(tmp_path) + + every = sorted(p.relative_to(tmp_path).as_posix() for p in tmp_path.rglob("*") if p.is_file()) + assert sorted(seen) == every + assert len(seen) == len(set(seen)) + # A settings template is read with its folder's own code, before every side file: here + # one in a folder that sorts after tests/, which a side-pass template would follow. + assert seen.index("zz_app/.env.example") < seen.index("tests/test_a.py") + assert seen.index("app/.env.example") < seen.index("app/test_helpers.py") + + +def test_a_template_beside_the_agent_is_read_before_a_big_tests_folder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(scan_walk, "_MAX_JOINED", 10) + _write(tmp_path, {f"tests/test_{index:02}.py": "import os\n" for index in range(25)}) + _write(tmp_path, {"zz_agent/agent.py": GRAPH, "zz_agent/.env.example": "OPENAI_API_KEY=\n"}) + + result = _walk(tmp_path) + + assert "OPENAI_API_KEY" in _agent(result, "zz_agent/agent.py").env_names + + +def test_a_reader_fault_on_every_file_logs_one_trace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _write(tmp_path, {f"m{index}.py": "x = 1\n" for index in range(5)}) + + def broken(data: bytes, shadowed: object) -> object: + raise LookupError("a fault in the kit's reader") + + monkeypatch.setattr(scan_walk, "_python_facts", broken) + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + result = _walk(tmp_path) + + assert result.unread_files == 5 + assert caplog.text.count("a file's reader failed") == 1 + assert "LookupError" in caplog.text + + +def test_a_reader_fault_on_every_described_file_logs_one_trace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _write(tmp_path, {"app/bot.py": GRAPH, **{f"app/m{index}.py": "x = 1\n" for index in range(5)}}) + + def broken(data: bytes, shadowed: object) -> object: + raise LookupError("a fault in the kit's reader") + + monkeypatch.setattr(scan_walk, "_python_facts", broken) + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + candidate = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert candidate is not None and candidate.partial is True + assert caplog.text.count("a file's reader failed") == 1 + + +def test_the_step_after_the_walk_watches_the_clock(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A crafted shape, small: many files that build a client, each importing a web of modules. + Joining them into agents must stop at its deadline and end as a clock stop, never run on + to the helper's hard stop.""" + files = { + f"pkg/m{index}.py": "".join(f"from .m{(index * 7 + step) % 200} import x\n" for step in range(1, 6)) + for index in range(200) + } + files.update( + { + f"pkg/c{index}.py": CLIENT + "".join(f"from .m{(index * 3 + step) % 200} import x\n" for step in range(5)) + for index in range(50) + } + ) + _write(tmp_path, files) + + complete = _walk(tmp_path) + assert complete.stopped == "" and complete.agents + + monkeypatch.setattr(scan_walk, "_RESULT_GRACE", 0.0) + calls: list[int] = [] + + def clock() -> float: + calls.append(1) + return float(len(calls)) + + # Each file read checks the clock once, so this deadline lets the walk finish and then + # gives the join only a few checks. + timed = walk_repo(tmp_path, deadline=float(len(files) + 5), clock=clock) + + assert timed.stopped == "clock" + assert timed.agents == () + assert len(calls) < len(files) + 50 + + +def test_the_step_after_the_walk_holds_a_bounded_number_of_edges( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A crafted shape, small: every candidate imports one package that re-exports modules, so + each candidate reaches every other and the pairs grow with the square of the candidates.""" + files = {"pkg/__init__.py": "".join(f"from .m{index} import x\n" for index in range(20))} + files.update({f"pkg/m{index}.py": "x = 1\n" for index in range(20)}) + files.update({f"c{index}.py": CLIENT + "import pkg\nfrom pkg import x\n" for index in range(30)}) + files.update({f"pkg/m{index}.py": CLIENT for index in range(20, 40)}) + files["pkg/__init__.py"] += "".join(f"from .m{index} import x\n" for index in range(20, 40)) + _write(tmp_path, files) + + assert _walk(tmp_path).stopped == "" + + monkeypatch.setattr(scan_walk, "_MAX_EDGES", 10) + bounded = _walk(tmp_path) + + assert bounded.stopped == "limit" + assert bounded.agents == () + + +# A UTF-16 byte-order mark followed by an odd number of bytes: the decoder can't read it. The +# same template with its last byte pair whole can. +ODD_UTF16_TEMPLATE = b"\xff\xfeA\x00=\x001\x00\n" +EVEN_UTF16_TEMPLATE = ODD_UTF16_TEMPLATE + b"\x00" + + +def test_a_template_the_decoder_rejects_marks_the_row_short( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + with pytest.raises(UnicodeDecodeError): + scan_walk._template_names(ODD_UTF16_TEMPLATE) + # The entry, and a file after the template that is still read: its setting is kept. + _write(tmp_path, {"app/bot.py": CLIENT, "app/tool.py": "import os\nos.getenv('TOOL_KEY')\n"}) + (tmp_path / "app" / ".env.example").write_bytes(EVEN_UTF16_TEMPLATE) + readable = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + (tmp_path / "app" / ".env.example").write_bytes(ODD_UTF16_TEMPLATE) + + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + rejected = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert readable is not None and readable.partial is False + assert set(readable.env_names) == {"A", "TOOL_KEY"} + assert rejected is not None and rejected.partial is True + assert set(rejected.env_names) == {"TOOL_KEY"} + # Short because the decoder rejected it, not because the kit's reader failed. + assert "agent description:" not in caplog.text + + +def test_the_walks_summary_line_counts_reader_faults_apart_from_unread_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _write(tmp_path, {f"m{index}.py": "x = 1\n" for index in range(3)}) + # An ordinary unreadable file: the tokenizer rejects it, which is no fault of the kit's. + _write(tmp_path, {"unterminated.py": "x = '''never closed\n"}) + real = scan_walk._python_facts + + def broken(data: bytes, shadowed: frozenset[str]) -> object: + if data == b"x = 1\n": + raise LookupError("a fault in the kit's reader") + return real(data, shadowed) + + monkeypatch.setattr(scan_walk, "_python_facts", broken) + with caplog.at_level("INFO", logger=scan_walk.logger.name): + _walk(tmp_path) + + assert "unread=4 reader_faults=3 " in caplog.text + + +def test_a_fault_in_the_description_keeps_what_was_read_and_logs_the_trace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """The description runs after the model was paid for, so a fault in the kit that escapes + it keeps what was read, as a partial row, instead of ending the scan in a paid retry. The + fault raised is outside the per-file catch, so only `describe_entry`'s own catch holds it.""" + _write( + tmp_path, + { + "app/bot.py": CLIENT, + "app/a_first.py": "import os\nos.getenv('FIRST_KEY')\n", + "app/z_last.py": "x = 1\n", + }, + ) + take = scan_walk._Description._take + calls: list[str] = [] + + def failing_after_the_first(self: object, directory: str, folder: str, name: str) -> None: + calls.append(name) + if len(calls) > 1: + raise RuntimeError("a fault in the kit") + take(self, directory, folder, name) # type: ignore[arg-type] + + monkeypatch.setattr(scan_walk._Description, "_take", failing_after_the_first) + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + candidate = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert calls[0] == "a_first.py" + assert candidate is not None + assert (candidate.entry, candidate.subfolder, candidate.partial) == ("app/bot.py", "app", True) + assert "FIRST_KEY" in candidate.env_names + assert "agent description stopped" in caplog.text + assert "RuntimeError: a fault in the kit" in caplog.text + + +# -- what the walk and its join keep (TH-9449), and its one capped reader (TH-9450) ---------- + + +def _unique_parts() -> Iterator[str]: + first = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_" + rest = first + "0123456789" + return ("".join(name) for name in itertools.product(first, rest, rest)) + + +def _crafted(parts: Iterator[str]) -> str: + """200 imports of 64 unique three-letter parts: the most kept for the bytes read.""" + return "".join("import " + ".".join(next(parts) for _ in range(64)) + "\n" for _ in range(200)) + + +def _files(root: Path, files: dict[str, str]) -> None: + for relative, content in files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _shape(root: Path, shape: str) -> None: + """Sixteen repository shapes, small, each growing one thing the walk or its join keeps.""" + if shape == "empty": + _files(root, {f"p{index % 3}/m{index:03}.py": "" for index in range(300)}) + elif shape == "plain_imports": + # Nine-character names: the allocator gives each 64 bytes, more than its size asked for. + _files(root, {f"m{i:02}.py": "".join(f"import mod{i:02}_{n:03}\n" for n in range(200)) for i in range(60)}) + elif shape == "dotted_imports": + _files( + root, + {f"m{i:02}.py": "".join(f"import pkg{i}_{n}.sub{n}.mod{n}\n" for n in range(200)) for i in range(60)}, + ) + elif shape == "from_import": + _files( + root, + { + f"m{i:02}.py": "from pkg.mod import (" + ", ".join(f"name_{i}_{n}" for n in range(300)) + ")\n" + for i in range(60) + }, + ) + elif shape == "long_names": + _files( + root, + { + f"m{i:02}.py": "".join( + "import " + ".".join([f"n{i}_{n}" + "q" * 250] * 16) + "\n" for n in range(200) + ) + for i in range(20) + }, + ) + elif shape == "settings": + _files( + root, + { + f"m{i:02}.py": "import os\n" + "".join(f"os.getenv('E{i}_{n}_{'X' * 120}')\n" for n in range(200)) + for i in range(60) + }, + ) + elif shape == "wide_names": + _files( + root, + { + f"m{i:02}.py": "".join(f"import 変数変数変数変数{i}_{n}.модульмодуль{n}\n" for n in range(200)) + for i in range(60) + }, + ) + elif shape == "crafted": + parts = _unique_parts() + _files(root, {f"m{i:02}.py": _crafted(parts) for i in range(12)}) + elif shape == "templates": + _files( + root, + {f"f{i:03}/.env.example": "".join(f"S{i}_{n}_NAME=\n" for n in range(200)) for i in range(200)}, + ) + elif shape == "graphs": + _files( + root, + { + f"g{i:02}/langgraph.json": json.dumps( + {"graphs": {f"x{n}": f"./deep/m{n}_{'y' * 40}.py:graph" for n in range(200)}} + ) + for i in range(20) + }, + ) + elif shape == "comb_and_trunk": + _files(root, {"/".join(f"c{c}d{d}" for d in range(100)) + "/m.py": "" for c in range(40)}) + trunk = "/".join(f"t{d}" for d in range(100)) + for depth in range(1, 101): + _files(root, {"/".join(trunk.split("/")[:depth]) + "/pyproject.toml": ""}) + _files(root, {f"{trunk}/leaf{n}/bot.py": "import os\nOpenAI()\n" for n in range(20)}) + elif shape == "importers_and_candidates": + deep = "/".join(f"level{d:02}_{'z' * 20}" for d in range(30)) + _files(root, {f"{deep}/mod{n:03}.py": "" for n in range(200)}) + _files( + root, + {f"{deep}/imp{n:02}.py": "".join(f"from . import mod{k:03}\n" for k in range(200)) for n in range(50)}, + ) + # Candidates that reach the importers, and many that reach nothing: the join's shares + # and import lists outweigh its fixed part. + _files(root, {f"{deep}/c{n:02}.py": f"from . import imp{n:02}\nOpenAI()\n" for n in range(50)}) + _files(root, {f"tiny/c{n:04}.py": "OpenAI()\n" for n in range(5_000)}) + elif shape == "project_per_file": + for index in range(300): + _files(root, {f"proj{index:03}/pyproject.toml": "", f"proj{index:03}/m.py": "x = 1\n"}) + elif shape == "rows_and_broom": + _files(root, {f"a{n:02}.py": "OpenAI()\n" for n in range(50)}) + _files( + root, + { + f"lib/m{i:03}.py": "import os\n" + "".join(f"os.getenv('L{i}_{n}')\n" for n in range(100)) + for i in range(100) + }, + ) + chain = root / "broom" + for depth in range(50): + chain = chain / f"d{depth}" + for sibling in range(20): + (chain.parent / f"s{depth}_{sibling}").mkdir(parents=True, exist_ok=True) + chain.mkdir(parents=True, exist_ok=True) + elif shape == "one_name_files": + _files(root, {"a.py": "OpenAI()\n"}) + _files(root, {f"n/f{n:04}.py": f"os.getenv('N{n:05}')\n" for n in range(2_000)}) + elif shape == "levels_over_256": + # Each import's level is an int of its own, and each client's line number too. + dots = "." * 300 + _files( + root, + { + f"m{i:02}.py": "".join(f"from {dots}m{n} import x\n" for n in range(200)) + "OpenAI()\n" * 100 + for i in range(60) + }, + ) + else: + raise AssertionError(shape) + + +SHAPES = [ + "empty", + "plain_imports", + "dotted_imports", + "from_import", + "long_names", + "settings", + "wide_names", + "crafted", + "templates", + "graphs", + "comb_and_trunk", + "importers_and_candidates", + "project_per_file", + "rows_and_broom", + "one_name_files", + "levels_over_256", +] +ORDINARY = SHAPES[1:7] + + +def _allocated(size: int) -> int: + """The block a request of ``size`` gets: pymalloc's 16-byte classes up to 512, then glibc's + malloc (an 8-byte header, 16-byte alignment, 32 at least), as on the Linux sandbox. Kept + apart from the code's own `_block`, so a fault in that can't hide here.""" + if size <= 512: + return max(16, (size + 15) // 16 * 16) + return max(32, (size + 8 + 15) // 16 * 16) + + +def _rounded_live() -> int: + return sum(_allocated(trace.size) for trace in tracemalloc.take_snapshot().traces) + + +def _largest_parse_peak(root: Path) -> int: + files = sorted(root.rglob("*.py"), key=lambda path: path.stat().st_size) + if not files: + return 0 + tracemalloc.start() + try: + base = tracemalloc.get_traced_memory()[0] + tracemalloc.reset_peak() + scan_walk._python_facts(files[-1].read_bytes(), frozenset()) + return tracemalloc.get_traced_memory()[1] - base + finally: + tracemalloc.stop() + + +def _hazard_peak(root: Path) -> int: + """A build hazard's own look at the root, the biggest any row's can be: bounded on its own + (`_HAZARD_PATH_BYTES`), not charged.""" + tracemalloc.start() + try: + base = tracemalloc.get_traced_memory()[0] + tracemalloc.reset_peak() + scan_walk.build_hazard(root, "") + return tracemalloc.get_traced_memory()[1] - base + finally: + tracemalloc.stop() + + +def _measure(root: Path) -> dict[str, int]: + """Walk and join ``root`` under tracemalloc: what was charged, what is really kept.""" + gc.collect() + tracemalloc.start() + try: + walk = scan_walk._Walk(root, deadline=FOREVER, clock=lambda: 0.0) + walk.run() + walk_peak = tracemalloc.get_traced_memory()[1] + gc.collect() + after_walk = tracemalloc.get_traced_memory()[0] + tracemalloc.reset_peak() + result = walk.result(upload=False) + join_peak = tracemalloc.get_traced_memory()[1] + gc.collect() + live = _rounded_live() + finally: + tracemalloc.stop() + join_charged = scan_walk._JOIN_BASE + join_charged += sum(scan_walk._held_for_join(path, facts) for path, facts in walk.facts.items()) + join_charged += sum(scan_walk._join_share(target) for target in walk.graphs) + assert result.stopped == "" + return { + "charged": walk.reader.held, + "live": live, + "walk_peak": walk_peak, + "join_rise": join_peak - after_walk, + "join_charged": join_charged, + } + + +@pytest.mark.parametrize("shape", SHAPES) +def test_what_is_charged_is_never_below_what_the_walk_and_its_join_keep( + tmp_path: Path, shape: str +) -> None: + """Checked against tracemalloc on CPython 3.12, the kit's Python, with each live block + rounded as the allocator rounds it (`_block`). The sizes in the estimate are 3.12's.""" + _shape(tmp_path, shape) + parse_peak = _largest_parse_peak(tmp_path) + hazard_peak = _hazard_peak(tmp_path) + + seen = _measure(tmp_path) + + assert seen["live"] <= seen["charged"], seen + assert seen["join_rise"] <= seen["join_charged"] + hazard_peak, (seen, hazard_peak) + assert seen["walk_peak"] <= seen["charged"] + parse_peak + 1_048_576, seen + if shape in ORDINARY: + # High, but not so high that an ordinary repository is cut early. + assert seen["charged"] <= 2.5 * seen["live"], seen + + +def test_the_walk_stops_at_what_it_keeps_not_at_how_many_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + parts = _unique_parts() + crafted = _crafted(parts) + _files(tmp_path / "crafted", {f"m{index:02}.py": _crafted(parts) for index in range(12)}) + _files(tmp_path / "small", {f"m{index:02}.py": "" for index in range(12)}) + one = scan_walk._held_facts("m00.py", scan_walk._python_facts(crafted.encode(), frozenset())) + monkeypatch.setattr(scan_walk, "_MAX_HELD", scan_walk._JOIN_BASE + 6 * one) + + stopped = _walk(tmp_path / "crafted") + whole = _walk(tmp_path / "small") + + assert stopped.stopped == "limit" + # The file whose charge crossed the budget is kept nowhere and isn't counted unread. + assert stopped.unread_files == 0 + assert whole.stopped == "" + + +def test_many_small_files_are_read_whole_and_only_joined_files_count( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A 40,000-file cap used to stop here, with the agent the last file read.""" + for folder in range(201): + package = tmp_path / "small" / f"p{folder:03}" + package.mkdir(parents=True) + for index in range(200): + (package / f"m{index:03}.py").touch() + _files(tmp_path / "small", {"zz_agent/agent.py": GRAPH}) + _files(tmp_path / "importing", {f"m{index:02}.py": "import os\n" for index in range(12)}) + + with caplog.at_level("INFO", logger=scan_walk.logger.name): + small = _walk(tmp_path / "small") + monkeypatch.setattr(scan_walk, "_MAX_JOINED", 10) + importing = _walk(tmp_path / "importing") + + assert small.stopped == "" + assert _entries(small) == ["zz_agent/agent.py"] + assert "files_read=40201 " in caplog.text + assert importing.stopped == "limit" + + +# For each thing kept: two repositories that differ only in it, and which measure shows it: +# what is live after the join, the walk's peak (the folders `os.walk` holds until it enters +# them), or the join's own rise (what the join builds for the files it reaches). +KEPT = { + "templates": "live", + "graphs": "live", + "prompt_paths": "live", + "project_sets": "live", + "pending_folders": "walk_peak", + "edges": "join", + "join_share": "join", + "import_levels": "live", + "line_numbers": "live", +} + + +def _kept_pair(with_it: Path, without: Path, structure: str) -> None: + if structure == "templates": + _files(with_it, {f"f{i:02}/.env.example": "".join(f"N{i}_{n}=\n" for n in range(100)) for i in range(40)}) + _files(without, {f"f{i:02}/readme.txt": "x\n" for i in range(40)}) + elif structure == "graphs": + graphs = {"graphs": {f"g{n}": f"./m{n}_{'y' * 30}.py:graph" for n in range(200)}} + _files(with_it, {f"a{i}/langgraph.json": json.dumps(graphs) for i in range(10)}) + _files(without, {f"a{i}/langgraph.json": "{}" for i in range(10)}) + elif structure == "prompt_paths": + _files(with_it, {f"prompts/prompt_{n:04}_{'p' * 60}.txt": "" for n in range(500)}) + _files(without, {f"others/other_{n:04}_{'p' * 60}.txt": "" for n in range(500)}) + elif structure == "project_sets": + for index in range(200): + _files(with_it, {f"p{index:03}/pyproject.toml": "", f"p{index:03}/m.py": "", f"p{index:03}/n.py": ""}) + _files(without, {f"p{index:03}/m.py": "", f"p{index:03}/n.py": ""}) + elif structure == "pending_folders": + for root, siblings in ((with_it, 20), (without, 0)): + chain = root + for depth in range(40): + chain = chain / f"d{depth}" + for sibling in range(siblings): + (chain.parent / f"s{depth}_{sibling}_{'w' * 30}").mkdir(parents=True) + chain.mkdir(parents=True) + elif structure == "edges": + # One candidate reaches 200 files that each import the other 199: the join keeps an + # import list for each. Without, the same files import nothing. + web = {f"m{n:03}.py": "".join(f"import m{k:03}\n" for k in range(200) if k != n) for n in range(200)} + _files(with_it, {"bot.py": "import m000\nOpenAI()\n", **web}) + _files(without, {"bot.py": "OpenAI()\n", **{name: "" for name in web}}) + elif structure == "import_levels": + # The same imports; only their levels pass 256, so each is an int of its own rather than + # a shared small one. + _files(with_it, {f"m{i:02}.py": "".join(f"from {'.' * 300}m{n} import x\n" for n in range(200)) for i in range(30)}) + _files(without, {f"m{i:02}.py": "".join(f"from .m{n} import x\n" for n in range(200)) for i in range(30)}) + elif structure == "line_numbers": + # One client a file, on a line past 256 or on line 1: an int of its own, or a shared one. + _files(with_it, {f"c{n:04}.py": "\n" * 300 + "OpenAI()\n" for n in range(2_000)}) + _files(without, {f"c{n:04}.py": "OpenAI()\n" + "\n" * 300 for n in range(2_000)}) + else: + _files(with_it, {f"c{n:04}.py": "OpenAI()\n" for n in range(5_000)}) + _files(without, {f"c{n:04}.py": "x = 1\n" for n in range(5_000)}) + + +@pytest.mark.parametrize("structure", list(KEPT)) +def test_each_thing_kept_is_charged_at_least_what_it_holds(tmp_path: Path, structure: str) -> None: + _kept_pair(tmp_path / "with", tmp_path / "without", structure) + + a, b = _measure(tmp_path / "with"), _measure(tmp_path / "without") + + if KEPT[structure] == "join": + # Charged at the walk for every file, whether the join reaches it or not, so the + # repository with the thing is checked on its own. + assert a["join_rise"] - b["join_rise"] > 0, (a, b) + assert a["join_rise"] <= a["join_charged"], a + else: + measure = KEPT[structure] + assert a["charged"] - b["charged"] >= a[measure] - b[measure] > 0, (a, b) + + +def _all_names(walk: scan_walk._Walk, subfolder: str) -> list[str]: + """What today's code built: every distinct name under the row's folder, sorted.""" + return sorted({name for path, facts in walk.facts.items() if scan_walk._under(path, subfolder) for name in facts.env}) + + +@pytest.mark.parametrize("names", [30, 300]) +def test_a_rows_setting_names_are_its_first_fifty(tmp_path: Path, names: int) -> None: + _files(tmp_path, {"bot.py": "from openai import OpenAI\nOpenAI()\n"}) + _files( + tmp_path, + {f"lib/m{i:02}.py": "import os\n" + "".join(f"os.getenv('K{(i * 37 + n) % names:04}')\n" for n in range(40)) for i in range(20)}, + ) + _files(tmp_path, {".env.example": "".join(f"T{n:03}=\n" for n in range(40))}) + walk = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + walk.run() + full = _all_names(walk, "") + expected = (*full, *sorted(walk.templates[""] - set(full)))[: scan_walk.ENV_NAMES_KEPT] + + row = _agent(walk.result(upload=False), "bot.py") + described = describe_entry(tmp_path, "bot.py", "", deadline=FOREVER) + + assert row.env_names == expected + assert described is not None and described.env_names == expected + assert scan_walk.ENV_NAMES_KEPT == 50 + from fi.alk.harness import scan_document + + assert scan_document.ENV_NAMES_KEPT is scan_walk.ENV_NAMES_KEPT + + +def test_a_hazard_look_gives_up_past_its_listed_paths(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _files(tmp_path, {"a/agent.py": "", "b/agent.py": ""}) + # Beside a and b, so the root's own listing passes the bound before either agent.py is + # seen, whatever order the file system lists folders in. + for index in range(40): + (tmp_path / f"folder_{index:02}_{'v' * 40}").mkdir() + assert scan_walk.build_hazard(tmp_path, "") == "agent_py" + + monkeypatch.setattr(scan_walk, "_HAZARD_PATH_BYTES", 500) + + assert scan_walk.build_hazard(tmp_path, "") == "" + + +@pytest.mark.parametrize( + ("weak", "stronger", "listed"), + [ + ("tests/test_client.py", "tests/test_graph.py", False), + ("app/client.py", "tests/test_graph.py", True), + ("app/client.py", "app/graph.py", False), + ], + ids=["side_beside_side", "own_beside_side_only", "own_beside_own"], +) +def test_a_side_candidate_never_decides_for_one_outside_side_folders( + tmp_path: Path, weak: str, stronger: str, listed: bool +) -> None: + _files(tmp_path, {"pyproject.toml": "", weak: "from openai import OpenAI\nOpenAI()\n", stronger: GRAPH}) + + assert (weak in _entries(_walk(tmp_path))) is listed + + +def test_the_join_is_linear_in_its_candidates(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _files(tmp_path, {"pyproject.toml": "", **{f"c{n:04}.py": "OpenAI()\n" for n in range(2_000)}}) + _files(tmp_path, {f"lib/m{n:04}.py": "" for n in range(2_000)}) + walk = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + walk.run() + project_of = scan_walk._Walk._project_of + side_checks = scan_walk._is_side_path + counted = {"project_of": 0, "side": 0} + + def counting_project_of(self: object, path: str) -> str: + counted["project_of"] += 1 + return project_of(self, path) # type: ignore[arg-type] + + def counting_side(path: str) -> bool: + counted["side"] += 1 + return side_checks(path) + + monkeypatch.setattr(scan_walk._Walk, "_project_of", counting_project_of) + monkeypatch.setattr(scan_walk, "_is_side_path", counting_side) + result = walk.result(upload=False) + + assert len(result.agents) == 50 + # Today's loops asked for each candidate's project once per other candidate, and walked + # every kept file once per row: millions here. + assert counted["project_of"] <= 3 * 2_000 + assert counted["side"] <= 10 * 4_000 + + +def test_a_folder_and_its_ancestors_are_looked_at_once(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # 150 levels: macOS paths stop at 1,024 characters. + deep = "/".join(f"d{depth}" for depth in range(150)) + _files(tmp_path, {f"{deep}/m{n:03}.py": f"import m{(n + 1) % 300:03}\n" for n in range(300)}) + file_in = scan_walk._Projects.file_in + looks: list[str] = [] + + def counting(self: object, folder: str) -> str | None: + looks.append(folder) + return file_in(self, folder) # type: ignore[arg-type] + + monkeypatch.setattr(scan_walk._Projects, "file_in", counting) + _walk(tmp_path) + + # 151 folders, the root included; a look a file at a time would be 300 times as many. + assert len(looks) <= 2 * 151 + assert len(set(looks)) == 151 + + +def test_the_capped_reader_counts_in_the_order_the_stop_points_need(tmp_path: Path) -> None: + for name, size in (("a", 6), ("b", 6), ("big", scan_walk._FILE_BYTES + 1)): + (tmp_path / name).write_bytes(b"x" * size) + missing = str(tmp_path / "missing") + + by_files = scan_walk._CappedReader(files=2, total_bytes=10**9) + assert by_files.read(str(tmp_path / "a")) == b"x" * 6 + with pytest.raises(FileNotFoundError): + by_files.read(missing) + assert by_files.files_read == 1 # a file that can't be read isn't counted + by_files.read(str(tmp_path / "b")) + with pytest.raises(scan_walk._CapReached): + by_files.read(missing) # the count is checked before reading + + by_bytes = scan_walk._CappedReader(files=None, total_bytes=10) + by_bytes.read(str(tmp_path / "a")) + with pytest.raises(scan_walk._CapReached): + by_bytes.read(str(tmp_path / "b")) # read and counted, then past the bytes + assert (by_bytes.files_read, by_bytes.bytes_read) == (2, 12) + + by_size = scan_walk._CappedReader(files=None, total_bytes=10**9) + with pytest.raises(scan_walk._Unread): + by_size.read(str(tmp_path / "big")) + assert by_size.files_read == 1 + + by_held = scan_walk._CappedReader(files=None, total_bytes=10**9, held=10) + by_held.charge(10) + with pytest.raises(scan_walk._CapReached): + by_held.charge(1) + scan_walk._CappedReader(files=None, total_bytes=10**9).charge(10**12) # no budget: never + assert not issubclass(scan_walk._CapReached, scan_walk._FILE_ERRORS) + + +def test_the_walk_and_the_description_read_through_the_one_capped_reader(tmp_path: Path) -> None: + walk = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + description = scan_walk._Description(os.fspath(tmp_path), "bot.py", "", deadline=FOREVER, clock=lambda: 0.0) + + assert isinstance(walk.reader, scan_walk._CappedReader) + assert isinstance(description.reader, scan_walk._CappedReader) + for reading in (scan_walk._Walk, scan_walk._Description): + assert "_read_file(" not in inspect.getsource(reading), reading.__name__ + + +@pytest.mark.parametrize("where", ["template", "source"]) +def test_the_descriptions_cap_stops_it_partial_at_either_read_site( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, where: str +) -> None: + _files(tmp_path, {"app/bot.py": CLIENT, "app/tool.py": "x = 1\n"}) + if where == "template": + # The cap is checked before each read: here before the first template is looked at. + _files(tmp_path, {"app/.env.example": "A=1\n"}) + monkeypatch.setattr(scan_walk, "_DESCRIBE_FILES", 0) + else: + # No template is there, so the first read counted is bot.py; tool.py is past the cap. + monkeypatch.setattr(scan_walk, "_DESCRIBE_FILES", 1) + # Past the cap the description reads nothing more: these come after it. + _files(tmp_path, {f"app/prompts/prompt_{n}.md": "Be brief.\n" for n in range(7)}) + + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + candidate = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert candidate is not None and candidate.partial is True + assert not candidate.prompt_files + assert "agent description stopped" not in caplog.text + + +def test_a_rows_pass_over_many_files_watches_the_clock( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """One row's pass over the kept files checks the clock as it goes, so a huge tree of files + that import nothing ends as a clock stop, never past the helper's hard stop.""" + _files(tmp_path, {"bot.py": "from openai import OpenAI\nOpenAI()\n"}) + for folder in range(50): + package = tmp_path / f"p{folder:02}" + package.mkdir() + for index in range(200): + (package / f"m{index:03}.py").touch() + now = [0.0] + side_checks = scan_walk._is_side_path + seen = [0] + + def counting(path: str) -> bool: + seen[0] += 1 + if seen[0] == 5_000: + now[0] = 10_000.0 # past the result's deadline, mid-pass + return side_checks(path) + + walk = scan_walk._Walk(tmp_path, deadline=100.0, clock=lambda: now[0]) + walk.run() + assert walk.stopped == "" + monkeypatch.setattr(scan_walk, "_is_side_path", counting) + + with pytest.raises(scan_walk._ResultOutOfTime): + walk.result(upload=False) + + +def test_a_project_lookup_is_charged_at_least_what_it_keeps(tmp_path: Path) -> None: + """Each folder the lookup keeps, with its link to its nearest project, measured on the + lookup alone: the walk also charges these folders as ones to enter, which would hide it.""" + folders = ["/".join(f"c{c}d{d}" for d in range(60)) for c in range(30)] + for folder in folders: + (tmp_path / folder).mkdir(parents=True) + (tmp_path / "pyproject.toml").touch() + charged = [0] + + def charge(size: int) -> None: + charged[0] += size + + projects = scan_walk._Projects(os.fspath(tmp_path), scan_walk._PROJECT_FILES, charge) + gc.collect() + tracemalloc.start() + try: + for folder in folders: + projects.nearest(folder) + gc.collect() + kept = _rounded_live() + finally: + tracemalloc.stop() + + assert projects.nearest(folders[0]) == "" + assert kept <= charged[0] + + +def test_the_first_names_hold_no_more_than_fifty_and_stop_early() -> None: + names = [f"N{n:04}" for n in range(300)] + taken: list[str] = [] + + def group(start: int) -> Iterator[str]: + for name in names[start:]: + taken.append(name) + yield name + + first = scan_walk._first_names([group(100), group(0), group(250)]) + + assert first == names[:50] + # Each group stops at its first name past the last of 50 kept: the first fills to 50 and + # stops at its 51st, the second replaces them all and stops at its 51st, and the third's + # first name is already past. + assert len(taken) == 51 + 51 + 1 + + +def test_a_rows_names_leave_out_a_deeper_listed_folder_and_a_sibling_with_the_same_start( + tmp_path: Path, +) -> None: + _files( + tmp_path, + { + "bot.py": GRAPH + "import os\nos.getenv('ROOT_KEY')\n", + "app/pyproject.toml": "", + "app/agent.py": GRAPH + "import os\nos.getenv('APP_KEY')\n", + # Not under app/, though its name starts the same way. + "app2/helper.py": "import os\nos.getenv('APP2_KEY')\n", + "plugins/x/pyproject.toml": "", + "plugins/x/agent.py": GRAPH + "import os\nos.getenv('PLUGIN_KEY')\n", + }, + ) + + result = _walk(tmp_path) + by_folder = {agent.subfolder: set(agent.env_names) for agent in result.agents} + + assert by_folder == { + "": {"ROOT_KEY", "APP2_KEY"}, + "app": {"APP_KEY"}, + "plugins/x": {"PLUGIN_KEY"}, + } + + +def test_the_estimate_rounds_each_object_up_to_its_block() -> None: + """The estimate's sizes are 3.12's, each rounded as the allocator rounds it. The whole-walk + comparison has margins beside the rounding, so the rounding is held here on its own.""" + import sys + + for size in range(1, 4_097): + assert scan_walk._block(size) >= _allocated(size), size + for name in ("", "a", "abcdefgh", "x" * 255, "変数", "модуль", "\U0001d400" * 3): + assert scan_walk._text(name) >= _allocated(sys.getsizeof(name)), name + for items in range(0, 210): + assert scan_walk._tuple(items) >= _allocated(sys.getsizeof(tuple(range(items)))), items + + +@pytest.mark.parametrize("structure", ["pending_folders", "prompt_paths"]) +def test_a_charge_made_outside_one_files_read_stops_the_walk_at_limit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, structure: str +) -> None: + """A folder's listing and a prompt path are charged outside any one file's read, so only the + walk's own catch turns their cap into `limit`: the walk keeps the row it found and never + raises out of `walk_repo`.""" + _files(tmp_path, {"bot.py": CLIENT}) + (tmp_path / "z").mkdir() + before = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + before.run() + for index in range(300): + if structure == "pending_folders": + (tmp_path / "z" / f"s{index:03}_{'w' * 30}").mkdir() + else: + (tmp_path / "z" / f"prompt_{index:03}.txt").touch() + monkeypatch.setattr(scan_walk, "_MAX_HELD", before.reader.held + 1_024) + + result = _walk(tmp_path) + + assert result.stopped == "limit" + assert _entries(result) == ["bot.py"] + + +def test_a_descriptions_byte_cap_marks_it_partial(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _files(tmp_path, {"app/bot.py": CLIENT, "app/tool.py": "x = 1\n" * 50}) + whole = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + monkeypatch.setattr(scan_walk, "_DESCRIBE_BYTES", len(CLIENT) + 10) + + capped = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert whole is not None and whole.partial is False + assert capped is not None and capped.partial is True + + +def test_a_project_lookup_charges_each_folder_and_each_link_it_keeps(tmp_path: Path) -> None: + """Every folder looked at is charged once for its entry, and every link to a nearest project + once more; the first look at a deep folder keeps a link for each ancestor, so a file below + one of them is answered from the links.""" + deep = "/".join(f"d{depth}" for depth in range(40)) + (tmp_path / deep).mkdir(parents=True) + (tmp_path / "d0" / "d1" / "pyproject.toml").touch() + charges: list[int] = [] + projects = scan_walk._Projects(os.fspath(tmp_path), scan_walk._PROJECT_FILES, charges.append) + + assert projects.nearest(deep) == "d0/d1" + + ancestors = [deep.rsplit("/", cut)[0] for cut in range(1, 40)] + [""] + assert set(projects._nearest) == {deep, *ancestors} + assert sum(charges) == sum(scan_walk._ENTRY + scan_walk._text(folder) for folder in projects._file) + ( + scan_walk._ENTRY * len(projects._nearest) + ) + looked = len(charges) + assert projects.nearest(ancestors[10]) == "d0/d1" + assert len(charges) == looked + + +def test_a_hazard_look_counts_each_listed_folder_by_its_full_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _files(tmp_path, {"a/agent.py": "", "b/agent.py": ""}) + names = [f"folder_{index:02}" for index in range(40)] + for name in names: + (tmp_path / name).mkdir() + # Above the names alone, below the names with the folder they sit in: the root's listing alone + # passes it, before either agent.py is found. + monkeypatch.setattr(scan_walk, "_HAZARD_PATH_BYTES", sum(len(name) + 1 for name in [*names, "a", "b"]) + 100) + + assert scan_walk.build_hazard(tmp_path, "") == "" + + +def test_the_file_whose_charge_crosses_the_budget_is_kept_nowhere( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _files(tmp_path, {"a.py": GRAPH, "b.py": GRAPH, "c.py": GRAPH}) + whole = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + whole.run() + # One byte short of the whole walk's charge: the last file's charge crosses it. + monkeypatch.setattr(scan_walk, "_MAX_HELD", whole.reader.held - 1) + + walk = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + walk.run() + result = _walk(tmp_path) + + assert walk.stopped == "limit" + assert len(walk.facts) == len(walk._keys) == 2 + assert result.stopped == "limit" and len(_entries(result)) == 2 + + +def test_an_import_inside_an_inner_project_resolves_at_an_outer_projects_root(tmp_path: Path) -> None: + """The places an import is looked for are every project at or above the file, nearest + first, then the repository's root: here the helper is found at the outer project's root, so + it joins the graph's agent rather than standing as one of its own.""" + _files( + tmp_path, + { + "svc/pyproject.toml": "", + "svc/a/pyproject.toml": "", + "svc/a/graph.py": GRAPH + "from helpers.llm import ask\n", + "svc/helpers/llm.py": CLIENT + "def ask():\n pass\n", + }, + ) + + assert _entries(_walk(tmp_path)) == ["svc/a/graph.py"] + + +def test_a_cap_reached_while_joining_ends_as_limit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _files(tmp_path, {"bot.py": CLIENT}) + + def charging(self: object, **kwargs: object) -> object: + raise scan_walk._CapReached() + + monkeypatch.setattr(scan_walk._Walk, "result", charging) + + assert _walk(tmp_path).stopped == "limit" + + +def test_each_folder_is_charged_once_across_the_two_passes(tmp_path: Path) -> None: + """The side pass walks the customer's own folders again; only the folders the first pass + never entered (side folders, and folders below one) are charged there.""" + for index in range(40): + (tmp_path / f"app{index:02}" / "inner").mkdir(parents=True) + (tmp_path / "tests" / f"t{index:02}" / "deep").mkdir(parents=True) + (tmp_path / f"app{index:02}" / "examples" / "e").mkdir(parents=True) + (tmp_path / f"app{index:02}" / ".hidden" / "x").mkdir(parents=True) + (tmp_path / ".github" / "workflows").mkdir(parents=True) + charged: dict[str, int] = {"own": 0, "side": 0, "once": 0} + + def into(key: str): + def charge(size: int) -> None: + charged[key] += size + + return charge + + top = str(tmp_path) + for _ in scan_walk._source_files(top, top, side=False, charge=into("own")): + pass + for _ in scan_walk._source_files(top, top, side=True, charge=into("side")): + pass + for _ in scan_walk._source_files(top, top, charge=into("once")): + pass + + assert charged["own"] > 0 and charged["side"] > 0 + assert charged["own"] + charged["side"] == charged["once"], charged + + +def test_the_estimates_sizes_were_measured_on_the_hosted_kits_python() -> None: + """The sizes are CPython 3.12's. On 3.14 a tuple's header is larger, so the estimate would be + low there. This fails when the hosted image's kit venv moves to another + Python, so the sizes are measured again before the bound is trusted.""" + dockerfile = (Path(__file__).resolve().parents[2] / "Dockerfile.hosted").read_text() + major, minor = scan_walk._SIZES_MEASURED_ON + made = [ + line + for line in dockerfile.splitlines() + if "/opt/alk-venv" in line and ("-m venv" in line or "uv venv" in line or "virtualenv" in line) + ] + assert len(made) == 1, made + assert f"python{major}.{minor} -m venv /opt/alk-venv" in made[0], made + assert made[0].count("-m venv") == 1, made + + +def test_a_memory_budget_stop_while_joining_says_so( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A held-budget stop during the join ends at `limit`, with its own cause in the log, not the + edge budget's.""" + _files(tmp_path, {"bot.py": CLIENT}) + + def reached(self, *, upload: bool): + raise scan_walk._CapReached() + + monkeypatch.setattr(scan_walk._Walk, "result", reached) + kept: list[int] = [] + real_empty = scan_walk._empty_result + + def empty(walk, stopped): + kept.append(len(walk.facts)) + return real_empty(walk, stopped) + + monkeypatch.setattr(scan_walk, "_empty_result", empty) + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + result = scan_walk.walk_repo(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + + assert result.stopped == "limit" + assert "passed its memory budget" in caplog.text + assert "edge budget" not in caplog.text + assert kept == [0] # the facts are let go before the empty result is built + + +@pytest.mark.parametrize("shape", ["one_entry", "two_entries"]) +def test_a_fan_out_join_is_charged_without_its_base(tmp_path: Path, shape: str) -> None: + """110 hubs of leaves that each import one shared module, about 22,000 files: the join walks + every leaf. One entry puts the import table just past its growth at 21,846 entries. Two + entries are dearer still: the first fills that table, then the second's own search grows its + seen set while the table stays full (about 59.5 bytes a file). What the join really holds is + covered by the per-file charge alone, without `_JOIN_BASE`, and only with `_JOIN_PER_FILE` + at 64: at 56 the two-entry shape fails. Leaves that import nothing share + CPython's empty tuple, which would hide the per-file cost.""" + per_hub = 199 if shape == "one_entry" else 198 + hubs = { + f"h{h:03}.py": "".join(f"import l{h:03}_{n:03}\n" for n in range(per_hub)) for h in range(110) + } + leaves = {f"l{h:03}_{n:03}.py": "import common\n" for h in range(110) for n in range(per_hub)} + entries = {"a.py": "".join(f"import h{h:03}\n" for h in range(110)) + CLIENT} + if shape == "two_entries": + entries["b.py"] = "".join(f"import h{h:03}\n" for h in range(100)) + CLIENT + _files(tmp_path, {**entries, "common.py": "x = 1\n", **hubs, **leaves}) + + seen = _measure(tmp_path) + + assert seen["join_rise"] > 1_000_000, seen + assert seen["join_rise"] <= seen["join_charged"] - scan_walk._JOIN_BASE, seen + + +@pytest.mark.parametrize("entries", ["folders", "files", "links"]) +def test_a_wide_folder_is_charged_its_listing(tmp_path: Path, entries: str) -> None: + """One folder of many entries: what `os.walk` holds for the folder it is in, its names, is + charged once when the walk reaches it, before any name is filtered out (links to a folder are + listed as folders and then left out), so the charge is never below the walk's peak.""" + target = tmp_path / "target" + target.mkdir() + wide = tmp_path / "wide" + wide.mkdir() + for index in range(80_000 if entries == "files" else 30_000): + if entries == "folders": + (wide / f"f{index:05}").mkdir() + elif entries == "files": + (wide / f"f{index:05}.txt").touch() + else: + (wide / f"f{index:05}").symlink_to(target) + + seen = _measure(tmp_path) + + assert seen["walk_peak"] <= seen["charged"], seen diff --git a/tests/harness/test_scenario_source_provision_agent.py b/tests/harness/test_scenario_source_provision_agent.py index 354a8afe5..03f820a69 100644 --- a/tests/harness/test_scenario_source_provision_agent.py +++ b/tests/harness/test_scenario_source_provision_agent.py @@ -12,6 +12,7 @@ import fi.alk from fi.alk.harness import scenario_source as ss from fi.alk.harness.contract import AgentContract +from fi.alk.harness.framework_detection import FrameworkFolders from fi.alk.harness.job import SourceKind from fi.alk.providers.agent import AgentBuildError, build_authored_agent @@ -56,15 +57,21 @@ def _contract(prompt: str) -> dict[str, Any]: ).model_dump(mode="json") -def _provision(bundle_dir: Path, kind: SourceKind) -> dict[str, Any]: +def _provision( + bundle_dir: Path, + kind: SourceKind, + source: ss.BundleScenarioSource | None = None, + *, + subfolder: str | None = None, +) -> dict[str, Any]: client = _RecordingClient() job = SimpleNamespace( run_id="job-1", metadata={}, - source=SimpleNamespace(kind=kind, repository="org/clinic"), + source=SimpleNamespace(kind=kind, repository="org/clinic", subfolder=subfolder), ) asyncio.run( - ss.BundleScenarioSource().build( + (source or ss.BundleScenarioSource()).build( job, object(), client, @@ -119,6 +126,29 @@ def test_provision_payload_carries_the_agent_built_from_the_sealed_contract( ) +def test_a_url_urlsplit_refuses_is_registered_redacted_and_logged_by_class_only( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + # A full-width number sign NFKC-normalises to "#", so urlsplit refuses the netloc, and its + # error repeats the whole userinfo. + leaky = "https://admin:hunter2\uff03x@db.example/" + _write_bundle(tmp_path, _contract(leaky)) + + with caplog.at_level(logging.DEBUG): + payload = _provision(tmp_path, SourceKind.GITHUB) + + assert payload["agent"]["agent"]["behaviour"][0]["instructions"] == "[redacted]" + assert "hunter2" not in json.dumps(payload["agent"]) + assert payload["agent_prompt"] == "[redacted]" + # Once for the document and once for the prompt. + logged = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] + assert [record.getMessage() for record in logged] == [ + "a URL was redacted whole: class=ValueError" + ] * 2 + assert all(record.exc_info is None for record in logged) + assert "hunter2" not in caplog.text + + @pytest.mark.parametrize( ("kind", "contract", "build_error", "logged"), [ @@ -152,7 +182,7 @@ def test_provision_payload_has_no_agent_for_remote_or_provider_sources_or_a_miss _write_bundle(tmp_path, contract) if build_error is not None: - def _raise(document: Any) -> dict[str, Any]: + def _raise(document: Any, **_: Any) -> dict[str, Any]: raise build_error monkeypatch.setattr(ss, "build_authored_agent", _raise) @@ -168,3 +198,257 @@ def _raise(document: Any) -> dict[str, Any]: else: assert any(logged in message for message in messages) assert not any("private_detail" in message for message in messages) + + +def _contract_from(prompt: str, folder: str) -> dict[str, Any]: + return { + **_contract(prompt), + "runtime": {"language": "python", "workdir": folder, "source_subfolder": folder}, + } + + +@pytest.mark.parametrize("value", ["agents/a", "my..agent"]) +def test_document_echoes_subfolder_exactly(tmp_path: Path, value: str) -> None: + _write_bundle(tmp_path, _contract_from("p", value)) + + payload = _provision(tmp_path, SourceKind.GITHUB, subfolder=value) + + assert payload["agent"]["agent"]["extension"]["source"] == {"subfolder": value} + + +@pytest.mark.parametrize( + ("built_from", "job_names"), + [("agents/a", "agents/b"), ("agents/a", None), (None, "agents/a")], +) +def test_no_document_when_the_bundle_came_from_another_folder( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + built_from: str | None, + job_names: str | None, +) -> None: + _write_bundle( + tmp_path, _contract_from("p", built_from) if built_from else _contract("p") + ) + + with caplog.at_level(logging.WARNING, logger=ss.__name__): + payload = _provision(tmp_path, SourceKind.GITHUB, subfolder=job_names) + + assert payload.get("agent") is None + # Told apart from any other bad document. + assert any("detail=subfolder_mismatch" in record.getMessage() for record in caplog.records) + + +@pytest.mark.parametrize("subfolder", [None, ""]) +def test_document_has_no_source_without_subfolder( + tmp_path: Path, subfolder: str | None +) -> None: + _write_bundle(tmp_path, _contract("p")) + + payload = _provision(tmp_path, SourceKind.GITHUB, subfolder=subfolder) + + assert "source" not in payload["agent"]["agent"]["extension"] + + +def test_no_subfolder_setup_hash_is_pinned(tmp_path: Path) -> None: + _write_bundle( + tmp_path, + { + "agent": "faq-bot", + "tools": [{"name": "search", "args": "query"}], + "runtime": {"language": "python", "workdir": "app", "command": ["python", "app.py"]}, + "real_use_cases": ["ask"], + }, + ) + + payload = _provision(tmp_path, SourceKind.GITHUB) + + # The value a build without a folder produced before the field existed. + assert ( + payload["agent"]["agent"]["setup_hash"] + == "sh1:27408adef9319e5f760d442d8e1e4c34ca2787f736e49ede86bc7398b323dd7b" + ) + + +def _source_with_crew(tmp_path: Path) -> ss.BundleScenarioSource: + agent = tmp_path / "agent" + agent.mkdir() + (agent / "main.py").write_text( + "from crewai import Crew\nCrew()\n", encoding="utf-8" + ) + return ss.BundleScenarioSource( + framework_folders=FrameworkFolders(agent, agent, None) + ) + + +def _framework_log_lines(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.getMessage().startswith("framework") + ] + + +def test_agent_document_carries_framework( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + bundle = tmp_path / "bundle" + contract = _contract("p") + _write_bundle(bundle, contract) + + with caplog.at_level(logging.INFO, logger="fi.alk.harness"): + payload = _provision(bundle, SourceKind.GITHUB, _source_with_crew(tmp_path)) + + lines = _framework_log_lines(caplog) + assert any("name=crewai" in line for line in lines) + assert not any(str(tmp_path) in line or "main.py" in line for line in lines) + framework = {"name": "crewai", "version": None} + assert payload["agent"]["agent"]["extension"]["framework"] == framework + assert payload["agent"]["agent"] == build_authored_agent( + contract, framework=framework + ) + + +def test_detector_crash_keeps_document( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + bundle = tmp_path / "bundle" + _write_bundle(bundle, _contract("p")) + secret = "/customer/private/path" + + def _raise(*_: Any, **__: Any) -> dict[str, Any]: + raise RuntimeError(secret) + + monkeypatch.setattr(ss, "detect_framework", _raise) + + with caplog.at_level(logging.INFO, logger="fi.alk.harness"): + payload = _provision(bundle, SourceKind.GITHUB, _source_with_crew(tmp_path)) + + lines = _framework_log_lines(caplog) + assert any("class=RuntimeError" in line for line in lines) + assert not any(secret in line for line in lines) + assert payload["agent"]["agent"]["extension"]["framework"] == { + "name": "unknown", + "version": None, + } + + +def test_no_folders_omits_framework(tmp_path: Path) -> None: + _write_bundle(tmp_path, _contract("p")) + + payload = _provision(tmp_path, SourceKind.GITHUB) + + assert set(payload["agent"]["agent"]["extension"]) == {"contract"} + + +@pytest.mark.parametrize( + ("kind", "expected_calls"), + [(SourceKind.GITHUB, 1), (SourceKind.ARCHIVE, 1), (SourceKind.REMOTE, 0)], + ids=["github", "archive", "not_for_a_remote_source"], +) +def test_detection_runs_once_off_the_event_loop( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + kind: SourceKind, + expected_calls: int, +) -> None: + bundle = tmp_path / "bundle" + _write_bundle(bundle, _contract("p")) + detections: list[Any] = [] + real_to_thread = asyncio.to_thread + + async def spy(function: Any, /, *args: Any, **kwargs: Any) -> Any: + if function is ss.detect_framework: + detections.append(args) + return await real_to_thread(function, *args, **kwargs) + + monkeypatch.setattr(ss.asyncio, "to_thread", spy) + + _provision(bundle, kind, _source_with_crew(tmp_path)) + + assert len(detections) == expected_calls + + +def test_detection_gets_all_three_folders( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + bundle = tmp_path / "bundle" + _write_bundle(bundle, _contract("p")) + calls: list[tuple[Any, dict[str, Any]]] = [] + + def record(folder: Any, **kwargs: Any) -> dict[str, Any]: + calls.append((folder, kwargs)) + return {"name": "none", "version": None} + + monkeypatch.setattr(ss, "detect_framework", record) + folders = FrameworkFolders(tmp_path / "a", tmp_path / "b", tmp_path / "c") + + _provision( + bundle, + SourceKind.GITHUB, + ss.BundleScenarioSource(framework_folders=folders), + ) + + assert calls == [ + (tmp_path / "a", {"repo_root": tmp_path / "b", "package_dir": tmp_path / "c"}) + ] + + +def test_a_password_in_a_prompt_url_reaches_neither_the_prompt_nor_the_document( + tmp_path: Path, +) -> None: + prompt = "You book visits. Post to https://bot:hunter2@hooks.example/v1?key=sk-1 after each." + _write_bundle(tmp_path, _contract(prompt)) + + payload = _provision(tmp_path, SourceKind.GITHUB) + + sent = json.dumps(payload) + assert "hunter2" not in sent + assert "sk-1" not in sent + assert payload["agent_prompt"] == "You book visits. Post to https://hooks.example/v1 after each." + assert payload["agent"]["agent"]["behaviour"][0]["instructions"] == payload["agent_prompt"] + + +def test_a_password_in_a_prompt_url_is_redacted_for_a_source_with_no_document( + tmp_path: Path, +) -> None: + _write_bundle(tmp_path, _contract("Use https://bot:hunter2@hooks.example/v1 to log.")) + + payload = _provision(tmp_path, SourceKind.REMOTE) + + assert "agent" not in payload + assert payload["agent_prompt"] == "Use https://hooks.example/v1 to log." + + +def test_only_the_urls_in_a_prompt_change_on_their_way_to_the_platform(tmp_path: Path) -> None: + """B1: whitespace and the tokens without a scheme start are byte-identical.""" + prompt = ( + "You book visits.\n\n\tRules: never share https://u:pw@h.example/x?key=sk-1#f\n" + "Then call postgres://db:hunter2@db.example:5432/app?ssl=1, and stop." + ) + _write_bundle(tmp_path, _contract(prompt)) + + payload = _provision(tmp_path, SourceKind.GITHUB) + + assert payload["agent_prompt"] == ( + "You book visits.\n\n\tRules: never share https://h.example/x\n" + "Then call postgres://db.example:5432/app and stop." + ) + + +def test_the_document_and_the_reading_hold_the_same_cleaned_prompt(tmp_path: Path) -> None: + """B2: the provision document, the reading's agent.json and agent_prompt agree.""" + prompt = "Post to https://bot:hunter2@hooks.example/v1?key=sk-1 after each." + _write_bundle(tmp_path, _contract(prompt)) + + payload = _provision(tmp_path, SourceKind.GITHUB) + reading = ss.authored_agent_document( + (tmp_path / "contract.json").read_text(encoding="utf-8"), subfolder=None, framework=None + ) + + cleaned = "Post to https://hooks.example/v1 after each." + assert payload["agent_prompt"] == cleaned + assert payload["agent"]["agent"]["behaviour"][0]["instructions"] == cleaned + assert reading["agent"]["behaviour"][0]["instructions"] == cleaned + assert "hunter2" not in json.dumps(reading) and "sk-1" not in json.dumps(reading) diff --git a/tests/harness/test_seed_contract.py b/tests/harness/test_seed_contract.py new file mode 100644 index 000000000..76c1bddf0 --- /dev/null +++ b/tests/harness/test_seed_contract.py @@ -0,0 +1,1747 @@ +"""A build handed a reading's raw contract uses it only when it was made from the same inputs. + +No model and no network: the reading's record, the build command's check of the handed file, +the builder's install at the opening of its first stage, and the round trip between them. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest +from pydantic import BaseModel, ValidationError + +from fi.alk.harness import ( + authoring_entrypoint, + cli, + reading_entry, + reading_result, + seed_contract, +) +from fi.alk.harness.contract import AgentContract +from fi.alk.harness.framework_detection import detect_framework +from fi.alk.harness.hosted_chat_entrypoint import HostedChatRuntime +from fi.alk.harness.job import HarnessJob +from fi.alk.harness.poc_guest_booking import PIN_ENV, TARGET_PHONE_ENV +from fi.alk.harness.reading_result import document +from fi.alk.harness.seed_contract import ( + RECORD_KEY, + SEED_ACCEPTED, + SEED_LIMIT_BYTES, + SEED_RESULT, + differs, + record_for, + use_seed, + with_record, +) +from fi.alk.harness.understand import load +from fi.alk.providers.agent import build_authored_agent + +FOLDER = "agents/a" +COMMIT = "0123456789abcdef0123456789abcdef01234567" +OTHER_COMMIT = "f" * 40 +TOKEN = "sk-SEEDTOKEN123" +GITHUB = { + "kind": "github", + "visibility": "public", + "repository": "owner/repo", + "commit_sha": COMMIT, + "subfolder": FOLDER, +} +EVALS = [{"name": "a"}] +CONTRACT = { + "agent": "support", + "modality": "chat", + "real_use_cases": ["order"], + "tools": [{"name": "add", "args": ["item_id"]}], + "runtime": {"language": "python", "workdir": FOLDER, "source_subfolder": FOLDER}, + "base_environment": { + "orders": [ + {"id": 1, "total": 0.1, "note": "café", "meta": {"tags": ["日本"]}}, + {"id": 2, "total": 12.5, "note": None, "meta": {"gift": None}}, + ], + "customers": [{"name": "日本", "credit": 0.1, "address": {"city": "Kyoto"}}], + }, +} + + +# -- the job, the source and the seed ---------------------------------------------------------- + + +def _job(*, execution="hosted", source=None, metadata=None, agent=None) -> HarnessJob: + return HarnessJob( + job_id="job", + run_id="run", + execution=execution, + source=source or GITHUB, + runtime={"isolation": "dedicated_vm"}, + agent=agent or {"connector": "auto"}, + metadata=( + {"available_evals": EVALS, "generic_harness_v1": True} + if metadata is None + else metadata + ), + ) + + +def _job_file(tmp_path: Path, job: HarnessJob) -> Path: + path = tmp_path / "job.json" + path.write_text(job.model_dump_json(), encoding="utf-8") + return path + + +def _source(tmp_path: Path) -> Path: + source = tmp_path / "source" + (source / FOLDER).mkdir(parents=True, exist_ok=True) + (source / FOLDER / "agent.py").write_text("x = 1\n", encoding="utf-8") + return source + + +def _seed_bytes(job: HarnessJob, body: dict = CONTRACT, *, record=None) -> bytes: + """The reading's bytes as it saved them, unlike any re-dump of them, with its record.""" + raw = json.dumps(body, indent=1, ensure_ascii=True).encode() + if record is False: + return raw + made_from = record_for(job) if record is None else record + return seed_contract._splice(raw, made_from, empty=not body) + + +def _seed_path(tmp_path: Path) -> Path: + return tmp_path / "seed" / "contract.json" + + +def _place(tmp_path: Path, data: bytes) -> Path: + path = _seed_path(tmp_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return path + + +def _reading(tmp_path: Path, data: bytes) -> Path: + reading = tmp_path / "reading" + reading.mkdir(exist_ok=True) + (reading / "contract.json").write_bytes(data) + return reading + + +# -- the record a reading leaves --------------------------------------------------------------- + + +def test_a_reading_records_what_it_was_made_from(tmp_path): + job = _job() + original = json.dumps(CONTRACT, indent=1).encode() + + saved = with_record(job, original) + + assert saved is not None + assert record_for(job) == { + "version": 1, + "source": {"kind": "github", "repository": "owner/repo", "commit_sha": COMMIT}, + "subfolder": FOLDER, + "evals_sha256": hashlib.sha256(b'[{"name":"a"}]').hexdigest(), + "generic_harness_v1": True, + } + assert json.loads(saved)[RECORD_KEY] == record_for(job) + compact = json.dumps(record_for(job), separators=(",", ":")).encode() + prefix = b'"reading_inputs": ' + compact + b", " + assert saved[1 : 1 + len(prefix)] == prefix + assert saved[:1] + saved[1 + len(prefix) :] == original + + +def test_the_record_never_reaches_the_agent_document(tmp_path): + job = _job() + source = _source(tmp_path) + reading = _reading(tmp_path, json.dumps(CONTRACT).encode()) + + assert document(job, source, reading) == 0 + + text = (reading / "agent.json").read_text(encoding="utf-8") + assert RECORD_KEY not in text + saved = json.loads((reading / "contract.json").read_text(encoding="utf-8")) + assert saved[RECORD_KEY] == record_for(job) + contract = AgentContract.model_validate(saved).model_dump(mode="json") + framework = detect_framework(source / FOLDER, repo_root=source, package_dir=None) + assert json.loads(text)["agent"] == build_authored_agent( + contract, subfolder=FOLDER, framework=framework + ) + + +@pytest.mark.parametrize( + "size", + [SEED_LIMIT_BYTES - 10, SEED_LIMIT_BYTES + 1], + ids=["near_the_cap", "over_the_cap"], +) +def test_the_record_is_skipped_near_the_size_cap(tmp_path, monkeypatch, size): + job = _job() + source = _source(tmp_path) + text = json.dumps(CONTRACT).encode() + # Trailing spaces still parse, so only the file's size changes. + data = text + b" " * (size - len(text)) + reading = _reading(tmp_path, data) + parsed: list[int] = [] + + def loads(value, *args, **kwargs): + parsed.append(len(value)) + return json.loads(value, *args, **kwargs) + + monkeypatch.setattr( + seed_contract, "json", SimpleNamespace(loads=loads, dumps=json.dumps) + ) + + code = document(job, source, reading) + + assert (reading / "contract.json").read_bytes() == data + if size < SEED_LIMIT_BYTES: + assert code == 0 + else: + assert parsed == [] + + +def test_a_record_is_never_parsed_from_a_contract_over_the_cap(monkeypatch): + parsed: list[int] = [] + + def loads(value, *args, **kwargs): + parsed.append(len(value)) + return json.loads(value, *args, **kwargs) + + monkeypatch.setattr( + seed_contract, "json", SimpleNamespace(loads=loads, dumps=json.dumps) + ) + text = json.dumps(CONTRACT).encode() + raw = text + b" " * (SEED_LIMIT_BYTES + 1 - len(text)) + + assert seed_contract.with_record(_job(), raw) is None + assert parsed == [] + + +def test_a_failed_record_keeps_the_reading(tmp_path, monkeypatch): + def broken(job, raw): + raise OSError("the disk refused the write") + + monkeypatch.setattr(seed_contract, "with_record", broken) + data = json.dumps(CONTRACT).encode() + reading = _reading(tmp_path, data) + + assert document(_job(), _source(tmp_path), reading) == 0 + + assert (reading / "contract.json").read_bytes() == data + + +def test_a_failed_record_keeps_its_traceback(tmp_path, monkeypatch, caplog): + def broken(job, raw): + raise RuntimeError("rec-9") + + monkeypatch.setattr(seed_contract, "with_record", broken) + reading = _reading(tmp_path, json.dumps(CONTRACT).encode()) + + with caplog.at_level(logging.WARNING, logger=reading_result.__name__): + assert document(_job(), _source(tmp_path), reading) == 0 + + [logged] = [r for r in caplog.records if r.name == reading_result.__name__] + assert logged.getMessage() == "reading record not written" + # The logger's mask keeps the trace only as masked text. + assert "Traceback" in logged.exc_text + assert "RuntimeError: rec-9" in logged.exc_text + + +LATER = json.dumps({**CONTRACT, "agent": "saved-later"}).encode() + + +_READ = json.dumps(CONTRACT).encode() + + +@pytest.mark.parametrize( + "later", + [LATER, _READ.replace(b'"support"', b'"Support"', 1)], + ids=["longer", "same_length"], +) +def test_a_save_after_the_read_is_never_overwritten(tmp_path, monkeypatch, later): + job = _job() + reading = _reading(tmp_path, _READ) + real = seed_contract.with_record + + def saved_meanwhile(job, raw): + # The session saves again while the record is being made. + (reading / "contract.json").write_bytes(later) + return real(job, raw) + + monkeypatch.setattr(seed_contract, "with_record", saved_meanwhile) + + assert document(job, _source(tmp_path), reading) == 0 + + assert (reading / "contract.json").read_bytes() == later + + +@pytest.mark.parametrize("saved_during_the_read", [False, True], ids=["quiet", "saved"]) +def test_the_record_is_made_from_the_bytes_the_document_used( + tmp_path, monkeypatch, saved_during_the_read +): + job = _job() + data = json.dumps(CONTRACT, indent=1).encode() + reading = _reading(tmp_path, data) + real = seed_contract.with_record + seen: list[bytes] = [] + + def spy(job, raw): + seen.append(raw) + return real(job, raw) + + monkeypatch.setattr(seed_contract, "with_record", spy) + if saved_during_the_read: + landed: list[bool] = [] + + def saved_then_sized(descriptor): + # Right after the document's read: only a fresh read of the file would see it. + if not landed: + landed.append(True) + (reading / "contract.json").write_bytes(LATER) + return os.fstat(descriptor) + + monkeypatch.setattr( + reading_result, + "os", + SimpleNamespace(fstat=saved_then_sized, replace=os.replace), + ) + + assert document(job, _source(tmp_path), reading) == 0 + + assert seen == [data] + saved = (reading / "contract.json").read_bytes() + if saved_during_the_read: + assert saved == LATER + assert "saved-later" not in (reading / "agent.json").read_text() + else: + assert saved == real(job, data) + + +@pytest.mark.parametrize( + ("make", "made_from"), + [ + ( + lambda: _job( + execution="local", + source={k: v for k, v in GITHUB.items() if k != "commit_sha"}, + ), + None, + ), + ( + lambda: _job(source={"kind": "archive", "archive_artifact_id": "upload-1"}), + {"kind": "archive", "archive_artifact_id": "upload-1"}, + ), + ( + lambda: _job( + agent={ + "connector": "vapi", + "mode": "connect_only", + "config": {"assistant_id": "assistant-1"}, + } + ), + None, + ), + ], + ids=["github_without_a_commit", "archive", "connect_only"], +) +def test_no_record_without_a_source_identity(tmp_path, make, made_from): + job = make() + data = json.dumps(CONTRACT).encode() + + saved = with_record(job, data) + + if made_from is None: + assert record_for(job) is None + assert saved is None + else: + assert record_for(job)["source"] == made_from + assert record_for(job)["subfolder"] == "" + assert json.loads(saved)[RECORD_KEY] == record_for(job) + + +def test_the_record_lowercases_the_commit(): + job = _job(source={**GITHUB, "commit_sha": "AB" * 20}) + + assert job.source.commit_sha == "AB" * 20 + assert record_for(job)["source"]["commit_sha"] == "ab" * 20 + + +# -- the build command ------------------------------------------------------------------------- + + +def _watch_authoring(monkeypatch): + calls: list[str] = [] + + def compose(*_args, **_kwargs): + calls.append("compose") + + async def auto(_namespace) -> int: + calls.append("auto") + return 0 + + def await_harness(_output) -> int: + calls.append("await") + return 0 + + monkeypatch.setattr(authoring_entrypoint, "initialize_hosted_compose", compose) + monkeypatch.setattr(authoring_entrypoint, "_auto", auto) + monkeypatch.setattr(authoring_entrypoint, "_await_harness", await_harness) + monkeypatch.setattr( + authoring_entrypoint, "_persist_authored_scenario_count", lambda *a: None + ) + return calls + + +def _build(tmp_path: Path, *flags: str) -> int: + return authoring_entrypoint.main( + [ + str(_job_file(tmp_path, _job())), + "--source", + str(_source(tmp_path)), + "--output", + str(tmp_path / "authoring"), + *flags, + ] + ) + + +def _seeded(tmp_path: Path, seed: Path | None = None) -> tuple[str, ...]: + return ( + "--await-harness", + "--seed-contract", + str(seed if seed is not None else _seed_path(tmp_path)), + ) + + +def _marker(tmp_path: Path) -> Path: + return tmp_path / "authoring" / SEED_ACCEPTED + + +def test_await_mode_accepts_the_seed_at_its_place(tmp_path, monkeypatch): + calls = _watch_authoring(monkeypatch) + _place(tmp_path, _seed_bytes(_job())) + assert not (tmp_path / "authoring").exists() + + assert _build(tmp_path, *_seeded(tmp_path)) == 0 + + assert _marker(tmp_path).is_file() + assert not (tmp_path / "authoring" / "contract.json").exists() + assert not (tmp_path / "authoring" / "failure.json").exists() + assert calls == ["compose", "await"] + + +def _elsewhere(tmp_path: Path) -> Path: + other = tmp_path / "other" / "contract.json" + other.parent.mkdir(parents=True) + other.write_bytes(_seed_bytes(_job())) + return other + + +def _nothing(tmp_path: Path) -> Path: + return _seed_path(tmp_path) + + +def _a_link(tmp_path: Path) -> Path: + real = tmp_path / "real.json" + real.write_bytes(_seed_bytes(_job())) + path = _seed_path(tmp_path) + path.parent.mkdir(parents=True) + path.symlink_to(real) + return path + + +def _a_folder(tmp_path: Path) -> Path: + path = _seed_path(tmp_path) + path.mkdir(parents=True) + return path + + +def _a_pipe(tmp_path: Path) -> Path: + path = _seed_path(tmp_path) + path.parent.mkdir(parents=True) + os.mkfifo(path) + return path + + +def _holding(data: bytes): + return lambda tmp_path: _place(tmp_path, data) + + +def _over_the_cap(tmp_path: Path) -> Path: + raw = _seed_bytes(_job()) + return _place(tmp_path, raw + b" " * (SEED_LIMIT_BYTES + 1 - len(raw))) + + +INVALID = "seed_contract_invalid" + + +@pytest.mark.parametrize( + ("make", "code", "details"), + [ + (_elsewhere, INVALID, {"reason": "wrong_path"}), + (_nothing, INVALID, {"reason": "unreadable"}), + (_a_link, INVALID, {"reason": "unreadable"}), + (_a_folder, INVALID, {"reason": "not_a_file"}), + (_a_pipe, INVALID, {"reason": "not_a_file"}), + (_holding(b"\xff\xfe"), INVALID, {"reason": "not_json"}), + (_holding(b"not json"), INVALID, {"reason": "not_json"}), + (_holding(b"[]"), INVALID, {"reason": "not_an_object"}), + (_holding(b"[" * 100000), INVALID, {"reason": "not_json"}), + ( + _over_the_cap, + "seed_contract_too_large", + {"contract_bytes": SEED_LIMIT_BYTES + 1}, + ), + ], + ids=[ + "wrong_path", + "missing", + "symlink", + "directory", + "fifo", + "not_utf8", + "not_json", + "array", + "too_deep", + "too_large", + ], +) +def test_a_bad_seed_is_refused_by_the_build_command( + tmp_path, monkeypatch, make, code, details +): + calls = _watch_authoring(monkeypatch) + seed = make(tmp_path) + + assert _build(tmp_path, *_seeded(tmp_path, seed)) == 78 + + failure = json.loads((tmp_path / "authoring" / "failure.json").read_text()) + assert { + key: failure[key] for key in ("domain", "stage", "code", "retryable", "details") + } == { + "domain": "platform_sync", + "stage": "understanding_agent", + "code": code, + "retryable": False, + "details": details, + } + assert calls == [] + assert not _marker(tmp_path).exists() + + +def test_a_seed_exactly_at_the_cap_is_accepted(tmp_path, monkeypatch): + _watch_authoring(monkeypatch) + raw = _seed_bytes(_job()) + _place(tmp_path, raw + b" " * (SEED_LIMIT_BYTES - len(raw))) + assert _seed_path(tmp_path).stat().st_size == SEED_LIMIT_BYTES + + assert _build(tmp_path, *_seeded(tmp_path)) == 0 + + assert _marker(tmp_path).is_file() + + +@pytest.mark.parametrize("case", ["no_seed", "refused"]) +def test_the_build_command_clears_a_stale_mark(tmp_path, monkeypatch, case): + calls = _watch_authoring(monkeypatch) + _marker(tmp_path).parent.mkdir(parents=True) + _marker(tmp_path).write_bytes(b"") + + if case == "no_seed": + assert _build(tmp_path, "--await-harness") == 0 + assert calls == ["compose", "await"] + else: + assert _build(tmp_path, *_seeded(tmp_path, _elsewhere(tmp_path))) == 78 + assert calls == [] + + assert not _marker(tmp_path).exists() + + +def test_a_refusal_holds_none_of_the_seed(tmp_path, monkeypatch, capfd): + _watch_authoring(monkeypatch) + _place(tmp_path, b"not json " + TOKEN.encode()) + + assert _build(tmp_path, *_seeded(tmp_path)) == 78 + + assert TOKEN not in (tmp_path / "authoring" / "failure.json").read_text() + captured = capfd.readouterr() + assert TOKEN not in captured.err + assert TOKEN not in captured.out + + +def test_adjustments_mode_ignores_the_seed(tmp_path, monkeypatch, capsys): + calls = _watch_authoring(monkeypatch) + + code = _build( + tmp_path, + "--adjustments", + str(tmp_path / "adjustments.jsonl"), + "--seed-contract", + str(tmp_path / "other" / "contract.json"), + ) + + assert code == 0 + assert "auto" in calls + assert not (tmp_path / "authoring" / "failure.json").exists() + assert not _marker(tmp_path).exists() + assert "seed contract: not used" in capsys.readouterr().err + + +@pytest.mark.parametrize( + "flags", + [ + ["--certify-only", "--bundle-output", "x"], + ["--read-agent-only"], + ["--find-agents"], + ], + ids=["certify_only", "read_agent_only", "find_agents"], +) +def test_the_seed_cannot_be_combined(tmp_path, monkeypatch, flags): + def never(*_args, **_kwargs): + raise AssertionError("refused before anything runs") + + monkeypatch.setattr(authoring_entrypoint, "_certify_only", never) + monkeypatch.setattr(reading_entry, "run_reading_mode", never) + + with pytest.raises(SystemExit) as stopped: + _build(tmp_path, "--seed-contract", str(_seed_path(tmp_path)), *flags) + + assert stopped.value.code == 2 + + +# -- use_seed ---------------------------------------------------------------------------------- + + +def test_a_matching_seed_is_copied_byte_for_byte(tmp_path): + job = _job() + seed = _place(tmp_path, _seed_bytes(job)) + output = tmp_path / "authoring" + + assert use_seed(seed, job, _source(tmp_path), output) == ("used", []) + + assert (output / "contract.json").read_bytes() == seed.read_bytes() + # What the world stage compares the world it builds against. + assert load(output).base_environment == CONTRACT["base_environment"] + assert (output / "generic-harness" / "source-model.json").is_file() + + +@pytest.mark.parametrize("generic", [False, None], ids=["false", "absent"]) +def test_a_job_that_isnt_generic_installs_and_records_the_seed(tmp_path, generic): + metadata = {"available_evals": EVALS} + if generic is not None: + metadata["generic_harness_v1"] = generic + job = _job(metadata=metadata) + seed = _place(tmp_path, _seed_bytes(job)) + output = tmp_path / "authoring" + + assert use_seed(seed, job, _source(tmp_path), output) == ("used", []) + + assert (output / "contract.json").read_bytes() == seed.read_bytes() + assert (output / seed_contract.SEED_INSTALLED).is_file() + # Nothing else of a generic-harness build is made for it. + assert not (output / "generic-harness" / "source-model.json").exists() + + +def test_a_reading_with_no_flag_seeds_a_build_whose_flag_is_false(tmp_path): + seed = _place(tmp_path, _seed_bytes(_job(metadata={"available_evals": EVALS}))) + build = _job(metadata={"available_evals": EVALS, "generic_harness_v1": False}) + + assert use_seed(seed, build, _source(tmp_path), tmp_path / "authoring") == ( + "used", + [], + ) + + +@pytest.mark.parametrize("generic", [False, None], ids=["false", "absent"]) +def test_a_builder_for_a_job_that_isnt_generic_uses_the_seed( + tmp_path, monkeypatch, generic +): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + metadata = {"available_evals": EVALS} + if generic is not None: + metadata["generic_harness_v1"] = generic + job = _job(metadata=metadata) + runtime.job = job.model_dump(mode="json") + _place(tmp_path, _seed_bytes(job)) + workspace = tmp_path / "authoring" + + assert _open(runtime).contract is not None + assert _outcome(tmp_path) == {"seed": "used", "differs": []} + assert (workspace / seed_contract.SEED_INSTALLED).is_file() + + # A restart after a contract edit and a lost outcome still says it was used. + (workspace / SEED_RESULT).unlink() + (workspace / "contract.json").write_text(json.dumps(CONTRACT), encoding="utf-8") + assert _open(_restarted(runtime)).contract is not None + assert _outcome(tmp_path) == {"seed": "used", "differs": []} + + +@pytest.mark.parametrize( + ("build", "body", "names"), + [ + ( + lambda: _job(source={**GITHUB, "commit_sha": OTHER_COMMIT}), + CONTRACT, + ["source"], + ), + ( + lambda: _job(source={**GITHUB, "repository": "owner/other"}), + CONTRACT, + ["source"], + ), + ( + lambda: _job(source={**GITHUB, "subfolder": "agents/b"}), + CONTRACT, + ["subfolder", "contract_subfolder"], + ), + ( + lambda: _job( + metadata={ + "available_evals": [{"name": "b"}], + "generic_harness_v1": True, + } + ), + CONTRACT, + ["evals_sha256"], + ), + ( + lambda: _job( + metadata={"available_evals": EVALS, "generic_harness_v1": False} + ), + CONTRACT, + ["generic_harness_v1"], + ), + ( + _job, + { + **CONTRACT, + "runtime": {**CONTRACT["runtime"], "source_subfolder": "agents/other"}, + }, + ["contract_subfolder"], + ), + ], + ids=["commit", "repository", "subfolder", "evals", "generic", "contract_folder"], +) +def test_a_mismatched_seed_reads_again(tmp_path, capsys, build, body, names): + seed = _place(tmp_path, _seed_bytes(_job(), body)) + job = build() + output = tmp_path / "authoring" + + assert differs(json.loads(seed.read_bytes()), job) == names + assert use_seed(seed, job, _source(tmp_path), output) == ("mismatch", names) + + assert not (output / "contract.json").exists() + assert not (output / "failure.json").exists() + assert f"differs: {', '.join(names)}" in capsys.readouterr().err + + +@pytest.mark.parametrize( + "body", + [ + CONTRACT, + { + **CONTRACT, + "runtime": {**CONTRACT["runtime"], "source_subfolder": "agents/b"}, + }, + ], + ids=["no_record", "no_record_and_another_folder"], +) +def test_a_seed_without_a_record_reads_again(tmp_path, body): + job = _job() + seed = _place(tmp_path, _seed_bytes(job, body, record=False)) + + assert differs(json.loads(seed.read_bytes()), job) == ["reading_inputs"] + assert use_seed(seed, job, _source(tmp_path), tmp_path / "authoring") == ( + "mismatch", + ["reading_inputs"], + ) + + +def test_a_rejected_contract_reads_again(tmp_path, capsys): + job = _job() + seed = _place(tmp_path, _seed_bytes(job, {**CONTRACT, "tools": TOKEN})) + output = tmp_path / "authoring" + + assert use_seed(seed, job, _source(tmp_path), output) == ("rejected", []) + + assert not (output / "contract.json").exists() + assert not (output / "failure.json").exists() + err = capsys.readouterr().err + assert "problems:" in err + assert TOKEN not in err + + +def test_a_failed_setup_step_reads_again(tmp_path, monkeypatch, capsys): + def half_done(destination, source, job): + model = destination / "generic-harness" / "source-model.json" + model.parent.mkdir(parents=True, exist_ok=True) + model.write_text("{}", encoding="utf-8") + raise RuntimeError("the source could not be modelled") + + monkeypatch.setattr(cli, "after_contract", half_done) + job = _job() + seed = _place(tmp_path, _seed_bytes(job)) + output = tmp_path / "authoring" + + assert use_seed(seed, job, _source(tmp_path), output) == ("setup_failed", []) + + assert not (output / "contract.json").exists() + assert not (output / "generic-harness" / "source-model.json").exists() + assert "class: RuntimeError" in capsys.readouterr().err + + +def test_a_seed_changed_after_it_was_accepted_reads_again(tmp_path): + seed = _place(tmp_path, b"not json") + output = tmp_path / "authoring" + + assert use_seed(seed, _job(), _source(tmp_path), output) == ("unreadable", []) + + assert not (output / "failure.json").exists() + assert not (output / "contract.json").exists() + + +# -- the builder ------------------------------------------------------------------------------- + + +class _TurnRan(Exception): + """Reached the point where a turn starts, which is as far as a test without a model goes.""" + + +class _Opened(Exception): + """A stage was opened, with the contract it was handed.""" + + def __init__(self, contract) -> None: + super().__init__("opened") + self.contract = contract + + +def _runtime(workspace: Path, *, state=None, control_only=True) -> HostedChatRuntime: + runtime = object.__new__(HostedChatRuntime) + runtime.workspace = workspace + runtime.job = {"scenario_count": 2} + emitted: list[tuple[str, dict]] = [] + + async def emit(kind, **fields): + emitted.append((kind, fields)) + + context = {"capabilities": {"control_only": control_only}} + if state is not None: + context["status"] = {"state": state} + runtime.client = SimpleNamespace( + turn_context=context, command_watermark=3, emit=emit + ) + runtime.emitted = emitted + runtime.conversation = SimpleNamespace(stage_name="understand", stage=None) + runtime._nudge = None + runtime._from_platform = False + runtime._last_progress, runtime._idle_turns, runtime._opened_with = None, 0, None + runtime._what_exists = lambda: (False, False) + runtime._after_saved = lambda: None + runtime.handled = [] + + async def handle(command): + runtime.handled.append(command) + + runtime._handle = handle + return runtime + + +def _command(*, from_platform=True) -> dict: + return { + "sequence": 7, + "message_id": "command-7", + "kind": "message", + "payload": {"content": "Build it.", "from_platform": from_platform}, + } + + +def _handle_command(runtime, command) -> None: + async def checkpoint(): + raise _TurnRan + + runtime._checkpoint = checkpoint + asyncio.run(runtime._handle_command(command)) + + +def _builder( + tmp_path: Path, monkeypatch, *, marker=True, state=None, control_only=True +) -> HostedChatRuntime: + monkeypatch.delenv(TARGET_PHONE_ENV, raising=False) + monkeypatch.delenv(PIN_ENV, raising=False) + job = _job() + workspace = tmp_path / "authoring" + workspace.mkdir() + runtime = _runtime(workspace, state=state, control_only=control_only) + runtime.job = job.model_dump(mode="json") + runtime.source = _source(tmp_path) + _place(tmp_path, _seed_bytes(job)) + if marker: + (workspace / SEED_ACCEPTED).write_bytes(b"") + + async def overview(): + return None + + runtime.client.overview = overview + order: list[str] = [] + + def after_saved(): + order.append("after_saved") + + def opening(contract): + order.append("open") + raise _Opened(contract) + + runtime._after_saved = after_saved + runtime._open_for_building = opening + runtime.order = order + return runtime + + +def _open(runtime) -> _Opened: + with pytest.raises(_Opened) as opened: + asyncio.run(runtime._open_builder()) + return opened.value + + +def test_opening_the_builder_installs_an_accepted_seed(tmp_path, monkeypatch): + runtime = _builder(tmp_path, monkeypatch) + runtime._what_exists = HostedChatRuntime._what_exists.__get__(runtime) + + opened = _open(runtime) + + assert opened.contract is not None + assert opened.contract.base_environment == CONTRACT["base_environment"] + installed = (tmp_path / "authoring" / "contract.json").read_bytes() + assert installed == _seed_path(tmp_path).read_bytes() + assert runtime.order == ["after_saved", "open"] + + +def test_a_seed_without_the_marker_is_not_used(tmp_path, monkeypatch): + runtime = _builder(tmp_path, monkeypatch, marker=False) + runtime._what_exists = HostedChatRuntime._what_exists.__get__(runtime) + + assert _open(runtime).contract is None + + assert not (tmp_path / "authoring" / "contract.json").exists() + + +@pytest.mark.parametrize("case", ["contract", "failure", "world", "build_over"]) +def test_the_seed_is_left_alone_when(tmp_path, monkeypatch, case): + runtime = _builder( + tmp_path, + monkeypatch, + state="canceled" if case == "build_over" else None, + # Outside control-only only the file itself stops it, not the build being over. + control_only=case != "failure", + ) + + def never(*_args, **_kwargs): + raise AssertionError("the seed must be left alone") + + monkeypatch.setattr(seed_contract, "use_seed", never) + workspace = tmp_path / "authoring" + kept = json.dumps(CONTRACT).encode() + if case == "contract": + (workspace / "contract.json").write_bytes(kept) + elif case == "failure": + (workspace / "failure.json").write_text("{}", encoding="utf-8") + elif case == "world": + runtime._what_exists = lambda: (True, True) + + def built(contract, overview, policy_job): + raise _Opened(contract) + + runtime._open_built = built + + _open(runtime) + + if case == "contract": + assert (workspace / "contract.json").read_bytes() == kept + else: + assert not (workspace / "contract.json").exists() + assert not (workspace / SEED_RESULT).exists() + + +def test_a_command_installs_nothing_before_a_stage_opens(tmp_path, monkeypatch): + runtime = _builder(tmp_path, monkeypatch) + + with pytest.raises(_TurnRan): + _handle_command(runtime, _command()) + + assert not (tmp_path / "authoring" / "contract.json").exists() + + +def test_a_failed_source_model_step_still_opens_the_world_stage( + tmp_path, monkeypatch, capsys +): + runtime = _builder(tmp_path, monkeypatch, marker=False) + (tmp_path / "authoring" / "contract.json").write_text( + json.dumps(CONTRACT), encoding="utf-8" + ) + + def failing(): + raise RuntimeError("the source could not be modelled") + + runtime._after_saved = failing + + assert _open(runtime).contract is not None + + assert "class: RuntimeError" in capsys.readouterr().err + + +def _outcome(tmp_path: Path) -> dict: + # The platform's path, spelled out: it is stored with the version but never hashed. + workspace = tmp_path / "authoring" + assert not (workspace / "seed-result.json").exists() + return json.loads((workspace / "generic-harness" / "seed-result.json").read_bytes()) + + +def _real_world(runtime) -> HostedChatRuntime: + runtime._what_exists = HostedChatRuntime._what_exists.__get__(runtime) + return runtime + + +def test_a_later_opening_never_installs(tmp_path, monkeypatch): + runtime = _real_world(_builder(tmp_path, monkeypatch, marker=False)) + workspace = tmp_path / "authoring" + + assert _open(runtime).contract is None + decided = (workspace / SEED_RESULT).read_bytes() + assert json.loads(decided) == {"seed": "not_accepted", "differs": []} + + # The build command's mark lands after the builder's first opening. + _marker(tmp_path).write_bytes(b"") + assert _open(runtime).contract is None + + assert not (workspace / "contract.json").exists() + assert _marker(tmp_path).is_file() + assert (workspace / SEED_RESULT).read_bytes() == decided + + # A new process finds only the file to go by. + assert _open(_restarted(runtime)).contract is None + assert not (workspace / "contract.json").exists() + assert (workspace / SEED_RESULT).read_bytes() == decided + + +def _restarted(runtime) -> HostedChatRuntime: + """The same workspace as a new process sees it: nothing remembered of the seed.""" + again = object.__new__(HostedChatRuntime) + again.__dict__.update( + { + key: value + for key, value in vars(runtime).items() + if not key.startswith("_seed") + } + ) + return again + + +def _from_another_commit(tmp_path, monkeypatch, runtime) -> None: + _place(tmp_path, _seed_bytes(_job(source={**GITHUB, "commit_sha": OTHER_COMMIT}))) + + +@pytest.mark.parametrize("outcome", ["used", "mismatch"]) +def test_the_mark_is_consumed_whatever_the_outcome(tmp_path, monkeypatch, outcome): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + if outcome == "mismatch": + _from_another_commit(tmp_path, monkeypatch, runtime) + + _open(runtime) + + assert not _marker(tmp_path).exists() + assert _outcome(tmp_path)["seed"] == outcome + + +def _as_placed(tmp_path, monkeypatch, runtime) -> None: + pass + + +def _not_a_contract(tmp_path, monkeypatch, runtime) -> None: + _place(tmp_path, _seed_bytes(_job(), {**CONTRACT, "tools": TOKEN})) + + +def _changed_since(tmp_path, monkeypatch, runtime) -> None: + _place(tmp_path, b"not json") + + +def _setup_fails(tmp_path, monkeypatch, runtime) -> None: + def fails(destination, source, job): + raise RuntimeError("the source could not be modelled") + + monkeypatch.setattr(cli, "after_contract", fails) + + +def _job_unreadable(tmp_path, monkeypatch, runtime) -> None: + runtime.job = {**runtime.job, "execution": "nowhere"} + + +@pytest.mark.parametrize( + ("arrange", "expected"), + [ + (_as_placed, {"seed": "used", "differs": []}), + (_from_another_commit, {"seed": "mismatch", "differs": ["source"]}), + (_not_a_contract, {"seed": "rejected", "differs": []}), + (_changed_since, {"seed": "unreadable", "differs": []}), + (_setup_fails, {"seed": "setup_failed", "differs": []}), + (_job_unreadable, {"seed": "error", "differs": []}), + ], + ids=["used", "mismatch", "rejected", "unreadable", "setup_failed", "error"], +) +def test_the_outcome_is_written_for_the_platform( + tmp_path, monkeypatch, arrange, expected +): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + arrange(tmp_path, monkeypatch, runtime) + + opened = _open(runtime) + + used = expected["seed"] == "used" + assert _outcome(tmp_path) == expected + assert (opened.contract is not None) == used + assert (tmp_path / "authoring" / "contract.json").exists() == used + + +def test_no_outcome_without_a_seed(tmp_path, monkeypatch): + runtime = _real_world(_builder(tmp_path, monkeypatch, marker=False)) + _seed_path(tmp_path).unlink() + + assert _open(runtime).contract is None + + assert not (tmp_path / "authoring" / SEED_RESULT).exists() + + +def test_no_outcome_after_a_refusal(tmp_path, monkeypatch): + # Outside control-only, so the refusal's own file is all that stops it. + runtime = _real_world( + _builder(tmp_path, monkeypatch, marker=False, control_only=False) + ) + (tmp_path / "authoring" / "failure.json").write_text("{}", encoding="utf-8") + + _open(runtime) + + assert not (tmp_path / "authoring" / SEED_RESULT).exists() + + +def _install_breaks(tmp_path, monkeypatch, runtime) -> None: + def breaks(destination, source, job): + raise RuntimeError("boom-7") + + monkeypatch.setattr(cli, "after_contract", breaks) + + +def _check_breaks(tmp_path, monkeypatch, runtime) -> None: + def breaks(body, job): + raise RuntimeError("boom-7") + + monkeypatch.setattr(seed_contract, "differs", breaks) + + +class _JobThatBreaks: + @classmethod + def model_validate(cls, value): + raise RuntimeError("boom-7") + + +def _job_breaks(tmp_path, monkeypatch, runtime) -> None: + # Imported first, so nothing the opening imports binds the stand-in for good. + from fi.alk.harness import builder # noqa: F401 + from fi.alk.harness import job as job_module + + monkeypatch.setattr(job_module, "HarnessJob", _JobThatBreaks) + + +def _source_model_breaks(tmp_path, monkeypatch, runtime) -> None: + (tmp_path / "authoring" / "contract.json").write_text( + json.dumps(CONTRACT), encoding="utf-8" + ) + + def breaks(): + raise RuntimeError("boom-7") + + runtime._after_saved = breaks + + +@pytest.mark.parametrize( + "arrange", + [_install_breaks, _check_breaks, _job_breaks, _source_model_breaks], + ids=["after_contract", "use_seed", "job", "after_saved"], +) +def test_an_unexpected_failure_keeps_its_traceback( + tmp_path, monkeypatch, capsys, arrange +): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + arrange(tmp_path, monkeypatch, runtime) + + _open(runtime) + + err = capsys.readouterr().err + assert "Traceback" in err + assert "boom-7" in err + + +def test_a_job_that_fails_validation_logs_no_values(tmp_path, monkeypatch, capfd): + secret = "sk-JOBTOKEN42" + runtime = _real_world(_builder(tmp_path, monkeypatch)) + runtime.job = {**runtime.job, "execution": secret} + + _open(runtime) + + err = capfd.readouterr().err + assert "ValidationError" in err + assert "Traceback" not in err + assert secret not in err + assert _outcome(tmp_path) == {"seed": "error", "differs": []} + assert secret not in (tmp_path / "authoring" / SEED_RESULT).read_text() + + +class _Holds(BaseModel): + value: int + + +def _holding_values() -> ValidationError: + try: + _Holds.model_validate({"value": TOKEN}) + except ValidationError as exc: + return exc + raise AssertionError("the value must fail") + + +def _install_holds_values(tmp_path, monkeypatch, runtime) -> None: + def fails(destination, source, job): + raise _holding_values() + + monkeypatch.setattr(cli, "after_contract", fails) + + +def _check_holds_values(tmp_path, monkeypatch, runtime) -> None: + def fails(body, job): + raise _holding_values() + + monkeypatch.setattr(seed_contract, "differs", fails) + + +def _source_model_holds_values(tmp_path, monkeypatch, runtime) -> None: + (tmp_path / "authoring" / "contract.json").write_text( + json.dumps(CONTRACT), encoding="utf-8" + ) + + def fails(): + raise _holding_values() + + runtime._after_saved = fails + + +def _chained_under_another(tmp_path, monkeypatch, runtime) -> None: + def fails(destination, source, job): + try: + raise _holding_values() + except ValidationError: + raise RuntimeError("boom-7") + + monkeypatch.setattr(cli, "after_contract", fails) + + +@pytest.mark.parametrize( + "arrange", + [ + _install_holds_values, + _check_holds_values, + _source_model_holds_values, + _chained_under_another, + ], + ids=["after_contract", "use_seed", "after_saved", "chained"], +) +def test_a_validation_error_never_prints_its_values( + tmp_path, monkeypatch, capfd, arrange +): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + arrange(tmp_path, monkeypatch, runtime) + + _open(runtime) + + err = capfd.readouterr().err + assert "class: " in err + assert "Traceback" not in err + assert TOKEN not in err + + +def _result_write_fails(monkeypatch, *, times: int) -> list[str]: + real = os.replace + failed: list[str] = [] + + def replace(src, dst): + if Path(dst).name == SEED_RESULT.name and len(failed) < times: + failed.append(str(dst)) + raise OSError(28, "No space left on device") + return real(src, dst) + + monkeypatch.setattr(seed_contract.os, "replace", replace) + return failed + + +def test_the_outcome_is_written_whole_or_not_at_all(tmp_path, monkeypatch, capsys): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + workspace = tmp_path / "authoring" + failed = _result_write_fails(monkeypatch, times=1) + + assert _open(runtime).contract is not None + + assert failed + assert not (workspace / SEED_RESULT).exists() + assert not list(workspace.rglob(".seed-*.tmp")) + err = capsys.readouterr().err + assert "seed result not written; class: OSError" in err + assert "Traceback" in err + + # The next opening writes what was decided, and installs nothing again. + installed = (workspace / "contract.json").read_bytes() + assert _open(runtime).contract is not None + assert _outcome(tmp_path) == {"seed": "used", "differs": []} + assert (workspace / "contract.json").read_bytes() == installed + + +def test_a_lost_outcome_never_lets_a_reopen_install(tmp_path, monkeypatch): + runtime = _real_world(_builder(tmp_path, monkeypatch, marker=False)) + workspace = tmp_path / "authoring" + _result_write_fails(monkeypatch, times=1) + + assert _open(runtime).contract is None + assert not (workspace / SEED_RESULT).exists() + + # The build command's mark lands while the reading is under way. + _marker(tmp_path).write_bytes(b"") + assert _open(runtime).contract is None + + assert not (workspace / "contract.json").exists() + assert _outcome(tmp_path) == {"seed": "not_accepted", "differs": []} + + +def test_a_restart_after_the_install_says_it_was_used(tmp_path, monkeypatch): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + workspace = tmp_path / "authoring" + # The process stopped after the install and before it could say so. + (workspace / "contract.json").write_bytes(_seed_path(tmp_path).read_bytes()) + + assert _open(runtime).contract is not None + + assert _outcome(tmp_path) == {"seed": "used", "differs": []} + assert not _marker(tmp_path).exists() + assert runtime.order == ["after_saved", "open"] + + +def test_a_restart_after_an_edit_and_a_lost_outcome_says_it_was_used( + tmp_path, monkeypatch +): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + workspace = tmp_path / "authoring" + _result_write_fails(monkeypatch, times=1) + assert _open(runtime).contract is not None + assert (workspace / seed_contract.SEED_INSTALLED).is_file() + + # A contract tool rewrites the contract without the record, then the process stops. + (workspace / "contract.json").write_text(json.dumps(CONTRACT), encoding="utf-8") + assert _open(_restarted(runtime)).contract is not None + + assert _outcome(tmp_path) == {"seed": "used", "differs": []} + + +def test_an_install_that_cannot_be_recorded_is_taken_back( + tmp_path, monkeypatch, capsys +): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + workspace = tmp_path / "authoring" + real = os.replace + + def replace(src, dst): + if Path(dst).name == seed_contract.SEED_INSTALLED.name: + raise OSError(28, "No space left on device") + return real(src, dst) + + monkeypatch.setattr(seed_contract.os, "replace", replace) + + assert _open(runtime).contract is None + + assert _outcome(tmp_path) == {"seed": "error", "differs": []} + assert not (workspace / "contract.json").exists() + assert not (workspace / seed_contract.SEED_INSTALLED).exists() + assert "Traceback" in capsys.readouterr().err + + +def test_a_contract_that_cannot_be_compared_is_never_claimed(tmp_path, monkeypatch): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + workspace = tmp_path / "authoring" + (workspace / "contract.json").write_bytes(_seed_path(tmp_path).read_bytes()) + + def breaks(path): + raise RuntimeError("boom-7") + + monkeypatch.setattr(seed_contract, "read_seed", breaks) + + assert _open(runtime).contract is not None + + assert not (workspace / SEED_RESULT).exists() + + +def test_every_file_is_written_beside_its_place(tmp_path, monkeypatch): + runtime = _real_world(_builder(tmp_path, monkeypatch, marker=False)) + workspace = tmp_path / "authoring" + real = os.replace + moves: list[tuple[Path, Path]] = [] + + def replace(src, dst): + moves.append((Path(src), Path(dst))) + return real(src, dst) + + monkeypatch.setattr(seed_contract.os, "replace", replace) + + assert seed_contract.accept_seed(_seed_path(tmp_path), workspace) is None + assert _open(runtime).contract is not None + + # Moved within one folder, so never across file systems, and never half written. + assert {dst.name for _, dst in moves} >= { + SEED_ACCEPTED, + "contract.json", + seed_contract.SEED_INSTALLED.name, + SEED_RESULT.name, + } + assert all(src.parent == dst.parent for src, dst in moves) + + +def test_a_source_model_already_there_survives_a_take_back(tmp_path, monkeypatch): + job = _job() + seed = _place(tmp_path, _seed_bytes(job)) + output = tmp_path / "authoring" + model = output / "generic-harness" / "source-model.json" + model.parent.mkdir(parents=True) + model.write_text('{"kept": true}', encoding="utf-8") + _setup_fails(tmp_path, monkeypatch, None) + + assert use_seed(seed, job, _source(tmp_path), output) == ("setup_failed", []) + + assert model.read_text(encoding="utf-8") == '{"kept": true}' + assert not (output / "contract.json").exists() + + +def test_a_group_holding_a_validation_error_prints_nothing(capsys): + import builtins + + group_type = getattr(builtins, "ExceptionGroup", None) + if group_type is None: + pytest.skip("no exception groups before 3.11") + + seed_contract.print_trace(group_type("several", [_holding_values()])) + + err = capsys.readouterr().err + assert TOKEN not in err + assert "Traceback" not in err + + +def test_a_chain_that_loops_still_ends(): + import threading + + first, second = RuntimeError("first"), RuntimeError("second") + first.__context__, second.__context__ = second, first + answers: list[bool] = [] + walk = threading.Thread( + target=lambda: answers.append(seed_contract.holds_values(first)), daemon=True + ) + walk.start() + walk.join(timeout=5) + + assert answers == [False] + + +def test_a_restart_after_a_lost_outcome_says_it_was_used(tmp_path, monkeypatch): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + _result_write_fails(monkeypatch, times=1) + assert _open(runtime).contract is not None + assert not (tmp_path / "authoring" / SEED_RESULT).exists() + + assert _open(_restarted(runtime)).contract is not None + + assert _outcome(tmp_path) == {"seed": "used", "differs": []} + + +def test_a_contract_that_is_not_the_seed_is_never_claimed(tmp_path, monkeypatch): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + workspace = tmp_path / "authoring" + seed = _seed_path(tmp_path).read_bytes() + # The same length, one byte apart: what a reading of its own saved. + saved = seed.replace(b'"support"', b'"Support"', 1) + assert saved != seed and len(saved) == len(seed) + (workspace / "contract.json").write_bytes(saved) + + _open(runtime) + + assert not (workspace / SEED_RESULT).exists() + assert (workspace / "contract.json").read_bytes() == saved + + +def _unlink_fails_for(monkeypatch, name: str) -> None: + real = Path.unlink + + def unlink(self, missing_ok=False): + if self.name == name: + raise PermissionError(13, "Permission denied") + return real(self, missing_ok=missing_ok) + + monkeypatch.setattr(Path, "unlink", unlink) + + +def test_a_mark_that_cannot_be_removed_still_leaves_the_outcome( + tmp_path, monkeypatch, capsys +): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + _unlink_fails_for(monkeypatch, SEED_ACCEPTED) + + assert _open(runtime).contract is not None + + assert _outcome(tmp_path) == {"seed": "used", "differs": []} + err = capsys.readouterr().err + assert "Traceback" in err + assert "PermissionError" in err + + +def test_a_contract_that_cannot_be_taken_back_is_used(tmp_path, monkeypatch): + runtime = _real_world(_builder(tmp_path, monkeypatch)) + _setup_fails(tmp_path, monkeypatch, runtime) + _unlink_fails_for(monkeypatch, "contract.json") + + assert _open(runtime).contract is not None + + assert _outcome(tmp_path) == {"seed": "used", "differs": []} + installed = (tmp_path / "authoring" / "contract.json").read_bytes() + assert installed == _seed_path(tmp_path).read_bytes() + + +# -- the round trip ---------------------------------------------------------------------------- + + +def test_a_reading_seeds_its_own_build(tmp_path): + job = _job() + source = _source(tmp_path) + reading = _reading(tmp_path, json.dumps(CONTRACT, indent=1).encode()) + + assert document(job, source, reading) == 0 + seed = reading / "contract.json" + + assert use_seed(seed, job, source, tmp_path / "build") == ("used", []) + assert (tmp_path / "build" / "contract.json").read_bytes() == seed.read_bytes() + other = _job(source={**GITHUB, "commit_sha": OTHER_COMMIT}) + assert differs(json.loads(seed.read_bytes()), other) == ["source"] + assert use_seed(seed, other, source, tmp_path / "other") == ("mismatch", ["source"]) + + +# -- the build command and the builder together, and the record's edge cases -- + + +def _saved_like_a_reading(tmp_path: Path, body: dict, job: HarnessJob) -> bytes: + # As the reading's save gate writes it, then the reading's record. + reading = tmp_path / "reading" + reading.mkdir() + dumped = AgentContract.model_validate(body).model_dump() + saved = json.dumps(dumped, indent=2, ensure_ascii=False).encode("utf-8") + recorded = with_record(job, saved) + assert recorded is not None + (reading / "contract.json").write_bytes(recorded) + return recorded + + +def test_the_build_command_then_the_builder(tmp_path, monkeypatch): + calls = _watch_authoring(monkeypatch) + job = _job() + raw = _saved_like_a_reading(tmp_path, CONTRACT, job) + _place(tmp_path, raw) + assert _build(tmp_path, *_seeded(tmp_path)) == 0 + assert calls == ["compose", "await"] + workspace = tmp_path / "authoring" + runtime = _runtime(workspace) + runtime.job = job.model_dump(mode="json") + runtime.source = _source(tmp_path) + runtime._what_exists = HostedChatRuntime._what_exists.__get__(runtime) + + def opening(contract): + raise _Opened(contract) + + runtime._open_for_building = opening + opened = _open(runtime) + assert opened.contract is not None + assert (workspace / "contract.json").read_bytes() == raw + assert load(workspace).base_environment == CONTRACT["base_environment"] + assert (workspace / "generic-harness" / "source-model.json").is_file() + assert _outcome(tmp_path) == {"seed": "used", "differs": []} + assert not _marker(tmp_path).exists() + + +def test_a_reading_of_the_repository_root_matches(tmp_path): + job = _job(source={k: v for k, v in GITHUB.items() if k != "subfolder"}) + body = {**CONTRACT, "runtime": {"language": "python", "workdir": "."}} + raw = _saved_like_a_reading(tmp_path, body, job) + assert differs(json.loads(raw), job) == [] + source = tmp_path / "source" + source.mkdir() + (source / "agent.py").write_text("x = 1\n") + assert use_seed( + tmp_path / "reading" / "contract.json", job, source, tmp_path / "out" + ) == ("used", []) + + +def test_a_reading_recorded_twice_keeps_one_record(tmp_path): + job = _job() + raw = _saved_like_a_reading(tmp_path, CONTRACT, job) + assert with_record(job, raw) is None + assert raw.count(b'"reading_inputs"') == 1 + + +@pytest.mark.parametrize("version", [2, True, "1", None]) +def test_a_record_of_another_version_reads_again(tmp_path, version): + job = _job() + record = {**record_for(job), "version": version} + seed = _place(tmp_path, _seed_bytes(job, record=record)) + assert differs(json.loads(seed.read_bytes()), job) == ["version"] + assert use_seed(seed, job, _source(tmp_path), tmp_path / "out") == ( + "mismatch", + ["version"], + ) + + +def test_an_open_reading_is_never_installed_under(tmp_path, monkeypatch): + from fi.alk.harness.hosted_chat_entrypoint import BuilderConversation + + runtime = _builder(tmp_path, monkeypatch, marker=False) + runtime._what_exists = HostedChatRuntime._what_exists.__get__(runtime) + conversation = BuilderConversation(runtime._open_builder) + said = [] + + class Reading: + async def say(self, message, on_event=None): + said.append(message) + return "ok" + + # A person's message opened the reading; the build command's mark comes after it. + conversation.stage = Reading() + (tmp_path / "authoring" / SEED_ACCEPTED).write_bytes(b"") + asyncio.run(conversation.say("Build it.")) + assert said == ["Build it."] + assert not (tmp_path / "authoring" / "contract.json").exists() + + +class _AtTheTurn(BaseException): + """Raised where the turn's message is put together, past every checkpoint.""" + + +def test_a_command_installs_nothing_up_to_its_turn(tmp_path, monkeypatch): + runtime = _builder(tmp_path, monkeypatch) + runtime.conversation.stage = None + runtime.conversation.say = lambda *a, **k: None + runtime.backend_name = "claude" + runtime._turn_acknowledged_through = 0 + runtime._tool_names = {} + runtime._journal_unsuccessful = lambda *a: None + + async def checkpoint(): + return {} + + async def pump(*a, **k): + return None + + def stop(): + raise _AtTheTurn + + runtime._checkpoint = checkpoint + runtime._pump_events = pump + runtime._ledger_note = stop + with pytest.raises(_AtTheTurn): + asyncio.run(runtime._handle_command(_command())) + assert not (tmp_path / "authoring" / "contract.json").exists() + + +def test_the_builder_hands_over_the_agents_source(tmp_path, monkeypatch): + runtime = _builder(tmp_path, monkeypatch) + runtime._what_exists = HostedChatRuntime._what_exists.__get__(runtime) + handed = [] + + def use(seed, job, source, output): + handed.append((source, output)) + return "mismatch", [] + + monkeypatch.setattr(seed_contract, "use_seed", use) + _open(runtime) + assert handed == [(runtime.source, runtime.workspace)] + + +def test_a_seed_that_grows_while_it_is_read_is_refused(tmp_path, monkeypatch): + seed = _place(tmp_path, b"{" + b" " * SEED_LIMIT_BYTES + b"}") + real = os.fstat + + def sized_small(descriptor): + found = real(descriptor) + return os.stat_result( + (found.st_mode, *list(found)[1:6], 10, *list(found)[7:10]) + ) + + monkeypatch.setattr(seed_contract.os, "fstat", sized_small) + _, problem, _ = seed_contract.read_seed(seed) + assert problem is not None and problem.code == "seed_contract_too_large" + + +def test_a_contract_the_reading_would_not_save_reads_again(tmp_path, capsys): + job = _job() + seed = _place(tmp_path, _seed_bytes(job, {**CONTRACT, "agent": ""})) + assert use_seed(seed, job, _source(tmp_path), tmp_path / "out") == ("rejected", []) + assert not (tmp_path / "out" / "contract.json").exists() + assert "problems: 1" in capsys.readouterr().err + + +def test_one_install_at_a_time(tmp_path, monkeypatch): + import threading + import time + + runtime = _builder(tmp_path, monkeypatch) + runtime._what_exists = HostedChatRuntime._what_exists.__get__(runtime) + running, most = [0], [0] + gate = threading.Lock() + + def slow(*_args): + with gate: + running[0] += 1 + most[0] = max(most[0], running[0]) + time.sleep(0.2) + with gate: + running[0] -= 1 + return "mismatch", [] + + monkeypatch.setattr(seed_contract, "use_seed", slow) + # Two processes' worth of memory, so only the lock and the file keep them apart. + openings = [runtime, _restarted(runtime)] + threads = [threading.Thread(target=one._take_seed) for one in openings] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert most[0] == 1 + + +def test_a_broken_stderr_does_not_undo_a_used_seed(tmp_path, monkeypatch): + class Closed: + def write(self, _text): + raise OSError("closed") + + def flush(self): + raise OSError("closed") + + job = _job() + seed = _place(tmp_path, _seed_bytes(job)) + monkeypatch.setattr(seed_contract.sys, "stderr", Closed()) + assert use_seed(seed, job, _source(tmp_path), tmp_path / "out") == ("used", []) + assert (tmp_path / "out" / "contract.json").read_bytes() == seed.read_bytes() + + # A traceback that can't be printed never turns a failure into a crash. + def fails(destination, source, job): + raise RuntimeError("boom-7") + + monkeypatch.setattr(cli, "after_contract", fails) + again = tmp_path / "again" + assert use_seed(seed, job, _source(tmp_path), again) == ("setup_failed", []) + assert not (again / "contract.json").exists() diff --git a/tests/harness/test_seed_rows.py b/tests/harness/test_seed_rows.py index a8ba8331d..ed208c9fe 100644 --- a/tests/harness/test_seed_rows.py +++ b/tests/harness/test_seed_rows.py @@ -168,12 +168,8 @@ def test_a_big_integer_key_the_world_holds_is_not_reported_missing(tmp_path): assert only_seed_rows(world, source, "drivers") -@pytest.mark.parametrize( - ("key_type", "seeded"), [("TEXT", "'UB7F3A2C91'"), ("BIGINT", str(_BIG))] -) -def test_a_seeded_in_flight_row_is_not_refused_as_left_over_from_building( - tmp_path, key_type, seeded -): +def _rides_saver(tmp_path, key_type, seeded, runtime=None): + """A world holding one booking, and a ``saved(source_root)`` that runs its save_world.""" from fi.alk.harness.catalogue import Catalogue, SubGoal, save_catalogue from fi.alk.harness.contract import AgentContract, ToolSpec from fi.alk.harness.simulator import save_simulator_prompt @@ -223,6 +219,7 @@ class Rides(GeneratedWorld): conversational=True, real_use_cases=["book a ride"], tools=[ToolSpec(name="book", args=["ref"], arg_values={"ref": ["UB_NEW"]})], + runtime=runtime, ) save_catalogue( Catalogue( @@ -248,6 +245,17 @@ def saved(source_root: str) -> dict: handler = next(one.handler for one in server.tools if one.name == "save_world") return asyncio.run(handler({})) + return saved + + +@pytest.mark.parametrize( + ("key_type", "seeded"), [("TEXT", "'UB7F3A2C91'"), ("BIGINT", str(_BIG))] +) +def test_a_seeded_in_flight_row_is_not_refused_as_left_over_from_building( + tmp_path, key_type, seeded +): + saved = _rides_saver(tmp_path, key_type, seeded) + refused = saved("") assert refused.get("is_error") and "left over from building" in str(refused) @@ -260,3 +268,61 @@ def saved(source_root: str) -> dict: kept = saved(str(source)) assert "left over from building" not in str(kept), kept assert not kept.get("is_error"), kept + + +def test_seed_rows_ignore_a_siblings_seed(tmp_path): + from fi.alk.harness.contract import Runtime + + source = tmp_path / "source" + for name, ref in (("a", "'UB_OWN'"), ("b", "'UB7F3A2C91'")): + (source / "agents" / name / "db").mkdir(parents=True) + (source / "agents" / name / "db" / "seed.sql").write_text( + f"INSERT INTO bookings (booking_ref, status) VALUES ({ref}, 'matched');\n", + encoding="utf-8", + ) + world = sqlite3.connect(":memory:") + world.executescript( + "CREATE TABLE bookings (booking_ref TEXT PRIMARY KEY, status TEXT);" + "INSERT INTO bookings VALUES ('UB7F3A2C91', 'matched');" + ) + + assert [name for name, _ in seed_files(source, "agents/a")] == ["agents/a/db/seed.sql"] + assert [name for name, _ in seed_files(source)] == [ + "agents/a/db/seed.sql", + "agents/b/db/seed.sql", + ] + assert set(missing_seed_rows(world, source, "agents/a")) == {"bookings"} + assert missing_seed_rows(world, source, "agents/b") == {} + assert not only_seed_rows(world, source, "bookings", "agents/a") + assert only_seed_rows(world, source, "bookings", "agents/b") + assert set(seed_rows(source, "agents/a")) == {"bookings"} + + folder = Runtime(source_subfolder="agents/a", workdir="agents/a") + refused = _rides_saver(tmp_path / "in", "TEXT", "'UB7F3A2C91'", folder)(str(source)) + assert refused.get("is_error") and "left over from building" in str(refused) + whole = _rides_saver(tmp_path / "out", "TEXT", "'UB7F3A2C91'")(str(source)) + assert "left over from building" not in str(whole), whole + + +def test_store_hints_leave_out_a_siblings_stores(tmp_path): + from fi.alk.harness.contract import AgentContract, Runtime + from fi.alk.harness.world.tools import world_tools + + source = tmp_path / "source" + for name in ("a", "b"): + (source / "agents" / name).mkdir(parents=True) + (source / "agents" / name / f"{name}_orders.db").write_text("x", encoding="utf-8") + (source / "shared.sqlite").write_text("x", encoding="utf-8") + + def said(runtime): + contract = AgentContract(agent="demo", runtime=runtime) + server, _world = world_tools(contract, tmp_path / "ws", source_root=str(source)) + handler = next(one.handler for one in server.tools if one.name == "adopt_store") + return asyncio.run(handler({"path": "missing.db"}))["content"][0]["text"] + + own = said(Runtime(source_subfolder="agents/a", workdir="agents/a")) + whole = said(None) + + assert "agents/a/a_orders.db" in own and "shared.sqlite" in own + assert "b_orders.db" not in own + assert "b_orders.db" in whole diff --git a/tests/harness/test_source_data_invariants_folder.py b/tests/harness/test_source_data_invariants_folder.py new file mode 100644 index 000000000..bdbb3f7c2 --- /dev/null +++ b/tests/harness/test_source_data_invariants_folder.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import asyncio +import json +import os +from pathlib import Path + +import pytest + +from fi.alk.harness import source_data_invariants as invariants + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + root = tmp_path / "repo" + (root / "agents" / "a" / "tools").mkdir(parents=True) + (root / "agents" / "b").mkdir(parents=True) + (root / "db.py").write_text("x = 1\n") + (root / "agents" / "shared.py").write_text("x = 2\n") + (root / "agents" / "a" / "tools" / "orders.py").write_text("x = 3\n") + (root / "agents" / "a" / "agent.py").write_text("x = 4\n") + (root / "agents" / "b" / "agent.py").write_text("x = 5\n") + return root + + +def _authoring(tmp_path: Path, folder: str | None) -> Path: + authoring = tmp_path / "authoring" + authoring.mkdir() + runtime = {"source_subfolder": folder} if folder else {} + (authoring / "contract.json").write_text(json.dumps({"runtime": runtime})) + return authoring + + +class _Stop(Exception): + pass + + +def _watch(monkeypatch) -> list[str]: + asked: list[str] = [] + + def spy(root, folder=""): + asked.append(folder) + raise _Stop + + monkeypatch.setattr(invariants, "source_files", spy) + return asked + + +def test_invariant_review_lists_only_folder_and_above(tmp_path, repo, monkeypatch): + listed = invariants.source_files(repo, "agents/a") + assert set(listed) == { + "db.py", + "agents/shared.py", + "agents/a/agent.py", + "agents/a/tools/orders.py", + } + + # Both reviews read the folder the contract recorded. + asked = _watch(monkeypatch) + authoring = _authoring(tmp_path, "agents/a") + with pytest.raises(_Stop): + asyncio.run(invariants.author_invariants(repo, authoring, None)) + with pytest.raises(_Stop): + asyncio.run( + invariants.review_while_writing( + repo, authoring, None, endpoints=None, world_key="w", done=None, expected=1 + ) + ) + assert asked == ["agents/a", "agents/a"] + + +def test_no_subfolder_lists_whole_repo(tmp_path, repo, monkeypatch): + assert set(invariants.source_files(repo)) == { + "db.py", + "agents/shared.py", + "agents/a/agent.py", + "agents/a/tools/orders.py", + "agents/b/agent.py", + } + + asked = _watch(monkeypatch) + with pytest.raises(_Stop): + asyncio.run(invariants.author_invariants(repo, _authoring(tmp_path, None), None)) + assert asked == [""] + + +def test_a_linked_folder_is_listed_by_what_it_points_at(repo): + os.symlink(repo / "agents" / "a", repo / "agents" / "alias") + + assert set(invariants.source_files(repo, "agents/alias")) == set( + invariants.source_files(repo, "agents/a") + ) + + +def test_a_folder_that_cannot_be_resolved_is_taken_as_written(repo): + assert set(invariants.source_files(repo, "agents/gone")) == {"db.py", "agents/shared.py"} diff --git a/tests/harness/test_source_directory_names.py b/tests/harness/test_source_directory_names.py index 13806d5cb..26c87ab94 100644 --- a/tests/harness/test_source_directory_names.py +++ b/tests/harness/test_source_directory_names.py @@ -173,3 +173,48 @@ def test_owned_output_cannot_equal_source_and_external_exclusion_preserves_sourc with pytest.raises(ProvisionError, match="source_fingerprint_exclusion_is_source"): source_fingerprint(source, exclude=source) assert source_fingerprint(source, exclude=tmp_path) == source_fingerprint(source) + + +def _two_agents(tmp_path): + for name in ("a", "b"): + (tmp_path / "agents" / name).mkdir(parents=True) + (tmp_path / "agents" / name / "agent.py").write_text("x = 1\n") + (tmp_path / "pyproject.toml").write_text("[project]\nname='top'\nversion='1'\n") + return tmp_path + + +def test_briefing_lists_only_subfolder(tmp_path): + root = _two_agents(tmp_path) + + briefing = RepoSource("app", root, subfolder="agents/a").briefing() + + assert "- agents/a/agent.py" in briefing + assert "agents/b/agent.py" not in briefing + assert "pyproject.toml" not in briefing + assert "The agent is the folder agents/a of this repository" in briefing + assert "Files above it, such as a project file, may apply to it." in briefing + + +def test_briefing_says_command_is_relative_to_workdir(tmp_path): + briefing = RepoSource("app", _two_agents(tmp_path), subfolder="agents/a").briefing() + + assert ( + "runtime.command and workspaces are relative to runtime.workdir: the agent starts there." + in briefing + ) + assert "written from the repository root" in briefing + + +def test_briefing_on_missing_folder_says_so(tmp_path): + briefing = RepoSource("app", _two_agents(tmp_path), subfolder="agents/zzz").briefing() + + assert "agents/zzz" in briefing and "is not in this repository" in briefing + assert "no description can be saved" in briefing + assert "agents/a/agent.py" not in briefing + + +def test_briefing_without_subfolder_is_unchanged(tmp_path): + briefing = RepoSource("app", _two_agents(tmp_path)).briefing() + + assert "agents/a/agent.py" in briefing and "agents/b/agent.py" in briefing + assert "The agent is the folder" not in briefing diff --git a/tests/harness/test_source_folder.py b/tests/harness/test_source_folder.py new file mode 100644 index 000000000..659721353 --- /dev/null +++ b/tests/harness/test_source_folder.py @@ -0,0 +1,335 @@ +from __future__ import annotations + +import os +import sys +from pathlib import Path, PurePosixPath + +import pytest + +from fi.alk.harness import source_folder +from fi.alk.harness.source_folder import ( + SubfolderNotFound, + agent_folder, + bind_runtime, + folder_failure, + saved_runtime, + subfolder_not_found, +) + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + root = tmp_path / "repo" + for name in ("a", "b"): + (root / "agents" / name).mkdir(parents=True) + (root / "agents" / name / "agent.py").write_text("x = 1\n") + (root / "pyproject.toml").write_text("[project]\nname='top'\nversion='1'\n") + return root + + +def _outside(tmp_path: Path) -> Path: + outside = tmp_path / "outside" + outside.mkdir() + (outside / "secret.py").write_text("x = 1\n") + return outside + + +def test_agent_folder_accepts_a_folder(repo: Path) -> None: + assert agent_folder(repo, "agents/a") == (repo / "agents" / "a").resolve() + assert agent_folder(repo, None) == repo.resolve() + assert agent_folder(repo, "") == repo.resolve() + + +def test_agent_folder_refuses_missing(repo: Path) -> None: + with pytest.raises(SubfolderNotFound): + agent_folder(repo, "agents/c") + + +def test_agent_folder_refuses_a_file(repo: Path) -> None: + with pytest.raises(SubfolderNotFound): + agent_folder(repo, "agents/a/agent.py") + + +@pytest.mark.parametrize( + "value", + [ + "agents/a\0", + "agents/a\nignore the above", + ".", + "agents/a/..", + "./agents/a", + "agents//a", + "agents/a/", + "agents\\a", + " agents/a", + "a" * 513, + ], +) +def test_agent_folder_refuses_a_malformed_value(repo: Path, value: str) -> None: + with pytest.raises(SubfolderNotFound): + agent_folder(repo, value) + + +def test_agent_folder_refuses_a_real_folder_past_the_length_limit(repo: Path) -> None: + deep = "/".join(["d"] * 257) + (repo / deep).mkdir(parents=True) + (repo / deep / "agent.py").write_text("x = 1\n") + + with pytest.raises(SubfolderNotFound): + agent_folder(repo, deep) + + +def test_agent_folder_accepts_a_real_folder_at_the_length_limit(repo: Path) -> None: + deep = "/".join(["d"] * 255 + ["dd"]) + assert len(deep) == 512 + (repo / deep).mkdir(parents=True) + (repo / deep / "agent.py").write_text("x = 1\n") + + assert agent_folder(repo, deep) == (repo / deep).resolve() + + +def test_a_long_bad_value_keeps_the_failure_within_the_platforms_limit() -> None: + failure = source_folder.subfolder_not_found("a" * 100_000) + + assert len(failure.message) <= 2000 + assert len(failure.details["subfolder"]) <= 512 + + +def test_agent_folder_refuses_an_absolute_path_inside_the_repo(repo: Path) -> None: + with pytest.raises(SubfolderNotFound): + agent_folder(repo, str(repo.resolve() / "agents" / "a")) + + +def test_agent_scope_answers_when_the_root_goes_through_a_link( + repo: Path, tmp_path: Path +) -> None: + link = tmp_path / "linked" + link.symlink_to(repo, target_is_directory=True) + + assert source_folder.agent_scope(link, "agents/a") == PurePosixPath("agents/a") + + +@pytest.mark.parametrize( + ("relative", "inside"), + [ + ("agents/a/agent.py", True), + ("agents/a/deep/x.py", True), + ("agents/x.py", True), + ("x.py", True), + ("agents/b/agent.py", False), + ("common/llm.py", False), + ], +) +def test_in_agent_scope(relative: str, inside: bool) -> None: + assert ( + source_folder.in_agent_scope(PurePosixPath(relative), PurePosixPath("agents/a")) + is inside + ) + + +def test_agent_folder_refuses_empty(repo: Path) -> None: + (repo / "agents" / "empty").mkdir() + with pytest.raises(SubfolderNotFound): + agent_folder(repo, "agents/empty") + + +def test_agent_folder_refuses_parent_escape(repo: Path, tmp_path: Path) -> None: + sibling = tmp_path / "x" + sibling.mkdir() + (sibling / "agent.py").write_text("x = 1\n") + with pytest.raises(SubfolderNotFound): + agent_folder(repo, "../x") + + +def test_agent_folder_refuses_absolute_outside(repo: Path, tmp_path: Path) -> None: + with pytest.raises(SubfolderNotFound): + agent_folder(repo, str(_outside(tmp_path))) + + +def test_agent_folder_refuses_link_out(repo: Path, tmp_path: Path) -> None: + os.symlink(_outside(tmp_path), repo / "agents" / "out") + with pytest.raises(SubfolderNotFound): + agent_folder(repo, "agents/out") + + +def test_agent_folder_refuses_link_loop(repo: Path) -> None: + os.symlink("two", repo / "one") + os.symlink("one", repo / "two") + with pytest.raises(SubfolderNotFound): + agent_folder(repo, "one") + + +def test_agent_folder_permission_fault_propagates( + repo: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def refuse(_path): + raise PermissionError(13, "denied") + + monkeypatch.setattr(source_folder.os, "scandir", refuse) + with pytest.raises(PermissionError): + agent_folder(repo, "agents/a") + + +def test_agent_folder_accepts_link_inside(repo: Path) -> None: + os.symlink(repo / "agents" / "a", repo / "alias") + assert agent_folder(repo, "alias") == (repo / "agents" / "a").resolve() + + +def test_agent_folder_accepts_dotted_name(repo: Path) -> None: + (repo / "my..agent").mkdir() + (repo / "my..agent" / "agent.py").write_text("x = 1\n") + assert agent_folder(repo, "my..agent") == (repo / "my..agent").resolve() + + +def test_failure_shape_is_exact() -> None: + failure = subfolder_not_found("agents/x") + body = failure.model_dump(mode="json") + assert "agents/x" in body.pop("message") + assert body == { + "domain": "agent", + "stage": "acquiring_source", + "code": "subfolder_not_found", + "retryable": False, + "details": {"subfolder": "agents/x"}, + } + + +def test_folder_failure_names_a_different_build_folder(repo: Path) -> None: + failure = folder_failure(repo, "agents/b", {"source_subfolder": "agents/a"}) + assert failure is not None + assert failure.code == "subfolder_not_found" + assert failure.retryable is False + assert "agents/a" in failure.message and "agents/b" in failure.message + assert failure.details == {"subfolder": "agents/b", "built_from": "agents/a"} + root_build = folder_failure(repo, "agents/b", {}) + assert root_build is not None and "repository root" in root_build.message + + +def test_saved_runtime_tells_no_contract_from_no_runtime(tmp_path: Path) -> None: + assert saved_runtime(tmp_path) is None + (tmp_path / "contract.json").write_text("not json") + assert saved_runtime(tmp_path) is None + for body in ('{"tools": []}', '{"runtime": null}', '{"runtime": []}'): + (tmp_path / "contract.json").write_text(body) + assert saved_runtime(tmp_path) == {} + (tmp_path / "contract.json").write_text('{"runtime": {"source_subfolder": "agents/a"}}') + assert saved_runtime(tmp_path) == {"source_subfolder": "agents/a"} + + +def test_folder_failure_is_none_when_all_is_well(repo: Path) -> None: + assert folder_failure(repo, "agents/a") is None + assert folder_failure(repo, None, {}) is None + assert folder_failure(repo, "agents/a", {"source_subfolder": "agents/a"}) is None + assert folder_failure(repo, "agents/zzz") is not None + + +def test_bind_runtime_defaults_workdir(repo: Path) -> None: + for workdir in ({}, {"workdir": ""}, {"workdir": "."}): + bound = bind_runtime(workdir, repo, "agents/a") + assert bound["workdir"] == "agents/a" + assert bound["source_subfolder"] == "agents/a" + assert bind_runtime(None, repo, "agents/a") == { + "source_subfolder": "agents/a", + "workdir": "agents/a", + } + + +def test_bind_runtime_refuses_outside(repo: Path) -> None: + for workdir in ("agents/b", ".."): + with pytest.raises(ValueError, match="runtime_workdir_outside_subfolder"): + bind_runtime({"workdir": workdir}, repo, "agents/a") + + +def test_bind_runtime_keeps_a_folder_inside(repo: Path) -> None: + (repo / "agents" / "a" / "app").mkdir() + bound = bind_runtime({"workdir": "agents/a/app"}, repo, "agents/a") + assert bound["workdir"] == "agents/a/app" + + +def test_bind_runtime_replaces_a_written_value(repo: Path) -> None: + bound = bind_runtime( + {"source_subfolder": "agents/b", "workdir": "", "service": "web"}, + repo, + "agents/a", + ) + assert bound["source_subfolder"] == "agents/a" + assert bound["service"] == "web" + + +def test_bind_runtime_is_idempotent(repo: Path) -> None: + once = bind_runtime({"command": ["python", "agent.py"]}, repo, "agents/a") + assert bind_runtime(once, repo, "agents/a") == once + + +def test_bind_runtime_no_subfolder_returns_input(repo: Path) -> None: + runtime = {"workdir": "agents/b"} + assert bind_runtime(runtime, repo, None) is runtime + assert bind_runtime(runtime, repo, "") is runtime + assert bind_runtime(None, repo, None) is None + assert bind_runtime(None, repo, "") is None + + +def test_bind_runtime_no_subfolder_strips_a_written_key(repo: Path) -> None: + for subfolder in (None, ""): + stripped = bind_runtime( + {"source_subfolder": "agents/b", "workdir": "x"}, repo, subfolder + ) + assert stripped == {"workdir": "x"} + + +def test_bind_runtime_refuses_climbing_command(repo: Path) -> None: + (repo / "shared.py").write_text("x = 1\n") + with pytest.raises(ValueError, match="runtime_command_outside_subfolder"): + bind_runtime( + {"command": ["python", "../../shared.py"]}, repo, "agents/a" + ) + ok = bind_runtime({"command": ["python", "agent.py", "--port", "8000"]}, repo, "agents/a") + assert ok["command"] == ["python", "agent.py", "--port", "8000"] + + +@pytest.mark.parametrize( + "command", + [ + ["uv", "run", "--directory=../b", "python", "agent.py"], + ["bun", "--cwd=../b", "index.ts"], + ["env", "PYTHONPATH=../b", "python", "agent.py"], + ], +) +def test_bind_runtime_refuses_a_climb_written_after_an_equals_sign( + repo: Path, command: list[str] +) -> None: + with pytest.raises(ValueError, match="runtime_command_outside_subfolder"): + bind_runtime({"command": command}, repo, "agents/a") + + +def test_bind_runtime_allows_an_equals_sign_that_stays_inside(repo: Path) -> None: + command = ["uv", "run", "--directory=.", "python", "agent.py", "--mode=fast"] + assert bind_runtime({"command": command}, repo, "agents/a")["command"] == command + + +def test_bind_runtime_command_absolute_paths(repo: Path) -> None: + program = ["python", str(repo / "agents" / "a" / "agent.py")] + assert bind_runtime({"command": program}, repo, "agents/a")["command"] == program + assert bind_runtime({"command": [sys.executable, "agent.py"]}, repo, "agents/a") + with pytest.raises(ValueError, match="runtime_command_outside_subfolder"): + bind_runtime( + {"command": ["python", str(repo / "agents" / "b" / "agent.py")]}, + repo, + "agents/a", + ) + + +def test_bind_runtime_refuses_sibling_compose_file(repo: Path) -> None: + (repo / "agents" / "b" / "compose.yaml").write_text("services: {}\n") + with pytest.raises(ValueError, match="runtime_compose_file_outside_subfolder"): + bind_runtime( + {"compose_file": "agents/b/compose.yaml"}, repo, "agents/a" + ) + + +def test_bind_runtime_allows_compose_file_above(repo: Path) -> None: + (repo / "compose.yaml").write_text("services: {}\n") + (repo / "agents" / "a" / "compose.yaml").write_text("services: {}\n") + for name in ("compose.yaml", "agents/a/compose.yaml"): + bound = bind_runtime({"compose_file": name}, repo, "agents/a") + assert bound["compose_file"] == name diff --git a/tests/harness/test_source_tool_evidence.py b/tests/harness/test_source_tool_evidence.py index 34b462d61..f896bc7d2 100644 --- a/tests/harness/test_source_tool_evidence.py +++ b/tests/harness/test_source_tool_evidence.py @@ -4,6 +4,7 @@ import pytest from fi.alk.harness.contract import AgentContract, validate_contract +from fi.alk.harness.source_tool_evidence import tool_evidence_problems from fi.alk.harness.tools import accept_contract @@ -778,3 +779,242 @@ def test_a_contract_without_a_prompt_source_keeps_its_quoted_excerpt(): contract = AgentContract.model_validate(payload() | {"system_prompt_excerpt": "Be brief."}) assert contract.system_prompt_from == [] assert "Be brief." in contract.brief() + + +def _entry_contract() -> AgentContract: + return AgentContract.model_validate( + { + "agent": "a", + "tool_entrypoints": [ + {"tool": "lookup", "mode": "import", "module": "agent", "callable": "lookup"} + ], + } + ) + + +_REAL = "def lookup():\n return 1\n" +_COMMENTED = "# def lookup():\n# return 1\n" + + +def test_tool_check_reads_the_agents_folder_first(tmp_path): + (tmp_path / "agents" / "a").mkdir(parents=True) + (tmp_path / "agent.py").write_text(_REAL) + (tmp_path / "agents" / "a" / "agent.py").write_text(_COMMENTED) + + assert tool_evidence_problems(_entry_contract(), tmp_path, "agents/a") + assert not tool_evidence_problems(_entry_contract(), tmp_path) + + +def test_tool_check_prefers_the_folders_copy_when_only_the_root_is_broken(tmp_path): + (tmp_path / "agents" / "a").mkdir(parents=True) + (tmp_path / "agent.py").write_text(_COMMENTED) + (tmp_path / "agents" / "a" / "agent.py").write_text(_REAL) + + assert not tool_evidence_problems(_entry_contract(), tmp_path, "agents/a") + + +def test_tool_check_falls_back_to_the_root_under_a_folder(tmp_path): + (tmp_path / "agents" / "a").mkdir(parents=True) + (tmp_path / "agents" / "a" / "other.py").write_text("x = 1\n") + (tmp_path / "agent.py").write_text(_COMMENTED) + + assert tool_evidence_problems(_entry_contract(), tmp_path, "agents/a") + + +def test_tool_check_with_a_bad_folder_reads_the_root(tmp_path): + (tmp_path / "agent.py").write_text(_REAL) + + assert not tool_evidence_problems(_entry_contract(), tmp_path, "agents/missing") + + +def test_tool_check_decodes_a_unicode_escape_module_as_python_does(tmp_path): + # \x23 is "#": Python reads both lines as comments, so no lookup is defined. + (tmp_path / "agent.py").write_bytes( + b"# coding: unicode_escape\n\\x23 def lookup():\n\\x23 return 1\n" + ) + + assert tool_evidence_problems(_entry_contract(), tmp_path) + + +_COMMENTED_METHOD = ( + "class Assistant:\n def __init__(self): pass\n" + " # async def lookup_weather(self, location): pass\n" +) + + +def _commented_source(tmp_path, text): + source = tmp_path / "source" + source.mkdir() + (source / "agent.py").write_text(text) + return source + + +def _named_twice(): + entry = payload()["tool_entrypoints"][0] + return AgentContract.model_validate( + payload() + | { + "tools": [{"name": "lookup_weather", "args": []}, {"name": "lookup_time", "args": []}], + "tool_entrypoints": [entry, entry | {"tool": "lookup_time"}], + } + ) + + +def test_a_module_many_entries_name_is_read_once(tmp_path, monkeypatch): + from fi.alk.harness import source_tool_evidence + + source = _commented_source(tmp_path, _COMMENTED_METHOD) + reads = [] + real = source_tool_evidence.read_module + monkeypatch.setattr( + source_tool_evidence, "read_module", lambda path, **kw: reads.append(path) or real(path, **kw) + ) + + assert len(tool_evidence_problems(_named_twice(), source)) == 2 + assert len(reads) == 1 + + +@pytest.mark.parametrize("over, problems", [(0, 1), (1, 0)]) +def test_a_named_module_is_read_up_to_the_limit_and_skipped_past_it( + tmp_path, monkeypatch, over, problems +): + from fi.alk.harness import python_source + from fi.alk.harness.python_source import LARGEST_PARSED_MODULE + + monkeypatch.setattr(python_source, "LARGEST_TOKENIZED_TOTAL", LARGEST_PARSED_MODULE) + text = _COMMENTED_METHOD + "#" * (LARGEST_PARSED_MODULE + over - len(_COMMENTED_METHOD)) + source = _commented_source(tmp_path, text) + + assert len(tool_evidence_problems(AgentContract.model_validate(payload()), source)) == problems + + +@pytest.mark.parametrize("total, problems", [(len(_COMMENTED_METHOD), 1), (len(_COMMENTED_METHOD) - 1, 0)]) +def test_the_named_modules_are_parsed_only_up_to_a_total(tmp_path, monkeypatch, total, problems): + from fi.alk.harness import python_source + + source = _commented_source(tmp_path, _COMMENTED_METHOD) + monkeypatch.setattr(python_source, "LARGEST_PARSED_TOTAL", total) + + assert len(tool_evidence_problems(AgentContract.model_validate(payload()), source)) == problems + + +@pytest.mark.parametrize( + "total, problems", [(len(_COMMENTED_METHOD), 1), (len(_COMMENTED_METHOD) - 1, 0)] +) +def test_the_named_modules_are_tokenized_only_up_to_a_total(tmp_path, monkeypatch, total, problems): + from fi.alk.harness import python_source + + source = _commented_source(tmp_path, _COMMENTED_METHOD) + monkeypatch.setattr(python_source, "LARGEST_TOKENIZED_TOTAL", total) + + assert len(tool_evidence_problems(AgentContract.model_validate(payload()), source)) == problems + + +@pytest.mark.parametrize("short_of, problems", [(0, 1), (1, 0)]) +def test_the_named_modules_are_tokenized_only_up_to_the_work(tmp_path, monkeypatch, short_of, problems): + import io + import tokenize + + from fi.alk.harness import python_source + + # Each token is charged the line it's read from, which 3.12.3 decodes again for each token. + work = sum( + len(token.line) for token in tokenize.generate_tokens(io.StringIO(_COMMENTED_METHOD).readline) + ) + source = _commented_source(tmp_path, _COMMENTED_METHOD) + monkeypatch.setattr(python_source, "LARGEST_TOKENIZED_WORK", work - short_of) + + assert len(tool_evidence_problems(AgentContract.model_validate(payload()), source)) == problems + + +@pytest.mark.parametrize("limit", ["LARGEST_TOKENIZED_WORK", "LARGEST_TOKENIZED_TOTAL"]) +def test_the_tokenize_limits_are_shared_by_the_named_modules(tmp_path, monkeypatch, limit): + import io + import tokenize + + from fi.alk.harness import python_source + + # Two modules, each under the limit alone and not together: one budget for the check. + source = _commented_source(tmp_path, _COMMENTED_METHOD) + other = _COMMENTED_METHOD.replace("lookup_weather", "lookup_time") + (source / "other.py").write_text(other) + entry = payload()["tool_entrypoints"][0] + contract = AgentContract.model_validate( + payload() + | { + "tools": [{"name": "lookup_weather", "args": []}, {"name": "lookup_time", "args": []}], + "tool_entrypoints": [ + entry, + entry | {"tool": "lookup_time", "module": "other", "callable": "Assistant.lookup_time"}, + ], + } + ) + if limit == "LARGEST_TOKENIZED_WORK": + def cost(text): + return sum(len(token.line) for token in tokenize.generate_tokens(io.StringIO(text).readline)) + else: + def cost(text): + return len(text.encode()) + assert len(tool_evidence_problems(contract, source)) == 2 + monkeypatch.setattr(python_source, limit, max(cost(_COMMENTED_METHOD), cost(other)) * 3 // 2) + + assert len(tool_evidence_problems(contract, source)) == 1 + + +@pytest.mark.parametrize("coding", ["rot13", "nosuch"]) +def test_a_module_with_a_coding_line_python_cant_read_is_unproven(tmp_path, coding): + source = _commented_source(tmp_path, f"# coding: {coding}\n" + _COMMENTED_METHOD) + + assert tool_evidence_problems(AgentContract.model_validate(payload()), source) == [] + + +def test_a_module_without_the_line_in_a_comment_is_not_tokenized(tmp_path, monkeypatch): + import tokenize + + source = _commented_source(tmp_path, "class Assistant:\n pass\n") + monkeypatch.setattr( + tokenize, "generate_tokens", lambda *args: pytest.fail("tokenized") + ) + + assert tool_evidence_problems(AgentContract.model_validate(payload()), source) == [] + + +def test_a_named_module_that_runs_the_parser_out_of_memory_is_left_alone(tmp_path): + source = _commented_source(tmp_path, _COMMENTED_METHOD + "x = " + "-" * 7000 + "1\n") + + assert tool_evidence_problems(AgentContract.model_validate(payload()), source) == [] + + +def test_tool_names_are_not_read_from_a_link_or_a_pipe_or_a_file_without_end( + tmp_path, monkeypatch +): + import io + import os + from pathlib import Path + + from fi.alk.harness.source_tool_evidence import defined_tool_names + + source = tmp_path / "source" + source.mkdir() + (source / "tools.py").write_text("def lookup_weather(location): pass\n") + (source / "deep.py").write_text("def nested(): pass\nx = " + "-" * 7000 + "1\n") + (source / "zero.py").write_text("") + os.mkfifo(source / "pipe.py") + + class Endless(io.RawIOBase): + def readable(self): + return True + + def read(self, size=-1): + assert size >= 0, "read to the end" + return b"#" * size + + opened = Path.open + + def open_(path, *args, **kwargs): + assert path.name != "pipe.py", "opened a pipe" + return Endless() if path.name == "zero.py" else opened(path, *args, **kwargs) + + monkeypatch.setattr(Path, "open", open_) + + assert defined_tool_names(source) == {"lookup_weather"} diff --git a/tests/harness/test_storage_decision_admission.py b/tests/harness/test_storage_decision_admission.py index bc8602b7d..aeb5d04b4 100644 --- a/tests/harness/test_storage_decision_admission.py +++ b/tests/harness/test_storage_decision_admission.py @@ -80,6 +80,7 @@ def contract_tools(destination, available_evals, **kwargs): workdir=lambda: tmp_path, servers=lambda: {}, builtin_tools=lambda: (), + subfolder="", ) understand.open_stage(source, out=tmp_path, require_setup_contract=True) assert captured["require_setup_contract"] is required @@ -221,3 +222,39 @@ def test_chat_interface_schema_teaches_the_supported_bridge(tmp_path, required): canonical = RuntimeInterface.model_json_schema()["properties"] assert canonical["kind"]["description"] == fields["kind"]["description"] assert canonical["protocol"]["description"] == fields["protocol"]["description"] + + +def test_understand_passes_subfolder(tmp_path, monkeypatch): + import asyncio + from types import SimpleNamespace + from fi.alk.harness import cli + + captured = {} + + def resolve(*args, **kwargs): + captured.update(kwargs) + return SimpleNamespace(name="fixture", kind="repo") + + async def converse(*args, **kwargs): + pass + + monkeypatch.setattr(cli, "resolve", resolve) + monkeypatch.setattr(cli, "open_stage", lambda *args, **kwargs: (object(), tmp_path)) + monkeypatch.setattr(cli, "_converse", converse) + monkeypatch.setattr(cli, "opening", lambda _: "") + monkeypatch.setattr(cli, "_guidance", lambda _: "") + monkeypatch.setattr(cli, "load", lambda _: None) + args = SimpleNamespace( + kind="repo", + name="fixture", + path=tmp_path, + out=tmp_path, + interactive=False, + job=SimpleNamespace( + metadata={}, source=SimpleNamespace(subfolder="agents/a") + ), + ) + + asyncio.run(cli._understand(args)) + + assert captured["subfolder"] == "agents/a" diff --git a/tests/providers/test_providers_agent_authored.py b/tests/providers/test_providers_agent_authored.py index 677e24b20..9cf867b21 100644 --- a/tests/providers/test_providers_agent_authored.py +++ b/tests/providers/test_providers_agent_authored.py @@ -166,3 +166,107 @@ def test_authored_keeps_the_host_of_a_url_with_a_placeholder_port() -> None: assert stored["braces"] == "http://{host}/v1" assert stored["shell"] == "http://localhost/v1" assert stored["numeric"] == "http://localhost:8080/v1" + + +def test_framework_written_beside_contract() -> None: + contract = _contract() + framework = {"name": "other", "version": "1.2", "label": "dspy"} + + agent = build_authored_agent(contract, framework=framework) + + assert set(agent["extension"]) == {"contract", "framework"} + assert agent["extension"]["framework"] == framework + assert agent["setup_hash"] == compute_setup_hash(agent) + assert agent["setup_hash"] != build_authored_agent(contract)["setup_hash"] + + +def test_no_framework_leaves_extension_and_hash() -> None: + recorded = json.loads((AUTHORED / "chat_source.json").read_text()) + + agent = build_authored_agent(recorded["contract"], framework=None) + + assert agent == recorded["agent"] + assert set(agent["extension"]) == {"contract"} + + +_ABSENT = object() + + +def _chat_contract() -> dict: + # A raw dict, not through AgentContract, so a model of any shape reaches + # build_authored_agent unchanged. + return copy.deepcopy( + json.loads((AUTHORED / "chat_source.json").read_text())["contract"] + ) + + +@pytest.mark.parametrize( + "model", + [_ABSENT, None, "gpt-4o", {"name": " "}], + ids=["absent", "none", "raw-string", "blank-name"], +) +def test_no_model_leaves_the_document_and_hash(model: object) -> None: + recorded = json.loads((AUTHORED / "chat_source.json").read_text()) + contract = _chat_contract() + expected = copy.deepcopy(recorded["agent"]) + if model is not _ABSENT: + contract["model"] = model + # The stored contract echoes the value whatever its shape, so only that + # echo and the hash over it may differ from the recorded document. + expected["extension"]["contract"]["model"] = model + expected["setup_hash"] = compute_setup_hash(expected) + + agent = build_authored_agent(contract) + + assert agent["model"] is None + assert agent == expected + + +def test_model_goes_to_the_document() -> None: + def built(model: dict) -> dict: + return build_authored_agent({**_chat_contract(), "model": model}) + + without = build_authored_agent(_chat_contract()) + saved = {"provider": " openai ", "name": " gpt-4o ", "path": "agent.py", "line": 9} + agent = built(saved) + + assert agent["model"] == { + "provider": "openai", + "name": "gpt-4o", + "temperature": None, + "max_tokens": None, + } + assert agent["extension"]["contract"]["model"] == saved + assert set(agent["extension"]) == {"contract"} + assert agent["setup_hash"] == compute_setup_hash(agent) + assert agent["setup_hash"] != without["setup_hash"] + + for provider in ("", " ", None): + assert built({"provider": provider, "name": "gpt-4o"})["model"] == { + "provider": None, + "name": "gpt-4o", + "temperature": None, + "max_tokens": None, + } + + long = built({"provider": "p" * 300, "name": "m" * 300, "path": "a.py", "line": 1}) + assert long["model"]["provider"] == "p" * 255 + assert long["model"]["name"] == "m" * 255 + + with_url = built( + { + "provider": "openai", + "name": "https://u:p@host.example:8443/v1/m?token=abc", + "path": "agent.py", + "line": 9, + } + ) + assert with_url["model"]["name"] == "https://host.example:8443/v1/m" + assert with_url["extension"]["contract"]["model"] == { + "provider": "openai", + "name": "https://host.example:8443/v1/m", + "path": "agent.py", + "line": 9, + } + assert "token=abc" not in json.dumps(with_url) + assert "u:p@" not in json.dumps(with_url) diff --git a/tests/providers/test_providers_agent_build.py b/tests/providers/test_providers_agent_build.py index dbc5a92b6..ba86233a6 100644 --- a/tests/providers/test_providers_agent_build.py +++ b/tests/providers/test_providers_agent_build.py @@ -1,11 +1,21 @@ from __future__ import annotations +import logging +import random +import re +import string +import subprocess +import sys +import time from typing import Any +from urllib.parse import urlsplit import pytest +from hypothesis import HealthCheck, Phase, given, settings +from hypothesis import strategies as st from fi.alk.harness.provider_import import _safe_profile_value -from fi.alk.providers.agent import _redact +from fi.alk.providers.agent import _redact, build_authored_agent PLAIN_DOCUMENTS: list[Any] = [ {"name": "Support line", "voice": {"voiceId": "marissa", "speed": 1.0}}, @@ -55,3 +65,509 @@ def test_added_markers_headers_and_parameters() -> None: "headers": "[redacted]", "url": "https://example.com/x", } + + +@pytest.mark.parametrize( + ("prose", "expected"), + [ + ( + "Call https://u:hunter2@api.example.com/v1 for data.", + "Call https://api.example.com/v1 for data.", + ), + ( + "Use https://api.example.com/v1?api_key=sk-123#frag always", + "Use https://api.example.com/v1 always", + ), + ("line1\nhttps://u:p@h.example/x", "line1\nhttps://h.example/x"), + ("See (https://u:p@h.example/x) first", "See (https://h.example/x) first"), + ( + "a https://u:p@one.example/1 and postgres://db:pw@two.example:5432/app?ssl=1", + "a https://one.example/1 and postgres://two.example:5432/app", + ), + # The host is lowercased, as it always was for a URL on its own. + ("x\thttp://{HOST}:{PORT}/hook y", "x\thttp://{host}/hook y"), + # A URL runs to the next whitespace, so the query takes the closing quote and comma + # with it: JSON quoted in prose stops being valid JSON (plan v5.1 §V9, design page). + ( + '{"hook": "https://u:p@h.example/x?k=1", "next": 1}', + '{"hook": "https://h.example/x "next": 1}', + ), + ], + ids=["password", "query", "after_newline", "in_brackets", "two_urls", "template", "quoted"], +) +def test_a_url_inside_prose_is_redacted_and_the_prose_kept(prose: str, expected: str) -> None: + assert _redact(prose) == expected + + +def test_a_refused_url_inside_prose_becomes_redacted_and_the_rest_stays( + caplog: pytest.LogCaptureFixture, +) -> None: + prose = "Before https://admin:hunter2#x@db.example/ and https://a:hunter2#y@db.example/ after" + + with caplog.at_level(logging.DEBUG, logger="fi.alk.providers.agent"): + assert _redact(prose) == "Before [redacted] and [redacted] after" + + # One line for the call, however many URLs in it were refused (#271). + [logged] = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] + assert logged.getMessage() == "a URL was redacted whole: class=ValueError" + assert "hunter2" not in caplog.text + + +@pytest.mark.parametrize( + "text", ["no link here", "mailto:someone@example.com", "ratio 3:1 at 10:30", ""] +) +def test_a_string_without_a_url_is_unchanged(text: str) -> None: + assert _redact(text) == text + + +def test_many_refused_urls_in_one_document_log_one_line( + caplog: pytest.LogCaptureFixture, +) -> None: + # A full-width number sign NFKC-normalises to "#", and "[" opens an IPv6 host that never + # closes: urlsplit refuses both. + leaky = "https://admin:hunter2\uff03x@db.example/" + refused = [leaky, *(["a://["] * 1000)] + document = { + "agent": "support", + "modality": "chat", + "system_prompt_excerpt": leaky, + "tools": [{"name": "add", "args": ["k"], "arg_values": {"k": refused}}], + } + + with caplog.at_level(logging.DEBUG, logger="fi.alk.providers.agent"): + agent = build_authored_agent(document) + + assert agent["behaviour"][0]["instructions"] == "[redacted]" + assert agent["tools"][0]["parameters"]["properties"]["k"]["enum"] == ["[redacted]"] * 1001 + [logged] = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] + assert logged.getMessage() == "a URL was redacted whole: class=ValueError" + assert logged.exc_info is None + assert "hunter2" not in caplog.text + + +# -- TH-9491: a URL inside prose (plan v5.1) ------------------------------------------------------- + +# No shrinking: each assertion names its input, and a planted bug then fails in seconds. +_FAST = (Phase.explicit, Phase.generate) +_PROPERTY = settings( + max_examples=400, + derandomize=True, + deadline=None, + phases=_FAST, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.filter_too_much], +) +_UNRESERVED = string.ascii_letters + string.digits + "-._~" +_SUB_DELIMS = "!$&'()*+,;=" +_QUOTES = "'\"<>`" +_NON_ASCII = "üé数据?#" + + +def _frozen_base_url(value: str) -> str: + """The base `_redact` string branch for a value that is one URL, frozen here. + + `_safe_profile_value` can't be the oracle: it raises on a non-numeric port. The copy carries + the plan's named changes from the base (V3, W2): an IPv6 host keeps its brackets, a URL with + a scheme but no host keeps its path and loses its query and fragment, and the result is built + by hand, so a scheme urllib doesn't know keeps its //. + """ + if "://" not in value: + return value + try: + parsed = urlsplit(value) + except ValueError: + return "[redacted]" + if not parsed.scheme: + return value + if not parsed.netloc: + return f"{parsed.scheme}://{parsed.path}" + hostname = parsed.hostname or "" + if ":" in hostname: + hostname = f"[{hostname}]" + try: + port = f":{parsed.port}" if parsed.port else "" + except ValueError: + port = "" + return f"{parsed.scheme}://{hostname}{port}{parsed.path}" + + +def _text(alphabet: str, *, min_size: int = 0, max_size: int = 8) -> st.SearchStrategy[str]: + return st.text(st.sampled_from(alphabet), min_size=min_size, max_size=max_size) + + +@st.composite +def _lone_urls(draw: Any) -> str: + """One URL, RFC 3986 shaped, with tabs and line breaks put somewhere after its ://.""" + scheme = draw(st.sampled_from(["https", "http", "postgres", "redis", "myapp", "git+ssh"])) + wide = _UNRESERVED + _SUB_DELIMS + _QUOTES + "[]%" + _NON_ASCII + userinfo = draw( + st.one_of( + st.just(""), + st.builds( + lambda user, password: f"{user}:{password}@", + _text(_UNRESERVED, min_size=1), + _text(wide, min_size=1), + ), + ) + ) + host = draw(st.one_of(_text(string.ascii_lowercase, min_size=1), st.just("[fd00::5]"))) + port = draw(st.sampled_from(["", ":8443", ":{PORT}", ":0"])) + path = draw(_text(wide + "/:@", max_size=10)) + query = draw(st.one_of(st.just(""), _text(wide + "/?:@=&", max_size=10).map("?".__add__))) + fragment = draw(st.one_of(st.just(""), _text(_UNRESERVED, max_size=4).map("#".__add__))) + url = f"{scheme}://{userinfo}{host}{port}{('/' + path) if path else ''}{query}{fragment}" + head = len(scheme) + 3 + for _ in range(draw(st.integers(0, 3))): + at = draw(st.integers(head, len(url))) + url = url[:at] + draw(st.sampled_from(["\t", "\n", "\r\n"])) + url[at:] + return url + + +@_PROPERTY +@given(_lone_urls()) +def test_a_single_url_field_is_cleaned_as_the_base_cleans_it(url: str) -> None: + """B5: a single-URL field, after W1's join, gives the frozen base's result.""" + from fi.alk.providers.agent import _START + + joined = url.replace("\t", "").replace("\r", "").replace("\n", "") + if len(_START.findall(joined)) != 1: + return + assert _redact(url) == _frozen_base_url(url), url + + +@st.composite +def _secret_urls(draw: Any, non_ascii: bool) -> tuple[str, str, str]: + """A URL with a password and a key, each between markers no honest text holds.""" + extra = _NON_ASCII if non_ascii else "" + password = "PWz" + draw(_text(_UNRESERVED + _SUB_DELIMS + _QUOTES + "%" + extra)) + "zQ" + key = "SKz" + draw(_text(_UNRESERVED + _SUB_DELIMS + _QUOTES + "/?" + extra)) + "zK" + user = draw(_text(_UNRESERVED, min_size=1, max_size=4)) + host = draw(_text(string.ascii_lowercase, min_size=1, max_size=6)) + ".example" + port = draw(st.sampled_from(["", ":8443", ":{PORT}"])) + # The path is kept (D2), so it never holds a marker's letters. + plain = "".join(c for c in _UNRESERVED + _SUB_DELIMS + _QUOTES + ":@" + extra if c not in "PQSK") + path = draw(_text(plain, max_size=6)) + query = draw(st.sampled_from([f"?key={key}", f"?a=1&key={key}", f"?next=/x&key={key}"])) + fragment = draw(st.sampled_from(["", "#Frag"])) + return f"https://{user}:{password}@{host}{port}/{path}{query}{fragment}", password, key + + +_WRAPPERS = [ + "{u}", + "Post to {u} after.", + "({u})", + "'{u}'", + '"{u}"', + "[x]({u}).", + "a\n{u}\nb", + '{{"k": "{u}"}}', + "get('{u}', t=3)", + "请{u}然后", + "go {u}`next` z", + "x {u},postgres://u:PWzpwzQ@b/db y", + # V1: text glued before the scheme, and a URL nested in a path. + "1.{u}", + "-{u}", + "...{u}", + "步骤2{u}", + "#42{u}", + "see https://h.example/a/2{u} ok", +] + + +def _leaked(out: str, password: str, key: str) -> bool: + return any(marker in out for marker in (password[:3], password[-2:], key[:3], key[-2:])) + + +@pytest.mark.parametrize("non_ascii", [False, True], ids=["ascii", "non_ascii"]) +@pytest.mark.parametrize("wrapper", _WRAPPERS) +def test_no_password_or_query_value_survives_in_any_wrapper(wrapper: str, non_ascii: bool) -> None: + """B8, as a property over each wrapper.""" + + @settings(max_examples=150, derandomize=True, deadline=None, phases=_FAST) + @given(_secret_urls(non_ascii)) + def check(drawn: tuple[str, str, str]) -> None: + url, password, key = drawn + out = _redact(wrapper.format(u=url)) + assert not _leaked(out, password, key), (url, out) + + check() + + +@pytest.mark.parametrize("non_ascii", [False, True], ids=["ascii", "non_ascii"]) +def test_no_password_or_query_value_survives_without_its_scheme(non_ascii: bool) -> None: + """B8, V1: a scheme given by a template (`${S}://…`) starts a URL too.""" + + @settings(max_examples=150, derandomize=True, deadline=None, phases=_FAST) + @given(_secret_urls(non_ascii)) + def check(drawn: tuple[str, str, str]) -> None: + url, password, key = drawn + out = _redact("Use ${DB_SCHEME}" + url.removeprefix("https") + " now") + assert not _leaked(out, password, key), (url, out) + + check() + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ( + "see https://h.example/x?next=https://u:pw@h2.example/y&key=sk-1 ok", + "see https://h.example/x ok", + ), + ( + "see https://h.example/x/https://u:pw@h2.example/y?key=sk-1 ok", + "see https://h.example/x/https://h2.example/y ok", + ), + ("postgres://u:pw@a/db,postgres://u:hunter2@b/db", "postgres://a/db,postgres://b/db"), + ( + "https://api.example.com/v1/customers//orders?api_key=sk-1", + "https://api.example.com/v1/customers//orders", + ), + ("https://h.example/x?city=Zürich&api_key=sk-1", "https://h.example/x"), + ("https://h.example/données?api_key=sk-1", "https://h.example/données"), + # W5: a URL with a scheme and no host loses its query, with and without a scheme name. + ("file:///etc/x?token=sk-1", "file:///etc/x"), + ("file:///etc/app.conf?token=sk-1", "file:///etc/app.conf"), + ("${S}:///etc/x?token=sk-1", "${S}:///etc/x"), + ("Use https://?key=sk-1 now", "Use https:// now"), + # W1: one URL broken by a tab or a line break, with a second :// in it. + ("https://h.example/cb\n?redirect=https://app.example/&key=sk-1", "https://h.example/cb"), + ( + "https://bot:hun\tter2@h.example/cb?redirect=https://app.example/", + "https://h.example/cb", + ), + ("postgres://u:hun\nter2@a/db,postgres://b/db", "postgres://a/db,postgres://b/db"), + ("https://bot:hun\tter2@h.example/v1?key=sk-1", "https://h.example/v1"), + ("https://bot:hunter2@h.example/v1\n?key=sk-1", "https://h.example/v1"), + # V1: the round-3 glued prefixes. + ("1.https://bot:hunter2@hooks.example/v1?key=sk-1", "1.https://hooks.example/v1"), + ("-https://bot:hunter2@h.example/v1?key=sk-1", "-https://h.example/v1"), + ("...https://bot:hunter2@h.example/v1?key=sk-1", "...https://h.example/v1"), + ("步骤2https://bot:hunter2@h.example/v1?key=sk-1", "步骤2https://h.example/v1"), + ("#42https://bot:hunter2@h.example/v1?key=sk-1", "#42https://h.example/v1"), + ("${DB_SCHEME}://app:hunter2@db.example/x?key=sk-1", "${DB_SCHEME}://db.example/x"), + ( + "see https://h.example/a/2https://bot:hunter2@h2.example/y?key=sk-1 ok", + "see https://h.example/a/2https://h2.example/y ok", + ), + ("x 1a.https://u:hunter2@h.example/?key=sk-1 y", "x 1a.https://h.example/ y"), + # V3: IPv6 keeps its brackets. + ("postgres://u:hunter2@[fd00::5]:5432/app?key=sk-1", "postgres://[fd00::5]:5432/app"), + # Change review r1, B-M1: a ? or # before the first scheme, then a second URL. The + # ? or # is searched for from the first start, not the token's (plant BC1). + ("ok?https://a.example/x,postgres://u:hunter2@b/db", "ok?https://a.example/x,postgres://b/db"), + ("#1https://a/,postgres://u:hunter2@b/db", "#1https://a/,postgres://b/db"), + # B-L2: a ? right before a scheme letter (plant BT3), and a refused schemeless URL, + # written back whole (plant BS3). + ("https://h.example/x?https://u:pw@a.example/y", "https://h.example/x"), + ("${S}://[", "${S}[redacted]"), + ("${S}://u:hunter2@[x/y", "${S}[redacted]"), + # Change review r2, B-M1: a # owns the rest of its token as a ? does, so a redirect URL + # and the token after it in an OAuth-style fragment go with it (plant N1). + ("https://app.example/cb#next=https://h.example/&access_token=sk-1", "https://app.example/cb"), + ( + "Login https://app.example/cb#state=x&redirect=https://h.example/done&token=sk-1 then", + "Login https://app.example/cb then", + ), + ], + ids=[ + "nested_in_query", + "nested_in_path", + "comma_joined", + "id_in_path", + "zurich", + "donnees", + "file_no_host", + "file_conf_no_host", + "template_no_host", + "empty_host", + "break_before_query", + "tab_in_password_two_starts", + "break_in_password_dsn", + "tab_in_password", + "break_before_key", + "digit_dot", + "dash", + "dots", + "cjk_digit", + "issue_number", + "template_scheme", + "nested_after_digit", + "letter_digit_dot", + "ipv6", + "query_before_first_scheme", + "fragment_before_first_scheme", + "query_right_before_a_scheme", + "refused_schemeless", + "refused_schemeless_password", + "fragment_owns_a_redirect", + "fragment_owns_a_redirect_in_prose", + ], +) +def test_named_urls_lose_their_password_and_query(text: str, expected: str) -> None: + """B8's named cases.""" + assert _redact(text) == expected + assert "hunter2" not in _redact(text) and "sk-1" not in _redact(text) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + # W2: a scheme urllib doesn't know keeps its //. + ("DATABASE_URL=sqlite:///./app.db", "DATABASE_URL=sqlite:///./app.db"), + ("sqlite:///./app.db", "sqlite:///./app.db"), + ("unix:///var/run/docker.sock", "unix:///var/run/docker.sock"), + ("postgresql:///mydb", "postgresql:///mydb"), + ("Use the myapp:// deep link.", "Use the myapp:// deep link."), + ("the :// separator", "the :// separator"), + # W1: a line break before a URL is kept, and the word is not joined into its scheme. + ("Steps\nhttps://u:p@h.example/x?key=sk-1", "Steps\nhttps://h.example/x"), + # The costs the design page names (V9, D4): more is removed, never less. + ("http://localhost:8000.", "http://localhost"), + ("[https://docs.example.com]", "[[redacted]"), + ( + "Docs (https://docs.example.com/page?lang=en) explain it.", + "Docs (https://docs.example.com/page explain it.", + ), + ( + "requests.get('https://u:p@h/x?key=sk-1', timeout=3)", + "requests.get('https://h/x timeout=3)", + ), + ("请访问https://api.example.com/v1?key=sk-1获取数据", "请访问https://api.example.com/v1"), + # A value that is one URL once its line breaks are removed is joined, as on the base. + ("https://h/x\nthen", "https://h/xthen"), + ], + ids=[ + "sqlite_in_env_line", + "sqlite", + "unix_socket", + "postgresql_no_host", + "deep_link", + "separator", + "word_then_break", + "port_before_punctuation", + "bracket_joined_to_host", + "closing_bracket_after_query", + "call_after_query", + "cjk_after_query", + "lone_url_then_break", + ], +) +def test_honest_text_changes_only_as_the_design_page_says(text: str, expected: str) -> None: + assert _redact(text) == expected + + +@pytest.mark.parametrize( + "value", + [ + "postgres://u:p@[fd00::5]:5432/app?key=sk-1", + "see postgres://u:p@[fd00::5]:5432/app now", + "file:///etc/x?token=sk-1", + "Use https://?key=sk-1 now", + "${S}:///etc/x?token=sk-1", + "sqlite:///./app.db", + "no link here", + "Call https://api.example.com/v1 for data.", + "", + ], + ids=["ipv6", "ipv6_in_prose", "empty_host", "empty_host_in_prose", "template", "sqlite", + "no_url", "nothing_to_clean", "empty"], +) +def test_cleaning_twice_changes_nothing(value: str) -> None: + """B12's cleaner half (V3, W3): IPv6, empty-host and nothing-to-clean values.""" + once = _redact(value) + assert _redact(once) == once + + +@_PROPERTY +@given(st.lists(_secret_urls(False), min_size=1, max_size=3)) +def test_cleaning_twice_changes_nothing_for_urls_with_a_host(drawn: list) -> None: + """W3: idempotent for every URL with a host, in prose.""" + once = _redact("Post " + " then ".join(url for url, _, _ in drawn) + " done") + assert _redact(once) == once + + +_WORD = st.text( + st.sampled_from(string.ascii_letters + string.digits + ".,;:!?'\"()[]/@#-数据é"), + min_size=1, + max_size=8, +).filter(lambda word: "://" not in word) +_GAP = st.sampled_from([" ", " ", "\n", "\t", " \n ", "\r\n", "\u3000", "\u00a0"]) + + +@_PROPERTY +@given( + st.lists( + st.tuples(st.one_of(_WORD, _secret_urls(True).map(lambda drawn: drawn[0])), _GAP), + min_size=1, + max_size=12, + ) +) +def test_honest_text_and_its_whitespace_are_kept_exactly(parts: list) -> None: + """B10 (and B1's whitespace half): removing the URLs from the input and the output leaves the + same text. The prose opens with a word, so it is never read as a single-URL field. The split + is the test's own, not the module's, so a change to the cleaner's split is seen (plant N5).""" + split = re.compile(r"(\s+)") + + text = "Note " + "".join(token + gap for token, gap in parts) + out = _redact(text) + given_parts, cleaned_parts = split.split(text), split.split(out) + assert len(given_parts) == len(cleaned_parts) + for given_part, cleaned_part in zip(given_parts, cleaned_parts): + if "://" not in given_part: + assert cleaned_part == given_part + + +def _timed(statement: str, timeout: float = 60.0) -> float: + """Seconds one call takes, in a child process: a quadratic scan is stopped, not waited for.""" + code = ( + "import logging, time\n" + "logging.disable(logging.WARNING)\n" + "from fi.alk.providers.agent import _redact\n" + f"value = {statement}\n" + "start = time.perf_counter()\n" + "_redact(value)\n" + "print(time.perf_counter() - start)\n" + ) + try: + done = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, timeout=timeout, check=True + ) + except subprocess.TimeoutExpired: + pytest.fail(f"cleaning {statement} took more than {timeout} s") + return float(done.stdout.strip()) + + +def test_a_long_run_of_scheme_characters_cleans_in_linear_time() -> None: + """B9: the lookbehind means a start inside a run of scheme characters is never tried.""" + assert _timed('"a" * 1_000_000 + "!://x"') < 1.0 + # Change review r2, B-L2: the lookbehind's other scheme characters too (plant N2). + assert _timed('"-" * 1_000_000 + "!://x"') < 1.0 + assert _timed('"a.-+" * 250_000 + "!://x"') < 1.0 + assert _timed('"p " + "1" * 1_000_000 + "://x"') < 1.0 + + +def test_a_token_of_many_starts_cleans_in_linear_time() -> None: + """B9c (W4): the ?/# position is cached, so one token of 400,000 starts is searched once.""" + assert _timed('"p " + "a://x/" * 400_000') < 2.0 + + +def test_five_mib_of_url_dense_prose_cleans_in_two_seconds() -> None: + """B9: varied URLs, so urlsplit's own cache doesn't flatter it (V4).""" + pick = random.Random(9) + sentences = [] + size = 0 + while size < 5 * 1024 * 1024: + host = "".join(pick.choice(string.ascii_lowercase) for _ in range(8)) + key = "".join(pick.choice(_UNRESERVED) for _ in range(10)) + sentence = f"Call https://u{size}:p{key}@{host}.example/v{size % 97}?key={key} then reply. " + sentences.append(sentence) + size += len(sentence) + prose = "".join(sentences) + start = time.perf_counter() + out = _redact(prose) + elapsed = time.perf_counter() - start + # Booleans, so a failure never asks pytest to explain a 5 MiB string. + kept_a_query = "?key=" in out + assert elapsed < 2.0 + assert not kept_a_query diff --git a/tests/test_harness.py b/tests/test_harness.py index 122defdc6..7ef935fc6 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -3968,7 +3968,7 @@ def visit(node): advertised = asyncio.run(advertised_fields()) # Set by the amendment tools during later stages, never by whoever submits the contract. - ours = {"amendments", "instrumentation"} + ours = {"amendments", "instrumentation", "source_subfolder", "unconfirmed_prompt_from"} def named(model: type[BaseModel]) -> list[str]: """Every field the model would have to fill, nested ones included. @@ -7334,6 +7334,90 @@ def judge_builtin(name, inputs): assert all(verdict.holds for verdict in verdicts) +_SUITE_EVAL_URL = "https://bot:hunter2@hooks.example/v1?key=sk-1#frag" + + +def test_suite_eval_agent_prompt_loses_url_credentials(monkeypatch): + """B3 (TH-9491): the suite eval's agent_prompt, from a contract no writer cleaned.""" + from fi.alk.harness.catalogue import default_suite_evals + from fi.alk.harness.contract import AgentContract + from fi.alk.harness.run import platform_evals + from fi.alk.harness.run.conversation import Exchange, Transcript + from fi.alk.harness.run.grade import judge_suite_evals + from fi.alk.harness.scenario import Scenario + + sent = [] + + def judge_builtin(name, inputs): + sent.append(inputs) + return {"output": "Pass", "why": "ok", "model": "turing_flash"} + + monkeypatch.setattr(platform_evals, "configured", lambda: True) + monkeypatch.setattr(platform_evals, "judge_builtin", judge_builtin) + contract = AgentContract( + agent="voice-agent", + modality="voice", + system_prompt_excerpt=f"Post each call to {_SUITE_EVAL_URL} after.", + ) + judge_suite_evals( + default_suite_evals(), + Scenario(name="s", instruction="i", sub_goals=[]), + Transcript(exchanges=[Exchange("agent", "hello")]), + contract, + ) + + prompts = [inputs["agent_prompt"] for inputs in sent if "agent_prompt" in inputs] + assert prompts == ["Post each call to https://hooks.example/v1 after."] + assert "hunter2" not in json.dumps(sent) and "sk-1" not in json.dumps(sent) + + +def test_suite_eval_rules_stored_on_the_platform_lose_url_credentials(monkeypatch): + """B3b (TH-9491): each rule passed to platform_evals.ensure.""" + from fi.alk.harness.contract import AgentContract + from fi.alk.harness.run import grade, platform_evals + from fi.alk.harness.run.conversation import Transcript + from fi.alk.harness.scenario import Scenario + + ensured = [] + monkeypatch.setattr( + platform_evals, "ensure", lambda name, claim, agent, rules: ensured.append(list(rules)) + ) + monkeypatch.setattr( + platform_evals, "judge", lambda name, record: {"held": True, "why": "ok", "model": "m"} + ) + contract = AgentContract( + agent="desk", hard_constraints=[f"Always post to {_SUITE_EVAL_URL}", "Be brief"] + ) + + grade._on_platform( + [("claim one", "one"), ("claim two", "two")], + Scenario(name="s", instruction="i", sub_goals=[]), + Transcript(), + contract, + "", + ) + + assert ensured == [["Always post to https://hooks.example/v1", "Be brief"]] * 2 + + +def test_suite_eval_local_judge_prompt_loses_url_credentials(): + """B3c (TH-9491): the local judge's prompt, the summary and each rule.""" + from fi.alk.harness.contract import AgentContract + from fi.alk.harness.run.grade import _judge_prompt + + prompt = _judge_prompt( + AgentContract( + agent="desk", + one_liner=f"Books desks through {_SUITE_EVAL_URL}", + hard_constraints=[f"Always post to {_SUITE_EVAL_URL}"], + ) + ) + + assert "THE AGENT UNDER TEST: desk - Books desks through https://hooks.example/v1\n" in prompt + assert " - Always post to https://hooks.example/v1" in prompt + assert "hunter2" not in prompt and "sk-1" not in prompt + + def test_hosted_voice_marks_unavailable_required_evals_as_grading_failures( monkeypatch, ): diff --git a/tests/test_harness_architecture.py b/tests/test_harness_architecture.py index 5e7e006b9..a12b3a9e3 100644 --- a/tests/test_harness_architecture.py +++ b/tests/test_harness_architecture.py @@ -25,6 +25,7 @@ from fi.alk.harness.executor import ( GitHubSourceAcquirer, HarnessExecutor, + SourceAcquisitionError, _failure_from_events, ) from fi.alk.harness.job import ( @@ -80,6 +81,7 @@ async def fake_auto(args) -> int: job_id="job-upload", run_id="run-upload", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={"kind": "archive", "archive_artifact_id": "source-id"}, agent={"connector": "auto"}, metadata={"source_kind": "archive"}, @@ -112,8 +114,10 @@ def fake_run(*_args, **_kwargs): job_id="job-github", run_id="run-github", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "visibility": "public", "repository": "customer/agent", "ref": "main", @@ -991,8 +995,10 @@ def test_job_carries_references_but_rejects_resolved_secrets() -> None: job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "installation_id": "installation", "repository": "customer/agent", "ref": "main", @@ -1001,9 +1007,9 @@ def test_job_carries_references_but_rejects_resolved_secrets() -> None: "connector": "livekit", "secret_refs": { "api_key": { - "manager": "futureagi", + "manager": "platform-vault", "key": "secret_livekit_key", - "purpose": "connect to agent", + "purpose": "target_provider", } }, }, @@ -1016,8 +1022,10 @@ def test_job_carries_references_but_rejects_resolved_secrets() -> None: job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "installation_id": "installation", "repository": "customer/agent", }, @@ -1030,6 +1038,7 @@ def test_public_github_job_needs_no_installation_but_is_commit_pinnable() -> Non job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", "visibility": "public", @@ -1049,8 +1058,10 @@ def test_public_github_branch_url_is_normalized_into_repository_and_ref() -> Non job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "visibility": "public", "repository": "https://github.com/future-agi/future-agi/tree/feat/harness", }, @@ -1091,8 +1102,10 @@ def test_hosted_job_rejects_unsafe_security_policy(security: dict) -> None: job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "visibility": "public", "repository": "customer/public-agent", }, @@ -1217,6 +1230,8 @@ def test_hosted_github_checkout_keeps_token_out_of_process_arguments( observed = {} def run(command, **kwargs): + if command[1] == "rev-parse": + return SimpleNamespace(returncode=0, stdout="a" * 40 + "\n", stderr="") observed["command"] = command observed["environment"] = kwargs["env"] Path(command[-1]).mkdir() @@ -1227,8 +1242,10 @@ def run(command, **kwargs): job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "installation_id": "installation", "repository": "customer/private-agent", "ref": "main", @@ -1254,6 +1271,8 @@ def token(_installation): raise AssertionError("public checkout must not request an installation token") def run(command, **kwargs): + if command[1] == "rev-parse": + return SimpleNamespace(returncode=0, stdout="a" * 40 + "\n", stderr="") observed["environment"] = kwargs["env"] Path(command[-1]).mkdir() return SimpleNamespace(returncode=0, stdout="", stderr="") @@ -1263,8 +1282,10 @@ def run(command, **kwargs): job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "visibility": "public", "repository": "customer/public-agent", }, @@ -1283,6 +1304,8 @@ def test_public_github_branch_url_clones_the_branch( observed = {} def run(command, **kwargs): + if command[1] == "rev-parse": + return SimpleNamespace(returncode=0, stdout="a" * 40 + "\n", stderr="") observed["command"] = command Path(command[-1]).mkdir() return SimpleNamespace(returncode=0, stdout="", stderr="") @@ -1292,8 +1315,10 @@ def run(command, **kwargs): job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "visibility": "public", "repository": "https://github.com/acme/agent/tree/feat/harness", }, @@ -1314,6 +1339,218 @@ def run(command, **kwargs): ] +def _recording_git(head_output: str): + calls = [] + + # Every call is recorded before it is answered, so rev-parse's arguments, folder and + # environment are checked as strictly as the clone's. + def run(command, **kwargs): + calls.append((command, kwargs["cwd"], kwargs["env"])) + if command[1] == "rev-parse": + return SimpleNamespace(returncode=0, stdout=head_output + "\n", stderr="") + Path(command[-1]).mkdir() + return SimpleNamespace(returncode=0, stdout="", stderr="") + + return run, calls + + +def test_executor_freezes_the_checked_out_commit_for_a_local_github_job( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # A value no job pins, so the build can only see it if the checkout's revision was frozen. + commit = "c" * 40 + observed: dict[str, str | None] = {} + calls = [] + + async def fake_auto(args) -> int: + observed["commit"] = args.job.source.commit_sha + return 0 + + def fake_run(command, **kwargs): + calls.append((command, kwargs["cwd"])) + return SimpleNamespace(returncode=0, stdout=commit + "\n", stderr="") + + monkeypatch.setattr("fi.alk.harness.cli._auto", fake_auto) + monkeypatch.setattr("fi.alk.harness.executor.subprocess.run", fake_run) + source = tmp_path / "repository" + source.mkdir() + # Hosted GitHub jobs must arrive pinned, so only a local job reaches the freeze. + job = HarnessJob( + job_id="job-local-github", + run_id="run-local-github", + execution="local", + source={ + "kind": "github", + "visibility": "public", + "repository": "customer/agent", + "ref": "main", + }, + agent={"connector": "auto"}, + ) + + status = asyncio.run( + HarnessExecutor().run(job, source=source, output=tmp_path / "artifacts") + ) + + assert status.stage.value == "completed" + assert observed["commit"] == commit + # The recorded provenance must be the checkout's revision, not that of a folder around it. + assert calls == [(["git", "rev-parse", "HEAD"], source)] + + +# The frozen job is not validated again, so a failed rev-parse must be refused even when it +# printed a commit, and output that isn't a commit even when git exits cleanly. +@pytest.mark.parametrize( + ("returncode", "stdout"), + [(1, "c" * 40 + "\n"), (0, "not-a-commit\n")], + ids=["rev-parse-fails", "output-is-not-a-commit"], +) +def test_executor_fails_a_local_github_job_whose_commit_cannot_be_resolved( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, returncode: int, stdout: str +) -> None: + async def fake_auto(args) -> int: + raise AssertionError("an unresolved commit must not reach the build") + + def fake_run(*_args, **_kwargs): + return SimpleNamespace(returncode=returncode, stdout=stdout, stderr="") + + monkeypatch.setattr("fi.alk.harness.cli._auto", fake_auto) + monkeypatch.setattr("fi.alk.harness.executor.subprocess.run", fake_run) + source = tmp_path / "repository" + source.mkdir() + job = HarnessJob( + job_id="job-local-github", + run_id="run-local-github", + execution="local", + source={ + "kind": "github", + "visibility": "public", + "repository": "customer/agent", + "ref": "main", + }, + agent={"connector": "auto"}, + ) + + status = asyncio.run( + HarnessExecutor().run(job, source=source, output=tmp_path / "artifacts") + ) + + assert status.stage.value == "failed" + assert status.failure.code == "github_commit_resolution_failed" + assert status.failure.domain is FailureDomain.INFRASTRUCTURE + assert status.failure.stage is HarnessStage.ACQUIRING_SOURCE + + +def test_hosted_github_checkout_keeps_token_out_of_git_arguments_and_the_pin_check( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + token = "installation-secret-token" + run, calls = _recording_git("a" * 40) + monkeypatch.setattr("fi.alk.harness.executor.subprocess.run", run) + job = HarnessJob( + job_id="job", + run_id="run", + execution="hosted", + runtime={"isolation": "dedicated_vm"}, + source={ + "kind": "github", + "installation_id": "installation", + "repository": "customer/private-agent", + "ref": "main", + "commit_sha": "a" * 40, + }, + agent={"connector": "http"}, + ) + + checkout = asyncio.run( + GitHubSourceAcquirer(lambda _installation: token).acquire(job, tmp_path) + ) + + assert checkout == tmp_path / "repository" + assert [command for command, _cwd, _env in calls] == [ + [ + "git", + "clone", + "--depth", + "1", + "--branch", + "main", + "https://github.com/customer/private-agent.git", + str(tmp_path / "repository"), + ], + ["git", "rev-parse", "HEAD"], + ] + assert all(token not in " ".join(command) for command, _cwd, _env in calls) + assert calls[0][2]["GIT_CONFIG_VALUE_0"].endswith(token) + # Verifying the pin needs no credential, so the token must not ride along into it. + _command, verify_cwd, verify_env = calls[1] + assert verify_cwd == tmp_path / "repository" + assert all(token not in value for value in verify_env.values()) + + +def test_hosted_github_checkout_refuses_a_checkout_that_is_not_the_pinned_commit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + token = "installation-secret-token" + # Differs from the pin only in its last character, so comparing a short prefix can't pass. + run, _calls = _recording_git("a" * 39 + "b") + monkeypatch.setattr("fi.alk.harness.executor.subprocess.run", run) + job = HarnessJob( + job_id="job", + run_id="run", + execution="hosted", + runtime={"isolation": "dedicated_vm"}, + source={ + "kind": "github", + "installation_id": "installation", + "repository": "customer/private-agent", + "ref": "main", + "commit_sha": "a" * 40, + }, + agent={"connector": "http"}, + ) + + with pytest.raises(SourceAcquisitionError, match="github_commit_mismatch"): + asyncio.run( + GitHubSourceAcquirer(lambda _installation: token).acquire(job, tmp_path) + ) + + +# None leaves the runtime out, which selects the shared-runner default; hosted customer code +# must never get that or any other isolation weaker than its own VM. +@pytest.mark.parametrize( + "isolation", + [ + None, + *( + isolation.value + for isolation in RuntimeIsolation + if isolation is not RuntimeIsolation.DEDICATED_VM + ), + ], +) +def test_hosted_job_refuses_any_isolation_but_a_dedicated_vm( + isolation: str | None, +) -> None: + # Pinned, so the commit rule passes and the isolation rule is the one that refuses. + common = { + "job_id": "job", + "run_id": "run", + "execution": "hosted", + "source": { + "kind": "github", + "visibility": "public", + "repository": "customer/public-agent", + "commit_sha": "a" * 40, + }, + "agent": {"connector": "auto"}, + } + runtime = {} if isolation is None else {"runtime": {"isolation": isolation}} + + with pytest.raises(ValueError, match="hosted_isolation_must_be_dedicated_vm"): + HarnessJob(**common, **runtime) + + def test_source_fingerprint_hashes_symlink_metadata_without_reading_target( tmp_path: Path, ) -> None: @@ -1329,3 +1566,37 @@ def test_source_fingerprint_hashes_symlink_metadata_without_reading_target( assert first == second + +def _github_source(**extra): + return { + "kind": "github", + "visibility": "public", + "repository": "customer/public-agent", + **extra, + } + + +def test_repository_source_keeps_subfolder() -> None: + from fi.alk.harness.job import RepositorySource + + # Kept exactly as sent: the platform compares the echo, so nothing is trimmed. + for value in ("agents/a", "agents/a/", " agents/a", "my..agent", "/abs"): + assert RepositorySource(**_github_source(subfolder=value)).subfolder == value + + +def test_empty_subfolder_is_none() -> None: + from fi.alk.harness.job import RepositorySource + + assert RepositorySource(**_github_source(subfolder="")).subfolder is None + assert RepositorySource(**_github_source()).subfolder is None + + +def test_subfolder_refused_on_other_kinds() -> None: + from fi.alk.harness.job import RepositorySource + + with pytest.raises(ValueError, match="subfolder_requires_github_source"): + RepositorySource(kind="provider", subfolder="agents/a") + with pytest.raises(ValueError, match="subfolder_requires_github_source"): + RepositorySource(kind="archive", archive_artifact_id="a", subfolder="agents/a") + assert RepositorySource(kind="provider", subfolder="").subfolder is None + diff --git a/tests/test_harness_credentials.py b/tests/test_harness_credentials.py index f4418f871..bc7420560 100644 --- a/tests/test_harness_credentials.py +++ b/tests/test_harness_credentials.py @@ -1,3 +1,4 @@ +import os from pathlib import Path import pytest @@ -620,3 +621,451 @@ def test_explicit_scan_paths_exclude_unused_optional_integrations( assert set(_requirements(manifest)) == {"OPENAI_API_KEY"} assert "retell" not in manifest.detected_connectors + + +def _agents_repo(root, files): + for name, body in files.items(): + (root / name).parent.mkdir(parents=True, exist_ok=True) + (root / name).write_text(body, encoding="utf-8") + return root + + +def _required(manifest): + return {item.environment_name: item.required for item in manifest.requirements} + + +def test_shared_code_the_folder_imports_requires_its_values(tmp_path): + root = _agents_repo( + tmp_path, + { + "common/__init__.py": "", + "common/llm.py": 'import os\nKEY = os.environ["SHARED_API_KEY"]\n', + "agents/a/agent.py": "from common import llm\n", + "agents/b/agent.py": 'import os\nKEY = os.environ["B_ONLY_API_KEY"]\n', + }, + ) + + required = _required(discover_credentials(root, agent_subfolder="agents/a")) + + assert required["SHARED_API_KEY"] is True + assert required["B_ONLY_API_KEY"] is False + + +def test_imports_are_followed_past_the_first_file(tmp_path): + root = _agents_repo( + tmp_path, + { + "common/first.py": "from . import second\n", + "common/second.py": 'import os\nKEY = os.environ["DEEP_API_KEY"]\n', + "agents/a/agent.py": "import common.first\n", + }, + ) + + assert _required(discover_credentials(root, agent_subfolder="agents/a"))["DEEP_API_KEY"] is True + + +def test_a_siblings_connector_requires_nothing(tmp_path): + root = _agents_repo( + tmp_path, {"agents/a/agent.py": "x = 1\n", "agents/b/agent.py": "import livekit\n"} + ) + with_folder = discover_credentials(root, agent_subfolder="agents/a") + without = discover_credentials(root) + + assert with_folder.detected_connectors == [] + assert without.detected_connectors != [] + + +def test_no_folder_keeps_every_hit_required(tmp_path): + root = _agents_repo( + tmp_path, {"agents/b/agent.py": 'import os\nKEY = os.environ["B_ONLY_API_KEY"]\n'} + ) + + assert _required(discover_credentials(root))["B_ONLY_API_KEY"] is True + + +@pytest.mark.parametrize( + "line", + [ + 'import { llm } from "../../packages/shared/llm";', + 'const llm = require("../../packages/shared/llm.ts");', + 'const llm = await import("../../packages/shared/llm");', + 'import { llm } from "packages/shared/llm";', + ], +) +def test_shared_scripts_the_folder_imports_require_their_values(tmp_path, line): + root = _agents_repo( + tmp_path, + { + "packages/shared/llm.ts": "export const key = process.env.SHARED_API_KEY;\n", + "packages/other/x.ts": "export const key = process.env.OTHER_API_KEY;\n", + "agents/a/index.ts": f"{line}\n", + }, + ) + + required = _required(discover_credentials(root, agent_subfolder="agents/a")) + + assert required["SHARED_API_KEY"] is True + assert required["OTHER_API_KEY"] is False + + +def test_a_script_import_reaches_an_index_and_a_compiled_name(tmp_path): + root = _agents_repo( + tmp_path, + { + "lib/index.ts": 'export * from "./real.js";\n', + "lib/real.ts": "export const key = process.env.REAL_API_KEY;\n", + "agents/a/index.js": 'import "../../lib";\n', + }, + ) + + assert _required(discover_credentials(root, agent_subfolder="agents/a"))["REAL_API_KEY"] is True + + +@pytest.mark.parametrize("make", ["link", "fifo"]) +def test_the_import_walk_does_not_read_a_link_or_a_pipe(tmp_path, make): + from fi.alk.harness.credentials import imported_by_agent + + root = _agents_repo(tmp_path, {"agents/a/agent.py": "x = 1\n"}) + target = root / "agents" / "a" / "x.py" + if make == "link": + target.symlink_to("/dev/zero") + else: + os.mkfifo(target) + + assert imported_by_agent(root, "agents/a") == frozenset() + + +def test_the_import_walk_does_not_follow_a_link_out_or_read_an_oversized_file(tmp_path): + from fi.alk.harness import credentials + + outside = tmp_path / "outside.py" + outside.write_text("x = 1\n") + root = _agents_repo( + tmp_path / "repo", + { + "common/big.py": "#" * (credentials._MAX_FILE_BYTES + 1), + "agents/a/agent.py": "import common.big\nimport linked\n", + }, + ) + (root / "linked.py").symlink_to(outside) + + assert credentials.imported_by_agent(root, "agents/a") == frozenset() + + +@pytest.mark.parametrize( + "budget, stopped", + [ + ({}, False), + ({"_MAX_WALK_IMPORTS": 1}, True), + # The folder's own two files, then nothing left for common/llm.py. + ({"_MAX_FILES": 2}, True), + ({"_MAX_WALK_BYTES": 1}, True), + ], + ids=["none", "names", "files", "bytes"], +) +def test_the_import_walk_says_when_a_limit_stopped_it(tmp_path, monkeypatch, caplog, budget, stopped): + from fi.alk.harness import credentials + + root = _agents_repo( + tmp_path, + { + "common/__init__.py": "", + "common/llm.py": 'import os\nKEY = os.environ["SHARED_API_KEY"]\n', + "agents/a/agent.py": "import nowhere\nimport common.llm\n", + # Read first, so a limit on names leaves no file waiting: only the cut says so. + "agents/a/aaa.py": "x = 1\n", + }, + ) + for name, value in budget.items(): + monkeypatch.setattr(credentials, name, value) + said: list[bool] = [] + + credentials.imported_by_agent(root, "agents/a", stopped=said) + with caplog.at_level("WARNING", logger="fi.alk.harness.credentials"): + manifest = credentials.discover_credentials(root, agent_subfolder="agents/a") + + assert said == [stopped] + # The key in the file it didn't reach is optional, and the manifest says the scan was cut. + assert manifest.truncated is stopped + assert ("stopped at its limits" in caplog.text) is stopped + + +def test_the_import_walk_stops_at_the_file_limit(tmp_path, monkeypatch): + from fi.alk.harness import credentials + + root = _agents_repo( + tmp_path, + { + "common/one.py": "import common.two\n", + "common/two.py": "x = 1\n", + "agents/a/agent.py": "import common.one\n", + }, + ) + monkeypatch.setattr(credentials, "_MAX_FILES", 1) + + assert credentials.imported_by_agent(root, "agents/a") == frozenset({(root / "common/one.py").resolve()}) + + +_STRICT_COMPOSE = "services:\n billing:\n environment:\n STRIPE_SECRET_KEY: ${STRIPE_SECRET_KEY:?}\n" + + +def test_a_root_compose_file_requires_nothing_unless_it_is_named(tmp_path): + root = _agents_repo( + tmp_path, {"docker-compose.yml": _STRICT_COMPOSE, "agents/a/agent.py": "x = 1\n"} + ) + + unnamed = discover_credentials(root, agent_subfolder="agents/a") + named = discover_credentials( + root, agent_subfolder="agents/a", agent_compose_file="docker-compose.yml" + ) + + assert _required(unnamed)["STRIPE_SECRET_KEY"] is False + assert _required(named)["STRIPE_SECRET_KEY"] is True + + +def test_a_compose_file_in_the_folder_is_the_agents(tmp_path): + root = _agents_repo(tmp_path, {"agents/a/docker-compose.yml": _STRICT_COMPOSE}) + + assert _required(discover_credentials(root, agent_subfolder="agents/a"))["STRIPE_SECRET_KEY"] is True + + +@pytest.mark.parametrize( + "line", + [ + "import" + " " * 400_000 + "y", + "import(" + " " * 400_000 + "y", + "require(" + " " * 400_000 + "y", + "from" + " " * 400_000 + "y", + "import type" + " " * 400_000 + "y", + "import " * 100_000, + "import type a " * 40_000, + ], + ids=["import", "import-call", "require", "from", "type", "repeated", "repeated-type"], +) +def test_an_odd_script_keeps_the_import_walk_quick(tmp_path, line): + import time + + from fi.alk.harness.credentials import imported_by_agent + + root = _agents_repo(tmp_path, {"agents/a/x.js": line}) + started = time.monotonic() + + imported_by_agent(root, "agents/a") + + assert time.monotonic() - started < 5 + + +@pytest.mark.parametrize( + ("name", "body"), + [ + ("routes.js", 'const note = from "/";\n'), + ("routes.ts", 'const url = from "";\nconst other = require("\\0");\n'), + ("deep.py", "from " + "." * 40 + " import x\n"), + ("deep2.py", "from " + "." * 40 + "pkg import x\n"), + ], +) +def test_one_odd_import_does_not_stop_the_walk(tmp_path, name, body): + from fi.alk.harness.credentials import imported_by_agent + + root = _agents_repo( + tmp_path, + { + "common/llm.py": "x = 1\n", + f"agents/a/{name}": body, + "agents/a/agent.py": "import common.llm\n", + }, + ) + + assert (root / "common/llm.py").resolve() in imported_by_agent(root, "agents/a") + + +def test_an_import_runs_the_packages_above_it(tmp_path): + root = _agents_repo( + tmp_path, + { + "pkg/__init__.py": "from . import config\n", + "pkg/config.py": 'import os\nKEY = os.environ["PACKAGE_API_KEY"]\n', + "pkg/mod.py": "x = 1\n", + "agents/a/agent.py": "from pkg.mod import x\n", + }, + ) + + assert _required(discover_credentials(root, agent_subfolder="agents/a"))["PACKAGE_API_KEY"] is True + + +def test_a_siblings_default_does_not_clear_the_agents_requirement(tmp_path): + root = _agents_repo( + tmp_path, + { + "agents/a/agent.py": 'import os\nNAME = os.environ["MODEL_NAME"]\n', + "agents/b/.env.example": "MODEL_NAME=gpt-4o\n", + }, + ) + + assert _required(discover_credentials(root, agent_subfolder="agents/a"))["MODEL_NAME"] is True + + +@pytest.mark.parametrize( + "line", + [ + '// import { key } from "../b/llm";', + '/* const key = require("../b/llm"); */', + 'import type { Key } from "../b/llm";', + ], +) +def test_a_comment_or_a_type_import_is_not_followed(tmp_path, line): + root = _agents_repo( + tmp_path, + { + "agents/a/ui.ts": f"{line}\n", + "agents/b/llm.ts": "export const key = process.env.BILLING_API_KEY;\n", + }, + ) + + assert _required(discover_credentials(root, agent_subfolder="agents/a"))["BILLING_API_KEY"] is False + + +def test_a_byte_order_mark_does_not_hide_a_files_imports(tmp_path): + root = _agents_repo( + tmp_path, + { + "common/llm.py": 'import os\nKEY = os.environ["SHARED_API_KEY"]\n', + "agents/a/agent.py": "\ufeffimport common.llm\n", + }, + ) + + assert _required(discover_credentials(root, agent_subfolder="agents/a"))["SHARED_API_KEY"] is True + + +def test_the_file_limit_reads_the_agents_files_first(tmp_path, monkeypatch): + from fi.alk.harness import credentials + + root = _agents_repo( + tmp_path, + { + "aaa/one.py": "x = 1\n", + "aaa/two.py": "x = 1\n", + "agents/a/agent.py": 'import os\nKEY = os.environ["A_ONLY_API_KEY"]\n', + }, + ) + monkeypatch.setattr(credentials, "_MAX_FILES", 1) + + assert _required(discover_credentials(root, agent_subfolder="agents/a"))["A_ONLY_API_KEY"] is True + + +def test_the_same_import_many_times_counts_once_against_the_budget(tmp_path): + from fi.alk.harness.credentials import _MAX_WALK_IMPORTS, imported_by_agent + + body = 'import"./gone";' * (_MAX_WALK_IMPORTS + 1) + 'import"../../common/real";' + root = _agents_repo( + tmp_path, + { + "common/real.js": 'import "./deep";\n', + "common/deep.js": "export const x = 1;\n", + "agents/a/x.js": body, + }, + ) + + assert (root / "common/deep.js").resolve() in imported_by_agent(root, "agents/a") + + +def test_the_import_walk_stops_at_its_byte_budget(tmp_path, monkeypatch): + from fi.alk.harness import credentials + + root = _agents_repo( + tmp_path, + { + "common/one.py": "import common.two\n", + "common/two.py": "x = 1\n", + "agents/a/agent.py": "import common.one\n", + }, + ) + monkeypatch.setattr(credentials, "_MAX_WALK_BYTES", 1) + + assert credentials.imported_by_agent(root, "agents/a") == frozenset( + {(root / "common/one.py").resolve()} + ) + + +def test_the_import_walk_stops_at_its_import_budget(tmp_path, monkeypatch): + from fi.alk.harness import credentials + + root = _agents_repo( + tmp_path, + { + "common/one.py": "import common.two\n", + "common/two.py": "x = 1\n", + "agents/a/agent.py": "import common.one\n", + }, + ) + monkeypatch.setattr(credentials, "_MAX_WALK_IMPORTS", 1) + + assert credentials.imported_by_agent(root, "agents/a") == frozenset( + {(root / "common/one.py").resolve()} + ) + + +def test_the_walk_reads_the_folders_own_files_before_a_chain(tmp_path, monkeypatch): + from fi.alk.harness import credentials + + root = _agents_repo( + tmp_path, + { + "agents/a/agent.py": "import common.web\n", + "agents/a/zz.py": "import chain.one\n", + "chain/one.py": "import chain.two\n", + "chain/two.py": "import chain.three\n", + "chain/three.py": "x = 1\n", + "common/web.py": "import common.deep\n", + "common/deep.py": "x = 1\n", + }, + ) + monkeypatch.setattr(credentials, "_MAX_WALK_IMPORTS", 3) + + assert (root / "common/deep.py").resolve() in credentials.imported_by_agent(root, "agents/a") + + +def test_the_walk_prefers_the_folders_copy_of_a_module(tmp_path): + from fi.alk.harness.credentials import imported_by_agent + + root = _agents_repo( + tmp_path, + { + "agents/a/agent.py": "import shared\n", + "agents/a/shared.py": "x = 1\n", + "shared.py": "x = 1\n", + }, + ) + + found = imported_by_agent(root, "agents/a") + + assert (root / "agents/a/shared.py").resolve() in found + assert (root / "shared.py").resolve() not in found + + +def test_an_odd_script_import_does_not_stop_the_files_other_imports(tmp_path): + from fi.alk.harness.credentials import imported_by_agent + + root = _agents_repo( + tmp_path, + { + "common/util.js": "export const x = 1;\n", + # "/" can't name a module, and makes the path code raise. + "agents/a/routes.js": 'import a from "/";\nimport b from "../../common/util.js";\n', + }, + ) + + assert (root / "common/util.js").resolve() in imported_by_agent(root, "agents/a") + + +def test_a_walk_that_cant_parse_a_file_marks_the_manifest_truncated(tmp_path): + root = tmp_path.resolve() + (root / "agents" / "a").mkdir(parents=True) + (root / "agents" / "a" / "main.py").write_text("import common.helpers\n") + (root / "common").mkdir() + (root / "common" / "__init__.py").write_text("") + (root / "common" / "helpers.py").write_text("from common import web\nNAME = 'a" + "\u0301" * 40 + "'\n") + (root / "common" / "web.py").write_text("import os\nKEY = os.environ['WEB_API_KEY']\n") + + manifest = discover_credentials(root, agent_subfolder="agents/a") + assert manifest.truncated diff --git a/tests/test_harness_sandbox_server.py b/tests/test_harness_sandbox_server.py index 3e1aff855..36ae964c9 100644 --- a/tests/test_harness_sandbox_server.py +++ b/tests/test_harness_sandbox_server.py @@ -210,6 +210,14 @@ def test_local_sandbox_reports_live_stage_timestamp_and_detail_from_events( source = tmp_path / "sources" / "agent" source.mkdir(parents=True) monkeypatch.setenv("ALK_SANDBOX_SOURCE_ROOTS", str(tmp_path / "sources")) + + # The stage comes from the event file, not the job. Left to run, the job reaches its + # worker's start, where the client's loop closing after the request cancels it and waits + # on a worker half started, or, past it, leaves a real worker running after the test. + async def idle(self, _job, _source): + await asyncio.sleep(0) + + monkeypatch.setattr(LocalSandbox, "_execute", idle) state_root = tmp_path / "state" client = TestClient(create_app(state_root)) submitted = client.post("/v1/jobs", json={"source_path": str(source)}).json() @@ -955,6 +963,220 @@ async def submit(): ) +class _Acquirer: + """Stands in for the clone: a repository holding two agents, counting how often it is asked.""" + + calls = 0 + on_acquire = None + + def __init__(self, _token_provider) -> None: + pass + + async def acquire(self, job, workspace): + type(self).calls += 1 + checkout = workspace / "repository" + for name, variable in (("a", "A_ONLY_API_KEY"), ("b", "B_ONLY_API_KEY")): + (checkout / "agents" / name).mkdir(parents=True) + (checkout / "agents" / name / "agent.py").write_text( + f'import os\nKEY = os.environ["{variable}"]\n' + ) + if type(self).on_acquire is not None: + type(self).on_acquire(job) + return checkout + + +@pytest.fixture +def cloning(monkeypatch): + from fi.alk.harness import sandbox_server + + _Acquirer.calls = 0 + _Acquirer.on_acquire = None + monkeypatch.setattr(sandbox_server, "GitHubSourceAcquirer", _Acquirer) + return sandbox_server + + +@pytest.fixture +def hosted_github_job(monkeypatch): + # submit() builds a GitHub job without the dedicated-VM runtime a hosted job requires, so + # today it refuses every GitHub submission. These tests cover what follows that step. + from fi.alk.harness import sandbox_server + from fi.alk.harness.job import RuntimeIsolation, RuntimeRequirements + + class HostedJob(sandbox_server.HarnessJob): + runtime: RuntimeRequirements = Field( + default_factory=lambda: RuntimeRequirements(isolation=RuntimeIsolation.DEDICATED_VM) + ) + + monkeypatch.setattr(sandbox_server, "HarnessJob", HostedJob) + + +def _github_request(subfolder: str, **extra) -> LocalSandboxRequest: + return LocalSandboxRequest( + github_repository="acme/agents", + github_commit_sha="a" * 40, + github_subfolder=subfolder, + metadata={"generic_harness_v1": True}, + **extra, + ) + + +def _executed(sandbox: LocalSandbox, request: LocalSandboxRequest): + async def run(): + response = sandbox.submit(request) + await sandbox._tasks[response.job.job_id] + return response.job.job_id + + job_id = asyncio.run(run()) + return json.loads((sandbox.jobs_root / job_id / "state.json").read_text()) + + +def test_github_subfolder_reaches_source(tmp_path, hosted_github_job): + sandbox = LocalSandbox(tmp_path / "state") + seen = {} + + async def idle(job, _source): + seen["subfolder"] = job.source.subfolder + + sandbox._execute = idle + + async def run(): + response = sandbox.submit(_github_request("agents/a")) + await sandbox._tasks[response.job.job_id] + + asyncio.run(run()) + + assert seen == {"subfolder": "agents/a"} + + +def test_github_subfolder_requires_repository(tmp_path): + with pytest.raises(ValueError, match="needs a GitHub repository"): + LocalSandboxRequest( + source_path=str(tmp_path), + github_subfolder="agents/a", + metadata={"generic_harness_v1": True}, + ) + assert LocalSandboxRequest(source_path=str(tmp_path), github_subfolder="").source_path + + +def test_github_subfolder_requires_generic(tmp_path): + with pytest.raises(ValueError, match="needs the generic harness"): + LocalSandboxRequest(github_repository="acme/agents", github_subfolder="agents/a") + + +def test_bad_subfolder_fails_once_without_retry(tmp_path, cloning, hosted_github_job): + sandbox = LocalSandbox(tmp_path / "state") + + state = _executed(sandbox, _github_request("agents/missing")) + + assert _Acquirer.calls == 1 + assert state["stage"] == "failed" + failure = state["failure"] + assert (failure["domain"], failure["stage"], failure["code"], failure["retryable"]) == ( + "agent", + "acquiring_source", + "subfolder_not_found", + False, + ) + assert failure["details"] == {"subfolder": "agents/missing"} + + +def test_bad_subfolder_keeps_canceled(tmp_path, cloning, hosted_github_job): + sandbox = LocalSandbox(tmp_path / "state") + + def cancel(job): + path = sandbox.jobs_root / job.job_id / "state.json" + state = json.loads(path.read_text()) + state["stage"] = "canceled" + path.write_text(json.dumps(state)) + + _Acquirer.on_acquire = cancel + + state = _executed(sandbox, _github_request("agents/missing")) + + assert state["stage"] == "canceled" + assert not state.get("failure") + + +def test_the_credentials_walk_runs_off_the_servers_loop( + tmp_path, cloning, monkeypatch, hosted_github_job +): + import threading + + from fi.alk.harness import sandbox_server + + # The walk reads the agent's files at bundling's limits: seconds, on a crafted repository. + sandbox = LocalSandbox(tmp_path / "state") + threads: list[threading.Thread] = [] + discover = sandbox_server.discover_credentials + + def recorded(*args, **kwargs): + threads.append(threading.current_thread()) + return discover(*args, **kwargs) + + def stop(*args, **kwargs): + raise RuntimeError("stop after the credentials are written") + + monkeypatch.setattr(sandbox_server, "discover_credentials", recorded) + monkeypatch.setattr(cloning, "resolve_worker_secrets", stop) + + async def run(): + response = sandbox.submit(_github_request("agents/a")) + await sandbox._tasks[response.job.job_id] + + asyncio.run(run()) + + assert threads and threading.main_thread() not in threads + + +def test_subfolder_scopes_credentials(tmp_path, cloning, monkeypatch, hosted_github_job): + sandbox = LocalSandbox(tmp_path / "state") + + def stop(*args, **kwargs): + raise RuntimeError("stop after the credentials are written") + + monkeypatch.setattr(cloning, "resolve_worker_secrets", stop) + + async def run(): + response = sandbox.submit(_github_request("agents/a")) + await sandbox._tasks[response.job.job_id] + return response.job.job_id + + job_id = asyncio.run(run()) + + manifest = json.loads((sandbox.jobs_root / job_id / "credentials.json").read_text()) + required = {item["environment_name"]: item["required"] for item in manifest["requirements"]} + assert required["A_ONLY_API_KEY"] is True + assert required["B_ONLY_API_KEY"] is False + + +def test_live_contract_output_loses_url_credentials_from_a_raw_contract(tmp_path): + """B3d (TH-9491, plan v5.1 §W9): a contract.json written directly, not by write_contract, + as a job from before the fix left it, is cleaned on its way to the live run view.""" + from fi.alk.harness.sandbox_server import _stage_outputs + + url = "https://bot:hunter2@hooks.example/v1?key=sk-1#frag" + (tmp_path / "contract.json").write_text( + json.dumps( + { + "one_liner": f"Books rides through {url}", + "modality": "voice", + "tools": [{"name": "book_ride"}], + "hard_constraints": [f"Always post to {url}", "Confirm before booking"], + } + ), + encoding="utf-8", + ) + + [contract] = [item for item in _stage_outputs(tmp_path) if item["kind"] == "contract"] + + assert contract["data"]["one_liner"] == "Books rides through https://hooks.example/v1" + assert contract["data"]["hard_constraints"] == [ + "Always post to https://hooks.example/v1", + "Confirm before booking", + ] + assert "hunter2" not in json.dumps(contract) and "sk-1" not in json.dumps(contract) + + @pytest.mark.parametrize("flag", [None, "0", "false", "", "true", "yes", " 1", "1 "]) def test_tokenless_server_refuses_jobs_unless_the_flag_is_exactly_one( tmp_path, monkeypatch, caplog, flag