From 41fbf932fd86dc28e6929f4a6f1a806d818d1475 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sun, 15 Nov 2026 01:45:59 +0530 Subject: [PATCH 1/3] feat(harness): record why a rebuild drops a scenario for a removed capability, and restore it under its key once the capability returns --- src/fi/alk/harness/dropped.py | 318 ++++++++++++++++ src/fi/alk/harness/hosted_chat_entrypoint.py | 18 + src/fi/alk/harness/nudge.py | 115 +++++- .../skills/builder/topics/agent-update.md | 7 + tests/harness/test_dropped_scenarios.py | 345 ++++++++++++++++++ 5 files changed, 801 insertions(+), 2 deletions(-) create mode 100644 src/fi/alk/harness/dropped.py create mode 100644 tests/harness/test_dropped_scenarios.py diff --git a/src/fi/alk/harness/dropped.py b/src/fi/alk/harness/dropped.py new file mode 100644 index 000000000..090a94ba1 --- /dev/null +++ b/src/fi/alk/harness/dropped.py @@ -0,0 +1,318 @@ +"""Scenarios a suite rebuild dropped because the agent lost a capability they need. + +Each drop is kept in ``dropped.json`` beside the scenarios, so it travels with every suite version: +what the scenario needed (tools its reference solution calls, sub-goals it names), what was gone, +the agent version that removed it, the reason, the last suite version that held it, and the +scenario's last content. A later suite rebuild in which everything it needed is back puts it in the +suite again under the same key, from that content, and proves it like any other scenario. One that +cannot be proved goes back to dropped and is reported with the reason. + +A scenario dropped for any other reason is not recorded: only a missing capability brings one back. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any + +from .scenario import Scenario, _slug + +DROPPED = "dropped.json" +FORMAT = 1 + + +def load(workspace: Path) -> dict[str, dict[str, Any]]: + """Each dropped scenario's record, by its key; empty when nothing was dropped.""" + try: + body = json.loads((Path(workspace) / DROPPED).read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + held = body.get("scenarios") if isinstance(body, dict) else None + if not isinstance(held, dict): + return {} + return {str(key): dict(one) for key, one in held.items() if isinstance(one, dict)} + + +def save(workspace: Path, records: Mapping[str, Mapping[str, Any]]) -> None: + """Write the records; with none left the file goes, so an untouched suite stays unchanged.""" + path = Path(workspace) / DROPPED + if not records: + path.unlink(missing_ok=True) + return + temporary = path.with_name(path.name + ".tmp") + temporary.write_text( + json.dumps( + { + "format": FORMAT, + "scenarios": {key: dict(records[key]) for key in sorted(records)}, + }, + indent=2, + ensure_ascii=False, + ), + encoding="utf-8", + ) + temporary.replace(path) + + +def needs(scenario: Scenario) -> dict[str, list[str]]: + """The capabilities a scenario needs: the tools its reference solution calls and its sub-goals.""" + return { + "tools": sorted({step.tool for step in scenario.solution if step.tool}), + "sub_goals": sorted(set(scenario.sub_goals)), + } + + +def missing( + needed: Mapping[str, Iterable[str]], tools: set[str], sub_goals: set[str] +) -> dict[str, list[str]]: + """What of ``needed`` the contract's tools and the catalogue's sub-goals no longer hold.""" + return { + "tools": sorted(one for one in needed.get("tools") or [] if one not in tools), + "sub_goals": sorted( + one for one in needed.get("sub_goals") or [] if one not in sub_goals + ), + } + + +def reason(gone: Mapping[str, Iterable[str]], agent_version: str = "") -> str: + """Why the scenario was dropped, for a person: "agent v5 no longer has `cancel_booking`".""" + agent = f"agent {agent_version}" if agent_version else "the agent" + parts = [] + tools = list(gone.get("tools") or []) + sub_goals = list(gone.get("sub_goals") or []) + if tools: + parts.append(f"{agent} no longer has " + ", ".join(f"`{one}`" for one in tools)) + if sub_goals: + parts.append( + ("the environment" if tools else f"for {agent} the environment") + + " retired " + + ", ".join(f"`{one}`" for one in sub_goals) + ) + return "; ".join(parts) + + +def record_drops( + workspace: Path, + before: Mapping[str, Mapping[str, Any]], + present: set[str], + *, + tools: set[str], + sub_goals: set[str], + agent_version: str = "", + suite_version: str = "", +) -> dict[str, dict[str, Any]]: + """Record each scenario of ``before`` the suite no longer holds because a capability it needs + is gone, and return those records by key. ``before`` is each scenario's content when the turn + began; ``present`` the keys the suite holds now.""" + records = load(workspace) + # One recorded by an earlier report of this turn and kept again after it is not dropped. + kept_again = {key for key in set(before) & present if key in records} + recorded: dict[str, dict[str, Any]] = {} + for key in sorted(set(before) - present): + try: + scenario = Scenario.model_validate(dict(before[key])) + except ValueError: + continue + needed = needs(scenario) + gone = missing(needed, tools, sub_goals) + if not gone["tools"] and not gone["sub_goals"]: + continue + recorded[key] = { + "name": scenario.name, + "capabilities": needed, + "missing": gone, + "agent_version": agent_version, + "reason": reason(gone, agent_version), + "last_suite_version": suite_version, + "content": dict(before[key]), + } + if recorded or kept_again: + save( + workspace, + { + **{key: one for key, one in records.items() if key not in kept_again}, + **recorded, + }, + ) + return recorded + + +def restorable( + records: Mapping[str, Mapping[str, Any]], tools: set[str], sub_goals: set[str] +) -> list[str]: + """The dropped keys whose every needed capability is back.""" + return sorted( + key + for key, one in records.items() + if not any(missing(one.get("capabilities") or {}, tools, sub_goals).values()) + and isinstance(one.get("content"), dict) + ) + + +def argument_changes(scenario: Scenario, contract: Any) -> list[str]: + """Steps of the reference solution passing arguments the tool no longer takes, or naming a + tool the contract lacks.""" + if contract is None: + return [] + by_name = {tool.name: tool for tool in contract.tools} + said = [] + for position, step in enumerate(scenario.solution, start=1): + tool = by_name.get(step.tool) + if tool is None: + said.append(f"step {position} calls {step.tool}, which the contract lacks") + continue + unknown = sorted(set(step.arguments) - set(tool.args)) if tool.args else [] + if unknown: + said.append( + f"step {position} {step.tool} no longer takes {', '.join(unknown)}" + ) + return said + + +def restore( + workspace: Path, + kept: list[Scenario], + *, + tools: set[str], + sub_goals: set[str], +) -> tuple[dict[str, dict[str, Any]], dict[str, str]]: + """Put each dropped scenario whose capabilities are all back into ``kept`` under its own key, + from its last content: ({key: its record} for those restored, {key: why not} for those that + could not be put back). + + ``kept`` is changed in place; the caller writes it. A restored record loses an earlier failed + attempt's reason, so the end of this rebuild decides it again. + """ + records = load(workspace) + keys = {one.scenario_key for one in kept} + names = {one.name: one.scenario_key for one in kept} + restored: dict[str, dict[str, Any]] = {} + refused: dict[str, str] = {} + for key in restorable(records, tools, sub_goals): + if key in keys: + continue + try: + scenario = Scenario.model_validate(records[key]["content"]) + except ValueError as exc: + refused[key] = f"its last content no longer reads as a scenario: {exc}"[ + :500 + ] + continue + scenario.scenario_key = key + if scenario.name in names: + refused[key] = f"its name {scenario.name} is held by {names[scenario.name]}" + continue + kept.append(scenario) + keys.add(key) + names[scenario.name] = key + records[key].pop("restore_failed", None) + restored[key] = dict(records[key]) + for key, why in refused.items(): + records[key]["restore_failed"] = why + if restored or refused: + save(workspace, records) + return restored, refused + + +def finish_restore( + workspace: Path, + restored: Mapping[str, Mapping[str, Any]], + problems: list[dict[str, Any]], + present: set[str], +) -> tuple[list[dict[str, str]], list[dict[str, str]], list[dict[str, Any]]]: + """Settle what this rebuild restored: (restored, failed, the gate's other problems). + + ``restored`` is each restored scenario's record as ``restore`` returned it, so settling again + after a later report decides from the same records. A restored scenario the gate proved, or + left for the live runtime to prove as it leaves any other, is in the suite again and its record + goes. One with a problem, or no longer in the suite, stays dropped with the reason; its problem + is reported as a failed restore instead of failing the rebuild. The caller removes the failed + ones from the suite. + """ + records = load(workspace) + why_by_key = { + str(one.get("item")): str(one.get("message") or "") + for one in problems + if one.get("kind") == "scenario" + } + done: list[dict[str, str]] = [] + failed: list[dict[str, str]] = [] + for key in sorted(restored): + record = dict(restored[key]) + if key in why_by_key: + why = "could not be proved again: " + why_by_key[key] + elif key not in present: + why = "dropped again in this rebuild" + else: + done.append({"key": key, "name": str(record.get("name") or key)}) + records.pop(key, None) + continue + records[key] = {**record, "restore_failed": why} + failed.append({"key": key, "why": why}) + save(workspace, records) + others = [ + one + for one in problems + if not (one.get("kind") == "scenario" and str(one.get("item")) in restored) + ] + return done, failed, others + + +def failed_restores(workspace: Path, keys: Iterable[str]) -> list[dict[str, str]]: + """The failed restore of each of ``keys`` that has one, as the result lists it.""" + records = load(workspace) + return [ + {"key": key, "why": str(records[key]["restore_failed"])} + for key in keys + if key in records and records[key].get("restore_failed") + ] + + +def submit_refusal( + workspace: Path, payload: Mapping[str, Any], present: set[str] +) -> str: + """Why a rebuild may not submit a new scenario under a dropped key, or "". + + A dropped key comes back only with its own scenario, once what it needs is back; a restored + one is already in the suite, so submitting it again is an edit and is allowed. + """ + name = str(payload.get("name") or "").strip() + key = str(payload.get("scenario_key") or "").strip() or _slug(name) + if key in present: + return "" + record = load(workspace).get(key) + if record is None: + return "" + return ( + f"Not kept: {key} was dropped ({record.get('reason') or 'a capability it needs is gone'}). " + "It comes back with its own content once what it needs is back; give a new scenario " + "a new name." + ) + + +def restore_note(workspace: Path, contract: Any, sub_goals: set[str]) -> str: + """The rebuild message's line on the scenarios this rebuild restores, or "".""" + tools = contract.tool_names() if contract is not None else set() + records = load(workspace) + keys = restorable(records, tools, sub_goals) + if not keys: + return "" + changes = [] + for key in keys: + try: + scenario = Scenario.model_validate(records[key]["content"]) + except ValueError: + continue + said = argument_changes(scenario, contract) + if said: + changes.append(f"{key}: " + "; ".join(said)) + return ( + "\n\nRestored, because what they need is back (same key, their last content): " + + ", ".join(keys) + + ". Keep each as it was; change a step only where its tool or arguments changed" + + (" (" + " | ".join(changes) + ")" if changes else "") + + ", then prove them like any other scenario. One that cannot be proved goes back to " + "dropped and is listed with the reason; never drop or rename one to pass." + ) diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index b873cf87b..8441e5af5 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -1393,19 +1393,37 @@ def _nudge_note(self) -> str: keys = [str(one) for one in self._nudge.get("user_draft_keys") or []] if self._nudge.get("stage") != SUITE: return _NUDGE_NOTE + _ENVIRONMENT_NUDGE_NOTE + restored = self._restore_note() if not keys: return ( _NUDGE_NOTE + "\n\nNo scenario is being edited by the person now: no list of edits in " "an earlier message applies any more, so repair every scenario this update " "breaks." + + restored ) return ( _NUDGE_NOTE + "\n\nScenarios the person is still editing (keep what they wrote; only carry " "over a renamed tool or argument): " + ", ".join(keys) + + restored ) + def _restore_note(self) -> str: + """The line on dropped scenarios this suite rebuild restores, or "".""" + from .catalogue import load_catalogue + from .dropped import restore_note + from .understand import load as load_contract + + try: + return restore_note( + self.workspace, + load_contract(self.workspace), + load_catalogue(self.workspace).names(), + ) + except (OSError, ValueError): + return "" + def _report_again_if_written(self) -> None: """A write after the last report_update runs the gate again over what lands.""" if self._nudge is None or not self._nudge_dirty or not self._nudge_reported: diff --git a/src/fi/alk/harness/nudge.py b/src/fi/alk/harness/nudge.py index b4e0f3683..cc4331815 100644 --- a/src/fi/alk/harness/nudge.py +++ b/src/fi/alk/harness/nudge.py @@ -18,6 +18,7 @@ from pydantic import ValidationError +from . import dropped as drops from .amend import _persist, drop_tool_references from .backends import FILE_TOOLS, SessionSpec, ToolServer, ToolSpec, tool, tool_server from .builder import _err, _ok, scenario_problems @@ -637,8 +638,14 @@ def write_result( warnings: list[str], not_checked: list[str], left_out: list[dict], + restored: list[dict] | None = None, + restore_failed: list[dict] | None = None, ) -> None: - """Write the result the platform lands; the last write wins.""" + """Write the result the platform lands; the last write wins. + + ``restored`` are the dropped scenarios this rebuild put back and proved; ``restore_failed`` + those it could not, each with why. + """ notes = _cut(notes, _NOTES_MAX) used = _size(notes) + 1024 listed: dict[str, list[Any]] = {} @@ -648,6 +655,8 @@ def write_result( ("warnings", warnings), ("not_checked", not_checked), ("left_out", left_out), + ("restored", restored or []), + ("restore_failed", restore_failed or []), ): listed[name] = [] for item in items[:_LISTED_MAX]: @@ -672,6 +681,8 @@ def write_result( "not_checked": listed["not_checked"], }, "left_out": listed["left_out"], + "restored": listed["restored"], + "restore_failed": listed["restore_failed"], } target = Path(workspace) / NUDGE_RESULT temporary = target.with_name(target.name + ".tmp") @@ -691,9 +702,16 @@ def report_update( started_with: set[str] | None = None, world: Mapping[str, Any] | None = None, invariants: Mapping[str, str] | None = None, + before: Mapping[str, Mapping[str, Any]] | None = None, + restored: Mapping[str, Mapping[str, Any]] | None = None, + unrestorable: list[str] | None = None, ) -> dict[str, Any]: """Run the stage gate and write the result file. ``nudge`` is the command payload. + ``before`` is each scenario's content when a suite turn began: one it no longer holds because a + capability it needs is gone is recorded as dropped. ``restored`` are the dropped scenarios the + turn put back, settled here; ``unrestorable`` those it could not put back. + ``world`` is the turn's world tracking: world changes not yet saved fail the gate. ``started_with`` is what the piece held when the turn began: its scenario keys (suite) or its contract's tools (environment). @@ -704,6 +722,8 @@ def report_update( stage = str(nudge.get("stage") or "") warnings: list[str] = [] left_out: list[dict] = [] + restored_rows: list[dict] = [] + restore_failed: list[dict] = [] if stage == ENVIRONMENT: problems, not_checked = environment_gate(workspace) if "world" not in not_checked: @@ -756,6 +776,15 @@ def report_update( started_with=started_with, new_sub_goals=list(nudge.get("new_sub_goals") or []), ) + problems, left_out, restored_rows, restore_failed = _settle_drops( + workspace, + nudge, + problems, + left_out, + before=before, + restored=restored or {}, + unrestorable=unrestorable or [], + ) write_result( workspace, nudge_id=str(nudge.get("nudge_id") or ""), @@ -766,6 +795,8 @@ def report_update( warnings=warnings, not_checked=not_checked, left_out=left_out, + restored=restored_rows, + restore_failed=restore_failed, ) return { "passed": not problems, @@ -773,9 +804,65 @@ def report_update( "warnings": warnings, "not_checked": not_checked, "left_out": left_out, + "restored": restored_rows, + "restore_failed": restore_failed, } +def _settle_drops( + workspace: Path, + nudge: Mapping[str, Any], + problems: list[dict], + left_out: list[dict], + *, + before: Mapping[str, Mapping[str, Any]] | None, + restored: Mapping[str, Mapping[str, Any]], + unrestorable: list[str], +) -> tuple[list[dict], list[dict], list[dict], list[dict]]: + """Settle a suite turn's restores and record its drops for a missing capability: + (problems, left_out, restored, restore_failed). + + A restored scenario that did not prove leaves the suite again; its problem becomes a failed + restore rather than failing the rebuild. + """ + present = {one.scenario_key for one in load_scenarios(workspace)} + done: list[dict] = [] + failed: list[dict] = [] + if restored: + done, failed, problems = drops.finish_restore( + workspace, restored, problems, present + ) + out = {one["key"] for one in failed} + if out & present: + catalogue = load_catalogue(workspace) + write_scenarios( + [one for one in load_scenarios(workspace) if one.scenario_key not in out], + workspace, + catalogue, + ) + present -= out + failed += drops.failed_restores(workspace, unrestorable) + if before is not None: + labels = nudge.get("base_labels") or {} + recorded = drops.record_drops( + workspace, + before, + present, + tools=_contract_tools(workspace), + sub_goals=load_catalogue(workspace).names(), + agent_version=str(nudge.get("agent_label") or ""), + suite_version=str(labels.get("suite") or "") if isinstance(labels, Mapping) else "", + ) + left_out = [ + {**one, "why": _cut(recorded[one["key"]]["reason"])} + if one.get("key") in recorded + else one + for one in left_out + ] + failed = [{"key": one["key"], "why": _cut(one["why"])} for one in failed] + return problems, left_out, done, failed + + def rename_refusal(workspace: Path, payload: dict[str, Any]) -> str: """Why a submitted scenario renames one already in the suite, or "" when it does not.""" name = str(payload.get("name") or "").strip() @@ -821,7 +908,11 @@ async def aimed(args: dict[str, Any], _aim=aim) -> dict[str, Any]: submit = one.handler async def guarded(args: dict[str, Any], _submit=submit) -> dict[str, Any]: - said = rename_refusal(workspace, args) + said = rename_refusal(workspace, args) or drops.submit_refusal( + workspace, + args, + {one.scenario_key for one in load_scenarios(workspace)}, + ) if said: return _err(said) return await _submit(args) @@ -1007,6 +1098,19 @@ def nudge_server( if stage == ENVIRONMENT else None ) + before: dict[str, dict[str, Any]] | None = None + restored: dict[str, dict[str, Any]] = {} + unrestorable: list[str] = [] + if stage == SUITE: + suite = kept if kept is not None else load_scenarios(workspace) + before = {one.scenario_key: one.model_dump(mode="json") for one in suite} + held = catalogue if catalogue is not None else load_catalogue(workspace) + restored, refused = drops.restore( + workspace, suite, tools=_contract_tools(workspace), sub_goals=held.names() + ) + unrestorable = sorted(refused) + if restored: + write_scenarios(suite, workspace, held) @tool( "retire_sub_goal", @@ -1040,8 +1144,15 @@ async def report(args: dict[str, Any]) -> dict[str, Any]: "started_with": started_with, "world": world, "invariants": invariants, + "before": before, + "restored": restored, + "unrestorable": unrestorable, } result = report_update(workspace, nudge=nudge, **done) + if kept is not None and result["restore_failed"]: + # Saving the session's list again must not bring a failed restore back. + out = {one["key"] for one in result["restore_failed"]} + kept[:] = [one for one in kept if one.scenario_key not in out] if reported is not None: reported.update(done) said = json.dumps(result, indent=1) diff --git a/src/fi/alk/harness/skills/builder/topics/agent-update.md b/src/fi/alk/harness/skills/builder/topics/agent-update.md index 01b8f4c6e..093ccd9d4 100644 --- a/src/fi/alk/harness/skills/builder/topics/agent-update.md +++ b/src/fi/alk/harness/skills/builder/topics/agent-update.md @@ -51,6 +51,13 @@ decided in the notes. - Every sub-goal the environment added must be named by at least one scenario; one scenario may cover several. - Drop a sub-goal from scenarios with `retire_sub_goal` only after the environment retired it. +- A scenario that needs a tool or sub-goal the environment no longer has is dropped, not rewritten + to do without it. `report_update` records why (the capability, the agent version) and keeps its + key and last content, so it comes back on its own when the agent has the capability again. +- Scenarios the message lists as restored are already back in the suite under their own key. Keep + each as it was and change a step only where its tool or arguments changed, then prove it. One + that cannot be proved goes back to dropped and is reported; never drop, rename or rewrite one to + pass, and never write a new scenario under a dropped key. - Prove everything with `prove_scenarios`. Scenarios the person is still editing are listed at the end of the message. Only this message's list counts: a scenario an earlier message listed is no longer being edited and is repaired like any other. In a listed scenario keep what the diff --git a/tests/harness/test_dropped_scenarios.py b/tests/harness/test_dropped_scenarios.py new file mode 100644 index 000000000..b0c385c62 --- /dev/null +++ b/tests/harness/test_dropped_scenarios.py @@ -0,0 +1,345 @@ +"""Scenarios a rebuild drops for a removed capability keep the reason and come back with it. + +The fixture walks one agent through v5, which removes `cancel_booking`, and v7, which brings it +back. Proving is stubbed: these tests pin what is recorded, restored and reported, not the prover. +""" + +import asyncio +import json +from pathlib import Path + +import pytest + +from fi.alk.harness import dropped, nudge +from fi.alk.harness.catalogue import Catalogue, SubGoal, load_catalogue, save_catalogue +from fi.alk.harness.contract import AgentContract, ToolSpec +from fi.alk.harness.prove import Proof +from fi.alk.harness.scenario import Scenario +from fi.alk.harness.scenario_tools import load_scenarios, write_scenarios + +CANCELS = ("cancels-a-booking", "cancels-after-a-change", "cancels-the-only-booking") +_CHECK = "def check(world, calls):\n return None\n" + + +def _contract( + root: Path, *, cancel: bool, cancel_args: tuple[str, ...] = ("booking_id",) +) -> None: + tools = [ + ToolSpec(name="find_booking", args=["name"]), + ToolSpec(name="book", args=["slot"]), + ] + if cancel: + tools.append(ToolSpec(name="cancel_booking", args=list(cancel_args))) + (root / "contract.json").write_text( + AgentContract(agent="bookings", tools=tools).model_dump_json(), encoding="utf-8" + ) + + +def _catalogue(root: Path, *, cancel: bool) -> Catalogue: + goals = [SubGoal(name="booked", what="a booking was made", check=_CHECK)] + if cancel: + goals.append( + SubGoal(name="booking-cancelled", what="the booking is gone", check=_CHECK) + ) + catalogue = Catalogue(sub_goals=goals) + save_catalogue(catalogue, root) + return catalogue + + +def _scenario(name: str, *, cancels: bool) -> Scenario: + last = ( + {"tool": "cancel_booking", "arguments": {"booking_id": "b-1"}} + if cancels + else {"tool": "book", "arguments": {"slot": "monday-10"}} + ) + return Scenario.model_validate( + { + "name": name, + "use_case": "manage a booking", + "instruction": f"{name}: do what the caller asks.", + "tests": f"whether the agent handles {name}", + "solution": [{"tool": "find_booking", "arguments": {"name": "Ana"}}, last], + "sub_goals": ["booking-cancelled"] if cancels else ["booked"], + } + ) + + +def _v4(root: Path) -> Path: + """The suite as agent v4 had it: one booking scenario and three that cancel.""" + _contract(root, cancel=True) + catalogue = _catalogue(root, cancel=True) + suite = [_scenario("books-a-slot", cancels=False)] + [ + _scenario(name, cancels=True) for name in CANCELS + ] + write_scenarios(suite, root, catalogue) + return root + + +def _environment(root: Path, *, cancel: bool, **contract) -> None: + """What the environment rebuild for that agent version leaves behind.""" + _contract(root, cancel=cancel, **contract) + _catalogue(root, cancel=cancel) + + +def _call(server, name: str, args: dict) -> dict: + handler = next(one.handler for one in server.tools if one.name == name) + return asyncio.run(handler(args)) + + +def _suite_turn(root: Path, agent: str, suite: str): + catalogue = load_catalogue(root) + kept = load_scenarios(root) + server = nudge.nudge_server( + root, + { + "nudge_id": f"n-{agent}", + "stage": "suite", + "agent_label": agent, + "base_labels": {"agent": agent, "environment": "v2", "suite": suite}, + }, + catalogue=catalogue, + kept=kept, + ) + return server, kept, catalogue + + +def _result(root: Path) -> dict: + return json.loads((root / nudge.NUDGE_RESULT).read_text(encoding="utf-8")) + + +@pytest.fixture(autouse=True) +def _proved(monkeypatch): + monkeypatch.setattr( + nudge, "prove", lambda *_args: Proof(ready=True, solvable=True, vacuous=False) + ) + + +def _drop_cancels(root: Path) -> Path: + """v5 removes `cancel_booking`; its suite turn drops the three cancel scenarios.""" + _environment(root, cancel=False) + server, kept, catalogue = _suite_turn(root, "v5", "v3") + kept[:] = [one for one in kept if one.name not in CANCELS] + write_scenarios(kept, root, catalogue) + said = _call( + server, "report_update", {"changed": True, "notes": "dropped the cancels"} + ) + assert not said.get("is_error"), said + return root + + +# Done when 1: v5 drops the three cancel scenarios and each records why. + + +def test_v5_drops_the_cancel_scenarios_and_records_capability_agent_and_reason( + tmp_path, +): + root = _drop_cancels(_v4(tmp_path)) + + records = dropped.load(root) + assert sorted(records) == sorted(CANCELS) + for key in CANCELS: + record = records[key] + assert record["capabilities"] == { + "tools": ["cancel_booking", "find_booking"], + "sub_goals": ["booking-cancelled"], + } + assert record["missing"] == { + "tools": ["cancel_booking"], + "sub_goals": ["booking-cancelled"], + } + assert record["agent_version"] == "v5" + assert record["reason"].startswith("agent v5 no longer has `cancel_booking`") + assert record["last_suite_version"] == "v3" + assert record["content"]["scenario_key"] == key + left_out = {one["key"]: one["why"] for one in _result(root)["left_out"]} + assert set(left_out) == set(CANCELS) + assert all( + why.startswith("agent v5 no longer has `cancel_booking`") + for why in left_out.values() + ) + assert _result(root)["checks"]["passed"] is True + + +def test_a_scenario_dropped_for_another_reason_is_not_recorded(tmp_path): + root = _v4(tmp_path) + server, kept, catalogue = _suite_turn(root, "v5", "v3") + kept[:] = [one for one in kept if one.name != "books-a-slot"] + write_scenarios(kept, root, catalogue) + + _call(server, "report_update", {"changed": True, "notes": "dropped a duplicate"}) + + assert dropped.load(root) == {} + assert not (root / dropped.DROPPED).exists() + + +def test_a_rebuild_without_the_capability_back_restores_nothing(tmp_path): + root = _drop_cancels(_v4(tmp_path)) + server, kept, _ = _suite_turn(root, "v6", "v4") + + _call(server, "report_update", {"changed": False, "notes": "nothing to change"}) + + assert {one.name for one in load_scenarios(root)} == {"books-a-slot"} + assert sorted(dropped.load(root)) == sorted(CANCELS) + assert _result(root)["restored"] == [] + + +# Done when 2: v7 brings `cancel_booking` back and the rebuild restores the three. + + +def test_v7_restores_the_cancel_scenarios_with_their_keys_and_last_content(tmp_path): + root = _drop_cancels(_v4(tmp_path)) + last = {key: dropped.load(root)[key]["content"] for key in CANCELS} + _environment(root, cancel=True) + + server, kept, _ = _suite_turn(root, "v7", "v4") + + on_disk = {one.scenario_key: one for one in load_scenarios(root)} + assert set(CANCELS) <= set(on_disk) + assert set(CANCELS) <= {one.scenario_key for one in kept} + for key in CANCELS: + assert on_disk[key].model_dump(mode="json") == last[key] + + said = _call( + server, "report_update", {"changed": True, "notes": "restored the cancels"} + ) + + assert not said.get("is_error"), said + result = _result(root) + assert [one["key"] for one in result["restored"]] == sorted(CANCELS) + assert result["restore_failed"] == [] and result["left_out"] == [] + assert dropped.load(root) == {} + + +def test_restored_scenarios_are_proved_like_any_other(tmp_path, monkeypatch): + root = _drop_cancels(_v4(tmp_path)) + _environment(root, cancel=True) + proved: list[str] = [] + + def prove(scenario, *_args): + proved.append(scenario.scenario_key) + return Proof(ready=True, solvable=True, vacuous=False) + + monkeypatch.setattr(nudge, "prove", prove) + server, _, _ = _suite_turn(root, "v7", "v4") + _call(server, "report_update", {"changed": True, "notes": "restored"}) + + assert set(CANCELS) <= set(proved) + + +def test_a_scenario_is_restored_only_once_every_capability_it_needed_is_back(tmp_path): + root = _drop_cancels(_v4(tmp_path)) + # The tool is back, the sub-goal is not. + _contract(root, cancel=True) + _catalogue(root, cancel=False) + + server, _, _ = _suite_turn(root, "v7", "v4") + _call(server, "report_update", {"changed": False, "notes": "half back"}) + + assert {one.name for one in load_scenarios(root)} == {"books-a-slot"} + assert sorted(dropped.load(root)) == sorted(CANCELS) + + +def test_a_settled_restore_is_still_listed_when_report_update_runs_again(tmp_path): + root = _drop_cancels(_v4(tmp_path)) + _environment(root, cancel=True) + server, _, _ = _suite_turn(root, "v7", "v4") + + _call(server, "report_update", {"changed": True, "notes": "first"}) + _call(server, "report_update", {"changed": True, "notes": "again"}) + + assert [one["key"] for one in _result(root)["restored"]] == sorted(CANCELS) + + +def test_the_message_names_the_restored_scenarios_and_a_changed_argument(tmp_path): + root = _drop_cancels(_v4(tmp_path)) + _environment(root, cancel=True, cancel_args=("reference",)) + contract = AgentContract.model_validate_json((root / "contract.json").read_text()) + + said = dropped.restore_note(root, contract, load_catalogue(root).names()) + + for key in CANCELS: + assert key in said + assert "cancel_booking no longer takes booking_id" in said + + +# Requirement 4: a restore that cannot be proved again stays dropped and is reported. + + +def test_a_restore_that_cannot_be_proved_stays_dropped_and_is_listed_with_why( + tmp_path, monkeypatch +): + root = _drop_cancels(_v4(tmp_path)) + _environment(root, cancel=True) + + def prove(scenario, *_args): + if scenario.scenario_key == "cancels-the-only-booking": + return Proof(ready=True, solvable=False, vacuous=False) + return Proof(ready=True, solvable=True, vacuous=False) + + monkeypatch.setattr(nudge, "prove", prove) + server, kept, _ = _suite_turn(root, "v7", "v4") + said = _call(server, "report_update", {"changed": True, "notes": "restored two"}) + + assert not said.get("is_error"), said + result = _result(root) + assert result["checks"]["passed"] is True + assert [one["key"] for one in result["restored"]] == [ + "cancels-a-booking", + "cancels-after-a-change", + ] + assert [one["key"] for one in result["restore_failed"]] == [ + "cancels-the-only-booking" + ] + assert result["restore_failed"][0]["why"].startswith("could not be proved again") + assert "cancels-the-only-booking" not in { + one.scenario_key for one in load_scenarios(root) + } + assert "cancels-the-only-booking" not in {one.scenario_key for one in kept} + record = dropped.load(root)["cancels-the-only-booking"] + assert record["restore_failed"].startswith("could not be proved again") + assert record["reason"].startswith("agent v5 no longer has") + + +def test_a_restored_scenario_dropped_again_in_the_turn_is_a_failed_restore(tmp_path): + root = _drop_cancels(_v4(tmp_path)) + _environment(root, cancel=True) + server, kept, catalogue = _suite_turn(root, "v7", "v4") + kept[:] = [one for one in kept if one.scenario_key != "cancels-a-booking"] + write_scenarios(kept, root, catalogue) + + _call(server, "report_update", {"changed": True, "notes": "dropped one again"}) + + failed = _result(root)["restore_failed"] + assert failed == [ + {"key": "cancels-a-booking", "why": "dropped again in this rebuild"} + ] + assert "cancels-a-booking" in dropped.load(root) + + +# Requirement 2: the key stays reserved; only the restore brings it back. + + +def test_a_rebuild_may_not_write_a_new_scenario_under_a_dropped_key(tmp_path): + root = _drop_cancels(_v4(tmp_path)) + + said = dropped.submit_refusal( + root, + {"name": "cancels-a-booking"}, + {one.scenario_key for one in load_scenarios(root)}, + ) + + assert said.startswith("Not kept: cancels-a-booking was dropped") + + +def test_editing_a_restored_scenario_is_not_refused(tmp_path): + root = _drop_cancels(_v4(tmp_path)) + _environment(root, cancel=True) + _suite_turn(root, "v7", "v4") + + said = dropped.submit_refusal( + root, + {"name": "cancels-a-booking"}, + {one.scenario_key for one in load_scenarios(root)}, + ) + + assert said == "" From 20d549a7e4b8c4bac32f36e8b7f5ac7540c59dc1 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sun, 15 Nov 2026 06:46:55 +0530 Subject: [PATCH 2/3] test(harness): with no drops left dropped.json is removed and the scenario digest never reads it --- tests/harness/test_dropped_scenarios.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/harness/test_dropped_scenarios.py b/tests/harness/test_dropped_scenarios.py index b0c385c62..7718c0528 100644 --- a/tests/harness/test_dropped_scenarios.py +++ b/tests/harness/test_dropped_scenarios.py @@ -183,6 +183,21 @@ def test_a_rebuild_without_the_capability_back_restores_nothing(tmp_path): assert _result(root)["restored"] == [] +def test_drops_sit_beside_the_scenarios_and_leave_no_file_once_none_remain(tmp_path): + """The platform hashes a suite with no dropped.json exactly as before drops existed.""" + from fi.alk.harness.authoring_runtime_validation import _artifact_digest + + root = _drop_cancels(_v4(tmp_path)) + with_drops = _artifact_digest(root / "scenarios") + records = dropped.load(root) + dropped.save(root, {}) + + assert not (root / dropped.DROPPED).exists() + assert _artifact_digest(root / "scenarios") == with_drops + dropped.save(root, records) + assert (root / dropped.DROPPED).exists() + + # Done when 2: v7 brings `cancel_booking` back and the rebuild restores the three. From 6787e2237872d764be3ba8e57c8c47bbea6a9b46 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sun, 15 Nov 2026 09:24:44 +0530 Subject: [PATCH 3/3] fix(harness): a rebuild adds back a tool the agent defines again with declare_tool; fix_tool and adopt_tool say so instead of claiming the agent lacks it --- src/fi/alk/harness/amend.py | 10 ++++++---- .../alk/harness/skills/builder/topics/agent-update.md | 3 +++ tests/test_harness.py | 11 ++++++++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/fi/alk/harness/amend.py b/src/fi/alk/harness/amend.py index 841568394..e6b080d14 100644 --- a/src/fi/alk/harness/amend.py +++ b/src/fi/alk/harness/amend.py @@ -47,8 +47,9 @@ def widen( spec = next((tool for tool in contract.tools if tool.name == tool_name), None) if spec is None: return False, ( - f"{tool_name!r} is not a tool this agent has. It has: " - f"{', '.join(sorted(contract.tool_names()))}" + f"{tool_name!r} is not in the contract. It has: " + f"{', '.join(sorted(contract.tool_names()))}. To add a tool the agent's source defines " + "(one it gained, or one an earlier version removed), use declare_tool." ) if argument not in spec.args: return False, ( @@ -242,8 +243,9 @@ def fix_tool( spec = next((tool for tool in contract.tools if tool.name == tool_name), None) if spec is None: return False, ( - f"{tool_name!r} is not a tool this agent has. It has: " - f"{', '.join(sorted(contract.tool_names()))}" + f"{tool_name!r} is not in the contract. It has: " + f"{', '.join(sorted(contract.tool_names()))}. To add a tool the agent's source defines " + "(one it gained, or one an earlier version removed), use declare_tool." ) if remove: diff --git a/src/fi/alk/harness/skills/builder/topics/agent-update.md b/src/fi/alk/harness/skills/builder/topics/agent-update.md index 093ccd9d4..233d2082b 100644 --- a/src/fi/alk/harness/skills/builder/topics/agent-update.md +++ b/src/fi/alk/harness/skills/builder/topics/agent-update.md @@ -25,6 +25,9 @@ decided in the notes. rebuilt in its own turn after this one lands. - There is no running world and no credentials here. Do not try to run the agent's tools; the checks are static. +- A tool the agent's source defines that the contract lacks (one it gained, or one an earlier version + removed and this one has again) is added with `declare_tool`, citing the line that defines it. + `fix_tool` and `adopt_tool` only change tools the contract already has. - For every tool the agent gained, add a sub-goal that checks what the tool lets the person using the agent do, so the suite stage adds a scenario for it. `report_update` warns about a new tool no sub-goal names; check that one covers it, by name or by its effect. diff --git a/tests/test_harness.py b/tests/test_harness.py index 9c8214b3c..83c017acf 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -1939,10 +1939,19 @@ def test_an_amendment_is_recorded_rather_than_blended_in(tmp_path): assert "mango_smoothie" in recorded[0] and "this morning" in recorded[0] +def test_fixing_a_tool_the_contract_lacks_points_at_declare_tool(tmp_path): + from fi.alk.harness.amend import fix_tool + + contract = _written_contract(tmp_path) + done, said = fix_tool(contract, tmp_path, tool_name="apply_discount", why="the agent has it again") + + assert not done and "is not in the contract" in said and "declare_tool" in said + + @pytest.mark.parametrize( "overrides,expected", [ - ({"tool_name": "nope"}, "is not a tool this agent has"), + ({"tool_name": "nope"}, "is not in the contract"), ({"argument": "colour"}, "takes no argument"), ({"why": " "}, "say why"), ({"values": ["big_mac"]}, "already accepts"),