From 7f1345319217db9a323fbe3dfa2bbb52160bc0b7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 22 Oct 2026 18:34:53 +0530 Subject: [PATCH 01/15] fix(harness): keep a scenario's key when it is renamed or replaced --- src/fi/alk/harness/scenario_tools.py | 17 +++++++++ src/fi/alk/harness/skills/builder/SKILL.md | 2 ++ tests/test_harness.py | 41 ++++++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index fbf3f9b95..af656e7c7 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -845,6 +845,18 @@ def accept_scenario( world.close() return _err(said) + # The scenario this one replaces: the one whose key it carries, else the one with its name. It + # keeps that key, so a rename is the same scenario under a new name, not a new one. + named = str(payload.get("scenario_key") or "").strip() + previous = next((one for one in kept if named and one.scenario_key == named), None) or next( + (one for one in kept if one.name == scenario.name), None + ) + if previous is not None and previous.name != scenario.name: + if any(one.name == scenario.name for one in kept): + return _err(f"Not kept. Another scenario is already named {scenario.name}; choose another name.") + kept[:] = [one for one in kept if one is not previous] + if previous is not None: + scenario.scenario_key = previous.scenario_key replaced = any(one.name == scenario.name for one in kept) if replaced and rename_on_collision: taken = {one.name for one in kept} @@ -1296,6 +1308,11 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "It becomes this scenario's folder name. It says what this scenario tests and " "carries, without a counter or an invented code, PIN or reference.", }, + "scenario_key": { + "type": "string", + "description": "Only when editing a scenario already kept: its scenario_key, read " + "from it. With a new name this renames it, keeping its number and its history.", + }, "use_case": { "type": "string", "description": "Which of the agent's use cases this belongs to.", diff --git a/src/fi/alk/harness/skills/builder/SKILL.md b/src/fi/alk/harness/skills/builder/SKILL.md index 08c7ce33b..d9ee3a0b3 100644 --- a/src/fi/alk/harness/skills/builder/SKILL.md +++ b/src/fi/alk/harness/skills/builder/SKILL.md @@ -139,6 +139,8 @@ what the gates check. Then: - **Edit one**: read it in full first, change only what was asked, and `submit_scenario` it under the same name with every other field kept exactly. That replaces it, keeps its number, and proves it. +- **Rename one**: read it in full, then `submit_scenario` it with its `scenario_key`, the new name and every + other field kept exactly. It stays the same scenario, with its number and its run history. - **Add**: `aim_for` the new total first, then write and submit each one. For more than a handful, brief `scenario_writer` sub-agents in one message. When the person asks for a level of an axis (no background noise, only one kind of caller), pass it in `aim_for`'s `asked_for` so it is not held to diff --git a/tests/test_harness.py b/tests/test_harness.py index d1fddad83..c78508132 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -2678,6 +2678,47 @@ def test_a_scenario_is_proved_before_it_is_kept(tmp_path): assert (root / "scenarios" / "adds-a-big-mac" / "scenario.json").exists() +def test_a_renamed_scenario_keeps_its_key_and_replaces_the_old_one(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + key = kept[0].scenario_key + said = accept_scenario( + _delta(name="orders-one-big-mac", scenario_key=key), world_root=root, catalogue=catalogue, kept=kept + ) + assert not said.get("is_error"), said + assert [(one.name, one.scenario_key) for one in kept] == [("orders-one-big-mac", key)] + assert (root / "scenarios" / "orders-one-big-mac" / "scenario.json").exists() + assert not (root / "scenarios" / "adds-a-big-mac").exists() + + +def test_an_edit_under_the_same_name_keeps_the_key_it_was_kept_with(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(scenario_key="kept-key"), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario(_delta(instruction="Order one Big Mac, please."), world_root=root, catalogue=catalogue, kept=kept) + assert [(one.name, one.scenario_key) for one in kept] == [("adds-a-big-mac", "kept-key")] + + +def test_a_rename_onto_another_scenario_s_name_is_refused(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario(_delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept) + key = kept[0].scenario_key + said = accept_scenario( + _delta(name="orders-again", scenario_key=key), world_root=root, catalogue=catalogue, kept=kept + ) + assert said.get("is_error") and "already named orders-again" in said["content"][0]["text"] + assert sorted(one.name for one in kept) == ["adds-a-big-mac", "orders-again"] + + def test_a_scenario_whose_solution_cannot_pass_its_own_checks_is_refused(tmp_path): """Either the scenario is impossible or the checks are wrong. Both have happened.""" from fi.alk.harness.scenario_tools import accept_scenario From 10f47bda6c393717008b546b6019b38de3a9219b Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 22 Oct 2026 18:58:03 +0530 Subject: [PATCH 02/15] fix(harness): rename a scenario without losing its key, and let a keyed rename pass its own duplicate checks --- src/fi/alk/harness/builder.py | 18 +++++++++- src/fi/alk/harness/hosted_chat_entrypoint.py | 1 + src/fi/alk/harness/scenario_tools.py | 17 +++++---- src/fi/alk/harness/skills/builder/SKILL.md | 3 +- tests/test_harness.py | 37 ++++++++++++++++++++ 5 files changed, 66 insertions(+), 10 deletions(-) diff --git a/src/fi/alk/harness/builder.py b/src/fi/alk/harness/builder.py index f87c50eb9..4aaf86c92 100644 --- a/src/fi/alk/harness/builder.py +++ b/src/fi/alk/harness/builder.py @@ -186,6 +186,22 @@ async def drop_scenarios(args: dict[str, Any]) -> dict[str, Any]: + (f". Not found: {', '.join(missing)}" if missing else "") ) + @tool( + "rename_scenario", + "Give a scenario a new name, by exact name. Only the name changes: it keeps its key, its " + "number and its run history, so it stays the same scenario.", + schema({"name": str, "new_name": str}, ["name", "new_name"]), + ) + async def rename_scenario(args: dict[str, Any]) -> dict[str, Any]: + name, new = str(args.get("name") or "").strip(), str(args.get("new_name") or "").strip() + if not any(one.name == name for one in kept): + return _err(f"{name} is not in the suite") + if not new or any(one.name == new for one in kept): + return _err(f"{new or 'that name'} is taken or empty; choose another") + kept[:] = [one.model_copy(update={"name": new}) if one.name == name else one for one in kept] + write_scenarios(kept, out, load_catalogue(out)) + return _ok(f"renamed {name} to {new}; same key, number and history") + @tool( "prove_scenarios", "Run the three gates (ready, solvable, not vacuous) again on existing scenarios by name, " @@ -314,7 +330,7 @@ async def declare_tool(args: dict[str, Any]) -> dict[str, Any]: return tool_server( name="suite", - tools=[drop_scenarios, prove_scenarios, check_health, impact_of, change_world, declare_tool], + tools=[drop_scenarios, rename_scenario, prove_scenarios, check_health, impact_of, change_world, declare_tool], ) diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 75bb0a545..cc11b382d 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -86,6 +86,7 @@ "submit_scenario", "drop_scenario", "drop_scenarios", + "rename_scenario", "save_scenarios", "add_sub_goal", "amend_contract", diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index af656e7c7..fa2bf1c80 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -1519,6 +1519,12 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] if problem: return _refuse(problem) + # A submission carrying a kept scenario's key replaces that one, under its new name if renamed, + # so the checks below never count it against itself. + key = str(args.get("scenario_key") or "").strip() + same = next( + (one.name for one in kept if key and one.scenario_key == key), str(args.get("name") or "") + ) # Cheap label checks run before the gates, since proving is expensive. strayed = _off_the_grid(args.get("coverage"), target.get("axes")) if strayed: @@ -1528,7 +1534,7 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] target.get("axes"), kept, cap(), - str(args.get("name") or ""), + same, target.get("asked"), ) if crowded_level: @@ -1537,15 +1543,13 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] short = _red_team_short(args, target.get("axes"), kept, cap()) if short: return _refuse(short) - twin = _already_in_the_suite( - args, kept, _first_names_on_disk(destination, str(args.get("name") or "")) - ) + twin = _already_in_the_suite({**args, "name": same}, kept, _first_names_on_disk(destination, same)) if twin: return _refuse(twin) if cap() and target.get("people") != "alike": # A replacement does not count against itself. crowded = crowded_field( - [one for one in kept if one.name != str(args.get("name") or "")], + [one for one in kept if one.name != same], Scenario.model_validate(args).persona, cap(), ) if args.get("persona") else "" @@ -1561,8 +1565,7 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] if stranger: return _refuse(stranger[0]) if cap(): - named = str(args.get("name") or "").strip() - already = any(one.name == named for one in kept) + already = any(one.name == same for one in kept) if not already and len(kept) >= cap(): return _err( f"This is complete: {len(kept)} of {cap()} written. Do not write another. " diff --git a/src/fi/alk/harness/skills/builder/SKILL.md b/src/fi/alk/harness/skills/builder/SKILL.md index d9ee3a0b3..03cd04371 100644 --- a/src/fi/alk/harness/skills/builder/SKILL.md +++ b/src/fi/alk/harness/skills/builder/SKILL.md @@ -139,8 +139,7 @@ what the gates check. Then: - **Edit one**: read it in full first, change only what was asked, and `submit_scenario` it under the same name with every other field kept exactly. That replaces it, keeps its number, and proves it. -- **Rename one**: read it in full, then `submit_scenario` it with its `scenario_key`, the new name and every - other field kept exactly. It stays the same scenario, with its number and its run history. +- **Rename one**: `rename_scenario`. Only the name changes; it keeps its number and its run history. - **Add**: `aim_for` the new total first, then write and submit each one. For more than a handful, brief `scenario_writer` sub-agents in one message. When the person asks for a level of an axis (no background noise, only one kind of caller), pass it in `aim_for`'s `asked_for` so it is not held to diff --git a/tests/test_harness.py b/tests/test_harness.py index c78508132..9257b6385 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -7889,6 +7889,43 @@ def test_a_scenario_placed_nowhere_is_refused_once_a_grid_is_dealt(tmp_path): assert not _off_the_grid(None, {}) +def test_renaming_through_the_tool_is_not_refused_as_a_duplicate_of_itself(tmp_path): + import asyncio + + from fi.alk.harness.scenario_tools import scenario_tools + + root, _contract, _catalogue = _built_environment(tmp_path) + server, kept = scenario_tools(_contract, root, root, wanted=2) + submit = next(one for one in server.tools if one.name == "submit_scenario") + person = {"persona": {"name": "Asha Rao", "personality": "Friendly and cooperative", "communication_style": "Direct and concise", "initial_message": "Hi, I want a Big Mac.", "languages": ["English"]}} + said = asyncio.run(submit.handler(_delta(**person))) + assert not said.get("is_error"), said + key = kept[0].scenario_key + said = asyncio.run(submit.handler(_delta(name="orders-one-big-mac", scenario_key=key, **person))) + assert not said.get("is_error"), said + assert [(one.name, one.scenario_key) for one in kept] == [("orders-one-big-mac", key)] + + +def test_the_builder_renames_a_scenario_keeping_its_key(tmp_path): + import asyncio + + from fi.alk.harness.builder import suite_server + from fi.alk.harness.scenario_tools import accept_scenario, load_scenarios + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario(_delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept) + key = kept[0].scenario_key + rename = next(one for one in suite_server(root, kept).tools if one.name == "rename_scenario") + said = asyncio.run(rename.handler({"name": "adds-a-big-mac", "new_name": "orders-one-big-mac"})) + assert not said.get("is_error"), said + on_disk = {one.name: one.scenario_key for one in load_scenarios(root)} + assert on_disk == {"orders-one-big-mac": key, "orders-again": kept[1].scenario_key} + taken = asyncio.run(rename.handler({"name": "orders-one-big-mac", "new_name": "orders-again"})) + assert taken.get("is_error") + + def test_a_redeclared_grid_only_ever_grows(tmp_path): import asyncio From 162ecf7fb7f51aab3a383d30826fee5b2847ea66 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 15:55:20 +0530 Subject: [PATCH 03/15] style(harness): format the scenario rename changes --- src/fi/alk/harness/builder.py | 20 ++++++-- src/fi/alk/harness/scenario_tools.py | 17 ++++--- tests/test_harness.py | 72 ++++++++++++++++++++++------ 3 files changed, 85 insertions(+), 24 deletions(-) diff --git a/src/fi/alk/harness/builder.py b/src/fi/alk/harness/builder.py index b9fa496b1..0004c2acd 100644 --- a/src/fi/alk/harness/builder.py +++ b/src/fi/alk/harness/builder.py @@ -193,12 +193,18 @@ async def drop_scenarios(args: dict[str, Any]) -> dict[str, Any]: schema({"name": str, "new_name": str}, ["name", "new_name"]), ) async def rename_scenario(args: dict[str, Any]) -> dict[str, Any]: - name, new = str(args.get("name") or "").strip(), str(args.get("new_name") or "").strip() + name, new = ( + str(args.get("name") or "").strip(), + str(args.get("new_name") or "").strip(), + ) if not any(one.name == name for one in kept): return _err(f"{name} is not in the suite") if not new or any(one.name == new for one in kept): return _err(f"{new or 'that name'} is taken or empty; choose another") - kept[:] = [one.model_copy(update={"name": new}) if one.name == name else one for one in kept] + kept[:] = [ + one.model_copy(update={"name": new}) if one.name == name else one + for one in kept + ] write_scenarios(kept, out, load_catalogue(out)) return _ok(f"renamed {name} to {new}; same key, number and history") @@ -328,7 +334,15 @@ async def declare_tool(args: dict[str, Any]) -> dict[str, Any]: return tool_server( name="suite", - tools=[drop_scenarios, rename_scenario, prove_scenarios, check_health, impact_of, change_world, declare_tool], + tools=[ + drop_scenarios, + rename_scenario, + prove_scenarios, + check_health, + impact_of, + change_world, + declare_tool, + ], ) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 0ea021eee..f37934db0 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -951,12 +951,14 @@ def accept_scenario( # The scenario this one replaces: the one whose key it carries, else the one with its name. It # keeps that key, so a rename is the same scenario under a new name, not a new one. named = str(payload.get("scenario_key") or "").strip() - previous = next((one for one in kept if named and one.scenario_key == named), None) or next( - (one for one in kept if one.name == scenario.name), None - ) + previous = next( + (one for one in kept if named and one.scenario_key == named), None + ) or next((one for one in kept if one.name == scenario.name), None) if previous is not None and previous.name != scenario.name: if any(one.name == scenario.name for one in kept): - return _err(f"Not kept. Another scenario is already named {scenario.name}; choose another name.") + return _err( + f"Not kept. Another scenario is already named {scenario.name}; choose another name." + ) kept[:] = [one for one in kept if one is not previous] if previous is not None: scenario.scenario_key = previous.scenario_key @@ -1697,7 +1699,8 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] # so the checks below never count it against itself. key = str(args.get("scenario_key") or "").strip() same = next( - (one.name for one in kept if key and one.scenario_key == key), str(args.get("name") or "") + (one.name for one in kept if key and one.scenario_key == key), + str(args.get("name") or ""), ) # Cheap label checks run before the gates, since proving is expensive. strayed = _off_the_grid(args.get("coverage"), target.get("axes")) @@ -1717,7 +1720,9 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] short = _red_team_short(args, target.get("axes"), kept, cap()) if short: return _refuse(short) - twin = _already_in_the_suite({**args, "name": same}, kept, _first_names_on_disk(destination, same)) + twin = _already_in_the_suite( + {**args, "name": same}, kept, _first_names_on_disk(destination, same) + ) if twin: return _refuse(twin) if cap() and target.get("people") != "alike": diff --git a/tests/test_harness.py b/tests/test_harness.py index bfda29ce1..b3c7f07a8 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -2686,10 +2686,15 @@ def test_a_renamed_scenario_keeps_its_key_and_replaces_the_old_one(tmp_path): accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) key = kept[0].scenario_key said = accept_scenario( - _delta(name="orders-one-big-mac", scenario_key=key), world_root=root, catalogue=catalogue, kept=kept + _delta(name="orders-one-big-mac", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, ) assert not said.get("is_error"), said - assert [(one.name, one.scenario_key) for one in kept] == [("orders-one-big-mac", key)] + assert [(one.name, one.scenario_key) for one in kept] == [ + ("orders-one-big-mac", key) + ] assert (root / "scenarios" / "orders-one-big-mac" / "scenario.json").exists() assert not (root / "scenarios" / "adds-a-big-mac").exists() @@ -2699,9 +2704,18 @@ def test_an_edit_under_the_same_name_keeps_the_key_it_was_kept_with(tmp_path): root, _contract, catalogue = _built_environment(tmp_path) kept = [] - accept_scenario(_delta(scenario_key="kept-key"), world_root=root, catalogue=catalogue, kept=kept) - accept_scenario(_delta(instruction="Order one Big Mac, please."), world_root=root, catalogue=catalogue, kept=kept) - assert [(one.name, one.scenario_key) for one in kept] == [("adds-a-big-mac", "kept-key")] + accept_scenario( + _delta(scenario_key="kept-key"), world_root=root, catalogue=catalogue, kept=kept + ) + accept_scenario( + _delta(instruction="Order one Big Mac, please."), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + assert [(one.name, one.scenario_key) for one in kept] == [ + ("adds-a-big-mac", "kept-key") + ] def test_a_rename_onto_another_scenario_s_name_is_refused(tmp_path): @@ -2710,12 +2724,20 @@ def test_a_rename_onto_another_scenario_s_name_is_refused(tmp_path): root, _contract, catalogue = _built_environment(tmp_path) kept = [] accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) - accept_scenario(_delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario( + _delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept + ) key = kept[0].scenario_key said = accept_scenario( - _delta(name="orders-again", scenario_key=key), world_root=root, catalogue=catalogue, kept=kept + _delta(name="orders-again", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + assert ( + said.get("is_error") + and "already named orders-again" in said["content"][0]["text"] ) - assert said.get("is_error") and "already named orders-again" in said["content"][0]["text"] assert sorted(one.name for one in kept) == ["adds-a-big-mac", "orders-again"] @@ -7912,13 +7934,25 @@ def test_renaming_through_the_tool_is_not_refused_as_a_duplicate_of_itself(tmp_p root, _contract, _catalogue = _built_environment(tmp_path) server, kept = scenario_tools(_contract, root, root, wanted=2) submit = next(one for one in server.tools if one.name == "submit_scenario") - person = {"persona": {"name": "Asha Rao", "personality": "Friendly and cooperative", "communication_style": "Direct and concise", "initial_message": "Hi, I want a Big Mac.", "languages": ["English"]}} + person = { + "persona": { + "name": "Asha Rao", + "personality": "Friendly and cooperative", + "communication_style": "Direct and concise", + "initial_message": "Hi, I want a Big Mac.", + "languages": ["English"], + } + } said = asyncio.run(submit.handler(_delta(**person))) assert not said.get("is_error"), said key = kept[0].scenario_key - said = asyncio.run(submit.handler(_delta(name="orders-one-big-mac", scenario_key=key, **person))) + said = asyncio.run( + submit.handler(_delta(name="orders-one-big-mac", scenario_key=key, **person)) + ) assert not said.get("is_error"), said - assert [(one.name, one.scenario_key) for one in kept] == [("orders-one-big-mac", key)] + assert [(one.name, one.scenario_key) for one in kept] == [ + ("orders-one-big-mac", key) + ] def test_the_builder_renames_a_scenario_keeping_its_key(tmp_path): @@ -7930,14 +7964,22 @@ def test_the_builder_renames_a_scenario_keeping_its_key(tmp_path): root, _contract, catalogue = _built_environment(tmp_path) kept = [] accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) - accept_scenario(_delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario( + _delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept + ) key = kept[0].scenario_key - rename = next(one for one in suite_server(root, kept).tools if one.name == "rename_scenario") - said = asyncio.run(rename.handler({"name": "adds-a-big-mac", "new_name": "orders-one-big-mac"})) + rename = next( + one for one in suite_server(root, kept).tools if one.name == "rename_scenario" + ) + said = asyncio.run( + rename.handler({"name": "adds-a-big-mac", "new_name": "orders-one-big-mac"}) + ) assert not said.get("is_error"), said on_disk = {one.name: one.scenario_key for one in load_scenarios(root)} assert on_disk == {"orders-one-big-mac": key, "orders-again": kept[1].scenario_key} - taken = asyncio.run(rename.handler({"name": "orders-one-big-mac", "new_name": "orders-again"})) + taken = asyncio.run( + rename.handler({"name": "orders-one-big-mac", "new_name": "orders-again"}) + ) assert taken.get("is_error") From 1c01b09e146aabe4cbd8fc6432b50e33b1106e5a Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 16:10:28 +0530 Subject: [PATCH 04/15] refactor(harness): rename through submit_scenario instead of a separate tool --- src/fi/alk/harness/builder.py | 32 +------------------- src/fi/alk/harness/hosted_chat_entrypoint.py | 1 - src/fi/alk/harness/skills/builder/SKILL.md | 3 +- tests/test_harness.py | 28 ----------------- 4 files changed, 3 insertions(+), 61 deletions(-) diff --git a/src/fi/alk/harness/builder.py b/src/fi/alk/harness/builder.py index 0004c2acd..277e7a56b 100644 --- a/src/fi/alk/harness/builder.py +++ b/src/fi/alk/harness/builder.py @@ -186,28 +186,6 @@ async def drop_scenarios(args: dict[str, Any]) -> dict[str, Any]: + (f". Not found: {', '.join(missing)}" if missing else "") ) - @tool( - "rename_scenario", - "Give a scenario a new name, by exact name. Only the name changes: it keeps its key, its " - "number and its run history, so it stays the same scenario.", - schema({"name": str, "new_name": str}, ["name", "new_name"]), - ) - async def rename_scenario(args: dict[str, Any]) -> dict[str, Any]: - name, new = ( - str(args.get("name") or "").strip(), - str(args.get("new_name") or "").strip(), - ) - if not any(one.name == name for one in kept): - return _err(f"{name} is not in the suite") - if not new or any(one.name == new for one in kept): - return _err(f"{new or 'that name'} is taken or empty; choose another") - kept[:] = [ - one.model_copy(update={"name": new}) if one.name == name else one - for one in kept - ] - write_scenarios(kept, out, load_catalogue(out)) - return _ok(f"renamed {name} to {new}; same key, number and history") - @tool( "prove_scenarios", "Run the three gates (ready, solvable, not vacuous) again on existing scenarios by name, " @@ -334,15 +312,7 @@ async def declare_tool(args: dict[str, Any]) -> dict[str, Any]: return tool_server( name="suite", - tools=[ - drop_scenarios, - rename_scenario, - prove_scenarios, - check_health, - impact_of, - change_world, - declare_tool, - ], + tools=[drop_scenarios, prove_scenarios, check_health, impact_of, change_world, declare_tool], ) diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index dba795801..7e83b4450 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -91,7 +91,6 @@ "submit_scenario", "drop_scenario", "drop_scenarios", - "rename_scenario", "save_scenarios", "add_sub_goal", "amend_contract", diff --git a/src/fi/alk/harness/skills/builder/SKILL.md b/src/fi/alk/harness/skills/builder/SKILL.md index cb11e6873..0cad4bfa1 100644 --- a/src/fi/alk/harness/skills/builder/SKILL.md +++ b/src/fi/alk/harness/skills/builder/SKILL.md @@ -141,7 +141,8 @@ what the gates check. Then: field that follows from it (setup, ready check, solution, instruction, sub-goals, labels), and `submit_scenario` it under the same name, keeping every field the change does not touch. That replaces it, keeps its number, and proves it. -- **Rename one**: `rename_scenario`. Only the name changes; it keeps its number and its run history. +- **Rename one**: `submit_scenario` it with its `scenario_key` and the new name, every other field + unchanged. It keeps its number and its run history. - **Add**: `aim_for` the new total first, then write and submit each one. For more than a handful, brief `scenario_writer` sub-agents in one message. When the person asks for a level of an axis (no background noise, only one kind of caller), pass it in `aim_for`'s `asked_for` so it is not held to diff --git a/tests/test_harness.py b/tests/test_harness.py index b3c7f07a8..af9f054bb 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -7955,34 +7955,6 @@ def test_renaming_through_the_tool_is_not_refused_as_a_duplicate_of_itself(tmp_p ] -def test_the_builder_renames_a_scenario_keeping_its_key(tmp_path): - import asyncio - - from fi.alk.harness.builder import suite_server - from fi.alk.harness.scenario_tools import accept_scenario, load_scenarios - - root, _contract, catalogue = _built_environment(tmp_path) - kept = [] - accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) - accept_scenario( - _delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept - ) - key = kept[0].scenario_key - rename = next( - one for one in suite_server(root, kept).tools if one.name == "rename_scenario" - ) - said = asyncio.run( - rename.handler({"name": "adds-a-big-mac", "new_name": "orders-one-big-mac"}) - ) - assert not said.get("is_error"), said - on_disk = {one.name: one.scenario_key for one in load_scenarios(root)} - assert on_disk == {"orders-one-big-mac": key, "orders-again": kept[1].scenario_key} - taken = asyncio.run( - rename.handler({"name": "orders-one-big-mac", "new_name": "orders-again"}) - ) - assert taken.get("is_error") - - def test_a_redeclared_grid_only_ever_grows(tmp_path): import asyncio From 65d71bbb03ec9eb4787155428ce8562f430368d7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 16:26:18 +0530 Subject: [PATCH 05/15] fix(harness): keep scenario keys unique and resolve edits by key --- src/fi/alk/harness/hosted_chat_entrypoint.py | 14 ++++- src/fi/alk/harness/scenario.py | 14 +++++ src/fi/alk/harness/scenario_tools.py | 7 +++ src/fi/alk/harness/scenarios.py | 8 ++- src/fi/alk/harness/skills/builder/SKILL.md | 3 +- tests/harness/test_builder_chat.py | 52 +++++++++++++++--- tests/test_harness.py | 58 +++++++++++++++++++- 7 files changed, 143 insertions(+), 13 deletions(-) diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 7e83b4450..f2d41e3dd 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -528,7 +528,19 @@ def _changes(self, args: dict[str, Any], fields: tuple[str, ...], whole_persona: `whole_persona` compares the persona the submission would save, so a changed language or a dropped persona counts: a submission replaces the scenario rather than patching it. """ - path = self.workspace / "scenarios" / str(args.get("name") or "") / "scenario.json" + from .scenario_tools import load_scenarios + + name, key = str(args.get("name") or ""), str(args.get("scenario_key") or "") + if key: + name = next( + ( + one.name + for one in load_scenarios(self.workspace) + if one.scenario_key == key + ), + name, + ) + path = self.workspace / "scenarios" / name / "scenario.json" try: held = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError): diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index e91f95598..42000b689 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -208,6 +208,20 @@ def _slug(name: str) -> str: return cleaned or "scenario-" + hashlib.sha256(name.encode()).hexdigest()[:12] +def unused_key(key: str, taken: set[str]) -> str: + """``key``, or it with a numbered suffix when another scenario already holds it. + + A renamed scenario keeps the key its first name gave it, so a new scenario under that name would + otherwise mint the same one. + """ + if not key or key not in taken: + return key + suffix = 2 + while f"{key}-{suffix}" in taken: + suffix += 1 + return f"{key}-{suffix}" + + class Scenario(BaseModel): """One test: what changes, what is asked, what a correct agent does, what must hold.""" diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index f37934db0..d5610eed6 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -66,6 +66,8 @@ redteam_problems, level_name, _pinned_identity, + _slug, + unused_key, _QUIET_INTERFACE, unpinned_callers, crowded_cells, @@ -962,6 +964,11 @@ def accept_scenario( kept[:] = [one for one in kept if one is not previous] if previous is not None: scenario.scenario_key = previous.scenario_key + else: + scenario.scenario_key = unused_key( + _slug(scenario.name) if named else scenario.scenario_key, + {one.scenario_key for one in kept}, + ) replaced = any(one.name == scenario.name for one in kept) if replaced and rename_on_collision: taken = {one.name for one in kept} diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index b114b3ca3..8b5594607 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -40,7 +40,7 @@ from .contract import AgentContract from .job import HarnessJob from .poc_guest_booking import guest_booking_pin_guidance, guest_booking_pin_scenario_problem -from .scenario import Scenario, voicemail_enabled +from .scenario import Scenario, unused_key, voicemail_enabled from .scenario_tools import ( SCENARIO_SERVER, TOOL_NAMES, @@ -919,6 +919,7 @@ def merged(written: list[list[Scenario]]) -> list[Scenario]: """ suite: list[Scenario] = [] taken: set[str] = set() + keys: set[str] = set() for batch in written: for one in batch: if one.name in taken: @@ -929,7 +930,12 @@ def merged(written: list[list[Scenario]]) -> list[Scenario]: update={"name": f"{stem}-{suffix}", "scenario_key": ""} ) logger.info("renamed a duplicate folder name to %s", one.name) + if one.scenario_key in keys and batch is not written[0]: + one = one.model_copy( + update={"scenario_key": unused_key(one.scenario_key, keys)} + ) taken.add(one.name) + keys.add(one.scenario_key) suite.append(one) return suite diff --git a/src/fi/alk/harness/skills/builder/SKILL.md b/src/fi/alk/harness/skills/builder/SKILL.md index 0cad4bfa1..ec732ae5f 100644 --- a/src/fi/alk/harness/skills/builder/SKILL.md +++ b/src/fi/alk/harness/skills/builder/SKILL.md @@ -142,7 +142,8 @@ what the gates check. Then: `submit_scenario` it under the same name, keeping every field the change does not touch. That replaces it, keeps its number, and proves it. - **Rename one**: `submit_scenario` it with its `scenario_key` and the new name, every other field - unchanged. It keeps its number and its run history. + unchanged. It keeps its number and its run history. Never drop a scenario and submit it again to + edit or rename it: it would come back as a new one, without its history. - **Add**: `aim_for` the new total first, then write and submit each one. For more than a handful, brief `scenario_writer` sub-agents in one message. When the person asks for a level of an axis (no background noise, only one kind of caller), pass it in `aim_for`'s `asked_for` so it is not held to diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index d57c86791..13696ebf2 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -270,16 +270,27 @@ def test_world_change_is_all_or_nothing_and_reports_counts(tmp_path): assert "a: 2, b: 1" in world_change(tmp_path, "UPDATE b SET id = '2'") -def test_meaning_changes_wait_for_a_yes_to_that_call_but_surface_edits_do_not(tmp_path, monkeypatch): +def test_meaning_changes_wait_for_a_yes_to_that_call_but_surface_edits_do_not( + tmp_path, monkeypatch +): from fi.alk.harness.backends import ToolSpec from fi.alk.harness.hosted_chat_entrypoint import _CONFIRM client = BuilderClient() runtime = _runtime(tmp_path, client, monkeypatch) (tmp_path / "scenarios" / "one" / "scenario.json").write_text( - json.dumps({"name": "one", "instruction": "ask for a refund", "persona": {"name": "Ana"}}) + json.dumps( + { + "name": "one", + "scenario_key": "one", + "instruction": "ask for a refund", + "persona": {"name": "Ana"}, + } + ) + ) + (tmp_path / "sub_goals.json").write_text( + json.dumps({"sub_goals": [{"name": "refunded", "description": "x"}]}) ) - (tmp_path / "sub_goals.json").write_text(json.dumps({"sub_goals": [{"name": "refunded", "description": "x"}]})) calls, asked, answers = [], [], [] async def handler(args): @@ -295,12 +306,39 @@ async def ask(tool, payload, context): drop = runtime._confirmed(ToolSpec("drop_scenarios", "", {}, handler)) goal = runtime._confirmed(ToolSpec("add_sub_goal", "", {}, handler)) - surface = {"name": "one", "instruction": "ask for a refund", "persona": {"name": "Ana", "accent": "Indian"}} - meaning = {"name": "one", "instruction": "ask for a cancellation", "persona": {"name": "Ana"}} + surface = { + "name": "one", + "instruction": "ask for a refund", + "persona": {"name": "Ana", "accent": "Indian"}, + } + meaning = { + "name": "one", + "instruction": "ask for a cancellation", + "persona": {"name": "Ana"}, + } assert asyncio.run(submit.handler(surface)).get("is_error") is None - assert asyncio.run(submit.handler({"name": "two", "instruction": "anything"})).get("is_error") is None + assert ( + asyncio.run(submit.handler({"name": "two", "instruction": "anything"})).get( + "is_error" + ) + is None + ) assert asyncio.run(goal.handler({"name": "new-goal"})).get("is_error") is None + assert ( + asyncio.run( + submit.handler({**surface, "name": "uno", "scenario_key": "one"}) + ).get("is_error") + is None + ) assert asked == [] + answers.append("No") + assert ( + asyncio.run(submit.handler({**meaning, "name": "uno", "scenario_key": "one"}))[ + "is_error" + ] + is True + ) + assert "ask for a cancellation" in asked.pop() answers.extend(["No, keep it", "Which scenarios use it?"]) assert asyncio.run(drop.handler({"names": ["one"]}))["is_error"] is True @@ -319,7 +357,7 @@ async def ask(tool, payload, context): answers.append("No") assert asyncio.run(submit.handler(meaning))["is_error"] is True assert asyncio.run(submit.handler(surface)).get("is_error") is None - assert len(calls) == 5 + assert len(calls) == 6 def test_the_harness_builds_from_an_empty_workspace_with_the_tools_for_what_exists(tmp_path, monkeypatch): diff --git a/tests/test_harness.py b/tests/test_harness.py index af9f054bb..bc848740a 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -2704,17 +2704,69 @@ def test_an_edit_under_the_same_name_keeps_the_key_it_was_kept_with(tmp_path): root, _contract, catalogue = _built_environment(tmp_path) kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + key = kept[0].scenario_key accept_scenario( - _delta(scenario_key="kept-key"), world_root=root, catalogue=catalogue, kept=kept + _delta(name="orders-one-big-mac", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, ) accept_scenario( - _delta(instruction="Order one Big Mac, please."), + _delta(name="orders-one-big-mac", instruction="Order one Big Mac, please."), world_root=root, catalogue=catalogue, kept=kept, ) assert [(one.name, one.scenario_key) for one in kept] == [ - ("adds-a-big-mac", "kept-key") + ("orders-one-big-mac", key) + ] + + +def test_a_new_scenario_never_takes_a_key_another_one_holds_or_one_it_made_up(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + key = kept[0].scenario_key + accept_scenario( + _delta(name="orders-one-big-mac", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario( + _delta(name="orders-again", scenario_key="not-in-this-suite"), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + keys = {one.name: one.scenario_key for one in kept} + assert keys["orders-one-big-mac"] == key + assert keys["adds-a-big-mac"] == f"{key}-2" + assert keys["orders-again"] == "orders-again" + + +def test_merging_writers_never_changes_a_saved_key_or_repeats_one(): + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenarios import merged + + def one(name, key=""): + return Scenario(name=name, instruction="x", scenario_key=key) + + saved = [ + one("renamed", key="first-name"), + one("twin", key="same"), + one("twin-too", key="same"), + ] + suite = merged([saved, [one("first-name")]]) + assert [(item.name, item.scenario_key) for item in suite] == [ + ("renamed", "first-name"), + ("twin", "same"), + ("twin-too", "same"), + ("first-name", "first-name-2"), ] From 222393280fb92c6ba8361ff223a7766b23933f99 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 16:32:20 +0530 Subject: [PATCH 06/15] test(harness): leave untouched test lines as they were --- tests/harness/test_builder_chat.py | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index 13696ebf2..db9e69750 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -270,9 +270,7 @@ def test_world_change_is_all_or_nothing_and_reports_counts(tmp_path): assert "a: 2, b: 1" in world_change(tmp_path, "UPDATE b SET id = '2'") -def test_meaning_changes_wait_for_a_yes_to_that_call_but_surface_edits_do_not( - tmp_path, monkeypatch -): +def test_meaning_changes_wait_for_a_yes_to_that_call_but_surface_edits_do_not(tmp_path, monkeypatch): from fi.alk.harness.backends import ToolSpec from fi.alk.harness.hosted_chat_entrypoint import _CONFIRM @@ -288,9 +286,7 @@ def test_meaning_changes_wait_for_a_yes_to_that_call_but_surface_edits_do_not( } ) ) - (tmp_path / "sub_goals.json").write_text( - json.dumps({"sub_goals": [{"name": "refunded", "description": "x"}]}) - ) + (tmp_path / "sub_goals.json").write_text(json.dumps({"sub_goals": [{"name": "refunded", "description": "x"}]})) calls, asked, answers = [], [], [] async def handler(args): @@ -306,23 +302,10 @@ async def ask(tool, payload, context): drop = runtime._confirmed(ToolSpec("drop_scenarios", "", {}, handler)) goal = runtime._confirmed(ToolSpec("add_sub_goal", "", {}, handler)) - surface = { - "name": "one", - "instruction": "ask for a refund", - "persona": {"name": "Ana", "accent": "Indian"}, - } - meaning = { - "name": "one", - "instruction": "ask for a cancellation", - "persona": {"name": "Ana"}, - } + surface = {"name": "one", "instruction": "ask for a refund", "persona": {"name": "Ana", "accent": "Indian"}} + meaning = {"name": "one", "instruction": "ask for a cancellation", "persona": {"name": "Ana"}} assert asyncio.run(submit.handler(surface)).get("is_error") is None - assert ( - asyncio.run(submit.handler({"name": "two", "instruction": "anything"})).get( - "is_error" - ) - is None - ) + assert asyncio.run(submit.handler({"name": "two", "instruction": "anything"})).get("is_error") is None assert asyncio.run(goal.handler({"name": "new-goal"})).get("is_error") is None assert ( asyncio.run( From d4fc0f9abc7d745ec2ef448fa1100f305920b285 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 17:28:45 +0530 Subject: [PATCH 07/15] fix(harness): compare a resubmitted scenario as saved, so defaults it omits are not changes --- src/fi/alk/harness/hosted_chat_entrypoint.py | 12 +++++++++++- tests/harness/test_builder_chat.py | 20 ++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index f2d41e3dd..7816fe986 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -545,7 +545,17 @@ def _changes(self, args: dict[str, Any], fields: tuple[str, ...], whole_persona: held = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError): return False - if any(field in args and args.get(field) != held.get(field) for field in fields): + from .scenario import Scenario + + try: + saved_now = Scenario.model_validate(held).model_dump() + submitted = Scenario.model_validate({**held, **args}).model_dump() + except ValueError: # an unreadable submission is never a safe unattended change + return True + if any( + field in args and submitted.get(field) != saved_now.get(field) + for field in fields + ): return True persona, before = args.get("persona") or {}, held.get("persona") or {} if whole_persona: diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index db9e69750..33f5d3bc0 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -283,10 +283,19 @@ def test_meaning_changes_wait_for_a_yes_to_that_call_but_surface_edits_do_not(tm "scenario_key": "one", "instruction": "ask for a refund", "persona": {"name": "Ana"}, + "solution": [ + { + "tool": "refund", + "arguments": {"id": "7"}, + "environment_arguments": {}, + } + ], } ) ) - (tmp_path / "sub_goals.json").write_text(json.dumps({"sub_goals": [{"name": "refunded", "description": "x"}]})) + (tmp_path / "sub_goals.json").write_text( + json.dumps({"sub_goals": [{"name": "refunded", "description": "x"}]}) + ) calls, asked, answers = [], [], [] async def handler(args): @@ -313,6 +322,13 @@ async def ask(tool, payload, context): ).get("is_error") is None ) + resubmitted = { + **surface, + "name": "uno", + "scenario_key": "one", + "solution": [{"tool": "refund", "arguments": {"id": "7"}}], + } + assert asyncio.run(submit.handler(resubmitted)).get("is_error") is None assert asked == [] answers.append("No") assert ( @@ -340,7 +356,7 @@ async def ask(tool, payload, context): answers.append("No") assert asyncio.run(submit.handler(meaning))["is_error"] is True assert asyncio.run(submit.handler(surface)).get("is_error") is None - assert len(calls) == 6 + assert len(calls) == 7 def test_the_harness_builds_from_an_empty_workspace_with_the_tools_for_what_exists(tmp_path, monkeypatch): From 64a1cfaad9da9b1153ab4668d3a613d3856350e8 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 17:33:41 +0530 Subject: [PATCH 08/15] fix(harness): fall back to the plain comparison for a scenario the model cannot read --- src/fi/alk/harness/hosted_chat_entrypoint.py | 4 ++-- src/fi/alk/harness/scenario.py | 6 +----- src/fi/alk/harness/scenario_tools.py | 2 +- tests/harness/test_builder_chat.py | 8 ++++++++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 7816fe986..fc419ca87 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -550,8 +550,8 @@ def _changes(self, args: dict[str, Any], fields: tuple[str, ...], whole_persona: try: saved_now = Scenario.model_validate(held).model_dump() submitted = Scenario.model_validate({**held, **args}).model_dump() - except ValueError: # an unreadable submission is never a safe unattended change - return True + except ValueError: + saved_now, submitted = held, {**held, **args} if any( field in args and submitted.get(field) != saved_now.get(field) for field in fields diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index 42000b689..948cf5308 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -209,11 +209,7 @@ def _slug(name: str) -> str: def unused_key(key: str, taken: set[str]) -> str: - """``key``, or it with a numbered suffix when another scenario already holds it. - - A renamed scenario keeps the key its first name gave it, so a new scenario under that name would - otherwise mint the same one. - """ + """``key``, or it with a numbered suffix when another scenario already holds it.""" if not key or key not in taken: return key suffix = 2 diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index d5610eed6..0ed149e74 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -1707,7 +1707,7 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] key = str(args.get("scenario_key") or "").strip() same = next( (one.name for one in kept if key and one.scenario_key == key), - str(args.get("name") or ""), + str(args.get("name") or "").strip(), ) # Cheap label checks run before the gates, since proving is expensive. strayed = _off_the_grid(args.get("coverage"), target.get("axes")) diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index 33f5d3bc0..9d86b5778 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -322,6 +322,14 @@ async def ask(tool, payload, context): ).get("is_error") is None ) + (tmp_path / "scenarios" / "old").mkdir() + (tmp_path / "scenarios" / "old" / "scenario.json").write_text( + json.dumps({"name": "old"}) + ) + answers.append("No") + unreadable = {"name": "old", "instruction": "anything"} + assert asyncio.run(submit.handler(unreadable))["is_error"] is True + asked.pop() resubmitted = { **surface, "name": "uno", From dfb608f51f314e8472a8e4db1bb71f3fa34ac7e5 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 17:41:43 +0530 Subject: [PATCH 09/15] feat(harness): edit a scenario by key with only the fields that change --- src/fi/alk/harness/hosted_chat_entrypoint.py | 7 ++-- src/fi/alk/harness/scenario_tools.py | 37 +++++++++++++++++--- src/fi/alk/harness/skills/builder/SKILL.md | 12 +++---- tests/test_harness.py | 13 ++++--- 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index fc419ca87..9c7ea2963 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -60,6 +60,7 @@ _NEEDS_CONFIRMATION = frozenset( { "submit_scenario", + "edit_scenario", "drop_scenario", "drop_scenarios", "change_world", @@ -71,6 +72,7 @@ "add_sub_goal", } ) +_SUBMITS = ("submit_scenario", "edit_scenario") _CONFIRM = "Yes, make this change" _RUN_FINISHED = frozenset({"completed", "failed", "canceled"}) _RUN_POLL_SECONDS = 20.0 @@ -89,6 +91,7 @@ _WRITING_TOOLS = frozenset( { "submit_scenario", + "edit_scenario", "drop_scenario", "drop_scenarios", "save_scenarios", @@ -461,7 +464,7 @@ def _confirmed(self, spec: Any) -> Any: async def guarded(args: dict[str, Any]) -> Any: if self._from_platform: if spec.name in ("drop_scenarios", "drop_scenario") or ( - spec.name == "submit_scenario" and self._changes_situation(args) + spec.name in _SUBMITS and self._changes_situation(args) ): return { "content": [ @@ -476,7 +479,7 @@ async def guarded(args: dict[str, Any]) -> Any: "is_error": True, } return await handler(args) - if spec.name == "submit_scenario" and not self._changes_meaning(args): + if spec.name in _SUBMITS and not self._changes_meaning(args): return await handler(args) if spec.name == "add_sub_goal" and not self._replaces_sub_goal(args): return await handler(args) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 0ed149e74..876ecdaaa 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -1473,11 +1473,6 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "It becomes this scenario's folder name. It says what this scenario tests and " "carries, without a counter or an invented code, PIN or reference.", }, - "scenario_key": { - "type": "string", - "description": "Only when editing a scenario already kept: its scenario_key, read " - "from it. With a new name this renames it, keeping its number and its history.", - }, "use_case": { "type": "string", "description": "Which of the agent's use cases this belongs to.", @@ -2299,6 +2294,36 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: ) return _ok(said) + edit_schema = { + **submit_scenario.input_schema, + "properties": { + **submit_scenario.input_schema["properties"], + "scenario_key": { + "type": "string", + "description": "The scenario's key, read from it.", + }, + }, + "required": ["scenario_key"], + } + + @tool( + "edit_scenario", + "Change a kept scenario: give its `scenario_key` and only the fields that change, such as a new " + "`name`. Every other field keeps its saved value, and the scenario keeps its key, number and " + "history. The result is checked exactly like `submit_scenario`.", + edit_schema, + ) + async def edit_scenario(args: dict[str, Any]) -> dict[str, Any]: + key = str(args.get("scenario_key") or "").strip() + held = next((one for one in kept if one.scenario_key == key), None) + if held is None: + return _err( + f"Not changed: no scenario in this suite has the scenario_key {key!r}." + ) + return await submit_scenario.handler( + {**held.model_dump(exclude_none=True), **args} + ) + server = tool_server( name=SCENARIO_SERVER, version="0.1.0", @@ -2309,6 +2334,7 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: try_check, add_sub_goal, submit_scenario, + edit_scenario, amend_contract, add_rule_tool, drop_rule_tool, @@ -2330,6 +2356,7 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: "try_check", "add_sub_goal", "submit_scenario", + "edit_scenario", "amend_contract", "add_rule", "drop_rule", diff --git a/src/fi/alk/harness/skills/builder/SKILL.md b/src/fi/alk/harness/skills/builder/SKILL.md index ec732ae5f..14327a2e0 100644 --- a/src/fi/alk/harness/skills/builder/SKILL.md +++ b/src/fi/alk/harness/skills/builder/SKILL.md @@ -137,13 +137,11 @@ selection is what "these" and "the selected ones" mean. Load `write-scenarios` before you write or rewrite any scenario in this conversation; its rules are what the gates check. Then: -- **Edit one**: read it in full first, change only what was asked, carry the change into every - field that follows from it (setup, ready check, solution, instruction, sub-goals, labels), and - `submit_scenario` it under the same name, keeping every field the change does not touch. That - replaces it, keeps its number, and proves it. -- **Rename one**: `submit_scenario` it with its `scenario_key` and the new name, every other field - unchanged. It keeps its number and its run history. Never drop a scenario and submit it again to - edit or rename it: it would come back as a new one, without its history. +- **Edit or rename one**: read it in full first, change only what was asked, carry the change into + every field that follows from it (setup, ready check, solution, instruction, sub-goals, labels), + and `edit_scenario` it with its `scenario_key` and only the fields that change. That keeps its + number and its run history, and proves it again. Never drop a scenario and submit it again to edit + or rename it: it would come back as a new one, without its history. - **Add**: `aim_for` the new total first, then write and submit each one. For more than a handful, brief `scenario_writer` sub-agents in one message. When the person asks for a level of an axis (no background noise, only one kind of caller), pass it in `aim_for`'s `asked_for` so it is not held to diff --git a/tests/test_harness.py b/tests/test_harness.py index bc848740a..d3a9cf1f2 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -7978,7 +7978,7 @@ def test_a_scenario_placed_nowhere_is_refused_once_a_grid_is_dealt(tmp_path): assert not _off_the_grid(None, {}) -def test_renaming_through_the_tool_is_not_refused_as_a_duplicate_of_itself(tmp_path): +def test_an_edit_sends_only_what_changes_and_keeps_the_rest(tmp_path): import asyncio from fi.alk.harness.scenario_tools import scenario_tools @@ -7986,6 +7986,7 @@ def test_renaming_through_the_tool_is_not_refused_as_a_duplicate_of_itself(tmp_p root, _contract, _catalogue = _built_environment(tmp_path) server, kept = scenario_tools(_contract, root, root, wanted=2) submit = next(one for one in server.tools if one.name == "submit_scenario") + edit = next(one for one in server.tools if one.name == "edit_scenario") person = { "persona": { "name": "Asha Rao", @@ -7997,14 +7998,18 @@ def test_renaming_through_the_tool_is_not_refused_as_a_duplicate_of_itself(tmp_p } said = asyncio.run(submit.handler(_delta(**person))) assert not said.get("is_error"), said + before = kept[0].model_dump(exclude={"name"}) key = kept[0].scenario_key - said = asyncio.run( - submit.handler(_delta(name="orders-one-big-mac", scenario_key=key, **person)) - ) + renamed = {"scenario_key": key, "name": "orders-one-big-mac"} + said = asyncio.run(edit.handler(renamed)) assert not said.get("is_error"), said assert [(one.name, one.scenario_key) for one in kept] == [ ("orders-one-big-mac", key) ] + assert kept[0].model_dump(exclude={"name"}) == before + unknown = asyncio.run(edit.handler({"scenario_key": "not-here", "name": "x"})) + assert unknown.get("is_error") and "not-here" in unknown["content"][0]["text"] + assert [one.name for one in kept] == ["orders-one-big-mac"] def test_a_redeclared_grid_only_ever_grows(tmp_path): From 088097a0fe44f8cacd9863643c67c44bcbbbeb4e Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 18:03:08 +0530 Subject: [PATCH 10/15] fix(harness): point submit_scenario at edit_scenario for kept scenarios --- src/fi/alk/harness/scenario_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 876ecdaaa..35d245968 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -1445,7 +1445,8 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: @tool( "submit_scenario", - "Keep one scenario. It is put through three gates before it is kept, and told which one " + "Keep one new scenario. To change one already kept, use `edit_scenario` with only the " + "fields that change. It is put through three gates before it is kept, and told which one " "failed if any does:\n" " 1. ready — the world is restored, setup_code runs, then ready_code. The world " "must end up holding what this scenario presumes.\n" From 8f24dc01613c8a5013bf8654b1805ccd08ded903 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 21:31:34 +0530 Subject: [PATCH 11/15] fix(harness): an edit changes only the persona fields it names --- src/fi/alk/harness/hosted_chat_entrypoint.py | 25 +++++++++---------- src/fi/alk/harness/scenario_tools.py | 26 +++++++++++++++----- tests/harness/test_builder_chat.py | 4 ++- tests/test_harness.py | 5 ++++ 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 9c7ea2963..9f73e03f8 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -531,18 +531,17 @@ def _changes(self, args: dict[str, Any], fields: tuple[str, ...], whole_persona: `whole_persona` compares the persona the submission would save, so a changed language or a dropped persona counts: a submission replaces the scenario rather than patching it. """ - from .scenario_tools import load_scenarios - - name, key = str(args.get("name") or ""), str(args.get("scenario_key") or "") - if key: - name = next( - ( - one.name - for one in load_scenarios(self.workspace) - if one.scenario_key == key - ), - name, - ) + from .scenario_tools import edited, load_scenarios + + key = str(args.get("scenario_key") or "") + target = next( + (one for one in load_scenarios(self.workspace) if key and one.scenario_key == key), + None, + ) + name = str(args.get("name") or "") if target is None else target.name + sent = args.get("persona") or {} + if target is not None: + args = edited(target, args) path = self.workspace / "scenarios" / name / "scenario.json" try: held = json.loads(path.read_text(encoding="utf-8")) @@ -572,7 +571,7 @@ def saved(one: dict[str, Any]) -> dict[str, Any]: except ValueError: # an unreadable persona is never a safe unattended change return True return any( - key in persona and persona.get(key) != before.get(key) + key in sent and persona.get(key) != before.get(key) for key in ("name", "gender", "age_group") ) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 35d245968..040e6e0d2 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -874,6 +874,21 @@ def _credentials_the_world_lacks(scenario: Scenario, trial: Any) -> list[str]: ] +def edited(held: Scenario, change: dict[str, Any]) -> dict[str, Any]: + """``held`` with ``change`` applied: a field it names is replaced, and a mapping such as the + persona changes only the entries it names.""" + saved = held.model_dump(exclude_none=True) + return { + **saved, + **{ + field: {**saved[field], **value} + if isinstance(value, dict) and isinstance(saved.get(field), dict) + else value + for field, value in change.items() + }, + } + + def accept_scenario( payload: dict[str, Any], *, @@ -2309,9 +2324,10 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: @tool( "edit_scenario", - "Change a kept scenario: give its `scenario_key` and only the fields that change, such as a new " - "`name`. Every other field keeps its saved value, and the scenario keeps its key, number and " - "history. The result is checked exactly like `submit_scenario`.", + "Change a kept scenario: give its `scenario_key` and only what changes, such as a new " + "`name` or the persona fields that change. Everything else keeps its saved value, and the " + "scenario keeps its key, number and history. The result is checked exactly like " + "`submit_scenario`.", edit_schema, ) async def edit_scenario(args: dict[str, Any]) -> dict[str, Any]: @@ -2321,9 +2337,7 @@ async def edit_scenario(args: dict[str, Any]) -> dict[str, Any]: return _err( f"Not changed: no scenario in this suite has the scenario_key {key!r}." ) - return await submit_scenario.handler( - {**held.model_dump(exclude_none=True), **args} - ) + return await submit_scenario.handler(edited(held, args)) server = tool_server( name=SCENARIO_SERVER, diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index 9d86b5778..67bb8de56 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -282,7 +282,7 @@ def test_meaning_changes_wait_for_a_yes_to_that_call_but_surface_edits_do_not(tm "name": "one", "scenario_key": "one", "instruction": "ask for a refund", - "persona": {"name": "Ana"}, + "persona": {"name": "Ana", "accent": "American"}, "solution": [ { "tool": "refund", @@ -346,6 +346,8 @@ async def ask(tool, payload, context): is True ) assert "ask for a cancellation" in asked.pop() + assert not runtime._changes_situation({"scenario_key": "one", "persona": {"name": "Ana"}}) + assert runtime._changes_situation({"scenario_key": "one", "persona": {"accent": "Indian"}}) answers.extend(["No, keep it", "Which scenarios use it?"]) assert asyncio.run(drop.handler({"names": ["one"]}))["is_error"] is True diff --git a/tests/test_harness.py b/tests/test_harness.py index d3a9cf1f2..563d6ac3f 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -8007,6 +8007,11 @@ def test_an_edit_sends_only_what_changes_and_keeps_the_rest(tmp_path): ("orders-one-big-mac", key) ] assert kept[0].model_dump(exclude={"name"}) == before + said = asyncio.run(edit.handler({"scenario_key": key, "persona": {"name": "Meera Iyer"}})) + assert not said.get("is_error"), said + assert kept[0].persona.name == "Meera Iyer" + assert kept[0].persona.initial_message == "Hi, I want a Big Mac." + assert kept[0].persona.languages == ["English"] unknown = asyncio.run(edit.handler({"scenario_key": "not-here", "name": "x"})) assert unknown.get("is_error") and "not-here" in unknown["content"][0]["text"] assert [one.name for one in kept] == ["orders-one-big-mac"] From 9f6e3567785238ebdaacd9e60d4ea3b49f24ce3a Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 26 Oct 2026 13:54:04 +0530 Subject: [PATCH 12/15] fix(harness): TH-8748 keep a renamed scenario's place, rename by name alone without proving again, clear fields with null, journal renames, repair repeated saved keys, count rename refusals, tidy _changes --- src/fi/alk/harness/hosted_chat_entrypoint.py | 23 +-- src/fi/alk/harness/scenario_tools.py | 136 ++++++++++++---- src/fi/alk/harness/scenarios.py | 3 +- src/fi/alk/harness/skills/builder/SKILL.md | 5 +- tests/test_harness.py | 154 ++++++++++++++++++- 5 files changed, 279 insertions(+), 42 deletions(-) diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 9f73e03f8..4bca27749 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -533,15 +533,18 @@ def _changes(self, args: dict[str, Any], fields: tuple[str, ...], whole_persona: """ from .scenario_tools import edited, load_scenarios - key = str(args.get("scenario_key") or "") + scenario_key = str(args.get("scenario_key") or "") target = next( - (one for one in load_scenarios(self.workspace) if key and one.scenario_key == key), + ( + one + for one in load_scenarios(self.workspace) + if scenario_key and one.scenario_key == scenario_key + ), None, ) name = str(args.get("name") or "") if target is None else target.name sent = args.get("persona") or {} - if target is not None: - args = edited(target, args) + change = args if target is None else edited(target, args) path = self.workspace / "scenarios" / name / "scenario.json" try: held = json.loads(path.read_text(encoding="utf-8")) @@ -551,15 +554,15 @@ def _changes(self, args: dict[str, Any], fields: tuple[str, ...], whole_persona: try: saved_now = Scenario.model_validate(held).model_dump() - submitted = Scenario.model_validate({**held, **args}).model_dump() + submitted = Scenario.model_validate({**held, **change}).model_dump() except ValueError: - saved_now, submitted = held, {**held, **args} + saved_now, submitted = held, {**held, **change} if any( - field in args and submitted.get(field) != saved_now.get(field) + field in change and submitted.get(field) != saved_now.get(field) for field in fields ): return True - persona, before = args.get("persona") or {}, held.get("persona") or {} + persona, before = change.get("persona") or {}, held.get("persona") or {} if whole_persona: from .scenario import Persona @@ -571,8 +574,8 @@ def saved(one: dict[str, Any]) -> dict[str, Any]: except ValueError: # an unreadable persona is never a safe unattended change return True return any( - key in sent and persona.get(key) != before.get(key) - for key in ("name", "gender", "age_group") + field in sent and persona.get(field) != before.get(field) + for field in ("name", "gender", "age_group") ) def _builder_platform_server(self): diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 040e6e0d2..59f23fe6c 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -322,7 +322,7 @@ def load_scenarios(destination: Path) -> list[Scenario]: JOURNAL = "written.jsonl" -def journal_scenario(scenario: Scenario, destination: Path) -> None: +def journal_scenario(scenario: Scenario, destination: Path, replaces: str = "") -> None: """Append one proved scenario to the journal, which is the only record a dead writer leaves. A delegated writer cannot write folders: saving the suite deletes every folder it does not know @@ -335,6 +335,11 @@ def journal_scenario(scenario: Scenario, destination: Path) -> None: path = Path(destination) / JOURNAL path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as journal: + # A rename drops the old name first, so recovering the journal never brings it back. + if replaces: + journal.write( + json.dumps({"dropped": replaces}, ensure_ascii=False) + "\n" + ) journal.write(json.dumps(scenario.model_dump(), ensure_ascii=False) + "\n") except Exception as broke: # noqa: BLE001 - a scenario is never lost over bookkeeping logger.warning("could not journal %s: %s", scenario.name, broke) @@ -354,7 +359,11 @@ def journalled(destination: Path) -> list[Scenario]: if not line.strip(): continue try: - one = Scenario.model_validate(json.loads(line)) + entry = json.loads(line) + if isinstance(entry, dict) and set(entry) == {"dropped"}: + found.pop(str(entry["dropped"]), None) + continue + one = Scenario.model_validate(entry) except Exception: # noqa: BLE001 - a half-written line is expected while writers run continue found[one.name] = one @@ -875,18 +884,53 @@ def _credentials_the_world_lacks(scenario: Scenario, trial: Any) -> list[str]: def edited(held: Scenario, change: dict[str, Any]) -> dict[str, Any]: - """``held`` with ``change`` applied: a field it names is replaced, and a mapping such as the - persona changes only the entries it names.""" - saved = held.model_dump(exclude_none=True) - return { - **saved, - **{ - field: {**saved[field], **value} - if isinstance(value, dict) and isinstance(saved.get(field), dict) - else value - for field, value in change.items() - }, - } + """``held`` with ``change`` applied: a field it names is replaced, a mapping such as the persona + changes only the entries it names, at any depth, and ``None`` clears a field back to its default. + """ + + def apply(saved: dict[str, Any], given: dict[str, Any]) -> dict[str, Any]: + out = dict(saved) + for field, value in given.items(): + if value is None: + out.pop(field, None) + elif isinstance(value, dict) and isinstance(out.get(field), dict): + out[field] = apply(out[field], value) + else: + out[field] = value + return out + + return apply(held.model_dump(exclude_none=True), change) + + +def rename_scenario( + held: Scenario, + name: str, + *, + kept: list[Scenario], + world_root: Path, + catalogue: Catalogue, + persist: bool = True, +) -> dict[str, Any]: + """Give a kept scenario a new name without proving it again: what it tests, its key, its place + in the suite and its proof are all unchanged, so the gates have nothing new to check.""" + try: + renamed = Scenario.model_validate({**held.model_dump(), "name": name}) + except ValueError as invalid: + return _err(f"Not renamed. {invalid}"[:600]) + if any(one.name == renamed.name for one in kept if one is not held): + return _err( + f"Not renamed. Another scenario is already named {renamed.name}; choose another name." + ) + spot = next(index for index, one in enumerate(kept) if one is held) + kept[spot] = renamed + if persist: + write_scenarios(kept, world_root, catalogue) + else: + journal_scenario(renamed, world_root, replaces=held.name) + return _ok( + f"{held.name} renamed to {renamed.name}. Nothing it tests changed, so it was not proved " + "again." + ) def accept_scenario( @@ -971,12 +1015,16 @@ def accept_scenario( previous = next( (one for one in kept if named and one.scenario_key == named), None ) or next((one for one in kept if one.name == scenario.name), None) + # A replaced or renamed scenario keeps its place in the suite. + spot = next((index for index, one in enumerate(kept) if one is previous), None) + renamed_from = "" if previous is not None and previous.name != scenario.name: if any(one.name == scenario.name for one in kept): return _err( f"Not kept. Another scenario is already named {scenario.name}; choose another name." ) - kept[:] = [one for one in kept if one is not previous] + renamed_from = previous.name + del kept[spot] if previous is not None: scenario.scenario_key = previous.scenario_key else: @@ -992,9 +1040,13 @@ def accept_scenario( suffix += 1 scenario = scenario.model_copy(update={"name": f"{stem}-{suffix}", "scenario_key": ""}) replaced = False + spot = None else: kept[:] = [one for one in kept if one.name != scenario.name] - kept.append(scenario) + if spot is None: + kept.append(scenario) + else: + kept.insert(spot, scenario) outside: list[str] = [] if vocabulary and scenario.keywords: allowed = {word.strip().casefold(): word.strip() for word in vocabulary if word.strip()} @@ -1017,7 +1069,7 @@ def accept_scenario( else: # A writer sharing the destination cannot write folders, so the journal is where its proved # work survives the session that proved it. - journal_scenario(scenario, world_root) + journal_scenario(scenario, world_root, replaces=renamed_from) # Say what the proof did not cover. On a lane where the target's tools have no endpoints, every # solution step is recorded without running, so "all three gates pass" is true and misleading: # the checks were exercised, the solution was not. @@ -1051,8 +1103,13 @@ def accept_scenario( if outside else "" ) + done = ( + f"{renamed_from} renamed to {scenario.name}" + if renamed_from + else f"{scenario.name} {'replaced' if replaced else 'kept'}" + ) return _ok( - f"{scenario.name} {'replaced' if replaced else 'kept'}. {proof_summary}" + f"{done}. {proof_summary}" f"{unproved}{strayed}\n{len(kept)} so far. These names are taken, so do not reuse one " f"and do not read them: {recent}{more}" ) @@ -1745,11 +1802,15 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] return _refuse(twin) if cap() and target.get("people") != "alike": # A replacement does not count against itself. - crowded = crowded_field( - [one for one in kept if one.name != same], - Scenario.model_validate(args).persona, - cap(), - ) if args.get("persona") else "" + crowded = ( + crowded_field( + [one for one in kept if one.name != same], + Scenario.model_validate(args).persona, + cap(), + ) + if args.get("persona") + else "" + ) if crowded: return _refuse(crowded) if args.get("persona") and args.get("fixture"): @@ -2325,9 +2386,9 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: @tool( "edit_scenario", "Change a kept scenario: give its `scenario_key` and only what changes, such as a new " - "`name` or the persona fields that change. Everything else keeps its saved value, and the " - "scenario keeps its key, number and history. The result is checked exactly like " - "`submit_scenario`.", + "`name` or the persona fields that change; `null` clears a field. Everything else keeps " + "its saved value, and the scenario keeps its key, number, place and history. A new name " + "alone is applied as it is; any other change is checked exactly like `submit_scenario`.", edit_schema, ) async def edit_scenario(args: dict[str, Any]) -> dict[str, Any]: @@ -2337,6 +2398,29 @@ async def edit_scenario(args: dict[str, Any]) -> dict[str, Any]: return _err( f"Not changed: no scenario in this suite has the scenario_key {key!r}." ) + change = { + field: value for field, value in args.items() if field != "scenario_key" + } + name = str(change.get("name") or "").strip() + if set(change) == {"name"} and name and name != held.name: + result = rename_scenario( + held, + name, + kept=kept, + world_root=world_root, + catalogue=catalogue, + persist=can_save, + ) + if not result.get("is_error"): + return result + said = str((result.get("content") or [{}])[0].get("text") or "") + seen = _count_refusal(refused, held.name, said) + if seen >= _REFUSALS_BEFORE_MOVING_ON: + return _err( + f"{said}\n\nThis is refusal {seen} of this rename for the same reason, so " + "asking again will not work. Keep the current name or choose another one." + ) + return result return await submit_scenario.handler(edited(held, args)) server = tool_server( diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index 8b5594607..aaa989309 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -930,7 +930,8 @@ def merged(written: list[list[Scenario]]) -> list[Scenario]: update={"name": f"{stem}-{suffix}", "scenario_key": ""} ) logger.info("renamed a duplicate folder name to %s", one.name) - if one.scenario_key in keys and batch is not written[0]: + # A key names one scenario, so a repeat, even one already saved, gets its own. + if one.scenario_key and one.scenario_key in keys: one = one.model_copy( update={"scenario_key": unused_key(one.scenario_key, keys)} ) diff --git a/src/fi/alk/harness/skills/builder/SKILL.md b/src/fi/alk/harness/skills/builder/SKILL.md index 14327a2e0..a2fd71455 100644 --- a/src/fi/alk/harness/skills/builder/SKILL.md +++ b/src/fi/alk/harness/skills/builder/SKILL.md @@ -140,8 +140,9 @@ what the gates check. Then: - **Edit or rename one**: read it in full first, change only what was asked, carry the change into every field that follows from it (setup, ready check, solution, instruction, sub-goals, labels), and `edit_scenario` it with its `scenario_key` and only the fields that change. That keeps its - number and its run history, and proves it again. Never drop a scenario and submit it again to edit - or rename it: it would come back as a new one, without its history. + number, place and run history, and proves it again unless only the name changed. Never drop a + scenario and submit it again to edit or rename it: it would come back as a new one, without its + history. - **Add**: `aim_for` the new total first, then write and submit each one. For more than a handful, brief `scenario_writer` sub-agents in one message. When the person asks for a level of an axis (no background noise, only one kind of caller), pass it in `aim_for`'s `asked_for` so it is not held to diff --git a/tests/test_harness.py b/tests/test_harness.py index 563d6ac3f..a782332f4 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -2749,7 +2749,7 @@ def test_a_new_scenario_never_takes_a_key_another_one_holds_or_one_it_made_up(tm assert keys["orders-again"] == "orders-again" -def test_merging_writers_never_changes_a_saved_key_or_repeats_one(): +def test_merging_writers_keeps_the_first_holder_of_a_key_and_gives_repeats_their_own(): from fi.alk.harness.scenario import Scenario from fi.alk.harness.scenarios import merged @@ -2765,7 +2765,7 @@ def one(name, key=""): assert [(item.name, item.scenario_key) for item in suite] == [ ("renamed", "first-name"), ("twin", "same"), - ("twin-too", "same"), + ("twin-too", "same-2"), ("first-name", "first-name-2"), ] @@ -8007,7 +8007,9 @@ def test_an_edit_sends_only_what_changes_and_keeps_the_rest(tmp_path): ("orders-one-big-mac", key) ] assert kept[0].model_dump(exclude={"name"}) == before - said = asyncio.run(edit.handler({"scenario_key": key, "persona": {"name": "Meera Iyer"}})) + said = asyncio.run( + edit.handler({"scenario_key": key, "persona": {"name": "Meera Iyer"}}) + ) assert not said.get("is_error"), said assert kept[0].persona.name == "Meera Iyer" assert kept[0].persona.initial_message == "Hi, I want a Big Mac." @@ -8017,6 +8019,152 @@ def test_an_edit_sends_only_what_changes_and_keeps_the_rest(tmp_path): assert [one.name for one in kept] == ["orders-one-big-mac"] +def test_a_rename_keeps_the_scenario_s_place_and_says_it_was_renamed(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario( + _delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept + ) + key = kept[0].scenario_key + said = accept_scenario( + _delta(name="renamed-first", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + assert not said.get("is_error"), said + assert said["content"][0]["text"].startswith( + "adds-a-big-mac renamed to renamed-first." + ) + assert [(one.name, one.scenario_key) for one in kept] == [ + ("renamed-first", key), + ("orders-again", kept[1].scenario_key), + ] + + +def test_a_replacement_under_the_same_name_keeps_its_place(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario( + _delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept + ) + said = accept_scenario( + _delta(instruction="Order a single Big Mac."), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + assert said["content"][0]["text"].startswith("adds-a-big-mac replaced.") + assert [one.name for one in kept] == ["adds-a-big-mac", "orders-again"] + assert kept[0].instruction == "Order a single Big Mac." + + +def test_a_new_name_alone_is_applied_without_proving_the_scenario_again( + tmp_path, monkeypatch +): + import asyncio + + from fi.alk.harness import scenario_tools as tools + + root, contract, _catalogue = _built_environment(tmp_path) + server, kept = tools.scenario_tools(contract, root, root, wanted=2) + submit = next(one for one in server.tools if one.name == "submit_scenario") + edit = next(one for one in server.tools if one.name == "edit_scenario") + assert not asyncio.run(submit.handler(_delta())).get("is_error") + assert not asyncio.run(submit.handler(_delta(name="orders-again"))).get("is_error") + first, second = kept[0].scenario_key, kept[1].scenario_key + + # An older scenario that a gate added since would now refuse still renames. + def refuse(*_args, **_kwargs): + raise AssertionError("a name-only rename must not prove the scenario again") + + monkeypatch.setattr(tools, "prove", refuse) + monkeypatch.setattr(tools, "validate_scenario", refuse) + said = asyncio.run(edit.handler({"scenario_key": first, "name": "siparis-tekrari"})) + assert not said.get("is_error"), said + assert said["content"][0]["text"].startswith( + "adds-a-big-mac renamed to siparis-tekrari." + ) + assert [(one.name, one.scenario_key) for one in kept] == [ + ("siparis-tekrari", first), + ("orders-again", second), + ] + saved = tools.load_scenarios(root) + assert sorted(one.name for one in saved) == ["orders-again", "siparis-tekrari"] + for _ in range(3): + taken = asyncio.run( + edit.handler({"scenario_key": first, "name": "orders-again"}) + ) + assert taken.get("is_error") + assert "already named orders-again" in taken["content"][0]["text"] + assert "refusal 3 of this rename" in taken["content"][0]["text"] + assert [one.name for one in kept] == ["siparis-tekrari", "orders-again"] + + +def test_an_edit_clears_a_field_given_as_null_and_merges_nested_mappings(): + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenario_tools import edited + + held = Scenario.model_validate( + { + **_delta(), + "fixture": { + "origin": "seed", + "order": {"item": "big_mac", "size": "large"}, + }, + "persona": { + "name": "Asha Rao", + "accent": "Scottish", + "languages": ["English"], + }, + } + ) + change = edited( + held, + {"persona": {"accent": None}, "fixture": {"order": {"size": "small"}}}, + ) + after = Scenario.model_validate(change) + assert ( + after.persona.accent + == Scenario.model_validate( + {**_delta(), "persona": {"name": "Asha Rao"}} + ).persona.accent + ) + assert after.persona.name == "Asha Rao" + assert after.persona.languages == ["English"] + assert after.fixture == { + "origin": "seed", + "order": {"item": "big_mac", "size": "small"}, + } + + +def test_a_writer_s_rename_never_brings_the_old_name_back_from_the_journal(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario, journalled + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario( + _delta(), world_root=root, catalogue=catalogue, kept=kept, persist=False + ) + key = kept[0].scenario_key + accept_scenario( + _delta(name="renamed-first", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, + persist=False, + ) + assert [(one.name, one.scenario_key) for one in journalled(root)] == [ + ("renamed-first", key) + ] + + def test_a_redeclared_grid_only_ever_grows(tmp_path): import asyncio From 16a1a7597a1bc9aa5eff46394fdc81307b54d14c Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 26 Oct 2026 18:19:21 +0530 Subject: [PATCH 13/15] fix(harness): TH-8748 edit_scenario accepts partial edits and a clear list on both backends, clears go through confirmation, writer edits replace in place, used keys are never reused, repeated keys repaired on open and merge, rename folded into accept_scenario, journal keeps siblings, changelog line --- CHANGELOG.md | 4 + src/fi/alk/harness/hosted_chat_entrypoint.py | 32 ++- src/fi/alk/harness/scenario.py | 29 ++ src/fi/alk/harness/scenario_tools.py | 275 +++++++++++-------- src/fi/alk/harness/scenarios.py | 22 +- tests/harness/test_builder_chat.py | 98 ++++++- tests/test_harness.py | 238 +++++++++++++++- 7 files changed, 566 insertions(+), 132 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf49aeea0..accb0ee1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ All notable release changes for Agent Learning Kit are tracked here. Post-rc.1 increments on the release branch. Each one adds gates on top of the 66 proved at rc.1; the rc.1 entry below is historical and unchanged. +- Builder chat: `edit_scenario` changes a kept scenario by its key with only the fields that change + (`clear` resets a field), keeping its key, number, place and history; a rename alone is not + proved again. A new scenario never takes a key the environment has used, a dropped scenario's + included, and a repeated saved key is repaired so each scenario keeps its own. - Runtime validation checks scenarios on one database lane per CPU by default, capped by the sandbox's cgroup CPU quota, instead of a fixed 16; `ALK_VALIDATION_DB_LANES` still overrides it. - Docs corpus: the full `docs/` tree (quickstarts, per-track guides, framework diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 4bca27749..3c4b08d07 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -254,6 +254,15 @@ def __init__( self.workspace = workspace self.source = source self.job = job + from .scenario import used_keys + + # Every key the environment has given a scenario, a dropped one's included, so a new + # scenario never takes one and inherits another's row and run history. + used_keys(workspace).update( + str(key) + for key in (job.get("metadata") or {}).get("used_scenario_keys") or () + if key + ) backend = resolve() self.backend_name = backend.name self.event_store: Any = None @@ -513,7 +522,9 @@ async def guarded(args: dict[str, Any]) -> Any: def _changes_situation(self, args: dict[str, Any]) -> bool: """Whether a submission changes what an existing scenario tests, not how it is graded.""" return self._changes( - args, ("instruction", "use_case", "branch", "tests", "coverage", "fixture"), whole_persona=True + args, + ("instruction", "use_case", "branch", "tests", "coverage", "fixture"), + whole_persona=True, ) def _replaces_sub_goal(self, args: dict[str, Any]) -> bool: @@ -543,7 +554,14 @@ def _changes(self, args: dict[str, Any], fields: tuple[str, ...], whole_persona: None, ) name = str(args.get("name") or "") if target is None else target.name - sent = args.get("persona") or {} + # What the submission names, a field it clears included. + cleared = [str(path) for path in args.get("clear") or ()] + named = set(args) | {path.split(".")[0] for path in cleared} + sent = set(args.get("persona") or {}) | { + path.split(".", 1)[1] for path in cleared if path.startswith("persona.") + } + if "persona" in cleared: + sent |= {"name", "gender", "age_group"} change = args if target is None else edited(target, args) path = self.workspace / "scenarios" / name / "scenario.json" try: @@ -554,11 +572,13 @@ def _changes(self, args: dict[str, Any], fields: tuple[str, ...], whole_persona: try: saved_now = Scenario.model_validate(held).model_dump() - submitted = Scenario.model_validate({**held, **change}).model_dump() + submitted = Scenario.model_validate( + change if target is not None else {**held, **change} + ).model_dump() except ValueError: saved_now, submitted = held, {**held, **change} if any( - field in change and submitted.get(field) != saved_now.get(field) + field in named and submitted.get(field) != saved_now.get(field) for field in fields ): return True @@ -594,9 +614,9 @@ async def call(request: Awaitable[dict[str, Any]]) -> dict[str, Any]: "Read the environment's scenarios exactly as the Scenarios tab shows them, with each " "one's tab number. Returns total_scenarios (the whole suite), matching (how many fit " "what you asked) and one page of rows. Narrow with any of: `numbers` (tab numbers, " - "e.g. [12, 250, 499]), `ranges` (e.g. [{\"from\": 3, \"to\": 7}]), `names`, `search` (every word must match a " + 'e.g. [12, 250, 499]), `ranges` (e.g. [{"from": 3, "to": 7}]), `names`, `search` (every word must match a ' "scenario's name, situation, use case or branch), `filters` (field to a list of " - "values, e.g. {\"persona.accent\": [\"Indian\"]}; fields: use_case, persona.name, " + 'values, e.g. {"persona.accent": ["Indian"]}; fields: use_case, persona.name, ' "persona.accent, persona.languages, persona.age_group, persona.gender, " "persona.location, persona.personality, persona.communication_style, " "background_noise, coverage.overlay, coverage.overlay_intensity, coverage.task, " diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index 948cf5308..b008ee25b 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -20,6 +20,7 @@ from collections import Counter, defaultdict from itertools import combinations from math import ceil, log +from pathlib import Path from typing import Any, ClassVar from pydantic import BaseModel, Field, field_validator, model_validator @@ -208,6 +209,16 @@ def _slug(name: str) -> str: return cleaned or "scenario-" + hashlib.sha256(name.encode()).hexdigest()[:12] +_USED_KEYS: dict[str, set[str]] = {} + + +def used_keys(destination: Any) -> set[str]: + """Every key the environment at ``destination`` has given a scenario, a dropped one's included, + so a new scenario never takes one and inherits another's row and run history. One live set per + environment: whatever adds to it is seen by every writer.""" + return _USED_KEYS.setdefault(str(Path(destination).resolve()), set()) + + def unused_key(key: str, taken: set[str]) -> str: """``key``, or it with a numbered suffix when another scenario already holds it.""" if not key or key not in taken: @@ -218,6 +229,24 @@ def unused_key(key: str, taken: set[str]) -> str: return f"{key}-{suffix}" +def with_unique_keys( + scenarios: list[Scenario], used: set[str] | None = None +) -> list[Scenario]: + """``scenarios`` with a key each: the first holder of a repeated key keeps it, and every other + one gets the next key no scenario holds or has held, the same one the platform gives it.""" + held = {one.scenario_key for one in scenarios} | (used or set()) + keys: set[str] = set() + out: list[Scenario] = [] + for one in scenarios: + if not one.scenario_key or one.scenario_key in keys: + key = unused_key(one.scenario_key or _slug(one.name), held | keys) + one = one.model_copy(update={"scenario_key": key}) + held.add(key) + keys.add(one.scenario_key) + out.append(one) + return out + + class Scenario(BaseModel): """One test: what changes, what is asked, what a correct agent does, what must hold.""" diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 59f23fe6c..f16ca47c5 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -68,6 +68,8 @@ _pinned_identity, _slug, unused_key, + used_keys, + with_unique_keys, _QUIET_INTERFACE, unpinned_callers, crowded_cells, @@ -305,9 +307,15 @@ def _forget_dropped(scenarios: list[Scenario], destination: Path) -> None: if not root.exists(): return keeping = {one.name for one in scenarios} + folded = {name.casefold(): name for name in keeping} for folder in root.iterdir(): - if folder.is_dir() and folder.name not in keeping: - shutil.rmtree(folder) + if not folder.is_dir() or folder.name in keeping: + continue + # On a case-insensitive disk a case-only rename is still this folder. + kept_as = root / folded.get(folder.name.casefold(), folder.name) + if kept_as != folder and kept_as.exists() and kept_as.samefile(folder): + continue + shutil.rmtree(folder) def load_scenarios(destination: Path) -> list[Scenario]: @@ -322,7 +330,9 @@ def load_scenarios(destination: Path) -> list[Scenario]: JOURNAL = "written.jsonl" -def journal_scenario(scenario: Scenario, destination: Path, replaces: str = "") -> None: +def journal_scenario( + scenario: Scenario, destination: Path, replaces: Scenario | None = None +) -> None: """Append one proved scenario to the journal, which is the only record a dead writer leaves. A delegated writer cannot write folders: saving the suite deletes every folder it does not know @@ -336,10 +346,10 @@ def journal_scenario(scenario: Scenario, destination: Path, replaces: str = "") path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as journal: # A rename drops the old name first, so recovering the journal never brings it back. - if replaces: - journal.write( - json.dumps({"dropped": replaces}, ensure_ascii=False) + "\n" - ) + # It names the entry it replaces, so a sibling's scenario of that name is untouched. + if replaces is not None: + dropped = {"dropped": replaces.name, "was": replaces.model_dump()} + journal.write(json.dumps(dropped, ensure_ascii=False) + "\n") journal.write(json.dumps(scenario.model_dump(), ensure_ascii=False) + "\n") except Exception as broke: # noqa: BLE001 - a scenario is never lost over bookkeeping logger.warning("could not journal %s: %s", scenario.name, broke) @@ -360,8 +370,11 @@ def journalled(destination: Path) -> list[Scenario]: continue try: entry = json.loads(line) - if isinstance(entry, dict) and set(entry) == {"dropped"}: - found.pop(str(entry["dropped"]), None) + if isinstance(entry, dict) and set(entry) == {"dropped", "was"}: + name = str(entry["dropped"]) + was = Scenario.model_validate(entry["was"]).model_dump() + if name in found and found[name].model_dump() == was: + del found[name] continue one = Scenario.model_validate(entry) except Exception: # noqa: BLE001 - a half-written line is expected while writers run @@ -852,8 +865,8 @@ def values(node: Any) -> list[str]: return [ "these fixed values are not where their source says: " + "; ".join(wrong) - + '. Write each source as `caller: ...` (the value is in what the caller says), `record: ...` ' - '(the world holds it) or `rule: ""` (quoted from the agent\'s ' + + ". Write each source as `caller: ...` (the value is in what the caller says), `record: ...` " + "(the world holds it) or `rule: \"\"` (quoted from the agent's " "instructions and containing the value). A value none of these gives is the agent's choice; " "take it out of `fixed`" ] @@ -884,8 +897,9 @@ def _credentials_the_world_lacks(scenario: Scenario, trial: Any) -> list[str]: def edited(held: Scenario, change: dict[str, Any]) -> dict[str, Any]: - """``held`` with ``change`` applied: a field it names is replaced, a mapping such as the persona - changes only the entries it names, at any depth, and ``None`` clears a field back to its default. + """``held`` with ``change`` applied: a field it names is replaced, and a mapping such as the + persona changes only the entries it names, at any depth. Each field ``clear`` names, a persona + field as ``persona.accent``, goes back to its default. """ def apply(saved: dict[str, Any], given: dict[str, Any]) -> dict[str, Any]: @@ -899,38 +913,18 @@ def apply(saved: dict[str, Any], given: dict[str, Any]) -> dict[str, Any]: out[field] = value return out - return apply(held.model_dump(exclude_none=True), change) - - -def rename_scenario( - held: Scenario, - name: str, - *, - kept: list[Scenario], - world_root: Path, - catalogue: Catalogue, - persist: bool = True, -) -> dict[str, Any]: - """Give a kept scenario a new name without proving it again: what it tests, its key, its place - in the suite and its proof are all unchanged, so the gates have nothing new to check.""" - try: - renamed = Scenario.model_validate({**held.model_dump(), "name": name}) - except ValueError as invalid: - return _err(f"Not renamed. {invalid}"[:600]) - if any(one.name == renamed.name for one in kept if one is not held): - return _err( - f"Not renamed. Another scenario is already named {renamed.name}; choose another name." - ) - spot = next(index for index, one in enumerate(kept) if one is held) - kept[spot] = renamed - if persist: - write_scenarios(kept, world_root, catalogue) - else: - journal_scenario(renamed, world_root, replaces=held.name) - return _ok( - f"{held.name} renamed to {renamed.name}. Nothing it tests changed, so it was not proved " - "again." + result = apply( + held.model_dump(exclude_none=True), + {field: value for field, value in change.items() if field != "clear"}, ) + for path in change.get("clear") or (): + *parents, last = str(path).split(".") + node: Any = result + for part in parents: + node = node.get(part) if isinstance(node, dict) else None + if isinstance(node, dict): + node.pop(last, None) + return result def accept_scenario( @@ -947,9 +941,15 @@ def accept_scenario( vocabulary: list[str] | None = None, spoken: bool = True, instructions: str = "", + used: set[str] | None = None, + renaming: bool = False, ) -> dict[str, Any]: """Validate one scenario, then prove it. A plain function so both halves are testable. + ``renaming`` keeps a proved scenario under a new name without proving it again: what it tests + is unchanged. ``used`` is every key the environment has given a scenario, so a new one never + takes a dropped scenario's key and inherits its run history. + ``persist`` is off for a writer that shares the destination with siblings: writing the suite removes every folder not in the writer's own list, so persisting here would delete whatever the others have proved. Those writers keep their work in ``kept`` and the caller saves once. @@ -968,46 +968,52 @@ def accept_scenario( place_for(scenario.name, scenario.fixture, scenario.instruction) or True ) - # Read against the world this scenario actually runs in, so a setup that creates the table - # a check reads is not reported as referring to something that does not exist. - trial, _applied, _ready = prepared(scenario, world_root) - try: - problems = validate_scenario( - scenario, - catalogue, - trial.state(), - simulator_prompt, - allow_empty_solution=allow_empty_solution, - spoken=spoken, - ) - problems.extend(contract_sequence_problems(scenario, hard_constraints or [])) - problems.extend(_credentials_the_world_lacks(scenario, trial)) - problems.extend(_identifiers_the_instruction_invents(scenario, trial)) - problems.extend(_fixed_sources_that_do_not_hold(scenario, trial, instructions)) - problems.extend(_steps_the_contract_refuses(scenario, world_root)) - problems.extend(_overlay_asserts_nothing(scenario, catalogue)) - problems.extend(_shared_check_bound_to_one_task(scenario, catalogue, kept)) - finally: - trial.close() + proof = None + if not renaming: + # Read against the world this scenario actually runs in, so a setup that creates the table + # a check reads is not reported as referring to something that does not exist. + trial, _applied, _ready = prepared(scenario, world_root) + try: + problems = validate_scenario( + scenario, + catalogue, + trial.state(), + simulator_prompt, + allow_empty_solution=allow_empty_solution, + spoken=spoken, + ) + problems.extend( + contract_sequence_problems(scenario, hard_constraints or []) + ) + problems.extend(_credentials_the_world_lacks(scenario, trial)) + problems.extend(_identifiers_the_instruction_invents(scenario, trial)) + problems.extend( + _fixed_sources_that_do_not_hold(scenario, trial, instructions) + ) + problems.extend(_steps_the_contract_refuses(scenario, world_root)) + problems.extend(_overlay_asserts_nothing(scenario, catalogue)) + problems.extend(_shared_check_bound_to_one_task(scenario, catalogue, kept)) + finally: + trial.close() - if problems: - return _err( - "Not kept. Fix these and submit again:\n - " + "\n - ".join(problems) - ) + if problems: + return _err( + "Not kept. Fix these and submit again:\n - " + "\n - ".join(problems) + ) - proof = prove(scenario, catalogue, world_root) - if not proof.holds: - said = f"Not kept. {proof.why()}" - # Code written against the wrong collection shape is the commonest way setup, ready and a - # check fail here, and the exception alone does not say which collections are mappings and - # which are lists. The world is asked, so the answer names them. - if "attribute" in said.lower() or "not subscriptable" in said.lower(): - world = restore(world_root) - try: - said += f"\n\n{world.shapes()}" - finally: - world.close() - return _err(said) + proof = prove(scenario, catalogue, world_root) + if not proof.holds: + said = f"Not kept. {proof.why()}" + # Code written against the wrong collection shape is the commonest way setup, ready and a + # check fail here, and the exception alone does not say which collections are mappings and + # which are lists. The world is asked, so the answer names them. + if "attribute" in said.lower() or "not subscriptable" in said.lower(): + world = restore(world_root) + try: + said += f"\n\n{world.shapes()}" + finally: + world.close() + return _err(said) # The scenario this one replaces: the one whose key it carries, else the one with its name. It # keeps that key, so a rename is the same scenario under a new name, not a new one. @@ -1025,20 +1031,29 @@ def accept_scenario( ) renamed_from = previous.name del kept[spot] + keys = {one.scenario_key for one in kept} | (used or set()) if previous is not None: scenario.scenario_key = previous.scenario_key else: scenario.scenario_key = unused_key( - _slug(scenario.name) if named else scenario.scenario_key, - {one.scenario_key for one in kept}, + _slug(scenario.name) if named else scenario.scenario_key, keys ) + if used is not None: + used.add(scenario.scenario_key) replaced = any(one.name == scenario.name for one in kept) - if replaced and rename_on_collision: + # An edit names its scenario by key, so it replaces that one; only a new submission that + # repeats a name is kept beside it under a numbered name. + if replaced and rename_on_collision and not (named and previous is not None): taken = {one.name for one in kept} stem, suffix = scenario.name, 2 while f"{stem}-{suffix}" in taken: suffix += 1 - scenario = scenario.model_copy(update={"name": f"{stem}-{suffix}", "scenario_key": ""}) + renamed = f"{stem}-{suffix}" + scenario = scenario.model_copy( + update={"name": renamed, "scenario_key": unused_key(_slug(renamed), keys)} + ) + if used is not None: + used.add(scenario.scenario_key) replaced = False spot = None else: @@ -1069,12 +1084,16 @@ def accept_scenario( else: # A writer sharing the destination cannot write folders, so the journal is where its proved # work survives the session that proved it. - journal_scenario(scenario, world_root, replaces=renamed_from) + journal_scenario( + scenario, world_root, replaces=previous if renamed_from else None + ) # Say what the proof did not cover. On a lane where the target's tools have no endpoints, every # solution step is recorded without running, so "all three gates pass" is true and misleading: # the checks were exercised, the solution was not. unproved = ( - "\nNOT PROVED: " + "" + if proof is None + else "\nNOT PROVED: " + f"{len(proof.assumed)} of {len(scenario.solution)} solution steps were recorded without " "running, because these tools have no endpoint in this environment: " + ", ".join(proof.assumed) @@ -1084,7 +1103,9 @@ def accept_scenario( else "" ) proof_summary = ( - "The world is ready for it, and its sub-goals will be judged from the call: the " + "Nothing it tests changed, so it was not proved again." + if proof is None + else "The world is ready for it, and its sub-goals will be judged from the call: the " "recording, the transcript, every action with its arguments and the records left." if proof.judged_only else "All three gates pass: the world is ready for it, the reference solution passes " @@ -1242,6 +1263,13 @@ def scenario_tools( catalogue = load_catalogue(destination) simulator_prompt = load_simulator_prompt(destination) target = {"count": wanted} + # Keys this environment has given out, a dropped scenario's included: never given to a new one. + used = used_keys(destination) + unique = with_unique_keys(kept, used) + if can_save and start_from is None and unique != kept: + write_scenarios(unique, destination, catalogue) + kept[:] = unique + used.update(one.scenario_key for one in kept) def cap() -> int: return int(target["count"] or 0) if can_grow else wanted @@ -1742,8 +1770,8 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "type": "object", "additionalProperties": {"type": "string"}, "description": "Where this scenario sits on the axes the plan varied, one " - "value per axis, for example {\"task\": \"book_ride\", \"overlay\": " - "\"interruption\"}. Copy it from your brief. It is what lets the suite " + 'value per axis, for example {"task": "book_ride", "overlay": ' + '"interruption"}. Copy it from your brief. It is what lets the suite ' "report how much of the space was tested; it never reaches the caller.", }, }, @@ -1845,7 +1873,10 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] rename_on_collision=rename_on_collision, vocabulary=target.get("keywords"), spoken=_is_spoken(contract), - instructions="\n".join([contract.system_prompt_excerpt, *contract.hard_constraints]), + instructions="\n".join( + [contract.system_prompt_excerpt, *contract.hard_constraints] + ), + used=used, ) if not result.get("is_error"): exploration["since_submit"] = 0 @@ -2371,14 +2402,30 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: ) return _ok(said) + def optional(schema: Any) -> Any: + """``schema`` with nothing required at any depth, so an edit sends only what changes.""" + if isinstance(schema, list): + return [optional(item) for item in schema] + if not isinstance(schema, dict): + return schema + return { + key: optional(value) for key, value in schema.items() if key != "required" + } + edit_schema = { **submit_scenario.input_schema, "properties": { - **submit_scenario.input_schema["properties"], + **optional(submit_scenario.input_schema["properties"]), "scenario_key": { "type": "string", "description": "The scenario's key, read from it.", }, + "clear": { + "type": "array", + "items": {"type": "string"}, + "description": "Fields to set back to their default, a persona field as " + "`persona.accent`.", + }, }, "required": ["scenario_key"], } @@ -2386,9 +2433,10 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: @tool( "edit_scenario", "Change a kept scenario: give its `scenario_key` and only what changes, such as a new " - "`name` or the persona fields that change; `null` clears a field. Everything else keeps " - "its saved value, and the scenario keeps its key, number, place and history. A new name " - "alone is applied as it is; any other change is checked exactly like `submit_scenario`.", + "`name` or the persona fields that change; name a field in `clear` to set it back to its " + "default. Everything else keeps its saved value, and the scenario keeps its key, number, " + "place and history. A new name alone is applied as it is; any other change is checked " + "exactly like `submit_scenario`.", edit_schema, ) async def edit_scenario(args: dict[str, Any]) -> dict[str, Any]: @@ -2402,17 +2450,29 @@ async def edit_scenario(args: dict[str, Any]) -> dict[str, Any]: field: value for field, value in args.items() if field != "scenario_key" } name = str(change.get("name") or "").strip() - if set(change) == {"name"} and name and name != held.name: - result = rename_scenario( - held, - name, - kept=kept, - world_root=world_root, - catalogue=catalogue, - persist=can_save, + if ( + set(change) != {"name"} + and name + and name != held.name + and any(one.name == name for one in kept) + ): + return _err( + f"Not changed. Another scenario is already named {name}; choose another name." ) - if not result.get("is_error"): - return result + if set(change) == {"name"} and name == held.name: + return _ok(f"Nothing changed: {held.name} already has that name.") + if set(change) != {"name"}: + return await submit_scenario.handler(edited(held, args)) + result = accept_scenario( + {**held.model_dump(), "name": name}, + world_root=world_root, + catalogue=catalogue, + kept=kept, + persist=can_save, + used=used, + renaming=True, + ) + if result.get("is_error"): said = str((result.get("content") or [{}])[0].get("text") or "") seen = _count_refusal(refused, held.name, said) if seen >= _REFUSALS_BEFORE_MOVING_ON: @@ -2420,8 +2480,7 @@ async def edit_scenario(args: dict[str, Any]) -> dict[str, Any]: f"{said}\n\nThis is refusal {seen} of this rename for the same reason, so " "asking again will not work. Keep the current name or choose another one." ) - return result - return await submit_scenario.handler(edited(held, args)) + return result server = tool_server( name=SCENARIO_SERVER, diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index aaa989309..2e1065cac 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -40,7 +40,7 @@ from .contract import AgentContract from .job import HarnessJob from .poc_guest_booking import guest_booking_pin_guidance, guest_booking_pin_scenario_problem -from .scenario import Scenario, unused_key, voicemail_enabled +from .scenario import Scenario, used_keys, voicemail_enabled, with_unique_keys from .scenario_tools import ( SCENARIO_SERVER, TOOL_NAMES, @@ -906,7 +906,9 @@ async def attempt_slice() -> Any: return list(kept) -def merged(written: list[list[Scenario]]) -> list[Scenario]: +def merged( + written: list[list[Scenario]], used: set[str] | None = None +) -> list[Scenario]: """One suite out of several writers, with folder-name collisions renamed rather than dropped. Asking for twenty scenarios has to return twenty. Two scenarios may legitimately share a use @@ -919,7 +921,6 @@ def merged(written: list[list[Scenario]]) -> list[Scenario]: """ suite: list[Scenario] = [] taken: set[str] = set() - keys: set[str] = set() for batch in written: for one in batch: if one.name in taken: @@ -930,15 +931,10 @@ def merged(written: list[list[Scenario]]) -> list[Scenario]: update={"name": f"{stem}-{suffix}", "scenario_key": ""} ) logger.info("renamed a duplicate folder name to %s", one.name) - # A key names one scenario, so a repeat, even one already saved, gets its own. - if one.scenario_key and one.scenario_key in keys: - one = one.model_copy( - update={"scenario_key": unused_key(one.scenario_key, keys)} - ) taken.add(one.name) - keys.add(one.scenario_key) suite.append(one) - return suite + # A key names one scenario, so a repeat, even one already saved, gets its own. + return with_unique_keys(suite, used) def _suite_summary(suite: list[Scenario]) -> str: @@ -1138,7 +1134,7 @@ async def guarded(mine: Slice, siblings: list[Slice], index: int) -> list[Scenar *(guarded(one, allocation, index) for index, one in enumerate(allocation)), return_exceptions=False, ) - proved = merged([load_scenarios(destination), *written]) + proved = merged([load_scenarios(destination), *written], used_keys(destination)) # What a writer proved but never handed back, because its session died after proving it. Matched # by name so a scenario already in hand is not added twice under a numbered name. recovered = [ @@ -1154,7 +1150,7 @@ async def guarded(mine: Slice, siblings: list[Slice], index: int) -> list[Scenar ) if on_event: on_event({"type": "recovered", "kept": len(recovered)}) - suite = merged([proved, recovered]) + suite = merged([proved, recovered], used_keys(destination)) # Read the whole thing and fill what nobody covered. Bounded, because a reviewer asked # twice will always find something smaller to say. @@ -1189,7 +1185,7 @@ async def guarded(mine: Slice, siblings: list[Slice], index: int) -> list[Scenar return_exceptions=False, ) before = len(suite) - suite = merged([suite, *more]) + suite = merged([suite, *more], used_keys(destination)) allocation = [*allocation, *missing] if len(suite) == before: break diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index 67bb8de56..e3a8d15e8 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -346,7 +346,9 @@ async def ask(tool, payload, context): is True ) assert "ask for a cancellation" in asked.pop() - assert not runtime._changes_situation({"scenario_key": "one", "persona": {"name": "Ana"}}) + assert not runtime._changes_situation( + {"scenario_key": "one", "persona": {"name": "Ana"}} + ) assert runtime._changes_situation({"scenario_key": "one", "persona": {"accent": "Indian"}}) answers.extend(["No, keep it", "Which scenarios use it?"]) @@ -369,7 +371,98 @@ async def ask(tool, payload, context): assert len(calls) == 7 -def test_the_harness_builds_from_an_empty_workspace_with_the_tools_for_what_exists(tmp_path, monkeypatch): +def test_an_edit_and_a_clear_go_through_the_same_confirmation_as_a_submission( + tmp_path, monkeypatch +): + from fi.alk.harness.backends import ToolSpec + + runtime = _runtime(tmp_path, BuilderClient(), monkeypatch) + (tmp_path / "scenarios" / "one" / "scenario.json").write_text( + json.dumps( + { + "name": "one", + "scenario_key": "one", + "instruction": "ask for a refund", + "persona": {"name": "Ana", "accent": "American"}, + "fixture": {"origin": "seed", "order": "7"}, + "solution": [ + { + "tool": "refund", + "arguments": {"id": "7"}, + "environment_arguments": {}, + } + ], + } + ) + ) + calls, asked, answers = [], [], [] + + async def handler(args): + calls.append(args) + return {"content": [{"type": "text", "text": "kept"}]} + + async def ask(tool, payload, context): + asked.append(payload["question"]) + return {"content": [{"type": "text", "text": answers.pop(0)}]} + + runtime._ask = ask + edit = runtime._confirmed(ToolSpec("edit_scenario", "", {}, handler)) + assert ( + asyncio.run( + edit.handler({"scenario_key": "one", "persona": {"accent": "Indian"}}) + ).get("is_error") + is None + ) + assert asked == [] + answers.append("No") + assert ( + asyncio.run(edit.handler({"scenario_key": "one", "clear": ["solution"]}))[ + "is_error" + ] + is True + ) + assert len(asked) == 1 + assert runtime._changes_situation({"scenario_key": "one", "clear": ["fixture"]}) + assert runtime._changes_situation({"scenario_key": "one", "clear": ["persona"]}) + assert runtime._changes_situation( + {"scenario_key": "one", "clear": ["persona.accent"]} + ) + runtime._from_platform = True + assert ( + asyncio.run(edit.handler({"scenario_key": "one", "clear": ["fixture"]}))[ + "is_error" + ] + is True + ) + assert len(calls) == 1 + + +def test_the_keys_the_platform_has_used_are_never_given_to_a_new_scenario( + tmp_path, monkeypatch +): + from fi.alk.harness.scenario import used_keys + + monkeypatch.setenv("ALK_HARNESS", "vertex-gemini") + policy = tmp_path / "policy.yaml" + policy.write_text( + "schema_version: 1\nharness_chat:\n enabled: true\n", encoding="utf-8" + ) + HostedChatRuntime( + client=BuilderClient(), # type: ignore[arg-type] + policy=load_chat_policy(policy), + workspace=tmp_path, + source=tmp_path, + job={ + "metadata": {"agent_name": "demo", "used_scenario_keys": ["two", "one"]}, + "scenario_count": 1, + }, + ) + assert {"one", "two"} <= used_keys(tmp_path) + + +def test_the_harness_builds_from_an_empty_workspace_with_the_tools_for_what_exists( + tmp_path, monkeypatch +): monkeypatch.setenv("ALK_HARNESS", "claude") policy = tmp_path / "policy.yaml" policy.write_text( @@ -423,7 +516,6 @@ async def overview(self): assert over["is_error"] and "the build asked for 3" in over["content"][0]["text"] assert not (workspace / ".futureagi-build-finished.json").exists() - async def commands(): return [{"sequence": 7, "message_id": "m7", "kind": "user_message", "payload": {"content": "add a lapsed policy"}}] diff --git a/tests/test_harness.py b/tests/test_harness.py index a782332f4..8bfa08242 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -8107,7 +8107,7 @@ def refuse(*_args, **_kwargs): assert [one.name for one in kept] == ["siparis-tekrari", "orders-again"] -def test_an_edit_clears_a_field_given_as_null_and_merges_nested_mappings(): +def test_an_edit_clears_what_clear_names_and_merges_nested_mappings(): from fi.alk.harness.scenario import Scenario from fi.alk.harness.scenario_tools import edited @@ -8127,7 +8127,7 @@ def test_an_edit_clears_a_field_given_as_null_and_merges_nested_mappings(): ) change = edited( held, - {"persona": {"accent": None}, "fixture": {"order": {"size": "small"}}}, + {"clear": ["persona.accent"], "fixture": {"order": {"size": "small"}}}, ) after = Scenario.model_validate(change) assert ( @@ -8165,6 +8165,240 @@ def test_a_writer_s_rename_never_brings_the_old_name_back_from_the_journal(tmp_p ] +def _edit_tools(tmp_path, **options): + from fi.alk.harness.scenario_tools import scenario_tools + + root, contract, catalogue = _built_environment(tmp_path) + server, kept = scenario_tools(contract, root, root, wanted=4, **options) + return root, catalogue, kept, {one.name: one for one in server.tools} + + +@pytest.mark.parametrize( + "args", + [ + {"scenario_key": "k", "persona": {"name": "Meera Iyer"}}, + {"scenario_key": "k", "clear": ["persona.accent", "fixture", "sub_goals"]}, + {"scenario_key": "k", "name": "renamed"}, + ], +) +def test_the_edits_edit_scenario_describes_pass_its_schema_on_both_backends( + tmp_path, args +): + import jsonschema + from claude_agent_sdk import SdkMcpTool, _build_input_schema + + from fi.alk.harness.backends.claude import _gateway_compatible_schema + + *_, tools = _edit_tools(tmp_path) + spec = tools["edit_scenario"] + sent = _build_input_schema( + SdkMcpTool( + name=spec.name, + description="", + input_schema=spec.input_schema, + handler=spec.handler, + ) + ) + jsonschema.validate(instance=args, schema=sent) + jsonschema.validate( + instance=args, schema=_gateway_compatible_schema(spec.input_schema) + ) + + +def test_a_writer_s_edit_by_key_replaces_its_scenario_rather_than_copying_it(tmp_path): + import asyncio + + _root, _catalogue, kept, tools = _edit_tools( + tmp_path, can_save=False, start_from=[] + ) + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + key = kept[0].scenario_key + said = asyncio.run( + tools["edit_scenario"].handler( + {"scenario_key": key, "instruction": "Order a single Big Mac."} + ) + ) + assert not said.get("is_error"), said + assert [(one.name, one.scenario_key) for one in kept] == [("adds-a-big-mac", key)] + + +def test_a_new_scenario_never_takes_a_key_the_environment_has_used(tmp_path): + import asyncio + + from fi.alk.harness.scenario import used_keys + + root, _catalogue, kept, tools = _edit_tools(tmp_path) + # The platform's keys, a dropped scenario's among them. + used_keys(root).add("adds-a-big-mac") + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + assert kept[0].scenario_key == "adds-a-big-mac-2" + + +def test_a_scenario_dropped_in_the_chat_never_lends_its_key_to_a_new_one(tmp_path): + import asyncio + + _root, _catalogue, kept, tools = _edit_tools(tmp_path) + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + dropped = kept[0].scenario_key + assert not asyncio.run( + tools["drop_scenario"].handler({"name": "adds-a-big-mac"}) + ).get("is_error") + assert not asyncio.run( + tools["submit_scenario"].handler(_delta(instruction="Order two Big Macs.")) + ).get("is_error") + assert kept[0].scenario_key not in ("", dropped) + + +def test_repeated_saved_keys_are_repaired_when_the_suite_is_opened(tmp_path): + import asyncio + + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenario_tools import ( + load_scenarios, + scenario_tools, + write_scenarios, + ) + + root, contract, catalogue = _built_environment(tmp_path) + first = Scenario.model_validate({**_delta(name="first"), "scenario_key": "k"}) + second = Scenario.model_validate({**_delta(name="second"), "scenario_key": "k"}) + write_scenarios([first, second], root, catalogue) + server, kept = scenario_tools(contract, root, root, wanted=2) + assert [(one.name, one.scenario_key) for one in load_scenarios(root)] == [ + ("first", "k"), + ("second", "k-2"), + ] + edit = next(one for one in server.tools if one.name == "edit_scenario") + said = asyncio.run(edit.handler({"scenario_key": "k-2", "name": "second-renamed"})) + assert not said.get("is_error"), said + assert [(one.name, one.scenario_key) for one in kept] == [ + ("first", "k"), + ("second-renamed", "k-2"), + ] + + +def test_merging_never_moves_a_repeat_onto_a_key_another_scenario_holds(): + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenarios import merged + + def one(name, key=""): + return Scenario.model_validate( + {"name": name, "instruction": "x", "scenario_key": key} + ) + + suite = merged([[one("a", "k"), one("b", "k"), one("c", "k-2")]]) + assert [(s.name, s.scenario_key) for s in suite] == [ + ("a", "k"), + ("b", "k-3"), + ("c", "k-2"), + ] + assert [(s.name, s.scenario_key) for s in merged([suite])] == [ + (s.name, s.scenario_key) for s in suite + ] + renamed = merged([[one("other", "x-2")], [one("x")], [one("x")]]) + keys = [s.scenario_key for s in renamed] + assert len(set(keys)) == len(keys) + + +def test_a_writer_s_rename_leaves_a_sibling_s_scenario_of_that_name_in_the_journal( + tmp_path, +): + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenario_tools import journal_scenario, journalled + + root, *_ = _built_environment(tmp_path) + mine = Scenario.model_validate(_delta(name="shared-name", instruction="Order two.")) + sibling = Scenario.model_validate(_delta(name="shared-name")) + journal_scenario(mine, root) + journal_scenario(sibling, root) + journal_scenario( + mine.model_copy(update={"name": "mine-renamed"}), root, replaces=mine + ) + assert sorted(one.name for one in journalled(root)) == [ + "mine-renamed", + "shared-name", + ] + + +def test_a_writer_s_rename_by_edit_scenario_is_journalled(tmp_path): + import asyncio + + from fi.alk.harness.scenario_tools import journalled + + root, _catalogue, kept, tools = _edit_tools(tmp_path, can_save=False, start_from=[]) + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + key = kept[0].scenario_key + said = asyncio.run( + tools["edit_scenario"].handler({"scenario_key": key, "name": "renamed"}) + ) + assert not said.get("is_error"), said + assert [(one.name, one.scenario_key) for one in journalled(root)] == [ + ("renamed", key) + ] + + +def test_a_case_only_rename_keeps_the_scenario_on_disk(tmp_path): + import asyncio + + from fi.alk.harness.scenario_tools import load_scenarios + + root, _catalogue, kept, tools = _edit_tools(tmp_path) + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + key = kept[0].scenario_key + said = asyncio.run( + tools["edit_scenario"].handler({"scenario_key": key, "name": "Adds-A-Big-Mac"}) + ) + assert not said.get("is_error"), said + assert [one.name for one in load_scenarios(root)] == ["Adds-A-Big-Mac"] + + +def test_renaming_a_scenario_to_its_own_name_changes_nothing(tmp_path, monkeypatch): + import asyncio + + from fi.alk.harness import scenario_tools as tools_module + + _root, _catalogue, kept, tools = _edit_tools(tmp_path) + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + monkeypatch.setattr( + tools_module, "prove", lambda *a, **k: (_ for _ in ()).throw(AssertionError) + ) + said = asyncio.run( + tools["edit_scenario"].handler( + {"scenario_key": kept[0].scenario_key, "name": "adds-a-big-mac"} + ) + ) + assert not said.get("is_error"), said + + +def test_an_edit_onto_a_taken_name_is_refused_before_anything_is_proved( + tmp_path, monkeypatch +): + import asyncio + + from fi.alk.harness import scenario_tools as tools_module + + _root, _catalogue, kept, tools = _edit_tools(tmp_path) + for name in ("adds-a-big-mac", "orders-again"): + assert not asyncio.run( + tools["submit_scenario"].handler( + _delta(name=name, instruction=f"Order {name}") + ) + ).get("is_error") + monkeypatch.setattr( + tools_module, "prove", lambda *a, **k: (_ for _ in ()).throw(AssertionError) + ) + said = asyncio.run( + tools["edit_scenario"].handler( + { + "scenario_key": kept[1].scenario_key, + "name": "adds-a-big-mac", + "instruction": "Order one", + } + ) + ) + assert said.get("is_error") and "already named" in said["content"][0]["text"] + + def test_a_redeclared_grid_only_ever_grows(tmp_path): import asyncio From f276ec506558c4d68daaa5c48ecf45871229f06e Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 26 Oct 2026 19:57:28 +0530 Subject: [PATCH 14/15] fix(harness): TH-8748 an edit that changes nothing says so and points to clear --- src/fi/alk/harness/scenario_tools.py | 5 +++++ tests/test_harness.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index f16ca47c5..a38c6580f 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -2462,6 +2462,11 @@ async def edit_scenario(args: dict[str, Any]) -> dict[str, Any]: if set(change) == {"name"} and name == held.name: return _ok(f"Nothing changed: {held.name} already has that name.") if set(change) != {"name"}: + if edited(held, args) == edited(held, {}): + return _ok( + f"Nothing changed: {held.name} already has these values. A field left out " + "keeps its saved value; to remove one, name it in `clear`." + ) return await submit_scenario.handler(edited(held, args)) result = accept_scenario( {**held.model_dump(), "name": name}, diff --git a/tests/test_harness.py b/tests/test_harness.py index 8bfa08242..d5bc110be 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -8014,6 +8014,13 @@ def test_an_edit_sends_only_what_changes_and_keeps_the_rest(tmp_path): assert kept[0].persona.name == "Meera Iyer" assert kept[0].persona.initial_message == "Hi, I want a Big Mac." assert kept[0].persona.languages == ["English"] + same = dict(kept[0].persona.model_dump(exclude_none=True)) + same.pop("languages") + said = asyncio.run(edit.handler({"scenario_key": key, "persona": same})) + assert not said.get("is_error"), said + assert said["content"][0]["text"].startswith("Nothing changed: orders-one-big-mac") + assert "`clear`" in said["content"][0]["text"] + assert kept[0].persona.languages == ["English"] unknown = asyncio.run(edit.handler({"scenario_key": "not-here", "name": "x"})) assert unknown.get("is_error") and "not-here" in unknown["content"][0]["text"] assert [one.name for one in kept] == ["orders-one-big-mac"] From 503a72dc6055f7359021dc54501021df1ac873e7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 26 Oct 2026 20:29:46 +0530 Subject: [PATCH 15/15] chore(harness): TH-8748 re-run checks