diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a33a43c8..a4d528e05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ All notable release changes for Agent Learning Kit are tracked here. Post-rc.1 increments on the release branch. Each one adds gates on top of the 66 proved at rc.1; the rc.1 entry below is historical and unchanged. +- Builder chat: `edit_scenario` changes a kept scenario by its key with only the fields that change + (`clear` resets a field), keeping its key, number, place and history; a rename alone is not + proved again. A new scenario never takes a key the environment has used, a dropped scenario's + included, and a repeated saved key is repaired so each scenario keeps its own. - A chat conversation never sends the scenario's instruction as the simulated person's first message: an empty or echoed opening is asked for again, then for a greeting in their language, and the person's own session records what the agent actually heard. diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 7e83b4450..3c4b08d07 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -60,6 +60,7 @@ _NEEDS_CONFIRMATION = frozenset( { "submit_scenario", + "edit_scenario", "drop_scenario", "drop_scenarios", "change_world", @@ -71,6 +72,7 @@ "add_sub_goal", } ) +_SUBMITS = ("submit_scenario", "edit_scenario") _CONFIRM = "Yes, make this change" _RUN_FINISHED = frozenset({"completed", "failed", "canceled"}) _RUN_POLL_SECONDS = 20.0 @@ -89,6 +91,7 @@ _WRITING_TOOLS = frozenset( { "submit_scenario", + "edit_scenario", "drop_scenario", "drop_scenarios", "save_scenarios", @@ -251,6 +254,15 @@ def __init__( self.workspace = workspace self.source = source self.job = job + from .scenario import used_keys + + # Every key the environment has given a scenario, a dropped one's included, so a new + # scenario never takes one and inherits another's row and run history. + used_keys(workspace).update( + str(key) + for key in (job.get("metadata") or {}).get("used_scenario_keys") or () + if key + ) backend = resolve() self.backend_name = backend.name self.event_store: Any = None @@ -461,7 +473,7 @@ def _confirmed(self, spec: Any) -> Any: async def guarded(args: dict[str, Any]) -> Any: if self._from_platform: if spec.name in ("drop_scenarios", "drop_scenario") or ( - spec.name == "submit_scenario" and self._changes_situation(args) + spec.name in _SUBMITS and self._changes_situation(args) ): return { "content": [ @@ -476,7 +488,7 @@ async def guarded(args: dict[str, Any]) -> Any: "is_error": True, } return await handler(args) - if spec.name == "submit_scenario" and not self._changes_meaning(args): + if spec.name in _SUBMITS and not self._changes_meaning(args): return await handler(args) if spec.name == "add_sub_goal" and not self._replaces_sub_goal(args): return await handler(args) @@ -510,7 +522,9 @@ async def guarded(args: dict[str, Any]) -> Any: def _changes_situation(self, args: dict[str, Any]) -> bool: """Whether a submission changes what an existing scenario tests, not how it is graded.""" return self._changes( - args, ("instruction", "use_case", "branch", "tests", "coverage", "fixture"), whole_persona=True + args, + ("instruction", "use_case", "branch", "tests", "coverage", "fixture"), + whole_persona=True, ) def _replaces_sub_goal(self, args: dict[str, Any]) -> bool: @@ -528,14 +542,47 @@ def _changes(self, args: dict[str, Any], fields: tuple[str, ...], whole_persona: `whole_persona` compares the persona the submission would save, so a changed language or a dropped persona counts: a submission replaces the scenario rather than patching it. """ - path = self.workspace / "scenarios" / str(args.get("name") or "") / "scenario.json" + from .scenario_tools import edited, load_scenarios + + scenario_key = str(args.get("scenario_key") or "") + target = next( + ( + one + for one in load_scenarios(self.workspace) + if scenario_key and one.scenario_key == scenario_key + ), + None, + ) + name = str(args.get("name") or "") if target is None else target.name + # What the submission names, a field it clears included. + cleared = [str(path) for path in args.get("clear") or ()] + named = set(args) | {path.split(".")[0] for path in cleared} + sent = set(args.get("persona") or {}) | { + path.split(".", 1)[1] for path in cleared if path.startswith("persona.") + } + if "persona" in cleared: + sent |= {"name", "gender", "age_group"} + change = args if target is None else edited(target, args) + path = self.workspace / "scenarios" / name / "scenario.json" try: held = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError): return False - if any(field in args and args.get(field) != held.get(field) for field in fields): + from .scenario import Scenario + + try: + saved_now = Scenario.model_validate(held).model_dump() + submitted = Scenario.model_validate( + change if target is not None else {**held, **change} + ).model_dump() + except ValueError: + saved_now, submitted = held, {**held, **change} + if any( + field in named and submitted.get(field) != saved_now.get(field) + for field in fields + ): return True - persona, before = args.get("persona") or {}, held.get("persona") or {} + persona, before = change.get("persona") or {}, held.get("persona") or {} if whole_persona: from .scenario import Persona @@ -547,8 +594,8 @@ def saved(one: dict[str, Any]) -> dict[str, Any]: except ValueError: # an unreadable persona is never a safe unattended change return True return any( - key in persona and persona.get(key) != before.get(key) - for key in ("name", "gender", "age_group") + field in sent and persona.get(field) != before.get(field) + for field in ("name", "gender", "age_group") ) def _builder_platform_server(self): @@ -567,9 +614,9 @@ async def call(request: Awaitable[dict[str, Any]]) -> dict[str, Any]: "Read the environment's scenarios exactly as the Scenarios tab shows them, with each " "one's tab number. Returns total_scenarios (the whole suite), matching (how many fit " "what you asked) and one page of rows. Narrow with any of: `numbers` (tab numbers, " - "e.g. [12, 250, 499]), `ranges` (e.g. [{\"from\": 3, \"to\": 7}]), `names`, `search` (every word must match a " + 'e.g. [12, 250, 499]), `ranges` (e.g. [{"from": 3, "to": 7}]), `names`, `search` (every word must match a ' "scenario's name, situation, use case or branch), `filters` (field to a list of " - "values, e.g. {\"persona.accent\": [\"Indian\"]}; fields: use_case, persona.name, " + 'values, e.g. {"persona.accent": ["Indian"]}; fields: use_case, persona.name, ' "persona.accent, persona.languages, persona.age_group, persona.gender, " "persona.location, persona.personality, persona.communication_style, " "background_noise, coverage.overlay, coverage.overlay_intensity, coverage.task, " diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index e91f95598..b008ee25b 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -20,6 +20,7 @@ from collections import Counter, defaultdict from itertools import combinations from math import ceil, log +from pathlib import Path from typing import Any, ClassVar from pydantic import BaseModel, Field, field_validator, model_validator @@ -208,6 +209,44 @@ def _slug(name: str) -> str: return cleaned or "scenario-" + hashlib.sha256(name.encode()).hexdigest()[:12] +_USED_KEYS: dict[str, set[str]] = {} + + +def used_keys(destination: Any) -> set[str]: + """Every key the environment at ``destination`` has given a scenario, a dropped one's included, + so a new scenario never takes one and inherits another's row and run history. One live set per + environment: whatever adds to it is seen by every writer.""" + return _USED_KEYS.setdefault(str(Path(destination).resolve()), set()) + + +def unused_key(key: str, taken: set[str]) -> str: + """``key``, or it with a numbered suffix when another scenario already holds it.""" + if not key or key not in taken: + return key + suffix = 2 + while f"{key}-{suffix}" in taken: + suffix += 1 + return f"{key}-{suffix}" + + +def with_unique_keys( + scenarios: list[Scenario], used: set[str] | None = None +) -> list[Scenario]: + """``scenarios`` with a key each: the first holder of a repeated key keeps it, and every other + one gets the next key no scenario holds or has held, the same one the platform gives it.""" + held = {one.scenario_key for one in scenarios} | (used or set()) + keys: set[str] = set() + out: list[Scenario] = [] + for one in scenarios: + if not one.scenario_key or one.scenario_key in keys: + key = unused_key(one.scenario_key or _slug(one.name), held | keys) + one = one.model_copy(update={"scenario_key": key}) + held.add(key) + keys.add(one.scenario_key) + out.append(one) + return out + + class Scenario(BaseModel): """One test: what changes, what is asked, what a correct agent does, what must hold.""" diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 0151271fc..a38c6580f 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -66,6 +66,10 @@ redteam_problems, level_name, _pinned_identity, + _slug, + unused_key, + used_keys, + with_unique_keys, _QUIET_INTERFACE, unpinned_callers, crowded_cells, @@ -303,9 +307,15 @@ def _forget_dropped(scenarios: list[Scenario], destination: Path) -> None: if not root.exists(): return keeping = {one.name for one in scenarios} + folded = {name.casefold(): name for name in keeping} for folder in root.iterdir(): - if folder.is_dir() and folder.name not in keeping: - shutil.rmtree(folder) + if not folder.is_dir() or folder.name in keeping: + continue + # On a case-insensitive disk a case-only rename is still this folder. + kept_as = root / folded.get(folder.name.casefold(), folder.name) + if kept_as != folder and kept_as.exists() and kept_as.samefile(folder): + continue + shutil.rmtree(folder) def load_scenarios(destination: Path) -> list[Scenario]: @@ -320,7 +330,9 @@ def load_scenarios(destination: Path) -> list[Scenario]: JOURNAL = "written.jsonl" -def journal_scenario(scenario: Scenario, destination: Path) -> None: +def journal_scenario( + scenario: Scenario, destination: Path, replaces: Scenario | None = None +) -> None: """Append one proved scenario to the journal, which is the only record a dead writer leaves. A delegated writer cannot write folders: saving the suite deletes every folder it does not know @@ -333,6 +345,11 @@ def journal_scenario(scenario: Scenario, destination: Path) -> None: path = Path(destination) / JOURNAL path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as journal: + # A rename drops the old name first, so recovering the journal never brings it back. + # It names the entry it replaces, so a sibling's scenario of that name is untouched. + if replaces is not None: + dropped = {"dropped": replaces.name, "was": replaces.model_dump()} + journal.write(json.dumps(dropped, ensure_ascii=False) + "\n") journal.write(json.dumps(scenario.model_dump(), ensure_ascii=False) + "\n") except Exception as broke: # noqa: BLE001 - a scenario is never lost over bookkeeping logger.warning("could not journal %s: %s", scenario.name, broke) @@ -352,7 +369,14 @@ def journalled(destination: Path) -> list[Scenario]: if not line.strip(): continue try: - one = Scenario.model_validate(json.loads(line)) + entry = json.loads(line) + if isinstance(entry, dict) and set(entry) == {"dropped", "was"}: + name = str(entry["dropped"]) + was = Scenario.model_validate(entry["was"]).model_dump() + if name in found and found[name].model_dump() == was: + del found[name] + continue + one = Scenario.model_validate(entry) except Exception: # noqa: BLE001 - a half-written line is expected while writers run continue found[one.name] = one @@ -841,8 +865,8 @@ def values(node: Any) -> list[str]: return [ "these fixed values are not where their source says: " + "; ".join(wrong) - + '. Write each source as `caller: ...` (the value is in what the caller says), `record: ...` ' - '(the world holds it) or `rule: ""` (quoted from the agent\'s ' + + ". Write each source as `caller: ...` (the value is in what the caller says), `record: ...` " + "(the world holds it) or `rule: \"\"` (quoted from the agent's " "instructions and containing the value). A value none of these gives is the agent's choice; " "take it out of `fixed`" ] @@ -872,6 +896,37 @@ def _credentials_the_world_lacks(scenario: Scenario, trial: Any) -> list[str]: ] +def edited(held: Scenario, change: dict[str, Any]) -> dict[str, Any]: + """``held`` with ``change`` applied: a field it names is replaced, and a mapping such as the + persona changes only the entries it names, at any depth. Each field ``clear`` names, a persona + field as ``persona.accent``, goes back to its default. + """ + + def apply(saved: dict[str, Any], given: dict[str, Any]) -> dict[str, Any]: + out = dict(saved) + for field, value in given.items(): + if value is None: + out.pop(field, None) + elif isinstance(value, dict) and isinstance(out.get(field), dict): + out[field] = apply(out[field], value) + else: + out[field] = value + return out + + result = apply( + held.model_dump(exclude_none=True), + {field: value for field, value in change.items() if field != "clear"}, + ) + for path in change.get("clear") or (): + *parents, last = str(path).split(".") + node: Any = result + for part in parents: + node = node.get(part) if isinstance(node, dict) else None + if isinstance(node, dict): + node.pop(last, None) + return result + + def accept_scenario( payload: dict[str, Any], *, @@ -886,9 +941,15 @@ def accept_scenario( vocabulary: list[str] | None = None, spoken: bool = True, instructions: str = "", + used: set[str] | None = None, + renaming: bool = False, ) -> dict[str, Any]: """Validate one scenario, then prove it. A plain function so both halves are testable. + ``renaming`` keeps a proved scenario under a new name without proving it again: what it tests + is unchanged. ``used`` is every key the environment has given a scenario, so a new one never + takes a dropped scenario's key and inherits its run history. + ``persist`` is off for a writer that shares the destination with siblings: writing the suite removes every folder not in the writer's own list, so persisting here would delete whatever the others have proved. Those writers keep their work in ``kept`` and the caller saves once. @@ -907,58 +968,100 @@ def accept_scenario( place_for(scenario.name, scenario.fixture, scenario.instruction) or True ) - # Read against the world this scenario actually runs in, so a setup that creates the table - # a check reads is not reported as referring to something that does not exist. - trial, _applied, _ready = prepared(scenario, world_root) - try: - problems = validate_scenario( - scenario, - catalogue, - trial.state(), - simulator_prompt, - allow_empty_solution=allow_empty_solution, - spoken=spoken, - ) - problems.extend(contract_sequence_problems(scenario, hard_constraints or [])) - problems.extend(_credentials_the_world_lacks(scenario, trial)) - problems.extend(_identifiers_the_instruction_invents(scenario, trial)) - problems.extend(_fixed_sources_that_do_not_hold(scenario, trial, instructions)) - problems.extend(_steps_the_contract_refuses(scenario, world_root)) - problems.extend(_overlay_asserts_nothing(scenario, catalogue)) - problems.extend(_shared_check_bound_to_one_task(scenario, catalogue, kept)) - finally: - trial.close() - - if problems: - return _err( - "Not kept. Fix these and submit again:\n - " + "\n - ".join(problems) - ) + proof = None + if not renaming: + # Read against the world this scenario actually runs in, so a setup that creates the table + # a check reads is not reported as referring to something that does not exist. + trial, _applied, _ready = prepared(scenario, world_root) + try: + problems = validate_scenario( + scenario, + catalogue, + trial.state(), + simulator_prompt, + allow_empty_solution=allow_empty_solution, + spoken=spoken, + ) + problems.extend( + contract_sequence_problems(scenario, hard_constraints or []) + ) + problems.extend(_credentials_the_world_lacks(scenario, trial)) + problems.extend(_identifiers_the_instruction_invents(scenario, trial)) + problems.extend( + _fixed_sources_that_do_not_hold(scenario, trial, instructions) + ) + problems.extend(_steps_the_contract_refuses(scenario, world_root)) + problems.extend(_overlay_asserts_nothing(scenario, catalogue)) + problems.extend(_shared_check_bound_to_one_task(scenario, catalogue, kept)) + finally: + trial.close() - proof = prove(scenario, catalogue, world_root) - if not proof.holds: - said = f"Not kept. {proof.why()}" - # Code written against the wrong collection shape is the commonest way setup, ready and a - # check fail here, and the exception alone does not say which collections are mappings and - # which are lists. The world is asked, so the answer names them. - if "attribute" in said.lower() or "not subscriptable" in said.lower(): - world = restore(world_root) - try: - said += f"\n\n{world.shapes()}" - finally: - world.close() - return _err(said) + if problems: + return _err( + "Not kept. Fix these and submit again:\n - " + "\n - ".join(problems) + ) + proof = prove(scenario, catalogue, world_root) + if not proof.holds: + said = f"Not kept. {proof.why()}" + # Code written against the wrong collection shape is the commonest way setup, ready and a + # check fail here, and the exception alone does not say which collections are mappings and + # which are lists. The world is asked, so the answer names them. + if "attribute" in said.lower() or "not subscriptable" in said.lower(): + world = restore(world_root) + try: + said += f"\n\n{world.shapes()}" + finally: + world.close() + return _err(said) + + # The scenario this one replaces: the one whose key it carries, else the one with its name. It + # keeps that key, so a rename is the same scenario under a new name, not a new one. + named = str(payload.get("scenario_key") or "").strip() + previous = next( + (one for one in kept if named and one.scenario_key == named), None + ) or next((one for one in kept if one.name == scenario.name), None) + # A replaced or renamed scenario keeps its place in the suite. + spot = next((index for index, one in enumerate(kept) if one is previous), None) + renamed_from = "" + if previous is not None and previous.name != scenario.name: + if any(one.name == scenario.name for one in kept): + return _err( + f"Not kept. Another scenario is already named {scenario.name}; choose another name." + ) + renamed_from = previous.name + del kept[spot] + keys = {one.scenario_key for one in kept} | (used or set()) + if previous is not None: + scenario.scenario_key = previous.scenario_key + else: + scenario.scenario_key = unused_key( + _slug(scenario.name) if named else scenario.scenario_key, keys + ) + if used is not None: + used.add(scenario.scenario_key) replaced = any(one.name == scenario.name for one in kept) - if replaced and rename_on_collision: + # An edit names its scenario by key, so it replaces that one; only a new submission that + # repeats a name is kept beside it under a numbered name. + if replaced and rename_on_collision and not (named and previous is not None): taken = {one.name for one in kept} stem, suffix = scenario.name, 2 while f"{stem}-{suffix}" in taken: suffix += 1 - scenario = scenario.model_copy(update={"name": f"{stem}-{suffix}", "scenario_key": ""}) + renamed = f"{stem}-{suffix}" + scenario = scenario.model_copy( + update={"name": renamed, "scenario_key": unused_key(_slug(renamed), keys)} + ) + if used is not None: + used.add(scenario.scenario_key) replaced = False + spot = None else: kept[:] = [one for one in kept if one.name != scenario.name] - kept.append(scenario) + if spot is None: + kept.append(scenario) + else: + kept.insert(spot, scenario) outside: list[str] = [] if vocabulary and scenario.keywords: allowed = {word.strip().casefold(): word.strip() for word in vocabulary if word.strip()} @@ -981,12 +1084,16 @@ def accept_scenario( else: # A writer sharing the destination cannot write folders, so the journal is where its proved # work survives the session that proved it. - journal_scenario(scenario, world_root) + journal_scenario( + scenario, world_root, replaces=previous if renamed_from else None + ) # Say what the proof did not cover. On a lane where the target's tools have no endpoints, every # solution step is recorded without running, so "all three gates pass" is true and misleading: # the checks were exercised, the solution was not. unproved = ( - "\nNOT PROVED: " + "" + if proof is None + else "\nNOT PROVED: " + f"{len(proof.assumed)} of {len(scenario.solution)} solution steps were recorded without " "running, because these tools have no endpoint in this environment: " + ", ".join(proof.assumed) @@ -996,7 +1103,9 @@ def accept_scenario( else "" ) proof_summary = ( - "The world is ready for it, and its sub-goals will be judged from the call: the " + "Nothing it tests changed, so it was not proved again." + if proof is None + else "The world is ready for it, and its sub-goals will be judged from the call: the " "recording, the transcript, every action with its arguments and the records left." if proof.judged_only else "All three gates pass: the world is ready for it, the reference solution passes " @@ -1015,8 +1124,13 @@ def accept_scenario( if outside else "" ) + done = ( + f"{renamed_from} renamed to {scenario.name}" + if renamed_from + else f"{scenario.name} {'replaced' if replaced else 'kept'}" + ) return _ok( - f"{scenario.name} {'replaced' if replaced else 'kept'}. {proof_summary}" + f"{done}. {proof_summary}" f"{unproved}{strayed}\n{len(kept)} so far. These names are taken, so do not reuse one " f"and do not read them: {recent}{more}" ) @@ -1149,6 +1263,13 @@ def scenario_tools( catalogue = load_catalogue(destination) simulator_prompt = load_simulator_prompt(destination) target = {"count": wanted} + # Keys this environment has given out, a dropped scenario's included: never given to a new one. + used = used_keys(destination) + unique = with_unique_keys(kept, used) + if can_save and start_from is None and unique != kept: + write_scenarios(unique, destination, catalogue) + kept[:] = unique + used.update(one.scenario_key for one in kept) def cap() -> int: return int(target["count"] or 0) if can_grow else wanted @@ -1424,7 +1545,8 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: @tool( "submit_scenario", - "Keep one scenario. It is put through three gates before it is kept, and told which one " + "Keep one new scenario. To change one already kept, use `edit_scenario` with only the " + "fields that change. It is put through three gates before it is kept, and told which one " "failed if any does:\n" " 1. ready — the world is restored, setup_code runs, then ready_code. The world " "must end up holding what this scenario presumes.\n" @@ -1648,8 +1770,8 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "type": "object", "additionalProperties": {"type": "string"}, "description": "Where this scenario sits on the axes the plan varied, one " - "value per axis, for example {\"task\": \"book_ride\", \"overlay\": " - "\"interruption\"}. Copy it from your brief. It is what lets the suite " + 'value per axis, for example {"task": "book_ride", "overlay": ' + '"interruption"}. Copy it from your brief. It is what lets the suite ' "report how much of the space was tested; it never reaches the caller.", }, }, @@ -1676,6 +1798,13 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] if problem: return _refuse(problem) + # A submission carrying a kept scenario's key replaces that one, under its new name if renamed, + # so the checks below never count it against itself. + key = str(args.get("scenario_key") or "").strip() + same = next( + (one.name for one in kept if key and one.scenario_key == key), + str(args.get("name") or "").strip(), + ) # Cheap label checks run before the gates, since proving is expensive. strayed = _off_the_grid(args.get("coverage"), target.get("axes")) if strayed: @@ -1685,7 +1814,7 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] target.get("axes"), kept, cap(), - str(args.get("name") or ""), + same, target.get("asked"), ) if crowded_level: @@ -1695,17 +1824,21 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] if short: return _refuse(short) twin = _already_in_the_suite( - args, kept, _first_names_on_disk(destination, str(args.get("name") or "")) + {**args, "name": same}, kept, _first_names_on_disk(destination, same) ) if twin: return _refuse(twin) if cap() and target.get("people") != "alike": # A replacement does not count against itself. - crowded = crowded_field( - [one for one in kept if one.name != str(args.get("name") or "")], - Scenario.model_validate(args).persona, - cap(), - ) if args.get("persona") else "" + crowded = ( + crowded_field( + [one for one in kept if one.name != same], + Scenario.model_validate(args).persona, + cap(), + ) + if args.get("persona") + else "" + ) if crowded: return _refuse(crowded) if args.get("persona") and args.get("fixture"): @@ -1718,8 +1851,7 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] if stranger: return _refuse(stranger[0]) if cap(): - named = str(args.get("name") or "").strip() - already = any(one.name == named for one in kept) + already = any(one.name == same for one in kept) if not already and len(kept) >= cap(): return _err( f"This is complete: {len(kept)} of {cap()} written. Do not write another. " @@ -1741,7 +1873,10 @@ def _refuse(said: str, as_given: dict[str, Any] | None = None) -> dict[str, Any] rename_on_collision=rename_on_collision, vocabulary=target.get("keywords"), spoken=_is_spoken(contract), - instructions="\n".join([contract.system_prompt_excerpt, *contract.hard_constraints]), + instructions="\n".join( + [contract.system_prompt_excerpt, *contract.hard_constraints] + ), + used=used, ) if not result.get("is_error"): exploration["since_submit"] = 0 @@ -2267,6 +2402,91 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: ) return _ok(said) + def optional(schema: Any) -> Any: + """``schema`` with nothing required at any depth, so an edit sends only what changes.""" + if isinstance(schema, list): + return [optional(item) for item in schema] + if not isinstance(schema, dict): + return schema + return { + key: optional(value) for key, value in schema.items() if key != "required" + } + + edit_schema = { + **submit_scenario.input_schema, + "properties": { + **optional(submit_scenario.input_schema["properties"]), + "scenario_key": { + "type": "string", + "description": "The scenario's key, read from it.", + }, + "clear": { + "type": "array", + "items": {"type": "string"}, + "description": "Fields to set back to their default, a persona field as " + "`persona.accent`.", + }, + }, + "required": ["scenario_key"], + } + + @tool( + "edit_scenario", + "Change a kept scenario: give its `scenario_key` and only what changes, such as a new " + "`name` or the persona fields that change; name a field in `clear` to set it back to its " + "default. Everything else keeps its saved value, and the scenario keeps its key, number, " + "place and history. A new name alone is applied as it is; any other change is checked " + "exactly like `submit_scenario`.", + edit_schema, + ) + async def edit_scenario(args: dict[str, Any]) -> dict[str, Any]: + key = str(args.get("scenario_key") or "").strip() + held = next((one for one in kept if one.scenario_key == key), None) + if held is None: + return _err( + f"Not changed: no scenario in this suite has the scenario_key {key!r}." + ) + change = { + field: value for field, value in args.items() if field != "scenario_key" + } + name = str(change.get("name") or "").strip() + if ( + set(change) != {"name"} + and name + and name != held.name + and any(one.name == name for one in kept) + ): + return _err( + f"Not changed. Another scenario is already named {name}; choose another name." + ) + if set(change) == {"name"} and name == held.name: + return _ok(f"Nothing changed: {held.name} already has that name.") + if set(change) != {"name"}: + if edited(held, args) == edited(held, {}): + return _ok( + f"Nothing changed: {held.name} already has these values. A field left out " + "keeps its saved value; to remove one, name it in `clear`." + ) + return await submit_scenario.handler(edited(held, args)) + result = accept_scenario( + {**held.model_dump(), "name": name}, + world_root=world_root, + catalogue=catalogue, + kept=kept, + persist=can_save, + used=used, + renaming=True, + ) + if result.get("is_error"): + said = str((result.get("content") or [{}])[0].get("text") or "") + seen = _count_refusal(refused, held.name, said) + if seen >= _REFUSALS_BEFORE_MOVING_ON: + return _err( + f"{said}\n\nThis is refusal {seen} of this rename for the same reason, so " + "asking again will not work. Keep the current name or choose another one." + ) + return result + server = tool_server( name=SCENARIO_SERVER, version="0.1.0", @@ -2277,6 +2497,7 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: try_check, add_sub_goal, submit_scenario, + edit_scenario, amend_contract, add_rule_tool, drop_rule_tool, @@ -2298,6 +2519,7 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: "try_check", "add_sub_goal", "submit_scenario", + "edit_scenario", "amend_contract", "add_rule", "drop_rule", diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index b114b3ca3..2e1065cac 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -40,7 +40,7 @@ from .contract import AgentContract from .job import HarnessJob from .poc_guest_booking import guest_booking_pin_guidance, guest_booking_pin_scenario_problem -from .scenario import Scenario, voicemail_enabled +from .scenario import Scenario, used_keys, voicemail_enabled, with_unique_keys from .scenario_tools import ( SCENARIO_SERVER, TOOL_NAMES, @@ -906,7 +906,9 @@ async def attempt_slice() -> Any: return list(kept) -def merged(written: list[list[Scenario]]) -> list[Scenario]: +def merged( + written: list[list[Scenario]], used: set[str] | None = None +) -> list[Scenario]: """One suite out of several writers, with folder-name collisions renamed rather than dropped. Asking for twenty scenarios has to return twenty. Two scenarios may legitimately share a use @@ -931,7 +933,8 @@ def merged(written: list[list[Scenario]]) -> list[Scenario]: logger.info("renamed a duplicate folder name to %s", one.name) taken.add(one.name) suite.append(one) - return suite + # A key names one scenario, so a repeat, even one already saved, gets its own. + return with_unique_keys(suite, used) def _suite_summary(suite: list[Scenario]) -> str: @@ -1131,7 +1134,7 @@ async def guarded(mine: Slice, siblings: list[Slice], index: int) -> list[Scenar *(guarded(one, allocation, index) for index, one in enumerate(allocation)), return_exceptions=False, ) - proved = merged([load_scenarios(destination), *written]) + proved = merged([load_scenarios(destination), *written], used_keys(destination)) # What a writer proved but never handed back, because its session died after proving it. Matched # by name so a scenario already in hand is not added twice under a numbered name. recovered = [ @@ -1147,7 +1150,7 @@ async def guarded(mine: Slice, siblings: list[Slice], index: int) -> list[Scenar ) if on_event: on_event({"type": "recovered", "kept": len(recovered)}) - suite = merged([proved, recovered]) + suite = merged([proved, recovered], used_keys(destination)) # Read the whole thing and fill what nobody covered. Bounded, because a reviewer asked # twice will always find something smaller to say. @@ -1182,7 +1185,7 @@ async def guarded(mine: Slice, siblings: list[Slice], index: int) -> list[Scenar return_exceptions=False, ) before = len(suite) - suite = merged([suite, *more]) + suite = merged([suite, *more], used_keys(destination)) allocation = [*allocation, *missing] if len(suite) == before: break diff --git a/src/fi/alk/harness/skills/builder/SKILL.md b/src/fi/alk/harness/skills/builder/SKILL.md index fc3c841af..a2fd71455 100644 --- a/src/fi/alk/harness/skills/builder/SKILL.md +++ b/src/fi/alk/harness/skills/builder/SKILL.md @@ -137,10 +137,12 @@ selection is what "these" and "the selected ones" mean. Load `write-scenarios` before you write or rewrite any scenario in this conversation; its rules are what the gates check. Then: -- **Edit one**: read it in full first, change only what was asked, carry the change into every - field that follows from it (setup, ready check, solution, instruction, sub-goals, labels), and - `submit_scenario` it under the same name, keeping every field the change does not touch. That - replaces it, keeps its number, and proves it. +- **Edit or rename one**: read it in full first, change only what was asked, carry the change into + every field that follows from it (setup, ready check, solution, instruction, sub-goals, labels), + and `edit_scenario` it with its `scenario_key` and only the fields that change. That keeps its + number, place and run history, and proves it again unless only the name changed. Never drop a + scenario and submit it again to edit or rename it: it would come back as a new one, without its + history. - **Add**: `aim_for` the new total first, then write and submit each one. For more than a handful, brief `scenario_writer` sub-agents in one message. When the person asks for a level of an axis (no background noise, only one kind of caller), pass it in `aim_for`'s `asked_for` so it is not held to diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index d57c86791..e3a8d15e8 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -277,9 +277,25 @@ def test_meaning_changes_wait_for_a_yes_to_that_call_but_surface_edits_do_not(tm client = BuilderClient() runtime = _runtime(tmp_path, client, monkeypatch) (tmp_path / "scenarios" / "one" / "scenario.json").write_text( - json.dumps({"name": "one", "instruction": "ask for a refund", "persona": {"name": "Ana"}}) + json.dumps( + { + "name": "one", + "scenario_key": "one", + "instruction": "ask for a refund", + "persona": {"name": "Ana", "accent": "American"}, + "solution": [ + { + "tool": "refund", + "arguments": {"id": "7"}, + "environment_arguments": {}, + } + ], + } + ) + ) + (tmp_path / "sub_goals.json").write_text( + json.dumps({"sub_goals": [{"name": "refunded", "description": "x"}]}) ) - (tmp_path / "sub_goals.json").write_text(json.dumps({"sub_goals": [{"name": "refunded", "description": "x"}]})) calls, asked, answers = [], [], [] async def handler(args): @@ -300,7 +316,40 @@ async def ask(tool, payload, context): assert asyncio.run(submit.handler(surface)).get("is_error") is None assert asyncio.run(submit.handler({"name": "two", "instruction": "anything"})).get("is_error") is None assert asyncio.run(goal.handler({"name": "new-goal"})).get("is_error") is None + assert ( + asyncio.run( + submit.handler({**surface, "name": "uno", "scenario_key": "one"}) + ).get("is_error") + is None + ) + (tmp_path / "scenarios" / "old").mkdir() + (tmp_path / "scenarios" / "old" / "scenario.json").write_text( + json.dumps({"name": "old"}) + ) + answers.append("No") + unreadable = {"name": "old", "instruction": "anything"} + assert asyncio.run(submit.handler(unreadable))["is_error"] is True + asked.pop() + resubmitted = { + **surface, + "name": "uno", + "scenario_key": "one", + "solution": [{"tool": "refund", "arguments": {"id": "7"}}], + } + assert asyncio.run(submit.handler(resubmitted)).get("is_error") is None assert asked == [] + answers.append("No") + assert ( + asyncio.run(submit.handler({**meaning, "name": "uno", "scenario_key": "one"}))[ + "is_error" + ] + is True + ) + assert "ask for a cancellation" in asked.pop() + assert not runtime._changes_situation( + {"scenario_key": "one", "persona": {"name": "Ana"}} + ) + assert runtime._changes_situation({"scenario_key": "one", "persona": {"accent": "Indian"}}) answers.extend(["No, keep it", "Which scenarios use it?"]) assert asyncio.run(drop.handler({"names": ["one"]}))["is_error"] is True @@ -319,10 +368,101 @@ async def ask(tool, payload, context): answers.append("No") assert asyncio.run(submit.handler(meaning))["is_error"] is True assert asyncio.run(submit.handler(surface)).get("is_error") is None - assert len(calls) == 5 + assert len(calls) == 7 + + +def test_an_edit_and_a_clear_go_through_the_same_confirmation_as_a_submission( + tmp_path, monkeypatch +): + from fi.alk.harness.backends import ToolSpec + + runtime = _runtime(tmp_path, BuilderClient(), monkeypatch) + (tmp_path / "scenarios" / "one" / "scenario.json").write_text( + json.dumps( + { + "name": "one", + "scenario_key": "one", + "instruction": "ask for a refund", + "persona": {"name": "Ana", "accent": "American"}, + "fixture": {"origin": "seed", "order": "7"}, + "solution": [ + { + "tool": "refund", + "arguments": {"id": "7"}, + "environment_arguments": {}, + } + ], + } + ) + ) + calls, asked, answers = [], [], [] + async def handler(args): + calls.append(args) + return {"content": [{"type": "text", "text": "kept"}]} -def test_the_harness_builds_from_an_empty_workspace_with_the_tools_for_what_exists(tmp_path, monkeypatch): + async def ask(tool, payload, context): + asked.append(payload["question"]) + return {"content": [{"type": "text", "text": answers.pop(0)}]} + + runtime._ask = ask + edit = runtime._confirmed(ToolSpec("edit_scenario", "", {}, handler)) + assert ( + asyncio.run( + edit.handler({"scenario_key": "one", "persona": {"accent": "Indian"}}) + ).get("is_error") + is None + ) + assert asked == [] + answers.append("No") + assert ( + asyncio.run(edit.handler({"scenario_key": "one", "clear": ["solution"]}))[ + "is_error" + ] + is True + ) + assert len(asked) == 1 + assert runtime._changes_situation({"scenario_key": "one", "clear": ["fixture"]}) + assert runtime._changes_situation({"scenario_key": "one", "clear": ["persona"]}) + assert runtime._changes_situation( + {"scenario_key": "one", "clear": ["persona.accent"]} + ) + runtime._from_platform = True + assert ( + asyncio.run(edit.handler({"scenario_key": "one", "clear": ["fixture"]}))[ + "is_error" + ] + is True + ) + assert len(calls) == 1 + + +def test_the_keys_the_platform_has_used_are_never_given_to_a_new_scenario( + tmp_path, monkeypatch +): + from fi.alk.harness.scenario import used_keys + + monkeypatch.setenv("ALK_HARNESS", "vertex-gemini") + policy = tmp_path / "policy.yaml" + policy.write_text( + "schema_version: 1\nharness_chat:\n enabled: true\n", encoding="utf-8" + ) + HostedChatRuntime( + client=BuilderClient(), # type: ignore[arg-type] + policy=load_chat_policy(policy), + workspace=tmp_path, + source=tmp_path, + job={ + "metadata": {"agent_name": "demo", "used_scenario_keys": ["two", "one"]}, + "scenario_count": 1, + }, + ) + assert {"one", "two"} <= used_keys(tmp_path) + + +def test_the_harness_builds_from_an_empty_workspace_with_the_tools_for_what_exists( + tmp_path, monkeypatch +): monkeypatch.setenv("ALK_HARNESS", "claude") policy = tmp_path / "policy.yaml" policy.write_text( @@ -376,7 +516,6 @@ async def overview(self): assert over["is_error"] and "the build asked for 3" in over["content"][0]["text"] assert not (workspace / ".futureagi-build-finished.json").exists() - async def commands(): return [{"sequence": 7, "message_id": "m7", "kind": "user_message", "payload": {"content": "add a lapsed policy"}}] diff --git a/tests/test_harness.py b/tests/test_harness.py index 5ddbc362d..d1debc2f3 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -2678,6 +2678,121 @@ def test_a_scenario_is_proved_before_it_is_kept(tmp_path): assert (root / "scenarios" / "adds-a-big-mac" / "scenario.json").exists() +def test_a_renamed_scenario_keeps_its_key_and_replaces_the_old_one(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + key = kept[0].scenario_key + said = accept_scenario( + _delta(name="orders-one-big-mac", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + assert not said.get("is_error"), said + assert [(one.name, one.scenario_key) for one in kept] == [ + ("orders-one-big-mac", key) + ] + assert (root / "scenarios" / "orders-one-big-mac" / "scenario.json").exists() + assert not (root / "scenarios" / "adds-a-big-mac").exists() + + +def test_an_edit_under_the_same_name_keeps_the_key_it_was_kept_with(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + key = kept[0].scenario_key + accept_scenario( + _delta(name="orders-one-big-mac", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + accept_scenario( + _delta(name="orders-one-big-mac", instruction="Order one Big Mac, please."), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + assert [(one.name, one.scenario_key) for one in kept] == [ + ("orders-one-big-mac", key) + ] + + +def test_a_new_scenario_never_takes_a_key_another_one_holds_or_one_it_made_up(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + key = kept[0].scenario_key + accept_scenario( + _delta(name="orders-one-big-mac", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario( + _delta(name="orders-again", scenario_key="not-in-this-suite"), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + keys = {one.name: one.scenario_key for one in kept} + assert keys["orders-one-big-mac"] == key + assert keys["adds-a-big-mac"] == f"{key}-2" + assert keys["orders-again"] == "orders-again" + + +def test_merging_writers_keeps_the_first_holder_of_a_key_and_gives_repeats_their_own(): + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenarios import merged + + def one(name, key=""): + return Scenario(name=name, instruction="x", scenario_key=key) + + saved = [ + one("renamed", key="first-name"), + one("twin", key="same"), + one("twin-too", key="same"), + ] + suite = merged([saved, [one("first-name")]]) + assert [(item.name, item.scenario_key) for item in suite] == [ + ("renamed", "first-name"), + ("twin", "same"), + ("twin-too", "same-2"), + ("first-name", "first-name-2"), + ] + + +def test_a_rename_onto_another_scenario_s_name_is_refused(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario( + _delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept + ) + key = kept[0].scenario_key + said = accept_scenario( + _delta(name="orders-again", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + assert ( + said.get("is_error") + and "already named orders-again" in said["content"][0]["text"] + ) + assert sorted(one.name for one in kept) == ["adds-a-big-mac", "orders-again"] + + def test_a_scenario_whose_solution_cannot_pass_its_own_checks_is_refused(tmp_path): """Either the scenario is impossible or the checks are wrong. Both have happened.""" from fi.alk.harness.scenario_tools import accept_scenario @@ -8008,6 +8123,434 @@ def test_a_scenario_placed_nowhere_is_refused_once_a_grid_is_dealt(tmp_path): assert not _off_the_grid(None, {}) +def test_an_edit_sends_only_what_changes_and_keeps_the_rest(tmp_path): + import asyncio + + from fi.alk.harness.scenario_tools import scenario_tools + + root, _contract, _catalogue = _built_environment(tmp_path) + server, kept = scenario_tools(_contract, root, root, wanted=2) + submit = next(one for one in server.tools if one.name == "submit_scenario") + edit = next(one for one in server.tools if one.name == "edit_scenario") + person = { + "persona": { + "name": "Asha Rao", + "personality": "Friendly and cooperative", + "communication_style": "Direct and concise", + "initial_message": "Hi, I want a Big Mac.", + "languages": ["English"], + } + } + said = asyncio.run(submit.handler(_delta(**person))) + assert not said.get("is_error"), said + before = kept[0].model_dump(exclude={"name"}) + key = kept[0].scenario_key + renamed = {"scenario_key": key, "name": "orders-one-big-mac"} + said = asyncio.run(edit.handler(renamed)) + assert not said.get("is_error"), said + assert [(one.name, one.scenario_key) for one in kept] == [ + ("orders-one-big-mac", key) + ] + assert kept[0].model_dump(exclude={"name"}) == before + said = asyncio.run( + edit.handler({"scenario_key": key, "persona": {"name": "Meera Iyer"}}) + ) + assert not said.get("is_error"), said + assert kept[0].persona.name == "Meera Iyer" + assert kept[0].persona.initial_message == "Hi, I want a Big Mac." + assert kept[0].persona.languages == ["English"] + same = dict(kept[0].persona.model_dump(exclude_none=True)) + same.pop("languages") + said = asyncio.run(edit.handler({"scenario_key": key, "persona": same})) + assert not said.get("is_error"), said + assert said["content"][0]["text"].startswith("Nothing changed: orders-one-big-mac") + assert "`clear`" in said["content"][0]["text"] + assert kept[0].persona.languages == ["English"] + unknown = asyncio.run(edit.handler({"scenario_key": "not-here", "name": "x"})) + assert unknown.get("is_error") and "not-here" in unknown["content"][0]["text"] + assert [one.name for one in kept] == ["orders-one-big-mac"] + + +def test_a_rename_keeps_the_scenario_s_place_and_says_it_was_renamed(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario( + _delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept + ) + key = kept[0].scenario_key + said = accept_scenario( + _delta(name="renamed-first", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + assert not said.get("is_error"), said + assert said["content"][0]["text"].startswith( + "adds-a-big-mac renamed to renamed-first." + ) + assert [(one.name, one.scenario_key) for one in kept] == [ + ("renamed-first", key), + ("orders-again", kept[1].scenario_key), + ] + + +def test_a_replacement_under_the_same_name_keeps_its_place(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + accept_scenario( + _delta(name="orders-again"), world_root=root, catalogue=catalogue, kept=kept + ) + said = accept_scenario( + _delta(instruction="Order a single Big Mac."), + world_root=root, + catalogue=catalogue, + kept=kept, + ) + assert said["content"][0]["text"].startswith("adds-a-big-mac replaced.") + assert [one.name for one in kept] == ["adds-a-big-mac", "orders-again"] + assert kept[0].instruction == "Order a single Big Mac." + + +def test_a_new_name_alone_is_applied_without_proving_the_scenario_again( + tmp_path, monkeypatch +): + import asyncio + + from fi.alk.harness import scenario_tools as tools + + root, contract, _catalogue = _built_environment(tmp_path) + server, kept = tools.scenario_tools(contract, root, root, wanted=2) + submit = next(one for one in server.tools if one.name == "submit_scenario") + edit = next(one for one in server.tools if one.name == "edit_scenario") + assert not asyncio.run(submit.handler(_delta())).get("is_error") + assert not asyncio.run(submit.handler(_delta(name="orders-again"))).get("is_error") + first, second = kept[0].scenario_key, kept[1].scenario_key + + # An older scenario that a gate added since would now refuse still renames. + def refuse(*_args, **_kwargs): + raise AssertionError("a name-only rename must not prove the scenario again") + + monkeypatch.setattr(tools, "prove", refuse) + monkeypatch.setattr(tools, "validate_scenario", refuse) + said = asyncio.run(edit.handler({"scenario_key": first, "name": "siparis-tekrari"})) + assert not said.get("is_error"), said + assert said["content"][0]["text"].startswith( + "adds-a-big-mac renamed to siparis-tekrari." + ) + assert [(one.name, one.scenario_key) for one in kept] == [ + ("siparis-tekrari", first), + ("orders-again", second), + ] + saved = tools.load_scenarios(root) + assert sorted(one.name for one in saved) == ["orders-again", "siparis-tekrari"] + for _ in range(3): + taken = asyncio.run( + edit.handler({"scenario_key": first, "name": "orders-again"}) + ) + assert taken.get("is_error") + assert "already named orders-again" in taken["content"][0]["text"] + assert "refusal 3 of this rename" in taken["content"][0]["text"] + assert [one.name for one in kept] == ["siparis-tekrari", "orders-again"] + + +def test_an_edit_clears_what_clear_names_and_merges_nested_mappings(): + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenario_tools import edited + + held = Scenario.model_validate( + { + **_delta(), + "fixture": { + "origin": "seed", + "order": {"item": "big_mac", "size": "large"}, + }, + "persona": { + "name": "Asha Rao", + "accent": "Scottish", + "languages": ["English"], + }, + } + ) + change = edited( + held, + {"clear": ["persona.accent"], "fixture": {"order": {"size": "small"}}}, + ) + after = Scenario.model_validate(change) + assert ( + after.persona.accent + == Scenario.model_validate( + {**_delta(), "persona": {"name": "Asha Rao"}} + ).persona.accent + ) + assert after.persona.name == "Asha Rao" + assert after.persona.languages == ["English"] + assert after.fixture == { + "origin": "seed", + "order": {"item": "big_mac", "size": "small"}, + } + + +def test_a_writer_s_rename_never_brings_the_old_name_back_from_the_journal(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario, journalled + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + accept_scenario( + _delta(), world_root=root, catalogue=catalogue, kept=kept, persist=False + ) + key = kept[0].scenario_key + accept_scenario( + _delta(name="renamed-first", scenario_key=key), + world_root=root, + catalogue=catalogue, + kept=kept, + persist=False, + ) + assert [(one.name, one.scenario_key) for one in journalled(root)] == [ + ("renamed-first", key) + ] + + +def _edit_tools(tmp_path, **options): + from fi.alk.harness.scenario_tools import scenario_tools + + root, contract, catalogue = _built_environment(tmp_path) + server, kept = scenario_tools(contract, root, root, wanted=4, **options) + return root, catalogue, kept, {one.name: one for one in server.tools} + + +@pytest.mark.parametrize( + "args", + [ + {"scenario_key": "k", "persona": {"name": "Meera Iyer"}}, + {"scenario_key": "k", "clear": ["persona.accent", "fixture", "sub_goals"]}, + {"scenario_key": "k", "name": "renamed"}, + ], +) +def test_the_edits_edit_scenario_describes_pass_its_schema_on_both_backends( + tmp_path, args +): + import jsonschema + from claude_agent_sdk import SdkMcpTool, _build_input_schema + + from fi.alk.harness.backends.claude import _gateway_compatible_schema + + *_, tools = _edit_tools(tmp_path) + spec = tools["edit_scenario"] + sent = _build_input_schema( + SdkMcpTool( + name=spec.name, + description="", + input_schema=spec.input_schema, + handler=spec.handler, + ) + ) + jsonschema.validate(instance=args, schema=sent) + jsonschema.validate( + instance=args, schema=_gateway_compatible_schema(spec.input_schema) + ) + + +def test_a_writer_s_edit_by_key_replaces_its_scenario_rather_than_copying_it(tmp_path): + import asyncio + + _root, _catalogue, kept, tools = _edit_tools( + tmp_path, can_save=False, start_from=[] + ) + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + key = kept[0].scenario_key + said = asyncio.run( + tools["edit_scenario"].handler( + {"scenario_key": key, "instruction": "Order a single Big Mac."} + ) + ) + assert not said.get("is_error"), said + assert [(one.name, one.scenario_key) for one in kept] == [("adds-a-big-mac", key)] + + +def test_a_new_scenario_never_takes_a_key_the_environment_has_used(tmp_path): + import asyncio + + from fi.alk.harness.scenario import used_keys + + root, _catalogue, kept, tools = _edit_tools(tmp_path) + # The platform's keys, a dropped scenario's among them. + used_keys(root).add("adds-a-big-mac") + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + assert kept[0].scenario_key == "adds-a-big-mac-2" + + +def test_a_scenario_dropped_in_the_chat_never_lends_its_key_to_a_new_one(tmp_path): + import asyncio + + _root, _catalogue, kept, tools = _edit_tools(tmp_path) + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + dropped = kept[0].scenario_key + assert not asyncio.run( + tools["drop_scenario"].handler({"name": "adds-a-big-mac"}) + ).get("is_error") + assert not asyncio.run( + tools["submit_scenario"].handler(_delta(instruction="Order two Big Macs.")) + ).get("is_error") + assert kept[0].scenario_key not in ("", dropped) + + +def test_repeated_saved_keys_are_repaired_when_the_suite_is_opened(tmp_path): + import asyncio + + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenario_tools import ( + load_scenarios, + scenario_tools, + write_scenarios, + ) + + root, contract, catalogue = _built_environment(tmp_path) + first = Scenario.model_validate({**_delta(name="first"), "scenario_key": "k"}) + second = Scenario.model_validate({**_delta(name="second"), "scenario_key": "k"}) + write_scenarios([first, second], root, catalogue) + server, kept = scenario_tools(contract, root, root, wanted=2) + assert [(one.name, one.scenario_key) for one in load_scenarios(root)] == [ + ("first", "k"), + ("second", "k-2"), + ] + edit = next(one for one in server.tools if one.name == "edit_scenario") + said = asyncio.run(edit.handler({"scenario_key": "k-2", "name": "second-renamed"})) + assert not said.get("is_error"), said + assert [(one.name, one.scenario_key) for one in kept] == [ + ("first", "k"), + ("second-renamed", "k-2"), + ] + + +def test_merging_never_moves_a_repeat_onto_a_key_another_scenario_holds(): + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenarios import merged + + def one(name, key=""): + return Scenario.model_validate( + {"name": name, "instruction": "x", "scenario_key": key} + ) + + suite = merged([[one("a", "k"), one("b", "k"), one("c", "k-2")]]) + assert [(s.name, s.scenario_key) for s in suite] == [ + ("a", "k"), + ("b", "k-3"), + ("c", "k-2"), + ] + assert [(s.name, s.scenario_key) for s in merged([suite])] == [ + (s.name, s.scenario_key) for s in suite + ] + renamed = merged([[one("other", "x-2")], [one("x")], [one("x")]]) + keys = [s.scenario_key for s in renamed] + assert len(set(keys)) == len(keys) + + +def test_a_writer_s_rename_leaves_a_sibling_s_scenario_of_that_name_in_the_journal( + tmp_path, +): + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenario_tools import journal_scenario, journalled + + root, *_ = _built_environment(tmp_path) + mine = Scenario.model_validate(_delta(name="shared-name", instruction="Order two.")) + sibling = Scenario.model_validate(_delta(name="shared-name")) + journal_scenario(mine, root) + journal_scenario(sibling, root) + journal_scenario( + mine.model_copy(update={"name": "mine-renamed"}), root, replaces=mine + ) + assert sorted(one.name for one in journalled(root)) == [ + "mine-renamed", + "shared-name", + ] + + +def test_a_writer_s_rename_by_edit_scenario_is_journalled(tmp_path): + import asyncio + + from fi.alk.harness.scenario_tools import journalled + + root, _catalogue, kept, tools = _edit_tools(tmp_path, can_save=False, start_from=[]) + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + key = kept[0].scenario_key + said = asyncio.run( + tools["edit_scenario"].handler({"scenario_key": key, "name": "renamed"}) + ) + assert not said.get("is_error"), said + assert [(one.name, one.scenario_key) for one in journalled(root)] == [ + ("renamed", key) + ] + + +def test_a_case_only_rename_keeps_the_scenario_on_disk(tmp_path): + import asyncio + + from fi.alk.harness.scenario_tools import load_scenarios + + root, _catalogue, kept, tools = _edit_tools(tmp_path) + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + key = kept[0].scenario_key + said = asyncio.run( + tools["edit_scenario"].handler({"scenario_key": key, "name": "Adds-A-Big-Mac"}) + ) + assert not said.get("is_error"), said + assert [one.name for one in load_scenarios(root)] == ["Adds-A-Big-Mac"] + + +def test_renaming_a_scenario_to_its_own_name_changes_nothing(tmp_path, monkeypatch): + import asyncio + + from fi.alk.harness import scenario_tools as tools_module + + _root, _catalogue, kept, tools = _edit_tools(tmp_path) + assert not asyncio.run(tools["submit_scenario"].handler(_delta())).get("is_error") + monkeypatch.setattr( + tools_module, "prove", lambda *a, **k: (_ for _ in ()).throw(AssertionError) + ) + said = asyncio.run( + tools["edit_scenario"].handler( + {"scenario_key": kept[0].scenario_key, "name": "adds-a-big-mac"} + ) + ) + assert not said.get("is_error"), said + + +def test_an_edit_onto_a_taken_name_is_refused_before_anything_is_proved( + tmp_path, monkeypatch +): + import asyncio + + from fi.alk.harness import scenario_tools as tools_module + + _root, _catalogue, kept, tools = _edit_tools(tmp_path) + for name in ("adds-a-big-mac", "orders-again"): + assert not asyncio.run( + tools["submit_scenario"].handler( + _delta(name=name, instruction=f"Order {name}") + ) + ).get("is_error") + monkeypatch.setattr( + tools_module, "prove", lambda *a, **k: (_ for _ in ()).throw(AssertionError) + ) + said = asyncio.run( + tools["edit_scenario"].handler( + { + "scenario_key": kept[1].scenario_key, + "name": "adds-a-big-mac", + "instruction": "Order one", + } + ) + ) + assert said.get("is_error") and "already named" in said["content"][0]["text"] + + def test_a_redeclared_grid_only_ever_grows(tmp_path): import asyncio