Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
7f13453
fix(harness): keep a scenario's key when it is renamed or replaced
KarthikAvinashFI Oct 22, 2026
10f47bd
fix(harness): rename a scenario without losing its key, and let a key…
KarthikAvinashFI Oct 22, 2026
e21e026
Merge remote-tracking branch 'upstream/dev' into fix/TH-8748-keep-sce…
KarthikAvinashFI Oct 22, 2026
162ecf7
style(harness): format the scenario rename changes
KarthikAvinashFI Oct 23, 2026
1c01b09
refactor(harness): rename through submit_scenario instead of a separa…
KarthikAvinashFI Oct 23, 2026
65d71bb
fix(harness): keep scenario keys unique and resolve edits by key
KarthikAvinashFI Oct 23, 2026
b80a182
Merge branch 'dev' of https://github.com/future-agi/agent-learning-ki…
KarthikAvinashFI Oct 23, 2026
2223932
test(harness): leave untouched test lines as they were
KarthikAvinashFI Oct 23, 2026
d4fc0f9
fix(harness): compare a resubmitted scenario as saved, so defaults it…
KarthikAvinashFI Oct 23, 2026
64a1cfa
fix(harness): fall back to the plain comparison for a scenario the mo…
KarthikAvinashFI Oct 23, 2026
dfb608f
feat(harness): edit a scenario by key with only the fields that change
KarthikAvinashFI Oct 23, 2026
088097a
fix(harness): point submit_scenario at edit_scenario for kept scenarios
KarthikAvinashFI Oct 23, 2026
8f24dc0
fix(harness): an edit changes only the persona fields it names
KarthikAvinashFI Oct 23, 2026
9f6e356
fix(harness): TH-8748 keep a renamed scenario's place, rename by name…
KarthikAvinashFI Oct 26, 2026
16a1a75
fix(harness): TH-8748 edit_scenario accepts partial edits and a clear…
KarthikAvinashFI Oct 26, 2026
f276ec5
fix(harness): TH-8748 an edit that changes nothing says so and points…
KarthikAvinashFI Oct 26, 2026
503a72d
chore(harness): TH-8748 re-run checks
KarthikAvinashFI Oct 26, 2026
fab10e3
Merge remote-tracking branch 'gh/dev' into fix/TH-8748-keep-scenario-key
KarthikAvinashFI Oct 27, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ All notable release changes for Agent Learning Kit are tracked here.
Post-rc.1 increments on the release branch. Each one adds gates on top of the
66 proved at rc.1; the rc.1 entry below is historical and unchanged.

- Builder chat: `edit_scenario` changes a kept scenario by its key with only the fields that change
(`clear` resets a field), keeping its key, number, place and history; a rename alone is not
proved again. A new scenario never takes a key the environment has used, a dropped scenario's
included, and a repeated saved key is repaired so each scenario keeps its own.
- A chat conversation never sends the scenario's instruction as the simulated person's first
message: an empty or echoed opening is asked for again, then for a greeting in their language,
and the person's own session records what the agent actually heard.
Expand Down
67 changes: 57 additions & 10 deletions src/fi/alk/harness/hosted_chat_entrypoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
_NEEDS_CONFIRMATION = frozenset(
{
"submit_scenario",
"edit_scenario",
"drop_scenario",
"drop_scenarios",
"change_world",
Expand All @@ -71,6 +72,7 @@
"add_sub_goal",
}
)
_SUBMITS = ("submit_scenario", "edit_scenario")
_CONFIRM = "Yes, make this change"
_RUN_FINISHED = frozenset({"completed", "failed", "canceled"})
_RUN_POLL_SECONDS = 20.0
Expand All @@ -89,6 +91,7 @@
_WRITING_TOOLS = frozenset(
{
"submit_scenario",
"edit_scenario",
"drop_scenario",
"drop_scenarios",
"save_scenarios",
Expand Down Expand Up @@ -251,6 +254,15 @@ def __init__(
self.workspace = workspace
self.source = source
self.job = job
from .scenario import used_keys

# Every key the environment has given a scenario, a dropped one's included, so a new
# scenario never takes one and inherits another's row and run history.
used_keys(workspace).update(
str(key)
for key in (job.get("metadata") or {}).get("used_scenario_keys") or ()
if key
)
backend = resolve()
self.backend_name = backend.name
self.event_store: Any = None
Expand Down Expand Up @@ -461,7 +473,7 @@ def _confirmed(self, spec: Any) -> Any:
async def guarded(args: dict[str, Any]) -> Any:
if self._from_platform:
if spec.name in ("drop_scenarios", "drop_scenario") or (
spec.name == "submit_scenario" and self._changes_situation(args)
spec.name in _SUBMITS and self._changes_situation(args)
):
return {
"content": [
Expand All @@ -476,7 +488,7 @@ async def guarded(args: dict[str, Any]) -> Any:
"is_error": True,
}
return await handler(args)
if spec.name == "submit_scenario" and not self._changes_meaning(args):
if spec.name in _SUBMITS and not self._changes_meaning(args):
return await handler(args)
if spec.name == "add_sub_goal" and not self._replaces_sub_goal(args):
return await handler(args)
Expand Down Expand Up @@ -510,7 +522,9 @@ async def guarded(args: dict[str, Any]) -> Any:
def _changes_situation(self, args: dict[str, Any]) -> bool:
"""Whether a submission changes what an existing scenario tests, not how it is graded."""
return self._changes(
args, ("instruction", "use_case", "branch", "tests", "coverage", "fixture"), whole_persona=True
args,
("instruction", "use_case", "branch", "tests", "coverage", "fixture"),
whole_persona=True,
)

def _replaces_sub_goal(self, args: dict[str, Any]) -> bool:
Expand All @@ -528,14 +542,47 @@ def _changes(self, args: dict[str, Any], fields: tuple[str, ...], whole_persona:
`whole_persona` compares the persona the submission would save, so a changed language or a
dropped persona counts: a submission replaces the scenario rather than patching it.
"""
path = self.workspace / "scenarios" / str(args.get("name") or "") / "scenario.json"
from .scenario_tools import edited, load_scenarios

scenario_key = str(args.get("scenario_key") or "")
target = next(
(
one
for one in load_scenarios(self.workspace)
if scenario_key and one.scenario_key == scenario_key
),
None,
)
name = str(args.get("name") or "") if target is None else target.name
# What the submission names, a field it clears included.
cleared = [str(path) for path in args.get("clear") or ()]
named = set(args) | {path.split(".")[0] for path in cleared}
sent = set(args.get("persona") or {}) | {
path.split(".", 1)[1] for path in cleared if path.startswith("persona.")
}
if "persona" in cleared:
sent |= {"name", "gender", "age_group"}
change = args if target is None else edited(target, args)
path = self.workspace / "scenarios" / name / "scenario.json"
try:
held = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return False
if any(field in args and args.get(field) != held.get(field) for field in fields):
from .scenario import Scenario

try:
saved_now = Scenario.model_validate(held).model_dump()
submitted = Scenario.model_validate(
change if target is not None else {**held, **change}
).model_dump()
except ValueError:
saved_now, submitted = held, {**held, **change}
if any(
field in named and submitted.get(field) != saved_now.get(field)
for field in fields
):
return True
persona, before = args.get("persona") or {}, held.get("persona") or {}
persona, before = change.get("persona") or {}, held.get("persona") or {}
if whole_persona:
from .scenario import Persona

Expand All @@ -547,8 +594,8 @@ def saved(one: dict[str, Any]) -> dict[str, Any]:
except ValueError: # an unreadable persona is never a safe unattended change
return True
return any(
key in persona and persona.get(key) != before.get(key)
for key in ("name", "gender", "age_group")
field in sent and persona.get(field) != before.get(field)
for field in ("name", "gender", "age_group")
)

def _builder_platform_server(self):
Expand All @@ -567,9 +614,9 @@ async def call(request: Awaitable[dict[str, Any]]) -> dict[str, Any]:
"Read the environment's scenarios exactly as the Scenarios tab shows them, with each "
"one's tab number. Returns total_scenarios (the whole suite), matching (how many fit "
"what you asked) and one page of rows. Narrow with any of: `numbers` (tab numbers, "
"e.g. [12, 250, 499]), `ranges` (e.g. [{\"from\": 3, \"to\": 7}]), `names`, `search` (every word must match a "
'e.g. [12, 250, 499]), `ranges` (e.g. [{"from": 3, "to": 7}]), `names`, `search` (every word must match a '
"scenario's name, situation, use case or branch), `filters` (field to a list of "
"values, e.g. {\"persona.accent\": [\"Indian\"]}; fields: use_case, persona.name, "
'values, e.g. {"persona.accent": ["Indian"]}; fields: use_case, persona.name, '
"persona.accent, persona.languages, persona.age_group, persona.gender, "
"persona.location, persona.personality, persona.communication_style, "
"background_noise, coverage.overlay, coverage.overlay_intensity, coverage.task, "
Expand Down
39 changes: 39 additions & 0 deletions src/fi/alk/harness/scenario.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from collections import Counter, defaultdict
from itertools import combinations
from math import ceil, log
from pathlib import Path
from typing import Any, ClassVar

from pydantic import BaseModel, Field, field_validator, model_validator
Expand Down Expand Up @@ -208,6 +209,44 @@ def _slug(name: str) -> str:
return cleaned or "scenario-" + hashlib.sha256(name.encode()).hexdigest()[:12]


_USED_KEYS: dict[str, set[str]] = {}


def used_keys(destination: Any) -> set[str]:
"""Every key the environment at ``destination`` has given a scenario, a dropped one's included,
so a new scenario never takes one and inherits another's row and run history. One live set per
environment: whatever adds to it is seen by every writer."""
return _USED_KEYS.setdefault(str(Path(destination).resolve()), set())


def unused_key(key: str, taken: set[str]) -> str:
"""``key``, or it with a numbered suffix when another scenario already holds it."""
if not key or key not in taken:
return key
suffix = 2
while f"{key}-{suffix}" in taken:
suffix += 1
return f"{key}-{suffix}"


def with_unique_keys(
scenarios: list[Scenario], used: set[str] | None = None
) -> list[Scenario]:
"""``scenarios`` with a key each: the first holder of a repeated key keeps it, and every other
one gets the next key no scenario holds or has held, the same one the platform gives it."""
held = {one.scenario_key for one in scenarios} | (used or set())
keys: set[str] = set()
out: list[Scenario] = []
for one in scenarios:
if not one.scenario_key or one.scenario_key in keys:
key = unused_key(one.scenario_key or _slug(one.name), held | keys)
one = one.model_copy(update={"scenario_key": key})
held.add(key)
keys.add(one.scenario_key)
out.append(one)
return out


class Scenario(BaseModel):
"""One test: what changes, what is asked, what a correct agent does, what must hold."""

Expand Down
Loading
Loading