diff --git a/CHANGELOG.md b/CHANGELOG.md index bf49aeea0..7a33a43c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ All notable release changes for Agent Learning Kit are tracked here. Post-rc.1 increments on the release branch. Each one adds gates on top of the 66 proved at rc.1; the rc.1 entry below is historical and unchanged. +- A chat conversation never sends the scenario's instruction as the simulated person's first + message: an empty or echoed opening is asked for again, then for a greeting in their language, + and the person's own session records what the agent actually heard. - Runtime validation checks scenarios on one database lane per CPU by default, capped by the sandbox's cgroup CPU quota, instead of a fixed 16; `ALK_VALIDATION_DB_LANES` still overrides it. - Docs corpus: the full `docs/` tree (quickstarts, per-track guides, framework diff --git a/src/fi/alk/harness/run/conversation.py b/src/fi/alk/harness/run/conversation.py index b3972dd84..2c999f775 100644 --- a/src/fi/alk/harness/run/conversation.py +++ b/src/fi/alk/harness/run/conversation.py @@ -13,7 +13,9 @@ import json import time +import unicodedata from dataclasses import dataclass, field +from itertools import accumulate from typing import Any, Callable from ..backends import SessionSpec @@ -126,32 +128,50 @@ def crashed(self) -> list[Call]: # as the agent instead of to it. OPENING = ( "The conversation is starting and you speak first. You are the person making contact, not " - "the agent being contacted. Say what you came to say, in your own words, and nothing else. " - "Do not offer to look anything up, do not answer on their behalf, and do not greet them and " - "wait: say the thing you actually want." + "the agent being contacted. What you write now are your first words to the agent, in the " + "language you speak. Say what you came to say, in your own words, and nothing else. Do not " + "offer to look anything up and do not answer on the agent's behalf." ) -# How an opening turn reads when the part has been swapped. Offers of help, not requests for it. -_AS_THE_AGENT = ( - "let me ", - "i'll look", - "i will look", - "i'd be happy to look", - "i can help you with that", - "how can i help", - "how may i help", - "what would you like to know", - "i'll check", - "i will check", - "let me check", - "sure! let me", +# When an opening line is not sent, the person is told so, because their own session keeps every +# line they wrote: the next one has to follow from what the agent actually heard, which is nothing. +_UNSENT = ( + "That line was not sent: it was empty, or it repeated the description of your situation " + "instead of saying something to the agent, so the agent has heard nothing yet. " +) +# The second try asks again for what they came to say; the last asks only for the greeting a person +# opens with, in the language they speak, so the request comes after the agent's reply. +_RETRY = ( + _UNSENT + + "You are the person making contact. Say what you came to say to the agent, in your own words." +) +_GREET = ( + _UNSENT + + "Open as a person does: say only a short greeting, in the language you speak." ) -def _answered_as_the_agent(said: str) -> bool: - """Whether the opening line is the agent's part rather than the person's.""" - opening = said.strip().lower() - return any(mark in opening for mark in _AS_THE_AGENT) +def _opening_line(said: str, instruction: str) -> str: + """The line the agent hears, or "" when it is not one: empty, only an end marker, or the + scenario's own instruction handed back instead of the person's words.""" + line = said.replace(DONE, "").replace(STUCK, "").strip() + written, heard = ( + " ".join(unicodedata.normalize("NFKC", text).casefold().split()) + for text in (instruction, line) + ) + # The instruction must sit on whole words of the line, so a line that only shares a phrase + # with it is still sent. Written without spaces, the spaces the line added are ignored. + if " " in written: + echoed = f" {written} " in f" {heard} " + else: + words = heard.split() + edges = {0, *accumulate(map(len, words))} + joined = "".join(words) + echoed = bool(written) and any( + joined.startswith(written, at) and at + len(written) in edges + for at in edges + ) + return "" if echoed else line def customer_prompt( @@ -231,16 +251,16 @@ def record(speaker: str, text: str) -> None: # The customer opens, in its own words. The scenario's instruction is written *about* # the caller ("orders two burgers and asks for..."), so speaking it verbatim would hand # the agent a stage direction instead of a person. - opening = await customer.say(OPENING) - said = opening.text.strip() or scenario.instruction - if _answered_as_the_agent(said): - # The opening turn is the one with no conversation behind it, and a model asked to - # speak into that gap will sometimes take the other part: it offers to look something - # up, the agent replies that no question was asked, and the run fails for a reason - # that has nothing to do with the agent. The instruction is the fallback, because a - # blunt version of the right question tests more than a fluent version of the wrong - # one. - said = scenario.instruction + said = _opening_line((await customer.say(OPENING)).text, scenario.instruction) + for again in (_RETRY, _GREET): + if said: + break + said = _opening_line((await customer.say(again)).text, scenario.instruction) + # Only when every ask came back unusable does the run open with a line the person did not + # write; they are told it with the agent's reply, so their session matches what was said. + opened_for_them = not said + if opened_for_them: + said = "Hello." record("customer", said) for _turn in range(max(1, scenario.max_turns)): try: @@ -252,7 +272,11 @@ def record(speaker: str, text: str) -> None: break record("agent", reply or "(said nothing)") - turn = await customer.say(reply or "(no response)") + heard = reply or "(no response)" + if opened_for_them: + heard = f"You opened with: {said}\nThe agent replied: {heard}" + opened_for_them = False + turn = await customer.say(heard) said = turn.text.strip() if DONE in said or STUCK in said: transcript.ended = GAVE_UP if STUCK in said else FINISHED diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index e47e75471..d3efaf784 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -324,6 +324,8 @@ the checks and the judge, so everything else stays out of their reach: own words, even when the caller asked for that very thing (the caller fixes what they want, not the words the agent records it under), and the same for a place or description the caller gave and the agent writes down (the identifier the world resolves it to is fixed, from the record); +- free text the agent composes to find something, such as a search query: many wordings find the + same thing, so check what it found (what came back, or what it acted on), not the words it used; - a value the agent's code fills in elsewhere, which does not fix what the agent's model passes; - an optional argument nothing fixes: leave it out, and the tool's default applies. diff --git a/tests/test_harness.py b/tests/test_harness.py index 320c18432..5ddbc362d 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6309,35 +6309,180 @@ def __init__(self): assert blind(survived) == [] -def test_an_opening_line_in_the_agents_voice_falls_back_to_the_instruction(): - """The opening turn is the one with no conversation behind it, and a model asked to speak into - that gap sometimes takes the other part. +ECHOED = "You contact support. You open by asking about your order." +RETRIED = "Hi, where is my order?" - Both of these are real opening lines from a run: the agent then replied that no question had - been asked, and the scenario failed for a reason that had nothing to do with the agent. - """ - from fi.alk.harness.run.conversation import OPENING, _answered_as_the_agent - assert _answered_as_the_agent( - "Sure! Let me find the database and make that update for you." +def _open_a_conversation(monkeypatch, openings, replies=("Done.",), instruction=ECHOED): + """Run converse with a caller that remembers every message it was sent, like a real session.""" + import asyncio + from types import SimpleNamespace + + from fi.alk.harness.contract import AgentContract + from fi.alk.harness.run import conversation + from fi.alk.harness.scenario import Scenario + + openings = iter(openings) + caller_saw = [] + agent_heard = [] + replies = iter(replies) + + async def noop(*_args): + return None + + async def caller_says(prompt): + caller_saw.append(prompt) + text = next(openings, None) + return SimpleNamespace(text=text if text is not None else "Thanks, bye. [DONE]") + + customer = SimpleNamespace( + __aenter__=noop, + __aexit__=noop, + say=caller_says, + spent_usd=0.0, + simulator_tokens=(0, 0), ) - assert _answered_as_the_agent( - "I'd be happy to look that up! Let me check the database." + monkeypatch.setattr(conversation, "Stage", lambda *_a, **_k: customer) + + async def agent_hears(text): + agent_heard.append(text) + reply = next(replies, None) + if reply is None: + raise conversation.TargetConversationEnded("Done.") + return reply + + target = SimpleNamespace(open=noop, say=agent_hears, close=noop, spent_usd=0.0) + scenario = Scenario(name="s", instruction=instruction) + asyncio.run(conversation.converse(target, scenario, AgentContract())) + return caller_saw, agent_heard + + +def test_the_opening_ask_says_who_speaks_first_and_in_which_language(): + """Which part the person plays is set by guidance, not by matching what they wrote.""" + from fi.alk.harness.run.conversation import OPENING + + assert "you speak first" in OPENING + assert "the person making contact, not the agent being contacted" in OPENING + assert "in the language you speak" in OPENING + + +@pytest.mark.parametrize( + "instruction, said, sent", + [ + (ECHOED, "", False), + (ECHOED, " \n\t", False), + (ECHOED, "[DONE]", False), + (ECHOED, "[STUCK]", False), + (ECHOED, ECHOED, False), + ( + ECHOED, + " you CONTACT support. You open by\nasking about your order. ", + False, + ), + (ECHOED, f"Hi! {ECHOED}", False), + ("Você quer saber do pedido.", "Voc\u0065\u0302 quer saber do pedido.", False), + ("我想取消订单", "我想 取消订单", False), + (ECHOED, "Let me explain, my order never came", True), + (ECHOED, "Merhaba, siparişim gelmedi.", True), + (ECHOED, "你好,我的订单还没到。", True), + ("Cancel my order", "Hi, I want to cancel my order.", True), + ("hi", "Is this the shop?", True), + ("", "Where is my order?", True), + ], +) +def test_an_opening_is_held_back_only_when_empty_a_marker_or_the_instruction( + monkeypatch, instruction, said, sent +): + _, agent_heard = _open_a_conversation( + monkeypatch, [said, RETRIED], replies=(), instruction=instruction ) - assert _answered_as_the_agent("What would you like to know about the database?") + assert agent_heard == [said.strip() if sent else RETRIED] + - # What a person actually says, which must survive untouched. - assert not _answered_as_the_agent("How many of your customers are based in Canada?") - assert not _answered_as_the_agent( - "I want all track prices changed to 0.99 for the promotion, can you update them?" +@pytest.mark.parametrize( + "openings, first", + [ + ([RETRIED], RETRIED), + (["", RETRIED], RETRIED), + ([ECHOED, RETRIED], RETRIED), + ([ECHOED, ECHOED, "Hi there."], "Hi there."), + (["", "", "Merhaba."], "Merhaba."), + (["", "", ""], "Hello."), + ([ECHOED, ECHOED, ECHOED], "Hello."), + ], +) +def test_an_unusable_opening_is_corrected_then_a_greeting_and_the_instruction_is_never_sent( + monkeypatch, openings, first +): + from fi.alk.harness.run.conversation import OPENING + + caller_saw, agent_heard = _open_a_conversation(monkeypatch, openings, replies=()) + assert agent_heard == [first] + # Each retry tells the person their last line was not sent, and the last asks for a greeting + # in their own language, so a retry is never the same question with the failure in context. + assert caller_saw[0] == OPENING + asks = caller_saw[1 : len(openings)] + assert all("not sent" in ask for ask in asks) + assert len(set(asks)) == len(asks) + if len(asks) == 2: + assert "greeting, in the language you speak" in asks[1] + + +LONG = "My order " + "never arrived and " * 180 + "I want a refund." + + +@pytest.mark.parametrize( + "openings, first", + [ + ([" \n\t", RETRIED], RETRIED), + (["[DONE]", RETRIED], RETRIED), + ([RETRIED + " [DONE]"], RETRIED), + (["مرحبا، طلبي لم يصل بعد"], "مرحبا، طلبي لم يصل بعد"), + (["", "", "你好"], "你好"), + ([ECHOED, "[STUCK]", "नमस्ते"], "नमस्ते"), + ([LONG], LONG.strip()), + ( + ["The website said 'how can I help you' and then crashed."], + "The website said 'how can I help you' and then crashed.", + ), + ], +) +def test_openings_in_any_script_or_shape_reach_the_agent_as_the_person_wrote_them( + monkeypatch, openings, first +): + _, agent_heard = _open_a_conversation(monkeypatch, openings, replies=()) + assert agent_heard == [first] + + +def test_after_a_corrected_opening_the_next_turn_is_only_the_agents_reply(monkeypatch): + caller_saw, agent_heard = _open_a_conversation( + monkeypatch, [ECHOED, RETRIED, "Thanks. [DONE]"], replies=("Let me check.",) ) - assert not _answered_as_the_agent( - "Which genre has the highest average listener rating?" + assert agent_heard == [RETRIED] + assert caller_saw[2] == "Let me check." + assert "not sent" in caller_saw[1] + + +def test_after_an_opening_the_person_did_not_write_their_next_turn_is_told_what_was_said( + monkeypatch, +): + caller_saw, agent_heard = _open_a_conversation( + monkeypatch, + ["", "", "", "My order never came."], + replies=("Hi, how can I help?",), + ) + assert agent_heard == ["Hello.", "My order never came."] + assert ( + caller_saw[-1] + == "You opened with: Hello.\nThe agent replied: Hi, how can I help?" ) - # And the instruction that produces the opening says which part to play. - assert "you speak first" in OPENING - assert "not \nthe agent being contacted" in OPENING or "not the agent" in OPENING + +def test_an_opening_the_person_wrote_reaches_them_without_a_note(monkeypatch): + caller_saw, _ = _open_a_conversation( + monkeypatch, [RETRIED, "Thanks. [DONE]"], replies=("Let me check.",) + ) + assert caller_saw[-1] == "Let me check." def test_a_refusal_scenario_is_not_vacuous_because_its_evidence_is_what_was_said():