From 2199e98b7e2f5cb1c77fd0d41838f661f97dac5f Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 14:59:21 +0530 Subject: [PATCH 01/10] fix(harness): never send the scenario instruction as the caller's opening, and keep free-text queries out of fixed values --- src/fi/alk/harness/run/conversation.py | 18 +++---- .../harness/skills/write-scenarios/SKILL.md | 2 + tests/test_harness.py | 48 +++++++++++++++++++ 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/src/fi/alk/harness/run/conversation.py b/src/fi/alk/harness/run/conversation.py index b3972dd84..cb9b0f765 100644 --- a/src/fi/alk/harness/run/conversation.py +++ b/src/fi/alk/harness/run/conversation.py @@ -232,15 +232,15 @@ def record(speaker: str, text: str) -> None: # the caller ("orders two burgers and asks for..."), so speaking it verbatim would hand # the agent a stage direction instead of a person. opening = await customer.say(OPENING) - said = opening.text.strip() or scenario.instruction - if _answered_as_the_agent(said): - # The opening turn is the one with no conversation behind it, and a model asked to - # speak into that gap will sometimes take the other part: it offers to look something - # up, the agent replies that no question was asked, and the run fails for a reason - # that has nothing to do with the agent. The instruction is the fallback, because a - # blunt version of the right question tests more than a fluent version of the wrong - # one. - said = scenario.instruction + said = opening.text.strip() + if not said or _answered_as_the_agent(said): + # A model speaking into an empty conversation sometimes says nothing or takes the + # agent's part. Ask once more; the instruction is stage directions, never sent as is. + said = (await customer.say(OPENING)).text.strip() + if not said or _answered_as_the_agent(said): + from ..hosted_scheduler import CallAborted + + raise CallAborted("simulator_no_opening: the caller gave no usable first message", code="simulator_stalled") record("customer", said) for _turn in range(max(1, scenario.max_turns)): try: diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index e47e75471..56df53ce5 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -324,6 +324,8 @@ the checks and the judge, so everything else stays out of their reach: own words, even when the caller asked for that very thing (the caller fixes what they want, not the words the agent records it under), and the same for a place or description the caller gave and the agent writes down (the identifier the world resolves it to is fixed, from the record); +- free text the agent composes to find something, such as a search query: many wordings find the + same thing, so check what it found (the record returned or acted on), not the words it searched with; - a value the agent's code fills in elsewhere, which does not fix what the agent's model passes; - an optional argument nothing fixes: leave it out, and the tool's default applies. diff --git a/tests/test_harness.py b/tests/test_harness.py index 320c18432..76607d82e 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6340,6 +6340,54 @@ def test_an_opening_line_in_the_agents_voice_falls_back_to_the_instruction(): assert "not \nthe agent being contacted" in OPENING or "not the agent" in OPENING +def test_a_caller_that_gives_no_usable_opening_aborts_instead_of_sending_its_instruction(monkeypatch): + import asyncio + + from fi.alk.harness.contract import AgentContract + from fi.alk.harness.hosted_scheduler import CallAborted + from fi.alk.harness.run import conversation + from fi.alk.harness.scenario import Scenario + + openings = iter(["", "Sure! Let me look that up for you."]) + sent = [] + + class Customer: + spent_usd = 0.0 + simulator_tokens = (0, 0) + + def __init__(self, *_args, **_kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + pass + + async def say(self, _prompt): + return type("Turn", (), {"text": next(openings)})() + + class Target: + spent_usd = 0.0 + + async def open(self): + pass + + async def say(self, text): + sent.append(text) + return "ok" + + async def close(self): + pass + + monkeypatch.setattr(conversation, "Stage", Customer) + scenario = Scenario(name="s", instruction="You contact support. You open by asking about Dune.") + with pytest.raises(CallAborted) as aborted: + asyncio.run(conversation.converse(Target(), scenario, AgentContract())) + assert aborted.value.code == "simulator_stalled" + assert sent == [] + + def test_a_refusal_scenario_is_not_vacuous_because_its_evidence_is_what_was_said(): """Where the right behaviour is to decline and touch nothing, every check about the world holds with nothing done, and the explanation is the only real evidence. From dc1a15c553bd5d5f4565497501bb7bebf30f98bf Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 14:59:52 +0530 Subject: [PATCH 02/10] test(harness): keep the opening test's instruction generic --- tests/test_harness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_harness.py b/tests/test_harness.py index 76607d82e..4abe18b2b 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6381,7 +6381,7 @@ async def close(self): pass monkeypatch.setattr(conversation, "Stage", Customer) - scenario = Scenario(name="s", instruction="You contact support. You open by asking about Dune.") + scenario = Scenario(name="s", instruction="You contact support. You open by asking about your order.") with pytest.raises(CallAborted) as aborted: asyncio.run(conversation.converse(Target(), scenario, AgentContract())) assert aborted.value.code == "simulator_stalled" From 3bdb7fd7237c19ce0c51c27a7138b59fe177b3e1 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 15:01:15 +0530 Subject: [PATCH 03/10] test(harness): shorten the opening test --- tests/test_harness.py | 39 +++++++++++---------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/tests/test_harness.py b/tests/test_harness.py index 4abe18b2b..ea146d59c 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6342,6 +6342,7 @@ def test_an_opening_line_in_the_agents_voice_falls_back_to_the_instruction(): def test_a_caller_that_gives_no_usable_opening_aborts_instead_of_sending_its_instruction(monkeypatch): import asyncio + from types import SimpleNamespace from fi.alk.harness.contract import AgentContract from fi.alk.harness.hosted_scheduler import CallAborted @@ -6351,43 +6352,25 @@ def test_a_caller_that_gives_no_usable_opening_aborts_instead_of_sending_its_ins openings = iter(["", "Sure! Let me look that up for you."]) sent = [] - class Customer: - spent_usd = 0.0 - simulator_tokens = (0, 0) - - def __init__(self, *_args, **_kwargs): - pass + async def noop(*_args): + return None - async def __aenter__(self): - return self + async def opening(_prompt): + return SimpleNamespace(text=next(openings)) - async def __aexit__(self, *_args): - pass + customer = SimpleNamespace(__aenter__=noop, __aexit__=noop, say=opening, spent_usd=0.0, simulator_tokens=(0, 0)) + monkeypatch.setattr(conversation, "Stage", lambda *_a, **_k: customer) - async def say(self, _prompt): - return type("Turn", (), {"text": next(openings)})() + async def say(text): + sent.append(text) - class Target: - spent_usd = 0.0 - - async def open(self): - pass - - async def say(self, text): - sent.append(text) - return "ok" - - async def close(self): - pass - - monkeypatch.setattr(conversation, "Stage", Customer) + target = SimpleNamespace(open=noop, say=say, close=noop, spent_usd=0.0) scenario = Scenario(name="s", instruction="You contact support. You open by asking about your order.") with pytest.raises(CallAborted) as aborted: - asyncio.run(conversation.converse(Target(), scenario, AgentContract())) + asyncio.run(conversation.converse(target, scenario, AgentContract())) assert aborted.value.code == "simulator_stalled" assert sent == [] - def test_a_refusal_scenario_is_not_vacuous_because_its_evidence_is_what_was_said(): """Where the right behaviour is to decline and touch nothing, every check about the world holds with nothing done, and the explanation is the only real evidence. From 19331bfec0bf862c43d2bd1b9584f18409e7f01b Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 15:50:00 +0530 Subject: [PATCH 04/10] test(harness): cover the opening retry that succeeds --- tests/test_harness.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/test_harness.py b/tests/test_harness.py index ea146d59c..3138660c6 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6340,7 +6340,16 @@ def test_an_opening_line_in_the_agents_voice_falls_back_to_the_instruction(): assert "not \nthe agent being contacted" in OPENING or "not the agent" in OPENING -def test_a_caller_that_gives_no_usable_opening_aborts_instead_of_sending_its_instruction(monkeypatch): +@pytest.mark.parametrize( + "openings, first", + [ + (["", "Sure! Let me look that up for you."], None), + (["Sure! Let me look that up for you.", "Hi, where is my order?"], "Hi, where is my order?"), + ], +) +def test_an_unusable_opening_is_asked_again_once_then_aborts_never_sending_the_instruction( + monkeypatch, openings, first +): import asyncio from types import SimpleNamespace @@ -6349,7 +6358,7 @@ def test_a_caller_that_gives_no_usable_opening_aborts_instead_of_sending_its_ins from fi.alk.harness.run import conversation from fi.alk.harness.scenario import Scenario - openings = iter(["", "Sure! Let me look that up for you."]) + openings = iter(openings) sent = [] async def noop(*_args): @@ -6363,13 +6372,18 @@ async def opening(_prompt): async def say(text): sent.append(text) + raise conversation.TargetConversationEnded("Done.") target = SimpleNamespace(open=noop, say=say, close=noop, spent_usd=0.0) scenario = Scenario(name="s", instruction="You contact support. You open by asking about your order.") - with pytest.raises(CallAborted) as aborted: + if first is None: + with pytest.raises(CallAborted) as aborted: + asyncio.run(conversation.converse(target, scenario, AgentContract())) + assert aborted.value.code == "simulator_stalled" + assert sent == [] + else: asyncio.run(conversation.converse(target, scenario, AgentContract())) - assert aborted.value.code == "simulator_stalled" - assert sent == [] + assert sent == [first] def test_a_refusal_scenario_is_not_vacuous_because_its_evidence_is_what_was_said(): """Where the right behaviour is to decline and touch nothing, every check about the world holds From b543234565a3a654a4e1746dac93e58408424f15 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 15:55:43 +0530 Subject: [PATCH 05/10] style(harness): tighten the opening retry comment and format it --- src/fi/alk/harness/run/conversation.py | 9 ++++++--- tests/test_harness.py | 19 ++++++++++++++++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/fi/alk/harness/run/conversation.py b/src/fi/alk/harness/run/conversation.py index cb9b0f765..df6f872c1 100644 --- a/src/fi/alk/harness/run/conversation.py +++ b/src/fi/alk/harness/run/conversation.py @@ -234,13 +234,16 @@ def record(speaker: str, text: str) -> None: opening = await customer.say(OPENING) said = opening.text.strip() if not said or _answered_as_the_agent(said): - # A model speaking into an empty conversation sometimes says nothing or takes the - # agent's part. Ask once more; the instruction is stage directions, never sent as is. + # Speaking into an empty conversation, a model sometimes says nothing or answers as + # the agent. Ask once more. said = (await customer.say(OPENING)).text.strip() if not said or _answered_as_the_agent(said): from ..hosted_scheduler import CallAborted - raise CallAborted("simulator_no_opening: the caller gave no usable first message", code="simulator_stalled") + raise CallAborted( + "simulator_no_opening: the caller gave no usable first message", + code="simulator_stalled", + ) record("customer", said) for _turn in range(max(1, scenario.max_turns)): try: diff --git a/tests/test_harness.py b/tests/test_harness.py index 3138660c6..8bfc17f7f 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6344,7 +6344,10 @@ def test_an_opening_line_in_the_agents_voice_falls_back_to_the_instruction(): "openings, first", [ (["", "Sure! Let me look that up for you."], None), - (["Sure! Let me look that up for you.", "Hi, where is my order?"], "Hi, where is my order?"), + ( + ["Sure! Let me look that up for you.", "Hi, where is my order?"], + "Hi, where is my order?", + ), ], ) def test_an_unusable_opening_is_asked_again_once_then_aborts_never_sending_the_instruction( @@ -6367,7 +6370,13 @@ async def noop(*_args): async def opening(_prompt): return SimpleNamespace(text=next(openings)) - customer = SimpleNamespace(__aenter__=noop, __aexit__=noop, say=opening, spent_usd=0.0, simulator_tokens=(0, 0)) + customer = SimpleNamespace( + __aenter__=noop, + __aexit__=noop, + say=opening, + spent_usd=0.0, + simulator_tokens=(0, 0), + ) monkeypatch.setattr(conversation, "Stage", lambda *_a, **_k: customer) async def say(text): @@ -6375,7 +6384,10 @@ async def say(text): raise conversation.TargetConversationEnded("Done.") target = SimpleNamespace(open=noop, say=say, close=noop, spent_usd=0.0) - scenario = Scenario(name="s", instruction="You contact support. You open by asking about your order.") + scenario = Scenario( + name="s", + instruction="You contact support. You open by asking about your order.", + ) if first is None: with pytest.raises(CallAborted) as aborted: asyncio.run(conversation.converse(target, scenario, AgentContract())) @@ -6385,6 +6397,7 @@ async def say(text): asyncio.run(conversation.converse(target, scenario, AgentContract())) assert sent == [first] + def test_a_refusal_scenario_is_not_vacuous_because_its_evidence_is_what_was_said(): """Where the right behaviour is to decline and touch nothing, every check about the world holds with nothing done, and the explanation is the only real evidence. From c5585786b52eb7512e83f5e3301844cd4085b88d Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 16:29:20 +0530 Subject: [PATCH 06/10] fix(harness): abort only when the caller gives no opening at all --- src/fi/alk/harness/run/conversation.py | 4 ++-- src/fi/alk/harness/skills/write-scenarios/SKILL.md | 2 +- tests/test_harness.py | 13 +++++++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/fi/alk/harness/run/conversation.py b/src/fi/alk/harness/run/conversation.py index df6f872c1..2ea96bc33 100644 --- a/src/fi/alk/harness/run/conversation.py +++ b/src/fi/alk/harness/run/conversation.py @@ -236,8 +236,8 @@ def record(speaker: str, text: str) -> None: if not said or _answered_as_the_agent(said): # Speaking into an empty conversation, a model sometimes says nothing or answers as # the agent. Ask once more. - said = (await customer.say(OPENING)).text.strip() - if not said or _answered_as_the_agent(said): + said = (await customer.say(OPENING)).text.strip() or said + if not said: from ..hosted_scheduler import CallAborted raise CallAborted( diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 56df53ce5..d3efaf784 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -325,7 +325,7 @@ the checks and the judge, so everything else stays out of their reach: words the agent records it under), and the same for a place or description the caller gave and the agent writes down (the identifier the world resolves it to is fixed, from the record); - free text the agent composes to find something, such as a search query: many wordings find the - same thing, so check what it found (the record returned or acted on), not the words it searched with; + same thing, so check what it found (what came back, or what it acted on), not the words it used; - a value the agent's code fills in elsewhere, which does not fix what the agent's model passes; - an optional argument nothing fixes: leave it out, and the tool's default applies. diff --git a/tests/test_harness.py b/tests/test_harness.py index 8bfc17f7f..9b3f3d337 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6343,14 +6343,23 @@ def test_an_opening_line_in_the_agents_voice_falls_back_to_the_instruction(): @pytest.mark.parametrize( "openings, first", [ - (["", "Sure! Let me look that up for you."], None), + (["", ""], None), ( ["Sure! Let me look that up for you.", "Hi, where is my order?"], "Hi, where is my order?", ), + (["", "Hi, let me ask about my order."], "Hi, let me ask about my order."), + ( + ["Hi, let me ask about my order.", "Hi, let me ask about my order."], + "Hi, let me ask about my order.", + ), + ( + ["Sure! Let me look that up for you.", ""], + "Sure! Let me look that up for you.", + ), ], ) -def test_an_unusable_opening_is_asked_again_once_then_aborts_never_sending_the_instruction( +def test_an_unusable_opening_is_asked_again_once_and_the_instruction_is_never_sent( monkeypatch, openings, first ): import asyncio From 7f4326454f2297c5219e59f9b13cbde462d6e9a7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 21:40:26 +0530 Subject: [PATCH 07/10] fix(harness): a caller with no usable opening greets instead of ending the call --- src/fi/alk/harness/run/conversation.py | 14 +++++--------- tests/test_harness.py | 24 +++++++----------------- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/src/fi/alk/harness/run/conversation.py b/src/fi/alk/harness/run/conversation.py index 2ea96bc33..b19cec859 100644 --- a/src/fi/alk/harness/run/conversation.py +++ b/src/fi/alk/harness/run/conversation.py @@ -235,15 +235,11 @@ def record(speaker: str, text: str) -> None: said = opening.text.strip() if not said or _answered_as_the_agent(said): # Speaking into an empty conversation, a model sometimes says nothing or answers as - # the agent. Ask once more. - said = (await customer.say(OPENING)).text.strip() or said - if not said: - from ..hosted_scheduler import CallAborted - - raise CallAborted( - "simulator_no_opening: the caller gave no usable first message", - code="simulator_stalled", - ) + # the agent. Ask once more; if that is no better, open as a caller does, with a + # greeting, and the request follows the agent's reply. + said = (await customer.say(OPENING)).text.strip() + if not said or _answered_as_the_agent(said): + said = "Hello." record("customer", said) for _turn in range(max(1, scenario.max_turns)): try: diff --git a/tests/test_harness.py b/tests/test_harness.py index 9b3f3d337..321eb40e7 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6343,19 +6343,16 @@ def test_an_opening_line_in_the_agents_voice_falls_back_to_the_instruction(): @pytest.mark.parametrize( "openings, first", [ - (["", ""], None), + (["", ""], "Hello."), + (["", "Hi, where is my order?"], "Hi, where is my order?"), ( ["Sure! Let me look that up for you.", "Hi, where is my order?"], "Hi, where is my order?", ), - (["", "Hi, let me ask about my order."], "Hi, let me ask about my order."), + (["Sure! Let me look that up for you.", ""], "Hello."), ( - ["Hi, let me ask about my order.", "Hi, let me ask about my order."], - "Hi, let me ask about my order.", - ), - ( - ["Sure! Let me look that up for you.", ""], - "Sure! Let me look that up for you.", + ["Sure! Let me look that up for you.", "Sure! Let me look that up for you."], + "Hello.", ), ], ) @@ -6366,7 +6363,6 @@ def test_an_unusable_opening_is_asked_again_once_and_the_instruction_is_never_se from types import SimpleNamespace from fi.alk.harness.contract import AgentContract - from fi.alk.harness.hosted_scheduler import CallAborted from fi.alk.harness.run import conversation from fi.alk.harness.scenario import Scenario @@ -6397,14 +6393,8 @@ async def say(text): name="s", instruction="You contact support. You open by asking about your order.", ) - if first is None: - with pytest.raises(CallAborted) as aborted: - asyncio.run(conversation.converse(target, scenario, AgentContract())) - assert aborted.value.code == "simulator_stalled" - assert sent == [] - else: - asyncio.run(conversation.converse(target, scenario, AgentContract())) - assert sent == [first] + asyncio.run(conversation.converse(target, scenario, AgentContract())) + assert sent == [first] def test_a_refusal_scenario_is_not_vacuous_because_its_evidence_is_what_was_said(): From 7e4e04b9699c56b2799c483aa1a28aae3521b051 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 26 Oct 2026 13:44:41 +0530 Subject: [PATCH 08/10] fix(harness): TH-8765 tell the caller when an opening was not sent, retry differently, greet in the caller's language, anchor the agent-voice check (also keeps the write-scenarios free-text search rule) --- src/fi/alk/harness/run/conversation.py | 78 ++++++++----- tests/test_harness.py | 156 ++++++++++++++++++++----- 2 files changed, 179 insertions(+), 55 deletions(-) diff --git a/src/fi/alk/harness/run/conversation.py b/src/fi/alk/harness/run/conversation.py index b19cec859..9a7b73bbf 100644 --- a/src/fi/alk/harness/run/conversation.py +++ b/src/fi/alk/harness/run/conversation.py @@ -12,6 +12,7 @@ from __future__ import annotations import json +import re import time from dataclasses import dataclass, field from typing import Any, Callable @@ -131,27 +132,47 @@ def crashed(self) -> list[Call]: "wait: say the thing you actually want." ) -# How an opening turn reads when the part has been swapped. Offers of help, not requests for it. -_AS_THE_AGENT = ( - "let me ", - "i'll look", - "i will look", - "i'd be happy to look", - "i can help you with that", - "how can i help", - "how may i help", - "what would you like to know", - "i'll check", - "i will check", - "let me check", - "sure! let me", +# When an opening line is not sent, the person is told so, because their own session keeps every +# line they wrote: the next one has to follow from what the agent actually heard, which is nothing. +_UNSENT = ( + "That line was not sent: it was empty, or it spoke as the agent instead of to them, so the " + "agent has heard nothing yet. " +) +# The second try asks again for what they came to say; the last asks only for the greeting a person +# opens with, in the language they speak, so the request comes after the agent's reply. +_RETRY = ( + _UNSENT + + "You are the person making contact. Say what you came to say, in your own words." +) +_GREET = ( + _UNSENT + + "Open as a person does: say only a short greeting, in the language you speak." +) + +# How an opening turn reads when the part has been swapped: it starts by offering help instead of +# asking for it, after at most a greeting, and the offer is aimed at the other person ("for you", +# "your", "that"). So a person who says "Let me check my email" or quotes what a website told them +# keeps their line. +_AS_THE_AGENT = re.compile( + r"^\W*(?:(?:hi|hello|hey|sure|of course|certainly|absolutely|okay|ok|great)\W+)*" + r"(?:let me (?:check|look|find|pull|see if|get)\b(?=.*\bfor you\b|\s+(?:that|this|it|into|up|your)\b)" + r"|i(?:'d| would) be happy to (?:help|look|check)" + r"|i can help you with" + r"|how (?:can|may) i (?:help|assist)(?:\s+you\b|\s*[?.!,]|\s*$)" + r"|what would you like to know" + r"|i(?:'ll| will) (?:look|check) (?:that|this|it|into))" ) def _answered_as_the_agent(said: str) -> bool: """Whether the opening line is the agent's part rather than the person's.""" - opening = said.strip().lower() - return any(mark in opening for mark in _AS_THE_AGENT) + return bool(_AS_THE_AGENT.search(said.strip().lower())) + + +def _opening_line(said: str) -> str: + """The line the agent hears, or "" when it is not one: empty, only an end marker, or the agent's part.""" + line = said.replace(DONE, "").replace(STUCK, "").strip() + return "" if _answered_as_the_agent(line) else line def customer_prompt( @@ -231,15 +252,16 @@ def record(speaker: str, text: str) -> None: # The customer opens, in its own words. The scenario's instruction is written *about* # the caller ("orders two burgers and asks for..."), so speaking it verbatim would hand # the agent a stage direction instead of a person. - opening = await customer.say(OPENING) - said = opening.text.strip() - if not said or _answered_as_the_agent(said): - # Speaking into an empty conversation, a model sometimes says nothing or answers as - # the agent. Ask once more; if that is no better, open as a caller does, with a - # greeting, and the request follows the agent's reply. - said = (await customer.say(OPENING)).text.strip() - if not said or _answered_as_the_agent(said): - said = "Hello." + said = _opening_line((await customer.say(OPENING)).text) + for again in (_RETRY, _GREET): + if said: + break + said = _opening_line((await customer.say(again)).text) + # Only when every ask came back unusable does the run open with a line the person did not + # write; they are told it with the agent's reply, so their session matches what was said. + opened_for_them = not said + if opened_for_them: + said = "Hello." record("customer", said) for _turn in range(max(1, scenario.max_turns)): try: @@ -251,7 +273,11 @@ def record(speaker: str, text: str) -> None: break record("agent", reply or "(said nothing)") - turn = await customer.say(reply or "(no response)") + heard = reply or "(no response)" + if opened_for_them: + heard = f"You opened with: {said}\nThe agent replied: {heard}" + opened_for_them = False + turn = await customer.say(heard) said = turn.text.strip() if DONE in said or STUCK in said: transcript.ended = GAVE_UP if STUCK in said else FINISHED diff --git a/tests/test_harness.py b/tests/test_harness.py index 321eb40e7..4bb7e76a1 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6309,7 +6309,7 @@ def __init__(self): assert blind(survived) == [] -def test_an_opening_line_in_the_agents_voice_falls_back_to_the_instruction(): +def test_an_opening_line_in_the_agents_voice_is_recognised(): """The opening turn is the one with no conversation behind it, and a model asked to speak into that gap sometimes takes the other part. @@ -6334,31 +6334,35 @@ def test_an_opening_line_in_the_agents_voice_falls_back_to_the_instruction(): assert not _answered_as_the_agent( "Which genre has the highest average listener rating?" ) + # A person who starts by explaining or checks their own details keeps their line. + assert not _answered_as_the_agent("Hi, let me explain, my order never came") + assert not _answered_as_the_agent("Let me explain what happened with my booking.") + assert not _answered_as_the_agent("I'll check my card, the number ends in 4417.") + assert not _answered_as_the_agent("Merhaba, siparişim gelmedi.") + assert _answered_as_the_agent("Okay, let me pull up your account.") + # Only the start of the line is read: a person quoting what they were told keeps their line. + assert not _answered_as_the_agent( + "The website said 'how can I help you' and then crashed, so I'm calling." + ) + assert not _answered_as_the_agent("مرحبا، طلبي لم يصل بعد") + assert not _answered_as_the_agent("你好,我的订单还没到。") + assert _answered_as_the_agent("Hi! How can I help you today?") + assert _answered_as_the_agent("How can I help?") + assert _answered_as_the_agent("Let me look that up.") + assert not _answered_as_the_agent("How can I help my daughter get a refund?") + assert not _answered_as_the_agent("Let me check my email for the order number.") + assert not _answered_as_the_agent("Let me see if I have the receipt here.") # And the instruction that produces the opening says which part to play. assert "you speak first" in OPENING assert "not \nthe agent being contacted" in OPENING or "not the agent" in OPENING -@pytest.mark.parametrize( - "openings, first", - [ - (["", ""], "Hello."), - (["", "Hi, where is my order?"], "Hi, where is my order?"), - ( - ["Sure! Let me look that up for you.", "Hi, where is my order?"], - "Hi, where is my order?", - ), - (["Sure! Let me look that up for you.", ""], "Hello."), - ( - ["Sure! Let me look that up for you.", "Sure! Let me look that up for you."], - "Hello.", - ), - ], -) -def test_an_unusable_opening_is_asked_again_once_and_the_instruction_is_never_sent( - monkeypatch, openings, first -): +AGENTLIKE = "Sure! Let me look that up for you." + + +def _open_a_conversation(monkeypatch, openings, replies=("Done.",)): + """Run converse with a caller that remembers every message it was sent, like a real session.""" import asyncio from types import SimpleNamespace @@ -6367,34 +6371,128 @@ def test_an_unusable_opening_is_asked_again_once_and_the_instruction_is_never_se from fi.alk.harness.scenario import Scenario openings = iter(openings) - sent = [] + caller_saw = [] + agent_heard = [] + replies = iter(replies) async def noop(*_args): return None - async def opening(_prompt): - return SimpleNamespace(text=next(openings)) + async def caller_says(prompt): + caller_saw.append(prompt) + text = next(openings, None) + return SimpleNamespace(text=text if text is not None else "Thanks, bye. [DONE]") customer = SimpleNamespace( __aenter__=noop, __aexit__=noop, - say=opening, + say=caller_says, spent_usd=0.0, simulator_tokens=(0, 0), ) monkeypatch.setattr(conversation, "Stage", lambda *_a, **_k: customer) - async def say(text): - sent.append(text) - raise conversation.TargetConversationEnded("Done.") + async def agent_hears(text): + agent_heard.append(text) + reply = next(replies, None) + if reply is None: + raise conversation.TargetConversationEnded("Done.") + return reply - target = SimpleNamespace(open=noop, say=say, close=noop, spent_usd=0.0) + target = SimpleNamespace(open=noop, say=agent_hears, close=noop, spent_usd=0.0) scenario = Scenario( name="s", instruction="You contact support. You open by asking about your order.", ) asyncio.run(conversation.converse(target, scenario, AgentContract())) - assert sent == [first] + return caller_saw, agent_heard + + +@pytest.mark.parametrize( + "openings, first", + [ + (["Hi, where is my order?"], "Hi, where is my order?"), + (["", "Hi, where is my order?"], "Hi, where is my order?"), + ([AGENTLIKE, "Hi, where is my order?"], "Hi, where is my order?"), + ([AGENTLIKE, AGENTLIKE, "Hi there."], "Hi there."), + (["", "", "Merhaba."], "Merhaba."), + (["", "", ""], "Hello."), + ([AGENTLIKE, AGENTLIKE, AGENTLIKE], "Hello."), + ], +) +def test_an_unusable_opening_is_corrected_then_a_greeting_and_the_instruction_is_never_sent( + monkeypatch, openings, first +): + from fi.alk.harness.run.conversation import OPENING, _GREET, _RETRY + + caller_saw, agent_heard = _open_a_conversation(monkeypatch, openings, replies=()) + assert agent_heard == [first] + # Each retry tells the person their last line was not sent, and the last asks for a greeting + # in their own language, so a retry is never the same question with the failure in context. + assert caller_saw == [OPENING, _RETRY, _GREET][: len(openings)] + assert "in the language you speak" in _GREET and "not sent" in _RETRY + + +LONG = "My order " + "never arrived and " * 180 + "I want a refund." + + +@pytest.mark.parametrize( + "openings, first", + [ + ([" \n\t", "Hi, where is my order?"], "Hi, where is my order?"), + (["[DONE]", "Hi, where is my order?"], "Hi, where is my order?"), + (["Hi, where is my order? [DONE]"], "Hi, where is my order?"), + (["مرحبا، طلبي لم يصل بعد"], "مرحبا، طلبي لم يصل بعد"), + (["", "", "你好"], "你好"), + ([AGENTLIKE, AGENTLIKE, "नमस्ते"], "नमस्ते"), + ([LONG], LONG.strip()), + ( + ["The website said 'how can I help you' and then crashed."], + "The website said 'how can I help you' and then crashed.", + ), + ], +) +def test_openings_in_any_script_or_shape_reach_the_agent_as_the_person_wrote_them( + monkeypatch, openings, first +): + _, agent_heard = _open_a_conversation(monkeypatch, openings, replies=()) + assert agent_heard == [first] + + +def test_after_a_corrected_opening_the_next_turn_is_only_the_agents_reply(monkeypatch): + caller_saw, agent_heard = _open_a_conversation( + monkeypatch, + [AGENTLIKE, "Hi, where is my order?", "Thanks. [DONE]"], + replies=("Let me check.",), + ) + from fi.alk.harness.run.conversation import OPENING, _RETRY + + assert agent_heard == ["Hi, where is my order?"] + assert caller_saw == [OPENING, _RETRY, "Let me check."] + + +def test_after_an_opening_the_person_did_not_write_their_next_turn_is_told_what_was_said( + monkeypatch, +): + caller_saw, agent_heard = _open_a_conversation( + monkeypatch, + ["", "", "", "My order never came."], + replies=("Hi, how can I help?",), + ) + assert agent_heard == ["Hello.", "My order never came."] + assert ( + caller_saw[-1] + == "You opened with: Hello.\nThe agent replied: Hi, how can I help?" + ) + + +def test_an_opening_the_person_wrote_reaches_them_without_a_note(monkeypatch): + caller_saw, _ = _open_a_conversation( + monkeypatch, + ["Hi, where is my order?", "Thanks. [DONE]"], + replies=("Let me check.",), + ) + assert caller_saw[-1] == "Let me check." def test_a_refusal_scenario_is_not_vacuous_because_its_evidence_is_what_was_said(): From 0f2716a82011f11c1f27863f250d6e10014ab6d7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 26 Oct 2026 14:33:31 +0530 Subject: [PATCH 09/10] fix(harness): TH-8765 remove the phrase check on the caller opening; structural checks only --- src/fi/alk/harness/run/conversation.py | 49 ++++++--------- tests/test_harness.py | 85 ++++++++++---------------- 2 files changed, 50 insertions(+), 84 deletions(-) diff --git a/src/fi/alk/harness/run/conversation.py b/src/fi/alk/harness/run/conversation.py index 9a7b73bbf..142d14972 100644 --- a/src/fi/alk/harness/run/conversation.py +++ b/src/fi/alk/harness/run/conversation.py @@ -12,7 +12,6 @@ from __future__ import annotations import json -import re import time from dataclasses import dataclass, field from typing import Any, Callable @@ -126,53 +125,39 @@ def crashed(self) -> list[Call]: # system prompt about an agent, and asked to speak with nothing preceding it, will sometimes reply # as the agent instead of to it. OPENING = ( - "The conversation is starting and you speak first. You are the person making contact, not " - "the agent being contacted. Say what you came to say, in your own words, and nothing else. " - "Do not offer to look anything up, do not answer on their behalf, and do not greet them and " - "wait: say the thing you actually want." + "The conversation is starting and you speak first. You are the caller: the person making " + "contact, not the agent being contacted. What you write now are your first words to the " + "agent, in the language you speak. Say what you came to say, in your own words, and nothing " + "else. Do not offer to look anything up, do not answer on their behalf, and do not greet them " + "and wait: say the thing you actually want." ) # When an opening line is not sent, the person is told so, because their own session keeps every # line they wrote: the next one has to follow from what the agent actually heard, which is nothing. _UNSENT = ( - "That line was not sent: it was empty, or it spoke as the agent instead of to them, so the " - "agent has heard nothing yet. " + "That line was not sent: it was empty, or it repeated the description of your situation " + "instead of saying something to the agent, so the agent has heard nothing yet. " ) # The second try asks again for what they came to say; the last asks only for the greeting a person # opens with, in the language they speak, so the request comes after the agent's reply. _RETRY = ( _UNSENT - + "You are the person making contact. Say what you came to say, in your own words." + + "You are the caller. Say what you came to say to the agent, in your own words." ) _GREET = ( _UNSENT + "Open as a person does: say only a short greeting, in the language you speak." ) -# How an opening turn reads when the part has been swapped: it starts by offering help instead of -# asking for it, after at most a greeting, and the offer is aimed at the other person ("for you", -# "your", "that"). So a person who says "Let me check my email" or quotes what a website told them -# keeps their line. -_AS_THE_AGENT = re.compile( - r"^\W*(?:(?:hi|hello|hey|sure|of course|certainly|absolutely|okay|ok|great)\W+)*" - r"(?:let me (?:check|look|find|pull|see if|get)\b(?=.*\bfor you\b|\s+(?:that|this|it|into|up|your)\b)" - r"|i(?:'d| would) be happy to (?:help|look|check)" - r"|i can help you with" - r"|how (?:can|may) i (?:help|assist)(?:\s+you\b|\s*[?.!,]|\s*$)" - r"|what would you like to know" - r"|i(?:'ll| will) (?:look|check) (?:that|this|it|into))" -) - - -def _answered_as_the_agent(said: str) -> bool: - """Whether the opening line is the agent's part rather than the person's.""" - return bool(_AS_THE_AGENT.search(said.strip().lower())) - -def _opening_line(said: str) -> str: - """The line the agent hears, or "" when it is not one: empty, only an end marker, or the agent's part.""" +def _opening_line(said: str, instruction: str) -> str: + """The line the agent hears, or "" when it is not one: empty, only an end marker, or the + scenario's own instruction handed back instead of the person's words.""" line = said.replace(DONE, "").replace(STUCK, "").strip() - return "" if _answered_as_the_agent(line) else line + written = " ".join(instruction.split()).casefold() + if written and written in " ".join(line.split()).casefold(): + return "" + return line def customer_prompt( @@ -252,11 +237,11 @@ def record(speaker: str, text: str) -> None: # The customer opens, in its own words. The scenario's instruction is written *about* # the caller ("orders two burgers and asks for..."), so speaking it verbatim would hand # the agent a stage direction instead of a person. - said = _opening_line((await customer.say(OPENING)).text) + said = _opening_line((await customer.say(OPENING)).text, scenario.instruction) for again in (_RETRY, _GREET): if said: break - said = _opening_line((await customer.say(again)).text) + said = _opening_line((await customer.say(again)).text, scenario.instruction) # Only when every ask came back unusable does the run open with a line the person did not # write; they are told it with the agent's reply, so their session matches what was said. opened_for_them = not said diff --git a/tests/test_harness.py b/tests/test_harness.py index 4bb7e76a1..61263641c 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6309,56 +6309,37 @@ def __init__(self): assert blind(survived) == [] -def test_an_opening_line_in_the_agents_voice_is_recognised(): - """The opening turn is the one with no conversation behind it, and a model asked to speak into - that gap sometimes takes the other part. - - Both of these are real opening lines from a run: the agent then replied that no question had - been asked, and the scenario failed for a reason that had nothing to do with the agent. - """ - from fi.alk.harness.run.conversation import OPENING, _answered_as_the_agent - - assert _answered_as_the_agent( - "Sure! Let me find the database and make that update for you." - ) - assert _answered_as_the_agent( - "I'd be happy to look that up! Let me check the database." - ) - assert _answered_as_the_agent("What would you like to know about the database?") - - # What a person actually says, which must survive untouched. - assert not _answered_as_the_agent("How many of your customers are based in Canada?") - assert not _answered_as_the_agent( - "I want all track prices changed to 0.99 for the promotion, can you update them?" - ) - assert not _answered_as_the_agent( - "Which genre has the highest average listener rating?" - ) - # A person who starts by explaining or checks their own details keeps their line. - assert not _answered_as_the_agent("Hi, let me explain, my order never came") - assert not _answered_as_the_agent("Let me explain what happened with my booking.") - assert not _answered_as_the_agent("I'll check my card, the number ends in 4417.") - assert not _answered_as_the_agent("Merhaba, siparişim gelmedi.") - assert _answered_as_the_agent("Okay, let me pull up your account.") - # Only the start of the line is read: a person quoting what they were told keeps their line. - assert not _answered_as_the_agent( - "The website said 'how can I help you' and then crashed, so I'm calling." - ) - assert not _answered_as_the_agent("مرحبا، طلبي لم يصل بعد") - assert not _answered_as_the_agent("你好,我的订单还没到。") - assert _answered_as_the_agent("Hi! How can I help you today?") - assert _answered_as_the_agent("How can I help?") - assert _answered_as_the_agent("Let me look that up.") - assert not _answered_as_the_agent("How can I help my daughter get a refund?") - assert not _answered_as_the_agent("Let me check my email for the order number.") - assert not _answered_as_the_agent("Let me see if I have the receipt here.") - - # And the instruction that produces the opening says which part to play. +def test_an_opening_is_checked_by_shape_only(): + """The opening is refused only for what can be checked without reading meaning into it: nothing + to say, only an end marker, or the scenario's own instruction handed back (the TH-8765 bug). + Which part the caller plays is set by the OPENING guidance, not by matching phrases.""" + from fi.alk.harness.run.conversation import OPENING, _opening_line + + instruction = "You contact support. You open by asking about your order." + assert _opening_line("", instruction) == "" + assert _opening_line(" \n\t", instruction) == "" + assert _opening_line("[DONE]", instruction) == "" + assert _opening_line("[STUCK]", instruction) == "" + assert _opening_line(instruction, instruction) == "" + assert _opening_line(" you CONTACT support. You open by\nasking about your order. ", instruction) == "" + assert _opening_line(f"Hi! {instruction}", instruction) == "" + # Anything a person could say is sent as written, in any language or script. + assert _opening_line("Hi, where is my order? [DONE]", instruction) == "Hi, where is my order?" + assert _opening_line("Let me explain, my order never came", instruction) == ( + "Let me explain, my order never came" + ) + assert _opening_line("Merhaba, siparişim gelmedi.", instruction) == "Merhaba, siparişim gelmedi." + assert _opening_line("你好,我的订单还没到。", instruction) == "你好,我的订单还没到。" + # With no instruction there is nothing to compare against, and nothing is refused for it. + assert _opening_line("Where is my order?", "") == "Where is my order?" + + # The guidance says which part to play and in which language. assert "you speak first" in OPENING - assert "not \nthe agent being contacted" in OPENING or "not the agent" in OPENING + assert "You are the caller" in OPENING and "not the agent being contacted" in OPENING + assert "in the language you speak" in OPENING -AGENTLIKE = "Sure! Let me look that up for you." +ECHOED = "You contact support. You open by asking about your order." def _open_a_conversation(monkeypatch, openings, replies=("Done.",)): @@ -6413,11 +6394,11 @@ async def agent_hears(text): [ (["Hi, where is my order?"], "Hi, where is my order?"), (["", "Hi, where is my order?"], "Hi, where is my order?"), - ([AGENTLIKE, "Hi, where is my order?"], "Hi, where is my order?"), - ([AGENTLIKE, AGENTLIKE, "Hi there."], "Hi there."), + ([ECHOED, "Hi, where is my order?"], "Hi, where is my order?"), + ([ECHOED, ECHOED, "Hi there."], "Hi there."), (["", "", "Merhaba."], "Merhaba."), (["", "", ""], "Hello."), - ([AGENTLIKE, AGENTLIKE, AGENTLIKE], "Hello."), + ([ECHOED, ECHOED, ECHOED], "Hello."), ], ) def test_an_unusable_opening_is_corrected_then_a_greeting_and_the_instruction_is_never_sent( @@ -6444,7 +6425,7 @@ def test_an_unusable_opening_is_corrected_then_a_greeting_and_the_instruction_is (["Hi, where is my order? [DONE]"], "Hi, where is my order?"), (["مرحبا، طلبي لم يصل بعد"], "مرحبا، طلبي لم يصل بعد"), (["", "", "你好"], "你好"), - ([AGENTLIKE, AGENTLIKE, "नमस्ते"], "नमस्ते"), + ([ECHOED, "[STUCK]", "नमस्ते"], "नमस्ते"), ([LONG], LONG.strip()), ( ["The website said 'how can I help you' and then crashed."], @@ -6462,7 +6443,7 @@ def test_openings_in_any_script_or_shape_reach_the_agent_as_the_person_wrote_the def test_after_a_corrected_opening_the_next_turn_is_only_the_agents_reply(monkeypatch): caller_saw, agent_heard = _open_a_conversation( monkeypatch, - [AGENTLIKE, "Hi, where is my order?", "Thanks. [DONE]"], + [ECHOED, "Hi, where is my order?", "Thanks. [DONE]"], replies=("Let me check.",), ) from fi.alk.harness.run.conversation import OPENING, _RETRY From 8436e8458087692c49588060ba85a38bda611b9a Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 26 Oct 2026 15:22:22 +0530 Subject: [PATCH 10/10] fix(harness): TH-8765 compare the opening with the instruction on whole words after normalising, neutral opening wording, behaviour tests through converse, changelog line --- CHANGELOG.md | 3 + src/fi/alk/harness/run/conversation.py | 34 ++++--- tests/test_harness.py | 117 +++++++++++++------------ 3 files changed, 90 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf49aeea0..7a33a43c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ All notable release changes for Agent Learning Kit are tracked here. Post-rc.1 increments on the release branch. Each one adds gates on top of the 66 proved at rc.1; the rc.1 entry below is historical and unchanged. +- A chat conversation never sends the scenario's instruction as the simulated person's first + message: an empty or echoed opening is asked for again, then for a greeting in their language, + and the person's own session records what the agent actually heard. - Runtime validation checks scenarios on one database lane per CPU by default, capped by the sandbox's cgroup CPU quota, instead of a fixed 16; `ALK_VALIDATION_DB_LANES` still overrides it. - Docs corpus: the full `docs/` tree (quickstarts, per-track guides, framework diff --git a/src/fi/alk/harness/run/conversation.py b/src/fi/alk/harness/run/conversation.py index 142d14972..2c999f775 100644 --- a/src/fi/alk/harness/run/conversation.py +++ b/src/fi/alk/harness/run/conversation.py @@ -13,7 +13,9 @@ import json import time +import unicodedata from dataclasses import dataclass, field +from itertools import accumulate from typing import Any, Callable from ..backends import SessionSpec @@ -125,11 +127,10 @@ def crashed(self) -> list[Call]: # system prompt about an agent, and asked to speak with nothing preceding it, will sometimes reply # as the agent instead of to it. OPENING = ( - "The conversation is starting and you speak first. You are the caller: the person making " - "contact, not the agent being contacted. What you write now are your first words to the " - "agent, in the language you speak. Say what you came to say, in your own words, and nothing " - "else. Do not offer to look anything up, do not answer on their behalf, and do not greet them " - "and wait: say the thing you actually want." + "The conversation is starting and you speak first. You are the person making contact, not " + "the agent being contacted. What you write now are your first words to the agent, in the " + "language you speak. Say what you came to say, in your own words, and nothing else. Do not " + "offer to look anything up and do not answer on the agent's behalf." ) # When an opening line is not sent, the person is told so, because their own session keeps every @@ -142,7 +143,7 @@ def crashed(self) -> list[Call]: # opens with, in the language they speak, so the request comes after the agent's reply. _RETRY = ( _UNSENT - + "You are the caller. Say what you came to say to the agent, in your own words." + + "You are the person making contact. Say what you came to say to the agent, in your own words." ) _GREET = ( _UNSENT @@ -154,10 +155,23 @@ def _opening_line(said: str, instruction: str) -> str: """The line the agent hears, or "" when it is not one: empty, only an end marker, or the scenario's own instruction handed back instead of the person's words.""" line = said.replace(DONE, "").replace(STUCK, "").strip() - written = " ".join(instruction.split()).casefold() - if written and written in " ".join(line.split()).casefold(): - return "" - return line + written, heard = ( + " ".join(unicodedata.normalize("NFKC", text).casefold().split()) + for text in (instruction, line) + ) + # The instruction must sit on whole words of the line, so a line that only shares a phrase + # with it is still sent. Written without spaces, the spaces the line added are ignored. + if " " in written: + echoed = f" {written} " in f" {heard} " + else: + words = heard.split() + edges = {0, *accumulate(map(len, words))} + joined = "".join(words) + echoed = bool(written) and any( + joined.startswith(written, at) and at + len(written) in edges + for at in edges + ) + return "" if echoed else line def customer_prompt( diff --git a/tests/test_harness.py b/tests/test_harness.py index 61263641c..5ddbc362d 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6309,40 +6309,11 @@ def __init__(self): assert blind(survived) == [] -def test_an_opening_is_checked_by_shape_only(): - """The opening is refused only for what can be checked without reading meaning into it: nothing - to say, only an end marker, or the scenario's own instruction handed back (the TH-8765 bug). - Which part the caller plays is set by the OPENING guidance, not by matching phrases.""" - from fi.alk.harness.run.conversation import OPENING, _opening_line - - instruction = "You contact support. You open by asking about your order." - assert _opening_line("", instruction) == "" - assert _opening_line(" \n\t", instruction) == "" - assert _opening_line("[DONE]", instruction) == "" - assert _opening_line("[STUCK]", instruction) == "" - assert _opening_line(instruction, instruction) == "" - assert _opening_line(" you CONTACT support. You open by\nasking about your order. ", instruction) == "" - assert _opening_line(f"Hi! {instruction}", instruction) == "" - # Anything a person could say is sent as written, in any language or script. - assert _opening_line("Hi, where is my order? [DONE]", instruction) == "Hi, where is my order?" - assert _opening_line("Let me explain, my order never came", instruction) == ( - "Let me explain, my order never came" - ) - assert _opening_line("Merhaba, siparişim gelmedi.", instruction) == "Merhaba, siparişim gelmedi." - assert _opening_line("你好,我的订单还没到。", instruction) == "你好,我的订单还没到。" - # With no instruction there is nothing to compare against, and nothing is refused for it. - assert _opening_line("Where is my order?", "") == "Where is my order?" - - # The guidance says which part to play and in which language. - assert "you speak first" in OPENING - assert "You are the caller" in OPENING and "not the agent being contacted" in OPENING - assert "in the language you speak" in OPENING - - ECHOED = "You contact support. You open by asking about your order." +RETRIED = "Hi, where is my order?" -def _open_a_conversation(monkeypatch, openings, replies=("Done.",)): +def _open_a_conversation(monkeypatch, openings, replies=("Done.",), instruction=ECHOED): """Run converse with a caller that remembers every message it was sent, like a real session.""" import asyncio from types import SimpleNamespace @@ -6381,20 +6352,59 @@ async def agent_hears(text): return reply target = SimpleNamespace(open=noop, say=agent_hears, close=noop, spent_usd=0.0) - scenario = Scenario( - name="s", - instruction="You contact support. You open by asking about your order.", - ) + scenario = Scenario(name="s", instruction=instruction) asyncio.run(conversation.converse(target, scenario, AgentContract())) return caller_saw, agent_heard +def test_the_opening_ask_says_who_speaks_first_and_in_which_language(): + """Which part the person plays is set by guidance, not by matching what they wrote.""" + from fi.alk.harness.run.conversation import OPENING + + assert "you speak first" in OPENING + assert "the person making contact, not the agent being contacted" in OPENING + assert "in the language you speak" in OPENING + + +@pytest.mark.parametrize( + "instruction, said, sent", + [ + (ECHOED, "", False), + (ECHOED, " \n\t", False), + (ECHOED, "[DONE]", False), + (ECHOED, "[STUCK]", False), + (ECHOED, ECHOED, False), + ( + ECHOED, + " you CONTACT support. You open by\nasking about your order. ", + False, + ), + (ECHOED, f"Hi! {ECHOED}", False), + ("Você quer saber do pedido.", "Voc\u0065\u0302 quer saber do pedido.", False), + ("我想取消订单", "我想 取消订单", False), + (ECHOED, "Let me explain, my order never came", True), + (ECHOED, "Merhaba, siparişim gelmedi.", True), + (ECHOED, "你好,我的订单还没到。", True), + ("Cancel my order", "Hi, I want to cancel my order.", True), + ("hi", "Is this the shop?", True), + ("", "Where is my order?", True), + ], +) +def test_an_opening_is_held_back_only_when_empty_a_marker_or_the_instruction( + monkeypatch, instruction, said, sent +): + _, agent_heard = _open_a_conversation( + monkeypatch, [said, RETRIED], replies=(), instruction=instruction + ) + assert agent_heard == [said.strip() if sent else RETRIED] + + @pytest.mark.parametrize( "openings, first", [ - (["Hi, where is my order?"], "Hi, where is my order?"), - (["", "Hi, where is my order?"], "Hi, where is my order?"), - ([ECHOED, "Hi, where is my order?"], "Hi, where is my order?"), + ([RETRIED], RETRIED), + (["", RETRIED], RETRIED), + ([ECHOED, RETRIED], RETRIED), ([ECHOED, ECHOED, "Hi there."], "Hi there."), (["", "", "Merhaba."], "Merhaba."), (["", "", ""], "Hello."), @@ -6404,14 +6414,18 @@ async def agent_hears(text): def test_an_unusable_opening_is_corrected_then_a_greeting_and_the_instruction_is_never_sent( monkeypatch, openings, first ): - from fi.alk.harness.run.conversation import OPENING, _GREET, _RETRY + from fi.alk.harness.run.conversation import OPENING caller_saw, agent_heard = _open_a_conversation(monkeypatch, openings, replies=()) assert agent_heard == [first] # Each retry tells the person their last line was not sent, and the last asks for a greeting # in their own language, so a retry is never the same question with the failure in context. - assert caller_saw == [OPENING, _RETRY, _GREET][: len(openings)] - assert "in the language you speak" in _GREET and "not sent" in _RETRY + assert caller_saw[0] == OPENING + asks = caller_saw[1 : len(openings)] + assert all("not sent" in ask for ask in asks) + assert len(set(asks)) == len(asks) + if len(asks) == 2: + assert "greeting, in the language you speak" in asks[1] LONG = "My order " + "never arrived and " * 180 + "I want a refund." @@ -6420,9 +6434,9 @@ def test_an_unusable_opening_is_corrected_then_a_greeting_and_the_instruction_is @pytest.mark.parametrize( "openings, first", [ - ([" \n\t", "Hi, where is my order?"], "Hi, where is my order?"), - (["[DONE]", "Hi, where is my order?"], "Hi, where is my order?"), - (["Hi, where is my order? [DONE]"], "Hi, where is my order?"), + ([" \n\t", RETRIED], RETRIED), + (["[DONE]", RETRIED], RETRIED), + ([RETRIED + " [DONE]"], RETRIED), (["مرحبا، طلبي لم يصل بعد"], "مرحبا، طلبي لم يصل بعد"), (["", "", "你好"], "你好"), ([ECHOED, "[STUCK]", "नमस्ते"], "नमस्ते"), @@ -6442,14 +6456,11 @@ def test_openings_in_any_script_or_shape_reach_the_agent_as_the_person_wrote_the def test_after_a_corrected_opening_the_next_turn_is_only_the_agents_reply(monkeypatch): caller_saw, agent_heard = _open_a_conversation( - monkeypatch, - [ECHOED, "Hi, where is my order?", "Thanks. [DONE]"], - replies=("Let me check.",), + monkeypatch, [ECHOED, RETRIED, "Thanks. [DONE]"], replies=("Let me check.",) ) - from fi.alk.harness.run.conversation import OPENING, _RETRY - - assert agent_heard == ["Hi, where is my order?"] - assert caller_saw == [OPENING, _RETRY, "Let me check."] + assert agent_heard == [RETRIED] + assert caller_saw[2] == "Let me check." + assert "not sent" in caller_saw[1] def test_after_an_opening_the_person_did_not_write_their_next_turn_is_told_what_was_said( @@ -6469,9 +6480,7 @@ def test_after_an_opening_the_person_did_not_write_their_next_turn_is_told_what_ def test_an_opening_the_person_wrote_reaches_them_without_a_note(monkeypatch): caller_saw, _ = _open_a_conversation( - monkeypatch, - ["Hi, where is my order?", "Thanks. [DONE]"], - replies=("Let me check.",), + monkeypatch, [RETRIED, "Thanks. [DONE]"], replies=("Let me check.",) ) assert caller_saw[-1] == "Let me check."