Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ All notable release changes for Agent Learning Kit are tracked here.
Post-rc.1 increments on the release branch. Each one adds gates on top of the
66 proved at rc.1; the rc.1 entry below is historical and unchanged.

- A chat conversation never sends the scenario's instruction as the simulated person's first
message: an empty or echoed opening is asked for again, then for a greeting in their language,
and the person's own session records what the agent actually heard.
- Runtime validation checks scenarios on one database lane per CPU by default, capped by the
sandbox's cgroup CPU quota, instead of a fixed 16; `ALK_VALIDATION_DB_LANES` still overrides it.
- Docs corpus: the full `docs/` tree (quickstarts, per-track guides, framework
Expand Down
88 changes: 56 additions & 32 deletions src/fi/alk/harness/run/conversation.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@

import json
import time
import unicodedata
from dataclasses import dataclass, field
from itertools import accumulate
from typing import Any, Callable

from ..backends import SessionSpec
Expand Down Expand Up @@ -126,32 +128,50 @@ def crashed(self) -> list[Call]:
# as the agent instead of to it.
OPENING = (
"The conversation is starting and you speak first. You are the person making contact, not "
"the agent being contacted. Say what you came to say, in your own words, and nothing else. "
"Do not offer to look anything up, do not answer on their behalf, and do not greet them and "
"wait: say the thing you actually want."
"the agent being contacted. What you write now are your first words to the agent, in the "
"language you speak. Say what you came to say, in your own words, and nothing else. Do not "
"offer to look anything up and do not answer on the agent's behalf."
)

# How an opening turn reads when the part has been swapped. Offers of help, not requests for it.
_AS_THE_AGENT = (
"let me ",
"i'll look",
"i will look",
"i'd be happy to look",
"i can help you with that",
"how can i help",
"how may i help",
"what would you like to know",
"i'll check",
"i will check",
"let me check",
"sure! let me",
# When an opening line is not sent, the person is told so, because their own session keeps every
# line they wrote: the next one has to follow from what the agent actually heard, which is nothing.
_UNSENT = (
"That line was not sent: it was empty, or it repeated the description of your situation "
"instead of saying something to the agent, so the agent has heard nothing yet. "
)
# The second try asks again for what they came to say; the last asks only for the greeting a person
# opens with, in the language they speak, so the request comes after the agent's reply.
_RETRY = (
_UNSENT
+ "You are the person making contact. Say what you came to say to the agent, in your own words."
)
_GREET = (
_UNSENT
+ "Open as a person does: say only a short greeting, in the language you speak."
)


def _answered_as_the_agent(said: str) -> bool:
"""Whether the opening line is the agent's part rather than the person's."""
opening = said.strip().lower()
return any(mark in opening for mark in _AS_THE_AGENT)
def _opening_line(said: str, instruction: str) -> str:
"""The line the agent hears, or "" when it is not one: empty, only an end marker, or the
scenario's own instruction handed back instead of the person's words."""
line = said.replace(DONE, "").replace(STUCK, "").strip()
written, heard = (
" ".join(unicodedata.normalize("NFKC", text).casefold().split())
for text in (instruction, line)
)
# The instruction must sit on whole words of the line, so a line that only shares a phrase
# with it is still sent. Written without spaces, the spaces the line added are ignored.
if " " in written:
echoed = f" {written} " in f" {heard} "
else:
words = heard.split()
edges = {0, *accumulate(map(len, words))}
joined = "".join(words)
echoed = bool(written) and any(
joined.startswith(written, at) and at + len(written) in edges
for at in edges
)
return "" if echoed else line


def customer_prompt(
Expand Down Expand Up @@ -231,16 +251,16 @@ def record(speaker: str, text: str) -> None:
# The customer opens, in its own words. The scenario's instruction is written *about*
# the caller ("orders two burgers and asks for..."), so speaking it verbatim would hand
# the agent a stage direction instead of a person.
opening = await customer.say(OPENING)
said = opening.text.strip() or scenario.instruction
if _answered_as_the_agent(said):
# The opening turn is the one with no conversation behind it, and a model asked to
# speak into that gap will sometimes take the other part: it offers to look something
# up, the agent replies that no question was asked, and the run fails for a reason
# that has nothing to do with the agent. The instruction is the fallback, because a
# blunt version of the right question tests more than a fluent version of the wrong
# one.
said = scenario.instruction
said = _opening_line((await customer.say(OPENING)).text, scenario.instruction)
for again in (_RETRY, _GREET):
if said:
break
said = _opening_line((await customer.say(again)).text, scenario.instruction)
# Only when every ask came back unusable does the run open with a line the person did not
# write; they are told it with the agent's reply, so their session matches what was said.
opened_for_them = not said
if opened_for_them:
said = "Hello."
record("customer", said)
for _turn in range(max(1, scenario.max_turns)):
try:
Expand All @@ -252,7 +272,11 @@ def record(speaker: str, text: str) -> None:
break
record("agent", reply or "(said nothing)")

turn = await customer.say(reply or "(no response)")
heard = reply or "(no response)"
if opened_for_them:
heard = f"You opened with: {said}\nThe agent replied: {heard}"
opened_for_them = False
turn = await customer.say(heard)
said = turn.text.strip()
if DONE in said or STUCK in said:
transcript.ended = GAVE_UP if STUCK in said else FINISHED
Expand Down
2 changes: 2 additions & 0 deletions src/fi/alk/harness/skills/write-scenarios/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,8 @@ the checks and the judge, so everything else stays out of their reach:
own words, even when the caller asked for that very thing (the caller fixes what they want, not the
words the agent records it under), and the same for a place or description the caller gave and
the agent writes down (the identifier the world resolves it to is fixed, from the record);
- free text the agent composes to find something, such as a search query: many wordings find the
same thing, so check what it found (what came back, or what it acted on), not the words it used;
- a value the agent's code fills in elsewhere, which does not fix what the agent's model passes;
- an optional argument nothing fixes: leave it out, and the tool's default applies.

Expand Down
187 changes: 166 additions & 21 deletions tests/test_harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -6309,35 +6309,180 @@ def __init__(self):
assert blind(survived) == []


def test_an_opening_line_in_the_agents_voice_falls_back_to_the_instruction():
"""The opening turn is the one with no conversation behind it, and a model asked to speak into
that gap sometimes takes the other part.
ECHOED = "You contact support. You open by asking about your order."
RETRIED = "Hi, where is my order?"

Both of these are real opening lines from a run: the agent then replied that no question had
been asked, and the scenario failed for a reason that had nothing to do with the agent.
"""
from fi.alk.harness.run.conversation import OPENING, _answered_as_the_agent

assert _answered_as_the_agent(
"Sure! Let me find the database and make that update for you."
def _open_a_conversation(monkeypatch, openings, replies=("Done.",), instruction=ECHOED):
"""Run converse with a caller that remembers every message it was sent, like a real session."""
import asyncio
from types import SimpleNamespace

from fi.alk.harness.contract import AgentContract
from fi.alk.harness.run import conversation
from fi.alk.harness.scenario import Scenario

openings = iter(openings)
caller_saw = []
agent_heard = []
replies = iter(replies)

async def noop(*_args):
return None

async def caller_says(prompt):
caller_saw.append(prompt)
text = next(openings, None)
return SimpleNamespace(text=text if text is not None else "Thanks, bye. [DONE]")

customer = SimpleNamespace(
__aenter__=noop,
__aexit__=noop,
say=caller_says,
spent_usd=0.0,
simulator_tokens=(0, 0),
)
assert _answered_as_the_agent(
"I'd be happy to look that up! Let me check the database."
monkeypatch.setattr(conversation, "Stage", lambda *_a, **_k: customer)

async def agent_hears(text):
agent_heard.append(text)
reply = next(replies, None)
if reply is None:
raise conversation.TargetConversationEnded("Done.")
return reply

target = SimpleNamespace(open=noop, say=agent_hears, close=noop, spent_usd=0.0)
scenario = Scenario(name="s", instruction=instruction)
asyncio.run(conversation.converse(target, scenario, AgentContract()))
return caller_saw, agent_heard


def test_the_opening_ask_says_who_speaks_first_and_in_which_language():
"""Which part the person plays is set by guidance, not by matching what they wrote."""
from fi.alk.harness.run.conversation import OPENING

assert "you speak first" in OPENING
assert "the person making contact, not the agent being contacted" in OPENING
assert "in the language you speak" in OPENING


@pytest.mark.parametrize(
"instruction, said, sent",
[
(ECHOED, "", False),
(ECHOED, " \n\t", False),
(ECHOED, "[DONE]", False),
(ECHOED, "[STUCK]", False),
(ECHOED, ECHOED, False),
(
ECHOED,
" you CONTACT support. You open by\nasking about your order. ",
False,
),
(ECHOED, f"Hi! {ECHOED}", False),
("Você quer saber do pedido.", "Voc\u0065\u0302 quer saber do pedido.", False),
("我想取消订单", "我想 取消订单", False),
(ECHOED, "Let me explain, my order never came", True),
(ECHOED, "Merhaba, siparişim gelmedi.", True),
(ECHOED, "你好,我的订单还没到。", True),
("Cancel my order", "Hi, I want to cancel my order.", True),
("hi", "Is this the shop?", True),
("", "Where is my order?", True),
],
)
def test_an_opening_is_held_back_only_when_empty_a_marker_or_the_instruction(
monkeypatch, instruction, said, sent
):
_, agent_heard = _open_a_conversation(
monkeypatch, [said, RETRIED], replies=(), instruction=instruction
)
assert _answered_as_the_agent("What would you like to know about the database?")
assert agent_heard == [said.strip() if sent else RETRIED]


# What a person actually says, which must survive untouched.
assert not _answered_as_the_agent("How many of your customers are based in Canada?")
assert not _answered_as_the_agent(
"I want all track prices changed to 0.99 for the promotion, can you update them?"
@pytest.mark.parametrize(
"openings, first",
[
([RETRIED], RETRIED),
(["", RETRIED], RETRIED),
([ECHOED, RETRIED], RETRIED),
([ECHOED, ECHOED, "Hi there."], "Hi there."),
(["", "", "Merhaba."], "Merhaba."),
(["", "", ""], "Hello."),
([ECHOED, ECHOED, ECHOED], "Hello."),
],
)
def test_an_unusable_opening_is_corrected_then_a_greeting_and_the_instruction_is_never_sent(
monkeypatch, openings, first
):
from fi.alk.harness.run.conversation import OPENING

caller_saw, agent_heard = _open_a_conversation(monkeypatch, openings, replies=())
assert agent_heard == [first]
# Each retry tells the person their last line was not sent, and the last asks for a greeting
# in their own language, so a retry is never the same question with the failure in context.
assert caller_saw[0] == OPENING
asks = caller_saw[1 : len(openings)]
assert all("not sent" in ask for ask in asks)
assert len(set(asks)) == len(asks)
if len(asks) == 2:
assert "greeting, in the language you speak" in asks[1]


LONG = "My order " + "never arrived and " * 180 + "I want a refund."


@pytest.mark.parametrize(
"openings, first",
[
([" \n\t", RETRIED], RETRIED),
(["[DONE]", RETRIED], RETRIED),
([RETRIED + " [DONE]"], RETRIED),
(["مرحبا، طلبي لم يصل بعد"], "مرحبا، طلبي لم يصل بعد"),
(["", "", "你好"], "你好"),
([ECHOED, "[STUCK]", "नमस्ते"], "नमस्ते"),
([LONG], LONG.strip()),
(
["The website said 'how can I help you' and then crashed."],
"The website said 'how can I help you' and then crashed.",
),
],
)
def test_openings_in_any_script_or_shape_reach_the_agent_as_the_person_wrote_them(
monkeypatch, openings, first
):
_, agent_heard = _open_a_conversation(monkeypatch, openings, replies=())
assert agent_heard == [first]


def test_after_a_corrected_opening_the_next_turn_is_only_the_agents_reply(monkeypatch):
caller_saw, agent_heard = _open_a_conversation(
monkeypatch, [ECHOED, RETRIED, "Thanks. [DONE]"], replies=("Let me check.",)
)
assert not _answered_as_the_agent(
"Which genre has the highest average listener rating?"
assert agent_heard == [RETRIED]
assert caller_saw[2] == "Let me check."
assert "not sent" in caller_saw[1]


def test_after_an_opening_the_person_did_not_write_their_next_turn_is_told_what_was_said(
monkeypatch,
):
caller_saw, agent_heard = _open_a_conversation(
monkeypatch,
["", "", "", "My order never came."],
replies=("Hi, how can I help?",),
)
assert agent_heard == ["Hello.", "My order never came."]
assert (
caller_saw[-1]
== "You opened with: Hello.\nThe agent replied: Hi, how can I help?"
)

# And the instruction that produces the opening says which part to play.
assert "you speak first" in OPENING
assert "not \nthe agent being contacted" in OPENING or "not the agent" in OPENING

def test_an_opening_the_person_wrote_reaches_them_without_a_note(monkeypatch):
caller_saw, _ = _open_a_conversation(
monkeypatch, [RETRIED, "Thanks. [DONE]"], replies=("Let me check.",)
)
assert caller_saw[-1] == "Let me check."


def test_a_refusal_scenario_is_not_vacuous_because_its_evidence_is_what_was_said():
Expand Down
Loading