Skip to content
Closed
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,13 @@ Post-rc.1 increments on the release branch. Each one adds gates on top of the
`on_tool` trigger waits for the action in full. An actor waiting for a record only it may write is refused when the
scenario is written. A timeout another actor answers reads `answered_by` only once that actor has acted or spoken,
else `limit_blocked` or `unresolved`.
- World changes on every trial with a door, actors or not: every new world is seeded with the full action log, and
a trial whose tools run where no door stands seals `{"recorded": false, "why": "in_process_tools"}`. The runner
snapshots the world's tables at a trial's start and end (read-only, by primary key, `_alk_*` never compared,
tables over 5,000 rows or without a key named in `skipped`, at most 100 changes listed) and seals `records`
and `start` (`seeded`, `differs` with the tables, or `unknown`). A check that reads an unreadable action log
(`EvidenceUnavailable`) is unmeasured, never settled on an empty log; the scenario errors as
`evidence_unmeasured`.
- Two more fault modes: `unavailable` (the world never sees the call; the agent hears 503 at once) and `error`
(the scenario's `status` 400-599 and `message`). The door answers the agent's keyed retry (`Idempotency-Key`)
from its first result, as it does an actor's; a fault before commit releases the key. A write the world commits
Expand Down
32 changes: 5 additions & 27 deletions src/fi/alk/harness/bundle_author_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,13 +139,7 @@ def __post_init__(self) -> None:

_IGNORED_ARTIFACT_PARTS = {".git", ".venv", "__pycache__", "node_modules"}
_CERTIFIED_BUNDLE_DIRECTORY = "certified-bundle"
# The agent's tool trace, as every world without actors is seeded.
_AGENT_TRACE_DDL = (
"CREATE TABLE IF NOT EXISTS _alk_tool_trace ("
"id bigserial PRIMARY KEY, name text NOT NULL, arguments jsonb NOT NULL, "
"result jsonb, ok boolean NOT NULL, error text, at double precision NOT NULL);\n"
)
# With actors, the world's action log: every action the tool proxy saw, who took it and what came of it.
# The world's action log: every action the door saw, who took it and what came of it.
_TOOL_TRACE_DDL = (
"CREATE TABLE IF NOT EXISTS _alk_tool_trace ("
"id bigserial PRIMARY KEY, name text NOT NULL, arguments jsonb NOT NULL, "
Expand All @@ -162,27 +156,11 @@ def __post_init__(self) -> None:
)


def _sets_trial_state(authoring: Path) -> bool:
"""Whether any scenario has a pre-run session or sets faults: both need the full action log."""
for document in Path(authoring).glob("scenarios/*/scenario.json"):
try:
body = json.loads(document.read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
if isinstance(body, dict) and (body.get("faults") or body.get("pre_run") or body.get("history")):
return True
return False


def _action_log_seed(authoring: Path) -> str:
"""The action log and who may do what in the world; a world without actors, pre-run sessions or faults is seeded exactly
as before."""
listing = load_actors(authoring)
return (
_TOOL_TRACE_DDL + actors_seed(listing)
if listing.actors or _sets_trial_state(authoring)
else _AGENT_TRACE_DDL
)
"""The world's full action log, and who may do what in it: every world records who did what and what came of it,
so World changes can be shown on every trial, actors or not. A world with no actors seeds no actor rows. A bundle
certified before this keeps the table it was seeded with; the door writes either shape."""
return _TOOL_TRACE_DDL + actors_seed(load_actors(authoring))


def _catalog_image(image: str) -> tuple[str, str]:
Expand Down
107 changes: 87 additions & 20 deletions src/fi/alk/harness/call_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import os
import stat
import tempfile
import threading
from dataclasses import dataclass, field, replace
from datetime import datetime, timezone
from pathlib import Path
Expand Down Expand Up @@ -1189,15 +1190,17 @@ def approvals(log: list[dict[str, Any]], decisions: Sequence[dict[str, Any]] = (

def _state_changes(
runtime: EnvironmentRuntime, bundle_dir: Path, decisions: Sequence[dict[str, Any]] = ()
) -> list[dict[str, Any]]:
) -> list[dict[str, Any]] | None:
"""What each participant attempted and what came of it; None when the action log could not be read, which the
trial seals as unreadable rather than as no change."""
database = _find_postgres_endpoint(runtime)
if database is None:
return []
return None
try:
log = _read_action_log(database.address)
except Exception as exc: # noqa: BLE001 - no state changes, never a failed call
logger.debug("state changes unreadable: %s", type(exc).__name__)
return []
return None
return trial_state_changes(log, load_actors(bundle_dir).by_id(), decisions)


Expand Down Expand Up @@ -2187,6 +2190,54 @@ def _materialize_vertex_adc(
# --- the runner ----------------------------------------------------------------------------


class TrialRecords:
"""The world's records at each trial's start, and the run's seeded start: what World changes compares a trial's
end against, and whether it started clean. Snapshots are read on a connection that cannot write; one that cannot
be read is sealed as unreadable, never a failed trial."""

def __init__(self) -> None:
self.seeded: dict[str, str] | None = None
self._seeded_taken = False
self._starts: dict[Any, tuple[Any, dict[str, Any]]] = {}
self._lock = threading.Lock()

def start(self, key: Any, dsn: str | None) -> None:
"""Read the world's records at a trial's start (after its reset and pre-run sessions). The first trial's start
is the run's seed."""
from .world_snapshot import digest, snapshot, start_state

try:
taken = snapshot(dsn) if dsn else None
except Exception as exc: # noqa: BLE001 - only the type: a database error can carry the world's password
logger.debug("trial start records unreadable: %s", type(exc).__name__)
taken = None
start = digest(taken) if taken is not None else None
with self._lock:
if not self._seeded_taken:
self._seeded_taken = True
self.seeded = start
self._starts[key] = (taken, start_state(self.seeded, start))

def end(self, key: Any, dsn: str | None) -> dict[str, Any]:
"""The trial's records before and after (`records`) and whether it started from the seed (`start`)."""
from .world_snapshot import diff, snapshot

with self._lock:
before, start = self._starts.pop(key, (None, {"state": "unknown", "tables": []}))
if before is None or not dsn:
return {"records": {"unreadable": True}, "start": start}
try:
after = snapshot(dsn)
except Exception as exc: # noqa: BLE001 - only the type: a database error can carry the world's password
logger.debug("trial end records unreadable: %s", type(exc).__name__)
return {"records": {"unreadable": True}, "start": start}
return {"records": diff(before, after), "start": start}


# Sealed instead of World changes when the agent's tools run where the world's door cannot see them.
NOT_RECORDED = {"recorded": False, "why": "in_process_tools"}


class CallRunnerImpl:
"""Satisfies `hosted_scheduler.CallRunner`. See the module docstring for the three
sub-systems this class implements."""
Expand All @@ -2202,6 +2253,7 @@ def __init__(
self._adapter = adapter
self._context = context
self._place_call = place_call
self._records = TrialRecords()
simulator_secret_values = _canonical_simulator_secrets(
context.simulator_provider_secret_values
)
Expand Down Expand Up @@ -2654,11 +2706,17 @@ async def place_with_actors() -> SimulationReport:
with contextlib.suppress(Exception):
_write_trial_state(runtime, [])

world_database = _find_postgres_endpoint(runtime)
records_dsn = world_database.address if world_database is not None else None

async def place_with_world_actors() -> SimulationReport:
if actors is None:
# The trial's starting records, after its reset: what World changes compares its end against.
await asyncio.to_thread(self._records.start, id(runtime), records_dsn)
return await place()
# What the actors wrote before T0 is in the world before the call starts.
await actors.replay()
await asyncio.to_thread(self._records.start, id(runtime), records_dsn)
acting = asyncio.create_task(actors.run())
try:
return await place()
Expand Down Expand Up @@ -2810,22 +2868,26 @@ async def _translate_report(
# from transcript prose.
if not calls and case is not None:
calls = _collect_provider_tool_calls(case)
# The world's door saw the agent's tools: a tool proxy in front of its tools service, or the door in its own
# process. Only then can the trial say what changed in the world.
door_seen = (
self._context.evidence_seam is EvidenceSeam.TOOL_TRACE or runtime.endpoints.get("world_door") is not None
)
acted = (
_actor_rows(runtime, self._context.bundle_dir, actor_decisions)
if case is not None
and self._context.evidence_seam is EvidenceSeam.TOOL_TRACE
if case is not None and door_seen
else []
)
changes = (
_state_changes(runtime, self._context.bundle_dir, actor_decisions)
if acted or actor_decisions
else []
)
duplicates = (
_duplicates(runtime)
if case is not None and self._context.evidence_seam is EvidenceSeam.TOOL_TRACE
else []
changes = _state_changes(runtime, self._context.bundle_dir, actor_decisions) if case is not None and door_seen else []
world_database = _find_postgres_endpoint(runtime)
records = (
await asyncio.to_thread(
self._records.end, id(runtime), world_database.address if world_database is not None else None
)
if case is not None and door_seen
else {}
)
duplicates = _duplicates(runtime) if case is not None and door_seen else []
# Hand-off records are not the world's actions: marks pair the graded calls with the action log.
graded = graded_calls(calls)
marks = _agent_marks(runtime, self._context.bundle_dir, graded) if acted else []
Expand All @@ -2846,7 +2908,7 @@ async def _translate_report(
recording = combined if combined and Path(combined).is_file() else None
notices = self._fault_notices(runtime, faults, calls)
# A call that ended with no words still says what did not take effect, and what actors did in the world,
# never only logging it.
# never only logging it; a call with words carries World changes, or why they were not recorded.
if result.transcript or notices or acted or actor_decisions or duplicates or took_part.get("call_actors"):
transcript_document = {
"schema_version": "futureagi.call-transcript.v1",
Expand All @@ -2865,18 +2927,23 @@ async def _translate_report(
"summary": summary,
"events": took_part.get("call_actor_events") or [],
}
if acted or actor_decisions:
# Kept out of the tool trace, whose every line is one of the agent's own calls. `decisions` is
# what each actor chose and why (live) or was scheduled to do (mocked); `actions` what the world
# made of it.
if door_seen:
# Every trial the door saw, actors or not: World changes is shown on each. Kept out of the tool
# trace, whose every line is one of the agent's own calls. `decisions` is what each actor chose
# and why (live) or was scheduled to do (mocked); `actions` what the world made of it.
transcript_document["world_actors"] = {
"actions": acted,
"decisions": list(actor_decisions),
"state_changes": changes,
"state_changes": changes or [],
"approvals": self._approvals(runtime, actor_decisions),
"limits": dict(limits or {}),
"usage": list(actor_usage),
**records,
**({"unreadable": True} if changes is None else {}),
}
else:
# The agent's tools run where the world's door cannot see them: said, never shown as no change.
transcript_document["world_actors"] = dict(NOT_RECORDED)
if duplicates:
# The world applied one of the agent's writes twice after its answer was lost: the scenario fails.
transcript_document.setdefault("world_actors", {})["duplicates"] = duplicates
Expand Down
28 changes: 23 additions & 5 deletions src/fi/alk/harness/chat_call_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -317,10 +317,14 @@ def _trial_log(door: Any, since: float = 0.0) -> list[dict[str, Any]]:
def _world_actors_record(
door: Any, listing: Any, dispatcher: Any, cast: Any, meter: Any, limits: dict[str, Any], since: float = 0.0
) -> dict[str, Any] | None:
"""What actors did in the world this trial and what it changed, sealed beside the transcript as a call seals it;
None when no actor took part."""
"""What actors did in the world this trial and what each participant changed, sealed beside the transcript as a
call seals it, on every trial with a door, actors or not (`actions` and `decisions` then empty); None only when
there is no door."""
from .call_runner import actor_rows, approvals, trial_state_changes

if door is None:
return None

listed = listing.by_id()
log = _trial_log(door, since)
decisions = dispatcher.record() if dispatcher is not None else []
Expand All @@ -343,8 +347,6 @@ def _world_actors_record(
if one.get("kind") == "timeout"
]
acted = actor_rows(log, listed, decisions)
if not acted and not decisions:
return None
return {
"actions": acted,
"decisions": decisions,
Expand Down Expand Up @@ -757,8 +759,11 @@ class HostedChatCallRunner:
"""Drive a repository chat ingress inside its leased world."""

def __init__(self, adapter: ArtifactUploader, context: CallRunnerContext) -> None:
from .call_runner import TrialRecords

self._adapter = adapter
self._context = context
self._records = TrialRecords()
contract_path = context.bundle_dir / "contract.json"
if not contract_path.is_file():
self._contract: AgentContract | None = None
Expand Down Expand Up @@ -860,6 +865,10 @@ async def run(
job_id = getattr(self._context.job, "job_id", None) or "probe"
trial = f"{job_id}:{scenario.scenario_key}"
trial_started = time.time()
records_dsn = world_db.address if world_db is not None and world_db.protocol == "postgres" else None
# The door sees the agent's tools when it runs them itself (tools the agent asks for over its interface), or
# when a tool proxy or the in-process door stands in front of the agent's own.
door_sees_tools = proxied or bool(interface is not None and getattr(interface, "include_tools", False))
dispatcher = build_dispatcher(
# What actors watch (an on_tool trigger) includes the agent's calls through its tool proxy.
document, self._context.bundle_dir, act=acts, read_log=lambda: _trial_log(door, trial_started),
Expand Down Expand Up @@ -913,6 +922,9 @@ async def run(
if dispatcher is not None:
# What the actors wrote before T0 is in the world before the conversation starts.
await dispatcher.replay()
# The trial's starting records, after its reset and pre-run sessions: what World changes compares against.
await asyncio.to_thread(self._records.start, id(runtime), records_dsn)
if dispatcher is not None:
acting = asyncio.create_task(dispatcher.run())
transcript = await _drive_conversation(
_HostedChatTarget(
Expand Down Expand Up @@ -993,7 +1005,13 @@ async def run(
transcript.calls[:] = _with_span_calls(transcript.calls, runtime)

world_record = _world_actors_record(door, listing, dispatcher, cast, meter, limits, since=trial_started)
from .call_runner import duplicate_commits
from .call_runner import NOT_RECORDED, duplicate_commits

if door_sees_tools:
world_record = {**(world_record or {}), **await asyncio.to_thread(self._records.end, id(runtime), records_dsn)}
else:
# The agent runs its own tools where no door stands: World changes says so, never "no change".
world_record = dict(NOT_RECORDED)

# The world applied one of the agent's writes twice after its answer was lost: the scenario fails.
duplicates = duplicate_commits(_trial_log(door, trial_started))
Expand Down
Loading
Loading