diff --git a/CHANGELOG.md b/CHANGELOG.md index 490215ff5..0380fecb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,13 @@ Post-rc.1 increments on the release branch. Each one adds gates on top of the `on_tool` trigger waits for the action in full. An actor waiting for a record only it may write is refused when the scenario is written. A timeout another actor answers reads `answered_by` only once that actor has acted or spoken, else `limit_blocked` or `unresolved`. +- World changes on every trial with a door, actors or not: every new world is seeded with the full action log, and + a trial whose tools run where no door stands seals `{"recorded": false, "why": "in_process_tools"}`. The runner + snapshots the world's tables at a trial's start and end (read-only, by primary key, `_alk_*` never compared, + tables over 5,000 rows or without a key named in `skipped`, at most 100 changes listed) and seals `records` + and `start` (`seeded`, `differs` with the tables, or `unknown`). A check that reads an unreadable action log + (`EvidenceUnavailable`) is unmeasured, never settled on an empty log; the scenario errors as + `evidence_unmeasured`. - Two more fault modes: `unavailable` (the world never sees the call; the agent hears 503 at once) and `error` (the scenario's `status` 400-599 and `message`). The door answers the agent's keyed retry (`Idempotency-Key`) from its first result, as it does an actor's; a fault before commit releases the key. A write the world commits diff --git a/src/fi/alk/harness/bundle_author_v2.py b/src/fi/alk/harness/bundle_author_v2.py index a1cd83782..35ba9145c 100644 --- a/src/fi/alk/harness/bundle_author_v2.py +++ b/src/fi/alk/harness/bundle_author_v2.py @@ -139,13 +139,7 @@ def __post_init__(self) -> None: _IGNORED_ARTIFACT_PARTS = {".git", ".venv", "__pycache__", "node_modules"} _CERTIFIED_BUNDLE_DIRECTORY = "certified-bundle" -# The agent's tool trace, as every world without actors is seeded. -_AGENT_TRACE_DDL = ( - "CREATE TABLE IF NOT EXISTS _alk_tool_trace (" - "id bigserial PRIMARY KEY, name text NOT NULL, arguments jsonb NOT NULL, " - "result jsonb, ok boolean NOT NULL, error text, at double precision NOT NULL);\n" -) -# With actors, the world's action log: every action the tool proxy saw, who took it and what came of it. +# The world's action log: every action the door saw, who took it and what came of it. _TOOL_TRACE_DDL = ( "CREATE TABLE IF NOT EXISTS _alk_tool_trace (" "id bigserial PRIMARY KEY, name text NOT NULL, arguments jsonb NOT NULL, " @@ -162,27 +156,11 @@ def __post_init__(self) -> None: ) -def _sets_trial_state(authoring: Path) -> bool: - """Whether any scenario has a pre-run session or sets faults: both need the full action log.""" - for document in Path(authoring).glob("scenarios/*/scenario.json"): - try: - body = json.loads(document.read_text(encoding="utf-8")) - except (OSError, ValueError): - continue - if isinstance(body, dict) and (body.get("faults") or body.get("pre_run") or body.get("history")): - return True - return False - - def _action_log_seed(authoring: Path) -> str: - """The action log and who may do what in the world; a world without actors, pre-run sessions or faults is seeded exactly - as before.""" - listing = load_actors(authoring) - return ( - _TOOL_TRACE_DDL + actors_seed(listing) - if listing.actors or _sets_trial_state(authoring) - else _AGENT_TRACE_DDL - ) + """The world's full action log, and who may do what in it: every world records who did what and what came of it, + so World changes can be shown on every trial, actors or not. A world with no actors seeds no actor rows. A bundle + certified before this keeps the table it was seeded with; the door writes either shape.""" + return _TOOL_TRACE_DDL + actors_seed(load_actors(authoring)) def _catalog_image(image: str) -> tuple[str, str]: diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py index ab23127dd..46e6ad1c4 100644 --- a/src/fi/alk/harness/call_runner.py +++ b/src/fi/alk/harness/call_runner.py @@ -40,6 +40,7 @@ import os import stat import tempfile +import threading from dataclasses import dataclass, field, replace from datetime import datetime, timezone from pathlib import Path @@ -1189,15 +1190,17 @@ def approvals(log: list[dict[str, Any]], decisions: Sequence[dict[str, Any]] = ( def _state_changes( runtime: EnvironmentRuntime, bundle_dir: Path, decisions: Sequence[dict[str, Any]] = () -) -> list[dict[str, Any]]: +) -> list[dict[str, Any]] | None: + """What each participant attempted and what came of it; None when the action log could not be read, which the + trial seals as unreadable rather than as no change.""" database = _find_postgres_endpoint(runtime) if database is None: - return [] + return None try: log = _read_action_log(database.address) except Exception as exc: # noqa: BLE001 - no state changes, never a failed call logger.debug("state changes unreadable: %s", type(exc).__name__) - return [] + return None return trial_state_changes(log, load_actors(bundle_dir).by_id(), decisions) @@ -2187,6 +2190,54 @@ def _materialize_vertex_adc( # --- the runner ---------------------------------------------------------------------------- +class TrialRecords: + """The world's records at each trial's start, and the run's seeded start: what World changes compares a trial's + end against, and whether it started clean. Snapshots are read on a connection that cannot write; one that cannot + be read is sealed as unreadable, never a failed trial.""" + + def __init__(self) -> None: + self.seeded: dict[str, str] | None = None + self._seeded_taken = False + self._starts: dict[Any, tuple[Any, dict[str, Any]]] = {} + self._lock = threading.Lock() + + def start(self, key: Any, dsn: str | None) -> None: + """Read the world's records at a trial's start (after its reset and pre-run sessions). The first trial's start + is the run's seed.""" + from .world_snapshot import digest, snapshot, start_state + + try: + taken = snapshot(dsn) if dsn else None + except Exception as exc: # noqa: BLE001 - only the type: a database error can carry the world's password + logger.debug("trial start records unreadable: %s", type(exc).__name__) + taken = None + start = digest(taken) if taken is not None else None + with self._lock: + if not self._seeded_taken: + self._seeded_taken = True + self.seeded = start + self._starts[key] = (taken, start_state(self.seeded, start)) + + def end(self, key: Any, dsn: str | None) -> dict[str, Any]: + """The trial's records before and after (`records`) and whether it started from the seed (`start`).""" + from .world_snapshot import diff, snapshot + + with self._lock: + before, start = self._starts.pop(key, (None, {"state": "unknown", "tables": []})) + if before is None or not dsn: + return {"records": {"unreadable": True}, "start": start} + try: + after = snapshot(dsn) + except Exception as exc: # noqa: BLE001 - only the type: a database error can carry the world's password + logger.debug("trial end records unreadable: %s", type(exc).__name__) + return {"records": {"unreadable": True}, "start": start} + return {"records": diff(before, after), "start": start} + + +# Sealed instead of World changes when the agent's tools run where the world's door cannot see them. +NOT_RECORDED = {"recorded": False, "why": "in_process_tools"} + + class CallRunnerImpl: """Satisfies `hosted_scheduler.CallRunner`. See the module docstring for the three sub-systems this class implements.""" @@ -2202,6 +2253,7 @@ def __init__( self._adapter = adapter self._context = context self._place_call = place_call + self._records = TrialRecords() simulator_secret_values = _canonical_simulator_secrets( context.simulator_provider_secret_values ) @@ -2654,11 +2706,17 @@ async def place_with_actors() -> SimulationReport: with contextlib.suppress(Exception): _write_trial_state(runtime, []) + world_database = _find_postgres_endpoint(runtime) + records_dsn = world_database.address if world_database is not None else None + async def place_with_world_actors() -> SimulationReport: if actors is None: + # The trial's starting records, after its reset: what World changes compares its end against. + await asyncio.to_thread(self._records.start, id(runtime), records_dsn) return await place() # What the actors wrote before T0 is in the world before the call starts. await actors.replay() + await asyncio.to_thread(self._records.start, id(runtime), records_dsn) acting = asyncio.create_task(actors.run()) try: return await place() @@ -2810,22 +2868,26 @@ async def _translate_report( # from transcript prose. if not calls and case is not None: calls = _collect_provider_tool_calls(case) + # The world's door saw the agent's tools: a tool proxy in front of its tools service, or the door in its own + # process. Only then can the trial say what changed in the world. + door_seen = ( + self._context.evidence_seam is EvidenceSeam.TOOL_TRACE or runtime.endpoints.get("world_door") is not None + ) acted = ( _actor_rows(runtime, self._context.bundle_dir, actor_decisions) - if case is not None - and self._context.evidence_seam is EvidenceSeam.TOOL_TRACE + if case is not None and door_seen else [] ) - changes = ( - _state_changes(runtime, self._context.bundle_dir, actor_decisions) - if acted or actor_decisions - else [] - ) - duplicates = ( - _duplicates(runtime) - if case is not None and self._context.evidence_seam is EvidenceSeam.TOOL_TRACE - else [] + changes = _state_changes(runtime, self._context.bundle_dir, actor_decisions) if case is not None and door_seen else [] + world_database = _find_postgres_endpoint(runtime) + records = ( + await asyncio.to_thread( + self._records.end, id(runtime), world_database.address if world_database is not None else None + ) + if case is not None and door_seen + else {} ) + duplicates = _duplicates(runtime) if case is not None and door_seen else [] # Hand-off records are not the world's actions: marks pair the graded calls with the action log. graded = graded_calls(calls) marks = _agent_marks(runtime, self._context.bundle_dir, graded) if acted else [] @@ -2846,7 +2908,7 @@ async def _translate_report( recording = combined if combined and Path(combined).is_file() else None notices = self._fault_notices(runtime, faults, calls) # A call that ended with no words still says what did not take effect, and what actors did in the world, - # never only logging it. + # never only logging it; a call with words carries World changes, or why they were not recorded. if result.transcript or notices or acted or actor_decisions or duplicates or took_part.get("call_actors"): transcript_document = { "schema_version": "futureagi.call-transcript.v1", @@ -2865,18 +2927,23 @@ async def _translate_report( "summary": summary, "events": took_part.get("call_actor_events") or [], } - if acted or actor_decisions: - # Kept out of the tool trace, whose every line is one of the agent's own calls. `decisions` is - # what each actor chose and why (live) or was scheduled to do (mocked); `actions` what the world - # made of it. + if door_seen: + # Every trial the door saw, actors or not: World changes is shown on each. Kept out of the tool + # trace, whose every line is one of the agent's own calls. `decisions` is what each actor chose + # and why (live) or was scheduled to do (mocked); `actions` what the world made of it. transcript_document["world_actors"] = { "actions": acted, "decisions": list(actor_decisions), - "state_changes": changes, + "state_changes": changes or [], "approvals": self._approvals(runtime, actor_decisions), "limits": dict(limits or {}), "usage": list(actor_usage), + **records, + **({"unreadable": True} if changes is None else {}), } + else: + # The agent's tools run where the world's door cannot see them: said, never shown as no change. + transcript_document["world_actors"] = dict(NOT_RECORDED) if duplicates: # The world applied one of the agent's writes twice after its answer was lost: the scenario fails. transcript_document.setdefault("world_actors", {})["duplicates"] = duplicates diff --git a/src/fi/alk/harness/chat_call_runner.py b/src/fi/alk/harness/chat_call_runner.py index f4dec7208..cc23fea00 100644 --- a/src/fi/alk/harness/chat_call_runner.py +++ b/src/fi/alk/harness/chat_call_runner.py @@ -317,10 +317,14 @@ def _trial_log(door: Any, since: float = 0.0) -> list[dict[str, Any]]: def _world_actors_record( door: Any, listing: Any, dispatcher: Any, cast: Any, meter: Any, limits: dict[str, Any], since: float = 0.0 ) -> dict[str, Any] | None: - """What actors did in the world this trial and what it changed, sealed beside the transcript as a call seals it; - None when no actor took part.""" + """What actors did in the world this trial and what each participant changed, sealed beside the transcript as a + call seals it, on every trial with a door, actors or not (`actions` and `decisions` then empty); None only when + there is no door.""" from .call_runner import actor_rows, approvals, trial_state_changes + if door is None: + return None + listed = listing.by_id() log = _trial_log(door, since) decisions = dispatcher.record() if dispatcher is not None else [] @@ -343,8 +347,6 @@ def _world_actors_record( if one.get("kind") == "timeout" ] acted = actor_rows(log, listed, decisions) - if not acted and not decisions: - return None return { "actions": acted, "decisions": decisions, @@ -757,8 +759,11 @@ class HostedChatCallRunner: """Drive a repository chat ingress inside its leased world.""" def __init__(self, adapter: ArtifactUploader, context: CallRunnerContext) -> None: + from .call_runner import TrialRecords + self._adapter = adapter self._context = context + self._records = TrialRecords() contract_path = context.bundle_dir / "contract.json" if not contract_path.is_file(): self._contract: AgentContract | None = None @@ -860,6 +865,10 @@ async def run( job_id = getattr(self._context.job, "job_id", None) or "probe" trial = f"{job_id}:{scenario.scenario_key}" trial_started = time.time() + records_dsn = world_db.address if world_db is not None and world_db.protocol == "postgres" else None + # The door sees the agent's tools when it runs them itself (tools the agent asks for over its interface), or + # when a tool proxy or the in-process door stands in front of the agent's own. + door_sees_tools = proxied or bool(interface is not None and getattr(interface, "include_tools", False)) dispatcher = build_dispatcher( # What actors watch (an on_tool trigger) includes the agent's calls through its tool proxy. document, self._context.bundle_dir, act=acts, read_log=lambda: _trial_log(door, trial_started), @@ -913,6 +922,9 @@ async def run( if dispatcher is not None: # What the actors wrote before T0 is in the world before the conversation starts. await dispatcher.replay() + # The trial's starting records, after its reset and pre-run sessions: what World changes compares against. + await asyncio.to_thread(self._records.start, id(runtime), records_dsn) + if dispatcher is not None: acting = asyncio.create_task(dispatcher.run()) transcript = await _drive_conversation( _HostedChatTarget( @@ -993,7 +1005,13 @@ async def run( transcript.calls[:] = _with_span_calls(transcript.calls, runtime) world_record = _world_actors_record(door, listing, dispatcher, cast, meter, limits, since=trial_started) - from .call_runner import duplicate_commits + from .call_runner import NOT_RECORDED, duplicate_commits + + if door_sees_tools: + world_record = {**(world_record or {}), **await asyncio.to_thread(self._records.end, id(runtime), records_dsn)} + else: + # The agent runs its own tools where no door stands: World changes says so, never "no change". + world_record = dict(NOT_RECORDED) # The world applied one of the agent's writes twice after its answer was lost: the scenario fails. duplicates = duplicate_commits(_trial_log(door, trial_started)) diff --git a/src/fi/alk/harness/hosted_scheduler.py b/src/fi/alk/harness/hosted_scheduler.py index 52ee54ead..dccc09b42 100644 --- a/src/fi/alk/harness/hosted_scheduler.py +++ b/src/fi/alk/harness/hosted_scheduler.py @@ -49,6 +49,7 @@ from .checks import reference_of, takes_reference from .joint_judge import OFF_SCRIPT from .judge import INCONCLUSIVE, NOT_APPLICABLE, UNMEASURED, verdict_named +from .world_door import EvidenceUnavailable from .judge import judge_sub_goal as _judge from .outbound import ( HostedAttemptSupersededError, @@ -427,6 +428,7 @@ async def receipt(self, receipt: ResultReceipt) -> None: ... "judge_undecided": FailureDomain.SIMULATOR, "caller_off_script": FailureDomain.SIMULATOR, "duplicate_commit": FailureDomain.AGENT, + "evidence_unmeasured": FailureDomain.SIMULATOR, "evidence_missing": FailureDomain.SIMULATOR, "actor_missing": FailureDomain.SIMULATOR, "world_usage": FailureDomain.SIMULATOR, @@ -682,6 +684,14 @@ def __init__(self, phase: str, cause: BaseException) -> None: self.cause = cause +class _PhaseEvidenceUnavailable(Exception): + """A check read evidence that could not be read (the world's action log): unmeasured, never passed or failed.""" + + def __init__(self, phase: str, cause: BaseException) -> None: + super().__init__(f"{phase}: {cause}") + self.cause = cause + + class _PhaseCrashed(Exception): def __init__(self, phase: str, cause: BaseException) -> None: super().__init__(f"{phase}: {cause}") @@ -806,6 +816,8 @@ async def _call() -> object: # the fixer report). The thread is abandoned, bounded by scenario count per the contract's # own accepted tradeoff; its world is discarded rather than reused (M13). raise _PhaseTimeout(phase) from exc + except EvidenceUnavailable as exc: + raise _PhaseEvidenceUnavailable(phase, exc) from exc except WorldUnavailable as exc: raise _PhaseWorldGone(phase, exc) from exc except WorldStateTooLarge as exc: @@ -843,6 +855,8 @@ async def _call() -> object: class _PhaseResult: value: object failure: ReceiptFailure | None + # Why the phase could not be measured: the evidence it read could not be read. + unmeasured: str | None = None async def _run_phase( @@ -877,6 +891,8 @@ async def _run_phase( None, _failure(_TIMEOUT_CODE_BY_PHASE[phase], f"{phase} exceeded its budget"), ) + except _PhaseEvidenceUnavailable as exc: + return _PhaseResult(None, None, unmeasured=str(exc.cause)) except _PhaseWorldGone as exc: return _PhaseResult(None, _failure("world_unavailable", str(exc.cause))) except _PhaseMisuse as exc: @@ -2423,6 +2439,14 @@ async def audit_log(**value): ) ) continue + if outcome.unmeasured is not None and (not goal.judged or getattr(goal, "both", False)): + # The check read the world's action log and the log could not be read: no verdict on missing evidence. + sub_goal_results.append( + SubGoalResult( + name=goal.name, held=None, reason=f"{UNMEASURED}{outcome.unmeasured}", judged=goal.judged != "" + ) + ) + continue verdict = _classify_check(outcome.value) both = bool(goal.judged) and getattr(goal, "both", False) if verdict.broken and (both or not goal.judged): @@ -2594,11 +2618,16 @@ async def _settle(goal: Any) -> Any: if any(result.held is False for result in sub_goal_results): status = "failed" elif any(result.held is None for result in sub_goal_results): + undecided = [result for result in sub_goal_results if result.held is None] + unmeasured = all(not result.judged and (result.reason or "").startswith(UNMEASURED) for result in undecided) return self._fault( scenario, world_index, attempt, - _failure("judge_undecided", "a judged sub-goal was not decided"), + _failure("evidence_unmeasured", "a sub-goal's evidence could not be read: " + + ", ".join(result.name for result in undecided)) + if unmeasured + else _failure("judge_undecided", "a judged sub-goal was not decided"), sub_goals=tuple(sub_goal_results), call=self._call_summary(call_outcome), retry=False, diff --git a/src/fi/alk/harness/world/handle.py b/src/fi/alk/harness/world/handle.py index 494b99988..2dfc79b5e 100644 --- a/src/fi/alk/harness/world/handle.py +++ b/src/fi/alk/harness/world/handle.py @@ -214,11 +214,21 @@ def events( self, actor_id: str | None = None, outcome: str | None = None ) -> list[dict[str, Any]]: """This trial's action log: every action attempted in the world, whose it was (`agent` for the - agent under test) and what came of it (`committed`, `refused`, `faulted`, `duplicate`).""" + agent under test) and what came of it (`committed`, `refused`, `faulted`, `duplicate`). A world with no log + has no events; a log that exists but cannot be read raises `EvidenceUnavailable`, so a check that needs it is + unmeasured, never settled on an empty log.""" + from ..world_door import EvidenceUnavailable + try: rows = self.query('SELECT * FROM "_alk_tool_trace" ORDER BY at, id') - except Exception: # noqa: BLE001 - a world without the log has no events - return [] + except Exception as exc: # noqa: BLE001 - told apart below: no log, or a log out of reach + try: + present = self.query("SELECT to_regclass('_alk_tool_trace') IS NOT NULL AS present") + except Exception: # noqa: BLE001 - the database itself is out of reach + raise EvidenceUnavailable("the world's action log could not be read") from exc + if present and not present[0].get("present"): + return [] + raise EvidenceUnavailable("the world's action log could not be read") from exc return [ row for row in rows diff --git a/src/fi/alk/harness/world_door.py b/src/fi/alk/harness/world_door.py index c14e04c36..dec6be54c 100644 --- a/src/fi/alk/harness/world_door.py +++ b/src/fi/alk/harness/world_door.py @@ -45,6 +45,10 @@ _IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,62}$") +class EvidenceUnavailable(Exception): + """The trial's action log could not be read: a check that needs it is unmeasured, never passed on an empty log.""" + + @dataclass(frozen=True) class ToolCall: """One attempt to act in the world, as every surface describes it.""" @@ -279,12 +283,13 @@ def enter(self, call: ToolCall) -> Outcome: def events( self, actor: str | None = None, outcome: str | None = None ) -> list[dict[str, Any]]: - """Every attempt in this trial's log, oldest first, optionally one participant's or one outcome's; nothing - when the log cannot be read.""" + """Every attempt in this trial's log, oldest first, optionally one participant's or one outcome's. Raises + `EvidenceUnavailable` when the log cannot be read: an empty answer would pass a check like "no refund was + committed" on missing evidence.""" try: rows = self.log.rows() - except Exception: # noqa: BLE001 - an unreadable log reads as empty this time - return [] + except Exception as exc: # noqa: BLE001 - whatever broke, the evidence is missing + raise EvidenceUnavailable("the world's action log could not be read") from exc return [ row for row in rows diff --git a/src/fi/alk/harness/world_snapshot.py b/src/fi/alk/harness/world_snapshot.py new file mode 100644 index 000000000..cd379080f --- /dev/null +++ b/src/fi/alk/harness/world_snapshot.py @@ -0,0 +1,170 @@ +"""The world's records before and after a trial. + +A trial's World changes says which records it added, changed or removed: the runner reads every table of the world's +database at the trial's start (after its reset and pre-run sessions) and again at its end, on a connection that cannot +write, and compares them by primary key. The harness's own tables (`_alk_*`) are never compared. A table with no +primary key, or over `max_rows`, is not compared and is named with why, never dropped silently; changes over +`max_changes` are counted, not listed. + +The same snapshot says whether a trial started from the seeded records (`digest`): the first trial's start is the +seed, and a later trial whose start differs did not start clean. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from typing import Any + +MAX_ROWS = 5000 +MAX_CHANGES = 100 +# Longest text a change shows for one field; a document or blob column would otherwise ride in every trial's record. +MAX_VALUE_CHARS = 2000 +INTERNAL_PREFIX = "_alk_" + + +@dataclass +class Snapshot: + """Each compared table's rows by primary key (the key's values as JSON text), and the tables left out with why.""" + + tables: dict[str, dict[str, dict[str, Any]]] = field(default_factory=dict) + skipped: list[dict[str, str]] = field(default_factory=list) + + +def _plain(row: dict[str, Any]) -> dict[str, Any]: + """A row as JSON data: dates, decimals and the like as text.""" + return json.loads(json.dumps(row, default=str)) + + +def _shown(row: dict[str, Any] | None) -> dict[str, Any] | None: + """A record as a change shows it: text past MAX_VALUE_CHARS is cut, and says how much was cut.""" + if row is None: + return None + shown: dict[str, Any] = {} + for name, value in row.items(): + text = value if isinstance(value, str) else json.dumps(value) if isinstance(value, (dict, list)) else None + if text is not None and len(text) > MAX_VALUE_CHARS: + value = f"{text[:MAX_VALUE_CHARS]}... ({len(text) - MAX_VALUE_CHARS} more characters)" + shown[name] = value + return shown + + +def from_rows( + tables: dict[str, tuple[list[str], list[dict[str, Any]]]], *, max_rows: int = MAX_ROWS +) -> Snapshot: + """A snapshot from each table's primary key columns and rows, by name; the reading is the caller's.""" + snap = Snapshot() + for table in sorted(tables): + if table.lower().startswith(INTERNAL_PREFIX): + continue + key, rows = tables[table] + if not key: + snap.skipped.append({"table": table, "why": "no primary key"}) + continue + if len(rows) > max_rows: + snap.skipped.append({"table": table, "why": f"over {max_rows} rows"}) + continue + snap.tables[table] = { + json.dumps([row.get(column) for column in key], default=str): _plain(row) for row in rows + } + return snap + + +_TABLES = ( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema = 'public' AND table_type = 'BASE TABLE' ORDER BY table_name" +) +_KEYS = ( + "SELECT kcu.table_name, kcu.column_name FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name " + "AND tc.table_schema = kcu.table_schema AND tc.table_name = kcu.table_name " + "WHERE tc.table_schema = 'public' AND tc.constraint_type = 'PRIMARY KEY' ORDER BY kcu.ordinal_position" +) + + +def snapshot(dsn: str, *, max_rows: int = MAX_ROWS, timeout_ms: int = 10000) -> Snapshot: + """Every non-internal table of the world's Postgres database, read on a connection that cannot write. Raises when + the database cannot be read; the caller records that, never failing the trial.""" + import psycopg + + options = f"-c default_transaction_read_only=on -c statement_timeout={timeout_ms}" + with psycopg.connect(dsn, autocommit=True, connect_timeout=5, options=options) as connection: + names = [ + str(row[0]) + for row in connection.execute(_TABLES).fetchall() + if not str(row[0]).lower().startswith(INTERNAL_PREFIX) + ] + keys: dict[str, list[str]] = {} + for table, column in connection.execute(_KEYS).fetchall(): + keys.setdefault(str(table), []).append(str(column)) + read: dict[str, tuple[list[str], list[dict[str, Any]]]] = {} + too_big: list[dict[str, str]] = [] + for table in names: + quoted = '"' + table.replace('"', '""') + '"' + if not keys.get(table): + read[table] = ([], []) + continue + count = connection.execute(f"SELECT count(*) FROM {quoted}").fetchone() # noqa: S608 - a listed table + if count and int(count[0]) > max_rows: + too_big.append({"table": table, "why": f"over {max_rows} rows"}) + continue + cursor = connection.execute(f"SELECT * FROM {quoted}") # noqa: S608 - a listed table + columns = [one[0] for one in cursor.description or []] + read[table] = (keys[table], [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()]) + snap = from_rows(read, max_rows=max_rows) + snap.skipped = sorted([*snap.skipped, *too_big], key=lambda one: one["table"]) + return snap + + +def diff(before: Snapshot, after: Snapshot, *, max_changes: int = MAX_CHANGES) -> dict[str, Any]: + """What the trial changed, table by table: each added, changed or removed record with its values before and + after and, for a change, the fields that changed. Tables either side left out are named in `skipped`; changes + past `max_changes` are counted in `truncated`.""" + changes: list[dict[str, Any]] = [] + total = 0 + skipped = {one["table"]: one for one in [*before.skipped, *after.skipped]} + for table in sorted(set(before.tables) | set(after.tables)): + if table in skipped: + continue + old = before.tables.get(table, {}) + new = after.tables.get(table, {}) + for key in sorted(set(old) | set(new)): + was, now = old.get(key), new.get(key) + if was == now: + continue + if was is None: + entry = {"table": table, "key": json.loads(key), "kind": "added", "before": None, "after": now, + "fields": []} + elif now is None: + entry = {"table": table, "key": json.loads(key), "kind": "removed", "before": was, "after": None, + "fields": []} + else: + fields = sorted(name for name in set(was) | set(now) if was.get(name) != now.get(name)) + entry = {"table": table, "key": json.loads(key), "kind": "changed", "before": was, "after": now, + "fields": fields} + total += 1 + if len(changes) < max_changes: + entry["before"], entry["after"] = _shown(entry["before"]), _shown(entry["after"]) + changes.append(entry) + return {"changes": changes, "skipped": list(skipped.values()), "truncated": max(total - len(changes), 0)} + + +def digest(snap: Snapshot) -> dict[str, str]: + """Each compared table's content as one hash, to tell whether two starts held the same records.""" + return { + table: hashlib.sha256(json.dumps(rows, sort_keys=True, default=str).encode()).hexdigest() + for table, rows in snap.tables.items() + } + + +def start_state(seeded: dict[str, str] | None, start: dict[str, str] | None) -> dict[str, Any]: + """Whether a trial started from the seeded records: `seeded`, `differs` with the tables that differ, or `unknown` + when either start could not be read.""" + if seeded is None or start is None: + return {"state": "unknown", "tables": []} + tables = sorted(name for name in set(seeded) | set(start) if seeded.get(name) != start.get(name)) + return {"state": "differs" if tables else "seeded", "tables": tables} + + +__all__ = ["MAX_CHANGES", "MAX_ROWS", "MAX_VALUE_CHARS", "Snapshot", "diff", "digest", "from_rows", "snapshot", "start_state"] diff --git a/tests/harness/test_bundle_author_v2.py b/tests/harness/test_bundle_author_v2.py index 5fdef7b98..a6440354b 100644 --- a/tests/harness/test_bundle_author_v2.py +++ b/tests/harness/test_bundle_author_v2.py @@ -5174,16 +5174,12 @@ def test_a_bundle_with_actors_seals_the_rules_its_release_checks_them_by( assert set(rules["records"]) == {"actors.json"} -def test_a_world_without_actors_is_seeded_with_the_tool_trace_it_always_had( - tmp_path: Path, -) -> None: +def test_a_world_without_actors_gets_the_full_action_log(tmp_path: Path) -> None: from fi.alk.harness.bundle_author_v2 import _action_log_seed - assert _action_log_seed(tmp_path) == ( - "CREATE TABLE IF NOT EXISTS _alk_tool_trace (" - "id bigserial PRIMARY KEY, name text NOT NULL, arguments jsonb NOT NULL, " - "result jsonb, ok boolean NOT NULL, error text, at double precision NOT NULL);\n" - ) + without = _action_log_seed(tmp_path) + assert "actor_id text" in without and "outcome text" in without and "effect text" in without + assert "INSERT INTO _alk_actors" not in without (tmp_path / "actors.json").write_text( '{"actors": [{"id": "reviewer", "name": "Robin"}]}\n', encoding="utf-8" ) diff --git a/tests/harness/test_call_runner.py b/tests/harness/test_call_runner.py index 42a5d56d3..7ed33bf75 100644 --- a/tests/harness/test_call_runner.py +++ b/tests/harness/test_call_runner.py @@ -2950,3 +2950,41 @@ def test_arguments_read_back_as_json_text_still_match(): _logged("issue_refund", {"order": 7}), ] assert len(duplicate_commits(log)) == 1 + + +def test_voice_trial_without_actors_seals_world_changes(tmp_path, monkeypatch) -> None: + _job_obj, context = _context(tmp_path=tmp_path, evidence_seam=EvidenceSeam.TOOL_TRACE) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + columns = ["name", "arguments", "result", "ok", "error", "at", "action_id", "actor_id", "outcome"] + rows = [("issue_refund", {"order": 7}, None, True, "", 2.0, "a1", "agent", "committed")] + monkeypatch.setitem(sys.modules, "psycopg", _FakePsycopg(rows, columns)) + + async def place_call(spec): + return _report() + + adapter = FakeAdapter() + runner = cr.CallRunnerImpl(adapter, context, place_call=place_call) + _run( + runner, + _FakeScenario("k1"), + _runtime(endpoints={"database": _postgres_endpoint()}, metadata={"livekit_agent_name": "agent-w0"}), + ) + sealed = {kind: data for kind, _key, data in adapter.uploads} + world = json.loads(sealed[cr.ArtifactKind.TRANSCRIPT])["world_actors"] + assert world["actions"] == [] and world["state_changes"][0]["committed_actions"] == ["issue_refund"] + assert "records" in world and "start" in world + + +def test_a_voice_trial_whose_tools_the_door_cannot_see_says_world_changes_were_not_recorded(tmp_path) -> None: + _job_obj, context = _context(tmp_path=tmp_path, evidence_seam=None) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + return _report() + + adapter = FakeAdapter() + runner = cr.CallRunnerImpl(adapter, context, place_call=place_call) + _run(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + sealed = {kind: data for kind, _key, data in adapter.uploads} + world = json.loads(sealed[cr.ArtifactKind.TRANSCRIPT])["world_actors"] + assert world == {"recorded": False, "why": "in_process_tools"} diff --git a/tests/harness/test_chat_call_runner.py b/tests/harness/test_chat_call_runner.py index dbc50801a..7cb509de2 100644 --- a/tests/harness/test_chat_call_runner.py +++ b/tests/harness/test_chat_call_runner.py @@ -1804,3 +1804,17 @@ def test_a_chat_actor_s_wait_still_open_at_the_end_is_recorded_with_its_timeout( assert record["decisions"] == [ {"actor": "manager", "play": "timeout", "fallback": None, "outcome": "unresolved_at_end", "timeout_seconds": 30} ] + + +def test_a_trial_without_actors_still_seals_world_changes() -> None: + from fi.alk.harness.actors import ActorList + from fi.alk.harness.world_door import AGENT, MemoryLog + + log = MemoryLog() + log.write({"name": "issue_refund", "actor_id": AGENT, "outcome": "committed", "ok": True, "at": 10.0}) + door = SimpleNamespace(log=log) + record = chat._world_actors_record(door, ActorList(), None, None, SimpleNamespace(rows=list), {}, since=9.0) + assert record is not None and record["actions"] == [] and record["decisions"] == [] + [agent] = record["state_changes"] + assert agent["committed_actions"] == ["issue_refund"] + assert chat._world_actors_record(None, ActorList(), None, None, SimpleNamespace(rows=list), {}) is None diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py index 5a0e3001b..603376b4d 100644 --- a/tests/harness/test_hosted_scheduler.py +++ b/tests/harness/test_hosted_scheduler.py @@ -4886,3 +4886,31 @@ async def _all(goals, world, calls, *, messages=()): ) clean, _ = _two_judged(monkeypatch, _all, goals=passing, outcome=_call_outcome(turns=2)) assert clean.status == "passed" and clean.failure is None + + +def test_a_check_reading_an_unreadable_log_is_unmeasured_not_passed(monkeypatch) -> None: + from fi.alk.harness.world_door import EvidenceUnavailable + + async def _all(goals, world, calls, *, messages=()): + return {} + + def reads_the_log(world, calls): + raise EvidenceUnavailable("the world's action log could not be read") + + goals = [FakeSubGoal("no_refund_committed", reads_the_log), FakeSubGoal("greeted", lambda w, c: None)] + receipt, _ = _two_judged(monkeypatch, _all, goals=goals, outcome=_call_outcome(turns=2)) + unmeasured, settled = receipt.sub_goals + assert unmeasured.held is None and unmeasured.reason.startswith(hs.UNMEASURED) + assert "action log could not be read" in unmeasured.reason + assert settled.held is True + assert receipt.status == "errored" and receipt.failure.code == "evidence_unmeasured" + + +def test_a_check_that_never_reads_the_log_still_settles(monkeypatch) -> None: + async def _all(goals, world, calls, *, messages=()): + return {} + + receipt, _ = _two_judged( + monkeypatch, _all, goals=[FakeSubGoal("greeted", lambda w, c: None)], outcome=_call_outcome(turns=2) + ) + assert receipt.status == "passed" diff --git a/tests/harness/test_world_door.py b/tests/harness/test_world_door.py index 9f67df4a6..affee1fd5 100644 --- a/tests/harness/test_world_door.py +++ b/tests/harness/test_world_door.py @@ -288,7 +288,11 @@ def rows(self): door = _door(_Executor(fail=True), log=_Broken()) outcome = door.enter(ToolCall(action="crm.update", name="update")) assert outcome.outcome == "faulted" and outcome.status == 502 - assert door.events() == [] + from fi.alk.harness.world_door import EvidenceUnavailable + + # The read side says the evidence is missing rather than reading as an empty log. + with pytest.raises(EvidenceUnavailable): + door.events() def test_an_unexpected_failure_inside_the_door_comes_back_as_an_outcome(): @@ -870,3 +874,15 @@ def test_each_call_is_logged_as_a_read_or_a_write_from_its_declared_effect_else_ assert door.enter(call).outcome == "committed" [row] = door.events() assert row["effect"] == effect + + +def test_events_raise_when_the_log_cannot_be_read(): + from fi.alk.harness.world_door import EvidenceUnavailable + + class _Unreadable(MemoryLog): + def rows(self): + raise RuntimeError("no read grant") + + door = _door(log=_Unreadable()) + with pytest.raises(EvidenceUnavailable): + door.events() diff --git a/tests/harness/test_world_snapshot.py b/tests/harness/test_world_snapshot.py new file mode 100644 index 000000000..0a7981392 --- /dev/null +++ b/tests/harness/test_world_snapshot.py @@ -0,0 +1,116 @@ +"""The world's records before and after a trial, and whether the trial started from the seed.""" + +from __future__ import annotations + +from fi.alk.harness.call_runner import TrialRecords +from fi.alk.harness.world_snapshot import diff, digest, from_rows, start_state + + +def _orders(*rows): + return {"orders": (["id"], [dict(row) for row in rows]), "refunds": (["id"], [])} + + +def test_diff_names_added_changed_and_removed_rows(): + before = from_rows({"orders": (["id"], [{"id": 1, "status": "paid"}, {"id": 2, "status": "paid"}])}) + after = from_rows( + {"orders": (["id"], [{"id": 1, "status": "refunded"}, {"id": 3, "status": "paid"}])} + ) + changes = {(one["key"][0], one["kind"]): one for one in diff(before, after)["changes"]} + assert set(changes) == {(1, "changed"), (2, "removed"), (3, "added")} + assert changes[(1, "changed")]["fields"] == ["status"] + assert changes[(1, "changed")]["before"] == {"id": 1, "status": "paid"} + assert changes[(1, "changed")]["after"] == {"id": 1, "status": "refunded"} + + +def test_internal_tables_are_never_compared(): + snap = from_rows({"_alk_tool_trace": (["id"], [{"id": 1}]), "orders": (["id"], [])}) + assert set(snap.tables) == {"orders"} + + +def test_a_table_without_a_primary_key_is_skipped_and_named(): + before = from_rows({"audit": ([], [{"line": "a"}]), "orders": (["id"], [])}) + after = from_rows({"audit": ([], [{"line": "a"}, {"line": "b"}]), "orders": (["id"], [])}) + found = diff(before, after) + assert found["changes"] == [] + assert found["skipped"] == [{"table": "audit", "why": "no primary key"}] + + +def test_a_table_over_the_row_cap_is_skipped_and_named(): + snap = from_rows({"events": (["id"], [{"id": n} for n in range(6)])}, max_rows=5) + assert snap.tables == {} and snap.skipped == [{"table": "events", "why": "over 5 rows"}] + + +def test_changes_over_the_cap_are_counted_not_dropped_silently(): + before = from_rows({"orders": (["id"], [])}) + after = from_rows({"orders": (["id"], [{"id": n} for n in range(5)])}) + found = diff(before, after, max_changes=2) + assert len(found["changes"]) == 2 and found["truncated"] == 3 + + +def test_a_second_trial_that_starts_from_changed_rows_reads_differs(monkeypatch): + import fi.alk.harness.world_snapshot as ws + + starts = iter( + [ + from_rows(_orders({"id": 1, "status": "paid"})), + from_rows(_orders({"id": 1, "status": "paid"})), + from_rows(_orders({"id": 1, "status": "refunded"})), + ] + ) + monkeypatch.setattr(ws, "snapshot", lambda dsn, **_: next(starts)) + records = TrialRecords() + records.start("w0", "dsn") + records._starts.pop("w0") + records.start("w0", "dsn") + assert records._starts.pop("w0")[1] == {"state": "seeded", "tables": []} + # The third start is compared with the seed, not with the trial before it. + records.start("w0", "dsn") + assert records._starts["w0"][1] == {"state": "differs", "tables": ["orders"]} + + +def test_start_state_is_unknown_when_a_start_could_not_be_read(): + assert start_state(None, {"orders": "x"})["state"] == "unknown" + snap = from_rows(_orders({"id": 1})) + assert start_state(digest(snap), digest(snap)) == {"state": "seeded", "tables": []} + + +def test_a_trial_whose_records_cannot_be_read_is_sealed_unreadable_never_failed(monkeypatch): + import fi.alk.harness.world_snapshot as ws + + def broken(dsn, **_): + raise RuntimeError("down") + + monkeypatch.setattr(ws, "snapshot", broken) + records = TrialRecords() + records.start("w0", "dsn") + assert records.end("w0", "dsn") == {"records": {"unreadable": True}, "start": {"state": "unknown", "tables": []}} + + +def test_world_changes_carry_the_rows_the_trial_changed(monkeypatch): + import fi.alk.harness.world_snapshot as ws + + taken = iter([from_rows(_orders({"id": 1042, "status": "paid"})), + from_rows({"orders": (["id"], [{"id": 1042, "status": "refunded"}]), + "refunds": (["id"], [{"id": 1, "order_id": 1042}])})]) + monkeypatch.setattr(ws, "snapshot", lambda dsn, **_: next(taken)) + records = TrialRecords() + records.start("w0", "dsn") + sealed = records.end("w0", "dsn") + kinds = sorted((one["table"], one["kind"]) for one in sealed["records"]["changes"]) + assert kinds == [("orders", "changed"), ("refunds", "added")] + assert sealed["start"]["state"] == "seeded" + + +def test_a_huge_field_is_cut_in_the_change_but_still_counts_as_changed(): + from fi.alk.harness.world_snapshot import MAX_VALUE_CHARS, diff, from_rows + + big = "x" * (MAX_VALUE_CHARS * 50) + before = from_rows({"docs": (["id"], [{"id": 1, "body": big, "notes": {"k": big}}])}) + after = from_rows({"docs": (["id"], [{"id": 1, "body": big + "y", "notes": {"k": big}}])}) + + [change] = diff(before, after)["changes"] + assert change["fields"] == ["body"] + assert len(change["after"]["body"]) < MAX_VALUE_CHARS + 50 + assert change["after"]["body"].endswith(f"({len(big) + 1 - MAX_VALUE_CHARS} more characters)") + assert len(change["before"]["notes"]) < MAX_VALUE_CHARS + 50 + assert change["after"]["id"] == 1