diff --git a/docs/methods/ahe_adapter.md b/docs/methods/ahe_adapter.md index 07e7cff3..f9069edf 100644 --- a/docs/methods/ahe_adapter.md +++ b/docs/methods/ahe_adapter.md @@ -154,7 +154,7 @@ this framework's own code. | Dimension | AHE (8b2a55d) | This Evolver | Why | |---|---|---|---| | When learning happens | Once per iteration, after k = 2 rollouts of every task; 10 iterations | The same: a step is every trial row, ten steps | Same | -| What the learner sees | The Debugger: each task's NexAU traces (with a per-turn timing breakdown for a timed-out rollout) and verifier output; the Evolve Agent: the overview, per-task reports, verdicts, history | The same, with the acting agent's event stream (Codex's or Claude Code's; whole, and one file per standard step) and the hook audit as the traces and the judge's feedback as the verifier output; no timing breakdown, since the stream has no timestamps | Framework: the host's trace format | +| What the learner sees | The Debugger: each task's NexAU traces (with a per-turn timing breakdown for a timed-out rollout) and verifier output; the Evolve Agent: the overview, per-task reports, verdicts, history | The same, with the acting agent's event stream (the one its CLI wrote; whole, and one file per standard step) and, on the hosts that run hooks (Codex, Claude Code), the hook audit as the traces, and the judge's feedback as the verifier output; no timing breakdown, which comes only from a Harbor trial's `agent/trajectory.json` | Framework: the host's trace format | | What is learned | NexAU's prompt, tool descriptions and implementations, middleware classes, skills, sub-agents, memory | The adapter's files (the table above), the same on either host; the host's loop and tools are fixed | Framework: the acting agent is fixed; the reason for the name | | The update | Edits plus a manifest of predictions, attributed at the next iteration; Algorithm 1 reverts HARMFUL changes, the released code leaves it to the agent | The same manifest and rules; `rollback: auto` (the paper) or `agent` (the code) | Decision: the paper by default | | Final harness | The best iteration by pass@1 | The best-ever commit, the harness its rollouts ran (Algorithm 1 line 15 names the harness after the next edit, which no rollout measured) | Decision | diff --git a/hermit/agent/evolver/methods/ahe_adapter/debugger.py b/hermit/agent/evolver/methods/ahe_adapter/debugger.py index 39ba9e9b..24a93c05 100644 --- a/hermit/agent/evolver/methods/ahe_adapter/debugger.py +++ b/hermit/agent/evolver/methods/ahe_adapter/debugger.py @@ -177,7 +177,16 @@ def trace_source(log: Path | None) -> tuple[Path, str] | None: # The texts the learners read name the acting agent's event stream as the framework's Codex agent keeps it; with # another acting agent the same sentences name its own (the Debugger and the Evolve Agent run on the Evolver's CLI). -STREAM_TEXTS = {"agent-claude.jsonl": (("agent-codex.jsonl", "agent-claude.jsonl"), +STREAM_TEXTS = {"agent-pi.jsonl": (("agent-codex.jsonl", "agent-pi.jsonl"), + ("the raw Codex event stream", "the raw Pi event stream"), + ("the Codex event stream", "the Pi event stream"), + ("`codex exec --json` event stream", "`pi --mode json` event stream")), + "agent-mini-swe-agent.jsonl": (("agent-codex.jsonl", "agent-mini-swe-agent.jsonl"), + ("the raw Codex event stream", "the raw mini-swe-agent event stream"), + ("the Codex event stream", "the mini-swe-agent event stream"), + ("`codex exec --json` event stream", + "`python -m hermit.agent.base.mini_swe_agent_run` event stream")), + "agent-claude.jsonl": (("agent-codex.jsonl", "agent-claude.jsonl"), ("the raw Codex event stream", "the raw Claude Code event stream"), ("the Codex event stream", "the Claude Code event stream"), ("`codex exec --json` event stream", diff --git a/tests/core/test_pi_agent.py b/tests/core/test_pi_agent.py index b2f38218..a6e82217 100644 --- a/tests/core/test_pi_agent.py +++ b/tests/core/test_pi_agent.py @@ -220,7 +220,8 @@ def test_system_instructions_are_appended_and_a_long_one_goes_in_a_file(fake): def test_a_pi_stream_is_read_back_by_its_name_or_by_its_events(tmp_path): - """The learning material keeps the acting agent's log: an Evolver reads Pi's steps from it like Codex's.""" + """The learning material keeps the acting agent's log: an Evolver reads Pi's steps from it like Codex's, and + records.parse_log reads its session and usage as Pi's (it read a Pi stream as Claude Code's: none of either).""" named = tmp_path / "log" named.mkdir() shutil.copy(STREAMS / "success.jsonl", named / trajectory.PI_STREAM) @@ -231,6 +232,9 @@ def test_a_pi_stream_is_read_back_by_its_name_or_by_its_events(tmp_path): raw = (STREAMS / "failure.jsonl").read_text() assert not base_cli.served("pi", raw) and base_cli.notices("pi", raw).startswith("503: ") assert base_cli.served("pi", (STREAMS / "success.jsonl").read_text()) + from hermit.agent.base.records import parse_log + session, [usage] = parse_log(STREAMS / "success.jsonl", "pi", "m") + assert session == SESSION and (usage.calls, usage.input_tokens, usage.output_tokens) == (4, 4000, 80) def test_the_pi_acting_agent_deploys_the_adapter_and_trusts_its_project_for_the_session(fake): diff --git a/tests/methods/ahe/test_ahe_hosts.py b/tests/methods/ahe/test_ahe_hosts.py index 441b1805..8fe78859 100644 --- a/tests/methods/ahe/test_ahe_hosts.py +++ b/tests/methods/ahe/test_ahe_hosts.py @@ -668,6 +668,27 @@ def test_a_state_with_an_unencodable_character_is_saved(tmp_path, capsys, monkey # --- the Evolver's texts ----------------------------------------------------------------------------------------- +def test_the_evolvers_texts_name_every_acting_agents_stream_it_reads(): + """A Pi or mini-swe-agent acting agent's sessions keep their own stream (agent-pi.jsonl, agent-mini-swe-agent.jsonl), + which rollout_stream finds: the learners' texts name it, not Codex's file, which is not there.""" + from hermit.agent.base.trajectory import CODEX_STREAM, STREAMS + from hermit.agent.evolver.methods.ahe_adapter import analysis as A, debugger as D + import test_ahe_analysis as AN + methods = T.ROOT / "hermit" / "agent" / "evolver" / "methods" / "ahe_adapter" + stats = A.compute_stats([AN.rec("a", 1, **AN.PASS), AN.rec("b", 1, **AN.FAIL)]) + query = A.build_evolution_query({"learning_material": "lm", "state": "st", "adapter": "ad"}, stats, step=2) + texts = [(methods / name).read_text() for name in ("START.md", "DEBUGGER.md")] + [query] + for stream in set(STREAMS) - {CODEX_STREAM}: + for text in texts: + words = D.for_stream(text, stream) + assert "agent-codex.jsonl" not in words and "Codex event stream" not in words, stream + assert "codex exec --json" not in words, stream + assert stream in D.for_stream(texts[0], stream) + for stream, command in (("agent-pi.jsonl", "pi --mode json"), + ("agent-mini-swe-agent.jsonl", "python -m hermit.agent.base.mini_swe_agent_run")): + assert f"`{command}` event stream" in D.for_stream(texts[1], stream) + + def test_the_evolvers_texts_name_the_acting_agents_stream(tmp_path): """START.md, the evolution query and the Debugger's question name Claude Code's stream when the rollouts are its sessions, and are the files' own for Codex's."""