From 000752a11d4c22967e98f4ca18d451b3a37ad55d Mon Sep 17 00:00:00 2001 From: ayushcodes10 Date: Thu, 3 Sep 2026 16:49:45 +0530 Subject: [PATCH] stop letting one tool claim to be another End-to-end testing through Codex's own config found the half of criterion 6 that was still forgeable. `written_by` has been derived from the fact's edge since 2026-08-29, so a caller cannot assert who wrote something. `recalled_by` was still a parameter, and a claude-code server passing recalled_by="codex" produced a counted cross-tool save: {"counts_toward_gate": true, "cross_tool_saves": 2} The criterion measures whether two tools were involved, and the agent supplying the evidence is the agent being graded, so neither side may come from the caller. The reader is now the server's own configured agent id, which comes from the config the client launched it with. `recalled_by` is gone from the tool contract. Both derived values are now returned on every response, not only when the save is rejected: the caller supplies neither, so the response is the only way it can see what the server concluded and check the save says what it meant. Verified by removing the fix and watching the guards go red, not merely by watching them pass. The 2048-char description guard added an hour earlier also caught this change bloating record_recall_save to 2096; the docstring is 1376 now. 415 tests pass, 4 added. --- src/echo_memory/server.py | 34 ++++---- .../test_recall_save_attribution.py | 81 +++++++++++++++++++ 2 files changed, 99 insertions(+), 16 deletions(-) create mode 100644 tests/integration/test_recall_save_attribution.py diff --git a/src/echo_memory/server.py b/src/echo_memory/server.py index b97a1e8..90bb0af 100644 --- a/src/echo_memory/server.py +++ b/src/echo_memory/server.py @@ -244,7 +244,6 @@ def record_recall_save( scope: str, fact_id: str, note: str, - recalled_by: str | None = None, ) -> dict: """Record that a fact you recalled from memory saved the user from re-explaining something to you. @@ -254,21 +253,15 @@ def record_recall_save( answered something the user would otherwise have had to tell you again, and the fact was originally written by a DIFFERENT tool or a past session. - That last part is the whole point, and it is why this takes `fact_id` - rather than a `written_by` string. Pass the `fact_id` of the fact that - helped - every query_memory result carries one. The server reads that - edge's own `agent_id` and uses it as `written_by`; the caller does not get - to assert who wrote a fact. + That is why this takes only a `fact_id`. Pass the one carried by the + query_memory result that helped; the server reads that edge's own agent_id + for the writer, and uses its own configured agent id for the reader. You + assert neither. Both used to be caller-supplied, and each in turn let the + model being graded type its own evidence. - Until 2026-08-29 `written_by` was free text supplied by the caller. Nothing - checked the fact existed, so the number gating v1a was a string typed by - the model being graded. A fact_id is checkable, so the reading is - admissible. - - recalled_by is you, defaulting to this server's own agent id. If the fact's - author and you are the same tool, the save is still recorded but does not - count toward the trial's bar - recalling your own note from ten minutes ago - is not the thing being measured. + If the fact's author and you are the same tool, the save is recorded but + does not count - recalling your own note from ten minutes ago is not the + thing being measured. note should be one sentence naming what it saved re-explaining, written so it still makes sense read cold in six months. Recording the identical note @@ -282,7 +275,10 @@ def record_recall_save( except ConfigError as e: return {"error": str(e)} - recalled_by = recalled_by or _state.config.agent_id + # Not a parameter: an agent asserting which tool it is, to a criterion that + # measures whether two tools are involved, is the same fault the written_by + # fix closed on 2026-08-29. + recalled_by = _state.config.agent_id try: with _state.pool.connection() as conn: written_by = _author_of(conn, group_id, fact_id) @@ -321,6 +317,12 @@ def record_recall_save( "recorded": True, "observation_id": recorded["id"], "already_recorded": not recorded["created"], + # Always returned, both branches. The caller supplies neither of these + # now, so the only way it can see what the server concluded about + # authorship - and check the save it just logged says what it meant - + # is for the response to state it. + "written_by": written_by, + "recalled_by": recalled_by, "counts_toward_gate": cross_tool, "cross_tool_saves": counts["cross_tool_saves"], "required": _observations.REQUIRED_SAVES, diff --git a/tests/integration/test_recall_save_attribution.py b/tests/integration/test_recall_save_attribution.py new file mode 100644 index 0000000..a1e49f8 --- /dev/null +++ b/tests/integration/test_recall_save_attribution.py @@ -0,0 +1,81 @@ +"""Neither side of a cross-tool save may be asserted by the caller. + +Criterion 6 counts a save only when the fact's author and its reader are +different tools, and the agent supplying the evidence is the agent being +graded. Both halves therefore have to come from somewhere the agent cannot +choose. + +The writer half was closed on 2026-08-29: `written_by` used to be free text, +so the number gating v1a was a string typed by the model being measured. It is +now read from the fact's own edge via fact_id. + +The reader half stayed open until end-to-end testing on 2026-09-03 passed +recalled_by="codex" from a claude-code server and watched it produce a counted +cross-tool save. The reader is now the server's own configured agent id, which +comes from the config the client launched it with, and is not a parameter.""" + +import inspect + +import pytest +from fake_embedder import REFERENCE, VectorEmbedder + +from echo_memory import server +from echo_memory.infra.config import Config + +FACT = "a fact written by one tool and read by another" + + +def _serve(migrated_db, agent_id): + config = Config( + user_id="ayush", agent_id=agent_id, database_url=migrated_db, project="echo-mem" + ) + server.startup(config=config, embedder=VectorEmbedder({"probe": REFERENCE, FACT: REFERENCE})) + return config + + +def _write(session_id): + result = server.write_episode( + "shared", session_id, + [{"name": "probe", "type": "test"}], + [{"source": "probe", "target": "probe", "relation_type": "is", + "fact": FACT, "confidence": "extracted"}], + entity_resolutions={"probe": {"resolved_to": "new"}}, + ) + return result["edges_created"][0] + + +def test_the_reader_cannot_be_claimed(migrated_db): + """The exact call that used to work: one tool asserting it is another.""" + _serve(migrated_db, "claude-code") + with pytest.raises(TypeError): + server.record_recall_save("shared", "1", "note", recalled_by="codex") + + +def test_recalled_by_is_not_part_of_the_tool_contract(migrated_db): + _serve(migrated_db, "claude-code") + assert "recalled_by" not in inspect.signature(server.record_recall_save).parameters + + +def test_one_tool_alone_cannot_produce_a_cross_tool_save(migrated_db): + _serve(migrated_db, "claude-code") + fact_id = _write("cc-1") + + saved = server.record_recall_save("shared", str(fact_id), "read my own note back") + + assert saved["recorded"] is True + assert saved["counts_toward_gate"] is False + assert saved["cross_tool_saves"] == 0 + + +def test_a_genuine_cross_tool_save_still_counts(migrated_db): + """Written by one server, read by another, exactly as two clients run.""" + _serve(migrated_db, "codex") + fact_id = _write("codex-1") + + _serve(migrated_db, "claude-code") + saved = server.record_recall_save("shared", str(fact_id), "codex knew this already") + + assert saved["counts_toward_gate"] is True + assert saved["cross_tool_saves"] == 1 + assert saved["written_by"] == "codex" + assert saved["recalled_by"] == "claude-code"