diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index e2a3610a..0c37491f 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -10,6 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Batch fact ids now stay with their own facts when part of a batch is skipped (#751).** `remember_batch` returns only the ids of the facts it stored. When a group was skipped as a duplicate, the list got shorter, and callers that matched ids to facts by position moved every later id onto the wrong fact. In auto-extraction, an `UPDATE` whose replacement was skipped could still delete (tombstone) the old fact, which lost data. In the write-quota retry queue (#747), skipped facts were dropped silently. + - `remember_batch` still returns a `list[str]` of stored ids, now a `BatchIds` with a `batch_results` attribute. It has one entry per input fact, holding the id or `None` when that fact was skipped, the same shape as `RelayWriteQuotaExceeded.batch_results`. Auto-extraction and the retry queue match ids to facts through it. + - A queued fact the batch writer skips goes back to the queue with one more attempt counted, and is dropped at the existing attempt limit. + - A store error counts as a duplicate only when `409` appears as a whole word. Before, `409` anywhere matched, including the digits inside a hex hash or address in an unrelated bundler error, and that error was then dropped silently. The managed relay never returns 409 for writes. The import engine's per-fact path uses the same check. + - **The session summary is no longer lost when the monthly write allowance is used up (#749).** A session Crystal the relay refused for quota was only logged. It now goes into the same encrypted retry queue as refused auto-extracted facts (#747) and is stored once saving works again, as the Crystal that would have been written at session end: same metadata and `session_id`, no embedding, and the parts the user has forgotten since it was queued are removed (#605 / #640). The refusal reaches the user through the once-per-session notice. Queued facts now keep the time they were refused as `created_at`, instead of taking the time the queue drains, so the vault groups them with their conversation. The notice's count reads correctly for one memory ("1 new memory is kept on this device"). Tests: `tests/test_write_quota_747.py` (section 7). - **A used-up monthly write allowance is no longer silent, and auto-extracted facts are no longer lost (#747).** The relay refuses a write over the cap with `403 {"error": "quota_exceeded"}` on `/v1/bundler`. The client never read that body, and `remember_batch` turned the error into a `RuntimeError` with no status and no "quota" in its text, so even the billing-cache invalidation added for #663 never ran. Auto-extraction logged "Fact not stored (batch failure)", marked the messages processed and moved on: 44 facts were dropped on one install over three days, and the user only found out by asking. Now: diff --git a/python/src/totalreclaw/agent/lifecycle.py b/python/src/totalreclaw/agent/lifecycle.py index e741833d..a63f453f 100644 --- a/python/src/totalreclaw/agent/lifecycle.py +++ b/python/src/totalreclaw/agent/lifecycle.py @@ -787,8 +787,9 @@ def _auto_extract_inner( continue wrote_ok = True - # fact_ids is a list[str] in the same order as chunk_dicts. - _process_batch_results(client, chunk, fact_ids, stored_texts, log_missing=True) + _process_batch_results( + client, chunk, _per_fact_ids(fact_ids, len(chunk)), stored_texts, log_missing=True + ) state.mark_messages_processed() if wrote_ok: @@ -845,6 +846,21 @@ def _process_batch_results( logger.warning("Failed to process batch result for fact: %s", e) +def _per_fact_ids(ids, n: int) -> list: + """One entry per submitted fact: its id, or ``None`` when it was not + stored (#751). + + ``remember_batch`` returns the stored ids only, so after a skipped group + the list is shorter than the input and position no longer says which + fact an id belongs to. Its ``batch_results`` does. A plain list (a stub + client) is taken as already lined up. + """ + results = getattr(ids, "batch_results", None) + if not isinstance(results, list): + results = list(ids or []) + return list(results[:n]) + [None] * (n - len(results)) + + # --------------------------------------------------------------------------- # Write-quota refusal: keep, tell, retry (#747) # @@ -1041,28 +1057,46 @@ def _drain_entries(state: "AgentState", client, queue: PendingStoreQueue, entrie if isinstance(e, InterpreterShutdownError) or is_interpreter_shutdown_error(e): queue.put_back(to_store_entries) return 0 - retry = [] - for entry in to_store_entries: - entry["attempts"] = int(entry.get("attempts") or 0) + 1 - if entry["attempts"] < _DRAIN_MAX_ATTEMPTS: - retry.append(entry) - dropped = len(to_store_entries) - len(retry) - queue.put_back(retry) + kept, dropped = _retry_later(queue, to_store_entries) logger.warning( "pending store: queued facts not stored yet (%s); %d kept, %d dropped after %d attempts", - e, len(retry), dropped, _DRAIN_MAX_ATTEMPTS, + e, kept, dropped, _DRAIN_MAX_ATTEMPTS, ) return 0 stored = 0 - for fact, fact_id in zip(to_store, ids): + not_stored = [] + for fact, entry, fact_id in zip(to_store, to_store_entries, _per_fact_ids(ids, len(to_store))): if fact_id: record_write(client, fact_id, fact["text"], fact.get("embedding")) stored += 1 + else: + not_stored.append(entry) + if not_stored: + # Skipped by the batch writer (#751). The relay never reports a + # duplicate, so this is not proof the fact is in the vault: retry it + # (the write gate drops it if it is), up to the attempt limit. + kept, dropped = _retry_later(queue, not_stored) + logger.warning( + "pending store: %d queued fact(s) not stored; %d kept, %d dropped after %d attempts", + len(not_stored), kept, dropped, _DRAIN_MAX_ATTEMPTS, + ) logger.info("pending store: saved %d queued fact(s)", stored) return stored +def _retry_later(queue: PendingStoreQueue, entries: list[dict]) -> tuple[int, int]: + """Put *entries* back with one more attempt counted, dropping those + that reach :data:`_DRAIN_MAX_ATTEMPTS`. Returns ``(kept, dropped)``.""" + retry = [] + for entry in entries: + entry["attempts"] = int(entry.get("attempts") or 0) + 1 + if entry["attempts"] < _DRAIN_MAX_ATTEMPTS: + retry.append(entry) + queue.put_back(retry) + return len(retry), len(entries) - len(retry) + + def crystal_gate_passes(n_messages: int, n_stored_facts: int) -> bool: """Content-aware length gate for a session Crystal. diff --git a/python/src/totalreclaw/client.py b/python/src/totalreclaw/client.py index 2257b04c..6e92312e 100644 --- a/python/src/totalreclaw/client.py +++ b/python/src/totalreclaw/client.py @@ -258,6 +258,25 @@ async def _derive_smart_account_address(mnemonic: str, rpc_url: str = "https://s return address.lower() +class BatchIds(list): + """What :meth:`TotalReclaw.remember_batch` returns: the ids of the stored + facts, in input order (a plain ``list[str]``), plus ``batch_results``. + + ``batch_results`` has one entry per input fact, in input order: the fact + id, or ``None`` when that fact was not stored (its group was skipped as a + duplicate). The list itself is shorter than the input whenever a fact was + skipped, so pair ids with input facts through ``batch_results``, never by + position in the list (#751). ``RelayWriteQuotaExceeded.batch_results`` + has the same shape for a batch the quota interrupted. + """ + + def __init__(self, ids=(), batch_results=None): + super().__init__(ids) + self.batch_results: list[Optional[str]] = ( + list(batch_results) if batch_results is not None else list(self) + ) + + class TotalReclaw: """High-level TotalReclaw client with remember/recall/forget/export/status. @@ -1001,7 +1020,7 @@ async def remember_batch( self, facts: list[dict], source: str = "python-client", - ) -> list[str]: + ) -> "BatchIds": """Store N facts via byte-capped, adaptive batched UserOperations. The batch analogue of :meth:`remember`. Accepts ANY number of facts @@ -1049,12 +1068,15 @@ async def remember_batch( Returns ------- - list[str] - Pre-assigned UUID fact IDs of the STORED facts, in input order. - Subsequent ``forget`` / ``pin_fact`` / ``unpin_fact`` calls - work against these IDs immediately. (A group rejected as a - duplicate contributes no ids and no error — it is silently - skipped, matching the import path's contract.) + BatchIds + A ``list[str]`` of the pre-assigned UUID fact IDs of the STORED + facts, in input order. Subsequent ``forget`` / ``pin_fact`` / + ``unpin_fact`` calls work against these IDs immediately. (A group + rejected as a duplicate contributes no ids and no error — it is + silently skipped, matching the import path's contract.) Its + ``batch_results`` attribute has one entry per input fact: the id, + or ``None`` when that fact was skipped. Use it to match ids to + facts (#751). Raises ------ @@ -1094,7 +1116,7 @@ async def remember_batch( lsh = self._get_lsh_hasher() if needs_lsh else None wallet = self._wallet_context(chain_id) - # #747: which input facts landed, and the first write-quota refusal. + # #747/#751: which input facts landed, and the first write-quota refusal. # The adaptive helper reduces failures to strings, and a group can be # refused after an earlier group already stored (the relay refuses a # batch whole when it does not fit), so record outcomes per fact here. @@ -1136,7 +1158,7 @@ async def _store_group(group: list[dict]) -> list[str]: f"remember_batch: {len(errors)} store group(s) failed: " + " | ".join(errors) ) - return ids + return BatchIds(ids, [stored_at.get(i) for i in range(len(facts))]) async def recall( self, diff --git a/python/src/totalreclaw/imports/engine.py b/python/src/totalreclaw/imports/engine.py index 746976e9..056aaf63 100644 --- a/python/src/totalreclaw/imports/engine.py +++ b/python/src/totalreclaw/imports/engine.py @@ -98,6 +98,7 @@ def _import_concurrency() -> int: MAX_BATCH_BYTES as _MAX_BATCH_BYTES, MAX_BATCH_GROUP_COUNT as IMPORT_MAX_BATCH_SIZE, group_and_store_adaptive, + is_duplicate_rejection, store_fact_batch, ) @@ -1881,7 +1882,7 @@ async def _store_group(group: list) -> list[str]: _record_stored_conv(stored_conv_ids, fact) except Exception as e: msg = str(e) - if '409' in msg or 'duplicate' in msg or 'fingerprint' in msg: + if is_duplicate_rejection(msg): logger.debug("Skipped duplicate: %s", fact.get('text', '')[:60]) # Counted as on-chain duplicate → conversation durably stored. _record_stored_conv(stored_conv_ids, fact) diff --git a/python/src/totalreclaw/operations.py b/python/src/totalreclaw/operations.py index 426dd397..1ecb3f7a 100644 --- a/python/src/totalreclaw/operations.py +++ b/python/src/totalreclaw/operations.py @@ -8,6 +8,7 @@ import asyncio import base64 import logging +import re import uuid from dataclasses import dataclass from datetime import datetime, timezone @@ -505,6 +506,21 @@ def group_payloads_by_size(payloads: list, max_count: int, max_bytes: int): yield group +_HTTP_409 = re.compile(r"\b409\b") + + +def is_duplicate_rejection(msg: str) -> bool: + """Whether a store error message is a duplicate rejection (HTTP 409 or a + duplicate content fingerprint). + + ``409`` must be a whole word (#751): matching it anywhere also matched + the digits inside a hex hash or address in an unrelated bundler error, + and that error was then silently dropped as a duplicate. The managed + relay's ``/v1/bundler`` never returns 409. + """ + return bool(_HTTP_409.search(msg)) or "duplicate" in msg or "fingerprint" in msg + + async def _store_group_adaptive(store_fn, group: list) -> tuple[list[str], list[str]]: """Store one payload group via ``store_fn``; halve-and-retry on a simulation-size revert. @@ -532,7 +548,7 @@ async def _store_group_adaptive(store_fn, group: list) -> tuple[list[str], list[ return list(ids), [] except Exception as e: msg = str(e) - if '409' in msg or 'duplicate' in msg or 'fingerprint' in msg: + if is_duplicate_rejection(msg): logger.debug("Batch of %d facts rejected as duplicate", len(group)) return [], [] msg_l = msg.lower() diff --git a/python/tests/test_batch_id_alignment_751.py b/python/tests/test_batch_id_alignment_751.py new file mode 100644 index 00000000..d6967656 --- /dev/null +++ b/python/tests/test_batch_id_alignment_751.py @@ -0,0 +1,288 @@ +"""Batch fact ids stay with their own facts when a group is skipped (#751). + +``remember_batch`` returns the ids of the facts it stored. A group skipped as +a "duplicate" contributes none, so the list is shorter than the input and +callers that paired it with the input by position moved every later id onto +the wrong fact. In auto-extraction that tombstoned the old fact of an UPDATE +whose replacement was never written; in the #747 drain, skipped queued facts +were lost. + +On the managed relay the skip only fires by accident: the duplicate check +matched ``409`` anywhere in an error message, e.g. inside a hex hash. +""" +from __future__ import annotations + +import os +import secrets +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from totalreclaw.agent.extraction import ExtractedFact + +_SIM_REVERT = "UserOperation reverted during simulation with reason: -32500 too big" +_DUPLICATE = "HTTP 409: duplicate fingerprint" +OLD_ID = "33333333-3333-4333-8333-333333333333" + + +def _client_shell(): + """A TotalReclaw with keys, relay and chain stubbed, so the real + ``remember_batch`` runs down to a patched ``store_fact_batch``.""" + from totalreclaw.client import TotalReclaw + + tr = TotalReclaw.__new__(TotalReclaw) + tr._ensure_address = AsyncMock() + tr._ensure_registered = AsyncMock() + tr._ensure_chain_id = AsyncMock(return_value=100) + tr._wallet_context = MagicMock(return_value=MagicMock(name="wallet")) + tr._get_lsh_hasher = MagicMock(return_value=None) + tr._relay = MagicMock(name="relay") + tr._data_edge_address = None + return tr + + +def _store_first_half_skipped(): + """``store_fact_batch`` stand-in: a group of more than one fact reverts + for size (so it is halved), a group holding the first fact is rejected + as a duplicate, anything else is stored with an id named after its text.""" + calls: list[list[str]] = [] + + async def _store(facts, *a, **kw): + texts = [f["text"] for f in facts] + calls.append(texts) + if len(facts) > 1: + raise RuntimeError(_SIM_REVERT) + if texts[0].startswith("first"): + raise RuntimeError(_DUPLICATE) + return [f"id:{t}" for t in texts] + + return _store, calls + + +# --------------------------------------------------------------------------- +# 1. remember_batch reports one outcome per input fact +# --------------------------------------------------------------------------- + + +class TestRememberBatchResults: + @pytest.mark.asyncio + async def test_a_skipped_group_keeps_later_ids_with_their_facts(self): + store, calls = _store_first_half_skipped() + client = _client_shell() + with patch("totalreclaw.client.store_fact_batch", new=store): + ids = await client.remember_batch( + [{"text": "first fact"}, {"text": "second fact"}], source="t" + ) + assert calls == [["first fact", "second fact"], ["first fact"], ["second fact"]] + # Unchanged contract: the ids of the stored facts. + assert list(ids) == ["id:second fact"] + # New: which input fact each id belongs to. + assert ids.batch_results == [None, "id:second fact"] + + @pytest.mark.asyncio + async def test_full_success_lines_up_one_to_one(self): + async def store(facts, *a, **kw): + return [f"id:{f['text']}" for f in facts] + + client = _client_shell() + with patch("totalreclaw.client.store_fact_batch", new=store): + ids = await client.remember_batch([{"text": "a"}, {"text": "b"}], source="t") + assert ids == ["id:a", "id:b"] + assert ids.batch_results == ["id:a", "id:b"] + + @pytest.mark.asyncio + async def test_all_skipped_gives_no_ids_and_all_none(self): + async def store(facts, *a, **kw): + raise RuntimeError(_DUPLICATE) + + client = _client_shell() + with patch("totalreclaw.client.store_fact_batch", new=store): + ids = await client.remember_batch([{"text": "a"}, {"text": "b"}], source="t") + assert ids == [] + assert ids.batch_results == [None, None] + + +# --------------------------------------------------------------------------- +# 2. A 409 inside a hex string is not a duplicate rejection +# --------------------------------------------------------------------------- + + +class TestDuplicateClassifier: + @pytest.mark.parametrize( + "msg", + [ + "HTTP 409: duplicate fingerprint", + "409 fingerprint duplicate", + "Client error '409 Conflict' for url 'https://relay.invalid/v1/store'", + ], + ) + def test_real_duplicate_rejections_match(self, msg): + from totalreclaw.operations import is_duplicate_rejection + + assert is_duplicate_rejection(msg) + + @pytest.mark.parametrize( + "msg", + [ + "AA21 didn't pay prefund sender 0x5409ab00c0ffee0000000000000000000000beef", + "Bundler error: userOpHash 0xdead4091beef failed", + "paymaster deposit too low: 14090 wei", + ], + ) + def test_409_inside_other_text_does_not_match(self, msg): + from totalreclaw.operations import is_duplicate_rejection + + assert not is_duplicate_rejection(msg) + + @pytest.mark.asyncio + async def test_hex_409_error_is_surfaced_not_swallowed(self): + """Before #751 this error was taken for a duplicate: no ids, no error, + and the facts silently dropped.""" + + async def store(facts, *a, **kw): + raise RuntimeError("AA21 didn't pay prefund sender 0x5409ab00c0ffee") + + client = _client_shell() + with patch("totalreclaw.client.store_fact_batch", new=store): + with pytest.raises(RuntimeError, match="AA21"): + await client.remember_batch([{"text": "a"}], source="t") + + +# --------------------------------------------------------------------------- +# 3. Auto-extraction: an UPDATE whose replacement was skipped tombstones nothing +# --------------------------------------------------------------------------- + + +def _plugin_state(client): + from totalreclaw.hermes.state import PluginState + + with patch.dict(os.environ, {}, clear=True): + with patch.object(Path, "exists", return_value=False): + state = PluginState() + state._client = client + return state + + +def _run_extract(state, facts): + from totalreclaw.agent.lifecycle import auto_extract + + async def fake_extract(*a, **kw): + return facts + + async def passthrough(fs, *a, **kw): + return fs + + with patch("totalreclaw.agent.lifecycle.extract_facts_llm", new=fake_extract), \ + patch("totalreclaw.agent.lifecycle.detect_and_resolve_contradictions", new=passthrough), \ + patch("totalreclaw.agent.lifecycle._fetch_recent_memories", return_value=[]), \ + patch("totalreclaw.embedding.get_embedding", return_value=None): + state.add_message("user", "My home city changed") + state.add_message("assistant", "Noted") + return auto_extract(state) + + +class TestAutoExtractAlignment: + def test_skipped_update_does_not_tombstone_and_ids_stay_put(self): + shell = _client_shell() + client = MagicMock() + client.recall = AsyncMock(return_value=[]) + client.forget = AsyncMock(return_value=True) + client.get_fact_pin_status = AsyncMock(return_value=False) + client.remember_batch = shell.remember_batch # the real method + state = _plugin_state(client) + facts = [ + ExtractedFact(text="first: home city is Porto", type="claim", importance=8, + action="UPDATE", existing_fact_id=OLD_ID, source="user"), + ExtractedFact(text="second: likes espresso", type="preference", importance=7, + action="ADD", source="user"), + ] + store, _ = _store_first_half_skipped() + with patch("totalreclaw.client.store_fact_batch", new=store), \ + patch("totalreclaw.agent.lifecycle.record_write") as rec: + stored = _run_extract(state, facts) + + # The UPDATE's replacement was not written: its old fact must stay. + client.forget.assert_not_awaited() + assert stored == ["second: likes espresso"] + assert [(c.args[1], c.args[2]) for c in rec.call_args_list] == [ + ("id:second: likes espresso", "second: likes espresso") + ] + + +# --------------------------------------------------------------------------- +# 4. The #747 drain: a skipped queued fact goes back to the queue +# --------------------------------------------------------------------------- + + +@pytest.fixture +def state_dir(tmp_path, monkeypatch) -> Path: + monkeypatch.setenv("TOTALRECLAW_STATE_DIR", str(tmp_path)) + return tmp_path + + +class _DrainClient: + resolved_chain_id = 100 + eoa_address = "0x000000000000000000000000000000000000cafe" + + def __init__(self): + from totalreclaw.agent.pending_store import PendingStoreQueue + + self._queue = PendingStoreQueue.for_owner(self.eoa_address, secrets.token_bytes(32)) + shell = _client_shell() + self.remember_batch = shell.remember_batch # the real method + + @property + def pending_store(self): + return self._queue + + async def fetch_recent_facts(self, limit=10): + return [] + + async def recall(self, *a, **kw): + return [] + + +class TestDrainAlignment: + def _drain(self, client, store): + from totalreclaw.agent.lifecycle import drain_pending_stores + + state = MagicMock() + with patch("totalreclaw.client.store_fact_batch", new=store), \ + patch("totalreclaw.embedding.get_embedding", return_value=None), \ + patch("totalreclaw.agent.lifecycle.record_write") as rec: + saved = drain_pending_stores(state, client) + return saved, rec + + def test_skipped_fact_is_kept_and_ids_stay_with_their_facts(self, state_dir): + client = _DrainClient() + client.pending_store.add( + [{"text": "first queued fact about trains"}, {"text": "second queued fact about boats"}], + reason="write_quota", + ) + store, _ = _store_first_half_skipped() + saved, rec = self._drain(client, store) + + assert saved == 1 + assert [(c.args[1], c.args[2]) for c in rec.call_args_list] == [ + ("id:second queued fact about boats", "second queued fact about boats") + ] + left = client.pending_store.take(10) + assert [e["fact"]["text"] for e in left] == ["first queued fact about trains"] + assert left[0]["attempts"] == 1 + + def test_skipped_fact_is_dropped_after_the_attempt_limit(self, state_dir): + from totalreclaw.agent.lifecycle import _DRAIN_MAX_ATTEMPTS + + client = _DrainClient() + client.pending_store.add([{"text": "first queued fact about trains"}], reason="write_quota") + entries = client.pending_store.take(10) + entries[0]["attempts"] = _DRAIN_MAX_ATTEMPTS - 1 + client.pending_store.put_back(entries) + + async def store(facts, *a, **kw): + raise RuntimeError(_DUPLICATE) + + saved, _ = self._drain(client, store) + assert saved == 0 + assert client.pending_store.count() == 0