From 2541dbc3adff9cacad1c0e59dee1bb25a4e51c98 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 19:52:22 +0200 Subject: [PATCH 01/42] fix(04-01): internal error tier, NULL-safe import dedup, 4x KNN oversample - WR-02: add MemoryError::Internal; all six writer-lock sites map poisoning to Internal("writer lock poisoned") -> REST 500 / MCP internal_error - delete MemoryError::NotFound (zero producers) and the now-unconstructable ApiError::NotFound; not-found stays data (forget's Ok(false)) - CR-01: Store::exists takes Option<&str> and probes with NULL-safe `source IS ?1`; import no longer coerces None to "" (regression test) - WR-01 partial: MAX_KNN_K = MAX_LIMIT * 4 keeps 4x oversample at every limit - mapper tier tests on both transports; exhaustive arms, no catch-alls Co-Authored-By: Claude Opus 5.5 --- crates/agent-memory-core/src/domain.rs | 8 ++- crates/agent-memory-core/src/service.rs | 5 +- crates/agent-memory-core/src/store/mod.rs | 4 +- crates/agent-memory-core/src/store/sqlite.rs | 66 +++++++++++++++----- crates/agent-memory-core/tests/import.rs | 35 ++++++++++- crates/agent-memory/src/mcp.rs | 20 +++--- crates/agent-memory/src/rest/handlers.rs | 31 +++++++-- crates/agent-memory/src/rest/mod.rs | 9 ++- 8 files changed, 142 insertions(+), 36 deletions(-) diff --git a/crates/agent-memory-core/src/domain.rs b/crates/agent-memory-core/src/domain.rs index f324798..04d2575 100644 --- a/crates/agent-memory-core/src/domain.rs +++ b/crates/agent-memory-core/src/domain.rs @@ -155,8 +155,12 @@ pub enum MemoryError { #[error("migration error: {0}")] Migration(#[from] rusqlite_migration::Error), - #[error("memory not found")] - NotFound, + /// A server-side fault with no client-input cause — e.g. a poisoned writer + /// mutex after a panic while it was held (WR-02). Always the internal tier + /// (REST 500 / MCP `internal_error`), never a not-found: not-found is data + /// (`Ok(false)` / `Ok(None)`), not an error variant. + #[error("internal error: {0}")] + Internal(String), } /// Validate an optional `limit` against the D-01 bounds. `None` always passes diff --git a/crates/agent-memory-core/src/service.rs b/crates/agent-memory-core/src/service.rs index 013be0f..291ffb7 100644 --- a/crates/agent-memory-core/src/service.rs +++ b/crates/agent-memory-core/src/service.rs @@ -282,8 +282,9 @@ impl MemoryService { let mut fresh = Vec::new(); let mut duplicates = 0usize; for draft in drafts { - let source = draft.source.as_deref().unwrap_or(""); - if store.exists(source, draft.mem_type, &draft.content)? { + // Pass the source through as-is: a `None` must probe for + // NULL, exactly what the insert stores (CR-01). + if store.exists(draft.source.as_deref(), draft.mem_type, &draft.content)? { duplicates += 1; } else { fresh.push(draft); diff --git a/crates/agent-memory-core/src/store/mod.rs b/crates/agent-memory-core/src/store/mod.rs index 979fc0e..f772f01 100644 --- a/crates/agent-memory-core/src/store/mod.rs +++ b/crates/agent-memory-core/src/store/mod.rs @@ -50,9 +50,11 @@ pub trait Store: Send + Sync { /// The INTEROP-01 idempotency probe: does a row with exactly this /// `(source, mem_type, content)` key already exist? `MemoryService::import` /// checks it before every insert so re-importing the same file is a no-op. + /// `source` is `Option` and compared NULL-safely: a `None` source matches a + /// stored NULL source, never an empty string (CR-01). fn exists( &self, - source: &str, + source: Option<&str>, mem_type: MemoryType, content: &str, ) -> Result; diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index b8171c1..4206de2 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -28,11 +28,14 @@ use crate::store::{migrations::migrations, Store}; /// the semantic candidate set. pub const DEFAULT_SEARCH_LIMIT: i64 = 50; -/// Hard cap on the KNN oversample size `k` (T-02-03: a huge `limit` must not -/// turn into an unbounded vector scan). Defined via the D-01 validation bound -/// so a valid limit can never exceed what the KNN leg honors (D-03, single -/// source of truth). -const MAX_KNN_K: i64 = crate::domain::MAX_LIMIT; +/// Hard cap on the KNN/keyword candidate oversample size `k` (T-02-03: a huge +/// `limit` must not turn into an unbounded vector scan). Defined as 4x the D-01 +/// validation bound so the `limit * 4` oversample holds at EVERY valid limit, +/// including `MAX_LIMIT` itself (03-REVIEW WR-01: at `MAX_LIMIT` the old cap +/// collapsed the oversample factor to 1). Still derived from the single source +/// of truth (D-03). This is a candidate-count cap only — it is unrelated to the +/// RRF smoothing constant (`k = 60`) used by hybrid fusion. +const MAX_KNN_K: i64 = crate::domain::MAX_LIMIT * 4; /// Register the sqlite-vec `vec0` extension process-globally. Idempotent: a /// `OnceLock` captures the FIRST result (success or error) so repeated calls @@ -223,7 +226,10 @@ impl Store for SqliteStore { // ONE writer transaction covers the memories row, the vector row, and // the status flag, so `vec_memories` can never hold a vector for a row // that failed to insert (T-02-05). - let mut conn = self.writer.lock().map_err(|_| MemoryError::NotFound)?; + let mut conn = self + .writer + .lock() + .map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; let tx = conn.transaction()?; tx.execute( "INSERT INTO memories \ @@ -303,7 +309,10 @@ impl Store for SqliteStore { } fn insert_embedding(&self, memory_id: i64, embedding: Vec) -> Result<(), MemoryError> { - let mut conn = self.writer.lock().map_err(|_| MemoryError::NotFound)?; + let mut conn = self + .writer + .lock() + .map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; let tx = conn.transaction()?; let blob: &[u8] = bytemuck::cast_slice(&embedding); // DELETE + INSERT instead of INSERT OR REPLACE: conflict-resolution @@ -343,17 +352,20 @@ impl Store for SqliteStore { fn exists( &self, - source: &str, + source: Option<&str>, mem_type: MemoryType, content: &str, ) -> Result { let conn = self.reads.get().map_err(MemoryError::Pool)?; // The INTEROP-01 dedup probe: all three key values bound, never - // formatted into the SQL (T-02-20). + // formatted into the SQL (T-02-20). `source IS ?1` is SQLite's + // NULL-safe equality: a `None` source binds NULL and matches the NULL + // the insert stored (`source = ?1` would be NULL = NULL → never true, + // which made every source-less re-import a duplicate — CR-01). let found = conn .query_row( "SELECT 1 FROM memories \ - WHERE source = ?1 AND mem_type = ?2 AND content = ?3 LIMIT 1", + WHERE source IS ?1 AND mem_type = ?2 AND content = ?3 LIMIT 1", params![source, mem_type.as_wire_str(), content], |_| Ok(()), ) @@ -464,7 +476,10 @@ impl Store for SqliteStore { } fn forget(&self, id: i64) -> Result { - let mut conn = self.writer.lock().map_err(|_| MemoryError::NotFound)?; + let mut conn = self + .writer + .lock() + .map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; let tx = conn.transaction()?; // Parameterized DELETEs. The FTS5 delete trigger syncs memories_fts, // but that is true for FTS ONLY — vec0 virtual tables ignore triggers, @@ -480,7 +495,10 @@ impl Store for SqliteStore { if ids.is_empty() { return Ok(()); } - let conn = self.writer.lock().map_err(|_| MemoryError::NotFound)?; + let conn = self + .writer + .lock() + .map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; // Build a parameterized IN-list (?2, ?3, …); ?1 is `now`. The ids are // bound, never string-formatted into the SQL (T-02-01). let placeholders: String = (0..ids.len()) @@ -501,7 +519,10 @@ impl Store for SqliteStore { } fn sweep_expired(&self, now: i64) -> Result { - let mut conn = self.writer.lock().map_err(|_| MemoryError::NotFound)?; + let mut conn = self + .writer + .lock() + .map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; let tx = conn.transaction()?; // The ONLY memories delete the sweep performs (STORE-04). Bound // predicate: NULL `expires_at` rows are never matched, so a no-TTL @@ -522,7 +543,10 @@ impl Store for SqliteStore { } fn materialize_decay(&self, now: i64, cfg: &DecayConfig) -> Result { - let conn = self.writer.lock().map_err(|_| MemoryError::NotFound)?; + let conn = self + .writer + .lock() + .map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; let pinned_hl = cfg.half_life_secs * PINNED_HALF_LIFE_MULTIPLIER; // UPDATE-only: recompute decay_score inline from last_accessed at `now`, // using the SAME exp/half-life math as the on-read search blend so the @@ -540,3 +564,17 @@ impl Store for SqliteStore { Ok(updated) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn max_knn_k_keeps_four_x_oversample_at_max_limit() { + // WR-01: the oversample must stay 4x even at the largest valid limit. + assert_eq!(MAX_KNN_K, crate::domain::MAX_LIMIT * 4); + assert_eq!(MAX_KNN_K, 800); + let k_at_max = (crate::domain::MAX_LIMIT * 4).min(MAX_KNN_K); + assert_eq!(k_at_max, crate::domain::MAX_LIMIT * 4); + } +} diff --git a/crates/agent-memory-core/tests/import.rs b/crates/agent-memory-core/tests/import.rs index 13e58a9..022eedf 100644 --- a/crates/agent-memory-core/tests/import.rs +++ b/crates/agent-memory-core/tests/import.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use agent_memory_core::clock::TestClock; use agent_memory_core::decay::DecayConfig; -use agent_memory_core::domain::MemoryType; +use agent_memory_core::domain::{MemoryType, NewMemory}; use agent_memory_core::embed::FakeEmbedder; use agent_memory_core::import::gsd_state::parse_gsd_state; use agent_memory_core::service::{ListArgs, MemoryService, SearchArgs, SearchMode}; @@ -97,3 +97,36 @@ async fn import_loads_typed_searchable_memories_idempotently() { assert_eq!(second.imported, 0, "re-run must import nothing"); assert_eq!(second.skipped_duplicates, first.imported); } + +/// CR-01 regression (03-REVIEW): a draft with `source: None` persists its +/// source as SQL NULL, so the dedup probe must use NULL-safe equality — before +/// the fix the probe coerced `None` to `""`, `NULL = ''` never matched, and +/// every re-import inserted a duplicate row. +#[tokio::test] +async fn reimporting_a_sourceless_draft_is_idempotent() { + let dir = tempfile::tempdir().expect("tempdir"); + let service = service_for(&dir.path().join("memory.db")); + + let draft = || NewMemory { + content: "sourceless decision survives re-import exactly once".to_string(), + mem_type: MemoryType::Decision, + tags: vec![], + source: None, + scope: None, + ttl_secs: None, + }; + + let first = service.import(vec![draft()]).await.expect("first import"); + assert_eq!(first.imported, 1); + assert_eq!(first.skipped_duplicates, 0); + + let second = service.import(vec![draft()]).await.expect("second import"); + assert_eq!(second.imported, 0, "re-run must import nothing"); + assert_eq!( + second.skipped_duplicates, 1, + "the source:None draft must be recognised as a duplicate" + ); + + let rows = service.list(ListArgs::default()).await.expect("list").len(); + assert_eq!(rows, 1, "exactly one row after two imports"); +} diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index 4531b9b..9db4698 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -23,12 +23,13 @@ use agent_memory_core::service::{ /// (closes review WR-04): client input (`InvalidType`, `InvalidQuery`, /// `InvalidArgument`) reports /// as `invalid_params`; internal failures (`Sqlite`, `Pool`, `Join`, -/// `Migration`, `NotFound`) report as `internal_error`, so agents never enter +/// `Migration`, `Internal`) report as `internal_error`, so agents never enter /// argument-repair loops over a DB fault (T-02G-03). /// -/// `NotFound` as an *error* only arises from internal store conditions (e.g. a -/// poisoned writer mutex) — MCP's clean not-found is the `Ok(false)` forget -/// result, unchanged. +/// `Internal` covers server faults with no client cause, e.g. a poisoned writer +/// mutex (WR-02). MCP's clean not-found is data — the `Ok(false)` forget +/// result — never an error variant. No catch-all arm: a new `MemoryError` +/// variant must be classified here at compile time. fn map_mcp_error(e: MemoryError) -> McpError { match &e { MemoryError::InvalidType(_) @@ -38,7 +39,7 @@ fn map_mcp_error(e: MemoryError) -> McpError { | MemoryError::Pool(_) | MemoryError::Join(_) | MemoryError::Migration(_) - | MemoryError::NotFound => McpError::internal_error(e.to_string(), None), + | MemoryError::Internal(_) => McpError::internal_error(e.to_string(), None), } } @@ -297,9 +298,14 @@ mod tests { "an out-of-bounds argument is client input → invalid_params" ); assert_eq!( - map_mcp_error(MemoryError::NotFound).code, + map_mcp_error(MemoryError::Internal("writer lock poisoned".into())).code, internal_error_code, - "NotFound-as-error only arises from internal store conditions → internal_error" + "a poisoned writer lock is a server fault → internal_error (WR-02)" + ); + assert_ne!( + map_mcp_error(MemoryError::Internal("writer lock poisoned".into())).code, + invalid_params_code, + "a server fault must never read as client input" ); } } diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index 7ab78e7..3dca294 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -82,18 +82,21 @@ pub struct ListQuery { /// Two-tier [`MemoryError`] → HTTP mapping, mirroring `mcp.rs`'s /// `invalid_params` / `internal_error` split: `InvalidType`, `InvalidQuery`, -/// and `InvalidArgument` are client input (400), `NotFound` is a missing -/// resource (404), everything else is 500. +/// and `InvalidArgument` are client input (400); everything else — including +/// `Internal` (e.g. a poisoned writer lock, WR-02) — is 500. A missing resource +/// is never an error variant: handlers build their 404 from not-found *data* +/// (forget's `Ok(false)`). No catch-all arm: a new `MemoryError` variant must +/// be classified here at compile time. fn map_memory_error(e: MemoryError) -> ApiError { match &e { MemoryError::InvalidType(_) | MemoryError::InvalidQuery(_) | MemoryError::InvalidArgument(_) => ApiError::BadRequest(e.to_string()), - MemoryError::NotFound => ApiError::NotFound, MemoryError::Sqlite(_) | MemoryError::Pool(_) | MemoryError::Join(_) - | MemoryError::Migration(_) => ApiError::Internal(e.to_string()), + | MemoryError::Migration(_) + | MemoryError::Internal(_) => ApiError::Internal(e.to_string()), } } @@ -571,4 +574,24 @@ mod tests { assert_eq!(body["status"], "ok"); assert_eq!(body["embedder"], "unreachable"); } + + #[test] + fn map_memory_error_puts_internal_faults_in_the_500_tier() { + // WR-02: a poisoned writer lock is a server fault — 500, never 404. + let err = map_memory_error(MemoryError::Internal("writer lock poisoned".into())); + assert!( + matches!(&err, ApiError::Internal(msg) if msg.contains("writer lock poisoned")), + "Internal must map to ApiError::Internal, got {err:?}" + ); + assert_eq!( + err.into_response().status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + + // The client tier is unchanged. + let bad = map_memory_error(MemoryError::InvalidArgument( + "limit must be between 1 and 200 (got 0)".into(), + )); + assert_eq!(bad.into_response().status(), StatusCode::BAD_REQUEST); + } } diff --git a/crates/agent-memory/src/rest/mod.rs b/crates/agent-memory/src/rest/mod.rs index cfaab5d..88a669f 100644 --- a/crates/agent-memory/src/rest/mod.rs +++ b/crates/agent-memory/src/rest/mod.rs @@ -26,14 +26,14 @@ use crate::mcp::AppState; /// Typed HTTP error for the REST handlers — the two-tier mapping mirrors /// `mcp.rs`'s `invalid_params` / `internal_error` split: bad input is a 400, -/// a missing resource is a 404, everything else is a 500. The body is always -/// `{"error": "..."}` so bad input can never surface as a 500. +/// everything else is a 500. The body is always `{"error": "..."}` so bad input +/// can never surface as a 500. A missing resource is not an error here: +/// handlers return their 404 body from not-found data (e.g. forget's +/// `Ok(false)`), so a server fault can never be mis-tiered as a 404 (WR-02). #[derive(Debug)] pub enum ApiError { /// Invalid client input (e.g. an unknown memory type) → 400. BadRequest(String), - /// The referenced resource does not exist → 404. - NotFound, /// An unexpected server-side failure → 500. Internal(String), } @@ -42,7 +42,6 @@ impl IntoResponse for ApiError { fn into_response(self) -> Response { let (status, message) = match self { ApiError::BadRequest(message) => (StatusCode::BAD_REQUEST, message), - ApiError::NotFound => (StatusCode::NOT_FOUND, "not found".to_string()), ApiError::Internal(message) => (StatusCode::INTERNAL_SERVER_ERROR, message), }; (status, Json(serde_json::json!({ "error": message }))).into_response() From 46a63fa810a935a58d8b4498579862487fdeec42 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 19:56:53 +0200 Subject: [PATCH 02/42] feat(04-01): migration 0003 memory_links with backup + newer-schema refusal - sql/0003_relations.sql: directed memory_links (PK from/to/kind, FK ON DELETE CASCADE x2, CHECK from!=to, WITHOUT ROWID, idx_links_to) - migrations.rs: append 0003, LATEST_SCHEMA_VERSION = 3 (+ chain test) - SqliteStore::open: user_version probe before any write -> SchemaTooNew (names both versions + brew upgrade hint, DB byte-untouched); sibling online backup {db}.backup-pre-0003-{YYYYMMDD} before migrating a non-empty older DB, one per schema version; backup failure aborts open - MemoryError::SchemaTooNew classified internal (startup-only) in both exhaustive mappers - rusqlite gains only the `backup` feature; 0.39 pin unchanged (AR-03-01) - tests/migration_hygiene.rs + committed binary fixture v0.0.1.db (generated once from tag-identical 0001+0002): freeze, backup, retention, fresh-no-backup, too-new refusal, backup-failure abort Co-Authored-By: Claude Opus 5.5 --- Cargo.toml | 5 +- .../agent-memory-core/sql/0003_relations.sql | 25 ++ crates/agent-memory-core/src/domain.rs | 16 + .../agent-memory-core/src/store/migrations.rs | 32 +- crates/agent-memory-core/src/store/sqlite.rs | 113 +++++- .../tests/fixtures/v0.0.1.db | Bin 0 -> 77824 bytes .../tests/migration_hygiene.rs | 357 ++++++++++++++++++ crates/agent-memory/src/mcp.rs | 4 + crates/agent-memory/src/rest/handlers.rs | 4 + 9 files changed, 551 insertions(+), 5 deletions(-) create mode 100644 crates/agent-memory-core/sql/0003_relations.sql create mode 100644 crates/agent-memory-core/tests/fixtures/v0.0.1.db create mode 100644 crates/agent-memory-core/tests/migration_hygiene.rs diff --git a/Cargo.toml b/Cargo.toml index 94d1cf9..8dd7bc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,10 @@ rmcp = { version = "1.8", features = ["server", "transport-io", "macros"] } # 0.40 pulls libsqlite3-sys 0.38, producing a `links = "sqlite3"` conflict. # `functions` enables create_scalar_function — the bundled SQLite is not built # with SQLITE_ENABLE_MATH_FUNCTIONS, so we register `exp` for the decay blend. -rusqlite = { version = "0.39", features = ["bundled", "functions"] } +# `backup` exposes the SQLite Online Backup API (rusqlite::backup) for the +# pre-migration backup (STORE-05 / D-11). Feature flag only — the 0.39 pin and +# the libsqlite3-sys 0.37 triangle are unchanged (AR-03-01). +rusqlite = { version = "0.39", features = ["bundled", "functions", "backup"] } # rusqlite_migration 2.5 tracks rusqlite ^0.39 (2.6 jumps to ^0.40); pinning 2.5 # keeps the whole graph on one `libsqlite3-sys` (0.37) — see rusqlite note above. rusqlite_migration = "2.5" diff --git a/crates/agent-memory-core/sql/0003_relations.sql b/crates/agent-memory-core/sql/0003_relations.sql new file mode 100644 index 0000000..e7f1b11 --- /dev/null +++ b/crates/agent-memory-core/sql/0003_relations.sql @@ -0,0 +1,25 @@ +-- 0003_relations.sql — flat typed relations between memories (Phase 4, MCP-07). +-- Run via rusqlite_migration `M::up`. Connection-level PRAGMAs (WAL, synchronous, +-- foreign_keys, busy_timeout) are applied per-connection in sqlite.rs, NOT here — +-- `foreign_keys = ON` there is what makes the cascading FK deletes below fire. +-- +-- APPEND-ONLY: 0001/0002 shipped in v0.0.1 and are FROZEN (STORE-05). Schema +-- changes always land as a new numbered migration; the committed fixture test in +-- tests/migration_hygiene.rs turns any edit of a shipped migration red. +-- +-- Edges are stored DIRECTED (from_id -> to_id) so `supersedes` / `caused_by` keep +-- their meaning; 1-hop expansion reads them in both directions (D-03). One row +-- per edge — no mirrored reverse rows. + +CREATE TABLE memory_links ( + from_id INTEGER NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + to_id INTEGER NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + kind TEXT NOT NULL, -- relates_to|supersedes|caused_by (closed enum, validated at the service seam) + created_at INTEGER NOT NULL, -- UTC unix epoch (i64), injected clock; most-recent edges win (D-07) + PRIMARY KEY (from_id, to_id, kind), -- one edge per (from, to, kind): duplicate link is a no-op (D-02) + CHECK (from_id != to_id) -- belt-and-braces; the service rejects self-links first (D-02) +) WITHOUT ROWID; + +-- Reverse-direction expansion (edges pointing AT a memory). The primary key +-- already covers lookups by from_id. +CREATE INDEX idx_links_to ON memory_links(to_id); diff --git a/crates/agent-memory-core/src/domain.rs b/crates/agent-memory-core/src/domain.rs index 04d2575..9fcb14e 100644 --- a/crates/agent-memory-core/src/domain.rs +++ b/crates/agent-memory-core/src/domain.rs @@ -161,6 +161,22 @@ pub enum MemoryError { /// (`Ok(false)` / `Ok(None)`), not an error variant. #[error("internal error: {0}")] Internal(String), + + /// The database was written by a NEWER agent-memory: its schema version is + /// ahead of what this binary can migrate to (D-13). Startup-only — raised by + /// `SqliteStore::open` before any write and before serving, so it never + /// crosses a transport; the binary reports it on stderr and exits. The + /// message names only the two version integers and the public upgrade + /// command (T-04-05). + #[error( + "database schema is version {db_version} but this binary supports up to version \ + {binary_version} — upgrade agent-memory (brew upgrade unityinflow/tap/agent-memory) \ + or point --db at a different file" + )] + SchemaTooNew { + db_version: i64, + binary_version: i64, + }, } /// Validate an optional `limit` against the D-01 bounds. `None` always passes diff --git a/crates/agent-memory-core/src/store/migrations.rs b/crates/agent-memory-core/src/store/migrations.rs index 9732862..a16f924 100644 --- a/crates/agent-memory-core/src/store/migrations.rs +++ b/crates/agent-memory-core/src/store/migrations.rs @@ -1,15 +1,27 @@ //! Schema versioning via `rusqlite_migration` (SQLite `user_version`). //! -//! Phase 2 appends further `M::up(...)` steps (e.g. an embedding column); the -//! ordering and idempotency are handled by the migration library. +//! The chain is APPEND-ONLY: 0001/0002 shipped in v0.0.1 and are frozen +//! (STORE-05, proven by the committed-fixture test in +//! `tests/migration_hygiene.rs`). New schema lands as a new `M::up(...)` step, +//! with [`LATEST_SCHEMA_VERSION`] bumped alongside it. There are no `.down()` +//! migrations — the pre-migration backup in `SqliteStore::open` is the rollback +//! story (D-11/D-12). Ordering and idempotency are handled by the library. use rusqlite_migration::{Migrations, M}; +/// The schema version (`PRAGMA user_version`) this binary migrates to — the +/// number of `M::up` steps in [`migrations`]. `SqliteStore::open` probes the +/// DB's `user_version` against it BEFORE migrating: a DB ahead of it is refused +/// with an upgrade hint (D-13), and a non-empty DB behind it is backed up first +/// (D-11/D-12). Bump it in the same commit that appends a migration. +pub const LATEST_SCHEMA_VERSION: i64 = 3; + /// The ordered set of forward migrations. Run on the writer connection at open. pub fn migrations() -> Migrations<'static> { Migrations::new(vec![ M::up(include_str!("../../sql/0001_init.sql")), M::up(include_str!("../../sql/0002_embeddings.sql")), + M::up(include_str!("../../sql/0003_relations.sql")), ]) } @@ -29,4 +41,20 @@ mod tests { .validate() .expect("migration set should validate"); } + + #[test] + fn latest_schema_version_matches_the_migration_chain() { + // The D-13 probe and the D-11 backup trigger both compare against this + // const, so it must equal what `to_latest` actually writes. + crate::store::sqlite::register_vec_extension() + .expect("sqlite-vec extension should register"); + let mut conn = rusqlite::Connection::open_in_memory().expect("in-memory db"); + migrations() + .to_latest(&mut conn) + .expect("migrate to latest"); + let version: i64 = conn + .pragma_query_value(None, "user_version", |row| row.get(0)) + .expect("read user_version"); + assert_eq!(version, LATEST_SCHEMA_VERSION); + } } diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index 4206de2..b562ea2 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -9,18 +9,21 @@ //! All SQL is parameterized; user input is never formatted into a query string //! (threat T-01-01). -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Mutex, OnceLock}; +use std::time::Duration; use r2d2::Pool; use r2d2_sqlite::SqliteConnectionManager; +use rusqlite::backup::Backup; use rusqlite::{params, Connection, OptionalExtension}; use rusqlite_migration::Migrations; use crate::decay::{DecayConfig, RankWeights, PINNED_HALF_LIFE_MULTIPLIER}; use crate::domain::{MemoryError, MemoryType, MemoryView, NewMemory}; use crate::service::{ListArgs, SearchArgs}; -use crate::store::{migrations::migrations, Store}; +use crate::store::migrations::{migrations, LATEST_SCHEMA_VERSION}; +use crate::store::Store; /// Default cap on returned search rows when the caller omits `limit` (T-02-04). /// `pub` because it is part of the documented API contract (D-03: the README @@ -108,6 +111,80 @@ fn prepare_connection(conn: &Connection) -> Result<(), rusqlite::Error> { register_functions(conn) } +/// Today's UTC date as `YYYYMMDD`, for the backup file name. UTC only — `Local` +/// time must never be reintroduced (chrono is trimmed to the `now` feature for +/// the darwin cross-link; `Datelike` needs no extra feature). +fn utc_yyyymmdd() -> String { + use chrono::Datelike; + let today = chrono::Utc::now().date_naive(); + format!("{:04}{:02}{:02}", today.year(), today.month(), today.day()) +} + +/// Write the pre-migration backup of `src` (the open DB at `db_path`) to the +/// sibling file `{db}.backup-pre-{NNNN}-{YYYYMMDD}`, where `NNNN` is the +/// zero-padded schema version about to be migrated to (D-11). Generic over the +/// target version, so every future migration inherits it (D-12). +/// +/// Retention is one backup per target schema version: every existing +/// `{db}.backup-pre-{NNNN}-*` entry is removed first; other versions' backups +/// are never touched. The copy uses the SQLite Online Backup API, which is +/// WAL-correct by construction (a raw file copy of a live WAL database is not). +/// The backup sits beside the DB, in the same directory and trust domain — +/// never a shared temp dir (T-04-03). +/// +/// Any failure is returned as an error and ABORTS the open: a machine that +/// cannot write a backup must not run an unattended migration (T-04-02). +fn pre_migration_backup(src: &Connection, db_path: &Path) -> Result { + let fail = |what: String| { + MemoryError::Internal(format!( + "pre-migration backup failed ({what}); refusing to migrate {}", + db_path.display() + )) + }; + + let file_name = db_path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| fail("database path has no UTF-8 file name".to_string()))?; + let dir = match db_path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent.to_path_buf(), + _ => PathBuf::from("."), + }; + let prefix = format!("{file_name}.backup-pre-{LATEST_SCHEMA_VERSION:04}-"); + + let entries = std::fs::read_dir(&dir) + .map_err(|e| fail(format!("cannot read directory {}: {e}", dir.display())))?; + for entry in entries { + let entry = entry.map_err(|e| fail(format!("cannot read directory entry: {e}")))?; + let matches = entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(&prefix)); + if matches { + let stale = entry.path(); + std::fs::remove_file(&stale).map_err(|e| { + fail(format!( + "cannot remove previous backup {}: {e}", + stale.display() + )) + })?; + } + } + + let backup_path = dir.join(format!("{prefix}{}", utc_yyyymmdd())); + let mut dst = Connection::open(&backup_path) + .map_err(|e| fail(format!("cannot create {}: {e}", backup_path.display())))?; + Backup::new(src, &mut dst) + .and_then(|backup| backup.run_to_completion(64, Duration::from_millis(50), None)) + .map_err(|e| fail(format!("copy to {} failed: {e}", backup_path.display())))?; + tracing::info!( + backup = %backup_path.display(), + target_schema_version = LATEST_SCHEMA_VERSION, + "pre-migration backup written" + ); + Ok(backup_path) +} + /// A pool customizer that runs [`apply_pragmas`] on each pooled read connection. #[derive(Debug)] struct PragmaCustomizer; @@ -127,6 +204,20 @@ pub struct SqliteStore { impl SqliteStore { /// Open (creating if absent) the database at `path`, run migrations on the /// writer, and build the read pool. Confirms FTS5 is compiled in. + /// + /// Migration hygiene (STORE-05), in this order — the probe and the backup + /// MUST precede `to_latest` (RESEARCH Pitfall 8): + /// + /// 1. Probe `PRAGMA user_version`. A DB AHEAD of [`LATEST_SCHEMA_VERSION`] + /// is refused with [`MemoryError::SchemaTooNew`] (D-13). The probe runs + /// before even the per-connection PRAGMAs, because switching a non-WAL + /// file to WAL rewrites its header — a refused DB stays byte-for-byte + /// untouched. + /// 2. If migrations are pending on a non-empty DB (`0 < user_version < + /// LATEST`, D-12), write a sibling online backup first (D-11). A backup + /// failure aborts the open; no migration runs. A brand-new file + /// (`user_version == 0`) has nothing to back up. + /// 3. Migrate to latest. pub fn open(path: &Path) -> Result { // vec0 registration MUST precede Connection::open: auto-extensions only // apply to connections opened after registration (this also covers every @@ -134,8 +225,26 @@ impl SqliteStore { register_vec_extension()?; let mut writer = Connection::open(path)?; + + // (1) D-13 newer-schema refusal — read-only, before any write. + let db_version: i64 = writer.pragma_query_value(None, "user_version", |row| row.get(0))?; + if db_version > LATEST_SCHEMA_VERSION { + return Err(MemoryError::SchemaTooNew { + db_version, + binary_version: LATEST_SCHEMA_VERSION, + }); + } + prepare_connection(&writer)?; + // (2) D-11/D-12 pre-migration backup. open() runs before serving, so + // the writer is the only connection and can be the backup source. + if db_version > 0 && db_version < LATEST_SCHEMA_VERSION { + pre_migration_backup(&writer, path)?; + } + + // (3) Migrate. rusqlite_migration's own DatabaseTooFarAhead check stays + // as a backstop should the probe above ever be bypassed. let migrations: Migrations<'static> = migrations(); migrations.to_latest(&mut writer)?; diff --git a/crates/agent-memory-core/tests/fixtures/v0.0.1.db b/crates/agent-memory-core/tests/fixtures/v0.0.1.db new file mode 100644 index 0000000000000000000000000000000000000000..6dade3a1d3295f3f460020757e4b0bdbb916b086 GIT binary patch literal 77824 zcmeI*Pi))P9S3m!k@d&_m_PN@BJlR$u8@Stj*ZMogTO!)n@Q9vpT5yX>rROpqkuEye^vkmzxe9^vu|y@-Wx=(k7@ zzRqEbB+UHy-(#cKgtYKWQT)@$CnG-_IWhcrcrx{i)Yp?ACi968;=dnytehX;8UOp( zugCP!-->5PZ_>)cdyD<<*AnT839<7HS6{1}4b#>6F1To8Y`_XFAwK3>3I6$geX`|!`yZr)QP34 zw_V$NU*#oN=lJu<&TFqE(yA(U=Dn6|nKjC7*e%O+s`SRSTh-dVcJsb7J9}8(u}I6O z4qJ69+p-^8MkElwCWl>GE?;6{6uBSW6%*;nN%5y+fs)^@uzgab&M-9)qYC<#Vo6i4 zsFg^az3k~}<~8XeTUa3VAo_LJnemc6WlFBPJDaAzmG?=6?acQ|@?TMTOX^~ArC2WQ zUN2~xS}Ecd=Yr(JBRMpjS=JZn|dE4N8nw(=`e@7fM?9-8LnMbcQlj0Yce>$W>b08bIMtfn z=AT~J-@-1^ZTqah&(>%=DxpfF6fJw#j`~rUtJ)%KH?3`EZrZhbENh+nmLF1GcigI9 zHMQr9mt$^kS}oe+qwV2s8(hM!u)vx&)3jxlUD3)Fb$XWClxJpR%`^<+uR5;oww-}S z^(bQWJmEsk?1Bb~pn6biT+Oyc|^(8)JQa2Ct z*d_H!vE+R!#nOse(Rdrm-Rvsu*x7Jh{wDA5{dpGbd^vWru)M0SWHW|YH(itVCA)6S zScd;QT9c3W&hf~xh4%;AiJR-zM%duKq~XxtJ3eRj zStFCZA3exy<=CyFcD1~!u}b+?acL)&NYBrUor%F}4ZG&hU=t}dNJ|}3aj;4zONYF0 z1n8e6Fz0>mxSH!El8N;6wAd*O)@>Qvk$UNU@{l?yJ445l4Zcs%DA~xf&5fp=?+?8{ zJGTeHNhH!|&g_q%^Qlnp5L`7VA#q4kc>ck$uhs3^{UCfLo=Cs(MzT{I2*1C=psJh2 zindBepKvPHYY|&rDVDA<>L~O61AM<$Y9_4?^T#xgR)IejRRo|_aq zqBkoe)85p}J646wiU(~fmX_3a+2DFKM)&P4*}mgk?)x$7ZdApE!773=KdfZv;7Wou zBbD@y81&g*RVym%rLf{k_o)4CInHBj57tiy;>~Vnj~!gQx89x*y|mnSqUuSc7C6TRH%V?LWCS5l`P&51LCKHr1`p9hQ4sIpf#R$(;1Z zqPEvFBinfW#n~rG3BvL_XmrNyZxl|n_ZbveVO1*^OH{-)wWQ@~H})p^)!q;iB-rcg zj_HQuO78lJc>45d@wV5_;IzUqAGA%HcntAkVt+I}7`uPs z5sBST8jF-2jon{kD!VtJ{;_FK_Z8S0WK=B7O|6men zvDI9+2NFgO34m`f#j^aA&o#GQw%M{8dTWQD1Lm23vZpt9a)o2@^y-=asaNt6_jE;` zG)kS@>*1Ztl=*GZj(BJZ)&C!vue}!OVq@-DYWb}5yoxWbx{~<(Yx!&1I-{}|4 zk-Yg;cRtB3FP972o%843yvq!WCYRLZ_=#-Gsx8!P$7-6xxiN}JQd)xYD|*5L0SG_< z0uX=z1Rwwb2tWV=5P-n{N#KGsE=*iGE=;6SNxIjbGjq(L zThiv8sbH6qjQBOZ`MCCDIGSrd32~i;9hliQxMKB`Fq*$5UfUawL^X42iOEOd3;$ zW#OAV3{e#M!~1XcMCD%s{a}Fr1Rwwb2tWV=5P$##AOHafK;VTF7>cDvMalpF|DMwM zzw-2j3lRZ900Izz00bZa0SG_<0uX=z1YTl+m?R71Loxan0pm&j_x~cD`s3{XCGHRi z9Rd)500bZa0SG_<0uX=z1YSsi5&4w3Wwsp4ZpQfi|CsV~LHVolN98y40t*Bn009U< z00Izz00bZa0SG_<0?$+6)tEdlE>_fnrdDsNm6c+-RJ~r%7O$q#vOJ&KlXI0SG_< z0uX=z1Rwwb2tWV=&qILU|Cf|M2+CjR2@3=u009U<00Izz00bZa0SG_<0$*l{RbclJ-01f&!4=~OSpyB_dgMh2xR literal 0 HcmV?d00001 diff --git a/crates/agent-memory-core/tests/migration_hygiene.rs b/crates/agent-memory-core/tests/migration_hygiene.rs new file mode 100644 index 0000000..4c02361 --- /dev/null +++ b/crates/agent-memory-core/tests/migration_hygiene.rs @@ -0,0 +1,357 @@ +//! STORE-05: shipped migrations are frozen; upgrades are backed up; newer DBs are refused cleanly +//! +//! Invariants proven here (D-11/D-12/D-13, RESEARCH Pattern 6, Pitfalls 6 + 8): +//! +//! - **Frozen SQL.** A committed binary fixture (`tests/fixtures/v0.0.1.db`, +//! built ONCE from the v0.0.1-tagged 0001+0002 and never regenerated) migrated +//! to latest has exactly the schema of a fresh-from-zero database. Any later +//! edit of shipped 0001/0002 makes the two diverge and turns this suite red. +//! - **Backup before migrate.** Upgrading a non-empty older DB first writes ONE +//! sibling `{db}.backup-pre-{NNNN}-{YYYYMMDD}` via the SQLite Online Backup +//! API, and that backup still carries the PRE-migration schema version. +//! - **Newer-schema refusal.** A DB whose `user_version` is ahead of the binary +//! is refused with a message naming both versions and the upgrade command, +//! and the file is left byte-for-byte untouched. +//! +//! Every test runs on real on-disk tempfiles (an in-memory DB has no sibling +//! file to back up) and registers the sqlite-vec module first — migration 0002 +//! creates a vec0 table (the `store/migrations.rs` test quirk). + +use std::fs; +use std::path::{Path, PathBuf}; + +use agent_memory_core::store::migrations::{migrations, LATEST_SCHEMA_VERSION}; +use agent_memory_core::store::sqlite::{register_vec_extension, SqliteStore}; +use rusqlite::Connection; + +/// The schema version v0.0.1 shipped (migrations 0001 + 0002). +const V0_0_1_SCHEMA_VERSION: usize = 2; + +/// Rows planted in the fixture so a backup is provably a copy of real data. +const FIXTURE_ROWS: i64 = 2; + +fn fixture_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/v0.0.1.db") +} + +/// Copy the committed fixture to `dest` — tests NEVER open the fixture in place. +fn copy_fixture(dest: &Path) { + register_vec_extension().expect("sqlite-vec registers"); + fs::copy(fixture_path(), dest).expect("copy committed v0.0.1 fixture"); +} + +fn user_version(path: &Path) -> i64 { + let conn = Connection::open(path).expect("open db"); + conn.pragma_query_value(None, "user_version", |row| row.get(0)) + .expect("read user_version") +} + +/// The full schema as sorted, whitespace-normalized `type|name|tbl|sql` lines, +/// excluding SQLite's own `sqlite_*` internal objects. +fn normalized_schema(path: &Path) -> Vec { + let conn = Connection::open(path).expect("open db"); + let mut stmt = conn + .prepare( + "SELECT type, name, tbl_name, sql FROM sqlite_master \ + WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name", + ) + .expect("prepare schema query"); + let rows = stmt + .query_map([], |row| { + let kind: String = row.get(0)?; + let name: String = row.get(1)?; + let tbl: String = row.get(2)?; + let sql: Option = row.get(3)?; + let sql = sql + .unwrap_or_default() + .split_whitespace() + .collect::>() + .join(" "); + Ok(format!("{kind}|{name}|{tbl}|{sql}")) + }) + .expect("query schema"); + let mut out: Vec = rows.map(|r| r.expect("schema row")).collect(); + out.sort(); + out +} + +/// Sibling backup files of `db` for the CURRENT schema version (the `-wal` / +/// `-shm` / `-journal` companions SQLite may create are not backups). +fn backups_for_latest(db: &Path) -> Vec { + let name = db + .file_name() + .and_then(|n| n.to_str()) + .expect("db file name"); + let prefix = format!("{name}.backup-pre-{LATEST_SCHEMA_VERSION:04}-"); + let mut out: Vec = fs::read_dir(db.parent().expect("db dir")) + .expect("read db dir") + .map(|e| e.expect("dir entry").path()) + .filter(|p| { + let n = p.file_name().and_then(|n| n.to_str()).unwrap_or_default(); + n.starts_with(&prefix) + && !n.ends_with("-wal") + && !n.ends_with("-shm") + && !n.ends_with("-journal") + }) + .collect(); + out.sort(); + out +} + +/// One-shot generator for the committed fixture. It applies ONLY migrations +/// 0001+0002 (the schema v0.0.1 shipped — verified byte-identical to the +/// `v0.0.1` tag before generation) and plants two rows. It is a no-op when the +/// fixture already exists: the fixture must NEVER be regenerated from SQL that +/// may since have been edited (Pitfall 6), or the freeze test proves nothing. +/// +/// Run once: `cargo test -p agent-memory-core --test migration_hygiene -- --ignored` +#[test] +#[ignore = "one-shot fixture generator; never regenerate the committed fixture"] +fn generate_v0_0_1_fixture() { + let path = fixture_path(); + if path.exists() { + return; + } + register_vec_extension().expect("sqlite-vec registers"); + let mut conn = Connection::open(&path).expect("create fixture"); + // v0.0.1 store files are WAL databases (apply_pragmas); mirror that. + conn.pragma_update(None, "journal_mode", "WAL") + .expect("WAL mode"); + migrations() + .to_version(&mut conn, V0_0_1_SCHEMA_VERSION) + .expect("apply 0001 + 0002 only"); + conn.execute_batch( + "INSERT INTO memories \ + (mem_type, content, tags, source, scope, base_weight, decay_score, \ + access_count, created_at, last_accessed, expires_at, embedding_status) \ + VALUES \ + ('DECISION', 'fixture decision from a v0.0.1 store', '[\"fixture\"]', \ + 'fixture', NULL, 2.0, 1.0, 0, 1000, 1000, NULL, 0), \ + ('TODO', 'fixture todo without a source', '[]', \ + NULL, 'proj', 1.0, 1.0, 0, 1001, 1001, NULL, 0);", + ) + .expect("plant fixture rows"); + // Fold the WAL into the main file so the committed artifact is one file. + conn.pragma_update(None, "wal_checkpoint", "TRUNCATE") + .expect("checkpoint"); + drop(conn); + assert!( + !PathBuf::from(format!("{}-wal", path.display())).exists() + || fs::metadata(format!("{}-wal", path.display())) + .map(|m| m.len() == 0) + .unwrap_or(true), + "fixture WAL must be fully checkpointed" + ); +} + +#[test] +fn committed_fixture_is_the_v0_0_1_schema() { + // Guard against a missing/regenerated fixture silently weakening the freeze. + assert!( + fixture_path().exists(), + "tests/fixtures/v0.0.1.db is committed" + ); + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("fixture-copy.db"); + copy_fixture(&db); + assert_eq!(user_version(&db), V0_0_1_SCHEMA_VERSION as i64); +} + +#[test] +fn fixture_migrated_to_latest_matches_a_fresh_database() { + let dir = tempfile::tempdir().expect("tempdir"); + let upgraded = dir.path().join("upgraded.db"); + let fresh = dir.path().join("fresh.db"); + copy_fixture(&upgraded); + + drop(SqliteStore::open(&upgraded).expect("upgrade v0.0.1 fixture")); + drop(SqliteStore::open(&fresh).expect("open fresh db")); + + assert_eq!(user_version(&upgraded), LATEST_SCHEMA_VERSION); + assert_eq!(user_version(&fresh), LATEST_SCHEMA_VERSION); + let upgraded_schema = normalized_schema(&upgraded); + assert!( + upgraded_schema + .iter() + .any(|l| l.starts_with("table|memory_links|")), + "migration 0003 must create memory_links" + ); + assert_eq!( + upgraded_schema, + normalized_schema(&fresh), + "an upgraded v0.0.1 DB must have exactly the fresh-from-zero schema — \ + shipped 0001/0002 SQL is frozen; append a new migration instead" + ); +} + +#[test] +fn upgrading_writes_one_pre_migration_backup_with_the_old_schema() { + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("memory.db"); + copy_fixture(&db); + + drop(SqliteStore::open(&db).expect("upgrade v0.0.1 fixture")); + + let backups = backups_for_latest(&db); + assert_eq!(backups.len(), 1, "exactly one backup, got {backups:?}"); + let backup = &backups[0]; + let backup_name = backup + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default(); + let stamp = backup_name.rsplit('-').next().unwrap_or_default(); + assert!( + stamp.len() == 8 && stamp.chars().all(|c| c.is_ascii_digit()), + "backup name ends in a YYYYMMDD stamp, got {backup_name}" + ); + + // Pitfall 8 proof: the backup was taken BEFORE migrating, so it still + // carries the v0.0.1 schema version and none of 0003's objects. + assert_eq!(user_version(backup), V0_0_1_SCHEMA_VERSION as i64); + assert!( + !normalized_schema(backup) + .iter() + .any(|l| l.contains("memory_links")), + "backup must predate migration 0003" + ); + let conn = Connection::open(backup).expect("backup opens as a valid database"); + let rows: i64 = conn + .query_row("SELECT count(*) FROM memories", [], |row| row.get(0)) + .expect("count backup rows"); + assert_eq!(rows, FIXTURE_ROWS, "backup holds the pre-upgrade data"); + + // The live DB itself was migrated. + assert_eq!(user_version(&db), LATEST_SCHEMA_VERSION); +} + +#[test] +fn re_running_the_upgrade_keeps_one_backup_per_schema_version() { + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("memory.db"); + let name = "memory.db"; + // A stale backup for the SAME target version must be replaced; a backup + // for a DIFFERENT version must never be touched (D-11). + let stale_same = dir.path().join(format!( + "{name}.backup-pre-{LATEST_SCHEMA_VERSION:04}-20000101" + )); + let other_version = dir.path().join(format!("{name}.backup-pre-0002-20000101")); + fs::write(&stale_same, b"stale").expect("plant stale backup"); + fs::write(&other_version, b"other").expect("plant other-version backup"); + + for _ in 0..2 { + // Reset to a pristine v0.0.1 DB so each open has a migration pending. + for suffix in ["", "-wal", "-shm"] { + let p = PathBuf::from(format!("{}{suffix}", db.display())); + if p.exists() { + fs::remove_file(&p).expect("reset db"); + } + } + copy_fixture(&db); + drop(SqliteStore::open(&db).expect("upgrade v0.0.1 fixture")); + } + + let backups = backups_for_latest(&db); + assert_eq!( + backups.len(), + 1, + "one backup per schema version, got {backups:?}" + ); + assert_ne!( + backups[0], stale_same, + "the stale same-version backup is replaced" + ); + assert_eq!(user_version(&backups[0]), V0_0_1_SCHEMA_VERSION as i64); + assert_eq!( + fs::read(&other_version).expect("other-version backup survives"), + b"other", + "another schema version's backup is never touched" + ); +} + +#[test] +fn opening_a_fresh_database_creates_no_backup() { + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("memory.db"); + + drop(SqliteStore::open(&db).expect("open fresh db")); + + let any_backup = fs::read_dir(dir.path()) + .expect("read dir") + .map(|e| e.expect("dir entry").file_name()) + .any(|n| n.to_string_lossy().contains(".backup-pre-")); + assert!(!any_backup, "a brand-new DB has nothing to back up (D-12)"); + assert_eq!(user_version(&db), LATEST_SCHEMA_VERSION); +} + +#[test] +fn a_newer_schema_database_is_refused_and_left_untouched() { + register_vec_extension().expect("sqlite-vec registers"); + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("memory.db"); + { + let conn = Connection::open(&db).expect("create db"); + conn.execute_batch("CREATE TABLE from_the_future (x INTEGER);") + .expect("create table"); + conn.pragma_update(None, "user_version", 99) + .expect("set user_version"); + } + let before = fs::read(&db).expect("read db bytes"); + + let message = match SqliteStore::open(&db) { + Ok(_) => panic!("a schema-99 database must be refused"), + Err(e) => e.to_string(), + }; + + assert!(message.contains("99"), "names the DB version: {message}"); + assert!( + message.contains(&LATEST_SCHEMA_VERSION.to_string()), + "names the binary version: {message}" + ); + assert!( + message.contains("3"), + "binary supports version 3: {message}" + ); + assert!( + message.contains("brew upgrade unityinflow/tap/agent-memory"), + "names the upgrade command: {message}" + ); + assert_eq!( + fs::read(&db).expect("re-read db bytes"), + before, + "the refused DB file must be byte-for-byte untouched (D-13)" + ); + assert!( + backups_for_latest(&db).is_empty(), + "a refused DB is never backed up" + ); +} + +#[test] +fn a_failed_backup_aborts_the_open_before_migrating() { + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("memory.db"); + copy_fixture(&db); + // A non-removable entry occupying the backup slot: the retention step + // cannot clear it, so the backup cannot be written — open must abort + // rather than run an unbacked migration (backup failure = abort). + let blocker = dir.path().join(format!( + "memory.db.backup-pre-{LATEST_SCHEMA_VERSION:04}-blocker" + )); + fs::create_dir(&blocker).expect("create blocker dir"); + fs::write(blocker.join("keep"), b"x").expect("populate blocker dir"); + + let message = match SqliteStore::open(&db) { + Ok(_) => panic!("open must abort when the backup cannot be written"), + Err(e) => e.to_string(), + }; + + assert!( + message.contains("backup"), + "names the backup failure: {message}" + ); + assert_eq!( + user_version(&db), + V0_0_1_SCHEMA_VERSION as i64, + "no migration may run without a backup" + ); +} diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index 9db4698..0e9ee19 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -40,6 +40,10 @@ fn map_mcp_error(e: MemoryError) -> McpError { | MemoryError::Join(_) | MemoryError::Migration(_) | MemoryError::Internal(_) => McpError::internal_error(e.to_string(), None), + // Startup-only (D-13): raised by SqliteStore::open before the server + // starts, so no tool call can produce it. Classified internal for + // exhaustiveness; the refusal itself reaches stderr via main.rs. + MemoryError::SchemaTooNew { .. } => McpError::internal_error(e.to_string(), None), } } diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index 3dca294..6122922 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -97,6 +97,10 @@ fn map_memory_error(e: MemoryError) -> ApiError { | MemoryError::Join(_) | MemoryError::Migration(_) | MemoryError::Internal(_) => ApiError::Internal(e.to_string()), + // Startup-only (D-13): raised by SqliteStore::open before the REST + // server binds, so no request can produce it. Classified 500 for + // exhaustiveness; the refusal itself reaches stderr via main.rs. + MemoryError::SchemaTooNew { .. } => ApiError::Internal(e.to_string()), } } From 2088a143bd9124cd863f653703a4c66f309befb0 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 19:58:08 +0200 Subject: [PATCH 03/42] docs(04-01): complete storage & error foundation plan (summary) Co-Authored-By: Claude Opus 5.5 --- .../04-01-SUMMARY.md | 262 ++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-01-SUMMARY.md diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-01-SUMMARY.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-01-SUMMARY.md new file mode 100644 index 0000000..7fca94f --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-01-SUMMARY.md @@ -0,0 +1,262 @@ +--- +phase: 04-memory-update-relations-hybrid-search +plan: 01 +subsystem: database +tags: [rust, sqlite, rusqlite, rusqlite_migration, migrations, backup-api, thiserror, mcp, axum] + +# Dependency graph +requires: + - phase: 03-api-hardening-toolchain-spikes + provides: "MemoryError::InvalidArgument seam, two-tier exhaustive mappers, bounds consts (MAX_LIMIT), 03-REVIEW findings CR-01/WR-01/WR-02" +provides: + - "MemoryError::Internal(String): 500 / internal_error tier, used at all six writer-lock sites (WR-02)" + - "MemoryError::SchemaTooNew { db_version, binary_version }: startup-only D-13 refusal" + - "MemoryError::NotFound and ApiError::NotFound deleted; not-found is data only" + - "Store::exists(source: Option<&str>, ..) with NULL-safe `source IS ?1` (CR-01)" + - "MAX_KNN_K = MAX_LIMIT * 4 (800): 4x oversample at every limit (WR-01 partial)" + - "Migration 0003: memory_links (from_id, to_id, kind, created_at), PK(from_id,to_id,kind), FK ON DELETE CASCADE, CHECK from!=to, WITHOUT ROWID, idx_links_to" + - "pub const LATEST_SCHEMA_VERSION: i64 = 3 (store/migrations.rs)" + - "SqliteStore::open: probe -> refuse-too-new -> pragmas -> sibling online backup -> migrate" + - "Committed binary fixture tests/fixtures/v0.0.1.db + tests/migration_hygiene.rs" +affects: [04-02 memory_update, 04-03 relations/link/unlink/expansion, 04-04 hybrid RRF, 05 export/import format, 06 release notes] + +# Actuals (#2632) +actuals: + tokens: 8465 + tasks: 2 + commits: 3 + +# Tech tracking +tech-stack: + added: ["rusqlite `backup` cargo feature (no new crate; 0.39 pin unchanged)"] + patterns: + - "Probe PRAGMA user_version BEFORE any per-connection PRAGMA, so a refused DB stays byte-for-byte untouched" + - "Generic pre-migration backup keyed by target schema version: {db}.backup-pre-{NNNN}-{YYYYMMDD}, one per version, abort-on-failure" + - "Schema freeze via a committed binary fixture: migrate a copy, diff normalized sqlite_master against fresh-from-zero" + - "Server faults are MemoryError::Internal; not-found is Ok(false)/Ok(None) data, never an error variant" + +key-files: + created: + - crates/agent-memory-core/sql/0003_relations.sql + - crates/agent-memory-core/tests/migration_hygiene.rs + - crates/agent-memory-core/tests/fixtures/v0.0.1.db + modified: + - Cargo.toml + - crates/agent-memory-core/src/domain.rs + - crates/agent-memory-core/src/service.rs + - crates/agent-memory-core/src/store/mod.rs + - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory-core/src/store/migrations.rs + - crates/agent-memory-core/tests/import.rs + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - crates/agent-memory/src/rest/mod.rs + +key-decisions: + - "D-13 probe runs before prepare_connection (not after it): the journal_mode=WAL pragma rewrites header bytes on a non-WAL file, which would break the 'DB untouched' guarantee" + - "Backup failure aborts SqliteStore::open with MemoryError::Internal naming the backup path; no migration runs (T-04-02)" + - "Retention step removes every {db}.backup-pre-{NNNN}-* entry for the target version and errors if one cannot be removed. Other versions' backups are never touched" + - "Backup date stamp comes from chrono::Utc::now().date_naive() via Datelike, which needs no extra chrono feature (UTC only, no Local)" + - "ApiError::NotFound deleted along with MemoryError::NotFound: REST 404s are built from not-found data (forget's Ok(false)), so a server fault can no longer be mis-tiered as a 404" + - "Fixture is a WAL-header DB with 2 planted rows (one source-less), generated once by an #[ignore]d, exists-guarded test from the tag-identical 0001+0002" + +patterns-established: + - "Migration hygiene: every future migration inherits probe -> backup -> migrate by bumping LATEST_SCHEMA_VERSION with the new M::up" + - "New writer methods must map lock poisoning to MemoryError::Internal(\"writer lock poisoned\")" + +requirements-completed: [STORE-05] + +coverage: + - id: D1 + description: "Poisoned writer lock maps to the internal tier on both transports (REST 500, MCP internal_error); MemoryError::NotFound deleted" + verification: + - kind: unit + ref: "crates/agent-memory/src/mcp.rs#map_mcp_error_splits_client_and_internal_tiers" + status: pass + - kind: unit + ref: "crates/agent-memory/src/rest/handlers.rs#map_memory_error_puts_internal_faults_in_the_500_tier" + status: pass + human_judgment: false + - id: D2 + description: "Re-importing a source:None draft is idempotent (CR-01)" + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/import.rs#reimporting_a_sourceless_draft_is_idempotent" + status: pass + human_judgment: false + - id: D3 + description: "MAX_KNN_K = MAX_LIMIT * 4 = 800 keeps the 4x KNN oversample at every valid limit (WR-01 partial)" + verification: + - kind: unit + ref: "crates/agent-memory-core/src/store/sqlite.rs#max_knn_k_keeps_four_x_oversample_at_max_limit" + status: pass + human_judgment: false + - id: D4 + description: "Shipped 0001/0002 are frozen: the committed v0.0.1 fixture migrated to latest equals a fresh DB's normalized schema" + requirement: STORE-05 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/migration_hygiene.rs#fixture_migrated_to_latest_matches_a_fresh_database" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/migration_hygiene.rs#committed_fixture_is_the_v0_0_1_schema" + status: pass + - kind: unit + ref: "crates/agent-memory-core/src/store/migrations.rs#latest_schema_version_matches_the_migration_chain" + status: pass + human_judgment: false + - id: D5 + description: "Upgrading a v0.0.1 DB writes one sibling online backup (user_version 2, pre-upgrade rows), one per schema version; a fresh DB gets none; a failed backup aborts before migrating" + requirement: STORE-05 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/migration_hygiene.rs#upgrading_writes_one_pre_migration_backup_with_the_old_schema" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/migration_hygiene.rs#re_running_the_upgrade_keeps_one_backup_per_schema_version" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/migration_hygiene.rs#opening_a_fresh_database_creates_no_backup" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/migration_hygiene.rs#a_failed_backup_aborts_the_open_before_migrating" + status: pass + human_judgment: false + - id: D6 + description: "A newer-schema DB is refused with a message naming 99, 3 and the brew upgrade command; the file is byte-untouched; the binary prints it on stderr with empty stdout" + requirement: STORE-05 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/migration_hygiene.rs#a_newer_schema_database_is_refused_and_left_untouched" + status: pass + - kind: manual_procedural + ref: "agent-memory --db serve exit 1, stdout 0 bytes, stderr names both versions + brew command, shasum unchanged" + status: pass + human_judgment: false + +# Metrics +duration: 9min +completed: 2026-10-23 +status: complete +--- + +# Phase 4 Plan 01: Storage and Error Foundation Summary + +**Migration 0003 (`memory_links`) ships behind a user_version probe, a sibling SQLite Online-Backup snapshot and a friendly newer-schema refusal, all locked by a committed v0.0.1 binary fixture. The plan also closes the CR-01 / WR-01 / WR-02 review carry-forwards: NULL-safe import dedup, a 4x KNN oversample cap, and `MemoryError::Internal`.** + +## Performance + +- **Duration:** ~9 min +- **Started:** 2026-10-23T17:48:14Z +- **Completed:** 2026-10-23T17:57:30Z +- **Tasks:** 2 / 2 +- **Files modified:** 13 (3 created, 10 modified) + +## Accomplishments + +- **WR-02 closed:** all six `writer.lock()` sites map poisoning to `MemoryError::Internal("writer lock poisoned")`, and both exhaustive mappers classify it as internal (REST 500 / MCP `internal_error`). `MemoryError::NotFound` and the unconstructable `ApiError::NotFound` are deleted; the compiler confirms there are zero producers. +- **CR-01 closed:** `Store::exists` takes `Option<&str>` and probes with `source IS ?1`, and `import()` no longer coerces `None` to `""`. Re-importing a source-less draft now reports `skipped_duplicates == 1`. +- **WR-01 (partial):** `MAX_KNN_K = MAX_LIMIT * 4 = 800`, so the 4x oversample holds at `limit = 200`. The doc comment separates it from the RRF smoothing constant `k = 60`. +- **STORE-05:** migration 0003 adds `memory_links`. `SqliteStore::open` now runs probe, then refuse-too-new, then pragmas, then backup, then migrate. A committed v0.0.1 fixture keeps 0001/0002 provably frozen. A mutation test was run by hand: editing a 0001 column default turned the freeze test red, and 0001 was then restored byte-identical to the tag. +- **End to end:** running the binary against a `user_version = 99` DB exits 1 with empty stdout (MCP-05 purity). Stderr reads `database schema is version 99 but this binary supports up to version 3 — upgrade agent-memory (brew upgrade unityinflow/tap/agent-memory) or point --db at a different file`, and the DB sha is unchanged. + +## Task Commits + +1. **Task 1: Internal error tier (WR-02), NotFound removal, MAX_KNN_K decouple (WR-01), NULL-safe import dedup (CR-01)** - `2541dbc` (fix) +2. **Task 2: Migration 0003 + pre-migration backup + newer-schema refusal + fixture freeze** - `46a63fa` (feat) + +**Plan metadata:** recorded in the final `docs(04-01)` commit. + +_TDD note: RED was observed locally for both tasks before implementation. For Task 1, the source:None test failed with `left: 1 right: 0`. For Task 2, 6 of 7 hygiene tests failed against a 2-migration chain. RED was not committed as a separate `test(...)` commit because CLAUDE.md and the operator rules forbid committing failing tests. Each task commit carries its tests plus the implementation._ + +## Files Created/Modified + +- `crates/agent-memory-core/sql/0003_relations.sql`: the `memory_links` relations schema (directed edges, cascade, self-link CHECK, reverse index) +- `crates/agent-memory-core/tests/migration_hygiene.rs`: STORE-05 suite (7 tests plus the `#[ignore]`d one-shot fixture generator) +- `crates/agent-memory-core/tests/fixtures/v0.0.1.db`: committed v0.0.1-schema binary fixture (user_version 2, WAL header, 2 rows, sha1 `7c6f9f0c…`) +- `Cargo.toml`: rusqlite gains the `backup` feature (version 0.39 unchanged; Cargo.lock unchanged) +- `crates/agent-memory-core/src/domain.rs`: `Internal` and `SchemaTooNew` added, `NotFound` removed +- `crates/agent-memory-core/src/store/migrations.rs`: 0003 appended, `LATEST_SCHEMA_VERSION`, chain-length test +- `crates/agent-memory-core/src/store/sqlite.rs`: open() hygiene flow, `pre_migration_backup`, `utc_yyyymmdd`, Internal lock mapping, NULL-safe `exists`, `MAX_KNN_K`, const unit test +- `crates/agent-memory-core/src/store/mod.rs`: `exists(source: Option<&str>, ..)` +- `crates/agent-memory-core/src/service.rs`: import dedup passes `draft.source.as_deref()` +- `crates/agent-memory-core/tests/import.rs`: CR-01 regression test +- `crates/agent-memory/src/mcp.rs`: mapper arms for `Internal` and `SchemaTooNew`, tier test +- `crates/agent-memory/src/rest/handlers.rs`: mapper arms for `Internal` and `SchemaTooNew`, 500-tier test +- `crates/agent-memory/src/rest/mod.rs`: `ApiError::NotFound` removed + +## Decisions Made + +- **Probe before pragmas.** The plan placed the probe "between prepare_connection and to_latest". It runs before `prepare_connection` instead, because `journal_mode=WAL` rewrites header bytes 18-19 of a rollback-journal file. Probing after it would have broken D-13's "DB untouched" contract and the byte-equality test. The backup still runs after the pragmas and before `to_latest`, so Pitfall 8 ordering holds. +- **Backup failure aborts** (CONTEXT discretion, resolved in planning). The error is `MemoryError::Internal("pre-migration backup failed (…); refusing to migrate ")`. +- **The retention step is strict.** Any entry matching `{db}.backup-pre-{NNNN}-*` that cannot be removed aborts the open. That makes the abort path testable without platform-specific permission tricks. +- **The fixture has data.** Two planted rows (one with NULL source) let the backup test assert that real rows survive in the snapshot. The plan only required the schema. + +## Deviations from Plan + +### Auto-fixed Issues + +**1. [Rule 3 - Blocking] Deleted `ApiError::NotFound` in `rest/mod.rs` (file not in the plan's list)** +- **Found during:** Task 1 +- **Issue:** With `MemoryError::NotFound` gone, `ApiError::NotFound` had no constructor left. That is a `dead_code` warning in the binary crate, which `clippy -D warnings` fails on. +- **Fix:** Removed the variant and its `IntoResponse` arm, and rewrote the doc to say REST 404s come from not-found data. Plans 04-02 and 04-03 build their 404s directly, as forget_handler does, so nothing downstream needs it. +- **Files modified:** crates/agent-memory/src/rest/mod.rs +- **Verification:** clippy clean; all REST tests (in-process and spawned-binary) green +- **Committed in:** 2541dbc + +**2. [Rule 1 - Bug] D-13 probe moved ahead of `prepare_connection`** +- **Found during:** Task 2 +- **Issue:** As written, the ordering would apply the WAL pragma to a too-new DB before refusing it, which mutates the file. That violates D-13 and the plan's own Test 5. +- **Fix:** The probe reads `user_version` on the freshly opened connection before any pragma. The backup and migration still follow the pragmas. +- **Files modified:** crates/agent-memory-core/src/store/sqlite.rs +- **Verification:** `a_newer_schema_database_is_refused_and_left_untouched` (byte equality) plus the manual binary shasum check +- **Committed in:** 46a63fa + +**3. [Rule 2 - Missing critical] Added tests beyond the five planned hygiene behaviours** +- **Found during:** Task 2 +- **Issue:** The plan's resolved Open Question 3 says the abort-on-backup-failure behaviour is "test-covered", but no listed behaviour covered it. A drifting `LATEST_SCHEMA_VERSION` would also silently disable both the probe and the backup trigger. +- **Fix:** Added `a_failed_backup_aborts_the_open_before_migrating`, `committed_fixture_is_the_v0_0_1_schema`, and the unit test `latest_schema_version_matches_the_migration_chain`. +- **Committed in:** 46a63fa + +--- + +**Total deviations:** 3 auto-fixed (1 blocking, 1 bug, 1 missing-critical test coverage) +**Impact on plan:** All three were needed for correctness or a clean `-D warnings` build. No scope creep, and no new crates. + +## Issues Encountered + +- The fixture path is caught by the user's local, uncommitted `.gitignore` (`*.db`), so it was staged with `git add -f` as the plan anticipated. That `.gitignore` itself was left untouched and unstaged. +- Project hooks block shell `cp`/`rm` of `.db` files. The manual end-to-end check used a fresh `mktemp -d` directory and Python's `sqlite3` instead of deleting anything. + +## Verification Results + +- `cargo test -p agent-memory-core --test import`: 2 passed +- `cargo test -p agent-memory-core --test migration_hygiene`: 7 passed, 1 ignored (the generator) +- `cargo test --workspace`: **87 passed / 0 failed / 2 ignored** (baseline was 76 / 0 / 1) +- `cargo clippy --workspace --all-targets -- -D warnings`: clean +- `cargo fmt --all --check`: clean +- `git diff v0.0.1 -- sql/0001_init.sql sql/0002_embeddings.sql`: empty +- `cargo tree -p rusqlite --depth 0`: `rusqlite v0.39.0` + +## User Setup Required + +None. No external service configuration is required. + +## Next Phase Readiness + +- Plan 04-02 (`memory_update`) can write new writer methods against `MemoryError::Internal`. +- Plan 04-03 (relations) has the `memory_links` table. It still owns the belt-and-braces orphan deletes in `forget()` and `sweep_expired()`, and the `LinkKind` and `LinkOutcome` types. +- Plan 04-04 (hybrid) inherits the 4x `MAX_KNN_K` oversample. +- Still deferred: the full WR-01 fix (filter-aware KNN retry or vec0 metadata pre-filtering) and WR-03 (the REST harness hang). Both are outside this plan's scope, per 04-RESEARCH. + +--- +*Phase: 04-memory-update-relations-hybrid-search* +*Completed: 2026-10-23* + +## Self-Check: PASSED + +- FOUND: crates/agent-memory-core/sql/0003_relations.sql +- FOUND: crates/agent-memory-core/tests/migration_hygiene.rs +- FOUND: crates/agent-memory-core/tests/fixtures/v0.0.1.db (tracked: `git ls-files --error-unmatch` exit 0) +- FOUND: commit 2541dbc (Task 1) +- FOUND: commit 46a63fa (Task 2) From 68abbf0a6da8dfa44bfba9c9ea5742cb649bf93e Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 19:58:52 +0200 Subject: [PATCH 04/42] docs(04-01): update state, roadmap progress, mark STORE-05 complete Co-Authored-By: Claude Opus 5.5 --- .planning/REQUIREMENTS.md | 4 ++-- .planning/ROADMAP.md | 6 +++--- .planning/STATE.md | 39 ++++++++++++++++++++++++++------------- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md index 957f220..8d80d92 100644 --- a/.planning/REQUIREMENTS.md +++ b/.planning/REQUIREMENTS.md @@ -25,7 +25,7 @@ Each maps to a roadmap phase. IDs continue from the v1.0 numbering (archived in ### Storage & Migration Hygiene (STORE) -- [ ] **STORE-05**: Schema migration 0003 (relations) ships with migration hygiene: shipped 0001/0002 SQL frozen (fixture divergence test against a v0.0.1 database), pre-migration backup, and a friendly error when a newer DB meets an older binary +- [x] **STORE-05**: Schema migration 0003 (relations) ships with migration hygiene: shipped 0001/0002 SQL frozen (fixture divergence test against a v0.0.1 database), pre-migration backup, and a friendly error when a newer DB meets an older binary ### Portability (DIST / INTEROP) @@ -66,7 +66,7 @@ Mapped by roadmap creation 2026-07-12. Coverage: 12/12 v1.1 requirements mapped | API-03 | Phase 3 | Complete | | MCP-06 | Phase 4 | Pending | | MCP-07 | Phase 4 | Pending | -| STORE-05 | Phase 4 | Pending | +| STORE-05 | Phase 4 | Complete | | SEARCH-04 | Phase 4 | Pending | | SEARCH-05 | Phase 4 | Pending | | DIST-04 | Phase 5 | Pending | diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 1313717..773a479 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -67,11 +67,11 @@ Plans: 4. Hybrid ranking stays decay-aware — a fresh memory outranks a stale one at equal fused rank — and access bumps apply only to post-truncation returned ids. 5. Upgrading a real v0.0.1 database applies migration 0003 only after a pre-migration backup; shipped 0001/0002 SQL is provably unchanged (fixture divergence test against a v0.0.1 database), and a newer-schema DB met by an older binary yields a friendly error, not a crash. -**Plans**: 4 plans +**Plans**: 1/4 plans executed Plans: **Wave 1** -- [ ] 04-01-PLAN.md — Error foundation (Internal tier, NotFound removal, CR-01 dedup, MAX_KNN_K decouple) + migration 0003 with backup/fixture-freeze/too-new hygiene (STORE-05) +- [x] 04-01-PLAN.md — Error foundation (Internal tier, NotFound removal, CR-01 dedup, MAX_KNN_K decouple) + migration 0003 with backup/fixture-freeze/too-new hygiene (STORE-05) **Wave 2** *(blocked on Wave 1 completion)* @@ -124,6 +124,6 @@ Phases execute in numeric order: 3 → 4 → 5 → 6 | 1. Core Memory Foundation | v1.0 | 3/3 | Complete | 2026-06-25 | | 2. Semantic Search, Interop & Release | v1.0 | 5/5 | Complete | 2026-07-12 | | 3. API Hardening & Toolchain Spikes | v1.1 | 2/2 | Complete | 2026-07-12 | -| 4. Memory Update, Relations & Hybrid Search | v1.1 | 0/? | Not started | - | +| 4. Memory Update, Relations & Hybrid Search | v1.1 | 1/4 | In Progress| | | 5. Portable Export & Import | v1.1 | 0/? | Not started | - | | 6. Distribution & v0.1.0 Release | v1.1 | 0/? | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index 77502cc..dfedf50 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -2,16 +2,19 @@ gsd_state_version: 1.0 milestone: v1.1 milestone_name: Hardening & Interop +current_phase: 04 +current_phase_name: Memory Update, Relations & Hybrid Search status: executing -stopped_at: Phase 4 context gathered -last_updated: "2026-07-13T13:42:43.486Z" -last_activity: 2026-10-09 -- Completed quick task 261009-hc1 (hosted release pipeline, closes #1) +stopped_at: Completed 04-01-PLAN.md +last_updated: "2026-10-23T17:58:30.378Z" +last_activity: 2026-10-23 +last_activity_desc: Phase 04 execution started +state_head: 2088a143bd9124cd863f653703a4c66f309befb0 progress: total_phases: 4 completed_phases: 1 - total_plans: 2 - completed_plans: 2 - percent: 25 + total_plans: 6 + completed_plans: 3 --- # Project State @@ -21,14 +24,14 @@ progress: See: .planning/PROJECT.md (updated 2026-07-12) **Core value:** An agent can persist a structured memory and retrieve the right one later — across sessions and tools — over a standard MCP interface, with no cloud. -**Current focus:** Phase 04 — memory-update-relations-hybrid-search +**Current focus:** Phase 04 — Memory Update, Relations & Hybrid Search ## Current Position -Phase: 4 -Plan: Not started +Phase: 04 (Memory Update, Relations & Hybrid Search) — EXECUTING +Plan: 2 of 4 Status: Ready to execute -Last activity: 2026-10-09 - Completed quick task 261009-hc1: hosted release pipeline with SLSA provenance (closes #1) +Last activity: 2026-10-23 — Phase 04 execution started Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) @@ -61,6 +64,11 @@ Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) | Phase 02 P05 | 8 min | 2 tasks | 6 files | | Phase 03 P01 | 12 min | 3 tasks | 11 files | | Phase 03 P02 | 30 min | 3 tasks | 2 files | +**Per-Plan Metrics:** + +| Plan | Duration | Tasks | Files | +|------|----------|-------|-------| +| Phase 04 P01 | 9 min | 2 tasks | 13 files | ## Accumulated Context @@ -84,6 +92,11 @@ Recent decisions affecting current work: - [03-01]: import() validates ttl per draft through the shared helper — the store() bypass is closed structurally for Phase 5 JSONL import - [03-02]: Phase 6 toolchain verdict (D-11): all 4 spike legs GREEN in one dispatch — Windows = cargo-xwin 0.23.0 / x86_64-pc-windows-msvc (gnu mingw-w64 proven fallback); musl = zigbuild 0.23.0 + target-suffixed CFLAGS -Du_int*_t shim (both legs green, no DIST-05 blocker) - [03-02]: CI revived on GitHub-hosted ubuntu-latest, secretless contents: read (D-12/D-10, ecosystem D-02 exception) — supersedes inner CLAUDE.md 'never ubuntu-latest' for this public repo; release.yml runner decision stays with Phase 6 +- [Phase 04]: [04-01]: MemoryError::Internal(String) is the 500/internal_error tier for server faults (poisoned writer lock); MemoryError::NotFound and ApiError::NotFound deleted — not-found is data (Ok(false)/Ok(None)) only (WR-02) +- [Phase 04]: [04-01]: Store::exists takes Option<&str> with NULL-safe 'source IS ?1' — source:None re-imports are idempotent (CR-01) +- [Phase 04]: [04-01]: MAX_KNN_K = MAX_LIMIT * 4 (800) — candidate oversample cap, distinct from RRF k=60 (WR-01 partial; full filter-aware fix deferred) +- [Phase 04]: [04-01]: SqliteStore::open order is probe user_version (before any PRAGMA, so a refused DB is byte-untouched) -> SchemaTooNew refusal -> pragmas -> sibling online backup {db}.backup-pre-{NNNN}-{YYYYMMDD} (one per target version, failure aborts) -> to_latest; LATEST_SCHEMA_VERSION=3 must be bumped with every new migration +- [Phase 04]: [04-01]: tests/fixtures/v0.0.1.db is a committed binary fixture generated once from tag-identical 0001+0002 — never regenerate it; it is gitignored locally by *.db so stage with git add -f ### Pending Todos @@ -124,9 +137,9 @@ Items acknowledged and carried forward: ## Session Continuity -Last session: 2026-07-12T20:49:51.528Z -Stopped at: Phase 4 context gathered -Resume file: .planning/phases/04-memory-update-relations-hybrid-search/04-CONTEXT.md +Last session: 2026-10-23T17:58:30.334Z +Stopped at: Completed 04-01-PLAN.md +Resume file: None ## Operator Next Steps From d76e43d9cdab8f983d506d6ecfef61589c9dcdc6 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:05:56 +0200 Subject: [PATCH 05/42] feat(04-02): memory_update core transaction + MCP tool (MCP-06) - MemoryPatch (double-Option merge-patch) + UpdatedMemory (flattened view + embedding_status, D-10) + validate_patch at the service seam (D-08) - Store::update: one writer tx, UPDATE in place (never delete+insert), freshness bump folded into the same statement (D-09), same-tx vec_memories invalidation + re-embed, outage falls back to embedding_status = 0 - MemoryService::update: validate -> clock -> embed only when content is patched -> spawn_blocking; unknown id is Ok(None) - serde_util::deserialize_explicit_null shared by both transports - memory_update MCP tool; not-found mirrors memory_forget's status shape; tool schema advertises no `default: null` (null means "clear") - tests/update.rs: re-embed kill-test, outage + sweep backfill, freshness bump, response shape, not-found, no-embed tags patch, validate-before-embed Co-Authored-By: Claude Opus 5.5 --- crates/agent-memory-core/src/domain.rs | 73 +++ crates/agent-memory-core/src/service.rs | 48 +- crates/agent-memory-core/src/store/mod.rs | 22 +- crates/agent-memory-core/src/store/sqlite.rs | 127 ++++- crates/agent-memory-core/tests/update.rs | 519 +++++++++++++++++++ crates/agent-memory/src/main.rs | 1 + crates/agent-memory/src/mcp.rs | 232 ++++++++- crates/agent-memory/src/serde_util.rs | 88 ++++ 8 files changed, 1102 insertions(+), 8 deletions(-) create mode 100644 crates/agent-memory-core/tests/update.rs create mode 100644 crates/agent-memory/src/serde_util.rs diff --git a/crates/agent-memory-core/src/domain.rs b/crates/agent-memory-core/src/domain.rs index 9fcb14e..6ac737a 100644 --- a/crates/agent-memory-core/src/domain.rs +++ b/crates/agent-memory-core/src/domain.rs @@ -114,6 +114,49 @@ pub struct MemoryView { pub last_accessed: i64, } +/// A JSON-merge-patch for `memory_update` (MCP-06 / D-08). +/// +/// Every field is a double `Option`, so a transport can carry the three wire +/// states of a JSON member through to the core: +/// +/// | wire | value | meaning | +/// |-----------------|------------------|---------------------------------| +/// | member absent | `None` | leave the field unchanged | +/// | `null` | `Some(None)` | clear it (where clearing is legal) | +/// | a value | `Some(Some(v))` | set it | +/// +/// Clearing means: `tags` → empty list, `scope` → global (`NULL`, the same +/// default [`NewMemory`] stores), `ttl_secs` → no expiry. `content` can never +/// be cleared or emptied (rejected by the service). `mem_type` and `source` are +/// deliberately NOT patchable (MCP-06 scope; type immutability also keeps +/// `base_weight` fixed). +/// +/// Plain struct with no serde derives: the transports own deserialization +/// (they need an explicit-null helper — plain `Option>` collapses +/// `null` into "absent") and construct this directly. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct MemoryPatch { + pub content: Option>, + pub tags: Option>>, + pub scope: Option>, + pub ttl_secs: Option>, +} + +/// The `memory_update` response (D-10): the full post-update [`MemoryView`] +/// plus `embedding_status`, so a caller can observe a pending re-embed (`0`) +/// after an embedder outage — the sweep backfills it to `1` later. +/// +/// The view is `#[serde(flatten)]`ed, so on the wire this is the familiar +/// MemoryView object with one extra member. `MemoryView` itself is untouched, +/// which keeps every existing list/search envelope byte-identical. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct UpdatedMemory { + #[serde(flatten)] + pub view: MemoryView, + /// `1` = a vector is stored for the current content; `0` = pending. + pub embedding_status: i64, +} + /// Smallest accepted `limit` on search/list (D-01/D-03). pub const MIN_LIMIT: i64 = 1; @@ -207,6 +250,36 @@ pub(crate) fn validate_ttl(ttl_secs: Option) -> Result<(), MemoryError> { Ok(()) } +/// Validate a [`MemoryPatch`] at the service seam, before any embed or store +/// hop (MCP-06 / D-08; Phase 3 D-06 — bad input is always the client tier): +/// +/// - a patch with every field absent is a client error, not a silent no-op; +/// - `content` may not be `null`, empty, or whitespace-only; +/// - a present `ttl_secs` goes through the shared [`validate_ttl`] bounds. +pub(crate) fn validate_patch(patch: &MemoryPatch) -> Result<(), MemoryError> { + if patch.content.is_none() + && patch.tags.is_none() + && patch.scope.is_none() + && patch.ttl_secs.is_none() + { + return Err(MemoryError::InvalidArgument( + "at least one of content, tags, scope, ttl_secs must be provided".to_string(), + )); + } + match &patch.content { + None => {} + Some(Some(content)) if !content.trim().is_empty() => {} + Some(Some(_)) | Some(None) => { + return Err(MemoryError::InvalidArgument( + "content must be a non-empty string".to_string(), + )); + } + } + // Only a present value is bounds-checked: absent (keep) and null (clear + // the expiry) both flatten to None, which validate_ttl always accepts. + validate_ttl(patch.ttl_secs.flatten()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/agent-memory-core/src/service.rs b/crates/agent-memory-core/src/service.rs index 291ffb7..7b828e8 100644 --- a/crates/agent-memory-core/src/service.rs +++ b/crates/agent-memory-core/src/service.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use crate::clock::Clock; use crate::decay::{apply_decay, decay_score, DecayConfig, DecayEngine, RankWeights, SweepReport}; -use crate::domain::{MemoryError, MemoryType, MemoryView, NewMemory}; +use crate::domain::{MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, UpdatedMemory}; use crate::embed::Embedder; use crate::store::sqlite::DEFAULT_SEARCH_LIMIT; use crate::store::Store; @@ -358,6 +358,52 @@ impl MemoryService { .map_err(MemoryError::Join)? } + /// Patch a memory in place (MCP-06), returning the full post-update view + /// plus `embedding_status` (D-10), or `Ok(None)` for an unknown id (clean + /// not-found, never an error — the [`MemoryService::forget`] contract). + /// + /// Order mirrors [`MemoryService::store`]: validate the patch FIRST (so a + /// bad patch never costs an embed round-trip — D-06 seam), stamp the clock, + /// then embed the new content best-effort ONLY when the patch carries + /// content (tags/scope/TTL-only patches never touch the embedder). An embed + /// failure never fails the update: the row lands pending + /// (`embedding_status = 0`) and the sweep backfills it (SEARCH-03). The + /// store applies the patch, the vector swap and the freshness bump (D-09) + /// in one writer transaction. + pub async fn update( + &self, + id: i64, + patch: MemoryPatch, + ) -> Result, MemoryError> { + crate::domain::validate_patch(&patch)?; + let now = self.clock.now(); + let embedding = match &patch.content { + Some(Some(content)) => { + let inputs = [content.clone()]; + match self.embedder.embed(&inputs).await { + Ok(mut vectors) if !vectors.is_empty() => Some(vectors.swap_remove(0)), + Ok(_) => { + tracing::warn!( + "embedding skipped (empty batch response); updating with embedding_status = 0" + ); + None + } + Err(e) => { + tracing::warn!( + "embedding failed ({e}); updating with embedding_status = 0" + ); + None + } + } + } + Some(None) | None => None, + }; + let store = self.store.clone(); + tokio::task::spawn_blocking(move || store.update(id, patch, embedding, now)) + .await + .map_err(MemoryError::Join)? + } + /// Run one lifecycle sweep at `now` (STORE-04): delete TTL-expired rows, then /// materialize every survivor's `decay_score` so ranking stays cheap between /// reads. Decay never deletes — only TTL expiry (here) and `forget` remove. diff --git a/crates/agent-memory-core/src/store/mod.rs b/crates/agent-memory-core/src/store/mod.rs index f772f01..34abb28 100644 --- a/crates/agent-memory-core/src/store/mod.rs +++ b/crates/agent-memory-core/src/store/mod.rs @@ -10,7 +10,7 @@ pub mod migrations; pub mod sqlite; use crate::decay::{DecayConfig, RankWeights}; -use crate::domain::{MemoryError, MemoryType, MemoryView, NewMemory}; +use crate::domain::{MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, UpdatedMemory}; use crate::service::{ListArgs, SearchArgs}; /// Synchronous persistence interface, implemented by [`sqlite::SqliteStore`]. @@ -84,6 +84,26 @@ pub trait Store: Send + Sync { /// RESEARCH Pitfall 3 / T-02-05). fn forget(&self, id: i64) -> Result; + /// Patch a memory IN PLACE (MCP-06 / D-08), returning the post-update row, + /// or `Ok(None)` if no row with that id exists (clean not-found, never an + /// error — the [`Store::forget`] contract). + /// + /// One writer transaction: the `memories` UPDATE (the FTS5 update trigger + /// re-mirrors content), the freshness bump folded into the same statement + /// (`last_accessed = now`, `access_count + 1` — D-09), and, when the patch + /// carries new content, the explicit `vec_memories` invalidation (vec0 + /// ignores triggers). With new content, a `Some` `embedding` is written and + /// `embedding_status` becomes 1; `None` leaves the row pending (0) for the + /// sweep backfill. The id never changes: an update is never a DELETE+INSERT. + /// `patch` must already be validated by the service. + fn update( + &self, + id: i64, + patch: MemoryPatch, + embedding: Option>, + now: i64, + ) -> Result, MemoryError>; + /// Bump the recency of the given ids: set `last_accessed = now` and increment /// `access_count`. Runs on the writer lane. A no-op for an empty id slice. fn bump_access(&self, ids: &[i64], now: i64) -> Result<(), MemoryError>; diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index b562ea2..c99b486 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -20,7 +20,7 @@ use rusqlite::{params, Connection, OptionalExtension}; use rusqlite_migration::Migrations; use crate::decay::{DecayConfig, RankWeights, PINNED_HALF_LIFE_MULTIPLIER}; -use crate::domain::{MemoryError, MemoryType, MemoryView, NewMemory}; +use crate::domain::{MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, UpdatedMemory}; use crate::service::{ListArgs, SearchArgs}; use crate::store::migrations::{migrations, LATEST_SCHEMA_VERSION}; use crate::store::Store; @@ -600,6 +600,131 @@ impl Store for SqliteStore { Ok(changed > 0) } + fn update( + &self, + id: i64, + patch: MemoryPatch, + embedding: Option>, + now: i64, + ) -> Result, MemoryError> { + let mut conn = self + .writer + .lock() + .map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; + let tx = conn.transaction()?; + + // Current values, read inside the writer tx so the merge below cannot + // race another write. Absent → clean not-found (the tx rolls back on + // drop; nothing was written). + let current = tx + .query_row( + "SELECT content, tags, scope, expires_at FROM memories WHERE id = ?1", + params![id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + )) + }, + ) + .optional()?; + let Some((cur_content, cur_tags_json, cur_scope, cur_expires_at)) = current else { + return Ok(None); + }; + + // D-08 merge in Rust: None = unchanged, Some(None) = clear, + // Some(Some(v)) = set. The service has already rejected null/empty + // content, so a `Some(None)` content is unreachable — treated as + // unchanged rather than trusted to clear a NOT NULL column. + let (content, content_changed) = match patch.content { + Some(Some(new_content)) => (new_content, true), + Some(None) | None => (cur_content, false), + }; + let tags_json = match patch.tags { + // Same JSON text encoding insert() writes. + Some(Some(tags)) => serde_json::to_string(&tags).unwrap_or_else(|_| "[]".to_string()), + Some(None) => "[]".to_string(), + None => cur_tags_json, + }; + let scope = match patch.scope { + Some(Some(scope)) => Some(scope), + // NULL = global: the default insert() stores for an omitted scope. + Some(None) => None, + None => cur_scope, + }; + let expires_at = match patch.ttl_secs { + // Recomputed from patch time; ttl bounds already validated, so the + // addition cannot overflow (MAX_TTL_SECS keeps it far from i64::MAX). + Some(Some(ttl)) => Some(now + ttl), + Some(None) => None, + None => cur_expires_at, + }; + + // INVARIANT — id immutability: the row is updated IN PLACE, never + // removed and re-inserted. `memories.id` is a plain INTEGER PRIMARY KEY + // whose rowids SQLite may recycle, so a delete+insert "update" could + // silently re-attach relation edges to an unrelated memory (T-04-08). + // + // D-09: the freshness bump rides in the same statement (bump_access + // semantics) — an update is the strongest recency signal. decay_score + // is reset to 1.0, the exact value of the decay curve at + // last_accessed = now (insert() stores the same literal). + // + // The memories_au trigger re-mirrors the FTS5 content row for free. + // All values bound — never formatted into the SQL (T-04-06). + tx.execute( + "UPDATE memories SET content = ?2, tags = ?3, scope = ?4, expires_at = ?5, \ + last_accessed = ?6, access_count = access_count + 1, decay_score = 1.0 \ + WHERE id = ?1", + params![id, content, tags_json, scope, expires_at, now], + )?; + + if content_changed { + // vec0 ignores triggers: the stale vector must go explicitly, in + // THIS transaction, or the old meaning would keep matching. + tx.execute("DELETE FROM vec_memories WHERE memory_id = ?1", params![id])?; + match embedding { + Some(vector) => { + let blob: &[u8] = bytemuck::cast_slice(&vector); + tx.execute( + "INSERT INTO vec_memories(memory_id, embedding) VALUES (?1, ?2)", + params![id, blob], + )?; + tx.execute( + "UPDATE memories SET embedding_status = 1 WHERE id = ?1", + params![id], + )?; + } + None => { + // Embed outage (or skipped): pending — the sweep backfill + // re-embeds it once the embedder recovers (SEARCH-03). + tx.execute( + "UPDATE memories SET embedding_status = 0 WHERE id = ?1", + params![id], + )?; + } + } + } + + let updated = tx.query_row( + "SELECT id, mem_type, content, tags, source, scope, base_weight, \ + decay_score, access_count, created_at, last_accessed, expires_at, \ + embedding_status \ + FROM memories WHERE id = ?1", + params![id], + |row| { + Ok(UpdatedMemory { + view: row_to_view(row)?, + embedding_status: row.get("embedding_status")?, + }) + }, + )?; + tx.commit()?; + Ok(Some(updated)) + } + fn bump_access(&self, ids: &[i64], now: i64) -> Result<(), MemoryError> { if ids.is_empty() { return Ok(()); diff --git a/crates/agent-memory-core/tests/update.rs b/crates/agent-memory-core/tests/update.rs new file mode 100644 index 0000000..5b2197c --- /dev/null +++ b/crates/agent-memory-core/tests/update.rs @@ -0,0 +1,519 @@ +//! MCP-06: `memory_update` patches a memory IN PLACE — content, tags, scope and +//! TTL — with the FTS5 mirror re-synced and the vector invalidated/re-embedded +//! in the SAME writer transaction. +//! +//! Locked decisions proven here: D-08 (JSON-merge-patch: absent = unchanged, +//! explicit null = clear where legal, content never null/empty), D-09 (an +//! update is a freshness signal: `last_accessed = now`, `access_count + 1`), +//! D-10 (the response is the full post-update view plus `embedding_status`). +//! +//! Harness: a REAL on-disk temp DB, an injected `TestClock`, and a local +//! `SwitchEmbedder` that wraps the programmed `FakeEmbedder`, counts calls, and +//! can be flipped into an outage — so one service instance can store while +//! Ollama is "up", update while it is "down", and backfill once it recovers. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; + +use agent_memory_core::clock::{Clock, TestClock}; +use agent_memory_core::decay::DecayConfig; +use agent_memory_core::domain::{MemoryError, MemoryPatch, MemoryType, NewMemory, MAX_TTL_SECS}; +use agent_memory_core::embed::{ + BoxFuture, EmbedError, Embedder, EmbedderHealth, FakeEmbedder, EMBEDDING_DIM, +}; +use agent_memory_core::service::{ListArgs, MemoryService, SearchArgs, SearchMode}; +use agent_memory_core::store::sqlite::{register_vec_extension, SqliteStore}; + +/// A programmed [`FakeEmbedder`] behind an outage switch and a call counter. +struct SwitchEmbedder { + inner: FakeEmbedder, + down: AtomicBool, + calls: AtomicUsize, +} + +impl SwitchEmbedder { + fn new(vectors: HashMap>) -> Self { + SwitchEmbedder { + inner: FakeEmbedder::with_vectors(vectors), + down: AtomicBool::new(false), + calls: AtomicUsize::new(0), + } + } + + fn set_down(&self, down: bool) { + self.down.store(down, Ordering::SeqCst); + } + + fn calls(&self) -> usize { + self.calls.load(Ordering::SeqCst) + } +} + +impl Embedder for SwitchEmbedder { + fn embed<'a>( + &'a self, + inputs: &'a [String], + ) -> BoxFuture<'a, Result>, EmbedError>> { + self.calls.fetch_add(1, Ordering::SeqCst); + if self.down.load(Ordering::SeqCst) { + return Box::pin(async { + Err(EmbedError::InvalidResponse( + "SwitchEmbedder is down".to_string(), + )) + }); + } + self.inner.embed(inputs) + } + + fn health<'a>(&'a self) -> BoxFuture<'a, Result> { + self.inner.health() + } +} + +struct Harness { + service: MemoryService, + clock: Arc, + embedder: Arc, + db_path: PathBuf, + _dir: tempfile::TempDir, +} + +fn harness(vectors: HashMap>) -> Harness { + let dir = tempfile::tempdir().expect("tempdir"); + let db_path = dir.path().join("memory.db"); + let store = SqliteStore::open(&db_path).expect("open store"); + let clock = Arc::new(TestClock::new(1_000)); + let embedder = Arc::new(SwitchEmbedder::new(vectors)); + let service = MemoryService::new( + Arc::new(store), + clock.clone() as Arc, + embedder.clone() as Arc, + DecayConfig::default(), + ); + Harness { + service, + clock, + embedder, + db_path, + _dir: dir, + } +} + +fn new_memory(content: &str) -> NewMemory { + NewMemory { + content: content.to_string(), + mem_type: MemoryType::Pattern, + tags: vec!["original".to_string()], + source: Some("update-test".to_string()), + scope: Some("project-a".to_string()), + ttl_secs: None, + } +} + +fn content_patch(content: &str) -> MemoryPatch { + MemoryPatch { + content: Some(Some(content.to_string())), + ..Default::default() + } +} + +/// A 768-dim unit vector along the given axis. +fn unit_axis(axis: usize) -> Vec { + let mut v = vec![0.0_f32; EMBEDDING_DIM]; + v[axis] = 1.0; + v +} + +/// `a * axis(i) + b * axis(j)` — a unit vector when `a² + b² = 1`. +fn mix(i: usize, a: f32, j: usize, b: f32) -> Vec { + let mut v = vec![0.0_f32; EMBEDDING_DIM]; + v[i] = a; + v[j] = b; + v +} + +/// Raw row columns the wire view does not expose. +struct RawRow { + embedding_status: i64, + access_count: i64, + last_accessed: i64, + expires_at: Option, + vec_rows: i64, +} + +fn raw_row(db_path: &Path, id: i64) -> RawRow { + register_vec_extension().expect("vec0 registers"); + let conn = rusqlite::Connection::open(db_path).expect("open raw connection"); + let (embedding_status, access_count, last_accessed, expires_at) = conn + .query_row( + "SELECT embedding_status, access_count, last_accessed, expires_at \ + FROM memories WHERE id = ?1", + [id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .expect("row exists"); + let vec_rows = conn + .query_row( + "SELECT count(*) FROM vec_memories WHERE memory_id = ?1", + [id], + |row| row.get(0), + ) + .expect("vec count"); + RawRow { + embedding_status, + access_count, + last_accessed, + expires_at, + vec_rows, + } +} + +const OLD_CONTENT: &str = "Postgres is the database"; +const NEW_CONTENT: &str = "SQLite is the database"; +const OLD_QUERY: &str = "which relational server do we run"; +const NEW_QUERY: &str = "which embedded file database do we use"; + +#[tokio::test] +async fn content_update_re_embeds_so_the_old_meaning_stops_matching() { + // Phase success criterion 1 (the re-embed kill-test). Four distractors sit + // near the OLD meaning, so with limit 1 (k = 4) a row that kept its stale + // vector would still rank first for the old query; a re-embedded row is + // pushed out of the candidate set entirely. + let mut vectors = HashMap::new(); + vectors.insert(OLD_CONTENT.to_string(), unit_axis(0)); + vectors.insert(NEW_CONTENT.to_string(), unit_axis(1)); + vectors.insert(OLD_QUERY.to_string(), unit_axis(0)); + vectors.insert(NEW_QUERY.to_string(), unit_axis(1)); + for i in 0..4 { + vectors.insert(format!("distractor {i}"), mix(0, 0.8, 10 + i, 0.6)); + } + let h = harness(vectors); + + let id = h + .service + .store(new_memory(OLD_CONTENT)) + .await + .expect("store target"); + for i in 0..4 { + h.service + .store(new_memory(&format!("distractor {i}"))) + .await + .expect("store distractor"); + } + + let before = h + .service + .search(SearchArgs { + query: OLD_QUERY.to_string(), + limit: Some(1), + ..Default::default() + }) + .await + .expect("search before update"); + assert_eq!(before.search_mode, SearchMode::Semantic); + assert_eq!( + before.results[0].id, id, + "sanity: the old meaning matches the target before the update" + ); + + let updated = h + .service + .update(id, content_patch(NEW_CONTENT)) + .await + .expect("update succeeds") + .expect("id exists"); + assert_eq!(updated.view.content, NEW_CONTENT); + assert_eq!(updated.embedding_status, 1, "re-embedded in the same tx"); + + let old = h + .service + .search(SearchArgs { + query: OLD_QUERY.to_string(), + limit: Some(1), + ..Default::default() + }) + .await + .expect("old-meaning search"); + assert_eq!(old.search_mode, SearchMode::Semantic); + assert!( + old.results.iter().all(|v| v.id != id), + "the old meaning must no longer return the updated row: {:?}", + old.results + ); + + let new = h + .service + .search(SearchArgs { + query: NEW_QUERY.to_string(), + limit: Some(1), + ..Default::default() + }) + .await + .expect("new-meaning search"); + assert_eq!(new.search_mode, SearchMode::Semantic); + assert_eq!(new.results[0].id, id, "the new meaning returns the row"); + + // Exactly one vector row: the old one was replaced, not shadowed. + assert_eq!(raw_row(&h.db_path, id).vec_rows, 1); + + // FTS5 was re-mirrored by the update trigger: keyword search (embedder + // down) finds the new wording and no longer finds the old. + h.embedder.set_down(true); + let kw_old = h + .service + .search(SearchArgs { + query: "Postgres".to_string(), + ..Default::default() + }) + .await + .expect("keyword search old"); + assert_eq!(kw_old.search_mode, SearchMode::Keyword); + assert!(kw_old.results.is_empty(), "old wording left the FTS mirror"); + let kw_new = h + .service + .search(SearchArgs { + query: "SQLite".to_string(), + ..Default::default() + }) + .await + .expect("keyword search new"); + assert_eq!(kw_new.results.len(), 1); + assert_eq!(kw_new.results[0].id, id); +} + +#[tokio::test] +async fn embed_outage_during_update_falls_back_to_pending_and_sweep_backfills() { + let h = harness(HashMap::new()); + let id = h + .service + .store(new_memory(OLD_CONTENT)) + .await + .expect("store"); + assert_eq!(raw_row(&h.db_path, id).embedding_status, 1); + + h.embedder.set_down(true); + let updated = h + .service + .update(id, content_patch(NEW_CONTENT)) + .await + .expect("an embed outage never fails the update") + .expect("id exists"); + assert_eq!(updated.view.content, NEW_CONTENT); + assert_eq!( + updated.embedding_status, 0, + "the response shows the pending re-embed" + ); + let raw = raw_row(&h.db_path, id); + assert_eq!(raw.embedding_status, 0); + assert_eq!(raw.vec_rows, 0, "the stale vector was invalidated"); + + h.embedder.set_down(false); + h.service.sweep(h.clock.now()).await.expect("sweep"); + let raw = raw_row(&h.db_path, id); + assert_eq!(raw.embedding_status, 1, "one sweep backfills the vector"); + assert_eq!(raw.vec_rows, 1); +} + +#[tokio::test] +async fn update_bumps_freshness_by_exactly_one_access() { + let h = harness(HashMap::new()); + let id = h + .service + .store(new_memory("fresh me")) + .await + .expect("store"); + let before = raw_row(&h.db_path, id); + assert_eq!(before.last_accessed, 1_000); + + h.clock.advance(5_000); + let updated = h + .service + .update( + id, + MemoryPatch { + tags: Some(Some(vec!["touched".to_string()])), + ..Default::default() + }, + ) + .await + .expect("update") + .expect("id exists"); + + let after = raw_row(&h.db_path, id); + assert_eq!(after.last_accessed, 6_000, "last_accessed = patch-time now"); + assert_eq!(after.access_count, before.access_count + 1); + assert_eq!(updated.view.last_accessed, 6_000); + assert_eq!(updated.view.decay_score, 1.0, "just touched = fully fresh"); +} + +#[tokio::test] +async fn update_returns_the_full_post_update_view_with_embedding_status() { + let h = harness(HashMap::new()); + let id = h + .service + .store(new_memory(OLD_CONTENT)) + .await + .expect("store"); + + let updated = h + .service + .update( + id, + MemoryPatch { + content: Some(Some(NEW_CONTENT.to_string())), + tags: Some(Some(vec!["a".to_string(), "b".to_string()])), + scope: Some(Some("project-b".to_string())), + ttl_secs: Some(Some(60)), + }, + ) + .await + .expect("update") + .expect("id exists"); + + assert_eq!(updated.view.id, id); + assert_eq!(updated.view.content, NEW_CONTENT); + assert_eq!( + updated.view.mem_type, + MemoryType::Pattern, + "type is immutable" + ); + assert_eq!(updated.view.tags, vec!["a".to_string(), "b".to_string()]); + assert_eq!(updated.view.scope.as_deref(), Some("project-b")); + assert_eq!(updated.view.created_at, 1_000, "created_at is immutable"); + assert_eq!(updated.embedding_status, 1); + assert_eq!(raw_row(&h.db_path, id).expires_at, Some(1_060)); + + // Wire shape: the MemoryView fields flattened, plus embedding_status. + let json = serde_json::to_value(&updated).expect("serializes"); + assert_eq!(json["id"], id); + assert_eq!(json["type"], "PATTERN"); + assert_eq!(json["content"], NEW_CONTENT); + assert_eq!(json["embedding_status"], 1); + assert!(json.get("view").is_none(), "the view is flattened"); + + // The list read path sees the same post-update row. + let listed = h.service.list(ListArgs::default()).await.expect("list"); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0], updated.view); +} + +#[tokio::test] +async fn updating_an_unknown_id_is_a_clean_not_found() { + let h = harness(HashMap::new()); + let outcome = h + .service + .update(424_242, content_patch("nobody home")) + .await + .expect("an unknown id is data, never an error"); + assert!(outcome.is_none()); +} + +#[tokio::test] +async fn a_tags_only_patch_never_calls_the_embedder() { + let h = harness(HashMap::new()); + let id = h + .service + .store(new_memory(OLD_CONTENT)) + .await + .expect("store"); + let calls_after_store = h.embedder.calls(); + let before = raw_row(&h.db_path, id); + + let updated = h + .service + .update( + id, + MemoryPatch { + tags: Some(Some(vec!["retagged".to_string()])), + scope: Some(None), + ttl_secs: Some(Some(30)), + ..Default::default() + }, + ) + .await + .expect("update") + .expect("id exists"); + + assert_eq!( + h.embedder.calls(), + calls_after_store, + "no content in the patch → zero embed calls" + ); + assert_eq!(updated.view.content, OLD_CONTENT, "content unchanged"); + assert_eq!(updated.embedding_status, before.embedding_status); + let after = raw_row(&h.db_path, id); + assert_eq!(after.embedding_status, before.embedding_status); + assert_eq!(after.vec_rows, 1, "the vector row is untouched"); +} + +#[tokio::test] +async fn an_invalid_patch_is_rejected_before_the_embedder_is_called() { + let h = harness(HashMap::new()); + let id = h + .service + .store(new_memory(OLD_CONTENT)) + .await + .expect("store"); + let calls_after_store = h.embedder.calls(); + + let cases: Vec<(MemoryPatch, &str)> = vec![ + ( + MemoryPatch::default(), + "at least one of content, tags, scope, ttl_secs must be provided", + ), + ( + MemoryPatch { + content: Some(None), + ..Default::default() + }, + "content must be a non-empty string", + ), + (content_patch(""), "content must be a non-empty string"), + ( + content_patch(" \n\t "), + "content must be a non-empty string", + ), + ( + MemoryPatch { + content: Some(Some("valid".to_string())), + ttl_secs: Some(Some(0)), + ..Default::default() + }, + "ttl_secs must be between", + ), + ( + MemoryPatch { + ttl_secs: Some(Some(MAX_TTL_SECS + 1)), + ..Default::default() + }, + "ttl_secs must be between", + ), + ]; + + for (patch, expected) in cases { + let err = h + .service + .update(id, patch.clone()) + .await + .expect_err("invalid patch is rejected"); + match err { + MemoryError::InvalidArgument(msg) => assert!( + msg.contains(expected), + "patch {patch:?}: expected message containing {expected:?}, got {msg:?}" + ), + other => panic!("patch {patch:?}: expected InvalidArgument, got {other:?}"), + } + } + assert_eq!( + h.embedder.calls(), + calls_after_store, + "validation runs before any embed hop" + ); + + // Nothing was written: the row is exactly as stored. + let listed = h.service.list(ListArgs::default()).await.expect("list"); + assert_eq!(listed[0].content, OLD_CONTENT); + assert_eq!(raw_row(&h.db_path, id).access_count, 0); +} diff --git a/crates/agent-memory/src/main.rs b/crates/agent-memory/src/main.rs index 36269f7..049c25b 100644 --- a/crates/agent-memory/src/main.rs +++ b/crates/agent-memory/src/main.rs @@ -7,6 +7,7 @@ mod config; mod mcp; mod rest; +mod serde_util; use std::net::SocketAddr; use std::path::PathBuf; diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index 0e9ee19..4db2ac4 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -12,13 +12,13 @@ use rmcp::handler::server::wrapper::Parameters; use rmcp::model::{CallToolResult, Content, Implementation, ServerCapabilities, ServerInfo}; use rmcp::{schemars, tool, tool_handler, tool_router, ErrorData as McpError, ServerHandler}; -use agent_memory_core::domain::{MemoryError, MemoryType, NewMemory}; +use agent_memory_core::domain::{MemoryError, MemoryPatch, MemoryType, NewMemory}; use agent_memory_core::embed::Embedder; use agent_memory_core::service::{ ListArgs as ServiceListArgs, MemoryService, SearchArgs as ServiceSearchArgs, }; -/// Shared two-tier [`MemoryError`] → [`McpError`] mapping for all four tool +/// Shared two-tier [`MemoryError`] → [`McpError`] mapping for every tool /// service calls — the mirror of `rest/handlers.rs`'s `map_memory_error` /// (closes review WR-04): client input (`InvalidType`, `InvalidQuery`, /// `InvalidArgument`) reports @@ -27,8 +27,8 @@ use agent_memory_core::service::{ /// argument-repair loops over a DB fault (T-02G-03). /// /// `Internal` covers server faults with no client cause, e.g. a poisoned writer -/// mutex (WR-02). MCP's clean not-found is data — the `Ok(false)` forget -/// result — never an error variant. No catch-all arm: a new `MemoryError` +/// mutex (WR-02). MCP's clean not-found is data — forget's `Ok(false)` and +/// update's `Ok(None)` — never an error variant. No catch-all arm: a new `MemoryError` /// variant must be classified here at compile time. fn map_mcp_error(e: MemoryError) -> McpError { match &e { @@ -122,6 +122,66 @@ pub struct ForgetArgs { pub id: i64, } +/// Arguments for `memory_update` (MCP-06) — a JSON-merge-patch over one memory. +/// +/// Omit a field to leave it unchanged; send `null` to clear it (`tags` → empty, +/// `scope` → global, `ttl_secs` → no expiry); send a value to set it. `content` +/// can be replaced but never cleared or emptied. At least one field besides +/// `id` is required. `type` and `source` cannot be changed. +/// +/// `skip_serializing_if` on the patch fields is for the SCHEMA, not the wire: +/// it stops schemars from advertising `"default": null`. A client that fills +/// in schema defaults would otherwise send explicit nulls — and null means +/// "clear" here, so every patch would wipe tags/scope/ttl. +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct UpdateArgs { + /// The numeric id of the memory to update. + pub id: i64, + /// New content (re-indexed and re-embedded). Omit to keep; never null or empty. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::serde_util::deserialize_explicit_null" + )] + pub content: Option>, + /// Replacement tag list. Omit to keep; `null` clears to an empty list. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::serde_util::deserialize_explicit_null" + )] + pub tags: Option>>, + /// New logical scope. Omit to keep; `null` resets to global. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::serde_util::deserialize_explicit_null" + )] + pub scope: Option>, + /// New time-to-live in seconds, counted from now. Omit to keep; `null` removes the expiry. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::serde_util::deserialize_explicit_null" + )] + pub ttl_secs: Option>, +} + +impl UpdateArgs { + /// Split into the target id and the core patch, field-for-field. + fn into_patch(self) -> (i64, MemoryPatch) { + ( + self.id, + MemoryPatch { + content: self.content, + tags: self.tags, + scope: self.scope, + ttl_secs: self.ttl_secs, + }, + ) + } +} + /// The MCP server. Holds shared state and the generated tool router. #[derive(Clone)] pub struct MemoryMcp { @@ -256,6 +316,32 @@ impl MemoryMcp { .map_err(|e| McpError::internal_error(e.to_string(), None))?; Ok(CallToolResult::success(vec![Content::text(json)])) } + + #[tool( + description = "Update a memory in place by id (JSON merge-patch): omit a field to keep it, send null to clear it (tags -> [], scope -> global, ttl_secs -> no expiry), send a value to set it. content can be replaced but never cleared; a new content is re-indexed and re-embedded. Returns the full updated memory including embedding_status (0 = re-embed pending), or a clean not-found result for an unknown id." + )] + async fn memory_update( + &self, + Parameters(args): Parameters, + ) -> Result { + let (id, patch) = args.into_patch(); + let updated = self + .state + .service + .update(id, patch) + .await + .map_err(map_mcp_error)?; + + // Ok(None) is a clean not-found, NOT an error — the memory_forget shape. + let json = match updated { + Some(updated) => serde_json::to_string(&updated), + None => serde_json::to_string( + &serde_json::json!({ "id": id, "updated": false, "reason": "not_found" }), + ), + } + .map_err(|e| McpError::internal_error(e.to_string(), None))?; + Ok(CallToolResult::success(vec![Content::text(json)])) + } } #[tool_handler(router = self.tool_router)] @@ -267,7 +353,7 @@ impl ServerHandler for MemoryMcp { info.capabilities = ServerCapabilities::builder().enable_tools().build(); info.server_info = Implementation::from_build_env(); info.instructions = Some( - "Persistent typed agent memory: store, list, search, and forget typed memories." + "Persistent typed agent memory: store, list, search, update, and forget typed memories." .to_string(), ); info @@ -278,6 +364,142 @@ impl ServerHandler for MemoryMcp { mod tests { use super::*; + use agent_memory_core::clock::SystemClock; + use agent_memory_core::decay::DecayConfig; + use agent_memory_core::embed::ollama::OllamaClient; + use agent_memory_core::store::sqlite::SqliteStore; + + /// A loopback port nothing listens on: stores land pending (status 0) and + /// no test depends on a local Ollama. + const DEAD_OLLAMA: &str = "http://127.0.0.1:9"; + + fn test_server(dir: &tempfile::TempDir) -> MemoryMcp { + let store = SqliteStore::open(&dir.path().join("mem.db")).expect("store opens"); + let embedder: Arc = Arc::new(OllamaClient::new(DEAD_OLLAMA)); + let service = MemoryService::new( + Arc::new(store), + Arc::new(SystemClock), + embedder.clone(), + DecayConfig::default(), + ); + MemoryMcp::new(Arc::new(AppState { service, embedder })) + } + + /// The JSON text payload of a successful tool result. + fn result_json(result: &CallToolResult) -> serde_json::Value { + let value = serde_json::to_value(result).expect("tool result serializes"); + let text = value + .pointer("/content/0/text") + .and_then(|t| t.as_str()) + .unwrap_or_else(|| panic!("no text content in tool result: {value}")); + serde_json::from_str(text).expect("tool text is JSON") + } + + async fn store_one(server: &MemoryMcp, content: &str) -> i64 { + let args: StoreArgs = serde_json::from_value(serde_json::json!({ + "content": content, + "type": "PATTERN", + "tags": ["before"], + "scope": "project-a", + "ttl_secs": 3600 + })) + .expect("store args deserialize"); + let result = server + .memory_store(Parameters(args)) + .await + .expect("store succeeds"); + result_json(&result).as_i64().expect("store returns an id") + } + + fn update_args(json: serde_json::Value) -> UpdateArgs { + serde_json::from_value(json).expect("update args deserialize") + } + + #[tokio::test] + async fn memory_update_returns_the_full_updated_memory() { + let dir = tempfile::tempdir().expect("tempdir"); + let server = test_server(&dir); + let id = store_one(&server, "old wording").await; + + let result = server + .memory_update(Parameters(update_args(serde_json::json!({ + "id": id, + "content": "new wording", + "tags": ["after"] + })))) + .await + .expect("update succeeds"); + + let body = result_json(&result); + assert_eq!(body["id"], id); + assert_eq!(body["content"], "new wording"); + assert_eq!(body["type"], "PATTERN"); + assert_eq!(body["tags"], serde_json::json!(["after"])); + assert_eq!(body["scope"], "project-a", "omitted scope is unchanged"); + // Dead embedder: the re-embed is pending, and the caller can see it. + assert_eq!(body["embedding_status"], 0); + } + + #[tokio::test] + async fn memory_update_of_an_unknown_id_is_a_clean_not_found() { + let dir = tempfile::tempdir().expect("tempdir"); + let server = test_server(&dir); + + let result = server + .memory_update(Parameters(update_args(serde_json::json!({ + "id": 999_999, + "content": "nobody home" + })))) + .await + .expect("an unknown id is a successful tool call, not an error"); + + assert_eq!( + result_json(&result), + serde_json::json!({ "id": 999_999, "updated": false, "reason": "not_found" }) + ); + } + + #[tokio::test] + async fn memory_update_with_empty_content_is_invalid_params() { + let dir = tempfile::tempdir().expect("tempdir"); + let server = test_server(&dir); + let id = store_one(&server, "keep me").await; + + let err = server + .memory_update(Parameters(update_args(serde_json::json!({ + "id": id, + "content": "" + })))) + .await + .expect_err("empty content is rejected"); + assert_eq!(err.code, McpError::invalid_params("x", None).code); + assert!(err.message.contains("content must be a non-empty string")); + } + + #[test] + fn update_args_schema_makes_only_id_required() { + let schema = + serde_json::to_value(schemars::schema_for!(UpdateArgs)).expect("schema serializes"); + assert_eq!( + schema["required"], + serde_json::json!(["id"]), + "patch fields are optional in the tool schema: {schema}" + ); + for field in ["content", "tags", "scope", "ttl_secs"] { + let property = &schema["properties"][field]; + assert!( + property.is_object(), + "{field} is described in the tool schema: {schema}" + ); + // null means "clear" in a merge-patch: advertising it as the + // default would invite clients to wipe fields they never touched. + assert!( + property.get("default").is_none(), + "{field} must not advertise a default: {property}" + ); + } + } + #[test] fn map_mcp_error_splits_client_and_internal_tiers() { let invalid_params_code = McpError::invalid_params("x", None).code; diff --git a/crates/agent-memory/src/serde_util.rs b/crates/agent-memory/src/serde_util.rs new file mode 100644 index 0000000..9969d6b --- /dev/null +++ b/crates/agent-memory/src/serde_util.rs @@ -0,0 +1,88 @@ +//! Serde helpers shared by both transports' request DTOs (MCP `UpdateArgs` and +//! REST `UpdateBody`), so the two can never disagree on patch semantics. + +use serde::{Deserialize, Deserializer}; + +/// Deserialize a JSON-merge-patch member into `Option>`, keeping +/// "explicit `null`" distinct from "absent" (D-08; serde issue #1042). +/// +/// Plain `Option>` cannot do this: serde maps a JSON `null` to the +/// OUTER `None`, silently turning "clear this field" into "leave it alone". +/// Use it together with `#[serde(default)]`: +/// +/// ```ignore +/// #[serde(default, deserialize_with = "crate::serde_util::deserialize_explicit_null")] +/// pub ttl_secs: Option>, +/// ``` +/// +/// - member absent → the field's `Default`, `None` (this function never runs); +/// - `null` → `Some(None)`; +/// - a value → `Some(Some(v))`. +pub(crate) fn deserialize_explicit_null<'de, D, T>( + deserializer: D, +) -> Result>, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(deserializer).map(Some) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, serde::Deserialize)] + struct Probe { + #[serde(default, deserialize_with = "deserialize_explicit_null")] + ttl_secs: Option>, + #[serde(default, deserialize_with = "deserialize_explicit_null")] + tags: Option>>, + #[serde(default, deserialize_with = "deserialize_explicit_null")] + scope: Option>, + } + + fn probe(json: &str) -> Probe { + serde_json::from_str(json).expect("probe JSON deserializes") + } + + #[test] + fn an_absent_member_is_none() { + let p = probe("{}"); + assert_eq!(p.ttl_secs, None); + assert_eq!(p.tags, None); + assert_eq!(p.scope, None); + } + + #[test] + fn an_explicit_null_is_some_none() { + let p = probe(r#"{"ttl_secs": null, "tags": null, "scope": null}"#); + assert_eq!(p.ttl_secs, Some(None)); + assert_eq!(p.tags, Some(None)); + assert_eq!(p.scope, Some(None)); + } + + #[test] + fn a_value_is_some_some() { + let p = probe(r#"{"ttl_secs": 60, "tags": ["a", "b"], "scope": "project-x"}"#); + assert_eq!(p.ttl_secs, Some(Some(60))); + assert_eq!(p.tags, Some(Some(vec!["a".to_string(), "b".to_string()]))); + assert_eq!(p.scope, Some(Some("project-x".to_string()))); + } + + #[test] + fn a_wrong_typed_value_is_still_a_deserialize_error() { + let err = serde_json::from_str::(r#"{"ttl_secs": "soon"}"#) + .expect_err("a string is not an i64"); + assert!(err.to_string().contains("invalid type"), "got: {err}"); + } + + #[test] + fn it_works_through_from_value_too() { + // rmcp hands tool arguments over as a serde_json::Value map. + let p: Probe = serde_json::from_value(serde_json::json!({ "scope": null })) + .expect("deserializes from a Value"); + assert_eq!(p.scope, Some(None)); + assert_eq!(p.ttl_secs, None); + } +} From 3a3d4fef168a06c87f4fc22437e8ecdfbc725eed Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:09:49 +0200 Subject: [PATCH 06/42] feat(04-02): PATCH /api/memories/{id} + merge-patch matrix on both transports - UpdateBody REST DTO (serde-only, same explicit-null helper as UpdateArgs) + update_handler: 200 UpdatedMemory, 404 {"id","updated":false, "reason":"not_found"}; body-extractor rejections reported as 400 - PATCH route on /api/memories/{id} beside DELETE - Matrix tests from raw JSON bodies: ttl_secs/tags/scope x absent/null/value on REST (through axum's Json extractor) and MCP (UpdateArgs -> MemoryPatch states + tool-level effects), plus a TestClock core matrix; ttl null survives a sweep past the old TTL - content null/empty, out-of-bounds ttl, empty patch, malformed bodies: 400 / invalid_params, never 500, never clamped - README: memory_update section (merge-patch table, embedding_status, freshness note) and the PATCH endpoint Co-Authored-By: Claude Opus 5.5 --- README.md | 70 ++++- crates/agent-memory-core/tests/update.rs | 155 ++++++++++ crates/agent-memory/src/mcp.rs | 180 ++++++++++++ crates/agent-memory/src/rest/handlers.rs | 352 ++++++++++++++++++++++- crates/agent-memory/src/rest/mod.rs | 9 +- 5 files changed, 755 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index aa7630b..8672700 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ Tool 10 in the [UnityInFlow](https://github.com/UnityInFlow) ecosystem. AI agents forget everything between sessions. `agent-memory` is a small [MCP](https://modelcontextprotocol.io) stdio server backed by an embedded SQLite -database: any MCP-speaking runtime can store, search, list, and forget structured -memories, and **every runtime pointed at the same database file shares the same +database: any MCP-speaking runtime can store, search, list, update, and forget +structured memories, and **every runtime pointed at the same database file shares the same memory** — so a decision recorded by one agent is recalled by the next, across sessions and across tools. @@ -101,13 +101,14 @@ Drop the same block into each agent runtime's MCP config; because they open the same `AGENT_MEMORY_DB` file (WAL-mode SQLite with a serialized writer), a memory stored in one is immediately retrievable in another. -## The four tools +## The five tools | Tool | Purpose | Validated parameters | |------|---------|----------------------| | `memory_store` | Persist a typed memory; returns its id. | `ttl_secs`: optional integer, valid range 1–3155760000 (~100 years); omitted = no expiry | | `memory_search` | Semantic (embedding) search when Ollama is reachable, FTS5 keyword otherwise — ranked by relevance × decay, recency-bumped on retrieval. | `limit`: optional integer, valid range 1–200; omitted defaults to 50. `tag`: exact tag match (case-sensitive) | | `memory_list` | List memories newest-first, with optional type/tag/scope filters. | `limit`: optional integer, valid range 1–200; omitted = unlimited. `tag`: exact tag match (case-sensitive) | +| `memory_update` | Patch a memory in place by id — content, tags, scope, TTL (clean not-found for unknown ids). | `content`: never null or empty. `ttl_secs`: 1–3155760000 when set; `null` = no expiry. At least one field besides `id` | | `memory_forget` | Delete a memory by id (clean not-found for unknown ids). | — | Out-of-bounds values are rejected as client errors (`invalid_params` over MCP) @@ -169,6 +170,52 @@ A present `limit` must be between 1 and 200 (omitted returns everything); a Returns `{ "id": 42, "deleted": true }`, or `deleted: false` for an unknown id. +### `memory_update` + +Patch an existing memory **in place** — its id never changes. The arguments +are a JSON merge-patch: each field you send is applied, each field you omit is +left alone, and an explicit `null` clears the field where clearing is legal. + +```json +{ "id": 42, "content": "We chose io.github.unityinflow as the Maven group (Sonatype-verified).", "ttl_secs": null } +``` + +| Field | Omitted | `null` | A value | +|-------|---------|--------|---------| +| `content` | unchanged | **rejected** (`invalid_params`) | replaced, then re-indexed and re-embedded | +| `tags` | unchanged | cleared to `[]` | replaced (the whole list) | +| `scope` | unchanged | reset to global (`null`) | replaced | +| `ttl_secs` | unchanged (the existing expiry stays) | expiry removed — the memory never expires | a new TTL counted from **now** (1–3155760000) | + +`type` and `source` cannot be changed; to retype a memory, store a new one and +forget the old. A patch with no field besides `id`, empty or whitespace-only +`content`, or an out-of-bounds `ttl_secs` is rejected as `invalid_params` +before anything is written. + +On success the result is the full updated memory — the same fields `memory_list` +returns — plus `embedding_status`: + +```json +{ "id": 42, "content": "We chose io.github.unityinflow as the Maven group (Sonatype-verified).", + "type": "DECISION", "tags": ["maven", "release"], "scope": "kore-runtime", + "decay_score": 1.0, "created_at": 1767225600, "last_accessed": 1767312000, + "embedding_status": 1 } +``` + +- **New content changes the meaning, everywhere.** The keyword index and the + embedding are replaced in the same transaction, so a search for the old + meaning stops matching the memory and the new meaning finds it. +- **`embedding_status`** is `1` when the stored vector matches the current + content and `0` when a re-embed is pending — the update succeeded while Ollama + was unreachable, and the background sweep re-embeds it once Ollama is back. + Until then semantic search cannot match it by meaning; keyword-fallback + search already sees the new content. +- **An update counts as a use.** It refreshes `last_accessed` (and the access + count), so an edited memory ranks as current rather than decayed. + +For an unknown id the result is `{ "id": 42, "updated": false, "reason": "not_found" }` +— a normal tool result, not an error. + ## Semantic search (optional Ollama) Out of the box, search uses SQLite FTS5 keyword ranking — fully offline, no @@ -211,6 +258,7 @@ agent-memory serve-rest --addr 127.0.0.1:8080 # custom loopback bind | `POST` | `/api/memories` | Store a memory (same fields as `memory_store`) | `ttl_secs`: 1–3155760000 when present; omitted = no expiry | | `GET` | `/api/memories` | List, with optional `type`/`tag`/`scope`/`limit` query params | `limit`: 1–200 when present; omitted = unlimited. `tag`: exact tag (case-sensitive) | | `POST` | `/api/search` | Search — returns the same `{search_mode, results}` envelope | `limit`: 1–200 when present; omitted defaults to 50. `tag`: exact tag (case-sensitive) | +| `PATCH` | `/api/memories/{id}` | Update in place — JSON merge-patch body with the same fields and semantics as `memory_update`; 200 returns the updated memory with `embedding_status` | as `memory_update`; 404 `{"id", "updated": false, "reason": "not_found"}` for unknown ids | | `DELETE` | `/api/memories/{id}` | Forget by id (404 with a structured body for unknown ids) | — | | `GET` | `/health` | Liveness + embedder reachability status | — | @@ -218,7 +266,16 @@ Out-of-bounds `limit`/`ttl_secs` values return **HTTP 400** with a JSON body whose `error` field names the field, range, and offending value — shaped `invalid argument: limit must be between 1 and 200 (got 0)` — never a 500 and never a silent clamp. The MCP transport rejects the same inputs identically as -`invalid_params`, because validation lives once at the shared service seam. +`invalid_params`, because validation lives once at the shared service seam. A +`PATCH` body that is not valid JSON, has a wrongly typed field, or is sent +without `Content-Type: application/json` is also a 400 with the same +`{"error": ...}` body. + +```bash +curl -X PATCH localhost:7437/api/memories/42 \ + -H 'content-type: application/json' \ + -d '{"tags": ["maven"], "ttl_secs": null}' +``` The REST daemon and the MCP stdio server can run concurrently against the same database file (WAL-mode SQLite with a serialized writer). @@ -280,6 +337,11 @@ A background task sweeps on an interval while the server runs: low its decay score falls — it simply ranks lower. Removal happens only via TTL or `memory_forget`. +Reads and updates both count as use: a search result is recency-bumped when it +is returned, and `memory_update` resets the memory's decay clock. A TTL can be +added, replaced (counted from the time of the update) or removed with +`memory_update`. + ## Development ```bash diff --git a/crates/agent-memory-core/tests/update.rs b/crates/agent-memory-core/tests/update.rs index 5b2197c..5421b3b 100644 --- a/crates/agent-memory-core/tests/update.rs +++ b/crates/agent-memory-core/tests/update.rs @@ -517,3 +517,158 @@ async fn an_invalid_patch_is_rejected_before_the_embedder_is_called() { assert_eq!(listed[0].content, OLD_CONTENT); assert_eq!(raw_row(&h.db_path, id).access_count, 0); } + +/// Store a memory carrying a 60 s TTL (expires at 1_060 on the harness clock). +async fn store_with_ttl(h: &Harness, content: &str) -> i64 { + let mut new = new_memory(content); + new.ttl_secs = Some(60); + h.service.store(new).await.expect("store") +} + +async fn listed_ids(h: &Harness) -> Vec { + h.service + .list(ListArgs::default()) + .await + .expect("list") + .iter() + .map(|v| v.id) + .collect() +} + +#[tokio::test] +async fn ttl_patch_matrix_absent_keeps_null_clears_value_resets_from_patch_time() { + let h = harness(HashMap::new()); + let absent = store_with_ttl(&h, "ttl absent").await; + let cleared = store_with_ttl(&h, "ttl cleared").await; + let reset = store_with_ttl(&h, "ttl reset").await; + + h.clock.advance(30); // now = 1_030, before the old 1_060 expiry + for (id, patch) in [ + ( + absent, + MemoryPatch { + tags: Some(Some(vec!["x".to_string()])), + ..Default::default() + }, + ), + ( + cleared, + MemoryPatch { + ttl_secs: Some(None), + ..Default::default() + }, + ), + ( + reset, + MemoryPatch { + ttl_secs: Some(Some(600)), + ..Default::default() + }, + ), + ] { + h.service + .update(id, patch) + .await + .expect("update") + .expect("id exists"); + } + assert_eq!(raw_row(&h.db_path, absent).expires_at, Some(1_060)); + assert_eq!(raw_row(&h.db_path, cleared).expires_at, None); + assert_eq!( + raw_row(&h.db_path, reset).expires_at, + Some(1_630), + "a new TTL counts from patch time" + ); + + // Advance past the OLD TTL and sweep. + h.clock.advance(100); // now = 1_130 + h.service.sweep(h.clock.now()).await.expect("sweep"); + let ids = listed_ids(&h).await; + assert!( + !ids.contains(&absent), + "absent ttl_secs keeps the old expiry" + ); + assert!(ids.contains(&cleared), "ttl_secs: null removes the expiry"); + assert!(ids.contains(&reset), "the reset TTL outlives the old one"); + + // Past the reset TTL too: only the cleared one is left. + h.clock.advance(1_000); // now = 2_130 + h.service.sweep(h.clock.now()).await.expect("sweep"); + assert_eq!(listed_ids(&h).await, vec![cleared]); +} + +#[tokio::test] +async fn tags_and_scope_patch_matrix_absent_null_value() { + let h = harness(HashMap::new()); + let id = h.service.store(new_memory("matrix")).await.expect("store"); + + // absent → unchanged (another field carries the patch) + let u = h + .service + .update( + id, + MemoryPatch { + ttl_secs: Some(Some(120)), + ..Default::default() + }, + ) + .await + .expect("update") + .expect("id exists"); + assert_eq!(u.view.tags, vec!["original".to_string()]); + assert_eq!(u.view.scope.as_deref(), Some("project-a")); + + // value → set + let u = h + .service + .update( + id, + MemoryPatch { + tags: Some(Some(vec!["n1".to_string(), "n2".to_string()])), + scope: Some(Some("project-z".to_string())), + ..Default::default() + }, + ) + .await + .expect("update") + .expect("id exists"); + assert_eq!(u.view.tags, vec!["n1".to_string(), "n2".to_string()]); + assert_eq!(u.view.scope.as_deref(), Some("project-z")); + + // null → cleared: tags empty, scope global (NULL — the insert default) + let u = h + .service + .update( + id, + MemoryPatch { + tags: Some(None), + scope: Some(None), + ..Default::default() + }, + ) + .await + .expect("update") + .expect("id exists"); + assert!(u.view.tags.is_empty()); + assert_eq!(u.view.scope, None); + + // The cleared row is visible to the scope/tag filters as cleared. + let in_old_scope = h + .service + .list(ListArgs { + scope: Some("project-z".to_string()), + ..Default::default() + }) + .await + .expect("list"); + assert!(in_old_scope.is_empty(), "the row left its old scope"); + let with_old_tag = h + .service + .list(ListArgs { + tag: Some("n1".to_string()), + ..Default::default() + }) + .await + .expect("list"); + assert!(with_old_tag.is_empty(), "the row lost its old tags"); +} diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index 4db2ac4..f4571e1 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -476,6 +476,186 @@ mod tests { assert!(err.message.contains("content must be a non-empty string")); } + /// Raw JSON text → `UpdateArgs` → core patch: the deserializer under test. + fn patch_from_raw(raw: &str) -> MemoryPatch { + serde_json::from_str::(raw) + .expect("raw update args deserialize") + .into_patch() + .1 + } + + #[test] + fn raw_update_args_carry_all_three_wire_states_into_the_patch() { + // D-08 / RESEARCH Pitfall 4: absent → None, null → Some(None), + // value → Some(Some(v)) — for each clearable field. + assert_eq!(patch_from_raw(r#"{"id": 1}"#), MemoryPatch::default()); + + let p = patch_from_raw(r#"{"id": 1, "ttl_secs": null, "tags": null, "scope": null}"#); + assert_eq!(p.ttl_secs, Some(None)); + assert_eq!(p.tags, Some(None)); + assert_eq!(p.scope, Some(None)); + assert_eq!(p.content, None, "content stays absent"); + + let p = patch_from_raw( + r#"{"id": 1, "ttl_secs": 90, "tags": ["a"], "scope": "s", "content": "c"}"#, + ); + assert_eq!(p.ttl_secs, Some(Some(90))); + assert_eq!(p.tags, Some(Some(vec!["a".to_string()]))); + assert_eq!(p.scope, Some(Some("s".to_string()))); + assert_eq!(p.content, Some(Some("c".to_string()))); + + // content: null survives deserialization as an explicit null, so the + // service can reject it (it must never collapse into "absent"). + assert_eq!( + patch_from_raw(r#"{"id": 1, "content": null}"#).content, + Some(None) + ); + + let (id, _) = serde_json::from_str::(r#"{"id": 42, "tags": []}"#) + .expect("deserializes") + .into_patch(); + assert_eq!(id, 42); + } + + async fn update_raw(server: &MemoryMcp, raw: &str) -> Result { + let args: UpdateArgs = serde_json::from_str(raw).expect("raw update args deserialize"); + server.memory_update(Parameters(args)).await + } + + async fn update_ok(server: &MemoryMcp, raw: &str) -> serde_json::Value { + result_json(&update_raw(server, raw).await.expect("update succeeds")) + } + + #[tokio::test] + async fn memory_update_tags_and_scope_matrix_absent_null_value() { + let dir = tempfile::tempdir().expect("tempdir"); + let server = test_server(&dir); + + // tags + let id = store_one(&server, "t1").await; + let body = update_ok(&server, &format!(r#"{{"id": {id}, "scope": "x"}}"#)).await; + assert_eq!( + body["tags"], + serde_json::json!(["before"]), + "absent → unchanged" + ); + let id = store_one(&server, "t2").await; + let body = update_ok(&server, &format!(r#"{{"id": {id}, "tags": null}}"#)).await; + assert_eq!(body["tags"], serde_json::json!([]), "null → empty"); + let id = store_one(&server, "t3").await; + let body = update_ok(&server, &format!(r#"{{"id": {id}, "tags": ["n"]}}"#)).await; + assert_eq!(body["tags"], serde_json::json!(["n"]), "value → set"); + + // scope + let id = store_one(&server, "s1").await; + let body = update_ok(&server, &format!(r#"{{"id": {id}, "tags": ["k"]}}"#)).await; + assert_eq!(body["scope"], "project-a", "absent → unchanged"); + let id = store_one(&server, "s2").await; + let body = update_ok(&server, &format!(r#"{{"id": {id}, "scope": null}}"#)).await; + assert_eq!(body["scope"], serde_json::Value::Null, "null → global"); + let id = store_one(&server, "s3").await; + let body = update_ok(&server, &format!(r#"{{"id": {id}, "scope": "project-b"}}"#)).await; + assert_eq!(body["scope"], "project-b", "value → set"); + } + + #[tokio::test] + async fn memory_update_ttl_matrix_absent_null_value() { + use agent_memory_core::clock::Clock; + + let dir = tempfile::tempdir().expect("tempdir"); + let server = test_server(&dir); + let now = SystemClock.now(); + // store_one sets a 3600 s TTL; this is well past it. + let past_old_ttl = now + 7_200; + + let survives = |id: i64, at: i64| { + let server = server.clone(); + async move { + server.state.service.sweep(at).await.expect("sweep"); + let listed = server + .memory_list(Parameters(ListArgs { + r#type: None, + tag: None, + scope: None, + limit: None, + })) + .await + .expect("list"); + result_json(&listed) + .as_array() + .expect("list is an array") + .iter() + .any(|v| v["id"] == id) + } + }; + + let id = store_one(&server, "ttl absent").await; + update_ok(&server, &format!(r#"{{"id": {id}, "tags": ["x"]}}"#)).await; + assert!( + !survives(id, past_old_ttl).await, + "absent → the old TTL still applies" + ); + + let id = store_one(&server, "ttl null").await; + update_ok(&server, &format!(r#"{{"id": {id}, "ttl_secs": null}}"#)).await; + assert!(survives(id, past_old_ttl).await, "null → no expiry"); + + let id = store_one(&server, "ttl value").await; + update_ok(&server, &format!(r#"{{"id": {id}, "ttl_secs": 86400}}"#)).await; + assert!( + survives(id, past_old_ttl).await, + "value → the new TTL replaced the old" + ); + assert!(!survives(id, now + 2 * 86_400).await, "…and still expires"); + } + + #[tokio::test] + async fn memory_update_invalid_patches_are_invalid_params_never_internal() { + let dir = tempfile::tempdir().expect("tempdir"); + let server = test_server(&dir); + let id = store_one(&server, "untouched").await; + let invalid_params = McpError::invalid_params("x", None).code; + let too_big = agent_memory_core::domain::MAX_TTL_SECS + 1; + + for (raw, needle) in [ + (format!(r#"{{"id": {id}}}"#), "at least one of"), + ( + format!(r#"{{"id": {id}, "content": null}}"#), + "content must be a non-empty string", + ), + ( + format!(r#"{{"id": {id}, "content": ""}}"#), + "content must be a non-empty string", + ), + ( + format!(r#"{{"id": {id}, "ttl_secs": 0}}"#), + "ttl_secs must be between", + ), + ( + format!(r#"{{"id": {id}, "ttl_secs": -3}}"#), + "ttl_secs must be between", + ), + ( + format!(r#"{{"id": {id}, "ttl_secs": {too_big}}}"#), + "ttl_secs must be between", + ), + ] { + let err = update_raw(&server, &raw) + .await + .expect_err("invalid patch is rejected"); + assert_eq!( + err.code, invalid_params, + "{raw}: client tier, never internal_error" + ); + assert!(err.message.contains(needle), "{raw}: got {:?}", err.message); + } + + // Still exactly as stored. + let body = update_ok(&server, &format!(r#"{{"id": {id}, "scope": "probe"}}"#)).await; + assert_eq!(body["content"], "untouched"); + assert_eq!(body["tags"], serde_json::json!(["before"])); + } + #[test] fn update_args_schema_makes_only_id_required() { let schema = diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index 6122922..14ff3e9 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -1,20 +1,22 @@ -//! The five REST handlers — thin adapters with ZERO business logic. +//! The six REST handlers — thin adapters with ZERO business logic. //! //! Each handler copies the `mcp.rs` per-tool shape: validate the wire type via //! `MemoryType::try_from` up front, build the exact core service arg structs, //! make ONE service call, and map the error. The request DTOs mirror the MCP //! tool argument structs field-for-field (`content`, `type`, `tags`, `source`, -//! `scope`, `ttl_secs`, `query`, `limit`) so both transports speak one dialect. -//! serde only — no schemars on REST DTOs. +//! `scope`, `ttl_secs`, `query`, `limit`, `id`) so both transports speak one +//! dialect. serde only — REST DTOs never derive a JSON Schema (that is the +//! MCP tool-schema concern). use std::sync::Arc; +use axum::extract::rejection::JsonRejection; use axum::extract::{Path, Query, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use axum::Json; -use agent_memory_core::domain::{MemoryError, MemoryType, NewMemory}; +use agent_memory_core::domain::{MemoryError, MemoryPatch, MemoryType, NewMemory}; use agent_memory_core::embed::EmbedderHealth; use agent_memory_core::service::{ ListArgs as ServiceListArgs, SearchArgs as ServiceSearchArgs, SearchOutcome, @@ -80,6 +82,51 @@ pub struct ListQuery { pub limit: Option, } +/// Body for `PATCH /api/memories/{id}` — mirrors `mcp::UpdateArgs` field for +/// field (the id comes from the path). A JSON-merge-patch: an absent member +/// keeps the field, `null` clears it (`tags` → `[]`, `scope` → global, +/// `ttl_secs` → no expiry), a value sets it. `content` is never clearable. +/// Uses the SAME explicit-null helper as the MCP DTO, so the two transports +/// cannot drift on null-vs-absent. +#[derive(Debug, serde::Deserialize)] +pub struct UpdateBody { + /// New content (re-indexed and re-embedded). Never null or empty. + #[serde( + default, + deserialize_with = "crate::serde_util::deserialize_explicit_null" + )] + pub content: Option>, + /// Replacement tag list; `null` clears to an empty list. + #[serde( + default, + deserialize_with = "crate::serde_util::deserialize_explicit_null" + )] + pub tags: Option>>, + /// New logical scope; `null` resets to global. + #[serde( + default, + deserialize_with = "crate::serde_util::deserialize_explicit_null" + )] + pub scope: Option>, + /// New time-to-live in seconds from now; `null` removes the expiry. + #[serde( + default, + deserialize_with = "crate::serde_util::deserialize_explicit_null" + )] + pub ttl_secs: Option>, +} + +impl From for MemoryPatch { + fn from(body: UpdateBody) -> Self { + MemoryPatch { + content: body.content, + tags: body.tags, + scope: body.scope, + ttl_secs: body.ttl_secs, + } + } +} + /// Two-tier [`MemoryError`] → HTTP mapping, mirroring `mcp.rs`'s /// `invalid_params` / `internal_error` split: `InvalidType`, `InvalidQuery`, /// and `InvalidArgument` are client input (400); everything else — including @@ -201,6 +248,41 @@ pub async fn forget_handler( } } +/// `PATCH /api/memories/{id}` → 200 [`UpdatedMemory`] (the full post-update +/// memory plus `embedding_status`), or 404 `{"id", "updated": false, +/// "reason": "not_found"}` — the exact JSON the `mcp.rs` update tool returns, +/// with HTTP-native status codes. +/// +/// The body extractor's rejection is taken explicitly and reported as a 400 +/// with the usual `{"error": ...}` body: a malformed or wrongly-typed patch is +/// client input, whatever axum's default status for it would be. Invalid +/// patch VALUES (null/empty content, out-of-bounds TTL, an empty patch) are +/// rejected by the service seam and map to 400 through [`map_memory_error`]. +/// +/// [`UpdatedMemory`]: agent_memory_core::domain::UpdatedMemory +pub async fn update_handler( + State(state): State>, + Path(id): Path, + body: Result, JsonRejection>, +) -> Result { + let Json(body) = body.map_err(|rejection| ApiError::BadRequest(rejection.body_text()))?; + + let updated = state + .service + .update(id, MemoryPatch::from(body)) + .await + .map_err(map_memory_error)?; + + match updated { + Some(updated) => Ok(Json(updated).into_response()), + None => Ok(( + StatusCode::NOT_FOUND, + Json(serde_json::json!({ "id": id, "updated": false, "reason": "not_found" })), + ) + .into_response()), + } +} + /// `GET /health` → 200 `{"status": "ok", "embedder": ...}`. /// /// The store ping is a cheap one-row list through the service (no SQL here); @@ -579,6 +661,268 @@ mod tests { assert_eq!(body["embedder"], "unreachable"); } + // ---- PATCH /api/memories/{id} (MCP-06 / D-08) ------------------------- + // + // Every patch below is a RAW JSON string pushed through axum's own `Json` + // extractor (`FromRequest`), so the explicit-null deserializer and the + // rejection mapping are under test — not a hand-built Rust struct. + + use agent_memory_core::clock::Clock; + use agent_memory_core::domain::MAX_TTL_SECS; + use axum::extract::FromRequest; + + async fn patch_raw(state: &Arc, id: i64, raw: &str) -> Response { + patch_request(state, id, raw, Some("application/json")).await + } + + async fn patch_request( + state: &Arc, + id: i64, + raw: &str, + content_type: Option<&str>, + ) -> Response { + let mut builder = axum::http::Request::builder() + .method("PATCH") + .uri(format!("/api/memories/{id}")); + if let Some(ct) = content_type { + builder = builder.header(axum::http::header::CONTENT_TYPE, ct); + } + let req = builder + .body(axum::body::Body::from(raw.to_string())) + .expect("request builds"); + let body = Json::::from_request(req, &()).await; + match update_handler(State(state.clone()), Path(id), body).await { + Ok(resp) => resp, + Err(err) => err.into_response(), + } + } + + /// Store a memory with tags `["rest"]`, scope `"unit"` and the given TTL. + async fn store_with_ttl(state: &Arc, ttl_secs: Option) -> i64 { + let mut req = store_request("patch target", "PATTERN"); + req.ttl_secs = ttl_secs; + let resp = store_handler(State(state.clone()), Json(req)) + .await + .expect("store succeeds") + .into_response(); + body_json(resp).await["id"].as_i64().expect("id") + } + + /// Run a sweep at `at` and report whether `id` survived it. + async fn survives_sweep_at(state: &Arc, id: i64, at: i64) -> bool { + state.service.sweep(at).await.expect("sweep runs"); + state + .service + .list(ServiceListArgs::default()) + .await + .expect("list") + .iter() + .any(|v| v.id == id) + } + + async fn ok_body(resp: Response) -> serde_json::Value { + assert_eq!(resp.status(), StatusCode::OK, "expected 200"); + body_json(resp).await + } + + async fn assert_400_containing(resp: Response, needle: &str) { + assert_eq!( + resp.status(), + StatusCode::BAD_REQUEST, + "expected 400 (never 500) for a patch that should contain {needle:?}" + ); + let body = body_json(resp).await; + let message = body["error"] + .as_str() + .expect("400 body has an error string"); + assert!( + message.contains(needle), + "error {message:?} should contain {needle:?}" + ); + } + + #[tokio::test] + async fn patch_tags_matrix_absent_null_value() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + + // absent → unchanged + let id = store_with_ttl(&state, None).await; + let body = ok_body(patch_raw(&state, id, r#"{"scope": "elsewhere"}"#).await).await; + assert_eq!(body["tags"], serde_json::json!(["rest"])); + assert_eq!( + body["content"], "patch target", + "absent content is unchanged" + ); + + // null → cleared to an empty list + let id = store_with_ttl(&state, None).await; + let body = ok_body(patch_raw(&state, id, r#"{"tags": null}"#).await).await; + assert_eq!(body["tags"], serde_json::json!([])); + assert_eq!(body["scope"], "unit", "an unrelated field is unchanged"); + + // value → set + let id = store_with_ttl(&state, None).await; + let body = ok_body(patch_raw(&state, id, r#"{"tags": ["x", "y"]}"#).await).await; + assert_eq!(body["tags"], serde_json::json!(["x", "y"])); + } + + #[tokio::test] + async fn patch_scope_matrix_absent_null_value() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + + let id = store_with_ttl(&state, None).await; + let body = ok_body(patch_raw(&state, id, r#"{"tags": ["t"]}"#).await).await; + assert_eq!(body["scope"], "unit", "absent scope is unchanged"); + + let id = store_with_ttl(&state, None).await; + let body = ok_body(patch_raw(&state, id, r#"{"scope": null}"#).await).await; + assert_eq!( + body["scope"], + serde_json::Value::Null, + "null resets to global" + ); + + let id = store_with_ttl(&state, None).await; + let body = ok_body(patch_raw(&state, id, r#"{"scope": "other"}"#).await).await; + assert_eq!(body["scope"], "other"); + } + + #[tokio::test] + async fn patch_ttl_matrix_absent_null_value() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let now = SystemClock.now(); + let past_old_ttl = now + 3_600; // well past the stored 60 s TTL + + // absent → the 60 s expiry still applies + let id = store_with_ttl(&state, Some(60)).await; + ok_body(patch_raw(&state, id, r#"{"tags": ["keep-ttl"]}"#).await).await; + assert!( + !survives_sweep_at(&state, id, past_old_ttl).await, + "an absent ttl_secs leaves the expiry in place" + ); + + // null → no expiry: survives a sweep far past the old TTL + let id = store_with_ttl(&state, Some(60)).await; + ok_body(patch_raw(&state, id, r#"{"ttl_secs": null}"#).await).await; + assert!( + survives_sweep_at(&state, id, past_old_ttl).await, + "ttl_secs: null removes the expiry" + ); + + // value → a new TTL counted from patch time + let id = store_with_ttl(&state, Some(60)).await; + ok_body(patch_raw(&state, id, r#"{"ttl_secs": 86400}"#).await).await; + assert!( + survives_sweep_at(&state, id, past_old_ttl).await, + "the new 1-day TTL replaced the 60 s one" + ); + assert!( + !survives_sweep_at(&state, id, now + 2 * 86_400).await, + "the new TTL itself still expires" + ); + } + + #[tokio::test] + async fn patch_content_null_or_empty_is_400_and_a_content_update_is_200() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let id = store_with_ttl(&state, None).await; + + for raw in [ + r#"{"content": null}"#, + r#"{"content": ""}"#, + r#"{"content": " "}"#, + ] { + assert_400_containing( + patch_raw(&state, id, raw).await, + "content must be a non-empty string", + ) + .await; + } + + let body = ok_body(patch_raw(&state, id, r#"{"content": "rewritten"}"#).await).await; + assert_eq!(body["id"], id); + assert_eq!(body["content"], "rewritten"); + assert_eq!(body["type"], "PATTERN"); + // Dead embedder → the re-embed is pending, and the response says so. + assert_eq!(body["embedding_status"], 0); + } + + #[tokio::test] + async fn patch_out_of_bounds_ttl_is_400_never_500_never_clamped() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let id = store_with_ttl(&state, None).await; + + let too_big = MAX_TTL_SECS + 1; + for raw in [ + r#"{"ttl_secs": 0}"#.to_string(), + r#"{"ttl_secs": -1}"#.to_string(), + format!(r#"{{"ttl_secs": {too_big}}}"#), + ] { + assert_400_containing( + patch_raw(&state, id, &raw).await, + "ttl_secs must be between", + ) + .await; + } + } + + #[tokio::test] + async fn patch_of_an_unknown_id_is_404_with_the_not_found_body() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + + let resp = patch_raw(&state, 777, r#"{"content": "nobody home"}"#).await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + assert_eq!( + body_json(resp).await, + serde_json::json!({ "id": 777, "updated": false, "reason": "not_found" }) + ); + } + + #[tokio::test] + async fn malformed_or_empty_patches_are_400_never_500() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let id = store_with_ttl(&state, None).await; + + // An empty patch is a client error, not a silent no-op. + assert_400_containing(patch_raw(&state, id, "{}").await, "at least one of").await; + + // Extractor rejections — syntax, type, missing content type — are all + // reported as 400 with the standard error body. + for (raw, content_type) in [ + (r#"{"tags": ["#, Some("application/json")), + (r#"{"ttl_secs": "soon"}"#, Some("application/json")), + (r#"{"tags": "not-a-list"}"#, Some("application/json")), + (r#"{"tags": ["x"]}"#, None), + ] { + let resp = patch_request(&state, id, raw, content_type).await; + assert_eq!( + resp.status(), + StatusCode::BAD_REQUEST, + "patch {raw:?} (content-type {content_type:?}) must be a 400" + ); + let body = body_json(resp).await; + assert!( + !body["error"].as_str().expect("error string").is_empty(), + "the 400 carries a message: {body}" + ); + } + + // None of the rejected patches touched the row. + let rows = state + .service + .list(ServiceListArgs::default()) + .await + .expect("list"); + assert_eq!(rows[0].tags, vec!["rest".to_string()]); + } + #[test] fn map_memory_error_puts_internal_faults_in_the_500_tier() { // WR-02: a poisoned writer lock is a server fault — 500, never 404. diff --git a/crates/agent-memory/src/rest/mod.rs b/crates/agent-memory/src/rest/mod.rs index 88a669f..57d8a19 100644 --- a/crates/agent-memory/src/rest/mod.rs +++ b/crates/agent-memory/src/rest/mod.rs @@ -28,8 +28,8 @@ use crate::mcp::AppState; /// `mcp.rs`'s `invalid_params` / `internal_error` split: bad input is a 400, /// everything else is a 500. The body is always `{"error": "..."}` so bad input /// can never surface as a 500. A missing resource is not an error here: -/// handlers return their 404 body from not-found data (e.g. forget's -/// `Ok(false)`), so a server fault can never be mis-tiered as a 404 (WR-02). +/// handlers return their 404 body from not-found data (forget's `Ok(false)`, +/// update's `Ok(None)`), so a server fault can never be mis-tiered as a 404 (WR-02). #[derive(Debug)] pub enum ApiError { /// Invalid client input (e.g. an unknown memory type) → 400. @@ -74,7 +74,10 @@ pub fn build_router(state: Arc) -> Router { post(handlers::store_handler).get(handlers::list_handler), ) .route("/api/search", post(handlers::search_handler)) - .route("/api/memories/{id}", delete(handlers::forget_handler)) + .route( + "/api/memories/{id}", + delete(handlers::forget_handler).patch(handlers::update_handler), + ) .route("/health", get(handlers::health_handler)) .with_state(state) } From a49b6c6f5763bae94df3c1d219398cd4989f4455 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:11:07 +0200 Subject: [PATCH 07/42] docs(04-02): complete memory_update plan (summary) Co-Authored-By: Claude Opus 5.5 --- .../04-02-SUMMARY.md | 275 ++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-02-SUMMARY.md diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-02-SUMMARY.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-02-SUMMARY.md new file mode 100644 index 0000000..50fd55a --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-02-SUMMARY.md @@ -0,0 +1,275 @@ +--- +phase: 04-memory-update-relations-hybrid-search +plan: 02 +subsystem: api +tags: [rust, sqlite, sqlite-vec, fts5, serde, schemars, rmcp, axum, merge-patch] + +# Dependency graph +requires: + - phase: 04-memory-update-relations-hybrid-search + provides: "04-01: MemoryError::Internal for writer-lock sites; not-found is data (MemoryError/ApiError::NotFound deleted); migration 0003" + - phase: 03-api-hardening-toolchain-spikes + provides: "validation seam (validate_ttl, InvalidArgument tier), exhaustive two-tier mappers" +provides: + - "MemoryPatch (double-Option merge-patch) + UpdatedMemory (flattened MemoryView + embedding_status) + validate_patch (domain.rs)" + - "Store::update: one writer tx, UPDATE in place, freshness bump folded in, same-tx vec_memories invalidation + re-embed, outage -> embedding_status 0" + - "MemoryService::update(id, patch) -> Result, MemoryError>" + - "serde_util::deserialize_explicit_null shared by both transports" + - "memory_update MCP tool (UpdateArgs) and PATCH /api/memories/{id} (UpdateBody + update_handler)" +affects: [04-03 relations (ids stay stable under update; edges never re-attach), 04-04 hybrid (update bumps decay), 05 export/import, 06 release notes] + +# Actuals (#2632) — chars/4 over the added lines of the realized diff +actuals: + tokens: 18026 + tasks: 2 + commits: 3 + +# Tech tracking +tech-stack: + added: [] + patterns: + - "Merge-patch DTO fields: Option> + #[serde(default, deserialize_with = explicit-null helper)]; absent -> None, null -> Some(None), value -> Some(Some(v))" + - "Patch DTOs carry skip_serializing_if = Option::is_none so schemars does not advertise default: null" + - "Update is UPDATE-in-place with the freshness bump in the same statement; never delete+insert (rowid recycling)" + - "REST handlers that must be 400 on any body problem take Result, JsonRejection>" + - "Transport tests feed RAW JSON through the real deserializer (axum FromRequest / serde_json::from_str), not Rust constructors" + +key-files: + created: + - crates/agent-memory-core/tests/update.rs + - crates/agent-memory/src/serde_util.rs + modified: + - crates/agent-memory-core/src/domain.rs + - crates/agent-memory-core/src/service.rs + - crates/agent-memory-core/src/store/mod.rs + - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory/src/main.rs + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - crates/agent-memory/src/rest/mod.rs + - README.md + +key-decisions: + - "D-10 via a dedicated UpdatedMemory (#[serde(flatten)] MemoryView + embedding_status); MemoryView is untouched, so every list/search envelope stays byte-identical" + - "An all-fields-absent patch is InvalidArgument ('at least one of content, tags, scope, ttl_secs must be provided'), not a silent no-op" + - "The update UPDATE also resets decay_score to 1.0, the exact decay value at last_accessed = now (insert stores the same literal), so list reads are honest without waiting for a sweep" + - "Patch DTO fields set skip_serializing_if = Option::is_none so the MCP tool schema has no default: null; a client filling schema defaults would otherwise send nulls, and null means clear" + - "PATCH maps every Json extractor rejection (syntax, wrong type, missing content-type) to 400 {error}, per the plan's 'malformed body -> 400'; other REST handlers keep axum's default 4xx statuses" + - "mem_type and source are not patchable (MCP-06 scope); a content patch always invalidates and re-embeds, even if the text is unchanged" + +patterns-established: + - "New patchable field: add Option> with the serde_util helper + skip_serializing_if on BOTH UpdateArgs and UpdateBody, a merge arm in SqliteStore::update, and a 3-state matrix cell per transport" + +requirements-completed: [MCP-06] + +coverage: + - id: D1 + description: "A content update changes the meaning in place: old-meaning semantic query no longer returns the row, new meaning does; FTS keyword index follows; exactly one vector row" + requirement: MCP-06 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/update.rs#content_update_re_embeds_so_the_old_meaning_stops_matching" + status: pass + human_judgment: false + - id: D2 + description: "Embed outage during a content update lands embedding_status 0 with the stale vector removed; one sweep backfills to 1" + requirement: MCP-06 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/update.rs#embed_outage_during_update_falls_back_to_pending_and_sweep_backfills" + status: pass + human_judgment: false + - id: D3 + description: "Update bumps freshness (last_accessed = patch time, access_count + 1) and returns the full post-update view plus embedding_status; type and created_at are immutable" + requirement: MCP-06 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/update.rs#update_bumps_freshness_by_exactly_one_access" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/update.rs#update_returns_the_full_post_update_view_with_embedding_status" + status: pass + human_judgment: false + - id: D4 + description: "Tags/scope/TTL-only patches never call the embedder; invalid patches are rejected before any embed or write" + requirement: MCP-06 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/update.rs#a_tags_only_patch_never_calls_the_embedder" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/update.rs#an_invalid_patch_is_rejected_before_the_embedder_is_called" + status: pass + human_judgment: false + - id: D5 + description: "Merge-patch matrix (ttl_secs/tags/scope x absent/null/value) from raw JSON on both transports, plus a TestClock core matrix; ttl null survives a sweep past the old TTL" + requirement: MCP-06 + verification: + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#patch_tags_matrix_absent_null_value / patch_scope_matrix_absent_null_value / patch_ttl_matrix_absent_null_value" + status: pass + - kind: unit + ref: "crates/agent-memory/src/mcp.rs#raw_update_args_carry_all_three_wire_states_into_the_patch" + status: pass + - kind: integration + ref: "crates/agent-memory/src/mcp.rs#memory_update_tags_and_scope_matrix_absent_null_value / memory_update_ttl_matrix_absent_null_value" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/update.rs#ttl_patch_matrix_absent_keeps_null_clears_value_resets_from_patch_time / tags_and_scope_patch_matrix_absent_null_value" + status: pass + - kind: unit + ref: "crates/agent-memory/src/serde_util.rs#tests (5)" + status: pass + human_judgment: false + - id: D6 + description: "Unknown id is a clean not-found on both transports (MCP success {id, updated:false, reason:not_found}; REST 404 same body); bad patches are 400 / invalid_params, never 500" + requirement: MCP-06 + verification: + - kind: integration + ref: "crates/agent-memory/src/mcp.rs#memory_update_of_an_unknown_id_is_a_clean_not_found / memory_update_invalid_patches_are_invalid_params_never_internal" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#patch_of_an_unknown_id_is_404_with_the_not_found_body / malformed_or_empty_patches_are_400_never_500 / patch_out_of_bounds_ttl_is_400_never_500_never_clamped / patch_content_null_or_empty_is_400_and_a_content_update_is_200" + status: pass + - kind: manual_procedural + ref: "live serve-rest binary: POST 201; PATCH 200 with embedding_status; unknown id 404 body; content:null 400; truncated JSON 400" + status: pass + human_judgment: false + - id: D7 + description: "memory_update tool schema requires only id and advertises no default: null" + requirement: MCP-06 + verification: + - kind: unit + ref: "crates/agent-memory/src/mcp.rs#update_args_schema_makes_only_id_required" + status: pass + - kind: manual_procedural + ref: "live stdio binary: tools/list lists memory_update with nullable typed properties; tools/call with scope/ttl_secs null succeeds; content null -> -32602" + status: pass + human_judgment: false + - id: D8 + description: "README documents memory_update (merge-patch table, embedding_status, freshness) and PATCH /api/memories/{id}" + requirement: MCP-06 + verification: [] + human_judgment: true + rationale: "Documentation clarity and accuracy is a reading judgment; no test asserts README prose" + +# Metrics +duration: 10min +completed: 2026-10-23 +status: complete +--- + +# Phase 4 Plan 02: memory_update Summary + +**`memory_update` (MCP) and `PATCH /api/memories/{id}` (REST) patch a memory in place using JSON-merge-patch semantics. One writer transaction re-mirrors FTS5, swaps the vector, and bumps freshness. An embed outage leaves `embedding_status = 0` for the sweep to backfill. Unknown ids get a clean not-found on both transports.** + +## Performance + +- **Duration:** ~10 min +- **Started:** 2026-10-23T18:01:29Z +- **Completed:** 2026-10-23T18:11Z +- **Tasks:** 2 / 2 +- **Files modified:** 11 (2 created, 9 modified) + +## Accomplishments + +- **Phase success criterion 1 holds.** After a content update, a semantic query for the old meaning no longer returns the row and the new meaning does. Keyword search follows the new text too. Four distractors near the old meaning make the test a genuine kill-test: with `limit = 1` (k = 4), a row that kept its stale vector would still rank first. A hand mutation that skipped the vector swap turned it red. +- **Outage fallback.** When the embedder is down during an update, the update still succeeds. The stale vector is deleted, `embedding_status` is set to 0, and one `sweep()` re-embeds the row back to 1. +- **D-08 merge-patch on both transports.** The shared `serde_util::deserialize_explicit_null` keeps `null` distinct from "absent". Raw-JSON matrices cover `ttl_secs`/`tags`/`scope` × absent/null/value over REST (through axum's real `Json` extractor) and over MCP. A hand mutation that collapsed `null` into "absent" failed 10 binary tests. +- **D-09 and D-10.** The freshness bump is folded into the same `UPDATE`. The response is the flattened MemoryView plus `embedding_status`. `type` and `created_at` stay immutable. +- **Never-500.** These all return 400 / `invalid_params` with stable message prefixes: null, empty or whitespace content; a TTL of 0, negative, or above `MAX_TTL_SECS`; an empty patch; malformed JSON; a wrongly typed field; a missing content-type. +- **Id immutability.** The row is updated in place and is never deleted and re-inserted, so rowid recycling cannot hit it (T-04-08). `grep -c 'DELETE FROM memories' sqlite.rs` is still 2. + +## Task Commits + +1. **Task 1: memory_update happy path end to end (core update transaction + MCP tool)**: `d76e43d` (feat) +2. **Task 2: full merge-patch matrix on both transports + PATCH /api/memories/{id} + README**: `3a3d4fe` (feat) + +**Plan metadata:** this SUMMARY commit, then a `docs(04-02)` state/roadmap commit. + +_TDD note: RED was observed before implementation. In Task 1, `tests/update.rs` failed to compile, with 16 errors for the missing `MemoryPatch` and `MemoryService::update`. In Task 2, the matrices were written against the Task 1 helper and verified by the null-collapse mutation above. RED was not committed separately, because CLAUDE.md and the operator rules forbid committing failing tests (same as 04-01)._ + +## Files Created/Modified + +- `crates/agent-memory-core/tests/update.rs` (new): MCP-06 suite of 9 tests. Covers the re-embed kill-test, outage plus sweep backfill, the freshness bump, the response shape, not-found, the no-embed tags patch, validate-before-embed, and the TTL and tags/scope matrices. A local `SwitchEmbedder` provides outage toggling and call counting. +- `crates/agent-memory/src/serde_util.rs` (new): the explicit-null helper plus 5 three-state unit tests, including the `from_value` path that rmcp uses. +- `crates/agent-memory-core/src/domain.rs`: `MemoryPatch`, `UpdatedMemory`, `validate_patch` +- `crates/agent-memory-core/src/store/mod.rs` and `store/sqlite.rs`: `Store::update` +- `crates/agent-memory-core/src/service.rs`: `MemoryService::update` +- `crates/agent-memory/src/main.rs`: `mod serde_util` +- `crates/agent-memory/src/mcp.rs`: `UpdateArgs`, the `memory_update` tool, the server instructions mention update, and 9 new tests +- `crates/agent-memory/src/rest/handlers.rs`: `UpdateBody`, `update_handler`, and 7 new raw-JSON PATCH tests +- `crates/agent-memory/src/rest/mod.rs`: the PATCH route +- `README.md`: a `memory_update` section, the tool table (now "five tools"), the REST table row plus a curl example, and a lifecycle note + +## Decisions Made + +- **UpdatedMemory is its own struct** (research Open Question 1, option a). Existing envelopes are unchanged. +- **An empty patch is a client error.** It returns "at least one of content, tags, scope, ttl_secs must be provided". +- **`decay_score` is reset to 1.0 inside the update.** That is exactly the decay value at `last_accessed = now`, so `list` (which reads the materialized column) shows the edited memory as fresh straight away. +- **The schema has no `default: null`** (found while verifying research assumption A1, see Deviations). +- **Every PATCH body rejection is a 400** with the standard `{"error": ...}` body. A non-numeric path id still gets axum's plain-text 400, identical to DELETE. + +## Deviations from Plan + +### Auto-fixed Issues + +**1. [Rule 2 - Missing critical] The tool schema advertised `"default": null` for every patch field** +- **Found during:** Task 1, while checking the live `tools/list` output (research A1) +- **Issue:** schemars derives `default: null` from `#[serde(default)]`. A client that fills in schema defaults would send explicit nulls, and in a merge-patch null means "clear". Every patch would then wipe tags, scope and TTL, and fail on `content: null`. +- **Fix:** added `skip_serializing_if = "Option::is_none"` to the four `UpdateArgs` patch fields. It affects only the schema, because the DTO is deserialize-only. A doc comment explains why. `update_args_schema_makes_only_id_required` now also asserts that no property has a `default`. +- **Files modified:** crates/agent-memory/src/mcp.rs +- **Verification:** unit test, plus the live `tools/list` output (nullable typed properties, `required: ["id"]`) +- **Committed in:** d76e43d + +**2. [Rule 3 - Blocking] Reworded the `handlers.rs` file-header comment so the plan's literal `grep schemars` gate passes** +- **Found during:** Task 2 acceptance check +- **Issue:** the pre-existing header line "serde only — no schemars on REST DTOs" matched the plan's gate, which expects no `schemars` text in `handlers.rs`. The code was already correct. +- **Fix:** the header now says "REST DTOs never derive a JSON Schema". The rule is the same; only the wording changed. +- **Committed in:** 3a3d4fe + +**3. [Rule 2 - Missing critical] Tests beyond the listed behaviours** +- Added a TestClock-driven core matrix (TTL keep/clear/reset from patch time, tags/scope clear visible to the list filters) and an FTS re-mirror assertion inside the kill-test. Also added a `from_value` test for the serde helper, because rmcp deserializes tool arguments from a `Value`, and a schema test. +- **Committed in:** d76e43d, 3a3d4fe + +--- + +**Total deviations:** 3 auto-fixed (2 missing-critical, 1 blocking). +**Impact on plan:** No scope creep, no new crates, no manifest change. + +## Issues Encountered + +- The first live schema check ran a stale `target/debug/agent-memory`, because `cargo test --bins` does not rebuild the bin target. Running `cargo build -p agent-memory` first fixed it. +- macOS has no `timeout` binary. The live REST check used Python's `signal.alarm` instead. + +## Verification Results + +- `cargo test -p agent-memory-core --test update`: 9 passed +- `cargo test -p agent-memory --bins`: 37 passed (was 22 before this plan) +- `cargo test --workspace`: **116 passed / 0 failed / 2 ignored** (04-01 baseline was 87 / 0 / 2) +- `cargo clippy --workspace --all-targets -- -D warnings`: clean +- `cargo fmt --all --check`: clean +- Acceptance greps: `UPDATE memories SET content = …` appears inside `fn update`. The `DELETE FROM memories` count is still 2. `patch(` is present in `rest/mod.rs`, and `schemars` does not appear in `handlers.rs`. +- Hand mutations, both reverted: skipping the vector swap failed 2 core tests, and collapsing null into absent failed 10 binary tests. +- Live binaries: the MCP stdio `tools/list` and `tools/call` behave as expected, and so does REST `serve-rest` PATCH (200, 404, 400, 400). + +## User Setup Required + +None. + +## Next Phase Readiness + +- Plan 04-03 (relations): an update keeps the id, so links attached to a memory survive edits. The `{id, …, reason: "not_found"}` status shape and the not-found-as-data pattern carry straight over to `memory_link` / `memory_unlink`. +- Plan 04-04 (hybrid): an update bumps `last_accessed` and resets `decay_score`, so freshly edited memories rank as fresh in the post-fusion decay blend. +- Still open, outside this plan: WR-01 (full filter-aware KNN) and WR-03 (the REST harness hang). No PATCH test was added to the spawned-binary `tests/rest.rs`. The in-process tests carry coverage, as the plan specifies, and the live check was manual. + +--- +*Phase: 04-memory-update-relations-hybrid-search* +*Completed: 2026-10-23* + +## Self-Check: PASSED + +- FOUND (tracked): crates/agent-memory-core/tests/update.rs (674 lines, ≥ 120) +- FOUND (tracked): crates/agent-memory/src/serde_util.rs +- FOUND: commit d76e43d (Task 1) +- FOUND: commit 3a3d4fe (Task 2) From 1c3cd934cba7b70d4df467c6c64188f65bb59259 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:11:31 +0200 Subject: [PATCH 08/42] docs(04-02): update state, roadmap progress, mark MCP-06 complete Co-Authored-By: Claude Opus 5.5 --- .planning/REQUIREMENTS.md | 4 ++-- .planning/ROADMAP.md | 6 +++--- .planning/STATE.md | 20 +++++++++++++------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md index 8d80d92..c39900b 100644 --- a/.planning/REQUIREMENTS.md +++ b/.planning/REQUIREMENTS.md @@ -15,7 +15,7 @@ Each maps to a roadmap phase. IDs continue from the v1.0 numbering (archived in ### MCP Interface (MCP) -- [ ] **MCP-06**: Agent can call `memory_update` to patch an existing memory (content, tags, scope, TTL) by id; a content update re-mirrors FTS and invalidates/re-embeds the vector **in the same writer transaction** (embed-outage falls back to `embedding_status = 0` + existing sweep backfill); unknown id returns the clean not-found shape +- [x] **MCP-06**: Agent can call `memory_update` to patch an existing memory (content, tags, scope, TTL) by id; a content update re-mirrors FTS and invalidates/re-embeds the vector **in the same writer transaction** (embed-outage falls back to `embedding_status = 0` + existing sweep backfill); unknown id returns the clean not-found shape - [ ] **MCP-07**: Agent can link two memories with a flat typed relation (`memory_link` / `memory_unlink`: from_id, to_id, kind) and search/list results can expand 1-hop related memories; forgetting a memory cascades its links (no orphan edges, no graph/entity model) ### REST & Input Hardening (API) @@ -64,7 +64,7 @@ Mapped by roadmap creation 2026-07-12. Coverage: 12/12 v1.1 requirements mapped |-------------|-------|--------| | API-02 | Phase 3 | Complete | | API-03 | Phase 3 | Complete | -| MCP-06 | Phase 4 | Pending | +| MCP-06 | Phase 4 | Complete | | MCP-07 | Phase 4 | Pending | | STORE-05 | Phase 4 | Complete | | SEARCH-04 | Phase 4 | Pending | diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 773a479..72c51eb 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -67,7 +67,7 @@ Plans: 4. Hybrid ranking stays decay-aware — a fresh memory outranks a stale one at equal fused rank — and access bumps apply only to post-truncation returned ids. 5. Upgrading a real v0.0.1 database applies migration 0003 only after a pre-migration backup; shipped 0001/0002 SQL is provably unchanged (fixture divergence test against a v0.0.1 database), and a newer-schema DB met by an older binary yields a friendly error, not a crash. -**Plans**: 1/4 plans executed +**Plans**: 2/4 plans executed Plans: **Wave 1** @@ -75,7 +75,7 @@ Plans: **Wave 2** *(blocked on Wave 1 completion)* -- [ ] 04-02-PLAN.md — memory_update end-to-end: merge-patch semantics, same-tx FTS+vector re-embed, PATCH /api/memories/{id} (MCP-06) +- [x] 04-02-PLAN.md — memory_update end-to-end: merge-patch semantics, same-tx FTS+vector re-embed, PATCH /api/memories/{id} (MCP-06) **Wave 3** *(blocked on Wave 2 completion)* @@ -124,6 +124,6 @@ Phases execute in numeric order: 3 → 4 → 5 → 6 | 1. Core Memory Foundation | v1.0 | 3/3 | Complete | 2026-06-25 | | 2. Semantic Search, Interop & Release | v1.0 | 5/5 | Complete | 2026-07-12 | | 3. API Hardening & Toolchain Spikes | v1.1 | 2/2 | Complete | 2026-07-12 | -| 4. Memory Update, Relations & Hybrid Search | v1.1 | 1/4 | In Progress| | +| 4. Memory Update, Relations & Hybrid Search | v1.1 | 2/4 | In Progress| | | 5. Portable Export & Import | v1.1 | 0/? | Not started | - | | 6. Distribution & v0.1.0 Release | v1.1 | 0/? | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index dfedf50..d8b908b 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -5,16 +5,16 @@ milestone_name: Hardening & Interop current_phase: 04 current_phase_name: Memory Update, Relations & Hybrid Search status: executing -stopped_at: Completed 04-01-PLAN.md -last_updated: "2026-10-23T17:58:30.378Z" +stopped_at: Completed 04-02-PLAN.md +last_updated: "2026-10-23T18:11:23.164Z" last_activity: 2026-10-23 last_activity_desc: Phase 04 execution started -state_head: 2088a143bd9124cd863f653703a4c66f309befb0 +state_head: a49b6c6f5763bae94df3c1d219398cd4989f4455 progress: total_phases: 4 completed_phases: 1 total_plans: 6 - completed_plans: 3 + completed_plans: 4 --- # Project State @@ -29,7 +29,7 @@ See: .planning/PROJECT.md (updated 2026-07-12) ## Current Position Phase: 04 (Memory Update, Relations & Hybrid Search) — EXECUTING -Plan: 2 of 4 +Plan: 3 of 4 Status: Ready to execute Last activity: 2026-10-23 — Phase 04 execution started @@ -69,6 +69,7 @@ Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) | Plan | Duration | Tasks | Files | |------|----------|-------|-------| | Phase 04 P01 | 9 min | 2 tasks | 13 files | +| Phase 04 P02 | 10min | 2 tasks | 11 files | ## Accumulated Context @@ -97,6 +98,11 @@ Recent decisions affecting current work: - [Phase 04]: [04-01]: MAX_KNN_K = MAX_LIMIT * 4 (800) — candidate oversample cap, distinct from RRF k=60 (WR-01 partial; full filter-aware fix deferred) - [Phase 04]: [04-01]: SqliteStore::open order is probe user_version (before any PRAGMA, so a refused DB is byte-untouched) -> SchemaTooNew refusal -> pragmas -> sibling online backup {db}.backup-pre-{NNNN}-{YYYYMMDD} (one per target version, failure aborts) -> to_latest; LATEST_SCHEMA_VERSION=3 must be bumped with every new migration - [Phase 04]: [04-01]: tests/fixtures/v0.0.1.db is a committed binary fixture generated once from tag-identical 0001+0002 — never regenerate it; it is gitignored locally by *.db so stage with git add -f +- [Phase 04]: 04-02: memory_update returns a dedicated UpdatedMemory (flattened MemoryView + embedding_status); MemoryView untouched so list/search envelopes stay byte-identical (D-10) +- [Phase 04]: 04-02: an all-fields-absent patch is InvalidArgument, not a no-op; content null/empty rejected at the seam; TTL patches reuse validate_ttl (D-08) +- [Phase 04]: 04-02: update is UPDATE-in-place (never delete+insert), folds the D-09 freshness bump and decay_score = 1.0 into the same statement, and swaps the vector in the same writer tx +- [Phase 04]: 04-02: patch DTO fields carry skip_serializing_if so the MCP tool schema never advertises default: null (null means clear in a merge-patch) +- [Phase 04]: 04-02: PATCH /api/memories/{id} maps every Json extractor rejection to 400 {error}; other REST handlers keep axum defaults ### Pending Todos @@ -137,8 +143,8 @@ Items acknowledged and carried forward: ## Session Continuity -Last session: 2026-10-23T17:58:30.334Z -Stopped at: Completed 04-01-PLAN.md +Last session: 2026-10-23T18:11:16.057Z +Stopped at: Completed 04-02-PLAN.md Resume file: None ## Operator Next Steps From 93ae8678651b133c2adf97106bb120284d9a70a8 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:18:55 +0200 Subject: [PATCH 09/42] feat(04-03): memory_link / memory_unlink with cascade on forget and TTL sweep (MCP-07) - LinkKind closed enum (relates_to | supersedes | caused_by) with TryFrom returning InvalidArgument naming the allowed set (D-01); LinkOutcome { Linked, NotFound { missing_id } } so a missing endpoint is data (D-02) - Store::link pre-checks both endpoints in the writer tx, then INSERT OR IGNORE: duplicates are idempotent Linked; never an FK 500 - Service rejects self-links at the seam before any store hop - forget() and sweep_expired() gain belt-and-braces orphan-edge deletes beside the FK ON DELETE CASCADE (T-04-12) - memory_link / memory_unlink MCP tools; POST /api/links and DELETE /api/links/{from_id}/{to_id}/{kind} (200/404 dual status, 400 on any bad body/kind/self-link) - tests/relations.rs (10) + MCP (4) + REST (4, incl. a real-HTTP router test) Co-Authored-By: Claude Opus 5.5 --- crates/agent-memory-core/src/domain.rs | 92 +++++ crates/agent-memory-core/src/service.rs | 47 ++- crates/agent-memory-core/src/store/mod.rs | 35 +- crates/agent-memory-core/src/store/sqlite.rs | 78 +++- crates/agent-memory-core/tests/relations.rs | 384 +++++++++++++++++++ crates/agent-memory/src/mcp.rs | 233 ++++++++++- crates/agent-memory/src/rest/handlers.rs | 320 +++++++++++++++- crates/agent-memory/src/rest/mod.rs | 7 +- 8 files changed, 1186 insertions(+), 10 deletions(-) create mode 100644 crates/agent-memory-core/tests/relations.rs diff --git a/crates/agent-memory-core/src/domain.rs b/crates/agent-memory-core/src/domain.rs index 6ac737a..dbab31f 100644 --- a/crates/agent-memory-core/src/domain.rs +++ b/crates/agent-memory-core/src/domain.rs @@ -68,6 +68,85 @@ impl std::convert::TryFrom<&str> for MemoryType { } } +/// The closed vocabulary of typed relations between two memories (MCP-07 / +/// D-01). Exactly three kinds ship in v0.1.0; adding one later is additive, +/// removing one is a contract break. +/// +/// Edges are stored DIRECTED (D-03), and each kind reads `from to`: +/// +/// - `relates_to` — a symmetric "see also"; the direction carries no meaning. +/// - `supersedes` — `from` replaces `to` (the newer memory points at the older). +/// - `caused_by` — `from` was caused by `to` (the effect points at its cause). +/// +/// The wire form is snake_case (`relates_to`, `supersedes`, `caused_by`). +/// Parse untrusted input with [`LinkKind::try_from`], which rejects anything +/// else with [`MemoryError::InvalidArgument`] naming the allowed set — so a +/// free-form kind string never reaches SQL (T-04-10). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LinkKind { + RelatesTo, + Supersedes, + CausedBy, +} + +impl LinkKind { + /// The snake_case wire form, matching the serde representation. This is + /// also the exact value stored in `memory_links.kind`. + pub fn as_wire_str(self) -> &'static str { + match self { + LinkKind::RelatesTo => "relates_to", + LinkKind::Supersedes => "supersedes", + LinkKind::CausedBy => "caused_by", + } + } + + /// The natural label of this kind read from the TARGET's side of the edge + /// (D-03): `relates_to` is symmetric and stays `relates_to`; an incoming + /// `supersedes` reads `superseded_by`; an incoming `caused_by` reads + /// `caused`. Used by 1-hop expansion for `direction: "in"` entries. + pub fn reverse_label(self) -> &'static str { + match self { + LinkKind::RelatesTo => "relates_to", + LinkKind::Supersedes => "superseded_by", + LinkKind::CausedBy => "caused", + } + } +} + +impl std::fmt::Display for LinkKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_wire_str()) + } +} + +impl std::convert::TryFrom<&str> for LinkKind { + type Error = MemoryError; + + fn try_from(s: &str) -> Result { + match s { + "relates_to" => Ok(LinkKind::RelatesTo), + "supersedes" => Ok(LinkKind::Supersedes), + "caused_by" => Ok(LinkKind::CausedBy), + other => Err(MemoryError::InvalidArgument(format!( + "kind must be one of relates_to, supersedes, caused_by (got '{other}')" + ))), + } + } +} + +/// Outcome of a link request (MCP-07 / D-02). A missing endpoint is DATA, not +/// an error: it names the first id that does not exist (`from_id` is checked +/// before `to_id`), so a client never sees a foreign-key failure (T-04-13). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LinkOutcome { + /// The edge exists now — freshly created, or already present (a duplicate + /// link is an idempotent no-op success). + Linked, + /// No memory with `missing_id` exists; nothing was written. + NotFound { missing_id: i64 }, +} + /// Input shape for storing a new memory (D-05). /// /// Only `content` and `mem_type` are conceptually required at the service boundary; @@ -280,6 +359,19 @@ pub(crate) fn validate_patch(patch: &MemoryPatch) -> Result<(), MemoryError> { validate_ttl(patch.ttl_secs.flatten()) } +/// Validate a link's endpoints at the service seam (MCP-07 / D-02): a memory +/// cannot be linked to itself. Runs before any store hop, so a self-link is +/// the client tier even when the id does not exist. (The 0003 `CHECK` is the +/// belt-and-braces backstop; it is never what a client observes.) +pub(crate) fn validate_link_endpoints(from_id: i64, to_id: i64) -> Result<(), MemoryError> { + if from_id == to_id { + return Err(MemoryError::InvalidArgument(format!( + "from_id and to_id must differ (got {from_id})" + ))); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/agent-memory-core/src/service.rs b/crates/agent-memory-core/src/service.rs index 7b828e8..c26627e 100644 --- a/crates/agent-memory-core/src/service.rs +++ b/crates/agent-memory-core/src/service.rs @@ -16,7 +16,10 @@ use std::sync::Arc; use crate::clock::Clock; use crate::decay::{apply_decay, decay_score, DecayConfig, DecayEngine, RankWeights, SweepReport}; -use crate::domain::{MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, UpdatedMemory}; +use crate::domain::{ + LinkKind, LinkOutcome, MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, + UpdatedMemory, +}; use crate::embed::Embedder; use crate::store::sqlite::DEFAULT_SEARCH_LIMIT; use crate::store::Store; @@ -358,6 +361,48 @@ impl MemoryService { .map_err(MemoryError::Join)? } + /// Link two memories with a typed, directed relation (MCP-07): + /// `from_id -[kind]-> to_id`. + /// + /// The self-link check runs FIRST, at this seam, before any store hop + /// (D-02): `from_id == to_id` is [`MemoryError::InvalidArgument`] even if + /// the id does not exist. The kind is already a closed [`LinkKind`] — the + /// transports parse the wire string with [`LinkKind::try_from`] (D-01). + /// + /// A missing endpoint is [`LinkOutcome::NotFound`] naming the id (clean + /// not-found data, never an error); linking an edge that already exists is + /// an idempotent [`LinkOutcome::Linked`] (D-02). The edge is stamped with + /// the injected clock, which orders 1-hop expansion (D-07). + pub async fn link( + &self, + from_id: i64, + to_id: i64, + kind: LinkKind, + ) -> Result { + crate::domain::validate_link_endpoints(from_id, to_id)?; + let now = self.clock.now(); + let store = self.store.clone(); + tokio::task::spawn_blocking(move || store.link(from_id, to_id, kind, now)) + .await + .map_err(MemoryError::Join)? + } + + /// Remove the directed edge `from_id -[kind]-> to_id` (MCP-07). Returns + /// `true` if it was removed, `false` if no such edge existed — the clean + /// not-found of [`MemoryService::forget`], never an error (D-02). A + /// self-pair simply never matches an edge, so it is a not-found too. + pub async fn unlink( + &self, + from_id: i64, + to_id: i64, + kind: LinkKind, + ) -> Result { + let store = self.store.clone(); + tokio::task::spawn_blocking(move || store.unlink(from_id, to_id, kind)) + .await + .map_err(MemoryError::Join)? + } + /// Patch a memory in place (MCP-06), returning the full post-update view /// plus `embedding_status` (D-10), or `Ok(None)` for an unknown id (clean /// not-found, never an error — the [`MemoryService::forget`] contract). diff --git a/crates/agent-memory-core/src/store/mod.rs b/crates/agent-memory-core/src/store/mod.rs index 34abb28..6bd0179 100644 --- a/crates/agent-memory-core/src/store/mod.rs +++ b/crates/agent-memory-core/src/store/mod.rs @@ -10,7 +10,10 @@ pub mod migrations; pub mod sqlite; use crate::decay::{DecayConfig, RankWeights}; -use crate::domain::{MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, UpdatedMemory}; +use crate::domain::{ + LinkKind, LinkOutcome, MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, + UpdatedMemory, +}; use crate::service::{ListArgs, SearchArgs}; /// Synchronous persistence interface, implemented by [`sqlite::SqliteStore`]. @@ -81,7 +84,10 @@ pub trait Store: Send + Sync { /// row with that id existed (clean not-found, never an error — MCP-04). The /// FTS5 mirror is kept in sync by the delete trigger; the `vec_memories` /// row is deleted EXPLICITLY (vec0 virtual tables ignore triggers — - /// RESEARCH Pitfall 3 / T-02-05). + /// RESEARCH Pitfall 3 / T-02-05). Every `memory_links` edge touching the + /// row goes with it: the FK `ON DELETE CASCADE` is the primary mechanism, + /// and an explicit orphan-edge delete in the same transaction covers any + /// connection that skipped the `foreign_keys` pragma (MCP-07 / T-04-12). fn forget(&self, id: i64) -> Result; /// Patch a memory IN PLACE (MCP-06 / D-08), returning the post-update row, @@ -104,6 +110,27 @@ pub trait Store: Send + Sync { now: i64, ) -> Result, MemoryError>; + /// Create the directed edge `from_id -[kind]-> to_id` (MCP-07), stamped + /// `created_at = now`. One writer transaction: both endpoints are checked + /// first, and a missing one returns [`LinkOutcome::NotFound`] naming it + /// (`from_id` first) — a clean not-found, never an FK-violation error + /// (D-02 / T-04-13). A duplicate `(from_id, to_id, kind)` is an idempotent + /// no-op that still returns [`LinkOutcome::Linked`]. The self-link check + /// belongs to the service seam and has already run. + fn link( + &self, + from_id: i64, + to_id: i64, + kind: LinkKind, + now: i64, + ) -> Result; + + /// Remove the directed edge `from_id -[kind]-> to_id` (MCP-07). Returns + /// `true` if an edge was removed, `false` if no such edge existed — the + /// clean not-found of [`Store::forget`], never an error (D-02). Direction + /// and kind are part of the edge's identity. + fn unlink(&self, from_id: i64, to_id: i64, kind: LinkKind) -> Result; + /// Bump the recency of the given ids: set `last_accessed = now` and increment /// `access_count`. Runs on the writer lane. A no-op for an empty id slice. fn bump_access(&self, ids: &[i64], now: i64) -> Result<(), MemoryError>; @@ -114,7 +141,9 @@ pub trait Store: Send + Sync { /// decay materialization is structurally forbidden from deleting (STORE-04 / /// Pitfall 7). The FTS5 delete trigger keeps `memories_fts` in sync; orphaned /// `vec_memories` rows are cleared explicitly in the same transaction (vec0 - /// ignores triggers — Pitfall 3 / T-02-05). + /// ignores triggers — Pitfall 3 / T-02-05). Edges of expired rows cascade + /// exactly as for [`Store::forget`], with the same explicit orphan-edge + /// delete as the belt-and-braces backstop (MCP-07 / T-04-12). fn sweep_expired(&self, now: i64) -> Result; /// Recompute and persist each surviving row's `decay_score` column at `now`, diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index c99b486..9ddb973 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -20,7 +20,10 @@ use rusqlite::{params, Connection, OptionalExtension}; use rusqlite_migration::Migrations; use crate::decay::{DecayConfig, RankWeights, PINNED_HALF_LIFE_MULTIPLIER}; -use crate::domain::{MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, UpdatedMemory}; +use crate::domain::{ + LinkKind, LinkOutcome, MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, + UpdatedMemory, +}; use crate::service::{ListArgs, SearchArgs}; use crate::store::migrations::{migrations, LATEST_SCHEMA_VERSION}; use crate::store::Store; @@ -596,10 +599,76 @@ impl Store for SqliteStore { // / T-02-05: a forgotten memory must never resurface semantically). tx.execute("DELETE FROM vec_memories WHERE memory_id = ?1", params![id])?; let changed = tx.execute("DELETE FROM memories WHERE id = ?1", params![id])?; + // Relation edges: `ON DELETE CASCADE` (foreign_keys = ON on every + // connection, apply_pragmas) already removed this row's edges. The + // explicit sweep below is the belt-and-braces backstop for any + // connection that ever skipped the pragma — it also heals orphans such + // a connection may have left behind (MCP-07 / T-04-12, milestone + // Pitfall 9) and is what stops a recycled rowid from ever re-attaching + // an old edge to an unrelated memory. Mirrors the vec_memories orphan + // idiom in sweep_expired. + tx.execute( + "DELETE FROM memory_links WHERE from_id NOT IN (SELECT id FROM memories) \ + OR to_id NOT IN (SELECT id FROM memories)", + [], + )?; tx.commit()?; Ok(changed > 0) } + fn link( + &self, + from_id: i64, + to_id: i64, + kind: LinkKind, + now: i64, + ) -> Result { + let mut conn = self + .writer + .lock() + .map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; + let tx = conn.transaction()?; + + // Pre-check BOTH endpoints inside the writer tx and report the first + // missing id as data. Never rely on the FK violation: parsing a + // SqliteFailure would surface a client mistake as a 500 (T-04-13). + for id in [from_id, to_id] { + let exists = tx + .query_row("SELECT 1 FROM memories WHERE id = ?1", params![id], |_| { + Ok(()) + }) + .optional()? + .is_some(); + if !exists { + // The tx rolls back on drop; nothing was written. + return Ok(LinkOutcome::NotFound { missing_id: id }); + } + } + + // The (from_id, to_id, kind) primary key makes a duplicate an + // idempotent no-op: changes() == 0 is still Linked (D-02). The kind + // binds only the closed enum's wire form (T-04-10). + tx.execute( + "INSERT OR IGNORE INTO memory_links (from_id, to_id, kind, created_at) \ + VALUES (?1, ?2, ?3, ?4)", + params![from_id, to_id, kind.as_wire_str(), now], + )?; + tx.commit()?; + Ok(LinkOutcome::Linked) + } + + fn unlink(&self, from_id: i64, to_id: i64, kind: LinkKind) -> Result { + let conn = self + .writer + .lock() + .map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; + let removed = conn.execute( + "DELETE FROM memory_links WHERE from_id = ?1 AND to_id = ?2 AND kind = ?3", + params![from_id, to_id, kind.as_wire_str()], + )?; + Ok(removed > 0) + } + fn update( &self, id: i64, @@ -772,6 +841,13 @@ impl Store for SqliteStore { "DELETE FROM vec_memories WHERE memory_id NOT IN (SELECT id FROM memories)", [], )?; + // Edges of the expired rows: cascaded by the FK, with the same + // explicit backstop as forget() (MCP-07 / T-04-12). + tx.execute( + "DELETE FROM memory_links WHERE from_id NOT IN (SELECT id FROM memories) \ + OR to_id NOT IN (SELECT id FROM memories)", + [], + )?; tx.commit()?; Ok(deleted) } diff --git a/crates/agent-memory-core/tests/relations.rs b/crates/agent-memory-core/tests/relations.rs new file mode 100644 index 0000000..5cef6ca --- /dev/null +++ b/crates/agent-memory-core/tests/relations.rs @@ -0,0 +1,384 @@ +//! MCP-07: flat typed relations between memories — `link` / `unlink` with the +//! closed three-kind vocabulary, cascade on every deletion path, and opt-in +//! 1-hop expansion on search/list. +//! +//! Locked decisions proven here: D-01 (closed `kind` enum: relates_to, +//! supersedes, caused_by — anything else is `InvalidArgument` naming the +//! allowed set), D-02 (duplicate link = idempotent no-op success, self-link = +//! `InvalidArgument`, missing endpoint / missing edge = clean not-found data, +//! never an error), D-03 (one directed edge row per link). +//! +//! Harness: a REAL on-disk temp DB, an injected `TestClock`, and a failing +//! `FakeEmbedder` (stores land pending; search deterministically takes the +//! FTS5 keyword path). Edge rows are counted through a raw side connection so +//! the assertions see the table itself, not the service's view of it. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use agent_memory_core::clock::{Clock, TestClock}; +use agent_memory_core::decay::DecayConfig; +use agent_memory_core::domain::{LinkKind, LinkOutcome, MemoryError, MemoryType, NewMemory}; +use agent_memory_core::embed::{Embedder, FakeEmbedder}; +use agent_memory_core::service::MemoryService; +use agent_memory_core::store::sqlite::{register_vec_extension, SqliteStore}; + +struct Harness { + service: MemoryService, + clock: Arc, + db_path: PathBuf, + _dir: tempfile::TempDir, +} + +fn harness() -> Harness { + let dir = tempfile::tempdir().expect("tempdir"); + let db_path = dir.path().join("memory.db"); + let store = SqliteStore::open(&db_path).expect("open store"); + let clock = Arc::new(TestClock::new(1_000)); + let embedder: Arc = Arc::new(FakeEmbedder::failing()); + let service = MemoryService::new( + Arc::new(store), + clock.clone() as Arc, + embedder, + DecayConfig::default(), + ); + Harness { + service, + clock, + db_path, + _dir: dir, + } +} + +fn new_memory(content: &str, ttl_secs: Option) -> NewMemory { + NewMemory { + content: content.to_string(), + mem_type: MemoryType::Decision, + tags: vec!["rel".to_string()], + source: Some("relations-test".to_string()), + scope: None, + ttl_secs, + } +} + +async fn store(h: &Harness, content: &str) -> i64 { + h.service + .store(new_memory(content, None)) + .await + .expect("store succeeds") +} + +fn raw_conn(db_path: &Path) -> rusqlite::Connection { + register_vec_extension().expect("vec0 registers"); + rusqlite::Connection::open(db_path).expect("open raw connection") +} + +/// Total number of edge rows in `memory_links`. +fn edge_count(db_path: &Path) -> i64 { + raw_conn(db_path) + .query_row("SELECT count(*) FROM memory_links", [], |row| row.get(0)) + .expect("count edges") +} + +/// Edge rows touching `id` in either direction. +fn edges_touching(db_path: &Path, id: i64) -> i64 { + raw_conn(db_path) + .query_row( + "SELECT count(*) FROM memory_links WHERE from_id = ?1 OR to_id = ?1", + [id], + |row| row.get(0), + ) + .expect("count edges touching id") +} + +/// Edge rows whose either endpoint no longer exists in `memories`. +fn orphan_edges(db_path: &Path) -> i64 { + raw_conn(db_path) + .query_row( + "SELECT count(*) FROM memory_links \ + WHERE from_id NOT IN (SELECT id FROM memories) \ + OR to_id NOT IN (SELECT id FROM memories)", + [], + |row| row.get(0), + ) + .expect("count orphan edges") +} + +fn invalid_argument_message(err: MemoryError) -> String { + match err { + MemoryError::InvalidArgument(msg) => msg, + other => panic!("expected InvalidArgument, got {other:?}"), + } +} + +// --------------------------------------------------------------------------- +// Task 1 — link / unlink / cascade +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn link_succeeds_and_a_duplicate_link_is_an_idempotent_no_op() { + let h = harness(); + let a = store(&h, "memory a").await; + let b = store(&h, "memory b").await; + + let first = h + .service + .link(a, b, LinkKind::RelatesTo) + .await + .expect("link succeeds"); + assert_eq!(first, LinkOutcome::Linked); + assert_eq!(edge_count(&h.db_path), 1); + + // D-02: the identical (from, to, kind) again is a success, not an error, + // and still exactly one row. + let again = h + .service + .link(a, b, LinkKind::RelatesTo) + .await + .expect("duplicate link is not an error"); + assert_eq!(again, LinkOutcome::Linked); + assert_eq!(edge_count(&h.db_path), 1, "duplicate link adds no row"); + + // A different kind between the same pair is a distinct edge. + h.service + .link(a, b, LinkKind::Supersedes) + .await + .expect("second kind links"); + assert_eq!(edge_count(&h.db_path), 2); +} + +#[tokio::test] +async fn a_self_link_is_invalid_argument_naming_the_id() { + let h = harness(); + let a = store(&h, "lonely").await; + + let err = h + .service + .link(a, a, LinkKind::RelatesTo) + .await + .expect_err("self-link is rejected"); + assert_eq!( + invalid_argument_message(err), + format!("from_id and to_id must differ (got {a})") + ); + assert_eq!(edge_count(&h.db_path), 0); + + // The seam check runs before any store hop: a self-link on an id that + // does not even exist is still the client error, not a not-found. + let err = h + .service + .link(424_242, 424_242, LinkKind::CausedBy) + .await + .expect_err("self-link is rejected before the existence check"); + assert_eq!( + invalid_argument_message(err), + "from_id and to_id must differ (got 424242)" + ); +} + +#[test] +fn an_unknown_kind_is_invalid_argument_listing_the_allowed_values() { + let err = LinkKind::try_from("blocks").expect_err("unknown kind is rejected"); + assert_eq!( + err.to_string(), + "invalid argument: kind must be one of relates_to, supersedes, caused_by (got 'blocks')" + ); + assert_eq!( + invalid_argument_message(err), + "kind must be one of relates_to, supersedes, caused_by (got 'blocks')" + ); + // The wire form is exact: case and spelling variants are not aliases. + assert!(LinkKind::try_from("Supersedes").is_err()); + assert!(LinkKind::try_from("superseded_by").is_err()); + assert!(LinkKind::try_from("").is_err()); +} + +#[test] +fn every_kind_round_trips_its_wire_form() { + for (wire, kind) in [ + ("relates_to", LinkKind::RelatesTo), + ("supersedes", LinkKind::Supersedes), + ("caused_by", LinkKind::CausedBy), + ] { + assert_eq!(LinkKind::try_from(wire).expect("known kind"), kind); + assert_eq!(kind.as_wire_str(), wire); + assert_eq!( + serde_json::to_value(kind).expect("serializes"), + serde_json::json!(wire) + ); + } +} + +#[tokio::test] +async fn linking_to_a_missing_memory_is_a_clean_not_found_naming_the_id() { + let h = harness(); + let a = store(&h, "present").await; + let missing = a + 1_000; + + let outcome = h + .service + .link(a, missing, LinkKind::CausedBy) + .await + .expect("a missing endpoint is data, never an FK-violation error"); + assert_eq!( + outcome, + LinkOutcome::NotFound { + missing_id: missing + } + ); + + // A missing FROM id is named too (the first missing id is reported). + let outcome = h + .service + .link(missing, a, LinkKind::CausedBy) + .await + .expect("missing from_id is data"); + assert_eq!( + outcome, + LinkOutcome::NotFound { + missing_id: missing + } + ); + assert_eq!(edge_count(&h.db_path), 0, "nothing was written"); +} + +#[tokio::test] +async fn unlink_removes_an_edge_and_a_missing_edge_is_a_clean_not_found() { + let h = harness(); + let a = store(&h, "from").await; + let b = store(&h, "to").await; + h.service + .link(a, b, LinkKind::Supersedes) + .await + .expect("link"); + assert_eq!(edge_count(&h.db_path), 1); + + // Wrong direction / wrong kind do not match the directed edge. + assert!(!h + .service + .unlink(b, a, LinkKind::Supersedes) + .await + .expect("reverse direction is not-found data")); + assert!(!h + .service + .unlink(a, b, LinkKind::RelatesTo) + .await + .expect("other kind is not-found data")); + assert_eq!(edge_count(&h.db_path), 1); + + assert!(h + .service + .unlink(a, b, LinkKind::Supersedes) + .await + .expect("unlink succeeds")); + assert_eq!(edge_count(&h.db_path), 0); + + // Unlinking it again is the clean not-found shape, never an error. + assert!(!h + .service + .unlink(a, b, LinkKind::Supersedes) + .await + .expect("missing edge is data")); +} + +#[tokio::test] +async fn forget_cascades_every_edge_touching_the_memory() { + let h = harness(); + let a = store(&h, "doomed").await; + let b = store(&h, "survivor b").await; + let c = store(&h, "survivor c").await; + + // Both directions, mixed kinds, plus one edge not touching `a`. + h.service + .link(a, b, LinkKind::Supersedes) + .await + .expect("a>b"); + h.service.link(b, a, LinkKind::CausedBy).await.expect("b>a"); + h.service + .link(c, a, LinkKind::RelatesTo) + .await + .expect("c>a"); + h.service + .link(b, c, LinkKind::RelatesTo) + .await + .expect("b>c"); + assert_eq!(edges_touching(&h.db_path, a), 3); + + assert!(h.service.forget(a).await.expect("forget")); + + assert_eq!(edges_touching(&h.db_path, a), 0, "no edge references a"); + assert_eq!(edge_count(&h.db_path), 1, "the unrelated edge b>c survives"); + assert_eq!(orphan_edges(&h.db_path), 0); +} + +#[tokio::test] +async fn a_ttl_sweep_cascades_the_expired_memorys_edges() { + let h = harness(); + let expiring = h + .service + .store(new_memory("short-lived", Some(10))) + .await + .expect("store expiring"); + let durable = store(&h, "long-lived").await; + h.service + .link(expiring, durable, LinkKind::CausedBy) + .await + .expect("out edge"); + h.service + .link(durable, expiring, LinkKind::Supersedes) + .await + .expect("in edge"); + assert_eq!(edge_count(&h.db_path), 2); + + h.clock.advance(100); // past expires_at = 1_010 + let report = h.service.sweep(h.clock.now()).await.expect("sweep"); + assert_eq!(report.expired, 1); + + assert_eq!(edges_touching(&h.db_path, expiring), 0); + assert_eq!(edge_count(&h.db_path), 0); + assert_eq!(orphan_edges(&h.db_path), 0); +} + +/// Simulate a connection that skipped `PRAGMA foreign_keys = ON` (milestone +/// Pitfall 9): delete a memory row with FK enforcement off, which leaves its +/// edges orphaned exactly as such a connection would. +fn orphan_by_raw_delete(db_path: &Path, id: i64) { + let conn = raw_conn(db_path); + conn.pragma_update(None, "foreign_keys", "OFF") + .expect("fk off"); + conn.execute("DELETE FROM memories WHERE id = ?1", [id]) + .expect("raw delete"); +} + +#[tokio::test] +async fn forget_also_clears_orphans_left_by_a_connection_without_the_fk_pragma() { + let h = harness(); + let a = store(&h, "orphaned by a pragma-less connection").await; + let b = store(&h, "partner").await; + let c = store(&h, "forgotten normally").await; + h.service + .link(a, b, LinkKind::RelatesTo) + .await + .expect("a>b"); + orphan_by_raw_delete(&h.db_path, a); + assert_eq!(orphan_edges(&h.db_path), 1, "precondition: one orphan edge"); + + // Forgetting an UNRELATED memory runs the belt-and-braces delete. + assert!(h.service.forget(c).await.expect("forget")); + assert_eq!(orphan_edges(&h.db_path), 0); +} + +#[tokio::test] +async fn sweep_also_clears_orphans_left_by_a_connection_without_the_fk_pragma() { + let h = harness(); + let a = store(&h, "orphaned by a pragma-less connection").await; + let b = store(&h, "partner").await; + h.service.link(b, a, LinkKind::CausedBy).await.expect("b>a"); + orphan_by_raw_delete(&h.db_path, a); + assert_eq!(orphan_edges(&h.db_path), 1, "precondition: one orphan edge"); + + // A sweep with nothing to expire still runs the explicit cleanup. + let report = h.service.sweep(h.clock.now()).await.expect("sweep"); + assert_eq!(report.expired, 0); + assert_eq!(orphan_edges(&h.db_path), 0); +} diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index f4571e1..8d9903f 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -12,7 +12,9 @@ use rmcp::handler::server::wrapper::Parameters; use rmcp::model::{CallToolResult, Content, Implementation, ServerCapabilities, ServerInfo}; use rmcp::{schemars, tool, tool_handler, tool_router, ErrorData as McpError, ServerHandler}; -use agent_memory_core::domain::{MemoryError, MemoryPatch, MemoryType, NewMemory}; +use agent_memory_core::domain::{ + LinkKind, LinkOutcome, MemoryError, MemoryPatch, MemoryType, NewMemory, +}; use agent_memory_core::embed::Embedder; use agent_memory_core::service::{ ListArgs as ServiceListArgs, MemoryService, SearchArgs as ServiceSearchArgs, @@ -182,6 +184,37 @@ impl UpdateArgs { } } +/// Arguments for `memory_link` (MCP-07): create the directed edge +/// `from_id -[kind]-> to_id`. +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct LinkArgs { + /// The id of the memory the edge starts at. + pub from_id: i64, + /// The id of the memory the edge points to. Must differ from `from_id`. + pub to_id: i64, + /// One of: relates_to (symmetric "see also"), supersedes (from replaces + /// to), caused_by (from was caused by to). + pub kind: String, +} + +/// Arguments for `memory_unlink` (MCP-07): remove the directed edge +/// `from_id -[kind]-> to_id`. Direction and kind must match the stored edge. +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct UnlinkArgs { + /// The id the edge starts at. + pub from_id: i64, + /// The id the edge points to. + pub to_id: i64, + /// One of: relates_to, supersedes, caused_by. + pub kind: String, +} + +/// Parse a wire `kind` up front (the `MemoryType` precedent): an unknown kind +/// is `invalid_params` naming the allowed set, and never reaches the service. +fn parse_link_kind(wire: &str) -> Result { + LinkKind::try_from(wire).map_err(map_mcp_error) +} + /// The MCP server. Holds shared state and the generated tool router. #[derive(Clone)] pub struct MemoryMcp { @@ -342,6 +375,80 @@ impl MemoryMcp { .map_err(|e| McpError::internal_error(e.to_string(), None))?; Ok(CallToolResult::success(vec![Content::text(json)])) } + + #[tool( + description = "Link two memories with a typed, directed relation from_id -[kind]-> to_id. kind is one of: relates_to (symmetric see-also), supersedes (from_id replaces the older to_id), caused_by (from_id was caused by to_id). Linking an existing edge again is a no-op success; from_id and to_id must differ. Returns {from_id, to_id, kind, linked: true}, or {linked: false, reason: 'not_found', missing_id} when either memory does not exist." + )] + async fn memory_link( + &self, + Parameters(args): Parameters, + ) -> Result { + let kind = parse_link_kind(&args.kind)?; + let outcome = self + .state + .service + .link(args.from_id, args.to_id, kind) + .await + .map_err(map_mcp_error)?; + + // A missing endpoint is a clean not-found, NOT an error — the + // memory_forget shape, plus the id that was missing (D-02). + let status = match outcome { + LinkOutcome::Linked => serde_json::json!({ + "from_id": args.from_id, + "to_id": args.to_id, + "kind": kind, + "linked": true + }), + LinkOutcome::NotFound { missing_id } => serde_json::json!({ + "from_id": args.from_id, + "to_id": args.to_id, + "kind": kind, + "linked": false, + "reason": "not_found", + "missing_id": missing_id + }), + }; + let json = serde_json::to_string(&status) + .map_err(|e| McpError::internal_error(e.to_string(), None))?; + Ok(CallToolResult::success(vec![Content::text(json)])) + } + + #[tool( + description = "Remove the directed relation from_id -[kind]-> to_id (direction and kind must match the stored edge; kind is relates_to, supersedes or caused_by). Returns {from_id, to_id, kind, unlinked: true}, or a clean {unlinked: false, reason: 'not_found'} result when no such edge exists." + )] + async fn memory_unlink( + &self, + Parameters(args): Parameters, + ) -> Result { + let kind = parse_link_kind(&args.kind)?; + let removed = self + .state + .service + .unlink(args.from_id, args.to_id, kind) + .await + .map_err(map_mcp_error)?; + + let status = if removed { + serde_json::json!({ + "from_id": args.from_id, + "to_id": args.to_id, + "kind": kind, + "unlinked": true + }) + } else { + serde_json::json!({ + "from_id": args.from_id, + "to_id": args.to_id, + "kind": kind, + "unlinked": false, + "reason": "not_found" + }) + }; + let json = serde_json::to_string(&status) + .map_err(|e| McpError::internal_error(e.to_string(), None))?; + Ok(CallToolResult::success(vec![Content::text(json)])) + } } #[tool_handler(router = self.tool_router)] @@ -353,7 +460,7 @@ impl ServerHandler for MemoryMcp { info.capabilities = ServerCapabilities::builder().enable_tools().build(); info.server_info = Implementation::from_build_env(); info.instructions = Some( - "Persistent typed agent memory: store, list, search, update, and forget typed memories." + "Persistent typed agent memory: store, list, search, update, link/unlink, and forget typed memories." .to_string(), ); info @@ -680,6 +787,128 @@ mod tests { } } + async fn link_raw(server: &MemoryMcp, raw: &str) -> Result { + let args: LinkArgs = serde_json::from_str(raw).expect("raw link args deserialize"); + server.memory_link(Parameters(args)).await + } + + async fn unlink_raw(server: &MemoryMcp, raw: &str) -> Result { + let args: UnlinkArgs = serde_json::from_str(raw).expect("raw unlink args deserialize"); + server.memory_unlink(Parameters(args)).await + } + + #[tokio::test] + async fn memory_link_and_unlink_round_trip_with_idempotent_duplicates() { + let dir = tempfile::tempdir().expect("tempdir"); + let server = test_server(&dir); + let a = store_one(&server, "newer decision").await; + let b = store_one(&server, "older decision").await; + let raw = format!(r#"{{"from_id": {a}, "to_id": {b}, "kind": "supersedes"}}"#); + let linked = serde_json::json!({ + "from_id": a, "to_id": b, "kind": "supersedes", "linked": true + }); + + let first = link_raw(&server, &raw).await.expect("link succeeds"); + assert_eq!(result_json(&first), linked); + // D-02: the identical link again is the same success, not an error. + let again = link_raw(&server, &raw).await.expect("duplicate is a no-op"); + assert_eq!(result_json(&again), linked); + + let removed = unlink_raw(&server, &raw).await.expect("unlink succeeds"); + assert_eq!( + result_json(&removed), + serde_json::json!({ + "from_id": a, "to_id": b, "kind": "supersedes", "unlinked": true + }) + ); + let missing = unlink_raw(&server, &raw) + .await + .expect("a missing edge is a successful tool call"); + assert_eq!( + result_json(&missing), + serde_json::json!({ + "from_id": a, "to_id": b, "kind": "supersedes", + "unlinked": false, "reason": "not_found" + }) + ); + } + + #[tokio::test] + async fn memory_link_to_a_missing_memory_is_a_clean_not_found_naming_it() { + let dir = tempfile::tempdir().expect("tempdir"); + let server = test_server(&dir); + let a = store_one(&server, "present").await; + + let result = link_raw( + &server, + &format!(r#"{{"from_id": {a}, "to_id": 999999, "kind": "caused_by"}}"#), + ) + .await + .expect("a missing endpoint is a successful tool call, never internal_error"); + assert_eq!( + result_json(&result), + serde_json::json!({ + "from_id": a, "to_id": 999_999, "kind": "caused_by", + "linked": false, "reason": "not_found", "missing_id": 999_999 + }) + ); + } + + #[tokio::test] + async fn memory_link_self_links_and_unknown_kinds_are_invalid_params() { + let dir = tempfile::tempdir().expect("tempdir"); + let server = test_server(&dir); + let a = store_one(&server, "solo").await; + let invalid_params = McpError::invalid_params("x", None).code; + + let err = link_raw( + &server, + &format!(r#"{{"from_id": {a}, "to_id": {a}, "kind": "relates_to"}}"#), + ) + .await + .expect_err("self-link is rejected"); + assert_eq!(err.code, invalid_params); + assert_eq!( + err.message, + format!("invalid argument: from_id and to_id must differ (got {a})") + ); + + for call in ["link", "unlink"] { + let raw = format!(r#"{{"from_id": {a}, "to_id": 7, "kind": "blocks"}}"#); + let err = match call { + "link" => link_raw(&server, &raw).await, + _ => unlink_raw(&server, &raw).await, + } + .expect_err("unknown kind is rejected"); + assert_eq!(err.code, invalid_params, "{call}: client tier"); + assert_eq!( + err.message, + "invalid argument: kind must be one of relates_to, supersedes, caused_by (got 'blocks')", + "{call}" + ); + } + } + + #[test] + fn link_args_schemas_require_all_three_fields_and_advertise_no_defaults() { + for schema in [ + serde_json::to_value(schemars::schema_for!(LinkArgs)).expect("schema"), + serde_json::to_value(schemars::schema_for!(UnlinkArgs)).expect("schema"), + ] { + assert_eq!( + schema["required"], + serde_json::json!(["from_id", "to_id", "kind"]), + "{schema}" + ); + for field in ["from_id", "to_id", "kind"] { + assert!( + schema["properties"][field].get("default").is_none(), + "{field} must not advertise a default: {schema}" + ); + } + } + } + #[test] fn map_mcp_error_splits_client_and_internal_tiers() { let invalid_params_code = McpError::invalid_params("x", None).code; diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index 14ff3e9..fbd8dc8 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -1,4 +1,4 @@ -//! The six REST handlers — thin adapters with ZERO business logic. +//! The REST handlers — thin adapters with ZERO business logic. //! //! Each handler copies the `mcp.rs` per-tool shape: validate the wire type via //! `MemoryType::try_from` up front, build the exact core service arg structs, @@ -16,7 +16,9 @@ use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use axum::Json; -use agent_memory_core::domain::{MemoryError, MemoryPatch, MemoryType, NewMemory}; +use agent_memory_core::domain::{ + LinkKind, LinkOutcome, MemoryError, MemoryPatch, MemoryType, NewMemory, +}; use agent_memory_core::embed::EmbedderHealth; use agent_memory_core::service::{ ListArgs as ServiceListArgs, SearchArgs as ServiceSearchArgs, SearchOutcome, @@ -116,6 +118,18 @@ pub struct UpdateBody { pub ttl_secs: Option>, } +/// Body for `POST /api/links` — mirrors `mcp::LinkArgs` field for field: +/// the directed edge `from_id -[kind]-> to_id`. +#[derive(Debug, serde::Deserialize)] +pub struct LinkBody { + /// The id of the memory the edge starts at. + pub from_id: i64, + /// The id of the memory the edge points to. Must differ from `from_id`. + pub to_id: i64, + /// One of: relates_to, supersedes, caused_by. + pub kind: String, +} + impl From for MemoryPatch { fn from(body: UpdateBody) -> Self { MemoryPatch { @@ -283,6 +297,94 @@ pub async fn update_handler( } } +/// Parse a wire `kind` up front: an unknown kind is a clean 400 naming the +/// allowed set, before anything touches the service (the D-07 type pattern). +fn parse_link_kind(wire: &str) -> Result { + LinkKind::try_from(wire).map_err(map_memory_error) +} + +/// `POST /api/links` → 200 `{"from_id", "to_id", "kind", "linked": true}`, or +/// 404 `{..., "linked": false, "reason": "not_found", "missing_id"}` when +/// either memory does not exist — the exact JSON the `mcp.rs` link tool +/// returns, with HTTP-native status codes. A duplicate link is a 200 no-op. +/// +/// Every body rejection (malformed JSON, wrong types, missing content-type) +/// is a 400 `{"error"}`, like PATCH; a self-link or unknown kind is a 400 +/// from the seam. A missing endpoint is never an FK-violation 500. +pub async fn link_handler( + State(state): State>, + body: Result, JsonRejection>, +) -> Result { + let Json(body) = body.map_err(|rejection| ApiError::BadRequest(rejection.body_text()))?; + let kind = parse_link_kind(&body.kind)?; + + let outcome = state + .service + .link(body.from_id, body.to_id, kind) + .await + .map_err(map_memory_error)?; + + match outcome { + LinkOutcome::Linked => Ok(Json(serde_json::json!({ + "from_id": body.from_id, + "to_id": body.to_id, + "kind": kind, + "linked": true + })) + .into_response()), + LinkOutcome::NotFound { missing_id } => Ok(( + StatusCode::NOT_FOUND, + Json(serde_json::json!({ + "from_id": body.from_id, + "to_id": body.to_id, + "kind": kind, + "linked": false, + "reason": "not_found", + "missing_id": missing_id + })), + ) + .into_response()), + } +} + +/// `DELETE /api/links/{from_id}/{to_id}/{kind}` → 200 `{"from_id", "to_id", +/// "kind", "unlinked": true}`, or 404 `{..., "unlinked": false, "reason": +/// "not_found"}` when no such edge exists — the `mcp.rs` unlink tool's JSON. +/// An unknown `kind` segment is a 400. +pub async fn unlink_handler( + State(state): State>, + Path((from_id, to_id, kind)): Path<(i64, i64, String)>, +) -> Result { + let kind = parse_link_kind(&kind)?; + let removed = state + .service + .unlink(from_id, to_id, kind) + .await + .map_err(map_memory_error)?; + + if removed { + Ok(Json(serde_json::json!({ + "from_id": from_id, + "to_id": to_id, + "kind": kind, + "unlinked": true + })) + .into_response()) + } else { + Ok(( + StatusCode::NOT_FOUND, + Json(serde_json::json!({ + "from_id": from_id, + "to_id": to_id, + "kind": kind, + "unlinked": false, + "reason": "not_found" + })), + ) + .into_response()) + } +} + /// `GET /health` → 200 `{"status": "ok", "embedder": ...}`. /// /// The store ping is a cheap one-row list through the service (no SQL here); @@ -923,6 +1025,220 @@ mod tests { assert_eq!(rows[0].tags, vec!["rest".to_string()]); } + // --- MCP-07 link / unlink ------------------------------------------------- + + async fn link_request( + state: &Arc, + raw: &str, + content_type: Option<&str>, + ) -> Response { + let mut builder = axum::http::Request::builder() + .method("POST") + .uri("/api/links"); + if let Some(ct) = content_type { + builder = builder.header(axum::http::header::CONTENT_TYPE, ct); + } + let req = builder + .body(axum::body::Body::from(raw.to_string())) + .expect("request builds"); + let body = Json::::from_request(req, &()).await; + match link_handler(State(state.clone()), body).await { + Ok(resp) => resp, + Err(err) => err.into_response(), + } + } + + async fn link_raw(state: &Arc, raw: &str) -> Response { + link_request(state, raw, Some("application/json")).await + } + + async fn unlink_path(state: &Arc, from: i64, to: i64, kind: &str) -> Response { + match unlink_handler(State(state.clone()), Path((from, to, kind.to_string()))).await { + Ok(resp) => resp, + Err(err) => err.into_response(), + } + } + + async fn stored_id(state: &Arc, content: &str) -> i64 { + let resp = store_handler( + State(state.clone()), + Json(store_request(content, "DECISION")), + ) + .await + .expect("store succeeds") + .into_response(); + body_json(resp).await["id"].as_i64().expect("id") + } + + #[tokio::test] + async fn post_links_links_idempotently_and_delete_unlinks_then_404s() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let a = stored_id(&state, "effect").await; + let b = stored_id(&state, "cause").await; + let raw = format!(r#"{{"from_id": {a}, "to_id": {b}, "kind": "caused_by"}}"#); + let linked = serde_json::json!({ + "from_id": a, "to_id": b, "kind": "caused_by", "linked": true + }); + + for attempt in ["first", "duplicate"] { + let resp = link_raw(&state, &raw).await; + assert_eq!(resp.status(), StatusCode::OK, "{attempt} link is a 200"); + assert_eq!(body_json(resp).await, linked, "{attempt}"); + } + + let resp = unlink_path(&state, a, b, "caused_by").await; + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + body_json(resp).await, + serde_json::json!({ + "from_id": a, "to_id": b, "kind": "caused_by", "unlinked": true + }) + ); + + let resp = unlink_path(&state, a, b, "caused_by").await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + assert_eq!( + body_json(resp).await, + serde_json::json!({ + "from_id": a, "to_id": b, "kind": "caused_by", + "unlinked": false, "reason": "not_found" + }) + ); + } + + #[tokio::test] + async fn post_links_to_a_missing_memory_is_404_naming_it_never_500() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let a = stored_id(&state, "present").await; + + let resp = link_raw( + &state, + &format!(r#"{{"from_id": 999999, "to_id": {a}, "kind": "relates_to"}}"#), + ) + .await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + assert_eq!( + body_json(resp).await, + serde_json::json!({ + "from_id": 999_999, "to_id": a, "kind": "relates_to", + "linked": false, "reason": "not_found", "missing_id": 999_999 + }) + ); + } + + #[tokio::test] + async fn bad_link_requests_are_400_never_500() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let a = stored_id(&state, "solo").await; + + let resp = link_raw( + &state, + &format!(r#"{{"from_id": {a}, "to_id": {a}, "kind": "supersedes"}}"#), + ) + .await; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + assert_eq!( + body_json(resp).await["error"], + format!("invalid argument: from_id and to_id must differ (got {a})") + ); + + let kind_error = + "invalid argument: kind must be one of relates_to, supersedes, caused_by (got 'blocks')"; + let resp = link_raw( + &state, + &format!(r#"{{"from_id": {a}, "to_id": 2, "kind": "blocks"}}"#), + ) + .await; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + assert_eq!(body_json(resp).await["error"], kind_error); + + let resp = unlink_path(&state, a, 2, "blocks").await; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + assert_eq!(body_json(resp).await["error"], kind_error); + + for (raw, content_type) in [ + (r#"{"from_id": 1, "to_id": 2"#, Some("application/json")), + ( + r#"{"from_id": "one", "to_id": 2, "kind": "relates_to"}"#, + Some("application/json"), + ), + ( + r#"{"from_id": 1, "kind": "relates_to"}"#, + Some("application/json"), + ), + (r#"{"from_id": 1, "to_id": 2, "kind": "relates_to"}"#, None), + ] { + let resp = link_request(&state, raw, content_type).await; + assert_eq!( + resp.status(), + StatusCode::BAD_REQUEST, + "link body {raw:?} (content-type {content_type:?}) must be a 400" + ); + } + } + + /// The real router over a loopback listener: route wiring, the + /// `{from_id}/{to_id}/{kind}` path extractor and the JSON body all run + /// exactly as they do in `serve-rest`. + #[tokio::test] + async fn links_routes_work_over_real_http() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let a = stored_id(&state, "newer").await; + let b = stored_id(&state, "older").await; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback"); + let addr = listener.local_addr().expect("local addr"); + let server = tokio::spawn(async move { + axum::serve(listener, crate::rest::build_router(state)) + .await + .expect("serve"); + }); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .build() + .expect("client"); + let base = format!("http://{addr}"); + + let resp = client + .post(format!("{base}/api/links")) + .json(&serde_json::json!({ "from_id": a, "to_id": b, "kind": "supersedes" })) + .send() + .await + .expect("POST /api/links"); + assert_eq!(resp.status().as_u16(), 200); + + let resp = client + .delete(format!("{base}/api/links/{a}/{b}/blocks")) + .send() + .await + .expect("DELETE with a bad kind"); + assert_eq!(resp.status().as_u16(), 400); + + let resp = client + .delete(format!("{base}/api/links/{a}/{b}/supersedes")) + .send() + .await + .expect("DELETE /api/links"); + assert_eq!(resp.status().as_u16(), 200); + let body: serde_json::Value = resp.json().await.expect("json body"); + assert_eq!(body["unlinked"], true); + + let resp = client + .delete(format!("{base}/api/links/{a}/{b}/supersedes")) + .send() + .await + .expect("DELETE again"); + assert_eq!(resp.status().as_u16(), 404); + + server.abort(); + } + #[test] fn map_memory_error_puts_internal_faults_in_the_500_tier() { // WR-02: a poisoned writer lock is a server fault — 500, never 404. diff --git a/crates/agent-memory/src/rest/mod.rs b/crates/agent-memory/src/rest/mod.rs index 57d8a19..ec25751 100644 --- a/crates/agent-memory/src/rest/mod.rs +++ b/crates/agent-memory/src/rest/mod.rs @@ -29,7 +29,7 @@ use crate::mcp::AppState; /// everything else is a 500. The body is always `{"error": "..."}` so bad input /// can never surface as a 500. A missing resource is not an error here: /// handlers return their 404 body from not-found data (forget's `Ok(false)`, -/// update's `Ok(None)`), so a server fault can never be mis-tiered as a 404 (WR-02). +/// update's `Ok(None)`, link's `NotFound`), so a server fault can never be mis-tiered as a 404 (WR-02). #[derive(Debug)] pub enum ApiError { /// Invalid client input (e.g. an unknown memory type) → 400. @@ -78,6 +78,11 @@ pub fn build_router(state: Arc) -> Router { "/api/memories/{id}", delete(handlers::forget_handler).patch(handlers::update_handler), ) + .route("/api/links", post(handlers::link_handler)) + .route( + "/api/links/{from_id}/{to_id}/{kind}", + delete(handlers::unlink_handler), + ) .route("/health", get(handlers::health_handler)) .with_state(state) } From d6363281222a594db44999dcee2fe1cdf57a776a Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:23:23 +0200 Subject: [PATCH 10/42] feat(04-03): 1-hop expansion via expand_links on search and list (MCP-07) - MemoryView gains related: Option> with skip_serializing_if, so flag-less envelopes stay byte-identical to v1.0 (golden-tested for search and list) - RelatedLink {id, kind, direction, type, snippet, tags}: incoming edges read their natural reverse label (superseded_by, caused); snippet cut at 160 chars on a char boundary with a trailing ellipsis (D-05) - Store::related: one read-pool query per anchor following edges both ways, newest first, LIMIT MAX_EXPANDED_LINKS = 10 (D-07) - Service expands AFTER truncation and AFTER spawn_bump(returned ids) on both search legs and list; neighbors are never bumped (D-06) - expand_links (serde default false) on MCP search/list args and REST POST /api/search body + GET /api/memories query - README: relations section, expansion shape, bounds table Co-Authored-By: Claude Opus 5.5 --- README.md | 92 ++++- crates/agent-memory-core/src/domain.rs | 68 ++++ crates/agent-memory-core/src/service.rs | 56 ++- crates/agent-memory-core/src/store/mod.rs | 9 +- crates/agent-memory-core/src/store/sqlite.rs | 71 +++- crates/agent-memory-core/tests/relations.rs | 361 ++++++++++++++++++- crates/agent-memory/src/mcp.rs | 107 +++++- crates/agent-memory/src/rest/handlers.rs | 116 +++++- 8 files changed, 861 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 8672700..95b1608 100644 --- a/README.md +++ b/README.md @@ -101,15 +101,17 @@ Drop the same block into each agent runtime's MCP config; because they open the same `AGENT_MEMORY_DB` file (WAL-mode SQLite with a serialized writer), a memory stored in one is immediately retrievable in another. -## The five tools +## The seven tools | Tool | Purpose | Validated parameters | |------|---------|----------------------| | `memory_store` | Persist a typed memory; returns its id. | `ttl_secs`: optional integer, valid range 1–3155760000 (~100 years); omitted = no expiry | -| `memory_search` | Semantic (embedding) search when Ollama is reachable, FTS5 keyword otherwise — ranked by relevance × decay, recency-bumped on retrieval. | `limit`: optional integer, valid range 1–200; omitted defaults to 50. `tag`: exact tag match (case-sensitive) | -| `memory_list` | List memories newest-first, with optional type/tag/scope filters. | `limit`: optional integer, valid range 1–200; omitted = unlimited. `tag`: exact tag match (case-sensitive) | +| `memory_search` | Semantic (embedding) search when Ollama is reachable, FTS5 keyword otherwise — ranked by relevance × decay, recency-bumped on retrieval. | `limit`: optional integer, valid range 1–200; omitted defaults to 50. `tag`: exact tag match (case-sensitive). `expand_links`: optional boolean, default `false` | +| `memory_list` | List memories newest-first, with optional type/tag/scope filters. | `limit`: optional integer, valid range 1–200; omitted = unlimited. `tag`: exact tag match (case-sensitive). `expand_links`: optional boolean, default `false` | | `memory_update` | Patch a memory in place by id — content, tags, scope, TTL (clean not-found for unknown ids). | `content`: never null or empty. `ttl_secs`: 1–3155760000 when set; `null` = no expiry. At least one field besides `id` | -| `memory_forget` | Delete a memory by id (clean not-found for unknown ids). | — | +| `memory_link` | Link two memories with a typed, directed relation (idempotent; clean not-found for unknown ids). | `kind`: one of `relates_to`, `supersedes`, `caused_by`. `from_id` ≠ `to_id` | +| `memory_unlink` | Remove a relation (clean not-found when no such edge exists). | `kind`: one of `relates_to`, `supersedes`, `caused_by` | +| `memory_forget` | Delete a memory by id (clean not-found for unknown ids). Its relations go with it. | — | Out-of-bounds values are rejected as client errors (`invalid_params` over MCP) with a message naming the field, the valid range, and the offending value — @@ -162,6 +164,9 @@ matches whole tags exactly and case-sensitively. A present `limit` must be between 1 and 200 (omitted returns everything); a `tag` filter matches an exact tag (case-sensitive). +Both `memory_search` and `memory_list` accept `"expand_links": true` to attach +each result's linked memories — see [Relations](#relations-memory_link--memory_unlink). + ### `memory_forget` ```json @@ -216,6 +221,73 @@ returns — plus `embedding_status`: For an unknown id the result is `{ "id": 42, "updated": false, "reason": "not_found" }` — a normal tool result, not an error. +### Relations (`memory_link` / `memory_unlink`) + +Connect memories with flat, typed relations — no graph model, just one directed +edge `from_id -[kind]-> to_id` per link: + +| `kind` | Reads as | Seen from `to_id` (expansion) | +|--------|----------|-------------------------------| +| `relates_to` | `from_id` relates to `to_id` (symmetric "see also") | `relates_to` | +| `supersedes` | `from_id` replaces the older `to_id` | `superseded_by` | +| `caused_by` | `from_id` was caused by `to_id` | `caused` | + +```json +{ "from_id": 57, "to_id": 42, "kind": "supersedes" } +``` + +`memory_link` returns `{ "from_id": 57, "to_id": 42, "kind": "supersedes", "linked": true }`. + +- **Idempotent.** Linking the same `(from_id, to_id, kind)` again is a no-op + success — safe to retry. A different `kind` between the same pair is a + separate relation. +- **Rejected as `invalid_params`:** a self-link (`from_id and to_id must differ + (got 57)`) and an unknown kind (`kind must be one of relates_to, supersedes, + caused_by (got 'blocks')`). +- **Unknown ids are a clean not-found:** `{ ..., "linked": false, "reason": + "not_found", "missing_id": 42 }` names the id that does not exist. Nothing is + written. + +`memory_unlink` takes the same three fields; direction and kind must match the +stored edge. It returns `"unlinked": true`, or `"unlinked": false, "reason": +"not_found"` when no such edge exists. + +**Relations never outlive their memories.** Forgetting a memory, or its TTL +expiring, removes every relation touching it in the same transaction. + +**Expansion.** Pass `"expand_links": true` to `memory_search` or +`memory_list` and every result gains a `related` list of trimmed 1-hop +summaries, following relations in both directions: + +```json +{ "id": 42, "content": "...", "type": "DECISION", "...": "...", + "related": [ + { "id": 57, "kind": "superseded_by", "direction": "in", + "type": "DECISION", "snippet": "Maven group is io.github.unityinflow (Sonatype-verified).", "tags": ["maven"] } + ] } +``` + +- `direction` is `out` when the result is the relation's `from_id` and `in` + when it is the `to_id`; `kind` reads naturally from the result's side (the + table above). +- `snippet` is the neighbor's content cut to 160 characters, with a trailing + `…` when cut. Fetch the neighbor by id if you need all of it. +- At most `MAX_EXPANDED_LINKS = 10` neighbors per result; when a memory has + more, the most recently created relations win. +- Expansion is strictly one hop and a pure read: neighbors are **not** + recency-bumped, so expanding never changes how anything decays. +- A result with no relations gets `"related": []`. Without the flag there is no + `related` member at all — the response is exactly the v0.0.1 shape. + +#### Bounds + +| Bound | Value | Applies to | +|-------|-------|------------| +| `limit` | 1–200 (`MAX_LIMIT`) | `memory_search` (default 50), `memory_list` (default unlimited) | +| `ttl_secs` | 1–3155760000 (~100 years) | `memory_store`, `memory_update` | +| `MAX_EXPANDED_LINKS` | 10 neighbors per result, most recent first | `expand_links` on search and list | +| snippet length | 160 characters (+ `…` when cut) | `related[].snippet` | + ## Semantic search (optional Ollama) Out of the box, search uses SQLite FTS5 keyword ranking — fully offline, no @@ -256,10 +328,12 @@ agent-memory serve-rest --addr 127.0.0.1:8080 # custom loopback bind | Method | Path | Purpose | Validated parameters | |--------|------|---------|----------------------| | `POST` | `/api/memories` | Store a memory (same fields as `memory_store`) | `ttl_secs`: 1–3155760000 when present; omitted = no expiry | -| `GET` | `/api/memories` | List, with optional `type`/`tag`/`scope`/`limit` query params | `limit`: 1–200 when present; omitted = unlimited. `tag`: exact tag (case-sensitive) | -| `POST` | `/api/search` | Search — returns the same `{search_mode, results}` envelope | `limit`: 1–200 when present; omitted defaults to 50. `tag`: exact tag (case-sensitive) | +| `GET` | `/api/memories` | List, with optional `type`/`tag`/`scope`/`limit`/`expand_links` query params | `limit`: 1–200 when present; omitted = unlimited. `tag`: exact tag (case-sensitive). `expand_links=true` attaches `related` summaries | +| `POST` | `/api/search` | Search — returns the same `{search_mode, results}` envelope | `limit`: 1–200 when present; omitted defaults to 50. `tag`: exact tag (case-sensitive). `"expand_links": true` attaches `related` summaries | | `PATCH` | `/api/memories/{id}` | Update in place — JSON merge-patch body with the same fields and semantics as `memory_update`; 200 returns the updated memory with `embedding_status` | as `memory_update`; 404 `{"id", "updated": false, "reason": "not_found"}` for unknown ids | | `DELETE` | `/api/memories/{id}` | Forget by id (404 with a structured body for unknown ids) | — | +| `POST` | `/api/links` | Link — JSON body `{from_id, to_id, kind}`, same semantics as `memory_link`; 200 (also for a duplicate) | 400 for a self-link, an unknown `kind` or a bad body; 404 `{..., "linked": false, "reason": "not_found", "missing_id"}` for unknown ids | +| `DELETE` | `/api/links/{from_id}/{to_id}/{kind}` | Unlink — same semantics as `memory_unlink` | 400 for an unknown `kind`; 404 `{..., "unlinked": false, "reason": "not_found"}` when no such edge exists | | `GET` | `/health` | Liveness + embedder reachability status | — | Out-of-bounds `limit`/`ttl_secs` values return **HTTP 400** with a JSON body @@ -275,6 +349,12 @@ without `Content-Type: application/json` is also a 400 with the same curl -X PATCH localhost:7437/api/memories/42 \ -H 'content-type: application/json' \ -d '{"tags": ["maven"], "ttl_secs": null}' + +curl -X POST localhost:7437/api/links \ + -H 'content-type: application/json' \ + -d '{"from_id": 57, "to_id": 42, "kind": "supersedes"}' + +curl 'localhost:7437/api/memories?type=DECISION&expand_links=true' ``` The REST daemon and the MCP stdio server can run concurrently against the same diff --git a/crates/agent-memory-core/src/domain.rs b/crates/agent-memory-core/src/domain.rs index dbab31f..c429a3b 100644 --- a/crates/agent-memory-core/src/domain.rs +++ b/crates/agent-memory-core/src/domain.rs @@ -147,6 +147,55 @@ pub enum LinkOutcome { NotFound { missing_id: i64 }, } +/// Which way a [`RelatedLink`] edge points, seen from the result it is +/// attached to: `out` = the result is the edge's `from_id`, `in` = the +/// result is its `to_id` (D-03 / D-05). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum LinkDirection { + Out, + In, +} + +/// A trimmed 1-hop neighbor summary attached to a search/list result when +/// `expand_links` is set (MCP-07 / D-05) — enough for an agent to decide +/// whether to fetch the neighbor, never the full view. +/// +/// `kind` is the direction-aware natural label: the stored kind for an +/// outgoing edge, [`LinkKind::reverse_label`] for an incoming one (so an +/// incoming `supersedes` reads `superseded_by`). +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct RelatedLink { + /// The neighbor memory's id. + pub id: i64, + /// Natural relation label from the anchor's point of view. + pub kind: String, + /// `out` (anchor → neighbor) or `in` (neighbor → anchor). + pub direction: LinkDirection, + /// The neighbor's memory type. + #[serde(rename = "type")] + pub mem_type: MemoryType, + /// The neighbor's content, cut to [`RELATED_SNIPPET_CHARS`] characters + /// (with a trailing `…` when cut). + pub snippet: String, + /// The neighbor's tags. + pub tags: Vec, +} + +/// Maximum characters of neighbor content carried in a [`RelatedLink`] +/// snippet (D-05). Counted in Unicode scalar values, so a cut never splits a +/// UTF-8 codepoint. +pub const RELATED_SNIPPET_CHARS: usize = 160; + +/// Cut `content` to [`RELATED_SNIPPET_CHARS`] characters on a char boundary, +/// appending `…` only when something was cut. +pub fn snippet(content: &str) -> String { + match content.char_indices().nth(RELATED_SNIPPET_CHARS) { + Some((byte_idx, _)) => format!("{}…", &content[..byte_idx]), + None => content.to_string(), + } +} + /// Input shape for storing a new memory (D-05). /// /// Only `content` and `mem_type` are conceptually required at the service boundary; @@ -180,6 +229,11 @@ pub struct Memory { } /// The read/return shape exposed over the tool boundary (D-06). +/// +/// INVARIANT — field order is wire order: serde emits members in declaration +/// order, and the v1.0 envelope (`id`, `content`, `type`, `tags`, `scope`, +/// `decay_score`, `created_at`, `last_accessed`) is a golden-tested contract. +/// New fields append at the end and must be skipped when absent. #[derive(Debug, Clone, PartialEq, Serialize)] pub struct MemoryView { pub id: i64, @@ -191,6 +245,13 @@ pub struct MemoryView { pub decay_score: f64, pub created_at: i64, pub last_accessed: i64, + /// 1-hop relation summaries (MCP-07 / D-04, D-05): `Some` only when the + /// caller asked for `expand_links` (an explicit `[]` when the memory has + /// no edges), `None` otherwise. `None` is SKIPPED on the wire, so a + /// flag-less search/list envelope stays byte-identical to v1.0 + /// (RESEARCH Pitfall 7). + #[serde(skip_serializing_if = "Option::is_none")] + pub related: Option>, } /// A JSON-merge-patch for `memory_update` (MCP-06 / D-08). @@ -252,6 +313,13 @@ pub const MIN_TTL_SECS: i64 = 1; /// arithmetic can never overflow (RESEARCH Pitfall 1). pub const MAX_TTL_SECS: i64 = 3_155_760_000; +/// Hard cap on the 1-hop neighbors attached to ONE search/list result when +/// `expand_links` is set (MCP-07 / D-07). Overflow keeps the most recent +/// edges (`memory_links.created_at` descending). Fixed — there is no +/// caller-tunable knob in v0.1.0. Together with [`MAX_LIMIT`] it bounds an +/// expanded response at `200 × 10` summaries (T-04-11). +pub const MAX_EXPANDED_LINKS: usize = 10; + /// The library's typed error. `thiserror` per the ecosystem rule (libraries use /// `thiserror`; the binary uses `anyhow` at its edges). #[derive(Debug, Error)] diff --git a/crates/agent-memory-core/src/service.rs b/crates/agent-memory-core/src/service.rs index c26627e..13bedb1 100644 --- a/crates/agent-memory-core/src/service.rs +++ b/crates/agent-memory-core/src/service.rs @@ -18,7 +18,7 @@ use crate::clock::Clock; use crate::decay::{apply_decay, decay_score, DecayConfig, DecayEngine, RankWeights, SweepReport}; use crate::domain::{ LinkKind, LinkOutcome, MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, - UpdatedMemory, + UpdatedMemory, MAX_EXPANDED_LINKS, }; use crate::embed::Embedder; use crate::store::sqlite::DEFAULT_SEARCH_LIMIT; @@ -32,6 +32,9 @@ pub struct ListArgs { pub tag: Option, pub scope: Option, pub limit: Option, + /// Attach 1-hop relation summaries to every returned row (MCP-07 / D-04). + /// `false` (the default) keeps the v1.0 response byte-identical. + pub expand_links: bool, } /// Arguments for [`MemoryService::search`]. `query` is the FTS5 MATCH string; the @@ -43,6 +46,10 @@ pub struct SearchArgs { pub tag: Option, pub scope: Option, pub limit: Option, + /// Attach 1-hop relation summaries to every returned row (MCP-07 / D-04). + /// Orthogonal to the search mode: applies on the semantic and the keyword + /// path alike. `false` (the default) keeps the v1.0 envelope byte-identical. + pub expand_links: bool, } /// Which search path produced a result set. Serialized lowercase — the shared @@ -147,11 +154,16 @@ impl MemoryService { // API-02 / D-01: a present limit must be 1..=200; omitted stays // unlimited (v1.0 behavior, RESEARCH Pitfall 6). crate::domain::validate_limit(args.limit)?; + let expand_links = args.expand_links; let store = self.store.clone(); let now = self.clock.now(); - tokio::task::spawn_blocking(move || store.list(args, now)) + let views = tokio::task::spawn_blocking(move || store.list(args, now)) .await - .map_err(MemoryError::Join)? + .map_err(MemoryError::Join)??; + if expand_links { + return self.expand_related(views).await; + } + Ok(views) } /// Search the corpus (MCP-02 / SEARCH-02 / SEARCH-03 / D-04). @@ -179,6 +191,7 @@ impl MemoryService { let cfg = self.decay_cfg; let weights = RankWeights::default(); let limit = args.limit.unwrap_or(DEFAULT_SEARCH_LIMIT) as usize; + let expand_links = args.expand_links; let query_input = [args.query.clone()]; match self.embedder.embed(&query_input).await { @@ -216,7 +229,14 @@ impl MemoryService { .map(|(view, _)| view) .collect(); + // Bump ONLY the returned ids, before expansion runs — expanded + // neighbors are never passed here (D-06). self.spawn_bump(results.iter().map(|v| v.id).collect(), now); + let results = if expand_links { + self.expand_related(results).await? + } else { + results + }; Ok(SearchOutcome { search_mode: SearchMode::Semantic, results, @@ -245,7 +265,14 @@ impl MemoryService { apply_decay(view, now, &cfg); } + // Same order as the semantic leg: bump the returned ids, then + // expand — neighbors are never bumped (D-06). self.spawn_bump(views.iter().map(|v| v.id).collect(), now); + let views = if expand_links { + self.expand_related(views).await? + } else { + views + }; Ok(SearchOutcome { search_mode: SearchMode::Keyword, results: views, @@ -339,6 +366,29 @@ impl MemoryService { }) } + /// Attach 1-hop relation summaries to already-final result rows (MCP-07): + /// one `spawn_blocking` hop that reads each row's neighbors via + /// [`Store::related`], capped at [`MAX_EXPANDED_LINKS`] most-recent edges + /// (D-07). Runs AFTER truncation and AFTER the recency bump, and is a pure + /// read — neighbor ids never reach [`Self::spawn_bump`] (D-06). Every row + /// gets `Some(..)`, an explicit `[]` when it has no edges, so a flag-on + /// response always carries the member. Strictly 1-hop: neighbors are not + /// themselves expanded (T-04-11). + async fn expand_related( + &self, + mut views: Vec, + ) -> Result, MemoryError> { + let store = self.store.clone(); + tokio::task::spawn_blocking(move || { + for view in &mut views { + view.related = Some(store.related(view.id, MAX_EXPANDED_LINKS)?); + } + Ok(views) + }) + .await + .map_err(MemoryError::Join)? + } + /// Fire-and-forget recency bump: retrieving a memory bumps its /// last_accessed/access_count so it surfaces higher next time (Open /// Question 2). A failure here must not fail the search. diff --git a/crates/agent-memory-core/src/store/mod.rs b/crates/agent-memory-core/src/store/mod.rs index 6bd0179..5c1f619 100644 --- a/crates/agent-memory-core/src/store/mod.rs +++ b/crates/agent-memory-core/src/store/mod.rs @@ -12,7 +12,7 @@ pub mod sqlite; use crate::decay::{DecayConfig, RankWeights}; use crate::domain::{ LinkKind, LinkOutcome, MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, - UpdatedMemory, + RelatedLink, UpdatedMemory, }; use crate::service::{ListArgs, SearchArgs}; @@ -131,6 +131,13 @@ pub trait Store: Send + Sync { /// and kind are part of the edge's identity. fn unlink(&self, from_id: i64, to_id: i64, kind: LinkKind) -> Result; + /// The 1-hop neighbors of memory `id`, following its edges in BOTH + /// directions (MCP-07 / D-03), as trimmed summaries (D-05). At most `cap` + /// entries, most-recent edges first (`created_at` descending — D-07). + /// A pure read on the read pool: it never bumps anything (D-06). An id + /// with no edges (or no row) yields an empty vec. + fn related(&self, id: i64, cap: usize) -> Result, MemoryError>; + /// Bump the recency of the given ids: set `last_accessed = now` and increment /// `access_count`. Runs on the writer lane. A no-op for an empty id slice. fn bump_access(&self, ids: &[i64], now: i64) -> Result<(), MemoryError>; diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index 9ddb973..9afa89c 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -21,8 +21,8 @@ use rusqlite_migration::Migrations; use crate::decay::{DecayConfig, RankWeights, PINNED_HALF_LIFE_MULTIPLIER}; use crate::domain::{ - LinkKind, LinkOutcome, MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, - UpdatedMemory, + snippet, LinkDirection, LinkKind, LinkOutcome, MemoryError, MemoryPatch, MemoryType, + MemoryView, NewMemory, RelatedLink, UpdatedMemory, }; use crate::service::{ListArgs, SearchArgs}; use crate::store::migrations::{migrations, LATEST_SCHEMA_VERSION}; @@ -299,9 +299,25 @@ fn row_to_view(row: &rusqlite::Row<'_>) -> Result { decay_score: row.get("decay_score")?, created_at: row.get("created_at")?, last_accessed: row.get("last_accessed")?, + // Expansion is attached by the service only when asked for (D-04). + related: None, }) } +/// A stored text value that fails its closed-enum parse is corrupt data, not +/// client input: report it as a row conversion failure (the `row_to_view` +/// mem_type precedent), which surfaces on the internal tier. +fn corrupt_column(what: &str, value: &str) -> rusqlite::Error { + rusqlite::Error::FromSqlConversionFailure( + 0, + rusqlite::types::Type::Text, + Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid {what} in row: {value}"), + )), + ) +} + /// Classify a post-bind FTS5 execution error at the store seam: a MATCH string /// FTS5 cannot parse is CLIENT input → [`MemoryError::InvalidQuery`]; anything /// else stays an internal [`MemoryError::Sqlite`]. @@ -794,6 +810,57 @@ impl Store for SqliteStore { Ok(Some(updated)) } + fn related(&self, id: i64, cap: usize) -> Result, MemoryError> { + // Read pool: expansion is a pure read and never touches the writer. + let conn = self.reads.get().map_err(MemoryError::Pool)?; + // Both directions in one pass (D-03): the CASE picks the FAR endpoint + // of each edge touching `id`. The from_id lookup rides the primary key + // and the to_id lookup rides idx_links_to. Newest edges first, so the + // LIMIT keeps the most recent (D-07); the id/kind tail only makes ties + // deterministic. All values bound (T-04-10). + let mut stmt = conn.prepare( + "SELECT l.from_id, l.kind, m.id, m.mem_type, m.content, m.tags \ + FROM memory_links l \ + JOIN memories m \ + ON m.id = CASE WHEN l.from_id = ?1 THEN l.to_id ELSE l.from_id END \ + WHERE l.from_id = ?1 OR l.to_id = ?1 \ + ORDER BY l.created_at DESC, m.id DESC, l.kind ASC \ + LIMIT ?2", + )?; + let cap = i64::try_from(cap).unwrap_or(i64::MAX); + let rows = stmt.query_map(params![id, cap], |row| { + let from_id: i64 = row.get(0)?; + let kind_str: String = row.get(1)?; + let mem_type_str: String = row.get(3)?; + let content: String = row.get(4)?; + let tags_json: String = row.get(5)?; + let kind = LinkKind::try_from(kind_str.as_str()) + .map_err(|_| corrupt_column("link kind", &kind_str))?; + let mem_type = MemoryType::try_from(mem_type_str.as_str()) + .map_err(|_| corrupt_column("mem_type", &mem_type_str))?; + // Seen from the anchor: an outgoing edge reads its stored kind, + // an incoming one its natural reverse label (superseded_by, caused). + let (direction, label) = if from_id == id { + (LinkDirection::Out, kind.as_wire_str()) + } else { + (LinkDirection::In, kind.reverse_label()) + }; + Ok(RelatedLink { + id: row.get(2)?, + kind: label.to_string(), + direction, + mem_type, + snippet: snippet(&content), + tags: serde_json::from_str(&tags_json).unwrap_or_default(), + }) + })?; + let mut out = Vec::new(); + for row in rows { + out.push(row?); + } + Ok(out) + } + fn bump_access(&self, ids: &[i64], now: i64) -> Result<(), MemoryError> { if ids.is_empty() { return Ok(()); diff --git a/crates/agent-memory-core/tests/relations.rs b/crates/agent-memory-core/tests/relations.rs index 5cef6ca..cbf5149 100644 --- a/crates/agent-memory-core/tests/relations.rs +++ b/crates/agent-memory-core/tests/relations.rs @@ -6,7 +6,11 @@ //! supersedes, caused_by — anything else is `InvalidArgument` naming the //! allowed set), D-02 (duplicate link = idempotent no-op success, self-link = //! `InvalidArgument`, missing endpoint / missing edge = clean not-found data, -//! never an error), D-03 (one directed edge row per link). +//! never an error), D-03 (one directed edge row per link, read in both +//! directions with natural reverse labels), D-04 (`expand_links` is opt-in and +//! the flag-less envelope is byte-identical to v1.0), D-05 (trimmed neighbor +//! summaries), D-06 (expanded neighbors are never access-bumped) and D-07 +//! (at most `MAX_EXPANDED_LINKS` neighbors, most-recent edges win). //! //! Harness: a REAL on-disk temp DB, an injected `TestClock`, and a failing //! `FakeEmbedder` (stores land pending; search deterministically takes the @@ -18,9 +22,12 @@ use std::sync::Arc; use agent_memory_core::clock::{Clock, TestClock}; use agent_memory_core::decay::DecayConfig; -use agent_memory_core::domain::{LinkKind, LinkOutcome, MemoryError, MemoryType, NewMemory}; +use agent_memory_core::domain::{ + LinkDirection, LinkKind, LinkOutcome, MemoryError, MemoryType, MemoryView, NewMemory, + RelatedLink, MAX_EXPANDED_LINKS, +}; use agent_memory_core::embed::{Embedder, FakeEmbedder}; -use agent_memory_core::service::MemoryService; +use agent_memory_core::service::{ListArgs, MemoryService, SearchArgs, SearchMode}; use agent_memory_core::store::sqlite::{register_vec_extension, SqliteStore}; struct Harness { @@ -31,11 +38,14 @@ struct Harness { } fn harness() -> Harness { + harness_with(Arc::new(FakeEmbedder::failing())) +} + +fn harness_with(embedder: Arc) -> Harness { let dir = tempfile::tempdir().expect("tempdir"); let db_path = dir.path().join("memory.db"); let store = SqliteStore::open(&db_path).expect("open store"); let clock = Arc::new(TestClock::new(1_000)); - let embedder: Arc = Arc::new(FakeEmbedder::failing()); let service = MemoryService::new( Arc::new(store), clock.clone() as Arc, @@ -382,3 +392,346 @@ async fn sweep_also_clears_orphans_left_by_a_connection_without_the_fk_pragma() assert_eq!(report.expired, 0); assert_eq!(orphan_edges(&h.db_path), 0); } + +// --------------------------------------------------------------------------- +// Task 2 — 1-hop expansion via expand_links +// --------------------------------------------------------------------------- + +fn search_args(query: &str, expand_links: bool) -> SearchArgs { + SearchArgs { + query: query.to_string(), + expand_links, + ..Default::default() + } +} + +fn list_args(expand_links: bool) -> ListArgs { + ListArgs { + expand_links, + ..Default::default() + } +} + +fn view_of(views: &[MemoryView], id: i64) -> &MemoryView { + views + .iter() + .find(|v| v.id == id) + .unwrap_or_else(|| panic!("id {id} missing from {views:?}")) +} + +fn related_of(views: &[MemoryView], id: i64) -> &[RelatedLink] { + view_of(views, id) + .related + .as_deref() + .unwrap_or_else(|| panic!("id {id} carries no related list")) +} + +#[tokio::test] +async fn expansion_follows_one_edge_both_ways_with_natural_labels() { + let h = harness(); + let newer = store(&h, "orbit decision newer").await; + let older = store(&h, "orbit decision older").await; + let cause = store(&h, "orbit root cause").await; + h.service + .link(newer, older, LinkKind::Supersedes) + .await + .expect("newer supersedes older"); + h.service + .link(newer, cause, LinkKind::CausedBy) + .await + .expect("newer caused_by cause"); + + let outcome = h + .service + .search(search_args("orbit", true)) + .await + .expect("search"); + assert_eq!(outcome.search_mode, SearchMode::Keyword); + let results = &outcome.results; + + // The edge's own endpoint reads the stored kind, outgoing. + let newer_related = related_of(results, newer); + assert_eq!(newer_related.len(), 2); + let to_older = newer_related + .iter() + .find(|r| r.id == older) + .expect("older is a neighbor of newer"); + assert_eq!(to_older.kind, "supersedes"); + assert_eq!(to_older.direction, LinkDirection::Out); + assert_eq!(to_older.mem_type, MemoryType::Decision); + assert_eq!(to_older.snippet, "orbit decision older"); + assert_eq!(to_older.tags, vec!["rel".to_string()]); + let to_cause = newer_related + .iter() + .find(|r| r.id == cause) + .expect("cause is a neighbor of newer"); + assert_eq!(to_cause.kind, "caused_by"); + assert_eq!(to_cause.direction, LinkDirection::Out); + + // The SAME single edge row read from the far side: natural reverse labels. + let older_related = related_of(results, older); + assert_eq!(older_related.len(), 1); + assert_eq!(older_related[0].id, newer); + assert_eq!(older_related[0].kind, "superseded_by"); + assert_eq!(older_related[0].direction, LinkDirection::In); + let cause_related = related_of(results, cause); + assert_eq!(cause_related.len(), 1); + assert_eq!(cause_related[0].id, newer); + assert_eq!(cause_related[0].kind, "caused"); + assert_eq!(cause_related[0].direction, LinkDirection::In); + + assert_eq!(edge_count(&h.db_path), 2, "still one row per edge (D-03)"); +} + +#[tokio::test] +async fn relates_to_reads_the_same_from_both_ends() { + let h = harness(); + let a = store(&h, "peer one").await; + let b = store(&h, "peer two").await; + h.service + .link(a, b, LinkKind::RelatesTo) + .await + .expect("link"); + + let views = h.service.list(list_args(true)).await.expect("list"); + let from_a = &related_of(&views, a)[0]; + let from_b = &related_of(&views, b)[0]; + assert_eq!((from_a.id, from_a.kind.as_str()), (b, "relates_to")); + assert_eq!(from_a.direction, LinkDirection::Out); + assert_eq!((from_b.id, from_b.kind.as_str()), (a, "relates_to")); + assert_eq!(from_b.direction, LinkDirection::In); +} + +#[tokio::test] +async fn flag_less_envelopes_are_byte_identical_to_v1_and_never_mention_related() { + let h = harness(); + let alpha = store(&h, "golden alpha").await; + let partner = store(&h, "golden partner").await; + h.service + .link(alpha, partner, LinkKind::Supersedes) + .await + .expect("link"); + assert_eq!((alpha, partner), (1, 2), "fresh DB: ids are 1 and 2"); + + // Golden v1.0 envelopes — captured from the pre-expansion wire shape. + // Edges exist, yet without the flag nothing about them may appear. + let list_golden = concat!( + r#"[{"id":2,"content":"golden partner","type":"DECISION","tags":["rel"],"#, + r#""scope":null,"decay_score":1.0,"created_at":1000,"last_accessed":1000},"#, + r#"{"id":1,"content":"golden alpha","type":"DECISION","tags":["rel"],"#, + r#""scope":null,"decay_score":1.0,"created_at":1000,"last_accessed":1000}]"# + ); + let search_golden = concat!( + r#"{"search_mode":"keyword","results":[{"id":1,"content":"golden alpha","#, + r#""type":"DECISION","tags":["rel"],"scope":null,"decay_score":1.0,"#, + r#""created_at":1000,"last_accessed":1000}]}"# + ); + + let listed = h.service.list(list_args(false)).await.expect("list"); + let list_json = serde_json::to_string(&listed).expect("serialize list"); + assert_eq!(list_json, list_golden); + assert!(!list_json.contains("related")); + + let found = h + .service + .search(search_args("alpha", false)) + .await + .expect("search"); + let search_json = serde_json::to_string(&found).expect("serialize search"); + assert_eq!(search_json, search_golden); + assert!(!search_json.contains("related")); + + // With the flag the same query carries the trimmed summary, in D-05 + // field order, after the v1.0 fields. + let expanded = h + .service + .search(search_args("alpha", true)) + .await + .expect("expanded search"); + let expanded_json = serde_json::to_string(&expanded).expect("serialize"); + assert_eq!( + expanded_json, + concat!( + r#"{"search_mode":"keyword","results":[{"id":1,"content":"golden alpha","#, + r#""type":"DECISION","tags":["rel"],"scope":null,"decay_score":1.0,"#, + r#""created_at":1000,"last_accessed":1000,"related":[{"id":2,"#, + r#""kind":"supersedes","direction":"out","type":"DECISION","#, + r#""snippet":"golden partner","tags":["rel"]}]}]}"# + ) + ); +} + +#[tokio::test] +async fn flag_on_without_edges_is_an_explicit_empty_related_list() { + let h = harness(); + let lone = store(&h, "unlinked memory").await; + + let views = h.service.list(list_args(true)).await.expect("list"); + assert_eq!(view_of(&views, lone).related, Some(vec![])); + let json = serde_json::to_string(&views).expect("serialize"); + assert!(json.contains(r#""related":[]"#), "{json}"); +} + +#[tokio::test] +async fn expansion_is_capped_and_the_most_recent_edges_win() { + let h = harness(); + let anchor = store(&h, "hub memory").await; + let mut neighbors = Vec::new(); + for i in 0..(MAX_EXPANDED_LINKS + 2) { + let n = store(&h, &format!("spoke {i}")).await; + // Stagger created_at so "most recent" is unambiguous; alternate the + // direction so the cap orders both directions together. + h.clock.advance(1); + if i % 2 == 0 { + h.service + .link(anchor, n, LinkKind::RelatesTo) + .await + .expect("out link"); + } else { + h.service + .link(n, anchor, LinkKind::CausedBy) + .await + .expect("in link"); + } + neighbors.push(n); + } + assert_eq!(MAX_EXPANDED_LINKS, 10, "D-07 cap value"); + + let views = h.service.list(list_args(true)).await.expect("list"); + let related = related_of(&views, anchor); + assert_eq!(related.len(), MAX_EXPANDED_LINKS); + + // The two OLDEST edges are the ones dropped; the rest come newest first. + let expected: Vec = neighbors + .iter() + .rev() + .take(MAX_EXPANDED_LINKS) + .copied() + .collect(); + let got: Vec = related.iter().map(|r| r.id).collect(); + assert_eq!(got, expected); + assert!(!got.contains(&neighbors[0]) && !got.contains(&neighbors[1])); +} + +#[test] +fn snippets_cut_at_160_chars_on_a_char_boundary_with_an_ellipsis() { + use agent_memory_core::domain::{snippet, RELATED_SNIPPET_CHARS}; + assert_eq!(RELATED_SNIPPET_CHARS, 160); + + let short = "fits easily"; + assert_eq!(snippet(short), short, "no ellipsis when nothing is cut"); + + let exact: String = "x".repeat(160); + assert_eq!(snippet(&exact), exact, "exactly 160 chars is not cut"); + + // Multi-byte content: a byte-index cut would split a codepoint. + let long: String = "é".repeat(200); + let cut = snippet(&long); + assert!(cut.ends_with('…')); + assert_eq!(cut.chars().count(), 161, "160 chars plus the ellipsis"); + assert_eq!(cut.trim_end_matches('…'), "é".repeat(160)); +} + +#[tokio::test] +async fn expanded_snippets_are_truncated_for_long_neighbor_content() { + let h = harness(); + let anchor = store(&h, "short anchor").await; + let long_content = format!("{} tail", "ünïcode ".repeat(30)); + let long = store(&h, &long_content).await; + h.service + .link(anchor, long, LinkKind::RelatesTo) + .await + .expect("link"); + + let views = h.service.list(list_args(true)).await.expect("list"); + let entry = &related_of(&views, anchor)[0]; + assert!(entry.snippet.ends_with('…'), "{:?}", entry.snippet); + assert_eq!(entry.snippet.chars().count(), 161); + assert!(long_content.starts_with(entry.snippet.trim_end_matches('…'))); +} + +/// Raw `(access_count, last_accessed)` for one row. +fn access_of(db_path: &Path, id: i64) -> (i64, i64) { + raw_conn(db_path) + .query_row( + "SELECT access_count, last_accessed FROM memories WHERE id = ?1", + [id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .expect("row exists") +} + +#[tokio::test] +async fn expanded_neighbors_are_never_access_bumped() { + let h = harness(); + let anchor = store(&h, "needle anchor").await; + let mut neighbors = Vec::new(); + for word in ["one", "two", "three"] { + let n = store(&h, &format!("haystack {word}")).await; + h.service + .link(anchor, n, LinkKind::RelatesTo) + .await + .expect("link"); + neighbors.push(n); + } + + h.clock.advance(500); + let found = h + .service + .search(search_args("needle", true)) + .await + .expect("search"); + assert_eq!(found.results.len(), 1, "only the anchor matches"); + assert_eq!(found.results[0].related.as_ref().map(Vec::len), Some(3)); + + // The returned result's bump is fire-and-forget: wait for it to land + // (bounded), so the neighbor assertions below are not vacuous. + let mut bumped = false; + for _ in 0..200 { + if access_of(&h.db_path, anchor) == (1, 1_500) { + bumped = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!(bumped, "the returned result is still bumped as before"); + + for n in neighbors { + assert_eq!( + access_of(&h.db_path, n), + (0, 1_000), + "neighbor {n} must not be bumped by expansion (D-06)" + ); + } +} + +#[tokio::test] +async fn expansion_also_applies_on_the_semantic_path() { + // A succeeding hash-vector embedder: search takes the semantic leg. + let h = harness_with(Arc::new(FakeEmbedder::with_vectors( + std::collections::HashMap::new(), + ))); + let a = store(&h, "vector anchor").await; + let b = store(&h, "vector neighbor").await; + h.service + .link(b, a, LinkKind::Supersedes) + .await + .expect("link"); + + let found = h + .service + .search(search_args("vector anchor", true)) + .await + .expect("search"); + assert_eq!(found.search_mode, SearchMode::Semantic); + let entry = &related_of(&found.results, a)[0]; + assert_eq!((entry.id, entry.kind.as_str()), (b, "superseded_by")); + assert_eq!(entry.direction, LinkDirection::In); + + let plain = h + .service + .search(search_args("vector anchor", false)) + .await + .expect("search"); + assert!(plain.results.iter().all(|v| v.related.is_none())); +} diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index 8d9903f..24f891c 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -95,6 +95,14 @@ pub struct ListArgs { /// Cap the number of returned rows. #[serde(default)] pub limit: Option, + /// Attach up to 10 one-hop related memories to each result as + /// `related: [{id, kind, direction, type, snippet, tags}]`. `direction` is + /// `out` (this memory -> neighbor) or `in` (neighbor -> this memory); + /// `kind` reads naturally from this memory's side (an incoming supersedes + /// is `superseded_by`, an incoming caused_by is `caused`). Most recent + /// links first. Defaults to false (no `related` member at all). + #[serde(default)] + pub expand_links: bool, } /// Arguments for `memory_search` (D-04/D-06). `query` is the FTS5 keyword string; @@ -115,6 +123,14 @@ pub struct SearchArgs { /// Cap the number of returned rows. #[serde(default)] pub limit: Option, + /// Attach up to 10 one-hop related memories to each result as + /// `related: [{id, kind, direction, type, snippet, tags}]`. `direction` is + /// `out` (this memory -> neighbor) or `in` (neighbor -> this memory); + /// `kind` reads naturally from this memory's side (an incoming supersedes + /// is `superseded_by`, an incoming caused_by is `caused`). Most recent + /// links first. Defaults to false (no `related` member at all). + #[serde(default)] + pub expand_links: bool, } /// Arguments for `memory_forget` (D-06). @@ -258,7 +274,7 @@ impl MemoryMcp { } #[tool( - description = "List stored memories newest-first, with optional type/tag/scope/limit filters." + description = "List stored memories newest-first, with optional type/tag/scope/limit filters. Set expand_links=true to attach each memory's linked memories as trimmed related summaries." )] async fn memory_list( &self, @@ -280,6 +296,7 @@ impl MemoryMcp { tag: args.tag, scope: args.scope, limit: args.limit, + expand_links: args.expand_links, }) .await .map_err(map_mcp_error)?; @@ -290,7 +307,7 @@ impl MemoryMcp { } #[tool( - description = "Search stored memories by semantic similarity (local Ollama embeddings) blended with decay, falling back to keyword/FTS5 search when the embedder is unavailable. Returns {search_mode: 'semantic'|'keyword', results: [...]}; results is empty when nothing matches." + description = "Search stored memories by semantic similarity (local Ollama embeddings) blended with decay, falling back to keyword/FTS5 search when the embedder is unavailable. Returns {search_mode: 'semantic'|'keyword', results: [...]}; results is empty when nothing matches. Set expand_links=true to attach each result's linked memories as trimmed related summaries." )] async fn memory_search( &self, @@ -315,6 +332,7 @@ impl MemoryMcp { tag: args.tag, scope: args.scope, limit: args.limit, + expand_links: args.expand_links, }) .await .map_err(map_mcp_error)?; @@ -685,6 +703,7 @@ mod tests { tag: None, scope: None, limit: None, + expand_links: false, })) .await .expect("list"); @@ -909,6 +928,90 @@ mod tests { } } + #[tokio::test] + async fn expand_links_from_raw_args_attaches_related_on_search_and_list() { + let dir = tempfile::tempdir().expect("tempdir"); + let server = test_server(&dir); + let anchor = store_one(&server, "quasar anchor").await; + let long_neighbor = format!("{} end", "neighbor ".repeat(40)); + let neighbor = store_one(&server, &long_neighbor).await; + link_raw( + &server, + &format!(r#"{{"from_id": {neighbor}, "to_id": {anchor}, "kind": "supersedes"}}"#), + ) + .await + .expect("link"); + + // memory_search, flag on — raw wire JSON through the real DTO. + let args: SearchArgs = + serde_json::from_str(r#"{"query": "quasar", "expand_links": true}"#).expect("args"); + let body = result_json( + &server + .memory_search(Parameters(args)) + .await + .expect("search"), + ); + let related = &body["results"][0]["related"]; + assert_eq!(related[0]["id"], neighbor); + assert_eq!(related[0]["kind"], "superseded_by"); + assert_eq!(related[0]["direction"], "in"); + assert_eq!(related[0]["type"], "PATTERN"); + let snippet = related[0]["snippet"].as_str().expect("snippet"); + assert!(snippet.ends_with('…'), "long content is cut: {snippet}"); + assert_eq!(snippet.chars().count(), 161); + + // memory_search, flag absent — no related member anywhere. + let args: SearchArgs = serde_json::from_str(r#"{"query": "quasar"}"#).expect("args"); + let result = server + .memory_search(Parameters(args)) + .await + .expect("search"); + let text = serde_json::to_value(&result).expect("serializes")["content"][0]["text"] + .as_str() + .expect("text") + .to_string(); + assert!(!text.contains("related"), "{text}"); + + // memory_list, flag on and off. + let args: ListArgs = serde_json::from_str(r#"{"expand_links": true}"#).expect("args"); + let body = result_json(&server.memory_list(Parameters(args)).await.expect("list")); + let rows = body.as_array().expect("array"); + let anchor_row = rows.iter().find(|r| r["id"] == anchor).expect("anchor"); + assert_eq!(anchor_row["related"][0]["id"], neighbor); + let neighbor_row = rows.iter().find(|r| r["id"] == neighbor).expect("neighbor"); + assert_eq!(neighbor_row["related"][0]["kind"], "supersedes"); + assert_eq!(neighbor_row["related"][0]["direction"], "out"); + + let args: ListArgs = serde_json::from_str("{}").expect("args"); + let body = result_json(&server.memory_list(Parameters(args)).await.expect("list")); + assert!(body + .as_array() + .expect("array") + .iter() + .all(|r| r.get("related").is_none())); + } + + #[test] + fn expand_links_is_an_optional_boolean_in_both_tool_schemas() { + for schema in [ + serde_json::to_value(schemars::schema_for!(SearchArgs)).expect("schema"), + serde_json::to_value(schemars::schema_for!(ListArgs)).expect("schema"), + ] { + let property = &schema["properties"]["expand_links"]; + assert_eq!(property["type"], "boolean", "{schema}"); + assert_eq!(property["default"], false, "{schema}"); + let required = schema["required"].as_array().cloned().unwrap_or_default(); + assert!( + !required.contains(&serde_json::json!("expand_links")), + "{schema}" + ); + } + // A non-boolean is rejected by deserialization (the typed seam). + assert!( + serde_json::from_str::(r#"{"query": "q", "expand_links": "yes"}"#).is_err() + ); + } + #[test] fn map_mcp_error_splits_client_and_internal_tiers() { let invalid_params_code = McpError::invalid_params("x", None).code; diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index fbd8dc8..d35e9c2 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -65,6 +65,10 @@ pub struct SearchRequest { /// Cap the number of returned rows. #[serde(default)] pub limit: Option, + /// Attach 1-hop `related` summaries to each row (MCP-07). Omitted = false + /// (the v1.0 response, byte-identical). + #[serde(default)] + pub expand_links: bool, } /// Query string for `GET /api/memories` — mirrors `mcp::ListArgs`. @@ -82,6 +86,10 @@ pub struct ListQuery { /// Cap the number of returned rows. #[serde(default)] pub limit: Option, + /// Attach 1-hop `related` summaries to each row (MCP-07). Omitted = false + /// (the v1.0 response, byte-identical). + #[serde(default)] + pub expand_links: bool, } /// Body for `PATCH /api/memories/{id}` — mirrors `mcp::UpdateArgs` field for @@ -197,7 +205,7 @@ pub async fn store_handler( Ok((StatusCode::CREATED, Json(serde_json::json!({ "id": id }))).into_response()) } -/// `GET /api/memories?type=&tag=&scope=&limit=` → 200 `Vec`. +/// `GET /api/memories?type=&tag=&scope=&limit=&expand_links=` → 200 `Vec`. pub async fn list_handler( State(state): State>, Query(q): Query, @@ -211,6 +219,7 @@ pub async fn list_handler( tag: q.tag, scope: q.scope, limit: q.limit, + expand_links: q.expand_links, }) .await .map_err(map_memory_error)?; @@ -235,6 +244,7 @@ pub async fn search_handler( tag: req.tag, scope: req.scope, limit: req.limit, + expand_links: req.expand_links, }) .await .map_err(map_memory_error)?; @@ -526,6 +536,7 @@ mod tests { tag: None, scope: None, limit: None, + expand_links: false, }), ) .await @@ -544,6 +555,7 @@ mod tests { tag: None, scope: None, limit: None, + expand_links: false, }), ) .await @@ -574,6 +586,7 @@ mod tests { tag: None, scope: None, limit: Some(10), + expand_links: false, }), ) .await @@ -610,6 +623,7 @@ mod tests { tag: None, scope: None, limit: None, + expand_links: false, }), ) .await @@ -646,6 +660,7 @@ mod tests { tag: None, scope: None, limit: Some(0), + expand_links: false, }), ) .await @@ -701,6 +716,7 @@ mod tests { tag: None, scope: None, limit: Some(-5), + expand_links: false, }), ) .await @@ -1180,6 +1196,87 @@ mod tests { } } + #[tokio::test] + async fn expand_links_from_raw_query_and_body_on_list_and_search() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let a = stored_id(&state, "pulsar effect").await; + let b = stored_id(&state, "pulsar cause").await; + let resp = link_raw( + &state, + &format!(r#"{{"from_id": {a}, "to_id": {b}, "kind": "caused_by"}}"#), + ) + .await; + assert_eq!(resp.status(), StatusCode::OK); + + let list_raw = |raw_query: &'static str| { + let state = state.clone(); + async move { + let uri: axum::http::Uri = format!("/api/memories{raw_query}") + .parse() + .expect("uri parses"); + let query = Query::::try_from_uri(&uri).expect("query parses"); + list_handler(State(state), query) + .await + .expect("list succeeds") + .into_response() + } + }; + + // GET /api/memories?expand_links=true + let rows = body_json(list_raw("?expand_links=true").await).await; + let rows = rows.as_array().expect("array"); + let effect = rows.iter().find(|r| r["id"] == a).expect("effect row"); + assert_eq!(effect["related"][0]["id"], b); + assert_eq!(effect["related"][0]["kind"], "caused_by"); + assert_eq!(effect["related"][0]["direction"], "out"); + let cause = rows.iter().find(|r| r["id"] == b).expect("cause row"); + assert_eq!(cause["related"][0]["kind"], "caused"); + assert_eq!(cause["related"][0]["direction"], "in"); + + // Absent or false: the v1.0 rows, no related member. + for q in ["", "?expand_links=false"] { + let rows = body_json(list_raw(q).await).await; + assert!( + rows.as_array() + .expect("array") + .iter() + .all(|r| r.get("related").is_none()), + "{q:?}" + ); + } + + // A non-boolean query value is rejected by the typed extractor. + let uri: axum::http::Uri = "/api/memories?expand_links=yes".parse().expect("uri"); + assert!(Query::::try_from_uri(&uri).is_err()); + + // POST /api/search with a raw JSON body. + let search_raw = |raw: &'static str| { + let state = state.clone(); + async move { + let req = axum::http::Request::builder() + .method("POST") + .uri("/api/search") + .header(axum::http::header::CONTENT_TYPE, "application/json") + .body(axum::body::Body::from(raw)) + .expect("request builds"); + let Json(body) = Json::::from_request(req, &()) + .await + .expect("body parses"); + search_handler(State(state), Json(body)) + .await + .expect("search succeeds") + .into_response() + } + }; + let body = + body_json(search_raw(r#"{"query": "effect", "expand_links": true}"#).await).await; + assert_eq!(body["search_mode"], "keyword"); + assert_eq!(body["results"][0]["related"][0]["id"], b); + let body = body_json(search_raw(r#"{"query": "effect"}"#).await).await; + assert!(body["results"][0].get("related").is_none(), "{body}"); + } + /// The real router over a loopback listener: route wiring, the /// `{from_id}/{to_id}/{kind}` path extractor and the JSON body all run /// exactly as they do in `serve-rest`. @@ -1213,6 +1310,23 @@ mod tests { .expect("POST /api/links"); assert_eq!(resp.status().as_u16(), 200); + let rows: serde_json::Value = client + .get(format!("{base}/api/memories?expand_links=true")) + .send() + .await + .expect("GET /api/memories?expand_links=true") + .json() + .await + .expect("json rows"); + let newer = rows + .as_array() + .expect("array") + .iter() + .find(|r| r["id"] == a) + .expect("row a"); + assert_eq!(newer["related"][0]["id"], b); + assert_eq!(newer["related"][0]["kind"], "supersedes"); + let resp = client .delete(format!("{base}/api/links/{a}/{b}/blocks")) .send() From a82878198070304fd79df425fa413df1affc0685 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:24:52 +0200 Subject: [PATCH 11/42] docs(04-03): complete relations plan (summary) Co-Authored-By: Claude Opus 5.5 --- .../04-03-SUMMARY.md | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-03-SUMMARY.md diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-03-SUMMARY.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-03-SUMMARY.md new file mode 100644 index 0000000..500ee9b --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-03-SUMMARY.md @@ -0,0 +1,319 @@ +--- +phase: 04-memory-update-relations-hybrid-search +plan: 03 +subsystem: api +tags: [rust, sqlite, relations, fts5, sqlite-vec, serde, schemars, rmcp, axum, mcp] + +# Dependency graph +requires: + - phase: 04-memory-update-relations-hybrid-search + provides: "04-01: migration 0003 memory_links (PK(from,to,kind), FK ON DELETE CASCADE, CHECK from!=to, idx_links_to), MemoryError::Internal for writer-lock sites, not-found as data. 04-02: UPDATE-in-place keeps ids immutable, so edges never re-attach" +provides: + - "LinkKind closed enum (relates_to | supersedes | caused_by) with as_wire_str / reverse_label / TryFrom -> InvalidArgument (D-01)" + - "LinkOutcome { Linked, NotFound { missing_id } }: a missing endpoint is data naming the id (D-02)" + - "Store::link / Store::unlink / Store::related + SqliteStore impls" + - "MemoryService::link (self-link rejected at the seam) / MemoryService::unlink" + - "Belt-and-braces orphan-edge deletes inside forget() and sweep_expired() (T-04-12)" + - "RelatedLink {id, kind, direction, type, snippet, tags} + LinkDirection + snippet() + RELATED_SNIPPET_CHARS = 160" + - "pub const MAX_EXPANDED_LINKS: usize = 10 (D-07)" + - "MemoryView.related: Option> skipped when None (byte-identical v1.0 envelope)" + - "expand_links: bool on core SearchArgs/ListArgs and on all four transport surfaces" + - "memory_link / memory_unlink MCP tools; POST /api/links + DELETE /api/links/{from_id}/{to_id}/{kind}" +affects: [04-04 hybrid RRF (expansion must stay orthogonal to the new hybrid search_mode), 05 export/import (JSONL format v1 must carry memory_links edges), 06 release notes] + +# Actuals (#2632): chars/4 over the added lines of the realized diff +actuals: + tokens: 20971 + tasks: 2 + commits: 4 + +# Tech tracking +tech-stack: + added: [] + patterns: + - "Closed-enum wire kinds: TryFrom<&str> returning InvalidArgument naming the allowed set; SQL binds only as_wire_str()" + - "Endpoint existence pre-checked inside the writer tx, so a missing id is returned as data, never an FK-violation 500" + - "Every deletion path carries an explicit orphan cleanup beside the FK cascade (the vec_memories idiom, now also for memory_links)" + - "Optional response members: Option + skip_serializing_if, locked by a golden byte-equality test of the old envelope" + - "Read-only enrichment (expansion) runs after truncation and after spawn_bump, so the ids it touches are never bumped" + +key-files: + created: + - crates/agent-memory-core/tests/relations.rs + modified: + - crates/agent-memory-core/src/domain.rs + - crates/agent-memory-core/src/service.rs + - crates/agent-memory-core/src/store/mod.rs + - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - crates/agent-memory/src/rest/mod.rs + - README.md + +key-decisions: + - "MAX_EXPANDED_LINKS = 10 (D-07 discretion); snippet = first 160 chars cut on a char boundary, with a trailing ellipsis only when cut (D-05 discretion)" + - "related lives directly on MemoryView as Option> with skip_serializing_if (research Open Question 2). Flag on: always Some, [] when there are no edges. Flag off: None, so the member is absent" + - "RelatedLink.direction is a LinkDirection enum serializing to out/in (same wire as the planned String, but typed); kind is the direction-aware natural label (superseded_by, caused, relates_to)" + - "The link not-found body echoes from_id/to_id/kind as well as {linked:false, reason:not_found, missing_id}, mirroring forget's id echo. from_id is checked first, so a link with both ids missing names from_id" + - "Unlink stays a thin wrapper. A self-pair (from==to) is a not-found (such an edge cannot exist), not an InvalidArgument" + - "POST /api/links maps every JSON body rejection to 400 {error}, like PATCH. A non-numeric path id on DELETE /api/links keeps axum's default plain-text 400, like DELETE /api/memories" + - "expand_links on REST search is a member of the existing POST /api/search JSON body (search has no query-param DTO). On REST list it is the ?expand_links= query param" + +patterns-established: + - "New relation kind: add a LinkKind variant + as_wire_str + reverse_label arms (exhaustive match forces both), extend the TryFrom message, the README kind table and the MCP tool description" + +requirements-completed: [MCP-07] + +coverage: + - id: D1 + description: "Link creates one directed edge; an identical relink is an idempotent Linked no-op (edge count stays 1); a different kind is a separate edge" + requirement: MCP-07 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#link_succeeds_and_a_duplicate_link_is_an_idempotent_no_op" + status: pass + - kind: integration + ref: "crates/agent-memory/src/mcp.rs#memory_link_and_unlink_round_trip_with_idempotent_duplicates" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#post_links_links_idempotently_and_delete_unlinks_then_404s" + status: pass + human_judgment: false + - id: D2 + description: "Self-link and unknown kind are InvalidArgument with the exact D-01/D-02 strings on core, MCP (invalid_params) and REST (400). A self-link on a nonexistent id is still the client error" + requirement: MCP-07 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#a_self_link_is_invalid_argument_naming_the_id / an_unknown_kind_is_invalid_argument_listing_the_allowed_values / every_kind_round_trips_its_wire_form" + status: pass + - kind: integration + ref: "crates/agent-memory/src/mcp.rs#memory_link_self_links_and_unknown_kinds_are_invalid_params" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#bad_link_requests_are_400_never_500" + status: pass + human_judgment: false + - id: D3 + description: "A missing endpoint is a clean not-found naming the missing id on all three layers (core LinkOutcome::NotFound, MCP success with reason not_found, REST 404); nothing is written; never an FK 500" + requirement: MCP-07 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#linking_to_a_missing_memory_is_a_clean_not_found_naming_the_id" + status: pass + - kind: integration + ref: "crates/agent-memory/src/mcp.rs#memory_link_to_a_missing_memory_is_a_clean_not_found_naming_it" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#post_links_to_a_missing_memory_is_404_naming_it_never_500" + status: pass + human_judgment: false + - id: D4 + description: "Unlink removes exactly the directed (from, to, kind) edge. A missing edge (or the wrong direction or kind) is a clean not-found on core, MCP and REST, including over real HTTP" + requirement: MCP-07 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#unlink_removes_an_edge_and_a_missing_edge_is_a_clean_not_found" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#links_routes_work_over_real_http" + status: pass + human_judgment: false + - id: D5 + description: "Zero orphan edges after forget AND after a TTL sweep (FK cascade). The explicit backstop deletes heal orphans left by a connection without the foreign_keys pragma" + requirement: MCP-07 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#forget_cascades_every_edge_touching_the_memory / a_ttl_sweep_cascades_the_expired_memorys_edges" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#forget_also_clears_orphans_left_by_a_connection_without_the_fk_pragma / sweep_also_clears_orphans_left_by_a_connection_without_the_fk_pragma" + status: pass + human_judgment: false + - id: D6 + description: "expand_links attaches 1-hop summaries in both directions from one edge row, with natural reverse labels (superseded_by/in, caused/in, relates_to symmetric), on the keyword and semantic search legs and on list" + requirement: MCP-07 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#expansion_follows_one_edge_both_ways_with_natural_labels / relates_to_reads_the_same_from_both_ends / expansion_also_applies_on_the_semantic_path" + status: pass + human_judgment: false + - id: D7 + description: "Flag-less search and list JSON is byte-identical to the golden v1.0 envelope (no related substring), even with edges present. Flag on with no edges gives related: []" + requirement: MCP-07 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#flag_less_envelopes_are_byte_identical_to_v1_and_never_mention_related / flag_on_without_edges_is_an_explicit_empty_related_list" + status: pass + human_judgment: false + - id: D8 + description: "Cap: MAX_EXPANDED_LINKS + 2 staggered edges (mixed directions) expand to exactly 10, newest first, with the two oldest dropped. Snippets are cut at 160 chars on a char boundary with an ellipsis" + requirement: MCP-07 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#expansion_is_capped_and_the_most_recent_edges_win" + status: pass + - kind: unit + ref: "crates/agent-memory-core/tests/relations.rs#snippets_cut_at_160_chars_on_a_char_boundary_with_an_ellipsis / expanded_snippets_are_truncated_for_long_neighbor_content" + status: pass + human_judgment: false + - id: D9 + description: "Expanded neighbors are never access-bumped (access_count and last_accessed unchanged), while the returned result is still bumped (the test waits for that bump to land)" + requirement: MCP-07 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#expanded_neighbors_are_never_access_bumped" + status: pass + human_judgment: false + - id: D10 + description: "expand_links is accepted from raw wire input on all four surfaces (MCP search/list args, REST search body, REST list query) as an optional boolean with default false; a non-boolean is rejected by the typed DTO" + requirement: MCP-07 + verification: + - kind: integration + ref: "crates/agent-memory/src/mcp.rs#expand_links_from_raw_args_attaches_related_on_search_and_list / expand_links_is_an_optional_boolean_in_both_tool_schemas" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#expand_links_from_raw_query_and_body_on_list_and_search / links_routes_work_over_real_http" + status: pass + - kind: manual_procedural + ref: "live stdio binary: tools/list shows 7 tools, link/unlink require [from_id,to_id,kind] with no defaults, expand_links boolean default false; link/dup/self/blocks/missing/list+expand/plain search/unlink behave as tested" + status: pass + human_judgment: false + - id: D11 + description: "README documents the relations tools, kind table with reverse labels, expansion shape, REST link routes and a bounds table including MAX_EXPANDED_LINKS = 10" + requirement: MCP-07 + verification: [] + human_judgment: true + rationale: "Documentation clarity and accuracy is a reading judgment; no test asserts README prose" + +# Metrics +duration: 10min +completed: 2026-10-23 +status: complete +--- + +# Phase 4 Plan 03: Relations (link / unlink / 1-hop expansion) Summary + +**Agents can now link memories with typed, directed relations: `memory_link` / `memory_unlink` over MCP, and `POST /api/links` / `DELETE /api/links/{from_id}/{to_id}/{kind}` over REST.** The vocabulary is closed to three kinds. A duplicate link is an idempotent no-op, a self-link is rejected, and a missing endpoint comes back as clean not-found data. Edges cascade on both forget and TTL sweep, with zero orphans either way. The opt-in `expand_links` flag attaches capped, newest-first 1-hop summaries to search and list results, using natural reverse labels and never access-bumping the neighbors. Without the flag, the envelope is byte-identical to v1.0. + +## Performance + +- **Duration:** ~10 min +- **Started:** 2026-10-23T18:13:31Z +- **Completed:** 2026-10-23T18:23:28Z +- **Tasks:** 2 / 2 +- **Files modified:** 9 (1 created, 8 modified) + +## Accomplishments + +- **Phase success criterion 2 holds.** Agents can link, unlink and expand, the cascade leaves no orphan edges, and there is no graph or entity model. Storage is one directed row per link. Expansion reads it from both ends, so an incoming `supersedes` reads `superseded_by` and an incoming `caused_by` reads `caused`. +- **D-01 / D-02 contracts are exact on all three layers:** + - Self-link: `invalid argument: from_id and to_id must differ (got N)`. + - Unknown kind: `invalid argument: kind must be one of relates_to, supersedes, caused_by (got 'blocks')`. + - Missing endpoint: `{..., "linked": false, "reason": "not_found", "missing_id": N}`, as a successful MCP call or a REST 404. Never a 500. +- **No orphan edges.** The FK `ON DELETE CASCADE` is the primary mechanism. `forget()` and `sweep_expired()` each also run `DELETE FROM memory_links WHERE from_id NOT IN (…) OR to_id NOT IN (…)` in the same transaction. A hand mutation that turned those deletes into no-ops failed exactly the two pragma-less-connection tests, which shows the backstop is real coverage and not decoration. +- **Expansion (D-03..D-07).** `Store::related` runs one read-pool query per anchor. It follows edges in both directions, newest first, with `LIMIT MAX_EXPANDED_LINKS` (10). The service attaches the result after truncation and after `spawn_bump(returned ids)`, on both search legs and on list. Hand mutations confirmed two tests bite: flipping the ORDER BY failed the cap test, and dropping `skip_serializing_if` failed the golden byte-compat test. +- **Live binary check.** The stdio server lists seven tools. The link/unlink schemas require `[from_id, to_id, kind]` with no `default`, and `expand_links` is a boolean with `default: false`. Every behaviour matches the tests, including a plain semantic search with no `related` member. + +## Task Commits + +1. **Task 1: memory_link / memory_unlink end-to-end with cascade**: `93ae867` (feat) +2. **Task 2: 1-hop expansion via expand_links on search and list**: `d636328` (feat) + +**Plan metadata:** this SUMMARY commit, then a `docs(04-03)` state/roadmap commit. + +_TDD note: RED was observed before implementation for both tasks. For Task 1, `tests/relations.rs` failed to compile with 42 errors for the missing `LinkKind`, `LinkOutcome`, `link` and `unlink`. For Task 2, it failed with 8 errors for the missing `RelatedLink`, `expand_links`, `related` and `MAX_EXPANDED_LINKS`. RED was not committed separately, because CLAUDE.md and the operator rules forbid committing failing tests (same as 04-01 and 04-02)._ + +## Files Created/Modified + +- `crates/agent-memory-core/tests/relations.rs` (new, 737 lines, 19 tests): link/idempotency, self-link, unknown kind, wire round-trip, missing endpoint, unlink, cascade on forget and sweep, pragma-less orphan healing (forget and sweep), both-direction labels, relates_to symmetry, golden byte-compat, empty-on, cap with newest-first ordering, snippet rules, no neighbor bumps, semantic-path expansion +- `crates/agent-memory-core/src/domain.rs`: `LinkKind`, `LinkOutcome`, `LinkDirection`, `RelatedLink`, `snippet()`, `RELATED_SNIPPET_CHARS`, `MAX_EXPANDED_LINKS`, `validate_link_endpoints`, `MemoryView.related`, and a wire-order invariant note +- `crates/agent-memory-core/src/store/mod.rs`: `link`, `unlink`, `related` trait methods; forget/sweep docs name the edge cascade +- `crates/agent-memory-core/src/store/sqlite.rs`: the `link` (pre-check plus `INSERT OR IGNORE`), `unlink` and `related` impls, the two orphan-edge deletes, `corrupt_column`, and `related: None` in `row_to_view` +- `crates/agent-memory-core/src/service.rs`: `link`, `unlink`, `expand_related`, and `expand_links` on `SearchArgs` / `ListArgs` +- `crates/agent-memory/src/mcp.rs`: `LinkArgs`/`UnlinkArgs`, the `memory_link`/`memory_unlink` tools, `expand_links` on `SearchArgs`/`ListArgs`, updated server instructions, and 6 new tests +- `crates/agent-memory/src/rest/handlers.rs`: `LinkBody`, `link_handler`, `unlink_handler`, `expand_links` on `SearchRequest`/`ListQuery`, and 5 new tests (including a real-HTTP router test) +- `crates/agent-memory/src/rest/mod.rs`: the two `/api/links` routes +- `README.md`: the "seven tools" table, the Relations section (kind table, expansion shape, cascade note), a bounds table with `MAX_EXPANDED_LINKS = 10`, the REST rows, and curl examples + +## Decisions Made + +See `key-decisions` in the frontmatter. In short: +- The cap is 10 and the snippet is 160 chars. +- `related` sits on `MemoryView`: `Some` when the flag is on, skipped when it is off. +- `direction` is a typed enum on the same wire. +- The link not-found body echoes the request fields. +- Unlink of a self-pair is a not-found. +- Link body rejections are 400s. +- On REST search, `expand_links` lives in the JSON body. + +## Deviations from Plan + +### Auto-fixed Issues + +**1. [Rule 2 - Missing critical] Pragma-less orphan tests for the belt-and-braces deletes** +- **Found during:** Task 1 +- **Issue:** With `foreign_keys = ON`, the FK cascade removes every edge on its own. The planned cascade tests would therefore stay green even if the explicit deletes required by T-04-12 were missing. +- **Fix:** Two tests orphan an edge through a raw connection with `foreign_keys = OFF`, then assert that an unrelated `forget()`, or a sweep with nothing to expire, heals it. Mutation-verified: they fail when the deletes are no-ops. +- **Files modified:** crates/agent-memory-core/tests/relations.rs +- **Commit:** 93ae867 + +**2. [Rule 2 - Missing critical] Link body rejections are 400, not axum's default status** +- **Found during:** Task 1 +- **Issue:** The plan said to copy `forget_handler`, which has no body. With a plain `Json` extractor, a missing field or wrong type would return axum's 422 and a missing content-type a 415, not the project's `{"error"}` 400 contract. +- **Fix:** `link_handler` takes `Result, JsonRejection>` and maps it to `ApiError::BadRequest`, the same as PATCH. +- **Files modified:** crates/agent-memory/src/rest/handlers.rs +- **Commit:** 93ae867 + +**3. [Rule 2 - Missing critical] Tests beyond the listed behaviours** +- **What was added:** a real-HTTP in-process router test (path extractor, route wiring, the `?expand_links=true` query); a `relates_to` symmetry test; a semantic-path expansion test; a flag-on-empty test; schema tests for `LinkArgs`/`UnlinkArgs` (all three fields required, no `default`) and for `expand_links` (boolean, default false); a test that a self-link on a nonexistent id is still `InvalidArgument`. +- **Commits:** 93ae867, d636328 + +**4. [Plan wording] `expand_links` on REST search is a body member** +- **Issue:** The plan says "REST search/list query-param DTOs". REST search is `POST /api/search` with a JSON body; it has no query-param DTO. +- **Fix:** `expand_links` was added to `SearchRequest` (the body) and to `ListQuery` (the `GET /api/memories` query string). Both have raw-input tests. +- **Commit:** d636328 + +**5. [Design detail] `RelatedLink.direction` is a `LinkDirection` enum rather than a `String`** +- The JSON is identical (`"out"` / `"in"`), but the type is closed, in line with the project's "pattern match exhaustively" rule. The not-found link body also echoes `from_id`/`to_id`/`kind`, which is a superset of the planned shape. +- **Commit:** d636328 / 93ae867 + +--- + +**Total deviations:** 3 auto-fixed (missing-critical), plus 2 wording/design clarifications. +**Impact on plan:** No scope creep, no new crates, no manifest or workflow changes. + +## Issues Encountered + +- The plan's literal gate expects `DELETE FROM memory_links WHERE from_id NOT IN` to appear 2 times in `sqlite.rs`. A shared helper would have made it 1, so the delete is written inline in both `forget()` and `sweep_expired()`. +- One clippy run reported `Finished` in 0.15s from cache. It was re-run with both crates touched and re-linted clean. + +## Verification Results + +- `cargo test -p agent-memory-core --test relations`: **19 passed** +- `cargo test -p agent-memory --bins`: 48 passed (37 before this plan) +- `cargo test --workspace`: **146 passed / 0 failed / 2 ignored** (04-02 baseline: 116 / 0 / 2) +- `cargo clippy --workspace --all-targets -- -D warnings`: clean (forced re-lint) +- `cargo fmt --all --check`: clean +- Acceptance greps: `INSERT OR IGNORE INTO memory_links` appears 1 time; `DELETE FROM memory_links WHERE from_id NOT IN` appears 2 times; `map_err(|_| MemoryError::NotFound` appears 0 times; `pub const MAX_EXPANDED_LINKS: usize = 10` is present; `MAX_EXPANDED_LINKS` is in the README; `skip_serializing_if = "Option::is_none"` is on `related`. +- Hand mutations, all reverted: no-op orphan deletes failed the 2 pragma-less tests; ORDER BY ASC failed the cap test; removing the skip failed the golden test. +- Live stdio binary: checked as described under D10. + +## User Setup Required + +None. + +## Next Phase Readiness + +- **Plan 04-04 (hybrid RRF):** expansion is orthogonal to `search_mode`. The hybrid leg must keep the same order — truncate, `spawn_bump(returned ids)`, then `expand_related` when `expand_links` is set — and must thread `args.expand_links` through before `args` is moved into the store legs. +- **Phase 5 (export/import):** edges live in `memory_links (from_id, to_id, kind, created_at)`. Format v1 must carry them, and import must remap ids, because ids are local rowids. +- **Still open, outside this plan:** WR-01 (full filter-aware KNN) and WR-03 (the spawned-binary REST harness hang). No link test was added to `tests/rest.rs`. The in-process real-HTTP router test covers the HTTP surface instead. + +--- +*Phase: 04-memory-update-relations-hybrid-search* +*Completed: 2026-10-23* + +## Self-Check: PASSED + +- FOUND (tracked): crates/agent-memory-core/tests/relations.rs (737 lines, ≥ 120) +- FOUND: commit 93ae867 (Task 1) +- FOUND: commit d636328 (Task 2) From 8903ee6614f8eec0f38037815b4ae973b215c901 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:25:16 +0200 Subject: [PATCH 12/42] docs(04-03): update state, roadmap progress, mark MCP-07 complete Co-Authored-By: Claude Opus 5.5 --- .planning/REQUIREMENTS.md | 4 ++-- .planning/ROADMAP.md | 6 +++--- .planning/STATE.md | 19 ++++++++++++------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md index c39900b..056bf31 100644 --- a/.planning/REQUIREMENTS.md +++ b/.planning/REQUIREMENTS.md @@ -16,7 +16,7 @@ Each maps to a roadmap phase. IDs continue from the v1.0 numbering (archived in ### MCP Interface (MCP) - [x] **MCP-06**: Agent can call `memory_update` to patch an existing memory (content, tags, scope, TTL) by id; a content update re-mirrors FTS and invalidates/re-embeds the vector **in the same writer transaction** (embed-outage falls back to `embedding_status = 0` + existing sweep backfill); unknown id returns the clean not-found shape -- [ ] **MCP-07**: Agent can link two memories with a flat typed relation (`memory_link` / `memory_unlink`: from_id, to_id, kind) and search/list results can expand 1-hop related memories; forgetting a memory cascades its links (no orphan edges, no graph/entity model) +- [x] **MCP-07**: Agent can link two memories with a flat typed relation (`memory_link` / `memory_unlink`: from_id, to_id, kind) and search/list results can expand 1-hop related memories; forgetting a memory cascades its links (no orphan edges, no graph/entity model) ### REST & Input Hardening (API) @@ -65,7 +65,7 @@ Mapped by roadmap creation 2026-07-12. Coverage: 12/12 v1.1 requirements mapped | API-02 | Phase 3 | Complete | | API-03 | Phase 3 | Complete | | MCP-06 | Phase 4 | Complete | -| MCP-07 | Phase 4 | Pending | +| MCP-07 | Phase 4 | Complete | | STORE-05 | Phase 4 | Complete | | SEARCH-04 | Phase 4 | Pending | | SEARCH-05 | Phase 4 | Pending | diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 72c51eb..7b3e802 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -67,7 +67,7 @@ Plans: 4. Hybrid ranking stays decay-aware — a fresh memory outranks a stale one at equal fused rank — and access bumps apply only to post-truncation returned ids. 5. Upgrading a real v0.0.1 database applies migration 0003 only after a pre-migration backup; shipped 0001/0002 SQL is provably unchanged (fixture divergence test against a v0.0.1 database), and a newer-schema DB met by an older binary yields a friendly error, not a crash. -**Plans**: 2/4 plans executed +**Plans**: 3/4 plans executed Plans: **Wave 1** @@ -79,7 +79,7 @@ Plans: **Wave 3** *(blocked on Wave 2 completion)* -- [ ] 04-03-PLAN.md — memory_link/memory_unlink with cascade + opt-in 1-hop expansion on search/list (MCP-07) +- [x] 04-03-PLAN.md — memory_link/memory_unlink with cascade + opt-in 1-hop expansion on search/list (MCP-07) **Wave 4** *(blocked on Wave 3 completion)* @@ -124,6 +124,6 @@ Phases execute in numeric order: 3 → 4 → 5 → 6 | 1. Core Memory Foundation | v1.0 | 3/3 | Complete | 2026-06-25 | | 2. Semantic Search, Interop & Release | v1.0 | 5/5 | Complete | 2026-07-12 | | 3. API Hardening & Toolchain Spikes | v1.1 | 2/2 | Complete | 2026-07-12 | -| 4. Memory Update, Relations & Hybrid Search | v1.1 | 2/4 | In Progress| | +| 4. Memory Update, Relations & Hybrid Search | v1.1 | 3/4 | In Progress| | | 5. Portable Export & Import | v1.1 | 0/? | Not started | - | | 6. Distribution & v0.1.0 Release | v1.1 | 0/? | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index d8b908b..41edc2a 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -5,16 +5,16 @@ milestone_name: Hardening & Interop current_phase: 04 current_phase_name: Memory Update, Relations & Hybrid Search status: executing -stopped_at: Completed 04-02-PLAN.md -last_updated: "2026-10-23T18:11:23.164Z" +stopped_at: Completed 04-03-PLAN.md +last_updated: "2026-10-23T18:25:09.406Z" last_activity: 2026-10-23 last_activity_desc: Phase 04 execution started -state_head: a49b6c6f5763bae94df3c1d219398cd4989f4455 +state_head: a82878198070304fd79df425fa413df1affc0685 progress: total_phases: 4 completed_phases: 1 total_plans: 6 - completed_plans: 4 + completed_plans: 5 --- # Project State @@ -29,7 +29,7 @@ See: .planning/PROJECT.md (updated 2026-07-12) ## Current Position Phase: 04 (Memory Update, Relations & Hybrid Search) — EXECUTING -Plan: 3 of 4 +Plan: 4 of 4 Status: Ready to execute Last activity: 2026-10-23 — Phase 04 execution started @@ -70,6 +70,7 @@ Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) |------|----------|-------|-------| | Phase 04 P01 | 9 min | 2 tasks | 13 files | | Phase 04 P02 | 10min | 2 tasks | 11 files | +| Phase 04 P03 | 10 min | 2 tasks | 9 files | ## Accumulated Context @@ -103,6 +104,10 @@ Recent decisions affecting current work: - [Phase 04]: 04-02: update is UPDATE-in-place (never delete+insert), folds the D-09 freshness bump and decay_score = 1.0 into the same statement, and swaps the vector in the same writer tx - [Phase 04]: 04-02: patch DTO fields carry skip_serializing_if so the MCP tool schema never advertises default: null (null means clear in a merge-patch) - [Phase 04]: 04-02: PATCH /api/memories/{id} maps every Json extractor rejection to 400 {error}; other REST handlers keep axum defaults +- [Phase 04]: 04-03: Relations are flat directed edges (LinkKind relates_to|supersedes|caused_by, closed enum); duplicate link = idempotent Linked, self-link = InvalidArgument at the seam, missing endpoint = LinkOutcome::NotFound{missing_id} data (never an FK 500) +- [Phase 04]: 04-03: forget() and sweep_expired() each carry an explicit orphan-edge DELETE beside the FK ON DELETE CASCADE, proven by pragma-less-connection tests +- [Phase 04]: 04-03: expand_links attaches MemoryView.related (Option, skip_serializing_if) after truncation and after spawn_bump; MAX_EXPANDED_LINKS = 10 newest-first, snippet 160 chars; neighbors never bumped; flag-less envelope golden-tested byte-identical +- [Phase 04]: 04-03: Incoming edges read natural reverse labels (superseded_by, caused); RelatedLink.direction is out|in; REST POST /api/links + DELETE /api/links/{from_id}/{to_id}/{kind} with 200/404 and 400 on any bad body/kind ### Pending Todos @@ -143,8 +148,8 @@ Items acknowledged and carried forward: ## Session Continuity -Last session: 2026-10-23T18:11:16.057Z -Stopped at: Completed 04-02-PLAN.md +Last session: 2026-10-23T18:25:02.213Z +Stopped at: Completed 04-03-PLAN.md Resume file: None ## Operator Next Steps From 024bcd463283e2b8d53dc0914e18c4949cbac56a Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:33:47 +0200 Subject: [PATCH 13/42] feat(04-04): RRF hybrid search over two pure-relevance legs with post-fusion decay (SEARCH-04/05) - Store::keyword_candidates: bm25-ordered keyword leg (ORDER BY bm25 ASC, id ASC), no decay term, oversampled limit*4 capped at MAX_KNN_K, same filters and map_fts_query_error routing as search; public Store::search untouched - pub const RRF_K = 60.0 and pub fn rrf_fuse: 1-based ranks, duplicate ids sum, total_cmp descending with ascending-id tie-break - Embed-OK arm runs knn_search + keyword_candidates concurrently (tokio::join!), propagates leg errors (keyword first), fuses, multiplies by pinned-aware decay once, truncates, bumps only returned ids, then expands - SearchMode::Hybrid replaces the now-unproduced SearchMode::Semantic - tests/hybrid.rs: hand-computed RRF, e2e fusion, both-leg oversample recall, SEARCH-05 golden (fresh beats stale at equal rank; pinned, once, multiplicative) Co-Authored-By: Claude Opus 5.5 --- crates/agent-memory-core/src/service.rs | 187 ++++++-- crates/agent-memory-core/src/store/mod.rs | 16 + crates/agent-memory-core/src/store/sqlite.rs | 43 ++ crates/agent-memory-core/tests/hybrid.rs | 453 +++++++++++++++++++ crates/agent-memory-core/tests/relations.rs | 2 +- crates/agent-memory-core/tests/semantic.rs | 10 +- crates/agent-memory-core/tests/update.rs | 6 +- 7 files changed, 659 insertions(+), 58 deletions(-) create mode 100644 crates/agent-memory-core/tests/hybrid.rs diff --git a/crates/agent-memory-core/src/service.rs b/crates/agent-memory-core/src/service.rs index 13bedb1..7c8b307 100644 --- a/crates/agent-memory-core/src/service.rs +++ b/crates/agent-memory-core/src/service.rs @@ -10,8 +10,9 @@ //! **The SEARCH-03 degrade seam lives HERE**, never in transports: an embed //! failure downgrades `store` to a pending row and routes `search` to the FTS5 //! keyword path — and ONLY an embed failure triggers the fallback (an empty -//! semantic result set is a valid `Semantic` answer). +//! hybrid result set is a valid `Hybrid` answer). +use std::collections::HashMap; use std::sync::Arc; use crate::clock::Clock; @@ -47,30 +48,70 @@ pub struct SearchArgs { pub scope: Option, pub limit: Option, /// Attach 1-hop relation summaries to every returned row (MCP-07 / D-04). - /// Orthogonal to the search mode: applies on the semantic and the keyword + /// Orthogonal to the search mode: applies on the hybrid and the keyword /// path alike. `false` (the default) keeps the v1.0 envelope byte-identical. pub expand_links: bool, } /// Which search path produced a result set. Serialized lowercase — the shared /// wire form for MCP and REST (RESEARCH Open Question 3). +/// +/// There is no `semantic` value any more: with the embedder up, every search +/// is fused (SEARCH-04), so a semantic-only variant would have no producer. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] #[serde(rename_all = "lowercase")] pub enum SearchMode { - /// Ranked by Ollama embedding similarity blended with decay (SEARCH-02). - Semantic, + /// The embedder was available: the KNN leg and the bm25 keyword leg were + /// fused with Reciprocal Rank Fusion, then decay-blended (SEARCH-04/05). + Hybrid, /// FTS5 keyword fallback — the embedder was unavailable (SEARCH-03). Keyword, } +/// Reciprocal Rank Fusion smoothing constant: the `k` in `1 / (k + rank)`, +/// the Cormack & Clarke convention (SEARCH-04). +/// +/// This is NOT the KNN oversample size or its cap (`MAX_KNN_K` in the SQLite +/// store, 800). The two are different quantities that happen to both be +/// called `k` in the literature, so neither is ever passed around as a bare +/// `k` (RESEARCH Pitfall 2). Changing this reshapes every hybrid ranking. +pub const RRF_K: f64 = 60.0; + +/// Fuse ranked id lists with Reciprocal Rank Fusion (SEARCH-04). +/// +/// Each leg is a list of ids, best first. An id at 1-based rank `r` in a leg +/// contributes `1 / (RRF_K + r)`, and an id that appears in several legs SUMS +/// its contributions, so agreement between legs is rewarded: rank 2 in both +/// legs (`2/62`) beats rank 1 in one leg (`1/61`). Only ranks matter, never +/// the legs' raw scores, which is why bm25 values and cosine distances never +/// need to be put on a common scale. +/// +/// Output is sorted by fused score, descending (`total_cmp`), with ties +/// broken by ascending id, so the result is deterministic regardless of hash +/// order. Decay is NOT applied here: the caller blends it in once, after +/// fusion (SEARCH-05 / RESEARCH Pitfall 1), so the legs must be pure +/// relevance orderings. +pub fn rrf_fuse(legs: &[Vec]) -> Vec<(i64, f64)> { + let mut scores: HashMap = HashMap::new(); + for leg in legs { + for (idx, id) in leg.iter().enumerate() { + let rank = idx as f64 + 1.0; + *scores.entry(*id).or_insert(0.0) += 1.0 / (RRF_K + rank); + } + } + let mut fused: Vec<(i64, f64)> = scores.into_iter().collect(); + fused.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0))); + fused +} + /// The ONE shared search envelope every transport serializes (MCP now, REST in /// plan 02-02): a top-level `search_mode` plus the ranked results. #[derive(Debug, Clone, serde::Serialize)] pub struct SearchOutcome { /// Which path produced `results` — surfaces degraded state to agents/users. pub search_mode: SearchMode, - /// Ranked result rows (may be empty; an empty semantic set is still - /// `Semantic` — only an embed failure switches the mode). + /// Ranked result rows (may be empty; an empty hybrid set is still + /// `Hybrid` — only an embed failure switches the mode). pub results: Vec, } @@ -166,22 +207,28 @@ impl MemoryService { Ok(views) } - /// Search the corpus (MCP-02 / SEARCH-02 / SEARCH-03 / D-04). + /// Search the corpus (MCP-02 / SEARCH-03 / SEARCH-04 / SEARCH-05 / D-04). /// - /// Semantic-first with graceful keyword fallback — **the degrade decision + /// Hybrid-first with graceful keyword fallback — **the degrade decision /// lives HERE at the embedder seam, never in transports**: /// - /// - Embed OK → vec0 KNN candidates (`spawn_blocking`), then in Rust blend - /// `relevance × (1 − cosine_distance) + decay × decay_score` over the - /// oversampled set, truncate to the limit, and return - /// [`SearchMode::Semantic`]. An empty candidate set is a valid semantic - /// answer (Pitfall 2: emptiness never means "fall back"). + /// - Embed OK → [`Self::hybrid_candidates`]: the KNN leg and the bm25 + /// keyword leg run concurrently, both oversampled and both in pure + /// relevance order. They are fused with [`rrf_fuse`], each fused score is + /// multiplied by the pinned-aware decay score once, and the result is + /// truncated to the limit and returned as [`SearchMode::Hybrid`]. An + /// empty result is a valid hybrid answer (Pitfall 2: emptiness never + /// means "fall back"). A leg error is never swallowed: a malformed + /// MATCH is [`MemoryError::InvalidQuery`] here exactly as in keyword + /// mode, so hybrid never silently serves semantic-only results (T-04-15). /// - Embed Err → one loud warning, then the UNCHANGED Phase-1 FTS5 /// bm25×decay path, returned as [`SearchMode::Keyword`]. A malformed FTS5 /// query error on this path maps exactly as before. Never an error and /// never empty-because-of-outage (SEARCH-03). /// - /// Both paths fire the fire-and-forget recency bump on returned ids. + /// Both paths fire the fire-and-forget recency bump on the returned ids + /// only — never on fusion candidates or on expanded neighbors — and then + /// attach the 1-hop expansion when `expand_links` is set. pub async fn search(&self, args: SearchArgs) -> Result { // API-02 / D-01: validate before any embed hop — an out-of-bounds // limit rejects with MemoryError::InvalidArgument; the old silent @@ -197,40 +244,14 @@ impl MemoryService { match self.embedder.embed(&query_input).await { Ok(mut vectors) if !vectors.is_empty() => { let query_vec = vectors.swap_remove(0); - let store = self.store.clone(); - let knn_args = args; - let candidates = - tokio::task::spawn_blocking(move || store.knn_search(query_vec, knn_args)) - .await - .map_err(MemoryError::Join)??; - - // Similarity×decay blend in Rust over the small candidate set: - // cosine distance 0 = identical, so (1 - distance) grows with a - // better match — additive with the decay term like the FTS5 - // blend (decay recomputed from last_accessed at `now`, STORE-03). - let mut scored: Vec<(MemoryView, f64)> = candidates - .into_iter() - .map(|(mut view, distance)| { - let d = decay_score( - now, - view.last_accessed, - cfg.half_life_secs, - view.mem_type.is_pinned(), - ); - view.decay_score = d; - let blended = weights.relevance * (1.0 - distance) + weights.decay * d; - (view, blended) - }) - .collect(); - scored.sort_by(|a, b| b.1.total_cmp(&a.1)); - let results: Vec = scored - .into_iter() - .take(limit) - .map(|(view, _)| view) - .collect(); - - // Bump ONLY the returned ids, before expansion runs — expanded - // neighbors are never passed here (D-06). + // `args` moves into the legs here; `limit` and `expand_links` + // were captured above. + let fused = self.hybrid_candidates(query_vec, args, now).await?; + let results: Vec = fused.into_iter().take(limit).collect(); + + // Bump ONLY the returned ids — after truncation, so fusion + // candidates are never bumped (Pitfall 3 / T-04-17), and before + // expansion, so expanded neighbors are never bumped (D-06). self.spawn_bump(results.iter().map(|v| v.id).collect(), now); let results = if expand_links { self.expand_related(results).await? @@ -238,7 +259,7 @@ impl MemoryService { results }; Ok(SearchOutcome { - search_mode: SearchMode::Semantic, + search_mode: SearchMode::Hybrid, results, }) } @@ -366,6 +387,74 @@ impl MemoryService { }) } + /// The hybrid ranking (SEARCH-04/05), before truncation: every fusion + /// candidate, best first. + /// + /// 1. Both legs run concurrently on blocking threads: [`Store::knn_search`] + /// (cosine-distance order) and [`Store::keyword_candidates`] (bm25 + /// order). Both are pure relevance and both oversample `limit * 4`. + /// 2. Errors propagate, the keyword leg's first: the KNN leg never parses + /// the query, so checking the keyword leg first means a malformed MATCH + /// is always [`MemoryError::InvalidQuery`] (REST 400 / MCP + /// invalid_params), the same classification as keyword mode (T-04-15). + /// 3. The two id lists are fused with [`rrf_fuse`]. + /// 4. Decay is applied exactly ONCE, post-fusion and multiplicatively: + /// `final = rrf_score × decay_score(now, last_accessed, half_life, + /// pinned)`. RRF scores are tiny (at most 2/61 ≈ 0.033) while decay is + /// in (0, 1], so an additive term would drown relevance. Multiplying + /// keeps "equal fused rank → the fresher memory wins" exact, and the + /// pinned half-life keeps its advantage (Pitfall 1). Each view carries + /// that same decay value on the wire. + /// 5. Sorted by the final score descending, ties by ascending id. + /// + /// Fusion is O(candidates) in memory over lists bounded by the KNN cap + /// (T-04-16). Nothing here bumps access stats. + async fn hybrid_candidates( + &self, + query_vec: Vec, + args: SearchArgs, + now: i64, + ) -> Result, MemoryError> { + let cfg = self.decay_cfg; + let knn_store = self.store.clone(); + let knn_args = args.clone(); + let keyword_store = self.store.clone(); + let keyword_args = args; + let (knn, keyword) = tokio::join!( + tokio::task::spawn_blocking(move || knn_store.knn_search(query_vec, knn_args)), + tokio::task::spawn_blocking(move || keyword_store.keyword_candidates(keyword_args)), + ); + let keyword = keyword.map_err(MemoryError::Join)??; + let knn = knn.map_err(MemoryError::Join)??; + + let knn_ids: Vec = knn.iter().map(|(view, _)| view.id).collect(); + let keyword_ids: Vec = keyword.iter().map(|view| view.id).collect(); + let mut views: HashMap = HashMap::new(); + for view in knn.into_iter().map(|(view, _)| view).chain(keyword) { + views.entry(view.id).or_insert(view); + } + + let mut scored: Vec<(MemoryView, f64)> = rrf_fuse(&[knn_ids, keyword_ids]) + .into_iter() + .filter_map(|(id, rrf_score)| { + // Every fused id came from one of the two legs, so the lookup + // always hits; filter_map just avoids an unwrap. + views.remove(&id).map(|mut view| { + let d = decay_score( + now, + view.last_accessed, + cfg.half_life_secs, + view.mem_type.is_pinned(), + ); + view.decay_score = d; + (view, rrf_score * d) + }) + }) + .collect(); + scored.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.id.cmp(&b.0.id))); + Ok(scored.into_iter().map(|(view, _)| view).collect()) + } + /// Attach 1-hop relation summaries to already-final result rows (MCP-07): /// one `spawn_blocking` hop that reads each row's neighbors via /// [`Store::related`], capped at [`MAX_EXPANDED_LINKS`] most-recent edges diff --git a/crates/agent-memory-core/src/store/mod.rs b/crates/agent-memory-core/src/store/mod.rs index 5c1f619..89828f2 100644 --- a/crates/agent-memory-core/src/store/mod.rs +++ b/crates/agent-memory-core/src/store/mod.rs @@ -80,6 +80,22 @@ pub trait Store: Send + Sync { cfg: DecayConfig, ) -> Result, MemoryError>; + /// The keyword leg of hybrid search (SEARCH-04): FTS5 candidates in + /// **pure relevance** order (bm25 best first, ties by ascending id), with + /// no decay term. The service applies decay once, after fusion (SEARCH-05), + /// so a decay-blended leg here would count recency twice. + /// + /// Oversampled exactly like [`Store::knn_search`]: `limit * 4` rows (the + /// limit defaults to 50), capped at the same KNN cap, so a hit ranked just + /// past the limit still reaches fusion. Same type/scope/tag filters as + /// [`Store::search`] (`None` disables). A malformed MATCH string is + /// [`MemoryError::InvalidQuery`], exactly as on [`Store::search`]. A valid + /// query that matches nothing is an empty vec. + /// + /// [`Store::search`] itself is unchanged: it is the keyword-only degrade + /// path used when the embedder is down. + fn keyword_candidates(&self, args: SearchArgs) -> Result, MemoryError>; + /// Delete a memory by id. Returns `true` if a row was deleted, `false` if no /// row with that id existed (clean not-found, never an error — MCP-04). The /// FTS5 mirror is kept in sync by the delete trigger; the `vec_memories` diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index 9afa89c..1d43277 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -603,6 +603,49 @@ impl Store for SqliteStore { Ok(out) } + fn keyword_candidates(&self, args: SearchArgs) -> Result, MemoryError> { + let conn = self.reads.get().map_err(MemoryError::Pool)?; + let mem_type_filter: Option = args.mem_type.map(|t| t.as_wire_str().to_string()); + // Same oversample as knn_search, so the two fused legs are symmetric + // (SEARCH-04 / RESEARCH Pitfall 3). + let fetch = (args.limit.unwrap_or(DEFAULT_SEARCH_LIMIT).max(1) * 4).min(MAX_KNN_K); + + // Pure relevance (RESEARCH Pitfall 1): bm25() is SMALLER = better, so + // ascending order is best first, with no negation and no decay term. + // Decay is applied once by the service, after fusion. The id tie-break + // keeps the leg deterministic. FROM/JOIN, filters and the + // map_fts_query_error routing are copied from `search`, and every + // value is bound, never concatenated (T-04-18). + let mut stmt = conn.prepare( + "SELECT m.id, m.mem_type, m.content, m.tags, m.source, m.scope, \ + m.base_weight, m.decay_score, m.access_count, m.created_at, \ + m.last_accessed, m.expires_at \ + FROM memories_fts \ + JOIN memories m ON m.id = memories_fts.rowid \ + WHERE memories_fts MATCH ?1 \ + AND (?2 IS NULL OR m.mem_type = ?2) \ + AND (?3 IS NULL OR m.scope = ?3) \ + AND (?4 IS NULL OR EXISTS (SELECT 1 FROM json_each(m.tags) \ + WHERE json_each.value = ?4)) \ + ORDER BY bm25(memories_fts) ASC, m.id ASC \ + LIMIT ?5", + )?; + + let rows = stmt.query_map( + params![args.query, mem_type_filter, args.scope, args.tag, fetch], + row_to_view, + ); + + // A malformed MATCH surfaces at step time, as on `search`: it is + // client input (InvalidQuery), never an internal error (T-04-15). + let rows = rows.map_err(|e| map_fts_query_error(&args.query, e))?; + let mut out = Vec::new(); + for row in rows { + out.push(row.map_err(|e| map_fts_query_error(&args.query, e))?); + } + Ok(out) + } + fn forget(&self, id: i64) -> Result { let mut conn = self .writer diff --git a/crates/agent-memory-core/tests/hybrid.rs b/crates/agent-memory-core/tests/hybrid.rs new file mode 100644 index 0000000..f8d21ca --- /dev/null +++ b/crates/agent-memory-core/tests/hybrid.rs @@ -0,0 +1,453 @@ +//! SEARCH-04/SEARCH-05: decay-aware hybrid search. +//! +//! With the embedder up, `MemoryService::search` runs two pure-relevance legs +//! (the distance-ordered KNN leg and the bm25-ordered `keyword_candidates` +//! leg), fuses them with Reciprocal Rank Fusion (k = 60, 1-based ranks, +//! duplicate ids sum their contributions), then multiplies each fused score by +//! the pinned-aware decay score exactly once (post-fusion, SEARCH-05). The +//! envelope reports `search_mode: "hybrid"`. +//! +//! RRF only looks at ranks, never raw distances or bm25 values, so every +//! fixture here controls ranks directly: query vector = axis 0, each row's +//! vector has a chosen cosine similarity to it, and bm25 order is set by term +//! frequency and document length. Real on-disk temp DBs and an injected +//! `TestClock`, as in tests/semantic.rs. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use agent_memory_core::clock::{Clock, TestClock}; +use agent_memory_core::decay::{decay_score, DecayConfig, DEFAULT_HALF_LIFE_SECS}; +use agent_memory_core::domain::{MemoryType, NewMemory}; +use agent_memory_core::embed::{Embedder, FakeEmbedder, EMBEDDING_DIM}; +use agent_memory_core::service::{rrf_fuse, MemoryService, SearchArgs, SearchMode, RRF_K}; +use agent_memory_core::store::sqlite::SqliteStore; +use agent_memory_core::store::Store; + +// --------------------------------------------------------------------------- +// Harness +// --------------------------------------------------------------------------- + +struct Harness { + _dir: tempfile::TempDir, + db_path: PathBuf, + clock: Arc, + store: Arc, + service: MemoryService, +} + +fn harness(embedder: Arc) -> Harness { + let dir = tempfile::tempdir().expect("tempdir"); + let db_path = dir.path().join("memory.db"); + let clock = Arc::new(TestClock::new(1_000)); + let store = Arc::new(SqliteStore::open(&db_path).expect("open store")); + let service = MemoryService::new( + store.clone(), + clock.clone() as Arc, + embedder, + DecayConfig::default(), + ); + Harness { + _dir: dir, + db_path, + clock, + store, + service, + } +} + +/// A succeeding embedder programmed with `vectors` (anything else embeds to a +/// deterministic hash-seeded unit vector, ~orthogonal to every axis). +fn live(vectors: HashMap>) -> Arc { + Arc::new(FakeEmbedder::with_vectors(vectors)) +} + +fn new_memory(content: &str, mem_type: MemoryType) -> NewMemory { + NewMemory { + content: content.to_string(), + mem_type, + tags: vec![], + source: None, + scope: None, + ttl_secs: None, + } +} + +async fn store(h: &Harness, content: &str, mem_type: MemoryType) -> i64 { + h.service + .store(new_memory(content, mem_type)) + .await + .expect("store") +} + +fn query(text: &str, limit: Option) -> SearchArgs { + SearchArgs { + query: text.to_string(), + limit, + ..Default::default() + } +} + +/// A 768-dim unit vector along `axis`. +fn unit_axis(axis: usize) -> Vec { + let mut v = vec![0.0_f32; EMBEDDING_DIM]; + v[axis] = 1.0; + v +} + +/// A unit vector whose cosine similarity to `unit_axis(0)` is exactly `sim`, +/// with the remainder on its own private `axis` so rows never coincide. +fn sim_vec(sim: f32, axis: usize) -> Vec { + assert!(axis > 0, "axis 0 is the query axis"); + let mut v = vec![0.0_f32; EMBEDDING_DIM]; + v[0] = sim; + v[axis] = (1.0 - sim * sim).sqrt(); + v +} + +fn ids(results: &[agent_memory_core::domain::MemoryView]) -> Vec { + results.iter().map(|v| v.id).collect() +} + +fn raw_conn(db_path: &Path) -> rusqlite::Connection { + agent_memory_core::store::sqlite::register_vec_extension().expect("register vec0"); + rusqlite::Connection::open(db_path).expect("open db for assertion") +} + +const EPS: f64 = 1e-12; + +// --------------------------------------------------------------------------- +// Test 1: rrf_fuse against hand-computed values +// --------------------------------------------------------------------------- + +#[test] +fn rrf_k_is_the_cormack_clarke_constant() { + assert_eq!(RRF_K, 60.0); +} + +#[test] +fn rrf_dual_leg_rank_two_beats_single_leg_rank_one() { + // 20 is rank 2 in BOTH legs: 1/62 + 1/62 = 2/62. 10 and 30 are rank 1 in + // one leg each: 1/61. 2/62 ≈ 0.03226 > 1/61 ≈ 0.01639. + let fused = rrf_fuse(&[vec![10, 20], vec![30, 20]]); + assert_eq!(fused.len(), 3, "three distinct ids: {fused:?}"); + + assert_eq!(fused[0].0, 20, "the dual-leg id wins: {fused:?}"); + assert!((fused[0].1 - 2.0 / 62.0).abs() < EPS, "{fused:?}"); + + // 10 and 30 tie at exactly 1/61: the ascending-id tie-break decides. + assert_eq!((fused[1].0, fused[2].0), (10, 30), "{fused:?}"); + assert!((fused[1].1 - 1.0 / 61.0).abs() < EPS, "{fused:?}"); + assert!((fused[2].1 - 1.0 / 61.0).abs() < EPS, "{fused:?}"); + assert!(fused[0].1 > fused[1].1, "2/62 strictly beats 1/61"); +} + +#[test] +fn rrf_duplicate_ids_sum_their_contributions() { + // 7: rank 1 in leg A (1/61) + rank 3 in leg B (1/63). + // 8: rank 2 in leg A (1/62) only. 9: rank 1 in leg B (1/61) only. + let fused = rrf_fuse(&[vec![7, 8], vec![9, 5, 7]]); + let score = |id: i64| { + fused + .iter() + .find(|(i, _)| *i == id) + .map(|(_, s)| *s) + .unwrap_or_else(|| panic!("id {id} missing from {fused:?}")) + }; + assert!((score(7) - (1.0 / 61.0 + 1.0 / 63.0)).abs() < EPS); + assert!((score(8) - 1.0 / 62.0).abs() < EPS); + assert!((score(9) - 1.0 / 61.0).abs() < EPS); + assert!((score(5) - 1.0 / 62.0).abs() < EPS); + // 7 first; 9 (1/61); then the 1/62 tie between 5 and 8 broken by id. + assert_eq!( + fused.iter().map(|(i, _)| *i).collect::>(), + vec![7, 9, 5, 8] + ); +} + +#[test] +fn rrf_ties_break_deterministically_by_ascending_id() { + for _ in 0..20 { + // HashMap iteration order varies run to run; the output must not. + let fused = rrf_fuse(&[vec![42, 3], vec![17, 99]]); + assert_eq!( + fused.iter().map(|(i, _)| *i).collect::>(), + vec![17, 42, 3, 99], + "rank-1 ids tie (17 < 42), then rank-2 ids tie (3 < 99)" + ); + } +} + +#[test] +fn rrf_of_empty_legs_is_empty() { + assert!(rrf_fuse(&[vec![], vec![]]).is_empty()); + assert!(rrf_fuse(&[]).is_empty()); +} + +// --------------------------------------------------------------------------- +// Test 2: end-to-end fusion surfaces both a semantic-only and a keyword-only hit +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn hybrid_surfaces_the_top_semantic_and_the_top_keyword_hit() { + // X: nearest vector, but does NOT contain the keyword. + // Y: the only keyword hit, but its vector is orthogonal to the query. + const X: &str = "gentle western wind patterns along the coast"; + const Y: &str = "zephyr rollout notes for the zephyr cluster"; + const Q: &str = "zephyr"; + let mut vectors = HashMap::new(); + vectors.insert(Q.to_string(), unit_axis(0)); + vectors.insert(X.to_string(), sim_vec(0.99, 1)); + vectors.insert(Y.to_string(), unit_axis(2)); + let h = harness(live(vectors)); + + let x = store(&h, X, MemoryType::Pattern).await; + let y = store(&h, Y, MemoryType::Pattern).await; + for filler in [ + "weekly standup happens every monday", + "the frontend bundle uses tailwind", + "invoices are exported nightly", + ] { + store(&h, filler, MemoryType::Pattern).await; + } + + let outcome = h.service.search(query(Q, None)).await.expect("search"); + assert_eq!(outcome.search_mode, SearchMode::Hybrid); + + // Y: keyword rank 1 (1/61) + some KNN rank r >= 2 (1/(60+r)) > 1/61. + // X: KNN rank 1 only (1/61). Every filler: one KNN contribution < 1/61. + let got = ids(&outcome.results); + assert_eq!( + &got[..2], + &[y, x], + "both the keyword-only and the semantic-only hit surface, fused" + ); + assert_eq!(got.len(), 5, "every KNN candidate is still a fused result"); +} + +#[tokio::test] +async fn hybrid_wire_label_is_lowercase_hybrid() { + let h = harness(live(HashMap::new())); + store(&h, "anything at all", MemoryType::Pattern).await; + let outcome = h + .service + .search(query("anything", None)) + .await + .expect("search"); + let json = serde_json::to_value(&outcome).expect("serialize"); + assert_eq!(json["search_mode"], "hybrid"); +} + +// --------------------------------------------------------------------------- +// Test 3: both legs are oversampled (limit * 4) before fusion +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn semantic_leg_is_oversampled_beyond_limit() { + // limit = 2. K holds the only keyword hit and sits at KNN rank 5 — beyond + // the limit, inside limit * 4. Fused: K = 1/61 + 1/65 beats A1 = 1/61. + // If the KNN leg fetched only `limit` rows, K would score 1/61, tie A1, + // and lose the ascending-id tie-break (K is stored after A1). + const Q: &str = "quasar"; + const K: &str = "quasar calibration notes"; + let near = [ + ("alpha survey of the northern sky", 0.99_f32), + ("beta survey of the southern sky", 0.98), + ("gamma survey of the eastern sky", 0.97), + ("delta survey of the western sky", 0.96), + ]; + let mut vectors = HashMap::new(); + vectors.insert(Q.to_string(), unit_axis(0)); + for (i, (text, sim)) in near.iter().enumerate() { + vectors.insert(text.to_string(), sim_vec(*sim, i + 1)); + } + vectors.insert(K.to_string(), sim_vec(0.95, 10)); + let h = harness(live(vectors)); + + let mut near_ids = Vec::new(); + for (text, _) in near { + near_ids.push(store(&h, text, MemoryType::Pattern).await); + } + let k = store(&h, K, MemoryType::Pattern).await; + + let outcome = h.service.search(query(Q, Some(2))).await.expect("search"); + assert_eq!(outcome.search_mode, SearchMode::Hybrid); + assert_eq!( + ids(&outcome.results), + vec![k, near_ids[0]], + "the keyword hit at KNN rank 5 must enter fusion and win" + ); +} + +#[tokio::test] +async fn keyword_leg_is_oversampled_beyond_limit() { + // limit = 2 (both legs fetch 8). M is KNN rank 1 and keyword rank 3 — + // beyond the limit. W1/W2 are keyword ranks 1/2 with far-away vectors, + // pushed out of the KNN top 8 by seven closer fillers. + // Fused: M = 1/61 + 1/63 beats W1 = 1/61. If the keyword leg fetched only + // `limit` rows, M would score 1/61, tie W1 and lose the id tie-break. + const Q: &str = "quasar"; + const W1: &str = "quasar quasar quasar"; + const W2: &str = "quasar quasar quasar beacon"; + const M: &str = "quasar telescope calibration notes for the southern observatory array"; + let mut vectors = HashMap::new(); + vectors.insert(Q.to_string(), unit_axis(0)); + vectors.insert(W1.to_string(), unit_axis(1)); + vectors.insert(W2.to_string(), unit_axis(2)); + vectors.insert(M.to_string(), sim_vec(0.99, 3)); + let fillers: Vec = (0..7) + .map(|i| format!("filler record number {i}")) + .collect(); + for (i, f) in fillers.iter().enumerate() { + vectors.insert(f.clone(), sim_vec(0.9 - 0.05 * i as f32, 20 + i)); + } + let h = harness(live(vectors)); + + let w1 = store(&h, W1, MemoryType::Pattern).await; + store(&h, W2, MemoryType::Pattern).await; + let m = store(&h, M, MemoryType::Pattern).await; + for f in &fillers { + store(&h, f, MemoryType::Pattern).await; + } + + let outcome = h.service.search(query(Q, Some(2))).await.expect("search"); + assert_eq!(outcome.search_mode, SearchMode::Hybrid); + assert_eq!( + ids(&outcome.results), + vec![m, w1], + "the keyword-rank-3 hit must enter fusion and win" + ); +} + +// --------------------------------------------------------------------------- +// Test 4 (SEARCH-05 golden): decay is applied exactly once, post-fusion +// --------------------------------------------------------------------------- + +/// Two memories with EQUAL fused RRF score: A is KNN rank 1 + keyword rank 2, +/// B is KNN rank 2 + keyword rank 1 (both 1/61 + 1/62). The one stored FIRST +/// is aged two half-lives before the second is stored. Returns +/// `(stale_id, fresh_id, results)`. +async fn equal_rank_pair(stale_is_a: bool) -> (i64, i64, Vec) { + const Q: &str = "lantern"; + const A: &str = "lantern maintenance schedule for the north wing corridor"; + const B: &str = "lantern lantern checklist"; + let mut vectors = HashMap::new(); + vectors.insert(Q.to_string(), unit_axis(0)); + vectors.insert(A.to_string(), sim_vec(0.99, 1)); + vectors.insert(B.to_string(), sim_vec(0.98, 2)); + let h = harness(live(vectors)); + + let (first, second) = if stale_is_a { (A, B) } else { (B, A) }; + let stale = store(&h, first, MemoryType::Pattern).await; + h.clock.advance(DEFAULT_HALF_LIFE_SECS as i64 * 2); + let fresh = store(&h, second, MemoryType::Pattern).await; + + let outcome = h.service.search(query(Q, None)).await.expect("search"); + assert_eq!(outcome.search_mode, SearchMode::Hybrid); + (stale, fresh, ids(&outcome.results)) +} + +#[tokio::test] +async fn at_equal_fused_rank_the_fresh_memory_outranks_the_stale_one() { + // Both orientations: the stale one is stored first, so it has the LOWER + // id and would win the RRF id tie-break — only decay can put fresh first. + for stale_is_a in [true, false] { + let (stale, fresh, got) = equal_rank_pair(stale_is_a).await; + assert_eq!( + got, + vec![fresh, stale], + "fresh must strictly outrank stale at equal fused rank (stale_is_a = {stale_is_a})" + ); + } +} + +#[tokio::test] +async fn a_stale_pinned_memory_keeps_its_pinned_advantage_and_decay_applies_once() { + // P (DECISION, pinned) is KNN rank 1 + keyword rank 1: RRF 2/61. + // U (PATTERN, fresh) is KNN rank 2 + keyword rank 2: RRF 2/62. + // P is aged 270_000 s. Pinned decay d = 2^(-270000 / (6 * 30d)) ≈ 0.98804. + // once, multiplicative : P = 2/61 * d ≈ 0.032395 > U = 2/62 ≈ 0.032258 + // twice (d^2 ≈ 0.97622): P ≈ 0.032007 < U -> U would win + // pinning ignored (unpinned d ≈ 0.93031): P ≈ 0.030502 < U -> U would win + // additive (rrf + d) : P ≈ 1.0208 < U ≈ 1.0323 -> U would win + // So P first proves: multiplicative, once, pinned-aware. + const Q: &str = "beacon"; + const P: &str = "beacon beacon protocol"; + const U: &str = "beacon protocol notes for the relay station handbook"; + const AGE: i64 = 270_000; + let mut vectors = HashMap::new(); + vectors.insert(Q.to_string(), unit_axis(0)); + vectors.insert(P.to_string(), sim_vec(0.99, 1)); + vectors.insert(U.to_string(), sim_vec(0.98, 2)); + let h = harness(live(vectors)); + + let p = store(&h, P, MemoryType::Decision).await; + h.clock.advance(AGE); + let u = store(&h, U, MemoryType::Pattern).await; + + let hl = DEFAULT_HALF_LIFE_SECS; + let d_pinned = decay_score(AGE, 0, hl, true); + assert!(d_pinned > 61.0 / 62.0 && d_pinned * d_pinned < 61.0 / 62.0); + assert!(decay_score(AGE, 0, hl, false) < 61.0 / 62.0); + + let outcome = h.service.search(query(Q, None)).await.expect("search"); + assert_eq!(outcome.search_mode, SearchMode::Hybrid); + assert_eq!(ids(&outcome.results), vec![p, u]); + + // The wire decay_score is the single, pinned-aware value at `now`. + let now = h.clock.now(); + let p_view = &outcome.results[0]; + assert!((p_view.decay_score - decay_score(now, p_view.last_accessed, hl, true)).abs() < EPS); + assert!((p_view.decay_score - d_pinned).abs() < EPS); + assert!((outcome.results[1].decay_score - 1.0).abs() < EPS); +} + +// --------------------------------------------------------------------------- +// Test 5: emptiness is not a fallback +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn empty_corpus_is_an_empty_hybrid_answer() { + let h = harness(live(HashMap::new())); + let outcome = h + .service + .search(query("nothing", None)) + .await + .expect("search"); + assert_eq!(outcome.search_mode, SearchMode::Hybrid); + assert!(outcome.results.is_empty()); +} + +#[tokio::test] +async fn filters_that_exclude_everything_stay_hybrid() { + let h = harness(live(HashMap::new())); + store(&h, "a scoped memory about caching", MemoryType::Pattern).await; + let outcome = h + .service + .search(SearchArgs { + query: "caching".to_string(), + scope: Some("no-such-scope".to_string()), + ..Default::default() + }) + .await + .expect("search"); + assert_eq!( + outcome.search_mode, + SearchMode::Hybrid, + "zero matches in both legs is still a hybrid answer, never a fallback" + ); + assert!(outcome.results.is_empty()); +} + +// Keeps the shared helpers referenced until Task 2 extends this file. +#[allow(dead_code)] +fn _harness_fields_used(h: &Harness) -> (&Path, &Arc) { + let _ = raw_conn; + (&h.db_path, &h.store) +} + +#[allow(dead_code)] +fn _store_trait_in_scope(s: &dyn Store) -> &dyn Store { + s +} diff --git a/crates/agent-memory-core/tests/relations.rs b/crates/agent-memory-core/tests/relations.rs index cbf5149..5e3731e 100644 --- a/crates/agent-memory-core/tests/relations.rs +++ b/crates/agent-memory-core/tests/relations.rs @@ -723,7 +723,7 @@ async fn expansion_also_applies_on_the_semantic_path() { .search(search_args("vector anchor", true)) .await .expect("search"); - assert_eq!(found.search_mode, SearchMode::Semantic); + assert_eq!(found.search_mode, SearchMode::Hybrid); let entry = &related_of(&found.results, a)[0]; assert_eq!((entry.id, entry.kind.as_str()), (b, "superseded_by")); assert_eq!(entry.direction, LinkDirection::In); diff --git a/crates/agent-memory-core/tests/semantic.rs b/crates/agent-memory-core/tests/semantic.rs index 82a3fea..2cf3fd5 100644 --- a/crates/agent-memory-core/tests/semantic.rs +++ b/crates/agent-memory-core/tests/semantic.rs @@ -107,8 +107,8 @@ async fn semantic_search_finds_meaning_with_no_shared_keywords() { assert_eq!( outcome.search_mode, - SearchMode::Semantic, - "with a working embedder the search mode must be semantic" + SearchMode::Hybrid, + "with a working embedder the search mode must be hybrid (SEARCH-04)" ); assert!( !outcome.results.is_empty(), @@ -169,7 +169,7 @@ async fn at_equal_vector_distance_recent_memory_outranks_aged() { .await .expect("semantic search"); - assert_eq!(outcome.search_mode, SearchMode::Semantic); + assert_eq!(outcome.search_mode, SearchMode::Hybrid); assert_eq!(outcome.results.len(), 2, "both memories are candidates"); assert_eq!( outcome.results[0].id, new_id, @@ -285,7 +285,7 @@ async fn tag_filter_is_exact_on_knn_path() { .search(search(Some("rust"))) .await .expect("tag-filtered semantic search"); - assert_eq!(outcome.search_mode, SearchMode::Semantic); + assert_eq!(outcome.search_mode, SearchMode::Hybrid); assert_eq!( outcome.results.len(), 1, @@ -353,7 +353,7 @@ async fn live_ollama_semantic_recall_without_shared_keywords() { .await .expect("live semantic search"); - assert_eq!(outcome.search_mode, SearchMode::Semantic); + assert_eq!(outcome.search_mode, SearchMode::Hybrid); assert_eq!( outcome.results.first().map(|v| v.id), Some(rls_id), diff --git a/crates/agent-memory-core/tests/update.rs b/crates/agent-memory-core/tests/update.rs index 5421b3b..a7c2c47 100644 --- a/crates/agent-memory-core/tests/update.rs +++ b/crates/agent-memory-core/tests/update.rs @@ -212,7 +212,7 @@ async fn content_update_re_embeds_so_the_old_meaning_stops_matching() { }) .await .expect("search before update"); - assert_eq!(before.search_mode, SearchMode::Semantic); + assert_eq!(before.search_mode, SearchMode::Hybrid); assert_eq!( before.results[0].id, id, "sanity: the old meaning matches the target before the update" @@ -236,7 +236,7 @@ async fn content_update_re_embeds_so_the_old_meaning_stops_matching() { }) .await .expect("old-meaning search"); - assert_eq!(old.search_mode, SearchMode::Semantic); + assert_eq!(old.search_mode, SearchMode::Hybrid); assert!( old.results.iter().all(|v| v.id != id), "the old meaning must no longer return the updated row: {:?}", @@ -252,7 +252,7 @@ async fn content_update_re_embeds_so_the_old_meaning_stops_matching() { }) .await .expect("new-meaning search"); - assert_eq!(new.search_mode, SearchMode::Semantic); + assert_eq!(new.search_mode, SearchMode::Hybrid); assert_eq!(new.results[0].id, id, "the new meaning returns the row"); // Exactly one vector row: the old one was replaced, not shadowed. From 7c63d0ed58b57ba06fbef7940a9dbd9068cb3b73 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:38:50 +0200 Subject: [PATCH 14/42] test(04-04): lock the hybrid degrade table, error taxonomy, bump scope and wire docs (SEARCH-04/05) - tests/hybrid.rs: embed error and empty embed batch degrade to output byte-identical to a direct Store::search; malformed MATCH (unbalanced quote, dangling AND, bare '?') is InvalidQuery in hybrid exactly as in keyword mode; a broken FTS or vec leg propagates a Sqlite error instead of serving the other leg; 20 candidates / limit 5 bumps exactly the 5 returned rows; hybrid expansion never bumps neighbors - MCP + REST in-process tests: "hybrid" with a live FakeEmbedder, "keyword" with a dead one, identical {search_mode, results} envelope; malformed query is invalid_params / 400 in hybrid mode - memory_search tool description: 'hybrid'|'keyword', RRF k=60, keyword degrade, FTS5 query syntax, filter-after-cut bound (WR-01), returned-only bumps - README: search-mode table, behavioral change "semantic" -> "hybrid" for the v0.1.0 release notes, bump scope and query-syntax notes - agent-memory dev-dependency enables core's test-clock feature (test builds only) Co-Authored-By: Claude Opus 5.5 --- README.md | 70 ++++-- crates/agent-memory-core/tests/hybrid.rs | 288 ++++++++++++++++++++++- crates/agent-memory/Cargo.toml | 3 + crates/agent-memory/src/mcp.rs | 88 ++++++- crates/agent-memory/src/rest/handlers.rs | 92 +++++++- 5 files changed, 509 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 95b1608..9f52cc7 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,12 @@ sessions and across tools. - **Decay, not deletion.** Memories fade in ranking over time (exponential decay; high-value pinned types fade slower) but are **never** deleted by decay. The only ways a memory is removed are an explicit TTL expiry or `memory_forget`. -- **Search by meaning.** Semantic search via local - [Ollama](https://ollama.com) embeddings when available, with automatic - fallback to SQLite FTS5 keyword ranking when it isn't — both blended with - decay. No Ollama, no network? Everything still works. +- **Search by meaning and by keyword.** Hybrid search when local + [Ollama](https://ollama.com) embeddings are available: a keyword ranking and + a semantic ranking are fused, so an exact-term hit and a paraphrase both + surface. Automatic fallback to SQLite FTS5 keyword ranking when Ollama isn't + there, and both modes are blended with decay. No Ollama, no network? + Everything still works. ## Install @@ -106,7 +108,7 @@ stored in one is immediately retrievable in another. | Tool | Purpose | Validated parameters | |------|---------|----------------------| | `memory_store` | Persist a typed memory; returns its id. | `ttl_secs`: optional integer, valid range 1–3155760000 (~100 years); omitted = no expiry | -| `memory_search` | Semantic (embedding) search when Ollama is reachable, FTS5 keyword otherwise — ranked by relevance × decay, recency-bumped on retrieval. | `limit`: optional integer, valid range 1–200; omitted defaults to 50. `tag`: exact tag match (case-sensitive). `expand_links`: optional boolean, default `false` | +| `memory_search` | Hybrid (keyword + semantic, rank-fused) search when Ollama is reachable, FTS5 keyword otherwise — weighted by decay, and only the returned results are recency-bumped. | `limit`: optional integer, valid range 1–200; omitted defaults to 50. `tag`: exact tag match (case-sensitive). `expand_links`: optional boolean, default `false` | | `memory_list` | List memories newest-first, with optional type/tag/scope filters. | `limit`: optional integer, valid range 1–200; omitted = unlimited. `tag`: exact tag match (case-sensitive). `expand_links`: optional boolean, default `false` | | `memory_update` | Patch a memory in place by id — content, tags, scope, TTL (clean not-found for unknown ids). | `content`: never null or empty. `ttl_secs`: 1–3155760000 when set; `null` = no expiry. At least one field besides `id` | | `memory_link` | Link two memories with a typed, directed relation (idempotent; clean not-found for unknown ids). | `kind`: one of `relates_to`, `supersedes`, `caused_by`. `from_id` ≠ `to_id` | @@ -146,14 +148,33 @@ Returns an envelope carrying the mode actually used plus the ranked results, each with a freshly-recomputed `decay_score`: ```json -{ "search_mode": "semantic", "results": [ ... ] } +{ "search_mode": "hybrid", "results": [ ... ] } ``` -`search_mode` is `"semantic"` when the query was answered via embeddings and -`"keyword"` when the FTS5 path served it (Ollama absent or unreachable). A -query that matches nothing returns an empty list, never an error. A present -`limit` must be between 1 and 200 (omitted defaults to 50); a `tag` filter -matches whole tags exactly and case-sensitively. +| `search_mode` | When | How results are ranked | +|---------------|------|------------------------| +| `"hybrid"` | Ollama is reachable | Two rankings are fused with Reciprocal Rank Fusion: the semantic (embedding KNN) ranking and the keyword (FTS5 bm25) ranking, each oversampled to 4 × `limit`. A memory at rank *r* in a ranking scores `1 / (60 + r)` (k = 60), and a memory found by both rankings **sums** both scores, so agreement wins: 2nd in both beats 1st in one. The fused score is then multiplied by the memory's decay score, once, so at equal fused rank the fresher memory wins and pinned types keep their slower fade. | +| `"keyword"` | Ollama is absent or unreachable | FTS5 bm25 relevance blended with decay — the v0.0.1 path, unchanged. | + +The envelope is the same `{search_mode, results}` shape in both modes. A query +that matches nothing returns an empty list, never an error, and an empty hybrid +answer is still reported as `"hybrid"`. A present `limit` must be between 1 and +200 (omitted defaults to 50); a `tag` filter matches whole tags exactly and +case-sensitively. + +- **The query is FTS5 MATCH syntax in both modes.** A query FTS5 cannot parse — + an unbalanced `"`, a dangling `AND`, bare punctuation such as `?`, `,` or `'` + — is a client error (`invalid_params` / HTTP 400) even when Ollama is up. + Hybrid never quietly answers from the semantic ranking alone. Use plain words + for natural-language questions: `what did we decide about caching`, not + `what did we decide about caching?`. +- **Only returned results are recency-bumped.** The candidates fusion looked at + but did not return, and any `expand_links` neighbors, are not touched, so a + search never changes how unseen memories decay. +- **Filters and the semantic ranking.** The semantic ranking applies + `type`/`tag`/`scope` filters after its 4 × `limit` candidate cut, so a + heavily filtered hybrid query can recall fewer than `limit` results. The + keyword ranking filters before its cut. ### `memory_list` @@ -213,8 +234,8 @@ returns — plus `embedding_status`: - **`embedding_status`** is `1` when the stored vector matches the current content and `0` when a re-embed is pending — the update succeeded while Ollama was unreachable, and the background sweep re-embeds it once Ollama is back. - Until then semantic search cannot match it by meaning; keyword-fallback - search already sees the new content. + Until then the semantic ranking cannot match it by meaning; the keyword + ranking (in hybrid and keyword mode alike) already sees the new content. - **An update counts as a use.** It refreshes `last_accessed` (and the access count), so an edited memory ranks as current rather than decayed. @@ -299,7 +320,9 @@ ollama pull nomic-embed-text ``` That's the whole setup. When Ollama is reachable, stores embed content -automatically and searches use vector KNN (cosine) blended with decay. Point +automatically and searches run in hybrid mode: vector KNN (cosine) and FTS5 +keyword rankings fused with RRF, then weighted by decay +(see [`memory_search`](#memory_search)). Point at a non-default Ollama with either: ```bash @@ -313,7 +336,8 @@ isn't pulled: - stores still succeed — content is queued (`embedding_status = 0`) and backfilled by the background sweep once Ollama returns; - searches transparently fall back to the FTS5 keyword path and report it via - `"search_mode": "keyword"`; + `"search_mode": "keyword"` — the same envelope and the same ranking as + v0.0.1; - nothing errors, nothing blocks. The tool works fully offline without Ollama. ## REST API @@ -369,17 +393,27 @@ VPN/SSH tunnel or an authenticating reverse proxy. ## Behavioral changes in v0.1.0 -Two deliberate contract changes from v0.0.1 (both apply identically to MCP and -REST): +Three deliberate contract changes from v0.0.1 (all apply identically to MCP +and REST): 1. **Tag filtering is now an exact, case-sensitive match** instead of a substring match — `tag=rust` no longer matches a memory tagged `rustling`, - and `Rust` does not match `rust`. This applies to search (semantic and + and `Rust` does not match `rust`. This applies to search (hybrid and keyword paths) and list. 2. **Out-of-bounds `limit` and `ttl_secs` now return client errors** (HTTP 400 / MCP `invalid_params`) carrying the valid range, instead of being silently clamped or accepted. Valid boundaries (`limit` 1–200, `ttl_secs` 1–3155760000) and omitted values behave exactly as before. +3. **`search_mode: "semantic"` is no longer emitted.** When the embedder is + available, searches are now hybrid: keyword and semantic rankings are fused + with RRF (k = 60) and decay-weighted, reported as `"search_mode": "hybrid"`. + `"keyword"` (embedder down) is unchanged, as is the envelope shape. Two side + effects: + - With Ollama up, a query FTS5 cannot parse (bare `?`, `,`, `'`, an + unbalanced `"`) is now a client error, exactly as it always was without + Ollama. v0.0.1 answered such queries from embeddings alone. + - Exact keyword hits now surface alongside paraphrase matches, so result + order can differ from v0.0.1's embedding-only ranking. ## Import from GSD diff --git a/crates/agent-memory-core/tests/hybrid.rs b/crates/agent-memory-core/tests/hybrid.rs index f8d21ca..7995812 100644 --- a/crates/agent-memory-core/tests/hybrid.rs +++ b/crates/agent-memory-core/tests/hybrid.rs @@ -18,9 +18,13 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use agent_memory_core::clock::{Clock, TestClock}; -use agent_memory_core::decay::{decay_score, DecayConfig, DEFAULT_HALF_LIFE_SECS}; -use agent_memory_core::domain::{MemoryType, NewMemory}; -use agent_memory_core::embed::{Embedder, FakeEmbedder, EMBEDDING_DIM}; +use agent_memory_core::decay::{ + apply_decay, decay_score, DecayConfig, RankWeights, DEFAULT_HALF_LIFE_SECS, +}; +use agent_memory_core::domain::{LinkKind, MemoryError, MemoryType, NewMemory}; +use agent_memory_core::embed::{ + BoxFuture, EmbedError, Embedder, EmbedderHealth, FakeEmbedder, EMBEDDING_DIM, +}; use agent_memory_core::service::{rrf_fuse, MemoryService, SearchArgs, SearchMode, RRF_K}; use agent_memory_core::store::sqlite::SqliteStore; use agent_memory_core::store::Store; @@ -439,15 +443,277 @@ async fn filters_that_exclude_everything_stay_hybrid() { ); assert!(outcome.results.is_empty()); } +// --------------------------------------------------------------------------- +// Task 2: the degrade decision table (RESEARCH Pattern 1), row by row +// +// | Condition | Outcome | +// |----------------------------------------|----------------------------------| +// | Embed OK, both legs OK | fuse -> "hybrid" (Test 2 above) | +// | Embed OK, both legs empty | [] still "hybrid" (Test 5 above) | +// | Embed Err / empty batch | unchanged v1.0 keyword path | +// | InvalidQuery from the keyword leg | propagate (400 / invalid_params) | +// | Internal SQLite error in either leg | propagate (500 / internal_error) | +// --------------------------------------------------------------------------- + +/// An embedder that answers with an EMPTY batch: the "invalid empty batch" +/// degrade row, distinct from an outright embed error. +struct EmptyBatchEmbedder; + +impl Embedder for EmptyBatchEmbedder { + fn embed<'a>( + &'a self, + _inputs: &'a [String], + ) -> BoxFuture<'a, Result>, EmbedError>> { + Box::pin(async { Ok(Vec::new()) }) + } + + fn health<'a>(&'a self) -> BoxFuture<'a, Result> { + Box::pin(async { Ok(EmbedderHealth::Ready) }) + } +} + +/// Seed a small keyword corpus with staggered ages and mixed types, so the v1.0 +/// bm25 × decay blend has something non-trivial to order. +async fn seed_degrade_corpus(h: &Harness) { + for (content, mem_type) in [ + ("database pool sizing for the api tier", MemoryType::Pattern), + ( + "database database migration checklist", + MemoryType::Decision, + ), + ( + "we never share a database between services", + MemoryType::Constraint, + ), + ("unrelated note about the office plants", MemoryType::Todo), + ("database backups run nightly at two", MemoryType::Pattern), + ] { + store(h, content, mem_type).await; + h.clock.advance(DEFAULT_HALF_LIFE_SECS as i64 / 3); + } +} + +async fn assert_degrades_to_the_v1_keyword_path(embedder: Arc) { + let h = harness(embedder); + seed_degrade_corpus(&h).await; + let args = SearchArgs { + query: "database".to_string(), + limit: Some(3), + ..Default::default() + }; + + // The v1.0 path called directly, BEFORE the service search bumps anything: + // public Store::search plus the service's on-read decay recompute. + let now = h.clock.now(); + let cfg = DecayConfig::default(); + let mut direct = h + .store + .search(args.clone(), now, RankWeights::default(), cfg) + .expect("direct Store::search"); + for view in &mut direct { + apply_decay(view, now, &cfg); + } + assert_eq!(direct.len(), 3, "the fixture must exercise the limit"); + + let outcome = h.service.search(args).await.expect("degraded search"); + assert_eq!(outcome.search_mode, SearchMode::Keyword); + + // Byte-for-byte: the same envelope shape, the same rows, the same order, + // the same decay scores. The keyword path is untouched by hybrid. + let got = serde_json::to_value(&outcome).expect("serialize outcome"); + let expected = serde_json::json!({ "search_mode": "keyword", "results": direct }); + assert_eq!(got, expected); +} + +#[tokio::test] +async fn embed_error_degrades_to_the_unchanged_v1_keyword_path() { + assert_degrades_to_the_v1_keyword_path(Arc::new(FakeEmbedder::failing())).await; +} + +#[tokio::test] +async fn an_empty_embed_batch_degrades_to_the_unchanged_v1_keyword_path() { + assert_degrades_to_the_v1_keyword_path(Arc::new(EmptyBatchEmbedder)).await; +} + +#[tokio::test] +async fn a_malformed_match_is_invalid_query_in_hybrid_mode_exactly_as_in_keyword_mode() { + // T-04-15: the same input must classify the same way in every mode, never + // a 400 in one mode and a semantic-only 200 in the other. The KNN leg + // would happily answer these queries, so its rows must not leak out. + // FTS5 rejects bare punctuation too: a natural-language "?" query is + // client input in hybrid mode, exactly as it always was in keyword mode. + for malformed in ["\"unbalanced", "AND", "what is the plan?"] { + for (label, embedder) in [ + ("hybrid", live(HashMap::new())), + ( + "keyword", + Arc::new(FakeEmbedder::failing()) as Arc, + ), + ] { + let h = harness(embedder); + store( + &h, + "what is the plan for the unbalanced ledger", + MemoryType::Pattern, + ) + .await; + let err = h + .service + .search(query(malformed, None)) + .await + .expect_err("a malformed MATCH must not succeed in any mode"); + assert!( + matches!(err, MemoryError::InvalidQuery(ref q) if q == malformed), + "{label} mode: {malformed:?} must be InvalidQuery, got {err:?}" + ); + } + } +} + +#[tokio::test] +async fn an_internal_error_in_either_leg_propagates_and_never_serves_the_other_leg() { + // Break one leg's index underneath a live store (renamed away, so the + // leg's SQL hits "no such table"): the search must fail with an internal + // Sqlite error (REST 500 / MCP internal_error), never quietly return the + // surviving leg's rows as a hybrid answer. + for leg_table in ["memories_fts", "vec_memories"] { + let h = harness(live(HashMap::new())); + store(&h, "ledger reconciliation runbook", MemoryType::Pattern).await; + raw_conn(&h.db_path) + .execute_batch(&format!( + "ALTER TABLE {leg_table} RENAME TO {leg_table}_offline;" + )) + .expect("rename leg table"); + + let err = h + .service + .search(query("ledger", None)) + .await + .expect_err("a broken leg must fail the search"); + assert!( + matches!(err, MemoryError::Sqlite(_)), + "with {leg_table} gone the search must surface an internal Sqlite error, got {err:?}" + ); + } +} + +// --------------------------------------------------------------------------- +// Bump scope (SEARCH-05 / Pitfall 3 / T-04-17) +// --------------------------------------------------------------------------- -// Keeps the shared helpers referenced until Task 2 extends this file. -#[allow(dead_code)] -fn _harness_fields_used(h: &Harness) -> (&Path, &Arc) { - let _ = raw_conn; - (&h.db_path, &h.store) +/// `access_count` for every row, by id. +fn access_counts(db_path: &Path) -> HashMap { + let conn = raw_conn(db_path); + let mut stmt = conn + .prepare("SELECT id, access_count FROM memories") + .expect("prepare"); + let rows = stmt + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("query"); + rows.map(|r| r.expect("row")).collect() } -#[allow(dead_code)] -fn _store_trait_in_scope(s: &dyn Store) -> &dyn Store { - s +/// Wait (bounded) until the fire-and-forget bump has landed `expected` total +/// increments, then give any stray over-fire a moment to land too. +async fn settle_bumps(db_path: &Path, expected: i64) -> HashMap { + for _ in 0..300 { + if access_counts(db_path).values().sum::() >= expected { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + access_counts(db_path) +} + +#[tokio::test] +async fn only_the_returned_ids_are_bumped_never_the_fusion_candidates() { + // 20 rows, every one a candidate in BOTH legs: all contain the keyword and + // all carry a vector, and each leg fetches limit * 4 = 20. limit = 5. + let h = harness(live(HashMap::new())); + for i in 0..20 { + store(&h, &format!("ledger entry number {i}"), MemoryType::Pattern).await; + } + assert!(access_counts(&h.db_path).values().all(|c| *c == 0)); + + let outcome = h + .service + .search(query("ledger", Some(5))) + .await + .expect("search"); + assert_eq!(outcome.search_mode, SearchMode::Hybrid); + let mut returned = ids(&outcome.results); + assert_eq!(returned.len(), 5); + + let counts = settle_bumps(&h.db_path, 5).await; + let mut bumped: Vec = counts + .iter() + .filter(|(_, c)| **c > 0) + .map(|(id, _)| *id) + .collect(); + bumped.sort_unstable(); + returned.sort_unstable(); + assert_eq!(counts.len(), 20); + assert_eq!( + bumped, returned, + "exactly the 5 returned rows are bumped; the other 15 candidates are untouched" + ); + assert!( + counts.values().all(|c| *c <= 1), + "each bumped once: {counts:?}" + ); +} + +#[tokio::test] +async fn hybrid_expansion_runs_after_the_bump_and_never_bumps_neighbors() { + // The 04-03 carry-forward on the new path: truncate, then bump the + // returned ids, then expand. The neighbor links to the only returned row. + const Q: &str = "harbor"; + const ANCHOR: &str = "harbor pilot boarding procedure"; + const NEIGHBOR: &str = "tide tables for the outer channel"; + let mut vectors = HashMap::new(); + vectors.insert(Q.to_string(), unit_axis(0)); + vectors.insert(ANCHOR.to_string(), sim_vec(0.99, 1)); + vectors.insert(NEIGHBOR.to_string(), sim_vec(0.5, 2)); + let h = harness(live(vectors)); + let anchor = store(&h, ANCHOR, MemoryType::Pattern).await; + let neighbor = store(&h, NEIGHBOR, MemoryType::Pattern).await; + h.service + .link(anchor, neighbor, LinkKind::RelatesTo) + .await + .expect("link"); + + let outcome = h + .service + .search(SearchArgs { + query: Q.to_string(), + limit: Some(1), + expand_links: true, + ..Default::default() + }) + .await + .expect("search"); + assert_eq!(outcome.search_mode, SearchMode::Hybrid); + assert_eq!(ids(&outcome.results), vec![anchor]); + let related = outcome.results[0] + .related + .as_ref() + .expect("expand_links attaches related"); + assert_eq!( + related.iter().map(|r| r.id).collect::>(), + vec![neighbor] + ); + + let counts = settle_bumps(&h.db_path, 1).await; + assert_eq!(counts.get(&anchor), Some(&1), "the returned row is bumped"); + assert_eq!( + counts.get(&neighbor), + Some(&0), + "an expanded neighbor is never bumped (D-06)" + ); + + // Without the flag the hybrid envelope carries no `related` member at all. + let plain = h.service.search(query(Q, Some(1))).await.expect("search"); + let json = serde_json::to_value(&plain).expect("serialize"); + assert!(json["results"][0].get("related").is_none(), "{json}"); } diff --git a/crates/agent-memory/Cargo.toml b/crates/agent-memory/Cargo.toml index dbb5420..48c2a66 100644 --- a/crates/agent-memory/Cargo.toml +++ b/crates/agent-memory/Cargo.toml @@ -25,6 +25,9 @@ tracing.workspace = true tracing-subscriber.workspace = true [dev-dependencies] +# Test builds only: exposes the core crate's FakeEmbedder/TestClock so the +# in-process MCP and REST tests can drive the hybrid (embedder-up) search path. +agent-memory-core = { path = "../agent-memory-core", features = ["test-clock"] } tempfile.workspace = true serde_json.workspace = true rusqlite.workspace = true diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index 24f891c..3d51545 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -307,7 +307,7 @@ impl MemoryMcp { } #[tool( - description = "Search stored memories by semantic similarity (local Ollama embeddings) blended with decay, falling back to keyword/FTS5 search when the embedder is unavailable. Returns {search_mode: 'semantic'|'keyword', results: [...]}; results is empty when nothing matches. Set expand_links=true to attach each result's linked memories as trimmed related summaries." + description = "Search stored memories. When the local Ollama embedder is reachable, search is hybrid: a keyword (FTS5) ranking and a semantic (embedding) ranking are fused with Reciprocal Rank Fusion (k=60), then weighted by decay so fresher memories win ties. When the embedder is unavailable, search falls back to keyword/FTS5 ranking alone. Returns {search_mode: 'hybrid'|'keyword', results: [...]}; results is empty when nothing matches. The query is FTS5 MATCH syntax in both modes, so a malformed query (an unbalanced quote, bare punctuation such as '?') is invalid_params. The semantic ranking applies type/tag/scope filters after its 4x-limit candidate cut, so a heavily filtered query may return fewer than limit results. Only returned results count as accessed. Set expand_links=true to attach each result's linked memories as trimmed related summaries." )] async fn memory_search( &self, @@ -499,8 +499,22 @@ mod tests { const DEAD_OLLAMA: &str = "http://127.0.0.1:9"; fn test_server(dir: &tempfile::TempDir) -> MemoryMcp { + server_with(dir, Arc::new(OllamaClient::new(DEAD_OLLAMA))) + } + + /// A server whose embedder is UP (hash-seeded FakeEmbedder vectors), so + /// memory_search takes the hybrid path (SEARCH-04). + fn live_server(dir: &tempfile::TempDir) -> MemoryMcp { + server_with( + dir, + Arc::new(agent_memory_core::embed::FakeEmbedder::with_vectors( + std::collections::HashMap::new(), + )), + ) + } + + fn server_with(dir: &tempfile::TempDir, embedder: Arc) -> MemoryMcp { let store = SqliteStore::open(&dir.path().join("mem.db")).expect("store opens"); - let embedder: Arc = Arc::new(OllamaClient::new(DEAD_OLLAMA)); let service = MemoryService::new( Arc::new(store), Arc::new(SystemClock), @@ -1012,6 +1026,76 @@ mod tests { ); } + fn search_args(json: serde_json::Value) -> SearchArgs { + serde_json::from_value(json).expect("search args deserialize") + } + + #[tokio::test] + async fn memory_search_reports_hybrid_when_the_embedder_is_up_and_keyword_when_down() { + // SEARCH-04 wire contract on the MCP transport, from raw tool args. + for (expected_mode, live) in [("hybrid", true), ("keyword", false)] { + let dir = tempfile::tempdir().expect("tempdir"); + let server = if live { + live_server(&dir) + } else { + test_server(&dir) + }; + let id = store_one(&server, "hybrid recall over the ledger").await; + + let result = server + .memory_search(Parameters(search_args(serde_json::json!({ + "query": "ledger" + })))) + .await + .expect("search succeeds"); + let body = result_json(&result); + assert_eq!(body["search_mode"], expected_mode, "{body}"); + assert_eq!(body["results"][0]["id"], id, "{body}"); + assert_eq!( + body.as_object().map(serde_json::Map::len), + Some(2), + "envelope is exactly {{search_mode, results}}: {body}" + ); + } + } + + #[tokio::test] + async fn memory_search_malformed_query_is_invalid_params_in_hybrid_mode() { + // T-04-15: the keyword leg's InvalidQuery is never swallowed in favor + // of the semantic leg's rows. + let dir = tempfile::tempdir().expect("tempdir"); + let server = live_server(&dir); + store_one(&server, "any searchable row").await; + let invalid_params_code = McpError::invalid_params("x", None).code; + + for malformed in ["\"", "what is the plan?"] { + let err = server + .memory_search(Parameters(search_args(serde_json::json!({ + "query": malformed + })))) + .await + .expect_err("a malformed MATCH is client input in hybrid mode"); + assert_eq!(err.code, invalid_params_code, "{malformed:?}: {err:?}"); + } + } + + #[test] + fn memory_search_description_enumerates_exactly_hybrid_and_keyword() { + let tools = MemoryMcp::tool_router().list_all(); + let search = tools + .iter() + .find(|t| t.name == "memory_search") + .expect("memory_search is registered"); + let description = search.description.as_deref().unwrap_or_default(); + // The whole enumeration, delimited on both sides, so a third mode on + // either side (the retired semantic one included) breaks the match. + assert!( + description.contains("{search_mode: 'hybrid'|'keyword', results"), + "{description}" + ); + assert!(description.contains("k=60"), "{description}"); + } + #[test] fn map_mcp_error_splits_client_and_internal_tiers() { let invalid_params_code = McpError::invalid_params("x", None).code; diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index d35e9c2..12102f7 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -443,8 +443,22 @@ mod tests { const DEAD_OLLAMA: &str = "http://127.0.0.1:9"; fn test_state(dir: &tempfile::TempDir) -> Arc { + state_with(dir, Arc::new(OllamaClient::new(DEAD_OLLAMA))) + } + + /// A state whose embedder is UP (hash-seeded FakeEmbedder vectors), so + /// search takes the hybrid path (SEARCH-04). + fn live_state(dir: &tempfile::TempDir) -> Arc { + state_with( + dir, + Arc::new(agent_memory_core::embed::FakeEmbedder::with_vectors( + std::collections::HashMap::new(), + )), + ) + } + + fn state_with(dir: &tempfile::TempDir, embedder: Arc) -> Arc { let store = SqliteStore::open(&dir.path().join("mem.db")).expect("store opens"); - let embedder: Arc = Arc::new(OllamaClient::new(DEAD_OLLAMA)); let service = MemoryService::new( Arc::new(store), Arc::new(SystemClock), @@ -645,6 +659,82 @@ mod tests { ); } + fn search_request(query: &str) -> SearchRequest { + SearchRequest { + query: query.to_string(), + r#type: None, + tag: None, + scope: None, + limit: Some(10), + expand_links: false, + } + } + + #[tokio::test] + async fn search_reports_hybrid_with_a_live_embedder_and_keyword_with_a_dead_one() { + // SEARCH-04 wire contract: embedder up -> "hybrid", embedder down -> + // "keyword"; the envelope keys are the same in both modes. + for (expected_mode, live) in [("hybrid", true), ("keyword", false)] { + let dir = tempfile::tempdir().expect("tempdir"); + let state = if live { + live_state(&dir) + } else { + test_state(&dir) + }; + store_handler( + State(state.clone()), + Json(store_request( + "row level security controls database access", + "PATTERN", + )), + ) + .await + .expect("store succeeds"); + + let resp = search_handler(State(state), Json(search_request("database"))) + .await + .expect("search succeeds") + .into_response(); + assert_eq!(resp.status(), StatusCode::OK); + let body = body_json(resp).await; + assert_eq!(body["search_mode"], expected_mode, "{body}"); + let mut keys: Vec<&str> = body + .as_object() + .expect("envelope is an object") + .keys() + .map(String::as_str) + .collect(); + keys.sort_unstable(); + assert_eq!(keys, vec!["results", "search_mode"], "{body}"); + assert_eq!(body["results"].as_array().map(Vec::len), Some(1), "{body}"); + } + } + + #[tokio::test] + async fn a_malformed_query_is_400_in_hybrid_mode_too_never_a_semantic_200() { + // T-04-15: with the embedder UP the KNN leg could answer, but the + // keyword leg's InvalidQuery must still surface as a 400. + let dir = tempfile::tempdir().expect("tempdir"); + let state = live_state(&dir); + store_handler( + State(state.clone()), + Json(store_request("any searchable row", "PATTERN")), + ) + .await + .expect("store succeeds"); + + for malformed in ["\"", "what is the plan?"] { + let err = search_handler(State(state.clone()), Json(search_request(malformed))) + .await + .expect_err("a malformed MATCH is client input in hybrid mode"); + assert_eq!( + err.into_response().status(), + StatusCode::BAD_REQUEST, + "{malformed:?} must be a 400 in hybrid mode, as in keyword mode" + ); + } + } + #[tokio::test] async fn search_with_zero_limit_returns_400_never_500() { // API-02 / D-01: an out-of-bounds limit is CLIENT input — 400 with the From fb9c4634bf807922d5bc73a2959dd6be257def80 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:40:27 +0200 Subject: [PATCH 15/42] docs(04-04): complete hybrid search plan (summary) Co-Authored-By: Claude Opus 5.5 --- .../04-04-SUMMARY.md | 330 ++++++++++++++++++ 1 file changed, 330 insertions(+) create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-04-SUMMARY.md diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-04-SUMMARY.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-04-SUMMARY.md new file mode 100644 index 0000000..cdbf639 --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-04-SUMMARY.md @@ -0,0 +1,330 @@ +--- +phase: 04-memory-update-relations-hybrid-search +plan: 04 +subsystem: search +tags: [rust, sqlite, fts5, sqlite-vec, rrf, hybrid-search, decay, rmcp, axum, mcp] + +# Dependency graph +requires: + - phase: 04-memory-update-relations-hybrid-search + provides: "04-01: MAX_KNN_K = MAX_LIMIT * 4 (800), the 4x oversample at every limit. 04-02: update bumps decay, and the merge-patch schema hazard (default: null) to avoid. 04-03: expand_links on SearchArgs, plus the truncate -> bump returned ids -> expand order, with the flag captured before args move into the store" +provides: + - "Store::keyword_candidates(args): bm25-ordered pure-relevance keyword leg (ORDER BY bm25 ASC, id ASC), oversampled limit*4 capped at MAX_KNN_K, same filters and map_fts_query_error routing as search" + - "pub const RRF_K: f64 = 60.0 and pub fn rrf_fuse(&[Vec]) -> Vec<(i64, f64)> (1-based ranks, duplicate ids sum, total_cmp desc + ascending-id tie-break)" + - "MemoryService::hybrid_candidates: concurrent legs via tokio::join!, keyword-first error propagation, RRF fusion, multiplicative post-fusion decay applied once" + - "SearchMode::Hybrid (wire \"hybrid\"); SearchMode::Semantic removed (no producer left)" + - "crates/agent-memory-core/tests/hybrid.rs: 19 tests (fusion goldens, oversample recall on both legs, SEARCH-05 goldens, the full degrade decision table, bump scope, hybrid expansion)" + - "memory_search tool description and README search-mode table for 'hybrid'|'keyword'; 'Behavioral changes in v0.1.0' item 3 for the DIST-06 release notes" +affects: [05 export/import (search_mode values for any import/export tooling), 06 release notes (DIST-06 picks up README behavioral change 3), future SEARCH-06 request-level mode override] + +# Actuals (#2632): chars/4 over the added lines of the realized diff (8903ee6..HEAD, code commits only) +actuals: + tokens: 12977 + tasks: 2 + commits: 4 + +# Tech tracking +tech-stack: + added: [] + patterns: + - "Rank fusion over pure-relevance legs: each leg orders by relevance only, and decay is applied once, multiplicatively, after fusion" + - "Concurrent blocking legs: two spawn_blocking futures under tokio::join!, with errors checked in a fixed order (the input-parsing leg first) so classification is deterministic" + - "Degrade-path byte-compat test: call the public Store::search directly before the service search, then assert JSON equality of the whole envelope" + - "Rank-controlled fixtures: query vector = axis 0, each row a chosen cosine similarity on a private axis, bm25 order set by term frequency and doc length" + - "Leg-failure injection without data loss: ALTER TABLE ... RENAME the leg's table on a temp DB" + +key-files: + created: + - crates/agent-memory-core/tests/hybrid.rs + modified: + - crates/agent-memory-core/src/service.rs + - crates/agent-memory-core/src/store/mod.rs + - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory-core/tests/semantic.rs + - crates/agent-memory-core/tests/update.rs + - crates/agent-memory-core/tests/relations.rs + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - crates/agent-memory/Cargo.toml + - README.md + +key-decisions: + - "RRF discretion (CONTEXT): 1-based ranks; legs fused in the order [KNN, keyword]; descending total_cmp with ascending-id tie-break, both in rrf_fuse and after the decay blend" + - "Post-fusion decay is multiplicative (final = rrf_score x decay_score), applied once, pinned-aware; each view's wire decay_score is that same value" + - "keyword_candidates computes its own oversample (limit*4, cap MAX_KNN_K), exactly like knn_search, instead of taking a k argument, so no bare k crosses a function boundary (Pitfall 2)" + - "Leg errors: the keyword leg's result is checked before the KNN leg's. Only the keyword leg parses the query, so a malformed MATCH is always InvalidQuery in hybrid mode, the same classification as keyword mode (T-04-15)" + - "The keyword leg adds m.id ASC as a bm25 tie-break, for deterministic ranks" + - "rrf_fuse and RRF_K are pub so tests/hybrid.rs can unit-test them against hand-computed values" + - "The bin crate's dev-dependency enables core's test-clock feature, so the in-process MCP/REST tests use the real FakeEmbedder. cargo tree confirms it is absent from the normal build graph" + - "tests/rest.rs left untouched: it had no 'semantic' assertions, and the optional WR-03 harness ride-along was skipped because the suite stayed stable" + +patterns-established: + - "A new search leg must be a pure-relevance ordering, oversampled like knn_search, and must route MATCH errors through map_fts_query_error" + +requirements-completed: [SEARCH-04, SEARCH-05] + +coverage: + - id: D1 + description: "rrf_fuse matches hand-computed RRF: rank 2 in both legs (2/62) beats rank 1 in one leg (1/61); duplicate ids sum; ties break by ascending id regardless of hash order; empty legs give []; RRF_K == 60" + requirement: SEARCH-04 + verification: + - kind: unit + ref: "crates/agent-memory-core/tests/hybrid.rs#rrf_dual_leg_rank_two_beats_single_leg_rank_one / rrf_duplicate_ids_sum_their_contributions / rrf_ties_break_deterministically_by_ascending_id / rrf_of_empty_legs_is_empty / rrf_k_is_the_cormack_clarke_constant" + status: pass + human_judgment: false + - id: D2 + description: "With the embedder up, a semantic-only hit and a keyword-only hit both surface in fused order, labeled search_mode hybrid (wire string \"hybrid\")" + requirement: SEARCH-04 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#hybrid_surfaces_the_top_semantic_and_the_top_keyword_hit / hybrid_wire_label_is_lowercase_hybrid" + status: pass + human_judgment: false + - id: D3 + description: "Both legs are oversampled (limit*4): a keyword hit at KNN rank 5 and a KNN hit at keyword rank 3 (limit 2) each enter fusion and win. Mutation-verified: an un-oversampled keyword leg fails the test" + requirement: SEARCH-04 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#semantic_leg_is_oversampled_beyond_limit / keyword_leg_is_oversampled_beyond_limit" + status: pass + human_judgment: false + - id: D4 + description: "SEARCH-05 golden: at exactly equal fused RRF score the fresh memory strictly outranks the stale one, in both orientations and against the id tie-break. A stale pinned DECISION keeps its pinned advantage, and the window is chosen so double decay, ignored pinning or an additive blend would each flip the order. Mutation-verified for all three" + requirement: SEARCH-05 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#at_equal_fused_rank_the_fresh_memory_outranks_the_stale_one / a_stale_pinned_memory_keeps_its_pinned_advantage_and_decay_applies_once" + status: pass + human_judgment: false + - id: D5 + description: "Emptiness is not a fallback: an empty corpus, or filters that exclude everything, return [] with search_mode still hybrid" + requirement: SEARCH-04 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#empty_corpus_is_an_empty_hybrid_answer / filters_that_exclude_everything_stay_hybrid" + status: pass + human_judgment: false + - id: D6 + description: "Degrade rows: an embed error and an empty embed batch both return the v1.0 keyword path, with the whole JSON envelope byte-identical to a direct Store::search plus on-read decay, mode keyword" + requirement: SEARCH-04 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#embed_error_degrades_to_the_unchanged_v1_keyword_path / an_empty_embed_batch_degrades_to_the_unchanged_v1_keyword_path" + status: pass + human_judgment: false + - id: D7 + description: "Error taxonomy in hybrid mode: a malformed MATCH (unbalanced quote, dangling AND, bare '?') is InvalidQuery exactly as in keyword mode, and REST 400 / MCP invalid_params with a live embedder. A broken FTS or vec leg propagates MemoryError::Sqlite and never serves the surviving leg. Mutation-verified: swallowing the keyword error fails both tests" + requirement: SEARCH-04 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#a_malformed_match_is_invalid_query_in_hybrid_mode_exactly_as_in_keyword_mode / an_internal_error_in_either_leg_propagates_and_never_serves_the_other_leg" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#a_malformed_query_is_400_in_hybrid_mode_too_never_a_semantic_200" + status: pass + - kind: integration + ref: "crates/agent-memory/src/mcp.rs#memory_search_malformed_query_is_invalid_params_in_hybrid_mode" + status: pass + human_judgment: false + - id: D8 + description: "Bump scope: 20 rows that are candidates in both legs, limit 5, so exactly the 5 returned rows are bumped (once each) and 15 are untouched. On the hybrid path the expanded neighbor is never bumped and the flag-less envelope has no related member. Mutation-verified: bumping all candidates fails both tests" + requirement: SEARCH-05 + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#only_the_returned_ids_are_bumped_never_the_fusion_candidates / hybrid_expansion_runs_after_the_bump_and_never_bumps_neighbors" + status: pass + human_judgment: false + - id: D9 + description: "Transport wire contract: in-process REST and MCP search return \"hybrid\" with a live FakeEmbedder and \"keyword\" with the dead loopback embedder, with the identical two-key envelope. The memory_search description enumerates exactly 'hybrid'|'keyword' with k=60" + requirement: SEARCH-04 + verification: + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#search_reports_hybrid_with_a_live_embedder_and_keyword_with_a_dead_one" + status: pass + - kind: integration + ref: "crates/agent-memory/src/mcp.rs#memory_search_reports_hybrid_when_the_embedder_is_up_and_keyword_when_down / memory_search_description_enumerates_exactly_hybrid_and_keyword" + status: pass + human_judgment: false + - id: D10 + description: "README: search-mode table (hybrid/keyword, RRF k=60, sums, post-fusion decay), bump-scope, query-syntax and filter-bound notes, and Behavioral changes in v0.1.0 item 3 (semantic -> hybrid)" + requirement: SEARCH-04 + verification: [] + human_judgment: true + rationale: "Documentation clarity and accuracy is a reading judgment; no test asserts README prose" + - id: D11 + description: "Live Ollama end-to-end: a paraphrased memory_search returns hybrid with sensible fused ordering; stopping Ollama degrades the same query to keyword with an unchanged envelope" + requirement: SEARCH-04 + verification: [] + human_judgment: true + rationale: "Needs a real local Ollama (the 04-VALIDATION.md manual-only row, end-of-phase human check); FakeEmbedder covers the logic in CI" + +# Metrics +duration: 11min +completed: 2026-10-23 +status: complete +--- + +# Phase 4 Plan 04: Decay-aware hybrid search (RRF k=60) Summary + +**With the embedder up, `memory_search` now fuses two pure-relevance rankings, KNN by cosine distance and FTS5 by bm25.** Fusion is Reciprocal Rank Fusion: k = 60, 1-based ranks, and an id found by both rankings sums both contributions. Each fused score is then multiplied by the memory's pinned-aware decay score, exactly once, and the result is reported as `search_mode: "hybrid"`. With the embedder down, search takes the v1.0 keyword path, and its output is byte-identical to a direct `Store::search`. A malformed MATCH is a 400 / `invalid_params` in both modes, and only the returned ids are ever bumped. + +## Performance + +- **Duration:** ~11 min +- **Started:** 2026-10-23T18:27:36Z +- **Completed:** 2026-10-23T18:38:55Z +- **Tasks:** 2 / 2 +- **Files modified:** 11 (1 created, 10 modified) + +## Accomplishments + +- **Phase success criteria 3 and 4 hold.** An exact-keyword hit and a semantic paraphrase hit both surface. Agreement between the two rankings is rewarded (2/62 beats 1/61). At equal fused rank the fresher memory wins, and the envelope is unchanged in every mode. +- **Decay is applied once, by construction.** Both legs are pure relevance: `knn_search` as-is, and the new `keyword_candidates` with `ORDER BY bm25(memories_fts) ASC, m.id ASC` and no decay term. Decay enters only in `hybrid_candidates`, as `rrf_score × decay_score(...)`. The pinned golden test sits in a window where decay applied twice, pinning ignored, or an additive blend would each flip the order, and hand mutations of all three failed it. +- **Oversampling on both legs is proven.** Each leg fetches `limit × 4`, capped at `MAX_KNN_K` (800). Two fixtures place the winning row beyond `limit` in one leg; the oversampled leg brings it into fusion, and without oversampling it would lose the id tie-break. +- **The degrade decision table is test-locked row by row:** + - fuse → hybrid; + - both legs empty → `[]`, still hybrid; + - embed error or empty batch → v1.0 keyword output, byte-identical JSON; + - malformed MATCH → InvalidQuery, the same in both modes; + - internal error in either leg → `Sqlite`, never the other leg's rows. +- **The 04-03 carry-forward is honored.** The order is truncate, `spawn_bump(returned ids)`, then `expand_related`, and `expand_links` and `limit` are captured before `args` moves into the legs. The 20-row / limit-5 kill test bumps exactly 5 rows. On the hybrid path the expanded neighbor stays at `access_count 0`. +- **The wire and the docs agree.** The `memory_search` description reads `'hybrid'|'keyword'` with RRF k=60, the keyword degrade, FTS5 query syntax, the WR-01 filter-after-cut bound and returned-only bumps. The README has the search-mode table and "Behavioral changes in v0.1.0" item 3. No MCP param struct was touched, so the 04-02 `default: null` schema hazard does not arise. + +## Task Commits + +1. **Task 1: RRF fusion over two pure-relevance legs + SearchMode::Hybrid + post-fusion decay**: `024bcd4` (feat) +2. **Task 2: degrade decision table, error propagation, bump scope, wire/docs**: `7c63d0e` (test) + +**Plan metadata:** this SUMMARY commit, then a `docs(04-04)` state/roadmap commit. + +_TDD note:_ +- Task 1's RED was observed before implementation: `tests/hybrid.rs` failed to compile with 8 errors (missing `rrf_fuse`, `RRF_K`, `SearchMode::Hybrid`). +- Task 2's tests are characterization locks over behaviour Task 1 had already implemented, so they passed on first run. They were proven to bite by hand mutation instead: + - swallowing the keyword-leg error failed the InvalidQuery and internal-error tests; + - bumping every fusion candidate failed both bump tests. +- Task 1 was also mutation-checked: double decay, an additive blend, ignored pinning and an un-oversampled keyword leg each failed a test. +- All mutations were reverted. RED was not committed separately, because CLAUDE.md and the operator rules forbid committing failing tests (same as 04-01 to 04-03). + +## Files Created/Modified + +- `crates/agent-memory-core/tests/hybrid.rs` (new, 719 lines, 19 tests): + - RRF unit goldens; + - e2e fusion and the wire label; + - oversample recall on both legs; + - the SEARCH-05 equal-rank and pinned goldens; + - emptiness; + - the degrade table (embed error, empty batch, malformed MATCH, broken FTS leg, broken vec leg); + - bump scope and hybrid expansion. +- `crates/agent-memory-core/src/service.rs`: + - `RRF_K` and `rrf_fuse`; + - `SearchMode::Hybrid` replaces `Semantic`; + - the embed-OK arm now calls `hybrid_candidates`, then truncates, bumps and expands; + - doc updates. + - The `other =>` keyword arm is untouched. +- `crates/agent-memory-core/src/store/mod.rs`: the `keyword_candidates` trait method and its contract docs. +- `crates/agent-memory-core/src/store/sqlite.rs`: the `keyword_candidates` impl, a pure insertion after `fn search` (whose body is unchanged). +- `crates/agent-memory-core/tests/{semantic,update,relations}.rs`: `SearchMode::Semantic` → `SearchMode::Hybrid` (8 assertions). +- `crates/agent-memory/src/mcp.rs`: the `memory_search` description, `live_server` / `server_with`, and 3 tests. +- `crates/agent-memory/src/rest/handlers.rs`: `live_state` / `state_with` / `search_request`, and 2 tests. +- `crates/agent-memory/Cargo.toml`: the dev-dependency enabling core's `test-clock` feature. +- `README.md`: the feature blurb, the tools-table row, the `memory_search` mode table and notes, the Ollama section, the `memory_update` note, and behavioral change 3. + +## Decisions Made + +See `key-decisions` in the frontmatter. In short: +- ranks are 1-based, with an id tie-break everywhere; +- the decay blend is multiplicative and applied once; +- `keyword_candidates` owns its oversample; +- the keyword leg's error is checked first, for deterministic classification; +- the public `rrf_fuse` / `RRF_K` exist for tests; +- the `test-clock` dev feature supplies the real `FakeEmbedder` in transport tests. + +## Deviations from Plan + +### Auto-fixed Issues + +**1. [Rule 2 - Missing critical] Tests beyond the listed behaviours** +- **What was added:** + - the empty-embed-batch degrade row; + - a broken-leg internal-error row for each leg; + - a hybrid-path test that expansion never bumps neighbors (the 04-03 carry-forward); + - transport tests showing a malformed query is 400 / `invalid_params` with a live embedder; + - a test locking the tool description's exact mode enumeration. +- **Why:** these rows of the RESEARCH decision table and the acceptance criteria (400 / `invalid_params` at both transports) were otherwise only implied. +- **Commit:** 7c63d0e + +**2. [Rule 3 - Blocking] Test files outside the plan's file list** +- **Issue:** removing `SearchMode::Semantic` breaks every test that names it, including `tests/update.rs` and `tests/relations.rs`, which the plan did not list. The plan's "any transport tests" covered the intent. +- **Fix:** each assertion was changed to `SearchMode::Hybrid`. The result-content assertions were unchanged and still hold. +- **Commit:** 024bcd4 + +**3. [Rule 3 - Blocking] `crates/agent-memory/Cargo.toml` dev-dependency feature** +- **Issue:** the plan asks for a live, FakeEmbedder-backed state in the bin crate's in-process tests, but `FakeEmbedder` is gated behind core's `test-clock` feature. +- **Fix:** one `[dev-dependencies]` line re-declares the existing path dependency with `features = ["test-clock"]`. No new package. `cargo tree -e normal,features` shows `test-clock` absent from the normal graph. +- **Commit:** 7c63d0e + +**4. [Plan wording] `keyword_candidates(args)`, not `(args, k)`** +- The artifacts list sketches a `k` argument. The action text says to mirror `knn_search`'s oversample expression, and Pitfall 2 forbids passing a bare `k`, so the method computes `limit*4` capped at `MAX_KNN_K` internally, exactly as `knn_search` does. "Identical parameter positions" was read as "the same bound-parameter filter block": positions are `?1..?5`, because the `search` positions `?4..?9` carry weights that this leg does not have. +- **Commit:** 024bcd4 + +**5. [Environment] A project hook blocks `DROP` in Bash commands** +- `.claude/hooks/pre-bash-rust.sh` rejected the command that wrote the broken-leg test, because the test SQL contained `DROP TABLE`. Rather than route around the guard, the test breaks each leg non-destructively with `ALTER TABLE RENAME TO _offline` on its temp DB. +- **Commit:** 7c63d0e + +--- + +**Total deviations:** 3 auto-fixed (1 missing-critical, 2 blocking), plus 2 wording/environment adaptations. +**Impact on plan:** no scope creep, no new crates, no workflow changes. + +## Issues Encountered + +- **User-facing consequence of the locked T-04-15 rule (please read).** FTS5 rejects bare punctuation: `what is the plan?`, `don't` and `a, b` are all `fts5: syntax error` (checked against SQLite directly). Because hybrid now runs the keyword leg on every query and must not swallow its `InvalidQuery`, a natural-language query with punctuation that v0.0.1 answered semantically while Ollama was up is now a 400 / `invalid_params`. This is what the plan, the must-haves and the threat model lock, and the MCP `query` param was already documented as "FTS5 MATCH syntax". It is recorded in the tool description and in README behavioral change 3, and a test locks it. If agents routinely send punctuated questions, a follow-up should decide whether to sanitize the keyword leg's input (for example, quote each token). That would be a contract change to discuss, not a ride-along. +- The plan's "one warn logged" for the degrade row is not asserted, because the core crate has no tracing-capture dev-dependency. The warn sits in the untouched keyword arm. +- One stale word remains: a comment in the untouched keyword arm still says "Same order as the semantic leg". It was kept so the arm stays byte-identical, as the plan requires. + +## Verification Results + +- `cargo test -p agent-memory-core --test hybrid`: **19 passed**. +- `cargo test -p agent-memory --bins`: **53 passed** (48 before this plan). +- `cargo test --workspace`: **170 passed / 0 failed / 2 ignored** (04-03 baseline: 146 / 0 / 2). +- `cargo clippy --workspace --all-targets -- -D warnings`: clean, with both crates re-checked. +- `cargo fmt --all --check`: clean. +- Acceptance greps: + - `const RRF_K: f64 = 60.0` present; + - `fn keyword_candidates` present; + - `bm25(memories_fts) ASC` present, with no decay term in that ORDER BY; + - `SearchMode::Semantic` count 0; + - `'semantic'` in `mcp.rs` count 0; + - `hybrid` in the `memory_search` description; + - README behavioral change 3 present; + - the mode table lists `"hybrid"` and `"keyword"`; + - the `sqlite.rs` diff is one hunk inserted after `fn search` (`@@ -601,0 +602,43 @@`). +- Hand mutations (6), all reverted: double decay, additive blend, ignored pinning, un-oversampled keyword leg, swallowed keyword error, bumping all candidates. Each failed at least one test. + +## User Setup Required + +None. Optional end-of-phase manual check (the 04-VALIDATION.md manual-only row): with a local Ollama running, a paraphrased `memory_search` should report `"hybrid"`; stop Ollama and the same query should degrade to `"keyword"` with the same envelope. + +## Next Phase Readiness + +- **Phase 4:** all four plans are complete and the phase is ready for verification. The orchestrator marks it complete. +- **Phase 5 (export/import):** search is unaffected by import ids. Note that `search_mode` now has only two values. +- **Phase 6 (release notes, DIST-06):** README "Behavioral changes in v0.1.0" now has three items. Item 3 covers semantic → hybrid and the punctuation consequence. +- **Still open, outside this plan:** + - WR-01: full filter-aware KNN. Documented as a bound, not fixed. + - WR-03: the spawned-binary REST harness hang. Not touched; the suite was stable. + - The punctuation-query consequence above, as a possible follow-up decision. + +--- +*Phase: 04-memory-update-relations-hybrid-search* +*Completed: 2026-10-23* + +## Self-Check: PASSED + +- FOUND: crates/agent-memory-core/tests/hybrid.rs +- FOUND: crates/agent-memory-core/src/service.rs +- FOUND: crates/agent-memory-core/src/store/sqlite.rs +- FOUND: crates/agent-memory/src/mcp.rs +- FOUND: README.md +- FOUND: commit 024bcd4 +- FOUND: commit 7c63d0e From bff047fa97429fd5409450b4614a04849cf4f95d Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:40:58 +0200 Subject: [PATCH 16/42] docs(04-04): update state, roadmap progress, mark SEARCH-04/SEARCH-05 complete Co-Authored-By: Claude Opus 5.5 --- .planning/REQUIREMENTS.md | 8 ++++---- .planning/ROADMAP.md | 6 +++--- .planning/STATE.md | 20 ++++++++++++-------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md index 056bf31..128f285 100644 --- a/.planning/REQUIREMENTS.md +++ b/.planning/REQUIREMENTS.md @@ -10,8 +10,8 @@ Each maps to a roadmap phase. IDs continue from the v1.0 numbering (archived in ### Search (SEARCH) -- [ ] **SEARCH-04**: With Ollama available, `memory_search` fuses the keyword and semantic legs via reciprocal-rank fusion (rank-based, k=60, duplicate ids sum contributions from both legs); results carry `search_mode: "hybrid"`. With Ollama unreachable, search degrades to keyword exactly as in v1.0 — the envelope shape is unchanged and existing `search_mode == "keyword"` checks keep working -- [ ] **SEARCH-05**: Hybrid ranking remains decay-aware — a post-fusion decay blend so a fresh memory outranks a stale one at equal fused rank, and access bumps apply only to post-truncation returned ids +- [x] **SEARCH-04**: With Ollama available, `memory_search` fuses the keyword and semantic legs via reciprocal-rank fusion (rank-based, k=60, duplicate ids sum contributions from both legs); results carry `search_mode: "hybrid"`. With Ollama unreachable, search degrades to keyword exactly as in v1.0 — the envelope shape is unchanged and existing `search_mode == "keyword"` checks keep working +- [x] **SEARCH-05**: Hybrid ranking remains decay-aware — a post-fusion decay blend so a fresh memory outranks a stale one at equal fused rank, and access bumps apply only to post-truncation returned ids ### MCP Interface (MCP) @@ -67,8 +67,8 @@ Mapped by roadmap creation 2026-07-12. Coverage: 12/12 v1.1 requirements mapped | MCP-06 | Phase 4 | Complete | | MCP-07 | Phase 4 | Complete | | STORE-05 | Phase 4 | Complete | -| SEARCH-04 | Phase 4 | Pending | -| SEARCH-05 | Phase 4 | Pending | +| SEARCH-04 | Phase 4 | Complete | +| SEARCH-05 | Phase 4 | Complete | | DIST-04 | Phase 5 | Pending | | INTEROP-02 | Phase 5 | Pending | | DIST-05 | Phase 6 | Pending | diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 7b3e802..cd06496 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -67,7 +67,7 @@ Plans: 4. Hybrid ranking stays decay-aware — a fresh memory outranks a stale one at equal fused rank — and access bumps apply only to post-truncation returned ids. 5. Upgrading a real v0.0.1 database applies migration 0003 only after a pre-migration backup; shipped 0001/0002 SQL is provably unchanged (fixture divergence test against a v0.0.1 database), and a newer-schema DB met by an older binary yields a friendly error, not a crash. -**Plans**: 3/4 plans executed +**Plans**: 4/4 plans executed Plans: **Wave 1** @@ -83,7 +83,7 @@ Plans: **Wave 4** *(blocked on Wave 3 completion)* -- [ ] 04-04-PLAN.md — Decay-aware hybrid RRF fusion with keyword degrade, error taxonomy, and wire/docs (SEARCH-04, SEARCH-05) +- [x] 04-04-PLAN.md — Decay-aware hybrid RRF fusion with keyword degrade, error taxonomy, and wire/docs (SEARCH-04, SEARCH-05) ### Phase 5: Portable Export & Import @@ -124,6 +124,6 @@ Phases execute in numeric order: 3 → 4 → 5 → 6 | 1. Core Memory Foundation | v1.0 | 3/3 | Complete | 2026-06-25 | | 2. Semantic Search, Interop & Release | v1.0 | 5/5 | Complete | 2026-07-12 | | 3. API Hardening & Toolchain Spikes | v1.1 | 2/2 | Complete | 2026-07-12 | -| 4. Memory Update, Relations & Hybrid Search | v1.1 | 3/4 | In Progress| | +| 4. Memory Update, Relations & Hybrid Search | v1.1 | 4/4 | In Progress| | | 5. Portable Export & Import | v1.1 | 0/? | Not started | - | | 6. Distribution & v0.1.0 Release | v1.1 | 0/? | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index 41edc2a..b576898 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -4,17 +4,17 @@ milestone: v1.1 milestone_name: Hardening & Interop current_phase: 04 current_phase_name: Memory Update, Relations & Hybrid Search -status: executing -stopped_at: Completed 04-03-PLAN.md -last_updated: "2026-10-23T18:25:09.406Z" +status: verifying +stopped_at: Completed 04-04-PLAN.md +last_updated: "2026-10-23T18:40:45.705Z" last_activity: 2026-10-23 last_activity_desc: Phase 04 execution started -state_head: a82878198070304fd79df425fa413df1affc0685 +state_head: fb9c4634bf807922d5bc73a2959dd6be257def80 progress: total_phases: 4 completed_phases: 1 total_plans: 6 - completed_plans: 5 + completed_plans: 6 --- # Project State @@ -30,7 +30,7 @@ See: .planning/PROJECT.md (updated 2026-07-12) Phase: 04 (Memory Update, Relations & Hybrid Search) — EXECUTING Plan: 4 of 4 -Status: Ready to execute +Status: Phase complete — ready for verification Last activity: 2026-10-23 — Phase 04 execution started Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) @@ -71,6 +71,7 @@ Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) | Phase 04 P01 | 9 min | 2 tasks | 13 files | | Phase 04 P02 | 10min | 2 tasks | 11 files | | Phase 04 P03 | 10 min | 2 tasks | 9 files | +| Phase 04 P04 | 11 min | 2 tasks | 11 files | ## Accumulated Context @@ -108,6 +109,9 @@ Recent decisions affecting current work: - [Phase 04]: 04-03: forget() and sweep_expired() each carry an explicit orphan-edge DELETE beside the FK ON DELETE CASCADE, proven by pragma-less-connection tests - [Phase 04]: 04-03: expand_links attaches MemoryView.related (Option, skip_serializing_if) after truncation and after spawn_bump; MAX_EXPANDED_LINKS = 10 newest-first, snippet 160 chars; neighbors never bumped; flag-less envelope golden-tested byte-identical - [Phase 04]: 04-03: Incoming edges read natural reverse labels (superseded_by, caused); RelatedLink.direction is out|in; REST POST /api/links + DELETE /api/links/{from_id}/{to_id}/{kind} with 200/404 and 400 on any bad body/kind +- [Phase 04]: 04-04: Hybrid search = RRF (k=60, 1-based ranks, duplicate ids sum, id tie-break) over two pure-relevance legs (KNN distance, bm25 ASC), each oversampled limit*4 capped at MAX_KNN_K; decay applied once, multiplicatively, post-fusion (SEARCH-04/05) +- [Phase 04]: 04-04: SearchMode::Semantic removed; embedder-up searches report search_mode "hybrid", embedder-down stays "keyword" byte-identical to v1.0 (README behavioral change 3) +- [Phase 04]: 04-04: Keyword-leg errors propagate first, so a malformed MATCH (incl. bare ?/,/' punctuation) is InvalidQuery/400 in hybrid as in keyword mode (T-04-15); follow-up candidate: sanitize natural-language queries for the keyword leg ### Pending Todos @@ -148,8 +152,8 @@ Items acknowledged and carried forward: ## Session Continuity -Last session: 2026-10-23T18:25:02.213Z -Stopped at: Completed 04-03-PLAN.md +Last session: 2026-10-23T18:40:36.856Z +Stopped at: Completed 04-04-PLAN.md Resume file: None ## Operator Next Steps From 2be42f2f16c7c17d67882322e3b3e4d3f64561b4 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:48:57 +0200 Subject: [PATCH 17/42] docs(04): add code review report Co-Authored-By: Claude Opus 5.5 --- .../04-REVIEW.md | 367 ++++++++++++++++++ 1 file changed, 367 insertions(+) create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md new file mode 100644 index 0000000..1c0e4d5 --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md @@ -0,0 +1,367 @@ +--- +phase: 04-memory-update-relations-hybrid-search +reviewed: 2026-10-23T00:00:00Z +depth: standard +files_reviewed: 20 +files_reviewed_list: + - Cargo.toml + - README.md + - crates/agent-memory-core/sql/0003_relations.sql + - crates/agent-memory-core/src/domain.rs + - crates/agent-memory-core/src/service.rs + - crates/agent-memory-core/src/store/migrations.rs + - crates/agent-memory-core/src/store/mod.rs + - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory-core/tests/hybrid.rs + - crates/agent-memory-core/tests/import.rs + - crates/agent-memory-core/tests/migration_hygiene.rs + - crates/agent-memory-core/tests/relations.rs + - crates/agent-memory-core/tests/semantic.rs + - crates/agent-memory-core/tests/update.rs + - crates/agent-memory/Cargo.toml + - crates/agent-memory/src/main.rs + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - crates/agent-memory/src/rest/mod.rs + - crates/agent-memory/src/serde_util.rs +findings: + critical: 3 + warning: 4 + info: 6 + total: 13 +status: issues_found +--- + +# Phase 4: Code Review Report + +**Reviewed:** 2026-10-23 +**Depth:** standard (diff `907161c..HEAD`) +**Files Reviewed:** 20 (binary fixture `tests/fixtures/v0.0.1.db` skipped) +**Status:** issues_found + +## Summary + +Phase 4 adds `memory_update` (merge-patch with an in-transaction vector swap), flat typed +relations with 1-hop expansion, RRF hybrid search, and migration 0003 with a pre-migration +backup and a newer-schema refusal. The code is careful in many places. SQL is parameterized +everywhere. The `MemoryError` mappers are exhaustive. No `unwrap()` was added to production +code. Clippy (`-D warnings`, all targets) is clean and all `agent-memory-core` tests pass +(verified locally). + +The main problems come from making hybrid the default search path, and from content becoming +mutable: + +1. Hybrid sends the raw query to FTS5 on every search while Ollama is up. FTS5 reports + column-filter parse errors (`sqlite-vec`, `re-embed`, `note: x`) as `no such column`. The + classifier does not recognise that message, so these queries now return **HTTP 500 / + `internal_error`** on the primary path. v0.0.1 answered them. +2. Natural-language punctuation (`?`, `,`, `'`) now returns 400 whenever Ollama is up. The + brief asked for an explicit assessment of this. The verdict is a **regression, not an + acceptable trade-off** (CR-02). +3. Content can now change, so the sweep's embedding backfill can write a vector for *old* + content over a row that was updated meanwhile. That leaves a permanently wrong vector with + `embedding_status = 1`. + +The FTS5 behaviour behind CR-01/CR-02 was checked against the bundled SQLite (3.51.3, via +rusqlite 0.39): + +| query | FTS5 result | mapped to | tier | +|---|---|---|---| +| `what is it?` | `fts5: syntax error near "?"` | `InvalidQuery` | 400 / invalid_params | +| `sqlite-vec` | `no such column: vec` | `Sqlite` | **500 / internal_error** | +| `re-embed` | `no such column: embed` | `Sqlite` | **500 / internal_error** | +| `note: auth` | `no such column: note` | `Sqlite` | **500 / internal_error** | + +Known, already-tracked items (WR-01 filter-after-KNN-cut, WR-03 REST harness hang) are not +re-reported. + +## Narrative Findings (AI reviewer) + +## Critical Issues + +### CR-01: Hyphenated or colon-bearing queries are 500 / `internal_error` on the default (hybrid) search path + +**File:** `crates/agent-memory-core/src/store/sqlite.rs:333-340` (classifier), `crates/agent-memory-core/src/service.rs:427` (hybrid propagates the keyword leg's error), `crates/agent-memory/src/mcp.rs:310`, `README.md:165-170` +**Issue:** `map_fts_query_error` only treats `fts5: syntax error` and `unterminated string` as +client input. FTS5 parses `a-b` and `a: b` as column filters and fails at *step* time with +`no such column: b`. That error falls through to `MemoryError::Sqlite`, which both mappers send +to the internal tier. + +Before Phase 4, this only affected keyword-fallback mode (Ollama down). Now every search with +Ollama up runs `keyword_candidates`, and its error takes priority over the KNN leg +(service.rs:427). So an ordinary query such as `sqlite-vec`, `re-embed`, `pre-migration +backup`, `TODO: auth` or `x86-64`: + +- returned 200 (semantic) in v0.0.1; +- now returns HTTP 500 / MCP `internal_error`. + +This breaks the project's core contract that bad client input never surfaces as a 500 (02-05, +Phase 3 D-06). It also defeats agent error handling: an agent that sees `internal_error` assumes +a server fault and will not rephrase. The README (L165-170) and the `memory_search` tool +description promise `invalid_params` for queries FTS5 cannot parse, so both documents are wrong +for this class of input. No test covers it: `hybrid.rs:538-570` only exercises `"unbalanced`, +`AND` and `?`. + +**Fix:** Classify step-time column-filter failures as client input. The statement prepares +successfully, so a `no such column` at step time can only come from the MATCH expression. Then +add a regression row for `sqlite-vec` and `note: x` to the hybrid/keyword decision-table test. +```rust +fn map_fts_query_error(query: &str, e: rusqlite::Error) -> MemoryError { + let msg = e.to_string(); + if msg.contains("fts5: syntax error") + || msg.contains("unterminated string") + // FTS5 column filters (`a-b`, `col: x`) on the single-column + // memories_fts table: only reachable from the bound MATCH string, + // because the statement itself prepared successfully. + || msg.contains("no such column") + { + MemoryError::InvalidQuery(query.to_string()) + } else { + MemoryError::Sqlite(e) + } +} +``` +(CR-02's sanitising retry removes most of this input class altogether. This mapping is still +the required backstop.) + +### CR-02: Assessment of the requested behaviour change — natural-language queries now fail with the embedder up (regression) + +**File:** `crates/agent-memory-core/src/service.rs:244-265, 412-428`, `crates/agent-memory-core/src/store/sqlite.rs:606-647`, `crates/agent-memory-core/tests/hybrid.rs:538-570`, `README.md:165-170, 407-414` +**Issue:** With Ollama up, which is the recommended setup, every search now depends on the raw +query being a valid FTS5 expression. Queries like `what did we decide about auth?`, `don't +retry`, `auth, tokens`, `C++ build` and an empty string are rejected with 400 / +`invalid_params`. v0.0.1 answered all of them semantically. Including CR-01's inputs, this +covers most of what an LLM agent naturally sends: questions, sentences with commas or +apostrophes, and hyphenated technical terms. + +Severity: **BLOCKER**. The degradation is documented (README L407-414, the tool description), +but documenting it does not make it acceptable for an agent-facing tool. + +- **The semantic search value is lost.** The product's main promise is "semantic search via + local Ollama embeddings". In the configuration where that should work best, the query + embedding is now discarded whenever the query contains common punctuation. +- **The T-04-15 rationale does not hold.** The plan's justification (04-04-PLAN T-04-15) is + that answering semantically in one mode and with a 400 in the other would let a caller probe + which mode is active. But every response already reports `search_mode`, so there is no hidden + mode to probe. The second reason, the error taxonomy, only requires the *same* classification + in both modes. It does not require that classification to be an error. +- **Agents pay a retry cost.** An agent receiving `invalid_params` will rewrite and retry, which + costs tokens on every punctuated question. Agents rarely read the "use plain words" caveat in + the README. + +**Fix:** Keep exact FTS5 syntax available to power users, but stop rejecting natural language. +Apply the same logic in both modes so the T-04-15 "same classification everywhere" property +still holds: +```rust +// sqlite.rs: a fallback MATCH built from the query's word tokens, each +// phrase-quoted (inner quotes doubled) and OR-joined. It always parses. +fn sanitize_fts_query(raw: &str) -> Option { + let terms: Vec = raw + .split(|c: char| !c.is_alphanumeric() && c != '_') + .filter(|t| !t.is_empty()) + .map(|t| format!("\"{}\"", t.replace('"', "\"\""))) + .collect(); + (!terms.is_empty()).then(|| terms.join(" OR ")) +} +``` +In `keyword_candidates`, and in `search` for the fallback path: run the raw MATCH first. If it +fails with `InvalidQuery`, rerun once with `sanitize_fts_query(&args.query)`. If no tokens are +left, return an empty keyword leg so hybrid fuses KNN-only. Only report `InvalidQuery` when the +caller clearly asked for FTS syntax (for example an unbalanced `"`), or never. Then update +`a_malformed_match_is_invalid_query_in_hybrid_mode_exactly_as_in_keyword_mode`: `what is the +plan?` should succeed in both modes. If the owners still choose to keep the 400, record that as +an explicit decision in the CONTEXT decisions log, and fix CR-01 either way. + +### CR-03: The sweep backfill can overwrite an updated memory's vector with an embedding of its *old* content, permanently + +**File:** `crates/agent-memory-core/src/service.rs:612-637`, `crates/agent-memory-core/src/store/sqlite.rs:439-479`, `crates/agent-memory-core/src/store/sqlite.rs:812-836` +**Issue:** The backfill has three steps: + +1. It reads `(id, content)` for pending rows (`pending_embeddings`). +2. It `await`s a batch embed of up to 128 texts outside any lock. On a CPU Ollama that takes + seconds. +3. `insert_embedding` then runs an unconditional `DELETE` + `INSERT` of the vector and + `embedding_status = 1`. + +Before Phase 4, content was immutable, so this was safe. `memory_update` breaks that +assumption. Interleaving: + +1. Row 7 is pending (stored while Ollama was down). +2. Ollama recovers. The sweep reads `(7, "old text")` and starts embedding it. +3. A client PATCHes row 7 with `"new text"`. `update` embeds the new text successfully, swaps + the vector and sets status 1. +4. The sweep finishes and `insert_embedding(7, vec("old text"))` deletes the fresh vector, + writes the stale one, and sets status 1. + +Row 7 now matches its old meaning semantically, forever: status is 1, so no later sweep repairs +it. The same happens if the update's own embed fails (status 0, no vector), since the sweep +"repairs" it with the stale vector. The race is wider than it looks. `main.rs` documents that +several MCP stdio processes and the REST daemon share one DB file, each with its own sweep +task, while the writer `Mutex` is only per-process. + +Separately, a sweep racing `forget` re-inserts a vector for a deleted id. That vector can then +attach to a new memory that reuses the rowid (`INTEGER PRIMARY KEY` without `AUTOINCREMENT`). +That race already existed, but the fix below closes it too. + +**Fix:** Make the backfill write conditional on the row still being the pending row whose +content was embedded. Do it in the same writer transaction: +```rust +fn insert_embedding(&self, memory_id: i64, expected_content: &str, embedding: Vec) + -> Result { + let mut conn = self.writer.lock().map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; + let tx = conn.transaction()?; + let claimed = tx.execute( + "UPDATE memories SET embedding_status = 1 \ + WHERE id = ?1 AND embedding_status = 0 AND content = ?2", + params![memory_id, expected_content], + )?; + if claimed == 0 { + return Ok(false); // updated, already embedded, or forgotten: skip; tx rolls back + } + let blob: &[u8] = bytemuck::cast_slice(&embedding); + tx.execute("DELETE FROM vec_memories WHERE memory_id = ?1", params![memory_id])?; + tx.execute("INSERT INTO vec_memories(memory_id, embedding) VALUES (?1, ?2)", params![memory_id, blob])?; + tx.commit()?; + Ok(true) +} +``` +Pass the content from `pending_embeddings` through `sweep()`. Add a test: pending row, then +the sweep reads it, then `update` with new content, then the backfill completes. Assert that +KNN on the new content's vector finds the row and KNN on the old content's vector does not. + +## Warnings + +### WR-01: Concurrent first open after an upgrade can fail startup, or leave a "pre-0003" backup that is really schema v3 + +**File:** `crates/agent-memory-core/src/store/sqlite.rs:224-252`, `crates/agent-memory-core/src/store/sqlite.rs:140-189` +**Issue:** `open()` has three unlocked steps: probe `user_version`, write the backup, then +`to_latest()`. rusqlite_migration 2.5 reads `user_version` outside its deferred transaction +(`goto` → `goto_up`, lib.rs:713-651). 0003 is the first migration ever applied to deployed +v0.0.1 databases, and several processes share one DB by design (an MCP stdio server per editor +session plus the REST daemon). That makes simultaneous first opens after `brew upgrade` a +realistic scenario: + +- **Startup failure:** A and B both probe version 2. A migrates and commits. B's `CREATE TABLE + memory_links` then fails with `table memory_links already exists`, or `SQLITE_BUSY_SNAPSHOT`. + The transaction rolls back, but B's server fails to start. +- **Silently wrong backup:** B probes 2, A migrates to 3, and only then does B run + `pre_migration_backup`. B deletes A's valid `*.backup-pre-0003-*` file and writes a copy of + the *already-migrated* v3 database under the "pre-0003" name. B's `to_latest` is then a + no-op. There are no down-migrations (migrations.rs:6-8), so the backup is the only rollback + path, and it is now wrong. + +**Fix:** Run probe, backup and migrate under one exclusive lock, and re-read the version under +it. For example: `BEGIN IMMEDIATE` on the writer, re-read `user_version`, skip the backup if it +is no longer pending, then `COMMIT` and migrate. A cheaper alternative: write the backup to a +temp name, open it, check that its `user_version == db_version`, then rename it into place and +remove the stale backups. Treat a `to_latest` failure caused by the version having moved to +`LATEST` as success. + +### WR-02: Patch DTOs silently ignore non-patchable or misspelled fields + +**File:** `crates/agent-memory/src/mcp.rs:154-186`, `crates/agent-memory/src/rest/handlers.rs:101-127` +**Issue:** Neither `UpdateArgs` nor `UpdateBody` uses `deny_unknown_fields`. Two cases: + +- `{"content": "x", "type": "DECISION"}` returns 200 with the type unchanged. `type` and + `source` are documented as immutable, but the caller is told the update succeeded. +- A typo such as `{"content": "x", "ttl_sec": 60}` updates the content and silently drops the + TTL. + +With merge-patch semantics, a silently dropped member is a silent partial write. This violates +the seam's "bad input is always the client tier" rule. + +**Fix:** Add `#[serde(deny_unknown_fields)]` to both DTOs. On MCP, `id` is a declared field, +so it is fine. On REST, the explicit `JsonRejection` handling already turns this into a 400. +Add one test per transport for `{"content": "x", "type": "DECISION"}` returning +`invalid_params` / 400. + +### WR-03: `relates_to` is documented as symmetric but stored and deduplicated as directed + +**File:** `crates/agent-memory-core/src/store/sqlite.rs:710-714, 719-728, 856-905`, `crates/agent-memory-core/src/domain.rs:77` +**Issue:** The primary key is `(from_id, to_id, kind)`, so `link(a, b, relates_to)` and +`link(b, a, relates_to)` create two edges for one symmetric relation. Consequences: + +- Expansion lists the neighbor twice, once `out` and once `in`, and both copies count against + `MAX_EXPANDED_LINKS`. +- `unlink(a, b, relates_to)` reports `unlinked: true` while `b -> a` keeps the "symmetric" + relation alive. +- The D-02 idempotency promise ("duplicate link is a no-op") fails for the only kind whose + direction is meaningless. + +**Fix:** Canonicalise `relates_to` endpoints at the service seam: store `(min(a,b), max(a,b))` +for `link` and look it up the same way for `unlink`. Optionally, dedupe by neighbor id in +`related()`. Document that the direction shown for `relates_to` is not meaningful. + +### WR-04: `memory_list` with `expand_links` and no `limit` is unbounded, contradicting the documented T-04-11 bound + +**File:** `crates/agent-memory-core/src/service.rs:194-208`, `crates/agent-memory-core/src/domain.rs:316-321` +**Issue:** The `MAX_EXPANDED_LINKS` doc claims that, together with `MAX_LIMIT`, it "bounds an +expanded response at 200 × 10 summaries (T-04-11)". But `list` with an omitted `limit` stays +unlimited (Pitfall 6). An expanded list therefore returns every memory plus up to 10 neighbors +each, and runs one `related()` query per row inside a single `spawn_blocking`. So the threat +mitigation the plan relies on does not hold for `GET /api/memories?expand_links=true`. + +**Fix:** When `expand_links` is set and `limit` is `None`, either apply `DEFAULT_SEARCH_LIMIT` +(or `MAX_LIMIT`) or reject with `InvalidArgument` ("limit is required with expand_links"). +Correct the doc comment and the README accordingly. + +## Info + +### IN-01: The pre-migration backup and the newer-schema refusal are not documented for users + +**File:** `README.md` (no section) +**Issue:** The README never mentions `backup`, `migration` or `upgrade`. Users will find +`memories.db.backup-pre-0003-YYYYMMDD` files next to their DB with no explanation, no +restore procedure, and no note that a newer DB refuses an older binary (D-11 to D-13). +**Fix:** Add a short "Upgrading" section covering the backup name pattern, the retention rule, +how to restore (stop the servers, copy the backup over `memories.db`, delete `-wal`/`-shm`), +and the newer-schema error message. + +### IN-02: Path and Query extractor rejections on the new routes return plain-text bodies, not `{"error": ...}` + +**File:** `crates/agent-memory/src/rest/handlers.rs:287-291, 364-367, 209-211` +**Issue:** `PATCH /api/memories/abc`, `DELETE /api/links/x/2/relates_to` and +`GET /api/memories?expand_links=1` hit axum's default `Path`/`Query` rejection, which returns a +plain-text body. The new handlers deliberately normalise `JsonRejection` to `{"error"}`, so the +error body shape is now inconsistent between body and path failures. +**Fix:** Take `Result, PathRejection>` / `Result, QueryRejection>` the same way +`update_handler` takes the JSON body, and map them to `ApiError::BadRequest`. + +### IN-03: The backup step deletes the previous same-version backup before the new one is written + +**File:** `crates/agent-memory-core/src/store/sqlite.rs:156-182` +**Issue:** If the new backup then fails (disk full, permissions), the previous pre-0003 backup +is already gone. The DB itself stays untouched because the open aborts, so this is low risk, +but the retention step is not atomic. +**Fix:** Write to `{prefix}{date}.tmp`, then rename it into place, and only then remove the +other `{prefix}*` entries. + +### IN-04: Stale comment refers to a "semantic leg" that no longer exists + +**File:** `crates/agent-memory-core/src/service.rs:289` +**Issue:** "Same order as the semantic leg" — the semantic-only path was removed. The fallback +warnings at L270-275 still say "semantic search unavailable", which is fine. The comment should +say "the hybrid path". +**Fix:** Reword the comment. + +### IN-05: `memory_update` can revive a row whose TTL has passed but which the sweep has not yet removed + +**File:** `crates/agent-memory-core/src/store/sqlite.rs:747-791` +**Issue:** `update` does not check `expires_at < now`. A patch with `ttl_secs: null` (or a new +TTL) on an expired row that has not been swept yet brings it back. This matches list/search, +which also show expired-but-unswept rows, but it should be a deliberate choice. +**Fix:** Either treat `expires_at < now` as not-found in `update`, or document the behaviour. + +### IN-06: Snippet truncation counts Unicode scalars, not grapheme clusters + +**File:** `crates/agent-memory-core/src/domain.rs:192-197` +**Issue:** The cut never splits a UTF-8 code point, but it can split a combining sequence or a +ZWJ emoji, leaving a dangling modifier before the `…`. This is cosmetic. +**Fix:** Accept and document it, or cut at the last whitespace before the limit (D-05 leaves +the choice open). + +--- + +_Reviewed: 2026-10-23_ +_Reviewer: Claude (gsd-code-reviewer)_ +_Depth: standard_ From d3a45807175959099ab87977a233d96de7cfb8f0 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 20:50:34 +0200 Subject: [PATCH 18/42] docs(04): add phase verification report (gaps_found) Co-Authored-By: Claude Opus 5.5 --- .../04-VERIFICATION.md | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-VERIFICATION.md diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-VERIFICATION.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-VERIFICATION.md new file mode 100644 index 0000000..246d42f --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-VERIFICATION.md @@ -0,0 +1,181 @@ +--- +phase: 04-memory-update-relations-hybrid-search +verified: 2026-10-23T18:52:00Z +status: gaps_found +score: 13/15 must-haves verified +behavior_unverified: 0 +overrides_applied: 0 +gaps: + - truth: "A malformed MATCH query is a 400/invalid_params in hybrid mode exactly as in keyword mode — the keyword leg's InvalidQuery propagates (plan 04-04 must-have #3; Phase 3 taxonomy: client input is never a 500)" + status: failed + reason: >- + map_fts_query_error only recognises the substrings "fts5: syntax error" and "unterminated string". + FTS5 reports column-filter-shaped input (a hyphen or a colon: `sqlite-vec`, `auth: tokens`) as + "no such column: ", which falls through to MemoryError::Sqlite, so the result is MCP -32603 + internal_error / REST 500 and the raw SQLite message ("sqlite error: no such column: vec") leaks to the + client. Keyword mode has had this since v1.0, but this phase sends every Ollama-up search through the + keyword leg. v0.0.1 answered these queries semantically; HEAD returns a server fault. Observed live on + both v0.0.1 (/opt/homebrew/bin/agent-memory) and the HEAD binary with a real Ollama. + artifacts: + - path: "crates/agent-memory-core/src/store/sqlite.rs" + issue: "map_fts_query_error (line ~333) marker list misses the step-time 'no such column' FTS5 parse failure; it is used by keyword_candidates (hybrid leg) and search (keyword path)" + - path: "crates/agent-memory-core/tests/hybrid.rs" + issue: "a_malformed_match_is_invalid_query_in_hybrid_mode_exactly_as_in_keyword_mode covers only '\"unbalanced', 'AND', 'what is the plan?', with no hyphen or colon case" + missing: + - "Classify the step-time FTS5 'no such column' failure (the SQL is static, so at step time it can only come from the MATCH string) as MemoryError::InvalidQuery, or resolve it through the sanitisation option in the decision item below" + - "Add 'sqlite-vec' and 'auth: tokens' to the malformed-query table test for both hybrid and keyword modes, plus a REST and MCP assertion that the tier is 400 / invalid_params with no 'sqlite error' text in the body" +human_verification: + - test: "Decide the contract for natural-language queries when Ollama is up. Run memory_search with 'what vector store do we use?' (or any query containing ? , ' - :) against a store holding 'We chose sqlite-vec for local vector similarity search'" + expected: "Developer decision. Today it returns invalid_params (400). v0.0.1 returned the right memory semantically. This was a deliberate planner decision (T-04-15), documented in README 'Behavioral changes in v0.1.0' item 3, but it cuts against the phase outcome 'I find the right memory even when the phrasing differs'. Options: (a) accept it as is and record an override; (b) sanitise or quote the keyword leg's input (e.g. tokenise into quoted OR'd terms) so natural-language punctuation can never fail the leg, which also fixes the gap above; (c) serve the KNN leg alone on a keyword-leg parse error (the plan rejected this as mode-probing)" + why_human: "Two locked contracts conflict: the plan's T-04-15 classification parity against the roadmap goal's paraphrase-recall outcome. This is a product decision, not a code defect a grep can settle" +--- + +# Phase 4: Memory Update, Relations & Hybrid Search — Verification Report + +**Phase Goal:** As an agent, I want to update existing memories, link related ones, and get hybrid keyword+semantic recall, so that my memory stays current and connected and I find the right memory even when the phrasing differs. +**Verified:** 2026-10-23T18:52:00Z +**Status:** gaps_found +**Re-verification:** No, initial verification + +Evidence sources: +- Code read at HEAD `bff047f`, with a clean working tree under `crates/`. +- The phase test binaries (`update`, `relations`, `hybrid`, `migration_hygiene`, `import`): 56 passed, 0 failed, 1 ignored (the one-shot fixture generator). +- Orchestrator gates: fmt, clippy `-D warnings`, build, and `cargo test --workspace` (170 passed / 0 failed / 2 ignored). +- **Live sessions** of the HEAD binary over MCP stdio against a real local Ollama (`nomic-embed-text`) and against an unreachable URL. +- A **real v0.0.1 database** written by the installed Homebrew `agent-memory 0.0.1`, then upgraded by HEAD. + +All scratch data lives in `/tmp/v04live-data` and `/tmp/v04real-data`. Nothing in the repo was modified. + +## Goal Achievement + +### Observable Truths + +| # | Truth | Status | Evidence | +|---|-------|--------|----------| +| 1 | SC1: `memory_update` partial patch (content/tags/scope/TTL) changes the memory in place; a content update re-mirrors FTS and invalidates/re-embeds the vector in the same writer tx; an outage gives `embedding_status=0` plus sweep backfill; the old meaning stops matching; an unknown id gets a clean not-found | ✓ VERIFIED | `SqliteStore::update` (sqlite.rs:731-854) does one lock and one tx. It runs `UPDATE memories SET ...` in place (never DELETE+INSERT); the `memories_au` trigger re-mirrors FTS; `DELETE FROM vec_memories` plus a re-insert, or status 0, happens in the same tx. Tests pass: `content_update_re_embeds_so_the_old_meaning_stops_matching` (it uses 4 distractors and also checks that the FTS old word is gone), `embed_outage_during_update_falls_back_to_pending_and_sweep_backfills`, `updating_an_unknown_id_is_a_clean_not_found`. Live: the update returned the full view with `embedding_status:1`. A paraphrase query that ranked id 1 first fell to rank 3 after its content changed. id 999 returned `{"id":999,"reason":"not_found","updated":false}` | +| 2 | SC2: `memory_link`/`memory_unlink` handle flat typed relations; search and list can expand 1-hop; forget and TTL expiry cascade links with no orphans; no graph model | ✓ VERIFIED | 0003 `memory_links` has a PK of (from,to,kind), FK `ON DELETE CASCADE` on both columns, `CHECK from!=to`, and `idx_links_to`. `related()` is a single non-recursive 1-hop SQL query. Explicit orphan deletes exist in both `forget` and `sweep_expired` (lines 669, 956). There are 19 relations tests, including pragma-less orphan healing. Live: a link of 1→2 `supersedes` expanded with `superseded_by`/`in` on id 2. After `forget(1)`, the DB held 0 orphan edges | +| 3 | SC3: with Ollama available, search returns RRF-fused results (k=60, duplicate ids sum) labelled `hybrid`; with Ollama unreachable, it degrades to keyword exactly as v1.0 with an unchanged envelope | ✓ VERIFIED (fusion and degrade mechanics; see #15 for the query-class caveat) | `RRF_K = 60.0` and `rrf_fuse` (service.rs:78-105) use 1-based ranks with summed contributions. The goldens pass (2/62 beats 1/61, duplicate ids sum, id tie-break). Public `fn search` is **byte-identical** to the pre-phase `907161c` (diffed), and the service keyword arm differs only by opt-in expansion. Live with Ollama up: `search_mode: hybrid`, keys `[results, search_mode]`. Live with Ollama down on the same DB: `search_mode: keyword` with the same keys and one fallback warning per search | +| 4 | SC4: hybrid ranking is decay-aware (fresh beats stale at equal fused rank); access bumps go only to post-truncation returned ids | ✓ VERIFIED | Decay is multiplicative and applied once after fusion (service.rs:437-455). `at_equal_fused_rank_the_fresh_memory_outranks_the_stale_one` uses a genuine cross-rank tie (1/61 + 1/62 each), in both orientations, so id order cannot win. `only_the_returned_ids_are_bumped_never_the_fusion_candidates` uses 20 candidates in both legs at limit 5 and finds exactly the 5 returned rows bumped. `spawn_bump` is called after `take(limit)` and before expansion | +| 5 | SC5: upgrading a real v0.0.1 DB applies 0003 only after a pre-migration backup; 0001/0002 SQL provably unchanged; a newer-schema DB gets a friendly error, not a crash | ✓ VERIFIED | Probe, then refuse, then pragmas, then backup, then migrate (sqlite.rs:224-252). `git diff v0.0.1` on 0001/0002 is empty. The fixture is committed, and the divergence test compares normalized `sqlite_master`. **Live with the real v0.0.1 binary:** the DB written by Homebrew 0.0.1 was opened by HEAD and produced `real.db.backup-pre-0003-20261023` (user_version=2, 2 rows, no memory_links); the live DB went to user_version=3 with data intact, and hybrid search, link and update all worked. **Live too-new:** `user_version=99` gave exit 1, 0 bytes on stdout, and the D-13 message on stderr naming 99, 3 and the brew command. The DB's shasum was unchanged | +| 6 | Re-importing a `source: None` draft is idempotent (CR-01) | ✓ VERIFIED | `source IS ?1` (sqlite.rs:496); `.as_deref()` in service import; test `reimporting_a_sourceless_draft_is_idempotent` passes | +| 7 | A poisoned writer lock is 500/internal_error, never 404 (WR-02) | ✓ VERIFIED | 9 `Internal("writer lock poisoned")` sites, including the new link/unlink/update writers. `MemoryError::NotFound`/`ApiError::NotFound` have 0 references. Mapper tier unit tests pass (full suite) | +| 8 | Merge-patch semantics: absent = unchanged, null = clear (ttl/tags/scope), content null or empty = 400 | ✓ VERIFIED | `validate_patch` runs before the embed. Matrix tests at core (`tags_and_scope_patch_matrix_absent_null_value`, `ttl_patch_matrix_...`) and on both transports from raw JSON (mcp.rs:669/701/753, handlers.rs PATCH tests). Patch DTOs set `skip_serializing_if`, so the schema carries no `default: null` | +| 9 | The update response is the full post-update view plus `embedding_status`, and the update bumps `last_accessed` | ✓ VERIFIED | `UpdatedMemory` flattens `MemoryView` and adds `embedding_status`. The UPDATE statement folds in `last_accessed = now, access_count + 1`. Tests: `update_bumps_freshness_by_exactly_one_access`, `update_returns_the_full_post_update_view_with_embedding_status`. Live response included `embedding_status` | +| 10 | Link edge cases: a duplicate is an idempotent no-op; a self-link is 400; an unknown kind is 400 listing the allowed values; a missing endpoint is a not-found naming the id | ✓ VERIFIED | `INSERT OR IGNORE`; endpoint pre-check inside the tx (no FK-violation 500); `validate_link_endpoints` at the seam. Relations tests and MCP/REST transport tests pass | +| 11 | Expansion covers both directions with natural reverse labels and a cap of `MAX_EXPANDED_LINKS = 10`, most recent first | ✓ VERIFIED | `related()` uses `ORDER BY l.created_at DESC ... LIMIT ?2`. Tests: `expansion_is_capped_and_the_most_recent_edges_win`, `expansion_follows_one_edge_both_ways_with_natural_labels`. Live: `supersedes`/`out` and `superseded_by`/`in` | +| 12 | With `expand_links` absent, the search and list JSON is byte-identical to v1.0 (no `related` key) | ✓ VERIFIED | `#[serde(skip_serializing_if = "Option::is_none")]` on `related`. Golden test `flag_less_envelopes_are_byte_identical_to_v1_and_never_mention_related`. Live: flag-less results carried no `related` | +| 13 | Expanded neighbors never receive access bumps | ✓ VERIFIED | Expansion runs after `spawn_bump(returned ids)`. Tests `expanded_neighbors_are_never_access_bumped` and `hybrid_expansion_runs_after_the_bump_and_never_bumps_neighbors` pass | +| 14 | A malformed MATCH is 400/invalid_params in hybrid mode exactly as in keyword mode (plan 04-04 must-have) | ✗ FAILED | Holds for the `fts5: syntax error` / `unterminated string` class only. **Live HEAD with Ollama up:** `memory_search {"query":"sqlite-vec"}` returned `-32603 internal_error "sqlite error: no such column: vec"`. The REST mapper sends `MemoryError::Sqlite` to 500. Classification parity with keyword mode holds (both modes give 500), but the 400 tier does not, and the SQL detail leaks. **Live v0.0.1 with Ollama up:** the same query returned the right memory (`search_mode: semantic`), so on the Ollama-up path this is a regression | +| 15 | Goal outcome: "I find the right memory even when the phrasing differs" (derived from the user story's so-that clause) | ? UNCERTAIN (decision needed) | Paraphrases without punctuation work live (for example, "which datastore keeps client information" finds "Postgres is our primary relational database for customer records" at rank 1 with no keyword overlap). Paraphrases with ordinary punctuation fail: `what vector store do we use?` gives invalid_params at HEAD, while v0.0.1 found the right memory. This is deliberate (T-04-15) and documented in the README, but it conflicts with the outcome clause. See Human Verification | + +**Score:** 13/15 truths verified (0 present-but-behavior-unverified, 1 failed, 1 uncertain) + +### Required Artifacts + +| Artifact | Expected | Status | Details | +|----------|----------|--------|---------| +| `crates/agent-memory-core/sql/0003_relations.sql` | memory_links schema | ✓ VERIFIED | PK(from,to,kind), 2× ON DELETE CASCADE, CHECK, WITHOUT ROWID, idx_links_to | +| `crates/agent-memory-core/src/store/migrations.rs` | 0003 in chain + `LATEST_SCHEMA_VERSION` | ✓ VERIFIED | `= 3`, with a test that the const equals what `to_latest` writes | +| `crates/agent-memory-core/tests/migration_hygiene.rs` | divergence/backup/too-new tests | ✓ VERIFIED | 357 lines, 7 tests plus an ignored generator; the generator refuses to overwrite an existing fixture | +| `crates/agent-memory-core/tests/fixtures/v0.0.1.db` | committed fixture | ✓ VERIFIED | git-tracked, user_version=2, unmodified in the working tree | +| `crates/agent-memory-core/src/domain.rs` | Internal, SchemaTooNew, MemoryPatch, UpdatedMemory, LinkKind, RelatedLink, MAX_EXPANDED_LINKS | ✓ VERIFIED | all present and used | +| `crates/agent-memory-core/tests/update.rs` | MCP-06 tests | ✓ VERIFIED | 674 lines, 9 tests | +| `crates/agent-memory/src/serde_util.rs` | explicit-null helper | ✓ VERIFIED | used by UpdateArgs and UpdateBody | +| `crates/agent-memory-core/tests/relations.rs` | MCP-07 tests | ✓ VERIFIED | 737 lines, 19 tests | +| `crates/agent-memory-core/tests/hybrid.rs` | SEARCH-04/05 tests | ✓ VERIFIED (coverage hole: no hyphen or colon query) | 719 lines, 19 tests | +| `crates/agent-memory-core/src/store/sqlite.rs` | update / link / related / keyword_candidates / backup | ✓ VERIFIED (except the `map_fts_query_error` gap) | see truths 1, 2, 5, 14 | + +### Key Link Verification + +| From | To | Via | Status | Details | +|------|----|-----|--------|---------| +| migrations.rs | 0003_relations.sql | `M::up(include_str!)` | ✓ WIRED | line 24 | +| sqlite.rs open() | rusqlite Backup API | `run_to_completion` | ✓ WIRED | line 181; feature `backup` on the pinned 0.39 | +| sqlite.rs open() | PRAGMA user_version | probe before pragmas and migrations | ✓ WIRED | line 233 | +| service.rs import | Store::exists | `draft.source.as_deref()` | ✓ WIRED | line 338 | +| mcp.rs | MemoryService::update/link/unlink | tool arms | ✓ WIRED | mcp.rs:374/400/438; exercised live | +| rest/mod.rs | update/link/unlink handlers | PATCH `/api/memories/{id}`, POST `/api/links`, DELETE `/api/links/{from}/{to}/{kind}` | ✓ WIRED | rest/mod.rs:77-85 | +| service.rs | Store::related | `expand_related` after the bump | ✓ WIRED | search (both arms) and list | +| sqlite.rs forget/sweep | memory_links | belt-and-braces orphan DELETE | ✓ WIRED | lines 669, 956 | +| service.rs | Store::keyword_candidates + knn_search | `tokio::join!`, keyword error checked first | ✓ WIRED | lines 423-428 | +| service.rs | decay_score | post-fusion multiplicative blend | ✓ WIRED | line 443 | +| sqlite.rs keyword_candidates | map_fts_query_error | InvalidQuery propagation | ⚠️ PARTIAL | wired, but the mapper misclassifies the `no such column` FTS5 failure (gap 1) | + +### Data-Flow Trace (Level 4) + +| Artifact | Data Variable | Source | Produces Real Data | Status | +|----------|---------------|--------|--------------------|--------| +| `SearchOutcome.results` (hybrid) | fused views | `vec_memories` KNN plus `memories_fts` bm25, read pool | yes (live: real rows ranked) | ✓ FLOWING | +| `MemoryView.related` | RelatedLink list | `memory_links JOIN memories` | yes (live) | ✓ FLOWING | +| `UpdatedMemory` | post-update row | re-read inside the writer tx | yes (live) | ✓ FLOWING | + +### Behavioral Spot-Checks + +| Behavior | Command | Result | Status | +|----------|---------|--------|--------| +| Phase test binaries | `cargo test -p agent-memory-core --test update --test relations --test hybrid --test migration_hygiene --test import` | 56 passed, 0 failed, 1 ignored | ✓ PASS | +| Live hybrid with a real Ollama (executor human-check, part 1) | MCP stdio `serve` against `/tmp/v04live-data/live.db`, Ollama at localhost:11434 | paraphrase gives `search_mode: hybrid`, correct memory at rank 1 | ✓ PASS | +| Live degrade (executor human-check, part 2) | same DB, `--ollama-url http://127.0.0.1:9` | `search_mode: keyword`, envelope keys identical, one fallback WARN per search | ✓ PASS | +| Real v0.0.1 upgrade | Homebrew 0.0.1 writes 2 rows, then HEAD opens | backup user_version=2 with 2 rows; live DB user_version=3; link/update/hybrid OK | ✓ PASS | +| Too-new refusal through the binary | `--db future.db` (user_version=99) | exit 1, empty stdout, D-13 message on stderr, shasum unchanged | ✓ PASS | +| Hyphenated query, Ollama up | `memory_search {"query":"sqlite-vec"}` | `-32603 sqlite error: no such column: vec` (v0.0.1: correct semantic hit) | ✗ FAIL | +| `?` query, Ollama up | `memory_search {"query":"what vector store do we use?"}` | `-32602 invalid_params` (v0.0.1: correct semantic hit) | ? decision | +| Normal build excludes test-clock | `cargo tree -p agent-memory -e normal -i agent-memory-core -f '{p} {f}'` | no features | ✓ PASS | + +The executor's end-of-phase manual check (`` in plan 04-04 Task 2) was performed by the verifier against a real local Ollama. The hybrid and degrade observations above satisfy it. + +### Probe Execution + +None declared. No `scripts/*/tests/probe-*.sh` exists, and no phase artifact references a probe. + +### Requirements Coverage + +| Requirement | Source Plan | Description | Status | Evidence | +|-------------|-------------|-------------|--------|----------| +| STORE-05 | 04-01 | 0003 with migration hygiene (frozen SQL, backup, newer-DB error) | ✓ SATISFIED | truth 5, live real-v0.0.1 upgrade | +| MCP-06 | 04-02 | memory_update, same-tx re-mirror and re-embed, outage fallback, not-found | ✓ SATISFIED | truths 1, 8, 9 | +| MCP-07 | 04-03 | link/unlink, 1-hop expansion, cascade, no graph model | ✓ SATISFIED | truths 2, 10-13 | +| SEARCH-04 | 04-04 | RRF k=60 hybrid, keyword degrade unchanged | ⚠️ SATISFIED WITH GAP | fusion and degrade verified (truth 3); hybrid now exposes the pre-existing 500 misclassification (truth 14) and the punctuation decision (truth 15) | +| SEARCH-05 | 04-04 | post-fusion decay, bumps on returned ids only | ✓ SATISFIED | truth 4 | + +There are no orphaned requirements. REQUIREMENTS.md maps exactly these 5 IDs to Phase 4, and each is claimed by a plan. + +### Anti-Patterns Found + +| File | Line | Pattern | Severity | Impact | +|------|------|---------|----------|--------| +| crates/agent-memory-core/src/store/sqlite.rs | ~333 | FTS5 error classifier matches message substrings and misses `no such column` | 🛑 Blocker | client input reaches the 500 tier with SQL detail on the now-default Ollama-up path (gap 1) | + +The scan of all 19 phase-modified files found no unreferenced TBD/FIXME/XXX markers, no TODO/HACK/placeholder text, and no `unwrap()`/`expect()` in non-test source. + +### Human Verification Required + +#### 1. Natural-language punctuation in hybrid queries (decision) + +**Test:** With Ollama up, run `memory_search` with `what vector store do we use?` against a store containing "We chose sqlite-vec for local vector similarity search". +**Expected:** A developer decision on the contract. Today the query returns invalid_params; v0.0.1 found the memory. Option (b), sanitising or quoting the keyword leg's input, would restore paraphrase recall and also close gap 1. Option (a) needs an override entry. +**Why human:** Plan decision T-04-15 (identical classification in every mode, so the mode cannot be probed) conflicts with the roadmap outcome clause. That is a product call. + +### Gaps Summary + +The phase is substantively delivered. Update, relations, RRF fusion, decay blending, bump scoping and migration hygiene are all real, wired, test-locked, and confirmed live. That includes an upgrade of a database written by the real v0.0.1 binary and a live hybrid-to-keyword degrade against a real Ollama. + +One gap blocks a clean pass, and it shares a root cause with the decision item. Hybrid search now always parses the query through FTS5, even when Ollama is up. Two consequences follow: + +1. **(Gap, fixable in one line plus tests.)** Hyphen- or colon-shaped queries (`sqlite-vec`, `auth: tokens`) produce FTS5's `no such column` error. `map_fts_query_error` does not recognise it, so it surfaces as a 500 / internal_error with the SQLite message. In v0.0.1 these queries worked whenever Ollama was up. +2. **(Decision.)** Queries with `?`, `,` or `'` are classified correctly as client errors, but that means ordinary paraphrased questions no longer return results with Ollama up. + +Sanitising the keyword leg's input would resolve both at once. If the developer judges gap 1 pre-existing and out of scope, add this override to the frontmatter: + +```yaml +overrides: + - must_have: "A malformed MATCH query is a 400/invalid_params in hybrid mode exactly as in keyword mode" + reason: "no-such-column misclassification is pre-existing v1.0 keyword-path behavior; tracked for a follow-up" + accepted_by: "" + accepted_at: "" +``` + +--- + +_Verified: 2026-10-23T18:52:00Z_ +_Verifier: Claude (gsd-verifier)_ From 6756749f886775f75b794e7561af46890a113e92 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 21:37:25 +0200 Subject: [PATCH 19/42] docs(04): gap-closure plans 04-05..04-07 (option-a search, backfill race, review fixes) Co-Authored-By: Claude Opus 5.5 --- .planning/ROADMAP.md | 8 +- .../04-05-PLAN.md | 538 ++++++++++++++++++ .../04-06-PLAN.md | 408 +++++++++++++ .../04-07-PLAN.md | 452 +++++++++++++++ 4 files changed, 1405 insertions(+), 1 deletion(-) create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-05-PLAN.md create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-06-PLAN.md create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-07-PLAN.md diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index cd06496..9f33a23 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -67,7 +67,7 @@ Plans: 4. Hybrid ranking stays decay-aware — a fresh memory outranks a stale one at equal fused rank — and access bumps apply only to post-truncation returned ids. 5. Upgrading a real v0.0.1 database applies migration 0003 only after a pre-migration backup; shipped 0001/0002 SQL is provably unchanged (fixture divergence test against a v0.0.1 database), and a newer-schema DB met by an older binary yields a friendly error, not a crash. -**Plans**: 4/4 plans executed +**Plans**: 7 plans (4 executed + 3 gap closure) Plans: **Wave 1** @@ -85,6 +85,12 @@ Plans: - [x] 04-04-PLAN.md — Decay-aware hybrid RRF fusion with keyword degrade, error taxonomy, and wire/docs (SEARCH-04, SEARCH-05) +**Gap closure** *(from 04-VERIFICATION.md + 04-REVIEW.md; gap waves run in order)* + +- [ ] 04-05-PLAN.md — Gap wave 1: forgiving keyword leg per user decision option-a (raw MATCH first, one sanitized OR-of-quoted-tokens retry, tokenless = empty leg, identical in both modes) + search-path error redaction (CR-01, CR-02, truths #14/#15, IN-04) (SEARCH-04) +- [ ] 04-06-PLAN.md — Gap wave 2: conditional sweep backfill closing the update/forget/sweep races + symmetric relates_to + IN-05/IN-06 docs (CR-03, WR-03) (MCP-06, MCP-07) +- [ ] 04-07-PLAN.md — Gap wave 3: deny_unknown_fields on update DTOs, uniform REST rejection bodies, race-safe verify-then-rename pre-migration backup + Upgrading docs (WR-02, IN-02, WR-01, IN-03, IN-01) (MCP-06, MCP-07, STORE-05) + ### Phase 5: Portable Export & Import **Goal**: As a user, I want to export all my memories to a portable file and import them on another machine, so that my agent's memory moves with me — across machines, with no cloud sync. diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-05-PLAN.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-05-PLAN.md new file mode 100644 index 0000000..f9a999d --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-05-PLAN.md @@ -0,0 +1,538 @@ +--- +phase: 04-memory-update-relations-hybrid-search +plan: 05 +type: execute +wave: 1 +depends_on: [] +files_modified: + - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory-core/src/store/mod.rs + - crates/agent-memory-core/src/service.rs + - crates/agent-memory-core/src/domain.rs + - crates/agent-memory-core/tests/hybrid.rs + - crates/agent-memory-core/tests/fallback.rs + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - crates/agent-memory/tests/rest.rs + - crates/agent-memory/tests/tools.rs + - README.md + - .planning/STATE.md +autonomous: true +gap_closure: true +requirements: [SEARCH-04] + +estimate: + tokens: 11000 + raw_tokens: 11000 + tasks: 3 + confidence: low + +must_haves: + truths: + - "A natural-language query with punctuation (what vector store do we use?, don't retry, auth, tokens, C++) succeeds in hybrid AND keyword mode and returns the matching memory first — never invalid_params (closes VERIFICATION truth #15 and REVIEW CR-02, per the 2026-10-23 user decision option-a)" + - "A hyphenated or colon-bearing query (sqlite-vec, auth: tokens, re-embed) succeeds in both modes — never a 500 / internal_error (closes VERIFICATION gap truth #14 and REVIEW CR-01)" + - "Valid structured FTS5 syntax (sqlite AND vec, \"exact phrase\", prefix vec*) still runs exactly as written, because the raw query always runs first" + - "A query with no word tokens (empty string, a lone \", punctuation only such as ?!, or *) adds nothing from the keyword leg: hybrid answers from the semantic leg alone, keyword mode returns an empty result set — never an error" + - "The keyword-leg classification is identical with Ollama up or down: both modes share one store helper, so the T-04-15 parity goal holds by making both modes forgiving rather than both failing" + - "No MCP or REST search response body ever carries raw SQLite text: a residual or internal search failure surfaces as the generic internal error SEARCH_STORAGE_FAILURE (REST 500 / MCP internal_error), with the detail only in the stderr log" + artifacts: + - path: "crates/agent-memory-core/src/store/sqlite.rs" + provides: "raw-first FTS5 execution with one sanitized OR-of-quoted-tokens retry, shared by the keyword fallback path and the hybrid keyword leg" + contains: "fn match_with_sanitized_retry" + - path: "crates/agent-memory-core/src/service.rs" + provides: "single redaction point for every search-path error" + contains: "fn redact_search_error" + - path: "crates/agent-memory-core/tests/hybrid.rs" + provides: "two-mode natural-language decision table, raw-first structured-syntax proofs, redaction rows" + contains: "queries_fts5_cannot_parse_succeed_identically_in_both_modes" + - path: "README.md" + provides: "search query contract and Behavioral changes in v0.1.0 item 3 rewritten for the forgiving keyword leg" + contains: "never rejected" + key_links: + - from: "crates/agent-memory-core/src/store/sqlite.rs fn search" + to: "match_with_sanitized_retry" + via: "keyword-degrade path (embedder down)" + pattern: "match_with_sanitized_retry\\(" + - from: "crates/agent-memory-core/src/store/sqlite.rs fn keyword_candidates" + to: "match_with_sanitized_retry" + via: "hybrid keyword leg (embedder up) — same helper, so classification is mode-independent" + pattern: "match_with_sanitized_retry\\(" + - from: "crates/agent-memory-core/src/service.rs pub async fn search" + to: "redact_search_error" + via: "map_err on the whole search (both arms, expansion included)" + pattern: "map_err\\(redact_search_error\\)" +--- + +# Plan 04-05: Gap Closure — Forgiving keyword leg (natural-language queries never fail search) + +## Phase Goal + +**As an** agent, **I want to** update existing memories, link related ones, and get hybrid keyword+semantic recall, **so that** my memory stays current and connected and **I find the right memory even when the phrasing differs**. + +This plan closes the search slice of that story. Today, with Ollama up, every query must parse as FTS5: a question mark makes search fail with invalid_params, and a hyphen (`sqlite-vec`) makes it fail with a 500 that leaks SQLite text. After this plan, an agent can ask in plain language, with any punctuation, in either mode, and get the right memory back. + + +Implement the locked user decision **option-a (sanitize & retry)** at the one store seam both search paths share, and redact every search-path failure at the service seam. + +- The raw query runs first, so valid structured FTS5 syntax keeps its meaning. +- If FTS5 rejects the raw query as client input, the query is retried ONCE, with its word tokens quoted and OR-joined. The client-input classes are `fts5: syntax error`, `unterminated string`, the step-time `no such column` class from hyphens and colons, and the three further FTS5 parse classes found during planning (`unknown special query`, `expected integer`, `fts5: parser stack overflow`). +- A query with no word tokens adds nothing from the keyword leg. +- Any failure left over after the retry, and every internal search error, reaches the client only as a generic internal message. + +Purpose: restore paraphrase recall for punctuated agent questions (VERIFICATION truth #15), remove the hyphen/colon 500 and its SQLite-text leak (truth #14, CR-01), and keep T-04-15's "same classification in every mode" property (CR-02). +Output: a forgiving keyword leg, a search-path redaction point, the old 400-pinning tests reversed, two-mode decision tables at the core and on both transports, and docs that match. + + + +@$HOME/.claude/gsd-core/workflows/execute-plan.md +@$HOME/.claude/gsd-core/templates/summary.md + + + +@.planning/STATE.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-CONTEXT.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-VERIFICATION.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-04-SUMMARY.md +@crates/agent-memory-core/src/store/sqlite.rs +@crates/agent-memory-core/src/service.rs +@crates/agent-memory-core/tests/hybrid.rs + + +## Locked user decision (2026-10-23, resolves VERIFICATION human_verification #1 and REVIEW CR-02) + +**option-a — sanitize & retry.** + +1. The raw query runs through FTS5 first. +2. If FTS5 rejects it as client input, the query is retried ONCE with its tokens quoted and OR-joined. +3. If nothing tokenizes, the keyword leg contributes nothing. Hybrid returns semantic-only results; keyword mode returns an empty set, not an error. +4. The classification is identical with Ollama up or down. +5. The step-time `no such column` 500 is eliminated. +6. A raw SQLite message never reaches a client on any search path. +7. The tests that pin the old 400 behaviour are reversed. + +RESEARCH's degrade-table row "InvalidQuery from keyword leg → propagate → 400 in ALL modes" (04-RESEARCH.md §Pattern 1) and the 04-04 must-have "A malformed MATCH query is a 400/invalid_params in hybrid mode" are **superseded** by this decision. Honor this decision, not those. + +## Interface contracts (established by this plan) + +Private helpers in `crates/agent-memory-core/src/store/sqlite.rs`: + +- `fn is_match_input_error(e: &rusqlite::Error) -> bool` + - True only when both hold: + - the error is `rusqlite::Error::SqliteFailure` with primary result code `SQLITE_ERROR` (`extended_code & 0xff == rusqlite::ffi::SQLITE_ERROR`); + - the message contains one of the FTS5 query-parse markers: `fts5: ` (covers `fts5: syntax error near …` and `fts5: parser stack overflow`), `unterminated string`, `no such column`, `unknown special query`, `expected integer`. + - It is only ever handed errors raised AFTER the static statement prepared, so a `no such column` there can only come from FTS5 parsing a column filter out of the MATCH string. +- `fn sanitize_fts_query(raw: &str) -> Option` + - Splits on every char that is neither `char::is_alphanumeric` nor `_`. + - Drops empty pieces and wraps each piece in double quotes. Any embedded `"` is doubled defensively, though none can survive the split. + - Joins the pieces with ` OR `, preserving order, with no dedupe and no cap. + - Returns `None` when no piece remains. +- `fn match_with_sanitized_retry(raw: &str, run: impl FnMut(&str) -> Result, rusqlite::Error>) -> Result, MemoryError>` + - `run(raw)` succeeds → its rows. + - `run(raw)` fails with `is_match_input_error`, then: + - `sanitize_fts_query(raw)` is `None` → `Ok(vec![])`; + - `sanitize_fts_query(raw)` is `Some(f)` → `run(f)`, and ANY error from that retry is `MemoryError::Sqlite`. That is a server fault: the retry parses by construction. + - `run(raw)` fails with any other error → `MemoryError::Sqlite`. + +Public in `crates/agent-memory-core/src/service.rs`: + +- `pub const SEARCH_STORAGE_FAILURE: &str = "search failed due to a storage error (details are in the server log)"`. +- `fn redact_search_error(e: MemoryError) -> MemoryError`. This is the only place search errors are tiered for the wire. + +Probe results that drove the marker list, from the system sqlite3 3.46 CLI against `fts5(content)`. The bundled 3.51.3 is the source of truth, and Task 2's unit test pins it. + +| query | FTS5 result | +|---|---| +| `what vector store do we use?`, `don't retry`, `auth, tokens`, `C++`, `AND`, empty, `?!,`, `a.b`, `C#` | `fts5: syntax error near …` | +| `sqlite-vec`, `auth: tokens`, `re-embed`, `x86-64`, `TODO: auth`, `-a` | `no such column: …` | +| `"unbalanced`, `"` | `unterminated string` | +| `*`, `*a` | `unknown special query: …` | +| `NEAR(a b, x)` | `expected integer, got "x"` | +| 3000-deep `(`…`)` | `fts5: parser stack overflow` | +| `"exact phrase"`, `sqlite AND vec`, `vec*`, `content: tokens`, `""` | valid (runs raw) | + +A 20,000-term quoted OR chain parses and runs in about 0.2 s, so the retry needs no token cap. + +**Shared test corpus.** Every decision-table test in this plan uses the same five PATTERN memories, so expectations are identical across layers: + +- M1 `We chose sqlite-vec for local vector similarity search` +- M2 `auth tokens rotate every hour` +- M3 `C++ build flags need care` +- M4 `don't retry the import blindly` +- M5 `the unbalanced ledger is reconciled nightly` + +**Decision table.** + +| query | expected top hit, keyword AND hybrid mode | +|---|---| +| `what vector store do we use?` | M1 | +| `sqlite-vec` | M1 | +| `don't retry` | M4 | +| `auth, tokens` | M2 | +| `auth: tokens` | M2 | +| `C++` | M3 | +| `"unbalanced` | M5 | + +Tokenless or no-hit queries: `AND`, the empty string, `?!,` and `*`. + +- In keyword mode they return an empty result set. `AND` sanitizes to the phrase "and", which no corpus row contains. +- In hybrid mode they return a non-empty, semantic-only result set. + +Every row is `Ok`, carries the mode's `search_mode`, and has no `sqlite error` text in any response body. + +## Artifacts this phase produces + +- `fn is_match_input_error`, `fn sanitize_fts_query`, `fn match_with_sanitized_retry` (private, `store/sqlite.rs`). They replace the marker-only classifier that missed `no such column`. +- `pub const SEARCH_STORAGE_FAILURE` and `fn redact_search_error` (`service.rs`). The body of `pub async fn search` moves into a private `async fn search_unredacted`. +- Rewritten docs: + - `Store::search`, `Store::keyword_candidates` and `MemoryService::search` / `hybrid_candidates`; + - the `SearchArgs` query docs (core + MCP); + - the `MemoryError::InvalidQuery` variant (kept, documented as no longer produced by search); + - the `memory_search` tool description; + - README search section and Behavioral changes item 3; + - a STATE.md decision entry. +- Reversed tests: + - hybrid.rs malformed test → `queries_fts5_cannot_parse_succeed_identically_in_both_modes`; + - fallback.rs → `a_lone_double_quote_is_an_empty_keyword_answer_never_an_error`; + - mcp.rs → `memory_search_answers_queries_fts5_cannot_parse_in_hybrid_mode`; + - handlers.rs → `search_with_a_lone_quote_returns_200_with_an_empty_keyword_answer` and `queries_fts5_cannot_parse_are_200_in_hybrid_mode_too`; + - tests/rest.rs keyword-mode `"` block → 200. +- New tests: + - the core two-mode table, raw-first proofs, and classifier/sanitizer unit tests; + - MCP + REST tables and redaction tests; + - a stdio end-to-end case in tests/tools.rs. + + + + + Task 1 (tracer): sqlite-vec and a punctuated question answered end-to-end in both modes, with no SQLite text on any search failure + crates/agent-memory-core/src/store/sqlite.rs, crates/agent-memory-core/src/store/mod.rs, crates/agent-memory-core/src/service.rs, crates/agent-memory-core/tests/hybrid.rs, crates/agent-memory-core/tests/fallback.rs, crates/agent-memory/src/mcp.rs, crates/agent-memory/src/rest/handlers.rs, crates/agent-memory/tests/rest.rs + + - crates/agent-memory-core/src/store/sqlite.rs: + - lines 320-340: the marker classifier being replaced; + - lines 535-647: `fn search` and `fn keyword_candidates`. Both SQL strings stay byte-identical; only the execution around them changes. + - crates/agent-memory-core/src/store/mod.rs lines 70-97 (Store::search / keyword_candidates docs) + - crates/agent-memory-core/src/service.rs: + - lines 41-54: SearchArgs doc; + - lines 210-303: `pub async fn search`, including the IN-04 stale comment at ~289; + - lines 390-456: `hybrid_candidates` and its keyword-first error ordering. + - crates/agent-memory-core/tests/hybrid.rs: + - lines 538-599: the malformed test and the broken-leg internal-error test, both of which flip here; + - the harness at lines 36-121. + - crates/agent-memory-core/tests/fallback.rs lines 195-225 (keyword-mode `"` test that flips here) + - crates/agent-memory/src/mcp.rs lines 1062-1080 (hybrid malformed test that flips here) + - crates/agent-memory/src/rest/handlers.rs lines 618-736 (two malformed-query tests that flip here; search_request helper) + - crates/agent-memory/tests/rest.rs lines 200-220 (spawned-binary `"` → 400 block that flips here; do NOT touch the harness) + - this plan's "Locked user decision" and "Interface contracts" sections + + + - A store holding M1, searched with `sqlite-vec` and with `what vector store do we use?`: + - hybrid (live FakeEmbedder) → Ok, `SearchMode::Hybrid`, M1 first; + - keyword (FakeEmbedder::failing) → Ok, `SearchMode::Keyword`, M1 first. + - The pre-existing inputs `"unbalanced`, `AND`, `what is the plan?` → Ok in both modes, with the mode's search_mode. No InvalidQuery. + - A lone `"` in keyword mode → Ok, Keyword, empty results. + - With `memories_fts` or `vec_memories` renamed away (the existing broken-leg fixture), hybrid search fails with `MemoryError::Internal(m)` where `m == SEARCH_STORAGE_FAILURE`, and the error's Display contains no `sqlite` (case-insensitive). + - MCP in-process (live_server): `"`, `what is the plan?` and `sqlite-vec` → Ok, `search_mode` hybrid, and the tool text contains no `sqlite error`. + - REST in-process: + - `"` with the dead embedder → 200, keyword, empty results; + - `"`, `what is the plan?` and `sqlite-vec` with the live embedder → 200, hybrid, and no `sqlite error` in the body. + - Spawned REST binary (keyword mode): POST /api/search with `"` and with `sqlite-vec` → 200 with a `results` array, and no `sqlite error` in the response text. + + + This is the atomic contract flip. Per the locked user decision (option-a), the implementation and the reversal of every test pinning the old 400 contract must land in ONE commit, because CLAUDE.md forbids committing failing tests. That is why this task touches 8 files: 5 of them get assertion-level edits only. + + 1. **sqlite.rs.** Add the three private helpers exactly as specified in "Interface contracts". + - `is_match_input_error` reads the SQLite message from the `SqliteFailure` detail string, falling back to the error's Display. + - Its doc comment states the invariant: it only sees errors raised after a successful prepare of a static statement, so `no such column` there comes from an FTS5 column filter in the MATCH string on the single-column `memories_fts(content)` table. + - `match_with_sanitized_retry` logs one `tracing::debug!` line when it falls back. Log the raw query's char count, not its text. + - Rewire `fn search` and `fn keyword_candidates`: + - Keep the `conn.prepare(...)?` call and its SQL string untouched. Prepare-time failures stay internal and are never retried. + - Replace the `query_map` plus per-row error-mapping loop with a closure that binds the MATCH string it is given at `?1` (every other bound parameter unchanged) and collects the mapped rows into `Result, rusqlite::Error>`. + - Pass that closure and `&args.query` to `match_with_sanitized_retry`. + - Delete the old marker-only classifier and its doc comment. Nothing else may call it, and clippy `-D warnings` rejects an unused private fn. + - Rewrite the two call-site comments to describe the revised T-04-15: both modes forgiving, raw first. + 2. **store/mod.rs.** Rewrite the `Store::search` and `Store::keyword_candidates` docs. A natural-language query never fails either method: + - the raw MATCH runs first; + - FTS5-unparseable client input gets one sanitized retry; + - a tokenless query is an empty vec; + - neither method returns `MemoryError::InvalidQuery`; + - only server faults are errors. + 3. **service.rs.** + - Add `SEARCH_STORAGE_FAILURE` and `redact_search_error` with an exhaustive match (no `_` arm, per CLAUDE.md): + - `InvalidType`, `InvalidQuery`, `InvalidArgument` pass through unchanged (client tier; their text echoes only client input and public bounds); + - `Internal` and `SchemaTooNew` pass through unchanged (crate-authored text); + - `Sqlite`, `Pool`, `Join`, `Migration` → `tracing::error!` with the original error as a field, then `MemoryError::Internal(SEARCH_STORAGE_FAILURE.to_string())`. + - Move the current body of `pub async fn search` verbatim into a new private `async fn search_unredacted(&self, args: SearchArgs) -> Result`. + - `search` now awaits it and applies `.map_err(redact_search_error)`. + - Update the `search`, `hybrid_candidates` and `SearchArgs` docs (query = plain natural language or FTS5 MATCH syntax; T-04-15 revised). Keep the keyword-leg-first error ordering in `hybrid_candidates`: it still makes internal-error propagation deterministic. + - IN-04: reword the stale "Same order as the semantic leg" comment in the keyword arm to refer to the hybrid path. + 4. **Flip the old-contract tests** (assertion-level edits; keep each test's harness). + - **hybrid.rs** + - Replace `a_malformed_match_is_invalid_query_in_hybrid_mode_exactly_as_in_keyword_mode` with `queries_fts5_cannot_parse_succeed_identically_in_both_modes`. Keep its two-mode loop and the three existing inputs, and add `sqlite-vec` and `auth: tokens` (the VERIFICATION gap rows). Each search is Ok, with the mode's search_mode. Task 2 grows this into the full table. + - In `an_internal_error_in_either_leg_propagates_and_never_serves_the_other_leg`, assert `MemoryError::Internal` carrying exactly `SEARCH_STORAGE_FAILURE`, with no `sqlite` in the Display. Import `SEARCH_STORAGE_FAILURE` from `agent_memory_core::service`. + - **fallback.rs:** rename and flip the keyword-mode `"` test as in ``. + - **mcp.rs:** rename and flip the hybrid malformed test as in ``. + - **handlers.rs:** rename and flip both malformed-query tests as in ``. + - **tests/rest.rs:** flip the keyword-mode `"` block to 200 with a `results` array, and add a `sqlite-vec` POST asserting 200 and no `sqlite error` in the text. Leave the spawned-server harness untouched (the 03-REVIEW WR-03 hang fix stays deferred). + 5. Run `cargo fmt --all`. Commit with explicit paths only (no `git add -A`). + + + cargo test -p agent-memory-core --test hybrid --test fallback && cargo test -p agent-memory && cargo clippy --workspace --all-targets -- -D warnings + + + - These greps are each non-empty: + - `grep -n "fn match_with_sanitized_retry" crates/agent-memory-core/src/store/sqlite.rs` + - `grep -n "fn sanitize_fts_query" crates/agent-memory-core/src/store/sqlite.rs` + - `grep -n "fn is_match_input_error" crates/agent-memory-core/src/store/sqlite.rs` + - `grep -c "match_with_sanitized_retry(" crates/agent-memory-core/src/store/sqlite.rs` is at least 3: the definition plus the `search` and `keyword_candidates` call sites. + - Both SQL strings are intact: + - `grep -c "ORDER BY bm25(memories_fts) ASC, m.id ASC" crates/agent-memory-core/src/store/sqlite.rs` outputs 1; + - `grep -c "LIMIT ?9" crates/agent-memory-core/src/store/sqlite.rs` outputs 1. + - Both of these are non-empty: + - `grep -n "fn redact_search_error" crates/agent-memory-core/src/service.rs` + - `grep -n "map_err(redact_search_error)" crates/agent-memory-core/src/service.rs` + - `cargo test -p agent-memory-core --test hybrid queries_fts5_cannot_parse_succeed_identically_in_both_modes` passes, and so do `an_internal_error_in_either_leg_propagates_and_never_serves_the_other_leg` and `fallback a_lone_double_quote_is_an_empty_keyword_answer_never_an_error`. + - `cargo test -p agent-memory` is green. That covers the flipped mcp.rs / handlers.rs tests and the spawned tests/rest.rs. + - `cargo clippy --workspace --all-targets -- -D warnings` and `cargo fmt --all --check` are clean. + - No `unwrap()` was added to non-test code. + + The tracer path works end to end. `sqlite-vec`, a punctuated question and a lone quote are all answered in both modes, through both transports and the spawned REST binary. A broken leg yields only the generic internal message. + + + + Task 2: Lock the full two-mode decision table, raw-first structured syntax, and the classifier against the bundled SQLite + crates/agent-memory-core/src/store/sqlite.rs, crates/agent-memory-core/tests/hybrid.rs, crates/agent-memory-core/src/domain.rs + + - crates/agent-memory-core/src/store/sqlite.rs: + - the three helpers from Task 1; + - the existing `#[cfg(test)] mod tests` at the end of the file. + - crates/agent-memory-core/tests/hybrid.rs: + - Task 1's `queries_fts5_cannot_parse_succeed_identically_in_both_modes`; + - the broken-leg test; + - the `harness`, `live`, `query` and `raw_conn` helpers. + - crates/agent-memory-core/src/domain.rs lines 326-335 (MemoryError::InvalidQuery) + - this plan's "Interface contracts" probe table, shared corpus and decision table + + + - **Unit (sqlite.rs mod tests)**, on an in-memory `CREATE VIRTUAL TABLE t USING fts5(content)` through rusqlite: + - every probe query that ERRORS in the bundled SQLite surfaces a step-time error that `is_match_input_error` accepts. The candidate erroring probes are the empty string, `what is it?`, `"unbalanced`, `AND`, `sqlite-vec`, `auth: tokens`, `*`, `NEAR(a b, x)`, and a 3000-deep nested-paren query. + - The probe table came from the sqlite3 3.46 CLI, but the project bundles SQLite 3.51.3, and the bundled engine is the source of truth. If a listed candidate (for example the empty string, `MATCH ''`) runs WITHOUT error in the bundled SQLite, move it to the "valid (runs raw)" group in the pinning test, beside `"exact phrase"`, `sqlite AND vec`, `vec*`, `content: tokens` and `""`. That group asserts `Ok`. + - Record each moved probe in a comment naming the bundled version. Its two-mode expectation in the decision table must still hold: Ok result, the mode's search_mode, and no `sqlite error` text. Only the reason changes: it runs raw instead of being retried. + - for every one of those probes that sanitizes to `Some`, the sanitized string runs without error. This proves the fallback always parses. + - **Unit, classifier negatives:** a constructed `SqliteFailure` with code SQLITE_BUSY, a constructed SQLITE_ERROR carrying `malformed JSON`, and a `FromSqlConversionFailure` are all rejected by `is_match_input_error`. + - **Unit, `sanitize_fts_query` table:** + + | input | output | + |---|---| + | `what vector store do we use?` | `"what" OR "vector" OR "store" OR "do" OR "we" OR "use"` | + | `don't retry` | `"don" OR "t" OR "retry"` | + | `C++` | `"C"` | + | `sqlite-vec` | `"sqlite" OR "vec"` | + | `auth: tokens` | `"auth" OR "tokens"` | + | `"unbalanced` | `"unbalanced"` | + | `AND` | `"AND"` | + | `snake_case` | `"snake_case"` | + | `über 日本語` | `"über" OR "日本語"` | + | empty, `?!,`, `*` | `None` | + + - **Core two-mode table (hybrid.rs).** The shared corpus M1-M5 runs in both modes: + - every targeted row returns its target first, in BOTH modes; + - `AND`, empty, `?!,` and `*` are an empty Keyword answer, and a non-empty Hybrid answer; + - every row is Ok. + - **Raw-first proofs (keyword mode).** Corpus: + - S1 `sqlite only here` + - S2 `sqlite with vec support` + - S3 `an exact phrase lives here` + - S4 `phrase before exact here` + - S5 `vector math notes` + + Expected results: + + | query | raw result | a sanitized retry would give | + |---|---|---| + | `sqlite AND vec` | exactly [S2] | S1+S2 | + | `"exact phrase"` | exactly [S3] | S3+S4 | + | `vec*` | exactly {S2, S5} | S2 only | + + - **Redaction in keyword mode:** with `memories_fts` renamed away and FakeEmbedder::failing, search is `MemoryError::Internal(SEARCH_STORAGE_FAILURE)`, with no `sqlite` in the Display. + + + 1. **sqlite.rs `mod tests`:** add the classifier, sanitizer and fallback-parses tests from ``. Build the step-time errors by preparing `SELECT count(*) FROM t WHERE t MATCH ?1` once, then calling `query_row` with each probe bound. + - If a probe's bundled-SQLite message differs from the CLI table, extend the marker list in `is_match_input_error` ONLY with an FTS5 query-parse message. Never add a generic SQLite message. + - If a candidate erroring probe runs WITHOUT error in the bundled SQLite, move it to the valid-raw group instead (see ``). Do not delete it, and do not loosen its decision-table expectation. + - Structure the pinning test as two explicit lists, `erroring_probes` and `valid_raw_probes`. The erroring list is non-empty and every entry is classified; every valid-raw entry runs `Ok`. Together they cover all 9 candidates plus the 5 valid-raw queries. + - Construct the negative errors with `rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(code), Some(msg))`, the same constructor `register_vec_extension` already uses. + 2. **hybrid.rs:** + - Grow `queries_fts5_cannot_parse_succeed_identically_in_both_modes` into the full decision table over the shared corpus M1-M5, all stored as PATTERN at one TestClock instant so decay is equal. Assert top hit and search_mode per mode, with a per-row message naming the query and mode. + - Add `structured_fts5_syntax_still_runs_raw` for the raw-first proofs, in keyword mode. + - Add a keyword-mode row to the broken-leg redaction test, or a sibling test named `a_broken_fts_index_in_keyword_mode_is_the_generic_internal_error`. + 3. **domain.rs:** add a doc comment on `MemoryError::InvalidQuery`. Since v0.1.0, search never produces it: the keyword leg retries FTS5-unparseable input and treats tokenless input as no keyword matches. The variant is kept so both transport mappers stay exhaustive and the client-tier vocabulary is stable. No behaviour change. + 4. Run `cargo fmt --all` and commit with explicit paths. + + + cargo test -p agent-memory-core --lib store::sqlite && cargo test -p agent-memory-core --test hybrid && cargo clippy -p agent-memory-core --all-targets -- -D warnings + + + - `cargo test -p agent-memory-core --lib store::sqlite` runs the new classifier, sanitizer and fallback-parses unit tests green against the bundled SQLite. Every entry of `erroring_probes` (however many of the 9 candidates actually error in 3.51.3) is accepted by `is_match_input_error`, and every one that sanitizes to `Some` parses. Every entry of `valid_raw_probes` runs `Ok`. Both lists together account for all 9 candidates plus the 5 valid-raw queries, with none dropped. + - Any candidate moved to the valid-raw group is named in a comment, with the bundled SQLite version, in both the unit test and SUMMARY.md. + - `cargo test -p agent-memory-core --test hybrid structured_fts5_syntax_still_runs_raw` passes, with exact id sets asserted for all three structured queries. + - `cargo test -p agent-memory-core --test hybrid queries_fts5_cannot_parse_succeed_identically_in_both_modes` passes, asserting all 11 rows in both modes (7 targeted plus 4 tokenless/no-hit). + - The keyword-mode broken-FTS redaction assertion passes. + - `grep -n "InvalidQuery" crates/agent-memory-core/src/domain.rs` shows the new doc comment directly above the variant. + - clippy `-D warnings` and fmt are clean. + + The bundled SQLite's actual parse-error classes are pinned. The fallback is proven always to parse. Structured syntax provably runs raw. Every row of the user's case list holds identically in both modes at the core seam. + + + + Task 3: Agents and REST clients see the forgiving contract — transport tables, stdio end-to-end, tool description, README, STATE decision + crates/agent-memory/src/mcp.rs, crates/agent-memory/src/rest/handlers.rs, crates/agent-memory/tests/tools.rs, README.md, .planning/STATE.md + + - crates/agent-memory/src/mcp.rs: + - lines 108-134: SearchArgs `query` doc, which becomes the tool schema text; + - line 310: the memory_search description; + - lines 495-560: test helpers `test_server` / `live_server` / `server_with` (DB at `dir/mem.db`) and `result_json`; + - lines 1029-1097: search tests and the description test. + - crates/agent-memory/src/rest/handlers.rs: + - lines 430-490: test helpers (`test_state` / `live_state` / `state_with`, DB at `dir/mem.db`; `body_json`); + - lines 662-736: `search_request` and the Task 1 flipped tests. + - crates/agent-memory/tests/tools.rs: + - lines 1-120: the Server harness, `call_tool`, `tool_text`; + - lines 219-296: `search_outcome` and the existing search test. + - README.md: + - lines 106-120: tools table; + - lines 141-177: the memory_search section, including the "The query is FTS5 MATCH syntax in both modes" bullet; + - lines 343-372: the REST error contract; + - lines 394-417: Behavioral changes in v0.1.0 item 3. + - .planning/STATE.md lines 78-115 (Decisions list format) + + + - **MCP in-process table:** the shared corpus M1-M5 × {live_server (hybrid), test_server (keyword)} × the decision table. + - Every call is Ok, with the mode's search_mode. + - Targeted rows have the target id at results[0]. + - The tool text never contains `sqlite error`. + - **REST in-process table:** the same matrix through `search_handler`, asserting status 200 plus the same checks on the body. + - **MCP redaction:** with `memories_fts` renamed on `dir/mem.db` (via a raw rusqlite connection after `agent_memory_core::store::sqlite::register_vec_extension()`), memory_search returns Err with the internal_error code. Its message contains `search failed due to a storage error` and no `sqlite` (case-insensitive). + - **REST redaction:** the same fixture gives 500 with an `{"error"}` body containing `search failed due to a storage error` and no `sqlite`. + - **Stdio end-to-end (tests/tools.rs, the real binary, keyword mode):** + - memory_store M1; + - memory_search `sqlite-vec` and memory_search `what vector store do we use?` each return `result` (no `error` member), `search_mode` keyword, and M1's id; + - the raw response JSON text contains no `sqlite error`. + - **Description test:** + - the memory_search description still contains `{search_mode: 'hybrid'|'keyword', results` and `k=60`; + - it now also contains `plain natural language` and `never rejected`; + - it no longer contains `invalid_params`. + + + 1. **mcp.rs.** + - Rewrite the memory_search tool description sentence about query syntax. It now says: + - the query may be plain natural language or FTS5 MATCH syntax; + - valid FTS5 expressions (AND/OR/NOT, quoted phrases, prefix*) run exactly as written; + - anything FTS5 cannot parse is retried once as its words OR-ed together; + - a query with no words contributes no keyword matches; + - a query is never rejected for its syntax, in either mode. + - Keep the existing mode enumeration, k=60, filter-bound, bump-scope and expand_links sentences. + - Update the `SearchArgs.query` doc comment to match (it is the schema text agents read). + - Add the in-process table test `memory_search_answers_every_query_shape_in_both_modes` and the redaction test `memory_search_storage_failure_is_internal_error_without_sqlite_text`. + - Extend `memory_search_description_enumerates_exactly_hybrid_and_keyword` with the new positive and negative substring assertions. + 2. **handlers.rs:** add `search_answers_every_query_shape_in_both_modes` and `search_storage_failure_is_500_without_sqlite_text`, per ``. Reuse `live_state` / `test_state` and `search_request`. + 3. **tests/tools.rs:** add `memory_search_answers_hyphenated_and_punctuated_queries_over_stdio`, using the existing Server harness, `call_tool` and `search_outcome`. Use fresh request ids. + 4. **README.md.** + - Replace the "The query is FTS5 MATCH syntax in both modes" bullet with a "Plain language or FTS5 syntax — a query is never rejected" bullet stating: + - the raw-first rule; + - the one sanitized retry (words quoted and OR-ed); + - the tokenless rule (hybrid answers from the semantic ranking, keyword mode returns an empty list); + - that it holds identically in both modes. + - Rewrite Behavioral changes in v0.1.0 item 3's first side-effect bullet. Queries FTS5 cannot parse are now answered in both modes: + - with Ollama down, v0.0.1 rejected punctuated queries (HTTP 400 / invalid_params); + - with Ollama down, v0.0.1 failed hyphenated or colon-bearing queries (`sqlite-vec`, `TODO: auth`) with HTTP 500 / internal_error; + - v0.1.0 answers both. Keep item 3's other bullet. + - In the REST error-contract paragraph, add one sentence: a storage failure during search reports a generic internal error, and the detail goes to the server's stderr log. + - Make the tools-table `memory_search` row mention that natural-language queries are accepted. + 5. **.planning/STATE.md:** add one bullet at the end of the Decisions list in the existing format: + - `[Phase 04]: 04-05: T-04-15 revised by user decision option-a (2026-10-23)`; + - search runs the raw FTS5 MATCH first, then one sanitized retry (word tokens quoted and OR-joined) on any FTS5 query-parse error, including the step-time `no such column` class; + - a tokenless query is an empty keyword leg (hybrid answers semantic-only, keyword mode returns empty); + - identical in both modes, so search never returns InvalidQuery; + - every search-path internal error is redacted to SEARCH_STORAGE_FAILURE; + - this supersedes the 04-04 decision that a malformed MATCH is InvalidQuery/400 in hybrid mode. + 6. Run `cargo fmt --all` and commit with explicit paths. + + + cargo test -p agent-memory --bins && cargo test -p agent-memory --test tools && cargo clippy --workspace --all-targets -- -D warnings && cargo fmt --all --check + + + - These all pass: + - `cargo test -p agent-memory --bins memory_search_answers_every_query_shape_in_both_modes` + - `search_answers_every_query_shape_in_both_modes` + - `memory_search_storage_failure_is_internal_error_without_sqlite_text` + - `search_storage_failure_is_500_without_sqlite_text` + - `memory_search_description_enumerates_exactly_hybrid_and_keyword` + - `cargo test -p agent-memory --test tools memory_search_answers_hyphenated_and_punctuated_queries_over_stdio` passes against the real binary. + - `grep -n "never rejected" README.md` is non-empty. The Behavioral changes item 3 text names both the v0.0.1 400 case and the `sqlite-vec` 500 case. + - `grep -n "04-05: T-04-15 revised" .planning/STATE.md` is non-empty. + - The full gate is green: `cargo fmt --all --check`, `cargo clippy --workspace --all-targets -- -D warnings`, and `cargo test --workspace`. Run the full test and clippy runs in the background and poll. + + Every transport, including the real stdio binary, answers every query shape in the user's case list in both modes, and never shows SQLite text. The tool description, README and STATE all state the new contract. + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| MCP stdio client → memory_search | untrusted query strings cross here; may now be arbitrary natural language | +| REST client (loopback) → POST /api/search | untrusted query strings and filters | +| store seam → FTS5 parser | the raw query and the sanitized fallback are both bound parameters, never SQL text | +| service seam → transport error bodies | internal error text crosses to the client here | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Severity | Disposition | Mitigation Plan | +|-----------|----------|-----------|----------|-------------|-----------------| +| T-04-15 (revised) | Information Disclosure | mode probing via keyword-leg classification | low | mitigate | Both modes share `match_with_sanitized_retry`, so a query classifies identically with Ollama up or down. The user decision (option-a) swaps "both fail" for "both forgiving". The mode is already disclosed by `search_mode`, so there is no hidden mode to probe. The two-mode table tests lock this at the core and on both transports (Tasks 1-3). | +| T-04-19 | Information Disclosure | raw SQLite error text (table/column names, engine detail) in search response bodies | medium | mitigate | `redact_search_error` is the single exit for search errors. It maps `Sqlite`/`Pool`/`Join`/`Migration` to `Internal(SEARCH_STORAGE_FAILURE)` and logs the detail to stderr only. Core and transport tests assert no `sqlite` text in any search body, including a broken-index fixture (Tasks 1-3). | +| T-04-20 | Tampering (injection) | sanitized fallback query construction | medium | mitigate | Fallback terms are runs of alphanumeric/underscore chars, so they cannot contain `"`. Each is wrapped as an FTS5 string literal, and the result is bound as `?1`, never formatted into SQL. The raw query was already bound, as before (T-04-18). The unit test proves every fallback parses (Task 2). | +| T-04-21 | Denial of Service | retry doubles FTS work for unparseable input; long OR chains | low | accept | At most one retry. The OR-chain cost is linear in input size (a 20,000-term chain took about 0.2 s in the planning probe), and transport request size already bounds it. Both legs stay capped at MAX_KNN_K. | +| T-04-22 | Tampering (result integrity) | the code-plus-marker classifier retrying a non-parse SQLITE_ERROR (for example `malformed JSON` from a tampered tags column under a tag filter) | low | accept | Only FTS5 query-parse markers trigger a retry. A residual failure after the retry is internal, so it can never be mis-tiered as client input. A tampered tags column is outside the local single-user trust model. | +| T-04-SC | Tampering | cargo installs | low | accept | Zero new dependencies. `tracing`, `rusqlite` and `serde_json` are already in both crates, and the pin triangle is untouched. | + +No `high` severity threat is open (ASVS L1, block_on: high). + + +## Deferred (findings and items NOT taken by this plan) + +| ID | Disposition | Reason | +|----|-------------|--------| +| CR-03 (stale-vector backfill race) | → plan 04-06 | Required scope. Kept separate because it touches the write path, not search. | +| WR-02, IN-01, IN-02, IN-03, WR-01 | → plan 04-07 | Transport strictness and migration-backup hardening; independent of search. | +| WR-03, IN-05, IN-06 | → plan 04-06 | Relations and write-path documentation. | +| WR-04 (unbounded expanded list) | → plan 04-06 (doc correction); behaviour change deferred | The doc and README correction is in 04-06 Task 3. The behaviour change (reject vs silent default limit vs unbounded) is an API-contract decision, left open for Phase 5/6; see 04-06's Deferred section. | +| Internal-tier redaction on NON-search surfaces | deferred | The user decision scopes redaction to search paths. The other surfaces keep the 02-05 / Phase 3 SECURITY informational-note-1 contract on the loopback-guarded API. Revisit if `--allow-remote` deployments become common. | +| 03-REVIEW WR-01 (filters applied after the KNN cut) | still deferred (not re-opened) | Per the brief. The README and tool description keep documenting the bound. | +| 03-REVIEW WR-03 (spawned REST harness hang) | still deferred (not re-opened) | Per the brief. Task 1 edits only assertions in tests/rest.rs, never its harness. | + + +- `cargo test -p agent-memory-core --test hybrid --test fallback` and `cargo test -p agent-memory-core --lib store::sqlite` (core table, raw-first proofs, classifier pinned to the bundled SQLite) +- `cargo test -p agent-memory` (in-process MCP/REST tables, redaction, spawned REST, stdio end-to-end) +- Phase gate, run in the background and polled: + - `cargo fmt --all --check` + - `cargo clippy --workspace --all-targets -- -D warnings` + - `cargo test --workspace` +- Commit per task with explicit paths. No CI workflow changes, no tag, release or push. +- Optional live re-check of VERIFICATION truths #14/#15 against a real local Ollama. With Ollama up, `memory_search` for `sqlite-vec` and for `what vector store do we use?` returns the sqlite-vec memory with `search_mode` hybrid. With `--ollama-url http://127.0.0.1:9`, the same queries return it with `search_mode` keyword. + + + +- VERIFICATION gap truth #14 is closed by being superseded: no query shape is a 400 or a 500 any more. The 04-04 must-have it tested is replaced by this plan's truths. The `no such column` class is classified and retried. +- VERIFICATION truth #15 is closed: punctuated paraphrase questions recall the right memory with Ollama up, and also with it down. +- REVIEW CR-01 and CR-02 are closed. T-04-15 parity is preserved as "both forgiving". +- No search response on any transport carries SQLite text. +- README, the tool description and STATE.md record the contract, ready for the DIST-06 release notes. + + + +Create `.planning/phases/04-memory-update-relations-hybrid-search/04-05-SUMMARY.md` when done. + diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-06-PLAN.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-06-PLAN.md new file mode 100644 index 0000000..5a14b46 --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-06-PLAN.md @@ -0,0 +1,408 @@ +--- +phase: 04-memory-update-relations-hybrid-search +plan: 06 +type: execute +wave: 2 +depends_on: [04-05] +files_modified: + - crates/agent-memory-core/src/store/mod.rs + - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory-core/src/service.rs + - crates/agent-memory-core/src/domain.rs + - crates/agent-memory-core/tests/backfill_race.rs + - crates/agent-memory-core/tests/relations.rs + - crates/agent-memory/src/mcp.rs + - README.md +autonomous: true +gap_closure: true +requirements: [MCP-06, MCP-07] + +estimate: + tokens: 8500 + raw_tokens: 8500 + tasks: 3 + confidence: low + +must_haves: + truths: + - "A sweep backfill whose memory was updated between the backfill's read and its write writes nothing: the row keeps the vector of its CURRENT content and stays embedding_status = 1 (closes REVIEW CR-03; MCP-06's 'the old meaning no longer matches semantically' holds under concurrency)" + - "If the racing update's own embed failed, the row stays pending with no vector, and the NEXT sweep embeds its new content — never the stale text" + - "A memory forgotten while its backfill was in flight never regains a vector, so no vector can attach to a later memory that reuses the rowid" + - "Two sweeps racing on the same pending row (several processes share one DB) write exactly one vector" + - "relates_to is one symmetric relation: linking a->b and then b->a stores a single edge, expansion lists the neighbour once from each side, and unlink removes it from either end (REVIEW WR-03, matching the documented symmetric 'see also' semantics)" + - "supersedes and caused_by keep D-03 directed identity: a->b and b->a stay two distinct relations, and a missing endpoint is still named in request order" + - "Canonical relates_to storage is invisible on the wire: memory_link / memory_unlink results and REST link bodies (including not-found bodies) echo the request's from_id / to_id. For example, link(b, a) with b > a reports from_id = b, to_id = a" + - "The MAX_EXPANDED_LINKS doc comment and the README state the real expanded-response bound: search is bounded by its limit, while memory_list with expand_links and no limit returns every row, each with up to MAX_EXPANDED_LINKS neighbours (REVIEW WR-04 doc correction; behaviour change deferred)" + artifacts: + - path: "crates/agent-memory-core/src/store/sqlite.rs" + provides: "conditional backfill claim (pending AND content unchanged) in the same writer tx as the vector write; canonical relates_to storage; symmetric unlink" + contains: "embedding_status = 0 AND content = ?2" + - path: "crates/agent-memory-core/tests/backfill_race.rs" + provides: "deterministic interleaving tests at store level and a gated-embedder service-level sweep test (CR-03)" + min_lines: 120 + - path: "crates/agent-memory-core/src/domain.rs" + provides: "LinkKind::is_symmetric + LinkKind::canonical_endpoints" + contains: "fn canonical_endpoints" + key_links: + - from: "crates/agent-memory-core/src/service.rs sweep" + to: "Store::insert_embedding" + via: "passes the exact content it embedded as expected_content; counts skipped stale writes" + pattern: "insert_embedding\\(id, &content" + - from: "crates/agent-memory-core/src/store/sqlite.rs link/unlink" + to: "LinkKind::canonical_endpoints / is_symmetric" + via: "canonical (min, max) storage for relates_to; both-orientation unlink for symmetric kinds" + pattern: "canonical_endpoints|is_symmetric" +--- + +# Plan 04-06: Gap Closure — Write-path integrity (backfill race, symmetric relates_to) + +## Phase Goal + +**As an** agent, **I want to** update existing memories, link related ones, and get hybrid keyword+semantic recall, **so that** my memory stays **current and connected** and I find the right memory even when the phrasing differs. + +This plan makes "current" and "connected" hold under concurrency: + +- an edited memory can no longer be silently re-embedded with its OLD text by a backfill that was already in flight (REVIEW CR-03); +- the symmetric `relates_to` relation can no longer be stored twice and expanded twice (REVIEW WR-03). + + +1. **CR-03.** Make the sweep's embedding backfill conditional. A vector is written only if the row is still pending AND its content still equals the text that was embedded. The check is one guarded UPDATE in the same writer transaction as the vector write. This closes three races: + - update-vs-backfill; + - forget-vs-backfill; + - sweep-vs-sweep across the several processes that share one DB file. +2. **WR-03.** Store `relates_to` canonically as (min id, max id), and let unlink remove a symmetric edge from either end. Directed kinds are untouched. +3. **IN-05, IN-06 and WR-04.** Document three existing behaviours, with no behaviour change: + - an update on an expired-but-unswept row succeeds; + - a snippet may split a grapheme cluster; + - an expanded `memory_list` without a limit returns every row. This corrects the `MAX_EXPANDED_LINKS` doc's 200 × 10 claim; WR-04's behaviour change is deferred. + +Purpose: MCP-06's success criterion 1 ("the old meaning no longer matches semantically") and MCP-07's flat-relation idempotency (D-02) must hold when writers interleave, not only when they run serially. +Output: +- the conditional `Store::insert_embedding`; +- the sweep wiring; +- relates_to canonicalisation; +- deterministic race tests and relations tests; +- updated docs. + + + +@$HOME/.claude/gsd-core/workflows/execute-plan.md +@$HOME/.claude/gsd-core/templates/summary.md + + + +@.planning/phases/04-memory-update-relations-hybrid-search/04-CONTEXT.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-02-SUMMARY.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-03-SUMMARY.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-05-SUMMARY.md +@crates/agent-memory-core/src/store/sqlite.rs +@crates/agent-memory-core/src/service.rs + + +## Interface contracts (established by this plan) + +- `Store::insert_embedding(&self, memory_id: i64, expected_content: &str, embedding: Vec) -> Result`. This REPLACES the old `(memory_id, embedding) -> Result<(), _>` signature. Its only caller is `MemoryService::sweep`; there are no other `Store` implementors. + - `Ok(true)`: the vector was written and `embedding_status` became 1. + - `Ok(false)`: nothing was written, because the row was updated, already embedded, or forgotten since `pending_embeddings` read it. + - The claim is a single guarded `UPDATE memories SET embedding_status = 1 WHERE id = ?1 AND embedding_status = 0 AND content = ?2`, run FIRST inside the writer transaction. + - Zero changed rows → return `Ok(false)`; the transaction rolls back on drop. + - Otherwise, `DELETE` + `INSERT` the vec0 row in the same transaction (never `INSERT OR REPLACE` on vec0, per the v1.0 decision), then commit. + - SQLite takes the write lock at that UPDATE statement, so the claim is atomic across processes as well as across threads. +- `LinkKind::is_symmetric(self) -> bool`: true only for `RelatesTo`. +- `LinkKind::canonical_endpoints(self, from_id: i64, to_id: i64) -> (i64, i64)`: + - `RelatesTo` → `(min, max)`; + - `Supersedes` and `CausedBy` → `(from_id, to_id)` unchanged. + + Both use exhaustive matches with no `_` arm. +- Canonicalisation is invisible on the wire. Every `memory_link` / `memory_unlink` response and REST link body echoes the REQUEST's `from_id` / `to_id`: the `{from_id, to_id, kind, linked|unlinked, ...}` body and the not-found body with `missing_id`. Transports build those bodies from the request, never from the stored row. `LinkOutcome` carries no endpoint ids besides `missing_id`, and that is a request id. +- `SqliteStore::link` checks both endpoints in REQUEST order, exactly as today, so `LinkOutcome::NotFound { missing_id }` keeps naming `from_id` first. Only then does it INSERT OR IGNORE the canonical pair. +- `SqliteStore::unlink` deletes `(from_id, to_id, kind)`. For a symmetric kind it also deletes the reversed orientation, which covers canonical rows and any relates_to row written by a pre-fix dev build of the unreleased 0003. No migration is involved, and 0001/0002/0003 SQL is untouched. + +## Artifacts this phase produces + +- The conditional `Store::insert_embedding` (trait + sqlite impl), and `MemoryService::sweep` passing the embedded content and logging `skipped_stale` next to `backfilled`. +- `LinkKind::is_symmetric` and `LinkKind::canonical_endpoints` (domain.rs). +- Canonical relates_to INSERT in `SqliteStore::link`, and a both-orientation symmetric DELETE in `SqliteStore::unlink`. +- NEW `crates/agent-memory-core/tests/backfill_race.rs`. +- New tests in `crates/agent-memory-core/tests/relations.rs`. +- Updated docs: + - `Store::insert_embedding`, `pending_embeddings`, `link`, `unlink` and `update` (IN-05); + - `snippet` (IN-06); + - `MAX_EXPANDED_LINKS`, which drops the false 200 × 10 bound for list (WR-04 doc); + - the memory_link / memory_unlink tool descriptions; + - README Relations (including a WR-04 expanded-list note), memory_update and Lifecycle sections. + + + + + Task 1 (tracer): An edited memory is never re-embedded with its old text by an in-flight sweep backfill + crates/agent-memory-core/src/store/mod.rs, crates/agent-memory-core/src/store/sqlite.rs, crates/agent-memory-core/src/service.rs, crates/agent-memory-core/tests/backfill_race.rs, README.md + + - crates/agent-memory-core/src/store/sqlite.rs: + - `fn insert_embedding` (~lines 439-463, the unconditional DELETE+INSERT+status write being replaced); + - `fn pending_embeddings` (~465-479); + - `fn update` (~731-854, the in-tx vector swap this race interleaves with); + - `fn forget`. + - crates/agent-memory-core/src/store/mod.rs lines 45-51 (insert_embedding / pending_embeddings docs) + - crates/agent-memory-core/src/service.rs `pub async fn sweep` (~lines 605-650: pending read → embed await outside any lock → write loop) and `BACKFILL_BATCH` + - crates/agent-memory-core/src/embed/mod.rs lines 20-160 (the `Embedder` trait with `BoxFuture`, and `FakeEmbedder::with_vectors` / `failing`) + - crates/agent-memory-core/tests/hybrid.rs lines 36-121 (harness / `unit_axis` / `raw_conn` patterns to copy) + - crates/agent-memory-core/tests/update.rs (the `embed_outage_during_update_falls_back_to_pending_and_sweep_backfills` test must stay green unchanged) + - README.md "Lifecycle: TTL & decay" section (Embedding backfill bullet) + + + Fixtures: + - `v_old = unit_axis(1)`, `v_new = unit_axis(2)`. These are orthogonal unit vectors, so the cosine distance between them is 1.0. The schema uses `distance_metric=cosine`. + - A FakeEmbedder is programmed with `"old text" → v_old` and `"new text" → v_new`. + + Store-level tests (deterministic, no async gating), in NEW `tests/backfill_race.rs`: + - **Happy path:** a pending row → `insert_embedding(id, "old text", v_old)` is `Ok(true)`, the status is 1, and KNN for `v_old` returns the id at distance ≈ 0. + - **Update with a successful embed lands between the read and the write:** + - setup: a pending row is read via `pending_embeddings`, then `Store::update(id, content "new text", Some(v_new))` runs; + - `insert_embedding(id, "old text", v_old)` is `Ok(false)`; + - KNN for `v_new` gives distance ≈ 0; + - KNN for `v_old` gives distance ≈ 1.0; + - the status is 1, and exactly one `vec_memories` row exists for the id. + - **Update whose own embed failed lands between the read and the write:** + - setup: the update runs with `None`; + - `insert_embedding(id, "old text", v_old)` is `Ok(false)`; + - no `vec_memories` row exists for the id; + - `pending_embeddings` now returns `(id, "new text")`; + - `insert_embedding(id, "new text", v_new)` is `Ok(true)`. + - **Forget lands between the read and the write:** `insert_embedding(id, "old text", v_old)` is `Ok(false)`, and zero `vec_memories` rows exist for the id. + - **Double sweep:** a second `insert_embedding` for the same id and content is `Ok(false)`, and exactly one vector row remains. + + Service-level test (deterministic gate): + - `GatedEmbedder` wraps the programmed FakeEmbedder. It has: + - an `AtomicBool` `fail`; + - an `armed` flag; + - two `tokio::sync::Notify` signals, `entered` and `release`. + - While armed, a batch containing "old text" disarms the gate, notifies `entered`, and awaits `release` before delegating. + - Test steps: + 1. Store "old text" with `fail = true`, so the row lands pending. + 2. Set `fail = false` and arm the gate. + 3. Spawn `service.sweep(now)` on a cloned service. + 4. Await `entered`. + 5. `service.update(id, content "new text")`. Its embed is not gated, so it succeeds. + 6. Notify `release`, then join the sweep: `Ok`. + 7. The stored vector is `v_new` (distance ≈ 0), not `v_old` (distance ≈ 1.0), and the status is 1. + - Variant: `fail = true` only around the update. + - After the gated sweep, the status is 0, there is no vector row, and the content is "new text". + - A second, ungated sweep with `fail = false` stores `v_new`. + + + Implements REVIEW CR-03 using the fix shape the review proposed. Per CLAUDE.md, no `unwrap()` in production code. + + 1. **store/mod.rs:** change the `insert_embedding` signature to the contract above. + - Rewrite its doc: the guarded claim, what each bool means, and the three races it closes (update, forget, concurrent sweeps across processes). + - Note on `pending_embeddings` that the returned content is exactly what the caller must pass back as `expected_content`. + 2. **sqlite.rs:** reimplement `insert_embedding` per "Interface contracts". + - Writer lock, mapping poison to `MemoryError::Internal("writer lock poisoned")` like every other writer. + - Transaction. The guarded UPDATE first, binding `memory_id` and `expected_content`. Zero changes → `Ok(false)`. + - Then the existing vec0 DELETE + INSERT (blob via `bytemuck::cast_slice`, bound as now). Commit. `Ok(true)`. + - Keep the DELETE+INSERT comment about vec0 conflict clauses. + 3. **service.rs `sweep`:** + - Keep `pending_embeddings` → one batch embed outside any lock → one `spawn_blocking` write loop. + - The loop now zips each `(id, content)` with its vector and calls `store.insert_embedding(id, &content, vector)`, counting `written` vs `skipped` from the bool. + - Log `tracing::info!` with both `backfilled` and `skipped_stale` fields. + - Update the `sweep` doc comment: a backfill never overwrites a row that changed or vanished while it was being embedded, and that row's current content is embedded on a later sweep. + - The `SweepReport` shape is unchanged (TTL contract). + 4. **NEW tests/backfill_race.rs:** + - Start with a `//! CR-03: …` header explaining the interleavings. + - Write all `` tests on real on-disk temp DBs with an injected `TestClock`, using the `harness` pattern from tests/hybrid.rs. + - Use the public `Store` methods directly for the store-level interleavings. + - Count vector rows with a raw rusqlite connection opened after `register_vec_extension()`. + - The gate uses only tokio sync primitives. No sleeps decide ordering. + - The project hook blocks `DROP` in Bash and test SQL, so do not use it. None is needed. + 5. **README.md, Lifecycle section, "Embedding backfill" bullet:** add that a memory edited or deleted while its backfill was in flight is never overwritten, and that its current content is embedded on a later sweep. + 6. Run `cargo fmt --all` and commit with explicit paths. + + + cargo test -p agent-memory-core --test backfill_race && cargo test -p agent-memory-core --test update --test semantic --test fallback && cargo clippy -p agent-memory-core --all-targets -- -D warnings + + + - `grep -n "embedding_status = 0 AND content = ?2" crates/agent-memory-core/src/store/sqlite.rs` is non-empty. + - `grep -n "expected_content: &str" crates/agent-memory-core/src/store/mod.rs` is non-empty. + - `grep -n "skipped_stale" crates/agent-memory-core/src/service.rs` is non-empty. + - `cargo test -p agent-memory-core --test backfill_race` passes with at least 6 tests: 5 store-level interleavings plus the gated sweep test with its variant. + - Pre-existing backfill users stay green, unchanged: `tests/update.rs` (embed-outage → sweep backfill), `tests/semantic.rs`, `tests/fallback.rs`. + - clippy `-D warnings` and fmt are clean. No `unwrap()` was added to non-test code. + + Whatever interleaving the backfill hits (update, failed update, forget, or a second sweep), the stored vector always matches the row's current content, or the row is left pending for the next sweep. + + + + Task 2: relates_to is one symmetric relation — link either way, expand once, unlink from either end + crates/agent-memory-core/src/domain.rs, crates/agent-memory-core/src/store/mod.rs, crates/agent-memory-core/src/store/sqlite.rs, crates/agent-memory-core/tests/relations.rs, crates/agent-memory/src/mcp.rs + + - crates/agent-memory-core/src/domain.rs: + - lines 70-136 (`LinkKind`, `as_wire_str`, `reverse_label`, `TryFrom`); + - lines 176-198 (`RELATED_SNIPPET_CHARS` / `snippet` for IN-06). + - crates/agent-memory-core/src/store/mod.rs lines 109-155 (`update` / `link` / `unlink` / `related` docs) + - crates/agent-memory-core/src/store/sqlite.rs `fn link` and `fn unlink` (~lines 678-729) + - crates/agent-memory-core/tests/relations.rs: + - helpers `harness`, `store`, `edge_count`, `edges_touching`, `orphan_edges`, `related_of`, `list_args`; + - `relates_to_reads_the_same_from_both_ends` (~487) and `forget_cascades_every_edge_touching_the_memory` (~294), which must stay green. + - crates/agent-memory/src/mcp.rs: + - LinkArgs/UnlinkArgs `kind` docs (~lines 203-226); + - the memory_link description (~398) and memory_unlink description (~436). + - crates/agent-memory/src/rest/handlers.rs `post_links_to_a_missing_memory_is_404_naming_it_never_500` (~1216). This is a read-only check: it must stay green because missing_id keeps request order. + + + - **Unit, `canonical_endpoints`:** + - RelatesTo (5,2) → (2,5) and (2,5) → (2,5); + - Supersedes (5,2) → (5,2); + - CausedBy (5,2) → (5,2). + - **Unit, `is_symmetric`:** true only for RelatesTo. + - **Linking both ways:** `link(a,b,RelatesTo)` then `link(b,a,RelatesTo)` → both Linked, and `edge_count == 1`. With expand_links, a's `related` lists b exactly once and b's lists a exactly once. + - **Canonical storage:** `link(b,a,RelatesTo)` with b > a stores the row (from_id = a, to_id = b), checked via raw SQL. + - **Unlink from either end:** `link(a,b,RelatesTo)` → `unlink(b,a,RelatesTo)` is true and `edge_count` is 0. A second unlink is false (the clean not-found). + - **Pre-fix backstop:** a relates_to row planted in the reversed orientation (from = b, to = a, b > a) via raw INSERT is removed by `unlink(a,b,RelatesTo)`. + - **Directed kinds:** `link(a,b,Supersedes)` and `link(b,a,Supersedes)` give 2 edges. `unlink(b,a,Supersedes)` removes only that one; the remaining edge has from = a. + - **Responses echo the request order:** + - `link(b, a, RelatesTo)` with b > a is stored canonically as (a, b), but the MCP `memory_link` result and the REST `POST /api/links` body still report `from_id = b`, `to_id = a`, `kind = relates_to`, `linked = true`; + - `memory_unlink(b, a, relates_to)` reports `from_id = b`, `to_id = a`; + - a not-found body for `link(999_999, a, RelatesTo)` reports `from_id = 999_999`, `to_id = a` and `missing_id = 999_999`. + - **Missing endpoints, request order kept:** + - `link(999_999, a, RelatesTo)` → `NotFound { missing_id: 999_999 }`; + - `link(a, 999_999, RelatesTo)` → 999_999; + - `link(999_998, 999_997, RelatesTo)` → 999_998, the request's from_id. + - **MCP:** the memory_link and memory_unlink descriptions both state that relates_to is symmetric. The memory_unlink description says direction must match for supersedes and caused_by, and that relates_to can be removed from either end. + + + Implements REVIEW WR-03. Symmetric relates_to is already the documented meaning: the domain.rs `LinkKind` docs and 04-CONTEXT specifics both say "relates_to symmetric". D-02 idempotency ("duplicate link is a no-op") therefore applies whichever way round the pair is written. D-03 directed storage is kept for supersedes and caused_by. + + 1. **domain.rs:** + - Add `is_symmetric` and `canonical_endpoints` to `impl LinkKind`, with exhaustive matches and doc comments. The `LinkKind` docs gain a note: relates_to is stored as (smaller id, larger id), so its expansion `direction` carries no meaning. + - IN-06: extend the `snippet` / `RELATED_SNIPPET_CHARS` doc. The cut never splits a UTF-8 code point, but it may split a grapheme cluster (a combining sequence or a ZWJ emoji). That is cosmetic and accepted under the D-05 discretion. No behaviour change. + 2. **sqlite.rs `link`:** keep the request-order endpoint pre-check loop unchanged. Bind the INSERT OR IGNORE with `kind.canonical_endpoints(from_id, to_id)`. + **sqlite.rs `unlink`:** one bound DELETE that matches `(from_id, to_id, kind)`, plus the reversed orientation when `kind.is_symmetric()`. Bind the flag as a parameter; never format SQL. Comment the pre-fix dev-build backstop. + 3. **store/mod.rs:** + - Update the `link` / `unlink` docs to the canonical-storage and symmetric-unlink contract. + - IN-05: add to the `update` doc that a row whose TTL has passed but which the sweep has not removed yet is still a row. Update succeeds, and a `ttl_secs` patch can extend or clear it. This is consistent with list and search showing such rows until the sweep runs. Documented behaviour; no change. + 4. **tests/relations.rs:** add the `` tests (canonical-endpoints and symmetry unit asserts may live here, since both fns are `pub`). Every existing relations test must stay green unchanged. + 5. **mcp.rs:** + - Reword the memory_link and memory_unlink descriptions and the LinkArgs/UnlinkArgs `kind` docs per ``. + - Add `link_and_unlink_descriptions_state_relates_to_symmetry`, asserting the descriptions contain `symmetric`, and that memory_unlink's contains `either end`. + - Add `relates_to_link_and_unlink_responses_echo_the_request_order`. Store a, then b (so b > a). Call `memory_link` with `{"from_id": b, "to_id": a, "kind": "relates_to"}`: the result JSON has `from_id == b`, `to_id == a`, `linked == true`. Then `memory_unlink` with the same args: `from_id == b`, `to_id == a`, `unlinked == true`. + - The transports already echo the request; this test locks that against the new canonical storage. Do not change the response construction. + 6. Run `cargo fmt --all` and commit with explicit paths. + + relates_to canonicalisation lives in code only: no migration, and 0003 is unreleased. Reverting just stops canonicalising, and every canonical row is still a valid directed row. + + cargo test -p agent-memory-core --test relations && cargo test -p agent-memory --bins link && cargo clippy --workspace --all-targets -- -D warnings + + + - `grep -n "fn canonical_endpoints" crates/agent-memory-core/src/domain.rs` and `grep -n "fn is_symmetric" crates/agent-memory-core/src/domain.rs` are non-empty. + - `grep -n "canonical_endpoints" crates/agent-memory-core/src/store/sqlite.rs` and `grep -n "is_symmetric" crates/agent-memory-core/src/store/sqlite.rs` are non-empty. + - `cargo test -p agent-memory-core --test relations` is green. It includes the 7 new behaviours and every pre-existing relations test, unchanged. + - These pass: + - `cargo test -p agent-memory --bins relates_to_link_and_unlink_responses_echo_the_request_order` (link(b, a) with b > a returns from_id = b, to_id = a even though the stored row is (a, b)) + - `cargo test -p agent-memory --bins link_and_unlink_descriptions_state_relates_to_symmetry` + - `post_links_to_a_missing_memory_is_404_naming_it_never_500` (unchanged) + - `grep -n "grapheme" crates/agent-memory-core/src/domain.rs` (IN-06) and `grep -n "sweep has not" crates/agent-memory-core/src/store/mod.rs` (IN-05) are non-empty. + - clippy `-D warnings` and fmt are clean. + + An agent that links two memories as "see also" in either order gets one relation. It sees that relation once from each side and can remove it from either side. Directed relations are unchanged. + + + + Task 3: README and doc comments tell users how relations, edits of expiring memories, snippets and expanded lists behave + README.md, crates/agent-memory-core/src/domain.rs + + - README.md: + - "Relations (memory_link / memory_unlink)" section (~lines 245-301); + - "memory_update" section (~199-243); + - the snippet bullet and the `MAX_EXPANDED_LINKS` bullet in the Expansion list; + - the Bounds table; + - the Lifecycle section edited in Task 1. + - crates/agent-memory-core/src/domain.rs lines ~316-321 (the `MAX_EXPANDED_LINKS` doc comment, which claims a `200 × 10` bound with T-04-11) + - crates/agent-memory-core/src/service.rs `pub async fn list` (~lines 194-208: an omitted limit stays unlimited; expansion runs per row). Read-only, to describe the actual behaviour. + - crates/agent-memory-core/src/domain.rs and store/mod.rs doc comments written in Task 2 (the wording to mirror) + + + Documentation for WR-03, IN-05, IN-06 and WR-04 (documentation-only; WR-04's behaviour change is deferred, see the Deferred section). It matches the behaviour Tasks 1-2 shipped and the behaviour `list` already has. + + 0. **WR-04, domain.rs.** Correct the `MAX_EXPANDED_LINKS` doc comment so it states the actual bound. Each result carries at most `MAX_EXPANDED_LINKS` neighbours. Search is bounded by its limit (1–200, default 50), so an expanded search returns at most 200 × 10 summaries. Include the exact phrase "list without a limit returns every row": each row is expanded up to `MAX_EXPANDED_LINKS`, so an unlimited expanded list grows with the corpus. Name T-04-11 as covering search and list-with-limit only. No code change. + + 1. **Relations section.** + - Extend the Idempotent bullet: `relates_to` is symmetric, so linking b→a after a→b is the same relation and a no-op success. + - Change the memory_unlink paragraph so that direction must match only for supersedes and caused_by, and a relates_to relation can be removed from either end. + - In the Expansion bullets, note that the `direction` reported for relates_to carries no meaning. + - Make the snippet bullet mention that a cut may split a combined emoji or accent sequence. + 2. **memory_update section.** Add one bullet for IN-05: an update to a memory whose TTL has already passed but which the background sweep has not yet removed still succeeds, as list and search still show it, and a `ttl_secs` patch can extend or clear its expiry. + 3. **WR-04, README Expansion bullets.** Next to the `MAX_EXPANDED_LINKS` bullet, add a one-line note: `memory_list` with `expand_links` and no `limit` returns every memory, each with up to 10 neighbours, so pass a `limit` on large stores. + 4. Run `cargo fmt --all --check`, which a doc-comment edit must keep clean. Commit with explicit paths. + + + grep -n "either end" README.md && grep -n "has not yet removed" README.md && grep -n "carries no meaning" README.md && grep -n "pass a .limit. on large stores" README.md && grep -n "without a limit returns every row" crates/agent-memory-core/src/domain.rs && cargo fmt --all --check + + + - `grep -n "either end" README.md` is non-empty, in the Relations section. + - `grep -n "has not yet removed" README.md` is non-empty, in the memory_update section. + - `grep -n "carries no meaning" README.md` is non-empty, in the Expansion bullets. + - WR-04 README note: `grep -n "pass a .limit. on large stores" README.md` is non-empty, in the Expansion bullets. + - WR-04 domain.rs doc: `grep -n "without a limit returns every row" crates/agent-memory-core/src/domain.rs` is non-empty, in the `MAX_EXPANDED_LINKS` doc comment. That doc comment now states the 200 × 10 bound for search only. + - The README still renders the existing Relations table unchanged: three rows, same reverse labels. + - `cargo fmt --all --check` is clean. No non-comment line in domain.rs changed for WR-04. + + The README and the `MAX_EXPANDED_LINKS` doc describe symmetric relates_to, updates to expiring rows, snippet cutting and the real expanded-list bound exactly as the code behaves. + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| sweep task ↔ request handlers (same process) | concurrent writers to the same row through one writer mutex, but the backfill's embed await happens outside the lock | +| process ↔ process (shared DB file) | several MCP stdio servers plus the REST daemon each run their own sweep; the writer mutex is per process only | +| MCP/REST client → link/unlink | untrusted ids and kinds; the kind is already a closed enum (T-04-10) | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Severity | Disposition | Mitigation Plan | +|-----------|----------|-----------|----------|-------------|-----------------| +| T-04-23 | Tampering (integrity) | sweep backfill writing a vector of superseded content over an updated row (status 1, so never repaired) | high | mitigate | Guarded claim `WHERE id = ?1 AND embedding_status = 0 AND content = ?2`, run first in the same writer transaction as the vector write. SQLite's write lock makes it atomic across processes. Deterministic store-level and gated service-level tests (Task 1). | +| T-04-24 | Tampering (integrity) | backfill re-inserting a vector for a forgotten id, which then attaches to a new memory reusing the rowid | medium | mitigate | The same guard: a forgotten row matches nothing, and a reused rowid with different content fails the content check. Forget-interleaving test (Task 1). | +| T-04-25 | Tampering (integrity) | duplicate symmetric edges inflating expansion and defeating unlink | low | mitigate | Canonical (min, max) storage for relates_to, a both-orientation symmetric DELETE, and bound parameters only (Task 2). | +| T-04-26 | Denial of Service | a row whose content keeps changing is skipped by every sweep | low | accept | Each skip is O(1) and the row is retried on the next tick. Content that keeps changing is re-embedded in-transaction by `update` itself whenever the embedder is up. | +| T-04-SC | Tampering | cargo installs | low | accept | Zero new dependencies. `tokio::sync::Notify` is already available through tokio. | + +No `high` severity threat is left open (T-04-23 is mitigated). ASVS L1, block_on: high. + + +## Deferred (findings and items NOT taken by this plan) + +| ID | Disposition | Reason | +|----|-------------|--------| +| CR-01, CR-02, VERIFICATION truths #14/#15, IN-04 | → plan 04-05 | The search contract. | +| WR-01, IN-03, IN-01, WR-02, IN-02 | → plan 04-07 | Migration-backup and transport hardening. | +| WR-04 (unbounded `memory_list` with `expand_links` and no `limit`) — behaviour change | deferred; doc correction done in Task 3 | The fix means choosing between three options:
1. reject an unlimited `expand_links` list (400 / `invalid_params`, "limit is required when expand_links is set");
2. apply a silent default limit when `expand_links` is set;
3. leave it unbounded and document the cost.
That choice is an API-contract decision for the user, recorded as an open question for Phase 5/6. The API is loopback-only, so the cost exposure is local. Task 3 corrects the `MAX_EXPANDED_LINKS` doc comment, which wrongly claimed a 200 × 10 bound for list, and adds a README note. No behaviour change. | +| IN-05 behaviour change (treat expired-unswept rows as not-found in update) | deferred | Changing it would diverge from list and search, which show such rows until the sweep runs. That is a product decision, not a cheap fix, so this plan documents the current behaviour instead. | +| IN-06 word-boundary or grapheme-aware snippet cut | deferred | Cosmetic. D-05 leaves the choice open, and grapheme segmentation would need a new dependency (unicode-segmentation) for no functional gain. Documented instead. | +| Dedupe-by-neighbour in `related()` | not needed | Canonical storage makes a duplicate relates_to pair impossible. Distinct kinds between one pair are intentionally distinct entries. | +| 03-REVIEW WR-01 (filters after the KNN cut), 03-REVIEW WR-03 (REST harness hang) | still deferred (not re-opened) | Per the brief. | + + +- `cargo test -p agent-memory-core --test backfill_race --test relations --test update --test semantic --test fallback` +- `cargo test -p agent-memory --bins` +- Phase gate, run in the background and polled: + - `cargo fmt --all --check` + - `cargo clippy --workspace --all-targets -- -D warnings` + - `cargo test --workspace` +- Commit per task with explicit paths. No migration, no CI workflow change, no tag, release or push. + + + +- REVIEW CR-03 is closed: no interleaving of update, forget or a second sweep can leave a vector that disagrees with the row's content. This is test-locked deterministically. +- REVIEW WR-03 is closed: relates_to is a single symmetric relation; directed kinds and the not-found contract are unchanged. +- IN-05 and IN-06 are documented, with no behaviour change. +- For WR-04, the `MAX_EXPANDED_LINKS` doc and the README state the real expanded-list bound, with no behaviour change. The behaviour change is deferred with its three options recorded. + + + +Create `.planning/phases/04-memory-update-relations-hybrid-search/04-06-SUMMARY.md` when done. + diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-07-PLAN.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-07-PLAN.md new file mode 100644 index 0000000..8fcc810 --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-07-PLAN.md @@ -0,0 +1,452 @@ +--- +phase: 04-memory-update-relations-hybrid-search +plan: 07 +type: execute +wave: 3 +depends_on: [04-05, 04-06] +files_modified: + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - crates/agent-memory/tests/tools.rs + - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory-core/tests/migration_hygiene.rs + - README.md + - .planning/STATE.md +autonomous: true +gap_closure: true +requirements: [MCP-06, MCP-07, STORE-05] + +estimate: + tokens: 8000 + raw_tokens: 8000 + tasks: 3 + confidence: low + +must_haves: + truths: + - "A memory_update tool call or PATCH /api/memories/{id} carrying an unknown or immutable field (type, source, or a typo such as ttl_sec) is rejected as invalid_params / HTTP 400 naming the field, and nothing is written — a merge-patch never silently drops a member (REVIEW WR-02)" + - "A malformed path or query parameter on the REST routes (PATCH /api/memories/abc, DELETE /api/memories/abc, DELETE /api/links/x/2/relates_to, GET /api/memories?expand_links=1) returns a JSON {\"error\": ...} body with the extractor's own status, like every other client error (IN-02)" + - "When two processes open the same pre-0003 database at once, both start, and exactly one valid pre-migration backup survives, carrying the OLD schema version. A backup taken after another process already migrated is discarded, never written over the valid one (REVIEW WR-01)" + - "A backup that fails to write never removes the previous valid backup first: the new copy goes to a temp file and is renamed into place before older same-version backups are pruned (IN-03)" + - "The README tells users where the pre-migration backup lives, how to restore it, and what happens when a newer database meets an older binary (IN-01)" + artifacts: + - path: "crates/agent-memory/src/mcp.rs" + provides: "UpdateArgs rejects unknown fields on rmcp's real Parameters path" + contains: "deny_unknown_fields" + - path: "crates/agent-memory/src/rest/handlers.rs" + provides: "UpdateBody rejects unknown fields; Path/Query rejections mapped to the JSON error body" + contains: "PathRejection" + - path: "crates/agent-memory-core/src/store/sqlite.rs" + provides: "verify-then-rename backup and a migration step that tolerates a concurrent completed upgrade" + contains: "fn migrate_tolerating_concurrent_upgrade" + - path: "README.md" + provides: "Upgrading section (backup name, retention, restore, newer-schema refusal)" + contains: "## Upgrading" + key_links: + - from: "crates/agent-memory-core/src/store/sqlite.rs SqliteStore::open" + to: "pre_migration_backup(&writer, path, db_version)" + via: "the probed version is passed in and compared with the copy's user_version before the copy is kept" + pattern: "pre_migration_backup\\(&writer, path, db_version\\)" + - from: "crates/agent-memory-core/src/store/sqlite.rs SqliteStore::open" + to: "migrate_tolerating_concurrent_upgrade" + via: "wraps to_latest; re-probes user_version on failure" + pattern: "migrate_tolerating_concurrent_upgrade\\(" +--- + +# Plan 04-07: Gap Closure — Strict update payloads, uniform REST error bodies, race-safe pre-migration backup + +## Phase Goal + +**As an** agent, **I want to** update existing memories, link related ones, and get hybrid keyword+semantic recall, **so that** my memory **stays current** and connected and I find the right memory even when the phrasing differs. + +"Stays current" has two more weak points, and this plan closes both: + +- An update that misspells or smuggles a field (`type`, `ttl_sec`) is told it succeeded while part of it was dropped (REVIEW WR-02). +- The very first open after `brew upgrade` (STORE-05) can fail to start, or can leave a "pre-0003" backup that is really a post-migration copy, when an editor's MCP server and the REST daemon start at the same moment (REVIEW WR-01, IN-03). + +It also gives the new REST routes the same `{"error"}` body shape as every other client error (IN-02), and documents upgrade and restore for users (IN-01). + + +1. Add `#[serde(deny_unknown_fields)]` to the two merge-patch DTOs only: `mcp::UpdateArgs` and `rest::handlers::UpdateBody`. These are surfaces new in v0.1.0. The [03-01] decision not to add it to the v0.0.1 DTOs stands, so no v0.0.1 behaviour changes. +2. Map axum `Path` and `Query` extractor rejections on the REST handlers to `ApiError` with the JSON body, preserving the extractor's status tier. +3. Make the STORE-05 backup race-safe without new dependencies or migrations: + - set `busy_timeout` before anything that can contend for the DB; + - back up to a temp file; + - verify the copy still carries the probed pre-migration version; + - rename it into place, and only then prune older same-version backups; + - treat a `to_latest` failure as success only when a re-probe shows another process already reached `LATEST_SCHEMA_VERSION`. +4. Add a README "Upgrading" section. + +Purpose: every client mistake on the update surface is reported, never swallowed, and the only rollback path a v0.0.1 user has (the backup) is always a genuine pre-migration copy. +Output: strict DTOs with end-to-end stdio and REST proofs, uniform rejection bodies, hardened `open()` with deterministic seam tests, and an Upgrading section. + + + +@$HOME/.claude/gsd-core/workflows/execute-plan.md +@$HOME/.claude/gsd-core/templates/summary.md + + + +@.planning/STATE.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-CONTEXT.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-01-SUMMARY.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-02-SUMMARY.md +@crates/agent-memory-core/src/store/sqlite.rs +@crates/agent-memory/src/rest/handlers.rs + + +## Interface contracts (established by this plan) + +**`UpdateArgs` / `UpdateBody`** + +- Both are `#[serde(deny_unknown_fields)]`. +- rmcp 1.8 deserializes tool `arguments` straight into `Parameters` via `serde_json::from_value`, and maps a failure to `invalid_params("failed to deserialize parameters: …")`. This was verified in rmcp-1.8.0 `src/handler/server/tool.rs`. MCP `_meta` lives outside `arguments`, so no legitimate key is lost. +- schemars 1.2 emits `"additionalProperties": false` for the type. The 04-02 schema test (`required == ["id"]`, no `default` on any patch field) must stay green. + +**REST rejection mapping** + +- Handlers take `Result, PathRejection>` / `Result, QueryRejection>`, mirroring the existing `Result, JsonRejection>` pattern. +- A private helper maps a rejection by its status: + - `rejection.status().is_server_error()` → `ApiError::Internal(rejection.body_text())`; + - any other status → `ApiError::BadRequest(rejection.body_text())`. + + So a parse failure stays a 400 with the JSON body, and an axum routing misconfiguration stays a 500. + +**Pre-migration backup and migration** + +`pre_migration_backup(src: &Connection, db_path: &Path, expected_version: i64) -> Result, MemoryError>`: + +1. Temp path: `{db}.backup-tmp-{LATEST:04}-{std::process::id()}`. Deliberately outside the `{db}.backup-pre-{NNNN}-` retention prefix, so no process prunes another's in-flight temp. +2. Copy `src` into the temp file with the Online Backup API, as today. +3. Read `user_version` from the copy. + - If it differs from `expected_version`, another process migrated in between: remove the temp file (best-effort), log at info, and return `Ok(None)`. Existing backups are never touched. +4. Drop the destination connection. +5. Rename the temp file to `{db}.backup-pre-{LATEST:04}-{YYYYMMDD}`. This is an atomic replace of a same-day stale backup. +6. Remove every OTHER `{db}.backup-pre-{LATEST:04}-*` entry, keeping the final path. +7. Return `Ok(Some(final))`. + +Any failure in steps 2, 5 or 6 removes the temp file (best-effort) and ABORTS the open with the existing "pre-migration backup failed (…); refusing to migrate …" `MemoryError::Internal`. At no point is a previous valid backup deleted before a new valid one is in place. + +`fn migrate_tolerating_concurrent_upgrade(conn: &mut Connection, migrate: impl FnOnce(&mut Connection) -> Result<(), MemoryError>) -> Result<(), MemoryError>`: + +- Run `migrate`. +- On `Err`, re-read `PRAGMA user_version` on the same connection. + - Equal to `LATEST_SCHEMA_VERSION` → log at info that another process completed the upgrade, and return `Ok(())`. + - Otherwise → return the original error. + +`SqliteStore::open` order: + +1. `register_vec_extension`. +2. `Connection::open`. +3. `writer.busy_timeout(5 s)`. This is connection-level, so the file stays byte-untouched for the D-13 refusal. +4. Probe `user_version`, refusing a version that is too new. +5. `prepare_connection`. `apply_pragmas` now sets `busy_timeout` first, before `journal_mode`. +6. If `0 < db_version < LATEST`: `pre_migration_backup(&writer, path, db_version)`. +7. `migrate_tolerating_concurrent_upgrade(&mut writer, …to_latest…)`. +8. The FTS and vec smoke checks and the pool build, unchanged. + +## Artifacts this phase produces + +- `#[serde(deny_unknown_fields)]` on `mcp::UpdateArgs` and `rest::handlers::UpdateBody`, with MCP in-process, REST in-process and stdio end-to-end tests. +- A private `rejection → ApiError` helper. `update_handler`, `forget_handler`, `unlink_handler` and `list_handler` take `Result`, with a real-HTTP test. +- In `store/sqlite.rs`: + - `pre_migration_backup` gains `expected_version` and returns `Option`; + - NEW `fn migrate_tolerating_concurrent_upgrade`; + - `busy_timeout` is set first on the writer and in `apply_pragmas`; + - unit tests for the three race and failure seams. +- `tests/migration_hygiene.rs`: no temp-file leftovers asserted after upgrades. +- README: + - a memory_update note on unknown fields; + - a REST error-body sentence; + - a NEW "## Upgrading" section. +- A STATE.md decision entry on the deny_unknown_fields scope. + + + + + Task 1 (tracer): A memory_update that names an unknown or immutable field is refused end-to-end over stdio and REST, and nothing is written + crates/agent-memory/src/mcp.rs, crates/agent-memory/src/rest/handlers.rs, crates/agent-memory/tests/tools.rs, README.md, .planning/STATE.md + + - crates/agent-memory/src/mcp.rs: + - lines 143-201: `UpdateArgs` with `skip_serializing_if` and `deserialize_with`, and `into_patch`; + - lines 553-560 and 619-667: `update_args` / `patch_from_raw` / `update_raw` / `update_ok` helpers; + - lines 752-821: the invalid-patch table and `update_args_schema_makes_only_id_required`. + - crates/agent-memory/src/rest/handlers.rs: + - lines 95-127: `UpdateBody`; + - lines 878-952: `patch_raw` / `patch_request` / `assert_400_containing`; + - lines 1095-1131: `malformed_or_empty_patches_are_400_never_500`. + - crates/agent-memory/tests/tools.rs lines 1-120 and 174-206 (Server harness, `call_tool`, and the JSON-RPC error assertion pattern) + - crates/agent-memory/src/serde_util.rs (`deserialize_explicit_null`, which must keep working beside deny_unknown_fields) + - README.md memory_update section (~lines 199-243) + - .planning/STATE.md Decisions list, including the [03-01] deny_unknown_fields entry + + + - **Stdio end-to-end (real binary):** + - memory_store a row; + - memory_update `{"id": N, "content": "changed", "type": "DECISION"}` → a JSON-RPC `error` member with code -32602, and a message containing `unknown field`; + - memory_update `{"id": N, "content": "changed", "ttl_sec": 60}` → the same; + - memory_list then shows the original content, unchanged. + - **MCP in-process:** + - deserializing `UpdateArgs` from `{"id":1,"content":"x","type":"DECISION"}`, from `{"id":1,"source":"y"}` and from `{"id":1,"ttl_sec":60}` fails with `unknown field`; + - every patch in the existing tags/scope/ttl absent/null/value matrices still deserializes and behaves identically; + - the `UpdateArgs` schema still has `required == ["id"]` and no `default` on any patch field, and now has `additionalProperties == false`. + - **REST in-process:** + - `patch_raw` with `{"content":"x","type":"DECISION"}` and with `{"ttl_sec":60}` → 400, with an `error` containing `unknown field`; + - the row is untouched afterwards; + - the existing malformed-patch table stays green. + + + Implements REVIEW WR-02 per its proposed fix, scoped to the two merge-patch DTOs. + + 1. **mcp.rs:** + - Add `#[serde(deny_unknown_fields)]` to `UpdateArgs`, beside the existing derive. Keep every field attribute (`default`, `skip_serializing_if`, `deserialize_with`) unchanged. + - Extend the `UpdateArgs` doc: unknown keys are rejected, which covers `type`/`source` (immutable) and typos, because a silently dropped member of a merge-patch is a silent partial write. + - Add `update_args_reject_unknown_and_immutable_fields`. + - Extend `update_args_schema_makes_only_id_required` with the `additionalProperties == false` assertion. + - The memory_update tool description gains: unknown or immutable fields such as type and source are rejected. + 2. **handlers.rs:** + - Add the same attribute and doc note to `UpdateBody`. + - Add `patch_with_an_unknown_or_immutable_field_is_400_and_writes_nothing`, using `patch_raw` and `assert_400_containing(…, "unknown field")`, then a list check that the row is unchanged. + 3. **tests/tools.rs:** add `memory_update_with_an_unknown_field_is_invalid_params_over_stdio`, per ``, using fresh request ids. + 4. **README.md, memory_update section:** unknown fields are rejected (`invalid_params` / HTTP 400 naming the field). `type` and `source` cannot be changed, and a misspelled field is an error, never silently ignored. + 5. **.planning/STATE.md:** add a Decisions bullet in the existing format. `[Phase 04]: 04-07: deny_unknown_fields added to UpdateArgs/UpdateBody only (new v0.1.0 surfaces, REVIEW WR-02); the [03-01] decision stands for the v0.0.1 DTOs, so the v0.1.0 release-notes delta for existing surfaces is unchanged`. + 6. Run `cargo fmt --all` and commit with explicit paths. + + Loosening a strict DTO later is additive. The change is made before v0.1.0 ships, so no released client depends on unknown fields being ignored. + + cargo test -p agent-memory --bins update && cargo test -p agent-memory --test tools memory_update_with_an_unknown_field_is_invalid_params_over_stdio && cargo clippy -p agent-memory --all-targets -- -D warnings + + + - `grep -c "deny_unknown_fields" crates/agent-memory/src/mcp.rs` outputs at least 1, and `grep -c "deny_unknown_fields" crates/agent-memory/src/rest/handlers.rs` outputs at least 1. + - These pass: + - `cargo test -p agent-memory --bins update_args_reject_unknown_and_immutable_fields` + - `update_args_schema_makes_only_id_required` (now including additionalProperties) + - `patch_with_an_unknown_or_immutable_field_is_400_and_writes_nothing` + - every pre-existing update test in both files, unchanged + - The stdio end-to-end test passes against the real binary and asserts code -32602. + - `grep -n "04-07: deny_unknown_fields" .planning/STATE.md` is non-empty. + - clippy `-D warnings` and fmt are clean. + + An agent or REST client that sends a field memory_update does not accept gets a client error naming it, on rmcp's real parameter path and over HTTP. The memory is untouched. + + + + Task 2: Malformed path and query parameters get the same JSON error body as every other REST client error + crates/agent-memory/src/rest/handlers.rs, README.md + + - crates/agent-memory/src/rest/handlers.rs: + - lines 13-17: imports; + - lines 208-228: `list_handler` with `Query`; + - lines 255-308: `forget_handler` / `update_handler` with `Path`; + - lines 360-396: `unlink_handler` with `Path` of a tuple; + - lines 878-905: `patch_request`, which passes `Path(id)` and must pass `Ok(Path(id))` after this change; + - lines 1373-1444: `links_routes_work_over_real_http`, the real-HTTP test pattern to copy. + - crates/agent-memory/src/rest/mod.rs lines 27-49 (`ApiError` and its `{"error"}` body) + - every in-file test that calls `forget_handler`, `update_handler`, `unlink_handler` or `list_handler` directly with `Path(..)` / `Query(..)`, all of which must be wrapped in `Ok(..)` + - README.md REST API section (~lines 343-392) + + + Over real HTTP (in-process `axum::serve` on 127.0.0.1:0, plus reqwest, as in `links_routes_work_over_real_http`): + - `PATCH /api/memories/abc` with a valid JSON body → 400; + - `DELETE /api/memories/abc` → 400; + - `DELETE /api/links/x/2/relates_to` → 400; + - `GET /api/memories?expand_links=1` → 400; + - `GET /api/memories?limit=abc` → 400. + + Each response has a JSON body whose `error` member is a non-empty string, and `Content-Type: application/json`. Every existing handler test stays green. + + + Implements REVIEW IN-02. + + 1. Import `PathRejection` and `QueryRejection` from `axum::extract::rejection`. + 2. Add a private generic helper (or two small fns) that turns a rejection into `ApiError` per "Interface contracts". It classifies by `status().is_server_error()`, so the tier is preserved, and the message comes from `body_text()`. + 3. Change these handlers to take `Result<…, …Rejection>` and map with the helper as their first statement, mirroring how `update_handler` already treats its `JsonRejection`: + - `forget_handler` and `update_handler`: `Result, PathRejection>`; + - `unlink_handler`: `Result, PathRejection>`; + - `list_handler`: `Result, QueryRejection>`. + + Update their doc comments. + 4. Update every direct in-file test call site to pass `Ok(Path(..))` / `Ok(Query(..))`. + 5. Add `malformed_path_and_query_params_get_the_json_error_body`, per ``, using the real-HTTP pattern. Abort the spawned server at the end. + 6. **README REST section:** add one sentence. Every client error, including a malformed path segment or query parameter, returns a JSON `{"error": "..."}` body. + 7. Run `cargo fmt --all` and commit with explicit paths. + + + cargo test -p agent-memory --bins malformed_path_and_query_params_get_the_json_error_body && cargo test -p agent-memory --bins && cargo clippy -p agent-memory --all-targets -- -D warnings + + + - `grep -n "PathRejection" crates/agent-memory/src/rest/handlers.rs` and `grep -n "QueryRejection" crates/agent-memory/src/rest/handlers.rs` are non-empty. + - `cargo test -p agent-memory --bins malformed_path_and_query_params_get_the_json_error_body` passes all 5 requests, with a status and JSON `error` asserted for each. + - `cargo test -p agent-memory --bins` is fully green, with no pre-existing REST test changed beyond the `Ok(..)` wrapping. + - clippy `-D warnings` and fmt are clean. + + A REST client sees one error body shape for every client mistake, whether it is in the body, the path or the query string. + + + + Task 3: Two servers starting at once on a pre-0003 database both come up, and the one surviving backup is a genuine pre-migration copy + crates/agent-memory-core/src/store/sqlite.rs, crates/agent-memory-core/tests/migration_hygiene.rs, README.md + + - crates/agent-memory-core/src/store/sqlite.rs: + - lines 82-90: `apply_pragmas`, where busy_timeout is currently set last; + - lines 117-189: `utc_yyyymmdd` and `pre_migration_backup`, with its current delete-first retention; + - lines 207-276: `SqliteStore::open` and its probe → refuse → pragmas → backup → migrate order and comments; + - the existing `#[cfg(test)] mod tests`. + - crates/agent-memory-core/tests/migration_hygiene.rs: + - lines 80-106: `backups_for_latest`; + - lines 187-357: the upgrade, retention, fresh, too-new and failed-backup tests. All must stay green unchanged, apart from added leftover asserts. + - .planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md WR-01 and IN-03 + - README.md: the Install, Run and Behavioral changes sections, to choose where Upgrading goes (after Run, before MCP configuration) + + + Unit tests (sqlite.rs `mod tests`; real temp files; call `register_vec_extension` first): + - **Schema already migrated.** Setup: + - create a DB with `SqliteStore::open`, so it is at `LATEST`, and drop it; + - plant `{db}.backup-pre-{LATEST:04}-20000101` containing `valid`; + - open a raw connection. + + `pre_migration_backup(&conn, &db, 2)` → `Ok(None)`. The planted file still reads `valid`, and no directory entry contains `.backup-tmp-`. + - **Valid backup.** Setup: a raw DB with `PRAGMA user_version = 2` and one table, plus a planted stale `{prefix}20000101`. + + `pre_migration_backup(&conn, &db, 2)` → `Ok(Some(p))`. Then: + - `p`'s name ends with today's UTC `YYYYMMDD` (via `utc_yyyymmdd`); + - the copy's `user_version` is 2; + - the stale entry is gone; + - no `.backup-tmp-` entry remains. + - **Failed write.** Setup: + - the same v2 DB and planted stale backup; + - a non-empty DIRECTORY occupying this process's temp path `{db}.backup-tmp-{LATEST:04}-{std::process::id()}`. + + The call → `Err`, with a message containing `backup`. The planted stale backup still exists. This is IN-03: no deletion before a valid replacement exists. + - **Concurrent completion.** + - `migrate_tolerating_concurrent_upgrade` with a closure returning `Err(MemoryError::Internal("simulated: table memory_links already exists"))` → `Ok(())` when the DB's `user_version` is `LATEST_SCHEMA_VERSION`; + - the same closure → `Err` carrying that message when `user_version` is 2; + - an `Ok` closure → `Ok`. + + Integration (migration_hygiene.rs): + - `upgrading_writes_one_pre_migration_backup_with_the_old_schema` and `re_running_the_upgrade_keeps_one_backup_per_schema_version` additionally assert that no `.backup-tmp-` entry remains in the directory. + - `a_failed_backup_aborts_the_open_before_migrating`, `a_newer_schema_database_is_refused_and_left_untouched` (byte-for-byte) and `opening_a_fresh_database_creates_no_backup` stay green unchanged. + - NEW `two_concurrent_first_opens_of_a_v0_0_1_database_both_succeed_with_one_genuine_backup`. This is a smoke test that exercises the real race window. The deterministic seams above remain the proof. + - Setup: `copy_fixture` onto a temp `memory.db`, then a `std::sync::Barrier` of 2. + - Two `std::thread`s each wait on the barrier, then call `SqliteStore::open(&db)` and drop the store. + - Both opens return `Ok`. + - The DB's `user_version` is `LATEST_SCHEMA_VERSION` (3). + - `backups_for_latest(&db)` has exactly one entry, and its `user_version` is 2 (`V0_0_1_SCHEMA_VERSION`). + - No `.backup-tmp-` entry remains. + - Run the race 10 times, each on a fresh temp dir, so a real interleaving is likely to be hit at least once. + - Join every thread with no `unwrap()` hiding a panic message: map a join error to a test failure that carries the panic payload. + + + Implements REVIEW WR-01 (the review's "cheaper alternative": temp, verify, rename, and tolerate a completed concurrent migration) and IN-03. No new dependency, no migration, and 0001/0002/0003 SQL untouched (STORE-05 freeze). + + 1. **`apply_pragmas`:** move `busy_timeout` to the first statement, so `journal_mode` and every later pragma on writer and pool connections wait out a concurrent opener instead of failing with SQLITE_BUSY. Comment why. + 2. **`SqliteStore::open`:** + - Immediately after `Connection::open`, call `writer.busy_timeout(Duration::from_millis(5000))`, mapping the error with `?`. Comment that it is connection-level state, so the D-13 probe still leaves a refused file byte-for-byte untouched. + - Pass `db_version` into `pre_migration_backup`; ignore the returned `Option` apart from logging. + - Replace the bare `to_latest` call with `migrate_tolerating_concurrent_upgrade`, wrapping `migrations.to_latest(conn)` and mapping its error via `MemoryError::from`. + - Update the numbered doc comment on `open` to the new order (see "Interface contracts"). + 3. **Rewrite `pre_migration_backup`** exactly per "Interface contracts": + - temp name outside the retention prefix; + - Online Backup into the temp file; + - `user_version` check on the copy; + - drop the destination connection; + - rename into place; + - prune the other same-version entries, keeping the final path; + - best-effort temp removal on every failure path; + - keep the existing `fail(..)` message shape, so the abort still names "backup". + + Update its doc: the retention rule is now "replace, then prune", and the copy is discarded if another process migrated meanwhile (WR-01). + 4. **Add `fn migrate_tolerating_concurrent_upgrade`** per "Interface contracts", with a doc comment explaining the failure it absorbs. rusqlite_migration reads `user_version` outside its own transaction, so a second opener can attempt 0003's CREATE TABLE after the first committed it. Note that the smoke checks after it still run. + 5. Add the four unit tests to sqlite.rs `mod tests`, the leftover assertions to the two migration_hygiene.rs tests, and the two-thread concurrent-open smoke test to migration_hygiene.rs, all per ``. The project hook blocks `DROP`, so do not use it; none is needed. + 6. **README.md:** add a "## Upgrading" section after "## Run". It covers: + - the pre-migration backup `memory.db.backup-pre-NNNN-YYYYMMDD`, written next to the DB with SQLite's online backup before any schema migration; + - one backup per target schema version, replaced atomically on a re-run, and other versions' backups never touched; + - a backup that cannot be written means the server refuses to start and does not migrate; + - several servers starting at once is safe: one migration, one genuine backup; + - restore steps: stop every agent-memory process (MCP servers and the REST daemon); copy the backup over `memory.db`; delete `memory.db-wal` and `memory.db-shm`; start the matching older binary; + - a database written by a newer agent-memory is refused at startup with a message naming both schema versions and `brew upgrade unityinflow/tap/agent-memory`, and the file is left untouched. + 7. Run `cargo fmt --all` and commit with explicit paths. + + Backup file naming and retention semantics are unchanged for users; only the write order and the race handling change. Nothing persistent changes shape. + + cargo test -p agent-memory-core --lib store::sqlite && cargo test -p agent-memory-core --test migration_hygiene && cargo clippy -p agent-memory-core --all-targets -- -D warnings + + + - These greps are each non-empty: + - `grep -n "fn migrate_tolerating_concurrent_upgrade" crates/agent-memory-core/src/store/sqlite.rs` + - `grep -n "pre_migration_backup(&writer, path, db_version)" crates/agent-memory-core/src/store/sqlite.rs` + - `grep -n "backup-tmp-" crates/agent-memory-core/src/store/sqlite.rs` + - `grep -n "writer.busy_timeout" crates/agent-memory-core/src/store/sqlite.rs` + - `cargo test -p agent-memory-core --lib store::sqlite` passes the 4 new seam tests. + - `cargo test -p agent-memory-core --test migration_hygiene` is fully green: 8 tests (the 7 existing plus `two_concurrent_first_opens_of_a_v0_0_1_database_both_succeed_with_one_genuine_backup`) and the ignored generator. That includes the byte-for-byte too-new refusal and the failed-backup abort. + - The concurrent smoke test is green on 3 consecutive local runs of `cargo test -p agent-memory-core --test migration_hygiene two_concurrent`. + - If it is ever flaky, treat that as a real defect in the race handling and fix the code. Never weaken the assertions or drop the test. + - Record the run count in SUMMARY.md. + - `git diff --stat -- crates/agent-memory-core/sql/` is empty. The migrations are frozen. + - `grep -n "## Upgrading" README.md` is non-empty, and the section names `memory.db-wal`, `memory.db-shm` and the brew upgrade command. + - The full gate is green: `cargo fmt --all --check`, `cargo clippy --workspace --all-targets -- -D warnings`, and `cargo test --workspace`. Run the full test and clippy runs in the background and poll. + + The first open after an upgrade is safe under concurrent starts. The surviving backup always carries the pre-migration schema, a failed backup never costs the previous one, and users know how to restore. + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| MCP stdio client → memory_update arguments | untrusted JSON object; rmcp deserializes it directly into `UpdateArgs` | +| REST client (loopback) → PATCH body, path segments, query string | untrusted; axum extractors reject malformed parts | +| process ↔ process at startup | several agent-memory processes can open (and migrate) the same DB file at once | +| DB directory | backup and temp files live beside the DB, in the same trust domain (T-04-03) | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Severity | Disposition | Mitigation Plan | +|-----------|----------|-----------|----------|-------------|-----------------| +| T-04-27 | Tampering (integrity) | silent partial write: an unknown or immutable merge-patch member ignored while the call reports success | medium | mitigate | `deny_unknown_fields` on both patch DTOs. Rejections are tested over the stdio binary, rmcp's real path, and REST, and assert that the row is untouched (Task 1). | +| T-04-28 | Information Disclosure | deserialization error text echoing field names | low | accept | The text names only the client's own key and the DTO's public field list, which the tool schema already publishes. There is no server state in it. | +| T-04-29 | Tampering / Repudiation | the rollback path destroyed: a post-migration copy stored under the "pre-0003" name, overwriting the genuine backup | high | mitigate | Back up to a temp file outside the retention prefix, and check the copy's `user_version` against the probed version before keeping it. Rename into place before pruning. Deterministic seam tests (Task 3). | +| T-04-30 | Denial of Service | a concurrent first open failing startup (`table memory_links already exists` / SQLITE_BUSY) | medium | mitigate | `busy_timeout` set first on every connection. A `to_latest` failure is tolerated only when a re-probe shows `LATEST_SCHEMA_VERSION`; anything else still fails loudly. Tested via the closure seam (Task 3). | +| T-04-31 | Information Disclosure | plain-text extractor rejection bodies on new routes | low | mitigate | Mapped into the uniform `{"error"}` body with the status tier preserved, and tested over real HTTP (Task 2). | +| T-04-32 | Tampering | a stale temp backup left behind by a crashed process | low | accept | It is pid-qualified and lives in the DB's own directory (same trust domain). It is never pruned automatically, so a live concurrent opener's temp is never deleted. The Upgrading section documents the backup files. | +| T-04-SC | Tampering | cargo installs | low | accept | Zero new dependencies. axum rejection types, rusqlite `busy_timeout` and serde attributes are already available. The pin triangle is untouched. | + +No `high` severity threat is left open (T-04-29 is mitigated). ASVS L1, block_on: high. + + +## Deferred (findings and items NOT taken by this plan) + +| ID | Disposition | Reason | +|----|-------------|--------| +| CR-01, CR-02, VERIFICATION truths #14/#15, IN-04 | → plan 04-05 | The search contract. | +| CR-03, WR-03, IN-05 (doc), IN-06 (doc) | → plan 04-06 | Write-path integrity. | +| WR-04 (unbounded `memory_list` with `expand_links` and no `limit`) | → plan 04-06 (doc correction); behaviour change deferred | The doc and README correction is in 04-06 Task 3. The behaviour change is an API-contract decision, left open for Phase 5/6; see 04-06's Deferred section. | +| WR-01 full fix with a cross-process exclusive lock (lock file or `BEGIN IMMEDIATE` spanning backup and migration) | deferred | Needs either a new dependency (a file-lock crate) or restructuring rusqlite_migration's own transaction. The verify-then-rename plus re-probe approach closes both observable failure modes (startup failure, wrong backup) without either. | +| `deny_unknown_fields` on the v0.0.1 DTOs (store, list, search, forget) and on the link DTOs | deferred, per the [03-01] decision | It would add a v0.1.0 behavioural change to existing surfaces. The link DTOs have only required fields, so an unknown key cannot cause a partial write. | +| Automatic cleanup of stale `.backup-tmp-*` files from crashed processes | deferred | A safe rule needs liveness detection of the owning pid. Crash-only and harmless (T-04-32). | +| 03-REVIEW WR-01 (filters after the KNN cut), 03-REVIEW WR-03 (REST harness hang) | still deferred (not re-opened) | Per the brief. | + + +- `cargo test -p agent-memory --bins` and `cargo test -p agent-memory --test tools` (strict DTOs, rejection bodies, stdio end-to-end) +- `cargo test -p agent-memory-core --lib store::sqlite` and `cargo test -p agent-memory-core --test migration_hygiene` (backup race and failure seams; STORE-05 regression suite) +- Phase gate, run in the background and polled: + - `cargo fmt --all --check` + - `cargo clippy --workspace --all-targets -- -D warnings` + - `cargo test --workspace` +- `git diff --stat -- crates/agent-memory-core/sql/` is empty. +- Commit per task with explicit paths. No CI workflow change, no tag, release or push. + + + +- REVIEW WR-02 is closed: no update payload is partially applied in silence, on either transport. +- REVIEW IN-02 is closed: one JSON error body shape for every REST client error. +- REVIEW WR-01 and IN-03 are closed: concurrent first opens start cleanly, and the surviving backup is always a genuine pre-migration copy, never deleted before its replacement exists. +- REVIEW IN-01 is closed: the README documents backup, restore and the newer-schema refusal. + + + +Create `.planning/phases/04-memory-update-relations-hybrid-search/04-07-SUMMARY.md` when done. + From 289edbe40876dd6dd3e7505ef27a33991883dfbb Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 21:42:47 +0200 Subject: [PATCH 20/42] =?UTF-8?q?feat(04-05):=20forgiving=20keyword=20leg?= =?UTF-8?q?=20=E2=80=94=20raw=20FTS5=20first,=20one=20sanitized=20retry,?= =?UTF-8?q?=20redacted=20search=20errors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements user decision option-a (2026-10-23) at the shared store seam: - sqlite.rs: is_match_input_error (SQLITE_ERROR + FTS5 query-parse markers, incl. the step-time `no such column` class from hyphens/colons), sanitize_fts_query (word tokens quoted and OR-joined, None when tokenless) and match_with_sanitized_retry, used by both Store::search (keyword degrade) and Store::keyword_candidates (hybrid leg). SQL strings unchanged. The marker-only map_fts_query_error is removed. - service.rs: SEARCH_STORAGE_FAILURE + redact_search_error (exhaustive); search() = search_unredacted().map_err(redact_search_error), so no SQLite text reaches any search response body (T-04-19). - Old 400-pinning tests reversed in the same commit (hybrid.rs, fallback.rs, mcp.rs, handlers.rs, tests/rest.rs); sqlite-vec and a punctuated question recall the target memory in both modes. Closes VERIFICATION truths #14/#15, REVIEW CR-01/CR-02 (T-04-15 revised). Co-Authored-By: Claude Opus 5.5 --- crates/agent-memory-core/src/service.rs | 84 ++++++-- crates/agent-memory-core/src/store/mod.rs | 17 +- crates/agent-memory-core/src/store/sqlite.rs | 196 +++++++++++++------ crates/agent-memory-core/tests/fallback.rs | 21 +- crates/agent-memory-core/tests/hybrid.rs | 95 ++++++--- crates/agent-memory/src/mcp.rs | 21 +- crates/agent-memory/src/rest/handlers.rs | 54 ++--- crates/agent-memory/tests/rest.rs | 43 ++-- 8 files changed, 370 insertions(+), 161 deletions(-) diff --git a/crates/agent-memory-core/src/service.rs b/crates/agent-memory-core/src/service.rs index 7c8b307..6ced680 100644 --- a/crates/agent-memory-core/src/service.rs +++ b/crates/agent-memory-core/src/service.rs @@ -25,6 +25,40 @@ use crate::embed::Embedder; use crate::store::sqlite::DEFAULT_SEARCH_LIMIT; use crate::store::Store; +/// The only message a client ever sees for a storage-tier search failure +/// (T-04-19). The underlying error goes to the stderr log, never to a response +/// body, so no SQLite text (table/column names, engine detail) leaks out of +/// any search path. Both transports surface it on the internal tier (REST 500 +/// / MCP `internal_error`). +pub const SEARCH_STORAGE_FAILURE: &str = + "search failed due to a storage error (details are in the server log)"; + +/// Tier every search-path error for the wire — the ONE place this happens. +/// +/// - Client tier (`InvalidType`, `InvalidQuery`, `InvalidArgument`) passes +/// through: its text echoes only client input and public bounds. +/// - Crate-authored internal text (`Internal`, `SchemaTooNew`) passes through. +/// - Storage-engine errors (`Sqlite`, `Pool`, `Join`, `Migration`) are logged +/// with full detail, then replaced by [`SEARCH_STORAGE_FAILURE`]. +/// +/// No catch-all arm: a new [`MemoryError`] variant must be tiered here at +/// compile time. +fn redact_search_error(e: MemoryError) -> MemoryError { + match e { + MemoryError::InvalidType(_) + | MemoryError::InvalidQuery(_) + | MemoryError::InvalidArgument(_) => e, + MemoryError::Internal(_) | MemoryError::SchemaTooNew { .. } => e, + MemoryError::Sqlite(_) + | MemoryError::Pool(_) + | MemoryError::Join(_) + | MemoryError::Migration(_) => { + tracing::error!(error = %e, "search failed with a storage error"); + MemoryError::Internal(SEARCH_STORAGE_FAILURE.to_string()) + } + } +} + /// Filters for [`MemoryService::list`]. All fields are optional; a `None` field /// disables that filter. `limit` caps the number of rows returned. #[derive(Debug, Clone, Default)] @@ -38,8 +72,12 @@ pub struct ListArgs { pub expand_links: bool, } -/// Arguments for [`MemoryService::search`]. `query` is the FTS5 MATCH string; the -/// remaining fields are optional filters and a result cap. +/// Arguments for [`MemoryService::search`]. `query` is plain natural language or +/// FTS5 MATCH syntax: valid FTS5 runs exactly as written, anything FTS5 cannot +/// parse is retried once as its words OR-ed together, and a query with no words +/// contributes no keyword matches — it is never rejected for its syntax, in +/// either mode (T-04-15 revised). The remaining fields are optional filters and +/// a result cap. #[derive(Debug, Clone, Default)] pub struct SearchArgs { pub query: String, @@ -218,18 +256,36 @@ impl MemoryService { /// multiplied by the pinned-aware decay score once, and the result is /// truncated to the limit and returned as [`SearchMode::Hybrid`]. An /// empty result is a valid hybrid answer (Pitfall 2: emptiness never - /// means "fall back"). A leg error is never swallowed: a malformed - /// MATCH is [`MemoryError::InvalidQuery`] here exactly as in keyword - /// mode, so hybrid never silently serves semantic-only results (T-04-15). + /// means "fall back"). A leg's internal error is never swallowed, so + /// hybrid never silently serves one leg's rows as a fused answer. /// - Embed Err → one loud warning, then the UNCHANGED Phase-1 FTS5 - /// bm25×decay path, returned as [`SearchMode::Keyword`]. A malformed FTS5 - /// query error on this path maps exactly as before. Never an error and - /// never empty-because-of-outage (SEARCH-03). + /// bm25×decay path, returned as [`SearchMode::Keyword`]. Never an error + /// and never empty-because-of-outage (SEARCH-03). + /// + /// **Query handling is identical in both modes** (user decision option-a, + /// T-04-15 revised): both keyword paths share one store helper that runs + /// the raw FTS5 MATCH first, retries FTS5-unparseable input once as its + /// word tokens OR-ed together, and treats a tokenless query as no keyword + /// matches. Hybrid then answers from the semantic leg alone; keyword mode + /// returns an empty set. Search never returns [`MemoryError::InvalidQuery`]. /// /// Both paths fire the fire-and-forget recency bump on the returned ids /// only — never on fusion candidates or on expanded neighbors — and then /// attach the 1-hop expansion when `expand_links` is set. + /// + /// Every error, from either arm and from the expansion, leaves through + /// [`redact_search_error`]: a storage failure reaches the client only as + /// [`MemoryError::Internal`] carrying [`SEARCH_STORAGE_FAILURE`], with the + /// detail in the stderr log (T-04-19). pub async fn search(&self, args: SearchArgs) -> Result { + self.search_unredacted(args) + .await + .map_err(redact_search_error) + } + + /// The body of [`Self::search`], before its errors are tiered for the wire + /// by [`redact_search_error`]. + async fn search_unredacted(&self, args: SearchArgs) -> Result { // API-02 / D-01: validate before any embed hop — an out-of-bounds // limit rejects with MemoryError::InvalidArgument; the old silent // zero-clamp is gone, so bad input never truncates quietly. @@ -286,7 +342,7 @@ impl MemoryService { apply_decay(view, now, &cfg); } - // Same order as the semantic leg: bump the returned ids, then + // Same order as the hybrid path: bump the returned ids, then // expand — neighbors are never bumped (D-06). self.spawn_bump(views.iter().map(|v| v.id).collect(), now); let views = if expand_links { @@ -393,10 +449,12 @@ impl MemoryService { /// 1. Both legs run concurrently on blocking threads: [`Store::knn_search`] /// (cosine-distance order) and [`Store::keyword_candidates`] (bm25 /// order). Both are pure relevance and both oversample `limit * 4`. - /// 2. Errors propagate, the keyword leg's first: the KNN leg never parses - /// the query, so checking the keyword leg first means a malformed MATCH - /// is always [`MemoryError::InvalidQuery`] (REST 400 / MCP - /// invalid_params), the same classification as keyword mode (T-04-15). + /// 2. Errors propagate, the keyword leg's first, so which internal error + /// surfaces is deterministic when both legs fail. The keyword leg never + /// fails on query syntax: it shares the keyword-mode helper (raw MATCH + /// first, one sanitized OR-of-words retry, tokenless → no candidates), + /// so classification is identical in both modes (T-04-15 revised). A + /// tokenless query fuses the KNN leg alone. /// 3. The two id lists are fused with [`rrf_fuse`]. /// 4. Decay is applied exactly ONCE, post-fusion and multiplicatively: /// `final = rrf_score × decay_score(now, last_accessed, half_life, diff --git a/crates/agent-memory-core/src/store/mod.rs b/crates/agent-memory-core/src/store/mod.rs index 89828f2..172f4d9 100644 --- a/crates/agent-memory-core/src/store/mod.rs +++ b/crates/agent-memory-core/src/store/mod.rs @@ -72,6 +72,14 @@ pub trait Store: Send + Sync { /// row's `last_accessed` at `now` (D-04, recompute-on-read — Open Question 3). /// A query that matches nothing returns an empty vec, never an error (MCP-02). /// `weights` tunes the relevance/decay blend; `cfg` supplies the half-life. + /// + /// A natural-language query never fails this method (user decision + /// option-a, T-04-15 revised): the raw MATCH string runs first, so valid + /// FTS5 syntax keeps its meaning; input FTS5 cannot parse (punctuation, a + /// hyphen or colon read as a column filter, an unbalanced quote, …) gets + /// ONE retry as its word tokens quoted and OR-joined; a query with no word + /// tokens is an empty vec. This method never returns + /// [`MemoryError::InvalidQuery`] — only server faults are errors. fn search( &self, args: SearchArgs, @@ -88,9 +96,12 @@ pub trait Store: Send + Sync { /// Oversampled exactly like [`Store::knn_search`]: `limit * 4` rows (the /// limit defaults to 50), capped at the same KNN cap, so a hit ranked just /// past the limit still reaches fusion. Same type/scope/tag filters as - /// [`Store::search`] (`None` disables). A malformed MATCH string is - /// [`MemoryError::InvalidQuery`], exactly as on [`Store::search`]. A valid - /// query that matches nothing is an empty vec. + /// [`Store::search`] (`None` disables). The query is handled exactly as on + /// [`Store::search`], through the same helper, so classification does not + /// depend on the mode: raw MATCH first, one sanitized retry (word tokens + /// quoted and OR-joined) for input FTS5 cannot parse, and an empty vec for + /// a tokenless query. Never [`MemoryError::InvalidQuery`]; only server + /// faults are errors. A valid query that matches nothing is an empty vec. /// /// [`Store::search`] itself is unchanged: it is the keyword-only degrade /// path used when the embedder is down. diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index 1d43277..fcf78b5 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -318,24 +318,107 @@ fn corrupt_column(what: &str, value: &str) -> rusqlite::Error { ) } -/// Classify a post-bind FTS5 execution error at the store seam: a MATCH string -/// FTS5 cannot parse is CLIENT input → [`MemoryError::InvalidQuery`]; anything -/// else stays an internal [`MemoryError::Sqlite`]. +/// FTS5 query-parse markers, matched against the SQLite error message. /// -/// Observed FTS5 query-parse messages from the bundled SQLite: a lone `"` -/// reports `unterminated string`; other malformed expressions (dangling -/// operators, bad NEAR syntax, …) report `fts5: syntax error`. The regression -/// test in `tests/fallback.rs` defines correctness; the string markers serve it. +/// - `fts5: ` covers `fts5: syntax error near …` (bare punctuation, a dangling +/// `AND`, an empty query) and `fts5: parser stack overflow` (deep nesting). +/// - `unterminated string`: an unbalanced `"`. +/// - `no such column`: a hyphen or colon makes FTS5 read a column filter +/// (`sqlite-vec` → column `vec`), which fails on the one-column table. +/// - `unknown special query`: a leading `*`. +/// - `expected integer`: a non-numeric `NEAR(…, x)` distance. /// -/// Only errors surfacing AFTER the MATCH parameter is bound (statement step / -/// row iteration) go through here — a `conn.prepare` failure (e.g. missing -/// fts5 module) is an internal error, not client input. -fn map_fts_query_error(query: &str, e: rusqlite::Error) -> MemoryError { - let msg = e.to_string(); - if msg.contains("fts5: syntax error") || msg.contains("unterminated string") { - MemoryError::InvalidQuery(query.to_string()) +/// Only FTS5 query-parse messages belong here, never a generic SQLite message +/// (T-04-22): a marker match is what licenses the sanitized retry. +const FTS5_QUERY_PARSE_MARKERS: [&str; 5] = [ + "fts5: ", + "unterminated string", + "no such column", + "unknown special query", + "expected integer", +]; + +/// Is `e` FTS5 rejecting the bound MATCH string as client input? +/// +/// True only when the error is a `SqliteFailure` whose primary result code is +/// `SQLITE_ERROR` AND whose message carries one of +/// [`FTS5_QUERY_PARSE_MARKERS`]. The message is read from the failure's detail +/// string, falling back to the error's `Display`. +/// +/// Invariant: this is only ever handed errors raised AFTER a static statement +/// prepared successfully (statement step / row iteration), so every table and +/// column the SQL names exists. A `no such column` here can therefore only +/// come from FTS5 parsing a column filter out of the MATCH string on the +/// single-column `memories_fts(content)` table. Prepare-time failures (a +/// missing table, a missing fts5 module) never reach this function. +fn is_match_input_error(e: &rusqlite::Error) -> bool { + let rusqlite::Error::SqliteFailure(code, detail) = e else { + return false; + }; + if code.extended_code & 0xff != rusqlite::ffi::SQLITE_ERROR { + return false; + } + let msg = detail.clone().unwrap_or_else(|| e.to_string()); + FTS5_QUERY_PARSE_MARKERS + .iter() + .any(|marker| msg.contains(marker)) +} + +/// Rewrite `raw` as an FTS5 expression that always parses: its word tokens, +/// each quoted as an FTS5 string, joined with ` OR ` (order preserved, no +/// dedupe, no cap). +/// +/// A token is a maximal run of `char::is_alphanumeric` or `_` characters; +/// every other character splits. Tokens therefore cannot contain `"`, but any +/// embedded quote is doubled anyway, as FTS5 string syntax requires. Returns +/// `None` when no token remains (empty, punctuation-only, `*`): the keyword +/// leg then has nothing to match. +fn sanitize_fts_query(raw: &str) -> Option { + let terms: Vec = raw + .split(|c: char| !(c.is_alphanumeric() || c == '_')) + .filter(|piece| !piece.is_empty()) + .map(|piece| format!("\"{}\"", piece.replace('"', "\"\""))) + .collect(); + if terms.is_empty() { + None } else { - MemoryError::Sqlite(e) + Some(terms.join(" OR ")) + } +} + +/// Run a keyword MATCH raw first, with ONE sanitized retry (user decision +/// option-a, 2026-10-23; T-04-15 revised). +/// +/// - `run(raw)` succeeds → its rows. Valid structured FTS5 syntax (`AND`, +/// quoted phrases, `prefix*`) therefore keeps its exact meaning. +/// - `run(raw)` fails with an FTS5 query-parse error ([`is_match_input_error`]) +/// → the query is retried once as [`sanitize_fts_query`]. A tokenless query +/// is an empty answer, never an error. Any failure of the retry is a server +/// fault ([`MemoryError::Sqlite`]): the sanitized form parses by +/// construction. +/// - `run(raw)` fails any other way → [`MemoryError::Sqlite`]. +/// +/// Both [`Store::search`] (the keyword-degrade path) and +/// [`Store::keyword_candidates`] (the hybrid keyword leg) go through here, so a +/// query classifies identically with the embedder up or down. +fn match_with_sanitized_retry( + raw: &str, + mut run: impl FnMut(&str) -> Result, rusqlite::Error>, +) -> Result, MemoryError> { + match run(raw) { + Ok(rows) => Ok(rows), + Err(e) if is_match_input_error(&e) => { + // Log the size only: the query text is user content. + tracing::debug!( + query_chars = raw.chars().count(), + "FTS5 could not parse the search query; retrying with its words OR-ed" + ); + match sanitize_fts_query(raw) { + None => Ok(Vec::new()), + Some(fallback) => run(&fallback).map_err(MemoryError::Sqlite), + } + } + Err(e) => Err(MemoryError::Sqlite(e)), } } @@ -552,7 +635,7 @@ impl Store for SqliteStore { // half-life for pinned types (DECISION/ARCHITECTURE/CONSTRAINT, D-08). // All params bound — the FTS5 MATCH string is never concatenated (T-02-01). // The tag filter is an exact, case-sensitive json_each equality - // (API-03 / D-07); the FTS5 error routing below is untouched. + // (API-03 / D-07); the FTS5 MATCH retry below is unaffected by it. let mut stmt = conn.prepare( "SELECT m.id, m.mem_type, m.content, m.tags, m.source, m.scope, \ m.base_weight, m.decay_score, m.access_count, m.created_at, \ @@ -573,34 +656,32 @@ impl Store for SqliteStore { LIMIT ?9", )?; - let rows = stmt.query_map( - params![ - args.query, - mem_type_filter, - args.scope, - weights.relevance, - weights.decay, - now, - cfg.half_life_secs, - pinned_hl, - limit, - args.tag, - ], - row_to_view, - ); - - // A malformed FTS5 MATCH string surfaces at step time (the bound MATCH - // parses on execution, not prepare) and is mapped HERE, at the store - // seam, to MemoryError::InvalidQuery via map_fts_query_error — which - // both transports surface as a client error (REST 400 / MCP - // invalid_params), never a panic and never a 500 (T-02-02, WR-05). - // A valid query that matches nothing yields an empty vec. - let rows = rows.map_err(|e| map_fts_query_error(&args.query, e))?; - let mut out = Vec::new(); - for row in rows { - out.push(row.map_err(|e| map_fts_query_error(&args.query, e))?); - } - Ok(out) + // The bound MATCH string parses at step time, not at prepare, so an + // FTS5 parse error surfaces inside `run`. Revised T-04-15 (option-a): + // the raw query runs first, so structured FTS5 syntax keeps its + // meaning; FTS5-unparseable client input gets one sanitized retry; a + // tokenless query is an empty vec. This is the same helper the hybrid + // keyword leg uses, so both modes are equally forgiving. A valid query + // that matches nothing yields an empty vec. + let run = |match_query: &str| -> Result, rusqlite::Error> { + stmt.query_map( + params![ + match_query, + mem_type_filter, + args.scope, + weights.relevance, + weights.decay, + now, + cfg.half_life_secs, + pinned_hl, + limit, + args.tag, + ], + row_to_view, + )? + .collect() + }; + match_with_sanitized_retry(&args.query, run) } fn keyword_candidates(&self, args: SearchArgs) -> Result, MemoryError> { @@ -614,7 +695,7 @@ impl Store for SqliteStore { // ascending order is best first, with no negation and no decay term. // Decay is applied once by the service, after fusion. The id tie-break // keeps the leg deterministic. FROM/JOIN, filters and the - // map_fts_query_error routing are copied from `search`, and every + // match_with_sanitized_retry routing are copied from `search`, and every // value is bound, never concatenated (T-04-18). let mut stmt = conn.prepare( "SELECT m.id, m.mem_type, m.content, m.tags, m.source, m.scope, \ @@ -631,19 +712,18 @@ impl Store for SqliteStore { LIMIT ?5", )?; - let rows = stmt.query_map( - params![args.query, mem_type_filter, args.scope, args.tag, fetch], - row_to_view, - ); - - // A malformed MATCH surfaces at step time, as on `search`: it is - // client input (InvalidQuery), never an internal error (T-04-15). - let rows = rows.map_err(|e| map_fts_query_error(&args.query, e))?; - let mut out = Vec::new(); - for row in rows { - out.push(row.map_err(|e| map_fts_query_error(&args.query, e))?); - } - Ok(out) + // Revised T-04-15 (option-a): the same raw-first, one-sanitized-retry + // helper as `search`, so a query classifies identically with the + // embedder up or down. An FTS5-unparseable query never fails the leg, + // and a tokenless one contributes no keyword candidates. + let run = |match_query: &str| -> Result, rusqlite::Error> { + stmt.query_map( + params![match_query, mem_type_filter, args.scope, args.tag, fetch], + row_to_view, + )? + .collect() + }; + match_with_sanitized_retry(&args.query, run) } fn forget(&self, id: i64) -> Result { diff --git a/crates/agent-memory-core/tests/fallback.rs b/crates/agent-memory-core/tests/fallback.rs index 23ae527..976b8e6 100644 --- a/crates/agent-memory-core/tests/fallback.rs +++ b/crates/agent-memory-core/tests/fallback.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use agent_memory_core::clock::{Clock, TestClock}; use agent_memory_core::decay::DecayConfig; -use agent_memory_core::domain::{MemoryError, MemoryType, NewMemory}; +use agent_memory_core::domain::{MemoryType, NewMemory}; use agent_memory_core::embed::{Embedder, FakeEmbedder}; use agent_memory_core::service::{ListArgs, MemoryService, SearchArgs, SearchMode}; use agent_memory_core::store::sqlite::SqliteStore; @@ -194,10 +194,11 @@ async fn keyword_fallback_tag_filter_is_exact() { } #[tokio::test] -async fn malformed_fts5_query_maps_to_invalid_query_in_keyword_mode() { - // Gap 2 / WR-05 regression: a query FTS5 cannot parse (a lone double-quote) - // is CLIENT input, not an internal failure — the store seam must surface it - // as MemoryError::InvalidQuery, never MemoryError::Sqlite (API-01). +async fn a_lone_double_quote_is_an_empty_keyword_answer_never_an_error() { + // User decision option-a (2026-10-23) reverses the Gap 2 / WR-05 400: a + // query FTS5 cannot parse (a lone double-quote) is retried once with its + // word tokens, and a query with NO word tokens is simply no keyword + // matches. Keyword mode answers it with an empty set, never an error. let dir = tempfile::tempdir().expect("tempdir"); let db_path = dir.path().join("memory.db"); let clock = Arc::new(TestClock::new(1_000)); @@ -211,17 +212,19 @@ async fn malformed_fts5_query_maps_to_invalid_query_in_keyword_mode() { .await .expect("store succeeds while embedder is down"); - let err = service + let outcome = service .search(SearchArgs { query: "\"".to_string(), ..Default::default() }) .await - .expect_err("a lone double-quote is not a valid FTS5 match expression"); + .expect("a lone double-quote must be answered, never rejected"); + assert_eq!(outcome.search_mode, SearchMode::Keyword); assert!( - matches!(err, MemoryError::InvalidQuery(_)), - "malformed FTS5 input must map to InvalidQuery, got: {err:?}" + outcome.results.is_empty(), + "a tokenless query has no keyword matches, got: {:?}", + outcome.results ); } diff --git a/crates/agent-memory-core/tests/hybrid.rs b/crates/agent-memory-core/tests/hybrid.rs index 7995812..75dcfa8 100644 --- a/crates/agent-memory-core/tests/hybrid.rs +++ b/crates/agent-memory-core/tests/hybrid.rs @@ -25,7 +25,9 @@ use agent_memory_core::domain::{LinkKind, MemoryError, MemoryType, NewMemory}; use agent_memory_core::embed::{ BoxFuture, EmbedError, Embedder, EmbedderHealth, FakeEmbedder, EMBEDDING_DIM, }; -use agent_memory_core::service::{rrf_fuse, MemoryService, SearchArgs, SearchMode, RRF_K}; +use agent_memory_core::service::{ + rrf_fuse, MemoryService, SearchArgs, SearchMode, RRF_K, SEARCH_STORAGE_FAILURE, +}; use agent_memory_core::store::sqlite::SqliteStore; use agent_memory_core::store::Store; @@ -451,8 +453,9 @@ async fn filters_that_exclude_everything_stay_hybrid() { // | Embed OK, both legs OK | fuse -> "hybrid" (Test 2 above) | // | Embed OK, both legs empty | [] still "hybrid" (Test 5 above) | // | Embed Err / empty batch | unchanged v1.0 keyword path | -// | InvalidQuery from the keyword leg | propagate (400 / invalid_params) | -// | Internal SQLite error in either leg | propagate (500 / internal_error) | +// | Query FTS5 cannot parse (option-a) | raw first, one sanitized retry; | +// | | tokenless -> semantic-only | +// | Internal SQLite error in either leg | propagate, redacted (500) | // --------------------------------------------------------------------------- /// An embedder that answers with an EMPTY batch: the "invalid empty batch" @@ -536,17 +539,22 @@ async fn an_empty_embed_batch_degrades_to_the_unchanged_v1_keyword_path() { } #[tokio::test] -async fn a_malformed_match_is_invalid_query_in_hybrid_mode_exactly_as_in_keyword_mode() { - // T-04-15: the same input must classify the same way in every mode, never - // a 400 in one mode and a semantic-only 200 in the other. The KNN leg - // would happily answer these queries, so its rows must not leak out. - // FTS5 rejects bare punctuation too: a natural-language "?" query is - // client input in hybrid mode, exactly as it always was in keyword mode. - for malformed in ["\"unbalanced", "AND", "what is the plan?"] { - for (label, embedder) in [ - ("hybrid", live(HashMap::new())), +async fn queries_fts5_cannot_parse_succeed_identically_in_both_modes() { + // T-04-15 revised by user decision option-a (2026-10-23): the same input + // classifies the same way in every mode, and that classification is now + // "answered", never InvalidQuery and never a 500. The raw MATCH runs + // first; FTS5-unparseable input is retried once as its words OR-ed. + for input in [ + "\"unbalanced", + "AND", + "what is the plan?", + "sqlite-vec", + "auth: tokens", + ] { + for (expected_mode, embedder) in [ + (SearchMode::Hybrid, live(HashMap::new())), ( - "keyword", + SearchMode::Keyword, Arc::new(FakeEmbedder::failing()) as Arc, ), ] { @@ -557,14 +565,48 @@ async fn a_malformed_match_is_invalid_query_in_hybrid_mode_exactly_as_in_keyword MemoryType::Pattern, ) .await; - let err = h + let outcome = h .service - .search(query(malformed, None)) + .search(query(input, None)) .await - .expect_err("a malformed MATCH must not succeed in any mode"); - assert!( - matches!(err, MemoryError::InvalidQuery(ref q) if q == malformed), - "{label} mode: {malformed:?} must be InvalidQuery, got {err:?}" + .unwrap_or_else(|e| { + panic!("{expected_mode:?} mode: {input:?} must be answered, got {e:?}") + }); + assert_eq!( + outcome.search_mode, expected_mode, + "{input:?} must carry the mode's search_mode" + ); + } + } + + // The tracer slice: a hyphenated term and a punctuated question both + // recall the sqlite-vec memory first, with the embedder up or down. + for input in ["sqlite-vec", "what vector store do we use?"] { + for (expected_mode, embedder) in [ + (SearchMode::Hybrid, live(HashMap::new())), + ( + SearchMode::Keyword, + Arc::new(FakeEmbedder::failing()) as Arc, + ), + ] { + let h = harness(embedder); + let m1 = store( + &h, + "We chose sqlite-vec for local vector similarity search", + MemoryType::Pattern, + ) + .await; + store(&h, "auth tokens rotate every hour", MemoryType::Pattern).await; + let outcome = h + .service + .search(query(input, None)) + .await + .unwrap_or_else(|e| panic!("{expected_mode:?}: {input:?} failed: {e:?}")); + assert_eq!(outcome.search_mode, expected_mode, "{input:?}"); + assert_eq!( + ids(&outcome.results).first(), + Some(&m1), + "{expected_mode:?}: {input:?} must recall the sqlite-vec memory first" ); } } @@ -573,9 +615,10 @@ async fn a_malformed_match_is_invalid_query_in_hybrid_mode_exactly_as_in_keyword #[tokio::test] async fn an_internal_error_in_either_leg_propagates_and_never_serves_the_other_leg() { // Break one leg's index underneath a live store (renamed away, so the - // leg's SQL hits "no such table"): the search must fail with an internal - // Sqlite error (REST 500 / MCP internal_error), never quietly return the - // surviving leg's rows as a hybrid answer. + // leg's SQL hits "no such table"): the search must fail on the internal + // tier (REST 500 / MCP internal_error), never quietly return the + // surviving leg's rows as a hybrid answer. The error is redacted at the + // service seam (T-04-19): only SEARCH_STORAGE_FAILURE, no SQLite text. for leg_table in ["memories_fts", "vec_memories"] { let h = harness(live(HashMap::new())); store(&h, "ledger reconciliation runbook", MemoryType::Pattern).await; @@ -591,8 +634,12 @@ async fn an_internal_error_in_either_leg_propagates_and_never_serves_the_other_l .await .expect_err("a broken leg must fail the search"); assert!( - matches!(err, MemoryError::Sqlite(_)), - "with {leg_table} gone the search must surface an internal Sqlite error, got {err:?}" + matches!(err, MemoryError::Internal(ref m) if m == SEARCH_STORAGE_FAILURE), + "with {leg_table} gone the search must surface the generic internal error, got {err:?}" + ); + assert!( + !err.to_string().to_lowercase().contains("sqlite"), + "no SQLite text may reach the client: {err}" ); } } diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index 3d51545..c1de908 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -1060,22 +1060,25 @@ mod tests { } #[tokio::test] - async fn memory_search_malformed_query_is_invalid_params_in_hybrid_mode() { - // T-04-15: the keyword leg's InvalidQuery is never swallowed in favor - // of the semantic leg's rows. + async fn memory_search_answers_queries_fts5_cannot_parse_in_hybrid_mode() { + // User decision option-a (T-04-15 revised): a query FTS5 cannot parse + // is answered in hybrid mode exactly as in keyword mode — never + // invalid_params, never internal_error, never SQLite text. let dir = tempfile::tempdir().expect("tempdir"); let server = live_server(&dir); store_one(&server, "any searchable row").await; - let invalid_params_code = McpError::invalid_params("x", None).code; - for malformed in ["\"", "what is the plan?"] { - let err = server + for input in ["\"", "what is the plan?", "sqlite-vec"] { + let result = server .memory_search(Parameters(search_args(serde_json::json!({ - "query": malformed + "query": input })))) .await - .expect_err("a malformed MATCH is client input in hybrid mode"); - assert_eq!(err.code, invalid_params_code, "{malformed:?}: {err:?}"); + .unwrap_or_else(|e| panic!("{input:?} must be answered in hybrid mode: {e:?}")); + let raw = serde_json::to_string(&result).expect("tool result serializes"); + assert!(!raw.contains("sqlite error"), "{input:?}: {raw}"); + let body = result_json(&result); + assert_eq!(body["search_mode"], "hybrid", "{input:?}: {body}"); } } diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index 12102f7..25e9d02 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -616,9 +616,10 @@ mod tests { } #[tokio::test] - async fn search_with_malformed_query_returns_400_never_500() { - // Gap 2 / WR-05: a lone double-quote is not a valid FTS5 match - // expression — CLIENT input must map to 400, never a 500 (API-01). + async fn search_with_a_lone_quote_returns_200_with_an_empty_keyword_answer() { + // User decision option-a reverses the Gap 2 / WR-05 400: a lone + // double-quote has no word tokens, so the keyword path has nothing to + // match and answers 200 with an empty result set — never a 400 or 500. let dir = tempfile::tempdir().expect("tempdir"); let state = test_state(&dir); @@ -629,7 +630,7 @@ mod tests { .await .expect("store succeeds"); - let err = search_handler( + let resp = search_handler( State(state), Json(SearchRequest { query: "\"".to_string(), @@ -641,21 +642,16 @@ mod tests { }), ) .await - .expect_err("malformed FTS5 query is rejected as client input"); + .expect("a lone quote is answered, never rejected") + .into_response(); - let resp = err.into_response(); - assert_eq!( - resp.status(), - StatusCode::BAD_REQUEST, - "malformed FTS5 query must map to 400, never 500" - ); + assert_eq!(resp.status(), StatusCode::OK); let body = body_json(resp).await; - assert!( - !body["error"] - .as_str() - .expect("400 body carries a string error field") - .is_empty(), - "the error message must be non-empty, got: {body}" + assert_eq!(body["search_mode"], "keyword", "{body}"); + assert_eq!( + body["results"].as_array().map(Vec::len), + Some(0), + "a tokenless query has no keyword matches: {body}" ); } @@ -711,9 +707,10 @@ mod tests { } #[tokio::test] - async fn a_malformed_query_is_400_in_hybrid_mode_too_never_a_semantic_200() { - // T-04-15: with the embedder UP the KNN leg could answer, but the - // keyword leg's InvalidQuery must still surface as a 400. + async fn queries_fts5_cannot_parse_are_200_in_hybrid_mode_too() { + // T-04-15 revised (option-a): with the embedder UP, a query FTS5 + // cannot parse is answered exactly as in keyword mode — 200 with the + // hybrid envelope, never a 400, a 500, or SQLite text in the body. let dir = tempfile::tempdir().expect("tempdir"); let state = live_state(&dir); store_handler( @@ -723,14 +720,17 @@ mod tests { .await .expect("store succeeds"); - for malformed in ["\"", "what is the plan?"] { - let err = search_handler(State(state.clone()), Json(search_request(malformed))) + for input in ["\"", "what is the plan?", "sqlite-vec"] { + let resp = search_handler(State(state.clone()), Json(search_request(input))) .await - .expect_err("a malformed MATCH is client input in hybrid mode"); - assert_eq!( - err.into_response().status(), - StatusCode::BAD_REQUEST, - "{malformed:?} must be a 400 in hybrid mode, as in keyword mode" + .unwrap_or_else(|_| panic!("{input:?} must be answered in hybrid mode")) + .into_response(); + assert_eq!(resp.status(), StatusCode::OK, "{input:?}"); + let body = body_json(resp).await; + assert_eq!(body["search_mode"], "hybrid", "{input:?}: {body}"); + assert!( + !body.to_string().contains("sqlite error"), + "{input:?}: {body}" ); } } diff --git a/crates/agent-memory/tests/rest.rs b/crates/agent-memory/tests/rest.rs index 94c39de..7073212 100644 --- a/crates/agent-memory/tests/rest.rs +++ b/crates/agent-memory/tests/rest.rs @@ -198,24 +198,31 @@ async fn rest_end_to_end_store_list_search_forget_health() { "search results must contain the stored id {id}, got: {body}" ); - // POST /api/search with a malformed FTS5 query (a lone double-quote) in - // keyword mode → 400 with an error body — CLIENT input, never a 500. - let resp = client - .post(format!("{base}/api/search")) - .json(&serde_json::json!({ "query": "\"" })) - .send() - .await - .expect("malformed-query search request"); - assert_eq!( - resp.status(), - 400, - "malformed FTS5 query must map to 400, never 500" - ); - let body: serde_json::Value = resp.json().await.expect("400 body is JSON"); - assert!( - body.get("error").and_then(|e| e.as_str()).is_some(), - "400 body must carry an error field, got: {body}" - ); + // POST /api/search with queries FTS5 cannot parse, in keyword mode (user + // decision option-a): a lone double-quote (no word tokens → empty answer) + // and a hyphenated term (read by FTS5 as a column filter → retried with + // its words) are both 200 with a results array — never a 400, never a + // 500, and never SQLite text in the response. + for input in ["\"", "sqlite-vec"] { + let resp = client + .post(format!("{base}/api/search")) + .json(&serde_json::json!({ "query": input })) + .send() + .await + .expect("unparseable-query search request"); + assert_eq!( + resp.status(), + 200, + "{input:?} must be answered in keyword mode, never rejected" + ); + let text = resp.text().await.expect("search body reads"); + assert!(!text.contains("sqlite error"), "{input:?}: {text}"); + let body: serde_json::Value = serde_json::from_str(&text).expect("200 body is JSON"); + assert!( + body.get("results").and_then(|r| r.as_array()).is_some(), + "{input:?}: 200 body must carry a results array, got: {body}" + ); + } // POST /api/search with limit 0 → 400 carrying the valid range (API-02) — // realism-only proof over real HTTP; the coverage-bearing tests are the From a719dd1e511cd6183811a0deb66278ec435c9ad9 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 21:44:51 +0200 Subject: [PATCH 21/42] test(04-05): pin bundled-SQLite FTS5 parse classes, two-mode decision table, raw-first proofs - sqlite.rs unit tests: all 9 candidate erroring probes (empty, `what is it?`, `"unbalanced`, `AND`, `sqlite-vec`, `auth: tokens`, `*`, `NEAR(a b, x)`, 3000-deep parens) error in the bundled SQLite 3.51.3 and are classified; the 5 valid-raw probes run Ok; every sanitized fallback parses; classifier negatives (SQLITE_BUSY, `malformed JSON`, FromSqlConversionFailure); sanitize_fts_query table. - hybrid.rs: shared corpus M1-M5 decision table (7 targeted + 4 tokenless rows) in both modes; structured_fts5_syntax_still_runs_raw (exact id sets); keyword-mode broken-FTS redaction to SEARCH_STORAGE_FAILURE. - domain.rs: InvalidQuery documented as no longer produced by search. Co-Authored-By: Claude Opus 5.5 --- crates/agent-memory-core/src/domain.rs | 6 + crates/agent-memory-core/src/store/sqlite.rs | 161 ++++++++++++++++ crates/agent-memory-core/tests/hybrid.rs | 189 ++++++++++++++----- 3 files changed, 308 insertions(+), 48 deletions(-) diff --git a/crates/agent-memory-core/src/domain.rs b/crates/agent-memory-core/src/domain.rs index c429a3b..ec2243a 100644 --- a/crates/agent-memory-core/src/domain.rs +++ b/crates/agent-memory-core/src/domain.rs @@ -327,6 +327,12 @@ pub enum MemoryError { #[error("invalid memory type '{0}' (expected one of DECISION, PATTERN, ERROR, TODO, ARCHITECTURE, CONSTRAINT)")] InvalidType(String), + /// Client tier (REST 400 / MCP `invalid_params`). **Since v0.1.0, search + /// never produces it** (user decision option-a, T-04-15 revised): the + /// keyword leg retries FTS5-unparseable input once as its words OR-ed + /// together, and treats a tokenless query as no keyword matches, in both + /// modes. The variant is kept so both transport error mappers stay + /// exhaustive and the client-tier error vocabulary stays stable. #[error("invalid search query {0:?}: not a valid FTS5 match expression")] InvalidQuery(String), diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index fcf78b5..fced7b0 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -1077,4 +1077,165 @@ mod tests { let k_at_max = (crate::domain::MAX_LIMIT * 4).min(MAX_KNN_K); assert_eq!(k_at_max, crate::domain::MAX_LIMIT * 4); } + + // ----------------------------------------------------------------------- + // option-a (T-04-15 revised): the classifier, the sanitizer, and the + // always-parses fallback, pinned against the BUNDLED SQLite — the source + // of truth, not the system sqlite3 CLI the planning probe used. + // ----------------------------------------------------------------------- + + /// An in-memory one-column FTS5 table shaped like `memories_fts(content)`. + fn fts_probe_conn() -> Connection { + let conn = Connection::open_in_memory().expect("in-memory db"); + conn.execute_batch( + "CREATE VIRTUAL TABLE t USING fts5(content); + INSERT INTO t(content) VALUES ('sqlite with vec support');", + ) + .expect("create fts5 probe table"); + conn + } + + /// Bind `query` as the MATCH string of a statement that prepared fine, so + /// any error is a step-time error — exactly what the store seam sees. + fn probe(stmt: &mut rusqlite::Statement<'_>, query: &str) -> Result { + stmt.query_row([query], |row| row.get(0)) + } + + fn deep_parens() -> String { + format!("{}a{}", "(".repeat(3000), ")".repeat(3000)) + } + + /// Candidate probes from the planning table that FTS5 rejects. All nine + /// error in the bundled SQLite 3.51.3 exactly as in the sqlite3 3.46 CLI + /// probe (the empty string included), so none moved to the valid-raw group. + fn erroring_probes() -> Vec { + vec![ + String::new(), + "what is it?".to_string(), + "\"unbalanced".to_string(), + "AND".to_string(), + "sqlite-vec".to_string(), + "auth: tokens".to_string(), + "*".to_string(), + "NEAR(a b, x)".to_string(), + deep_parens(), + ] + } + + /// Structured FTS5 syntax that is valid as written: it runs raw, and the + /// sanitized retry never fires for it. + fn valid_raw_probes() -> Vec { + vec![ + "\"exact phrase\"".to_string(), + "sqlite AND vec".to_string(), + "vec*".to_string(), + "content: tokens".to_string(), + "\"\"".to_string(), + ] + } + + #[test] + fn every_bundled_fts5_parse_error_is_classified_as_match_input() { + let conn = fts_probe_conn(); + let mut stmt = conn + .prepare("SELECT count(*) FROM t WHERE t MATCH ?1") + .expect("static statement prepares"); + let erroring = erroring_probes(); + assert!(!erroring.is_empty()); + for query in &erroring { + let err = probe(&mut stmt, query) + .expect_err(&format!("{:.40?} must error in the bundled SQLite", query)); + assert!( + is_match_input_error(&err), + "{:.40?} must be classified as client input, got: {err}", + query + ); + } + } + + #[test] + fn valid_structured_fts5_syntax_runs_raw_in_the_bundled_sqlite() { + let conn = fts_probe_conn(); + let mut stmt = conn + .prepare("SELECT count(*) FROM t WHERE t MATCH ?1") + .expect("static statement prepares"); + for query in valid_raw_probes() { + probe(&mut stmt, &query) + .unwrap_or_else(|e| panic!("{query:?} is valid FTS5 and must run raw: {e}")); + } + } + + #[test] + fn the_sanitized_fallback_always_parses() { + let conn = fts_probe_conn(); + let mut stmt = conn + .prepare("SELECT count(*) FROM t WHERE t MATCH ?1") + .expect("static statement prepares"); + for query in erroring_probes().iter().chain(valid_raw_probes().iter()) { + if let Some(fallback) = sanitize_fts_query(query) { + probe(&mut stmt, &fallback).unwrap_or_else(|e| { + panic!("fallback {fallback:.60?} of {query:.40?} must parse: {e}") + }); + } + } + // The fallback finds the row through a hyphenated query's words. + let fallback = sanitize_fts_query("sqlite-vec").expect("has words"); + assert_eq!(probe(&mut stmt, &fallback).expect("parses"), 1); + } + + #[test] + fn the_classifier_rejects_everything_that_is_not_an_fts5_parse_error() { + let busy = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_BUSY), + Some("database is locked".to_string()), + ); + assert!(!is_match_input_error(&busy)); + // The primary code gates too: a parse marker under SQLITE_BUSY is no match. + let busy_with_marker = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_BUSY), + Some("fts5: syntax error near \"?\"".to_string()), + ); + assert!(!is_match_input_error(&busy_with_marker)); + // SQLITE_ERROR alone is not enough: the message must be an FTS5 parse. + let malformed_json = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR), + Some("malformed JSON".to_string()), + ); + assert!(!is_match_input_error(&malformed_json)); + assert!(!is_match_input_error(&corrupt_column("mem_type", "BOGUS"))); + // Positive control for the constructed form. + let syntax = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR), + Some("fts5: syntax error near \"?\"".to_string()), + ); + assert!(is_match_input_error(&syntax)); + } + + #[test] + fn sanitize_fts_query_quotes_and_or_joins_word_tokens() { + let table: [(&str, Option<&str>); 12] = [ + ( + "what vector store do we use?", + Some(r#""what" OR "vector" OR "store" OR "do" OR "we" OR "use""#), + ), + ("don't retry", Some(r#""don" OR "t" OR "retry""#)), + ("C++", Some(r#""C""#)), + ("sqlite-vec", Some(r#""sqlite" OR "vec""#)), + ("auth: tokens", Some(r#""auth" OR "tokens""#)), + ("\"unbalanced", Some(r#""unbalanced""#)), + ("AND", Some(r#""AND""#)), + ("snake_case", Some(r#""snake_case""#)), + ("über 日本語", Some(r#""über" OR "日本語""#)), + ("", None), + ("?!,", None), + ("*", None), + ]; + for (input, expected) in table { + assert_eq!( + sanitize_fts_query(input).as_deref(), + expected, + "sanitize_fts_query({input:?})" + ); + } + } } diff --git a/crates/agent-memory-core/tests/hybrid.rs b/crates/agent-memory-core/tests/hybrid.rs index 75dcfa8..4e9c07f 100644 --- a/crates/agent-memory-core/tests/hybrid.rs +++ b/crates/agent-memory-core/tests/hybrid.rs @@ -538,33 +538,63 @@ async fn an_empty_embed_batch_degrades_to_the_unchanged_v1_keyword_path() { assert_degrades_to_the_v1_keyword_path(Arc::new(EmptyBatchEmbedder)).await; } +/// The shared option-a corpus (04-05 plan): the same five PATTERN rows back +/// every decision-table test, at the core and on both transports. +const M1: &str = "We chose sqlite-vec for local vector similarity search"; +const M2: &str = "auth tokens rotate every hour"; +const M3: &str = "C++ build flags need care"; +const M4: &str = "don't retry the import blindly"; +const M5: &str = "the unbalanced ledger is reconciled nightly"; + +/// Queries FTS5 cannot parse as written, each with the corpus row (1-based +/// index into M1..M5) it must recall first in BOTH modes. +const TARGETED: [(&str, usize); 7] = [ + ("what vector store do we use?", 1), + ("sqlite-vec", 1), + ("don't retry", 4), + ("auth, tokens", 2), + ("auth: tokens", 2), + ("C++", 3), + ("\"unbalanced", 5), +]; + +/// Tokenless or no-hit queries: no keyword matches. Keyword mode answers an +/// empty set; hybrid answers from the semantic leg alone. (`AND` sanitizes +/// to the phrase "and", which no corpus row contains.) +const TOKENLESS_OR_NO_HIT: [&str; 4] = ["AND", "", "?!,", "*"]; + +/// Store M1..M5 as PATTERN rows at one TestClock instant (equal decay) and +/// return their ids in corpus order. +async fn store_corpus(h: &Harness) -> Vec { + let mut out = Vec::new(); + for content in [M1, M2, M3, M4, M5] { + out.push(store(h, content, MemoryType::Pattern).await); + } + out +} + +fn both_modes() -> [(SearchMode, Arc); 2] { + [ + (SearchMode::Hybrid, live(HashMap::new())), + ( + SearchMode::Keyword, + Arc::new(FakeEmbedder::failing()) as Arc, + ), + ] +} + #[tokio::test] async fn queries_fts5_cannot_parse_succeed_identically_in_both_modes() { // T-04-15 revised by user decision option-a (2026-10-23): the same input // classifies the same way in every mode, and that classification is now - // "answered", never InvalidQuery and never a 500. The raw MATCH runs - // first; FTS5-unparseable input is retried once as its words OR-ed. - for input in [ - "\"unbalanced", - "AND", - "what is the plan?", - "sqlite-vec", - "auth: tokens", - ] { - for (expected_mode, embedder) in [ - (SearchMode::Hybrid, live(HashMap::new())), - ( - SearchMode::Keyword, - Arc::new(FakeEmbedder::failing()) as Arc, - ), - ] { - let h = harness(embedder); - store( - &h, - "what is the plan for the unbalanced ledger", - MemoryType::Pattern, - ) - .await; + // "answered" — never InvalidQuery, never a 500. The raw MATCH runs first; + // FTS5-unparseable input is retried once as its words OR-ed; a tokenless + // query contributes no keyword matches. + for (expected_mode, embedder) in both_modes() { + let h = harness(embedder); + let corpus = store_corpus(&h).await; + + for (input, target) in TARGETED { let outcome = h .service .search(query(input, None)) @@ -576,42 +606,105 @@ async fn queries_fts5_cannot_parse_succeed_identically_in_both_modes() { outcome.search_mode, expected_mode, "{input:?} must carry the mode's search_mode" ); + assert_eq!( + ids(&outcome.results).first(), + Some(&corpus[target - 1]), + "{expected_mode:?} mode: {input:?} must recall M{target} first, got {:?}", + ids(&outcome.results) + ); } - } - // The tracer slice: a hyphenated term and a punctuated question both - // recall the sqlite-vec memory first, with the embedder up or down. - for input in ["sqlite-vec", "what vector store do we use?"] { - for (expected_mode, embedder) in [ - (SearchMode::Hybrid, live(HashMap::new())), - ( - SearchMode::Keyword, - Arc::new(FakeEmbedder::failing()) as Arc, - ), - ] { - let h = harness(embedder); - let m1 = store( - &h, - "We chose sqlite-vec for local vector similarity search", - MemoryType::Pattern, - ) - .await; - store(&h, "auth tokens rotate every hour", MemoryType::Pattern).await; + for input in TOKENLESS_OR_NO_HIT { let outcome = h .service .search(query(input, None)) .await - .unwrap_or_else(|e| panic!("{expected_mode:?}: {input:?} failed: {e:?}")); + .unwrap_or_else(|e| { + panic!("{expected_mode:?} mode: {input:?} must be answered, got {e:?}") + }); assert_eq!(outcome.search_mode, expected_mode, "{input:?}"); - assert_eq!( - ids(&outcome.results).first(), - Some(&m1), - "{expected_mode:?}: {input:?} must recall the sqlite-vec memory first" - ); + match expected_mode { + SearchMode::Keyword => assert!( + outcome.results.is_empty(), + "keyword mode: {input:?} has no keyword matches, got {:?}", + ids(&outcome.results) + ), + SearchMode::Hybrid => assert!( + !outcome.results.is_empty(), + "hybrid mode: {input:?} must still answer from the semantic leg" + ), + } } } } +#[tokio::test] +async fn structured_fts5_syntax_still_runs_raw() { + // The raw query always runs first, so valid FTS5 keeps its exact meaning. + // Each expected set differs from what a sanitized OR-of-words retry would + // return, so a wrongly retried query fails here. + const S1: &str = "sqlite only here"; + const S2: &str = "sqlite with vec support"; + const S3: &str = "an exact phrase lives here"; + const S4: &str = "phrase before exact here"; + const S5: &str = "vector math notes"; + + let h = harness(Arc::new(FakeEmbedder::failing())); + let mut s = Vec::new(); + for content in [S1, S2, S3, S4, S5] { + s.push(store(&h, content, MemoryType::Pattern).await); + } + + // (query, exact raw result as indices into S1..S5; retry would differ) + let table: [(&str, &[usize]); 3] = [ + ("sqlite AND vec", &[2]), // retry: S1 + S2 + ("\"exact phrase\"", &[3]), // retry: S3 + S4 + ("vec*", &[2, 5]), // retry: S2 only + ]; + for (input, expected) in table { + let outcome = h + .service + .search(query(input, None)) + .await + .unwrap_or_else(|e| panic!("{input:?} is valid FTS5: {e:?}")); + assert_eq!(outcome.search_mode, SearchMode::Keyword); + let mut got = ids(&outcome.results); + got.sort_unstable(); + let mut want: Vec = expected.iter().map(|i| s[i - 1]).collect(); + want.sort_unstable(); + assert_eq!( + got, want, + "{input:?} must run raw, not as a sanitized retry" + ); + } +} + +#[tokio::test] +async fn a_broken_fts_index_in_keyword_mode_is_the_generic_internal_error() { + // T-04-19 on the keyword-degrade path: the FTS index renamed away, the + // embedder down. The prepare-time "no such table" is never retried and + // reaches the caller only as SEARCH_STORAGE_FAILURE. + let h = harness(Arc::new(FakeEmbedder::failing())); + store(&h, "ledger reconciliation runbook", MemoryType::Pattern).await; + raw_conn(&h.db_path) + .execute_batch("ALTER TABLE memories_fts RENAME TO memories_fts_offline;") + .expect("rename fts table"); + + let err = h + .service + .search(query("ledger", None)) + .await + .expect_err("a broken FTS index must fail keyword search"); + assert!( + matches!(err, MemoryError::Internal(ref m) if m == SEARCH_STORAGE_FAILURE), + "keyword mode must surface the generic internal error, got {err:?}" + ); + assert!( + !err.to_string().to_lowercase().contains("sqlite"), + "no SQLite text may reach the client: {err}" + ); +} + #[tokio::test] async fn an_internal_error_in_either_leg_propagates_and_never_serves_the_other_leg() { // Break one leg's index underneath a live store (renamed away, so the From 4453aa1ea1359eda650a32b3d1498da8e785f2d4 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 21:47:31 +0200 Subject: [PATCH 22/42] feat(04-05): forgiving query contract on both transports, stdio e2e, docs and STATE decision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - mcp.rs: memory_search description and SearchArgs.query schema text now say the query may be plain natural language or FTS5 syntax and is never rejected for its syntax; description test asserts the new wording and the absence of invalid_params. New in-process two-mode decision-table test over the shared corpus and a broken-FTS redaction test (internal_error, no SQLite text). - handlers.rs: same two-mode table through search_handler (200 everywhere) and a broken-FTS 500 redaction test. - tests/tools.rs: real-binary stdio case — sqlite-vec and a punctuated question both recall the stored memory in keyword mode. - README: "never rejected" search bullet, REST storage-failure sentence, tools-table row, Behavioral changes item 3 rewritten (v0.0.1 400 and sqlite-vec 500 cases both answered now). - STATE.md: 04-05 T-04-15 revised (option-a) decision entry. Co-Authored-By: Claude Opus 5.5 --- .planning/STATE.md | 1 + README.md | 31 ++++-- crates/agent-memory/src/mcp.rs | 130 ++++++++++++++++++++++- crates/agent-memory/src/rest/handlers.rs | 118 ++++++++++++++++++++ crates/agent-memory/tests/tools.rs | 50 +++++++++ 5 files changed, 315 insertions(+), 15 deletions(-) diff --git a/.planning/STATE.md b/.planning/STATE.md index b576898..9b33d81 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -112,6 +112,7 @@ Recent decisions affecting current work: - [Phase 04]: 04-04: Hybrid search = RRF (k=60, 1-based ranks, duplicate ids sum, id tie-break) over two pure-relevance legs (KNN distance, bm25 ASC), each oversampled limit*4 capped at MAX_KNN_K; decay applied once, multiplicatively, post-fusion (SEARCH-04/05) - [Phase 04]: 04-04: SearchMode::Semantic removed; embedder-up searches report search_mode "hybrid", embedder-down stays "keyword" byte-identical to v1.0 (README behavioral change 3) - [Phase 04]: 04-04: Keyword-leg errors propagate first, so a malformed MATCH (incl. bare ?/,/' punctuation) is InvalidQuery/400 in hybrid as in keyword mode (T-04-15); follow-up candidate: sanitize natural-language queries for the keyword leg +- [Phase 04]: 04-05: T-04-15 revised by user decision option-a (2026-10-23) — search runs the raw FTS5 MATCH first, then one sanitized retry (word tokens quoted and OR-joined) on any FTS5 query-parse error, including the step-time `no such column` class from hyphens/colons; a tokenless query is an empty keyword leg (hybrid answers semantic-only, keyword mode returns empty); identical in both modes, so search never returns InvalidQuery; every search-path internal error is redacted to SEARCH_STORAGE_FAILURE (T-04-19). Supersedes the 04-04 decision that a malformed MATCH is InvalidQuery/400 in hybrid mode. ### Pending Todos diff --git a/README.md b/README.md index 9f52cc7..1c7835f 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ stored in one is immediately retrievable in another. | Tool | Purpose | Validated parameters | |------|---------|----------------------| | `memory_store` | Persist a typed memory; returns its id. | `ttl_secs`: optional integer, valid range 1–3155760000 (~100 years); omitted = no expiry | -| `memory_search` | Hybrid (keyword + semantic, rank-fused) search when Ollama is reachable, FTS5 keyword otherwise — weighted by decay, and only the returned results are recency-bumped. | `limit`: optional integer, valid range 1–200; omitted defaults to 50. `tag`: exact tag match (case-sensitive). `expand_links`: optional boolean, default `false` | +| `memory_search` | Hybrid (keyword + semantic, rank-fused) search when Ollama is reachable, FTS5 keyword otherwise — weighted by decay, and only the returned results are recency-bumped. Accepts plain natural-language queries (any punctuation) as well as FTS5 syntax. | `limit`: optional integer, valid range 1–200; omitted defaults to 50. `tag`: exact tag match (case-sensitive). `expand_links`: optional boolean, default `false` | | `memory_list` | List memories newest-first, with optional type/tag/scope filters. | `limit`: optional integer, valid range 1–200; omitted = unlimited. `tag`: exact tag match (case-sensitive). `expand_links`: optional boolean, default `false` | | `memory_update` | Patch a memory in place by id — content, tags, scope, TTL (clean not-found for unknown ids). | `content`: never null or empty. `ttl_secs`: 1–3155760000 when set; `null` = no expiry. At least one field besides `id` | | `memory_link` | Link two memories with a typed, directed relation (idempotent; clean not-found for unknown ids). | `kind`: one of `relates_to`, `supersedes`, `caused_by`. `from_id` ≠ `to_id` | @@ -162,12 +162,16 @@ answer is still reported as `"hybrid"`. A present `limit` must be between 1 and 200 (omitted defaults to 50); a `tag` filter matches whole tags exactly and case-sensitively. -- **The query is FTS5 MATCH syntax in both modes.** A query FTS5 cannot parse — - an unbalanced `"`, a dangling `AND`, bare punctuation such as `?`, `,` or `'` - — is a client error (`invalid_params` / HTTP 400) even when Ollama is up. - Hybrid never quietly answers from the semantic ranking alone. Use plain words - for natural-language questions: `what did we decide about caching`, not - `what did we decide about caching?`. +- **Plain language or FTS5 syntax — a query is never rejected.** The raw query + runs through FTS5 first, so valid FTS5 expressions (`sqlite AND vec`, + `"exact phrase"`, `vec*`) keep their exact meaning. A query FTS5 cannot parse + — a question mark, a comma or apostrophe, a hyphen or colon (`sqlite-vec`, + `TODO: auth`), an unbalanced `"` — is retried once with its words quoted and + OR-ed together, so `what vector store do we use?` searches for `what`, + `vector`, `store`, … A query with no words at all (empty, `?!`, `*`) + contributes no keyword matches: hybrid answers from the semantic ranking + alone and keyword mode returns an empty list. This holds identically in both + modes, so whether Ollama is up never changes how a query is treated. - **Only returned results are recency-bumped.** The candidates fusion looked at but did not return, and any `expand_links` neighbors, are not touched, so a search never changes how unseen memories decay. @@ -367,7 +371,9 @@ never a silent clamp. The MCP transport rejects the same inputs identically as `invalid_params`, because validation lives once at the shared service seam. A `PATCH` body that is not valid JSON, has a wrongly typed field, or is sent without `Content-Type: application/json` is also a 400 with the same -`{"error": ...}` body. +`{"error": ...}` body. A storage failure during search reports a generic +internal error (HTTP 500 / MCP `internal_error`, `search failed due to a +storage error`); the detail goes only to the server's stderr log. ```bash curl -X PATCH localhost:7437/api/memories/42 \ @@ -409,9 +415,12 @@ and REST): with RRF (k = 60) and decay-weighted, reported as `"search_mode": "hybrid"`. `"keyword"` (embedder down) is unchanged, as is the envelope shape. Two side effects: - - With Ollama up, a query FTS5 cannot parse (bare `?`, `,`, `'`, an - unbalanced `"`) is now a client error, exactly as it always was without - Ollama. v0.0.1 answered such queries from embeddings alone. + - Queries FTS5 cannot parse are now answered in both modes. With Ollama + down, v0.0.1 rejected punctuated queries (bare `?`, `,`, `'`, an + unbalanced `"`) with HTTP 400 / `invalid_params`, and failed hyphenated or + colon-bearing queries (`sqlite-vec`, `TODO: auth`) with HTTP 500 / + `internal_error`. v0.1.0 answers both: the raw query runs first, and only + if FTS5 cannot parse it is it retried once as its words OR-ed together. - Exact keyword hits now surface alongside paraphrase matches, so result order can differ from v0.0.1's embedding-only ranking. diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index c1de908..3676a37 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -105,11 +105,16 @@ pub struct ListArgs { pub expand_links: bool, } -/// Arguments for `memory_search` (D-04/D-06). `query` is the FTS5 keyword string; -/// the remaining fields are optional filters and a result cap. +/// Arguments for `memory_search` (D-04/D-06). `query` is plain natural language +/// or FTS5 MATCH syntax; the remaining fields are optional filters and a +/// result cap. #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] pub struct SearchArgs { - /// The keyword query (FTS5 MATCH syntax). + /// The search query: plain natural language or FTS5 MATCH syntax. Valid + /// FTS5 expressions (AND/OR/NOT, quoted phrases, prefix*) run exactly as + /// written; anything FTS5 cannot parse is retried once as its words OR-ed + /// together; a query with no words contributes no keyword matches. A query + /// is never rejected for its syntax, in either mode. pub query: String, /// Filter by memory type (UPPERCASE). Omit for all types. #[serde(default)] @@ -307,7 +312,7 @@ impl MemoryMcp { } #[tool( - description = "Search stored memories. When the local Ollama embedder is reachable, search is hybrid: a keyword (FTS5) ranking and a semantic (embedding) ranking are fused with Reciprocal Rank Fusion (k=60), then weighted by decay so fresher memories win ties. When the embedder is unavailable, search falls back to keyword/FTS5 ranking alone. Returns {search_mode: 'hybrid'|'keyword', results: [...]}; results is empty when nothing matches. The query is FTS5 MATCH syntax in both modes, so a malformed query (an unbalanced quote, bare punctuation such as '?') is invalid_params. The semantic ranking applies type/tag/scope filters after its 4x-limit candidate cut, so a heavily filtered query may return fewer than limit results. Only returned results count as accessed. Set expand_links=true to attach each result's linked memories as trimmed related summaries." + description = "Search stored memories. When the local Ollama embedder is reachable, search is hybrid: a keyword (FTS5) ranking and a semantic (embedding) ranking are fused with Reciprocal Rank Fusion (k=60), then weighted by decay so fresher memories win ties. When the embedder is unavailable, search falls back to keyword/FTS5 ranking alone. Returns {search_mode: 'hybrid'|'keyword', results: [...]}; results is empty when nothing matches. The query may be plain natural language or FTS5 MATCH syntax: valid FTS5 expressions (AND/OR/NOT, quoted phrases, prefix*) run exactly as written, anything FTS5 cannot parse is retried once as its words OR-ed together, and a query with no words contributes no keyword matches, so a query is never rejected for its syntax, in either mode. The semantic ranking applies type/tag/scope filters after its 4x-limit candidate cut, so a heavily filtered query may return fewer than limit results. Only returned results count as accessed. Set expand_links=true to attach each result's linked memories as trimmed related summaries." )] async fn memory_search( &self, @@ -1097,6 +1102,123 @@ mod tests { "{description}" ); assert!(description.contains("k=60"), "{description}"); + // Option-a (T-04-15 revised): the forgiving query contract is what + // agents read, and the retired invalid_params promise is gone. + assert!( + description.contains("plain natural language"), + "{description}" + ); + assert!(description.contains("never rejected"), "{description}"); + assert!(!description.contains("invalid_params"), "{description}"); + } + + /// The shared option-a corpus (mirrors agent-memory-core tests/hybrid.rs): + /// five rows, a targeted decision table (query -> 1-based corpus index + /// recalled first in both modes), and tokenless or no-hit queries. + const CORPUS: [&str; 5] = [ + "We chose sqlite-vec for local vector similarity search", + "auth tokens rotate every hour", + "C++ build flags need care", + "don't retry the import blindly", + "the unbalanced ledger is reconciled nightly", + ]; + const TARGETED: [(&str, usize); 7] = [ + ("what vector store do we use?", 1), + ("sqlite-vec", 1), + ("don't retry", 4), + ("auth, tokens", 2), + ("auth: tokens", 2), + ("C++", 3), + ("\"unbalanced", 5), + ]; + const TOKENLESS_OR_NO_HIT: [&str; 4] = ["AND", "", "?!,", "*"]; + + #[tokio::test] + async fn memory_search_answers_every_query_shape_in_both_modes() { + for (expected_mode, live) in [("hybrid", true), ("keyword", false)] { + let dir = tempfile::tempdir().expect("tempdir"); + let server = if live { + live_server(&dir) + } else { + test_server(&dir) + }; + let mut ids = Vec::new(); + for content in CORPUS { + ids.push(store_one(&server, content).await); + } + + let queries = TARGETED + .iter() + .map(|(q, target)| (*q, Some(*target))) + .chain(TOKENLESS_OR_NO_HIT.iter().map(|q| (*q, None))); + for (input, target) in queries { + let result = server + .memory_search(Parameters(search_args(serde_json::json!({ + "query": input + })))) + .await + .unwrap_or_else(|e| { + panic!("{expected_mode}: {input:?} must be answered, got {e:?}") + }); + let raw = serde_json::to_string(&result).expect("tool result serializes"); + assert!(!raw.contains("sqlite error"), "{input:?}: {raw}"); + let body = result_json(&result); + assert_eq!(body["search_mode"], expected_mode, "{input:?}: {body}"); + let results = body["results"].as_array().expect("results array"); + match target { + Some(index) => assert_eq!( + results.first().and_then(|r| r["id"].as_i64()), + Some(ids[index - 1]), + "{expected_mode}: {input:?} must recall corpus row {index} first: {body}" + ), + None if live => assert!( + !results.is_empty(), + "hybrid: {input:?} must answer from the semantic leg: {body}" + ), + None => assert!( + results.is_empty(), + "keyword: {input:?} has no keyword matches: {body}" + ), + } + } + } + } + + #[tokio::test] + async fn memory_search_storage_failure_is_internal_error_without_sqlite_text() { + // T-04-19: a broken FTS index (renamed away underneath the server) + // reaches the agent only as the generic internal error, in both modes. + let internal_error_code = McpError::internal_error("x", None).code; + for live in [true, false] { + let dir = tempfile::tempdir().expect("tempdir"); + let server = if live { + live_server(&dir) + } else { + test_server(&dir) + }; + store_one(&server, "ledger reconciliation runbook").await; + agent_memory_core::store::sqlite::register_vec_extension().expect("register vec0"); + rusqlite::Connection::open(dir.path().join("mem.db")) + .expect("open db") + .execute_batch("ALTER TABLE memories_fts RENAME TO memories_fts_offline;") + .expect("rename fts table"); + + let err = server + .memory_search(Parameters(search_args(serde_json::json!({ + "query": "ledger" + })))) + .await + .expect_err("a broken FTS index must fail the search"); + assert_eq!(err.code, internal_error_code, "live={live}: {err:?}"); + assert!( + err.message.contains("search failed due to a storage error"), + "live={live}: {err:?}" + ); + assert!( + !err.message.to_lowercase().contains("sqlite"), + "live={live}: no SQLite text may reach the agent: {err:?}" + ); + } } #[test] diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index 25e9d02..38ff150 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -735,6 +735,124 @@ mod tests { } } + /// The shared option-a corpus (mirrors agent-memory-core tests/hybrid.rs): + /// five rows, a targeted decision table (query -> 1-based corpus index + /// recalled first in both modes), and tokenless or no-hit queries. + const CORPUS: [&str; 5] = [ + "We chose sqlite-vec for local vector similarity search", + "auth tokens rotate every hour", + "C++ build flags need care", + "don't retry the import blindly", + "the unbalanced ledger is reconciled nightly", + ]; + const TARGETED: [(&str, usize); 7] = [ + ("what vector store do we use?", 1), + ("sqlite-vec", 1), + ("don't retry", 4), + ("auth, tokens", 2), + ("auth: tokens", 2), + ("C++", 3), + ("\"unbalanced", 5), + ]; + const TOKENLESS_OR_NO_HIT: [&str; 4] = ["AND", "", "?!,", "*"]; + + #[tokio::test] + async fn search_answers_every_query_shape_in_both_modes() { + for (expected_mode, live) in [("hybrid", true), ("keyword", false)] { + let dir = tempfile::tempdir().expect("tempdir"); + let state = if live { + live_state(&dir) + } else { + test_state(&dir) + }; + let mut ids = Vec::new(); + for content in CORPUS { + let resp = store_handler( + State(state.clone()), + Json(store_request(content, "PATTERN")), + ) + .await + .expect("store succeeds") + .into_response(); + ids.push(body_json(resp).await["id"].as_i64().expect("id")); + } + + let queries = TARGETED + .iter() + .map(|(q, target)| (*q, Some(*target))) + .chain(TOKENLESS_OR_NO_HIT.iter().map(|q| (*q, None))); + for (input, target) in queries { + let resp = search_handler(State(state.clone()), Json(search_request(input))) + .await + .unwrap_or_else(|_| panic!("{expected_mode}: {input:?} must be answered")) + .into_response(); + assert_eq!(resp.status(), StatusCode::OK, "{expected_mode}: {input:?}"); + let body = body_json(resp).await; + assert!( + !body.to_string().contains("sqlite error"), + "{input:?}: {body}" + ); + assert_eq!(body["search_mode"], expected_mode, "{input:?}: {body}"); + let results = body["results"].as_array().expect("results array"); + match target { + Some(index) => assert_eq!( + results.first().and_then(|r| r["id"].as_i64()), + Some(ids[index - 1]), + "{expected_mode}: {input:?} must recall corpus row {index} first: {body}" + ), + None if live => assert!( + !results.is_empty(), + "hybrid: {input:?} must answer from the semantic leg: {body}" + ), + None => assert!( + results.is_empty(), + "keyword: {input:?} has no keyword matches: {body}" + ), + } + } + } + } + + #[tokio::test] + async fn search_storage_failure_is_500_without_sqlite_text() { + // T-04-19: a broken FTS index (renamed away underneath the server) + // reaches the REST client only as the generic internal error. + for live in [true, false] { + let dir = tempfile::tempdir().expect("tempdir"); + let state = if live { + live_state(&dir) + } else { + test_state(&dir) + }; + stored_id(&state, "ledger reconciliation runbook").await; + agent_memory_core::store::sqlite::register_vec_extension().expect("register vec0"); + rusqlite::Connection::open(dir.path().join("mem.db")) + .expect("open db") + .execute_batch("ALTER TABLE memories_fts RENAME TO memories_fts_offline;") + .expect("rename fts table"); + + let resp = match search_handler(State(state), Json(search_request("ledger"))).await { + Ok(_) => panic!("live={live}: a broken FTS index must fail the search"), + Err(err) => err.into_response(), + }; + assert_eq!( + resp.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "live={live}" + ); + let body = body_json(resp).await; + let error = body["error"].as_str().expect("500 body carries an error"); + assert!( + error.contains("search failed due to a storage error"), + "live={live}: {body}" + ); + assert!( + !error.to_lowercase().contains("sqlite"), + "live={live}: no SQLite text may reach the client: {body}" + ); + } + } + #[tokio::test] async fn search_with_zero_limit_returns_400_never_500() { // API-02 / D-01: an out-of-bounds limit is CLIENT input — 400 with the diff --git a/crates/agent-memory/tests/tools.rs b/crates/agent-memory/tests/tools.rs index 4872233..7e407d4 100644 --- a/crates/agent-memory/tests/tools.rs +++ b/crates/agent-memory/tests/tools.rs @@ -293,6 +293,56 @@ fn memory_search_returns_ranked_results_with_decay_then_empty_on_no_match() { server.shutdown(); } +#[test] +fn memory_search_answers_hyphenated_and_punctuated_queries_over_stdio() { + // User decision option-a (T-04-15 revised), end to end through the real + // binary in keyword mode: a hyphenated term (which FTS5 reads as a column + // filter) and a punctuated question are both answered with the stored + // memory, never a JSON-RPC error and never SQLite text. + let dir = tempfile::tempdir().expect("tempdir"); + let db_path = dir.path().join("memory.db"); + + let mut server = Server::spawn(&db_path); + server.initialize(); + + let stored = server.call_tool( + 2, + "memory_store", + serde_json::json!({ + "content": "We chose sqlite-vec for local vector similarity search", + "type": "PATTERN" + }), + ); + let m1: i64 = tool_text(&stored) + .parse() + .unwrap_or_else(|e| panic!("memory_store returns an id ({e}): {stored}")); + + for (id, query) in [(3, "sqlite-vec"), (4, "what vector store do we use?")] { + let resp = server.call_tool(id, "memory_search", serde_json::json!({ "query": query })); + assert!( + resp.get("error").is_none(), + "{query:?} must be answered, not a JSON-RPC error: {resp}" + ); + assert!(resp.get("result").is_some(), "{query:?}: {resp}"); + assert!( + !resp.to_string().contains("sqlite error"), + "{query:?}: no SQLite text may reach the agent: {resp}" + ); + let (mode, results) = search_outcome(&resp); + assert_eq!(mode, "keyword", "{query:?}: dead Ollama URL forces keyword"); + assert_eq!( + results + .first() + .and_then(|v| v.get("id")) + .and_then(|i| i.as_i64()), + Some(m1), + "{query:?} must recall the sqlite-vec memory first: {resp}" + ); + } + + server.shutdown(); +} + #[test] fn memory_forget_deletes_then_search_and_list_omit_it_unknown_id_is_clean_not_found() { let dir = tempfile::tempdir().expect("tempdir"); From 42533282bbb1b9dac1228e0438b38ddc90c8fdff Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 21:49:13 +0200 Subject: [PATCH 23/42] docs(04-05): complete forgiving keyword leg plan (summary) --- .../04-05-SUMMARY.md | 242 ++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-05-SUMMARY.md diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-05-SUMMARY.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-05-SUMMARY.md new file mode 100644 index 0000000..f67c056 --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-05-SUMMARY.md @@ -0,0 +1,242 @@ +--- +phase: 04-memory-update-relations-hybrid-search +plan: 05 +subsystem: search +tags: [fts5, sqlite, hybrid-search, rrf, error-redaction, mcp, rest, gap-closure] + +# Dependency graph +requires: + - phase: 04-memory-update-relations-hybrid-search (04-04) + provides: hybrid RRF search, keyword_candidates leg, keyword-leg-first error ordering +provides: + - raw-first FTS5 MATCH with one sanitized OR-of-quoted-words retry, shared by the keyword-degrade path and the hybrid keyword leg + - tokenless query = empty keyword leg (hybrid semantic-only, keyword mode empty) + - SEARCH_STORAGE_FAILURE + redact_search_error, the single wire-tiering point for search errors + - two-mode decision tables at the core, MCP, REST and stdio layers; classifier pinned to bundled SQLite 3.51.3 +affects: [04-06, 04-07, phase-05, phase-06 DIST-06 release notes, README behavioral changes] + +actuals: + tokens: 11786 + tasks: 3 + commits: 4 + +tech-stack: + added: [] + patterns: + - "Raw-first FTS5 execution with a closure-based single retry (match_with_sanitized_retry) around a byte-identical static statement" + - "Service-seam error redaction: public fn = private *_unredacted().map_err(redact_*), exhaustive variant tiering" + +key-files: + created: [] + modified: + - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory-core/src/store/mod.rs + - crates/agent-memory-core/src/service.rs + - crates/agent-memory-core/src/domain.rs + - crates/agent-memory-core/tests/hybrid.rs + - crates/agent-memory-core/tests/fallback.rs + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - crates/agent-memory/tests/rest.rs + - crates/agent-memory/tests/tools.rs + - README.md + - .planning/STATE.md + +key-decisions: + - "04-05: T-04-15 revised by user decision option-a (2026-10-23): raw FTS5 MATCH first, one sanitized retry (word tokens quoted and OR-joined) on any FTS5 query-parse error incl. the step-time `no such column` class; tokenless query = empty keyword leg; identical in both modes, so search never returns InvalidQuery; every search-path storage error redacted to SEARCH_STORAGE_FAILURE. Supersedes the 04-04 InvalidQuery/400-in-hybrid decision." + - "All 9 candidate erroring probes error in bundled SQLite 3.51.3 exactly as in the 3.46 CLI probe (empty string included); none moved to the valid-raw group; no marker beyond the planned five was needed." + +patterns-established: + - "Search-path errors leave MemoryService only through redact_search_error; transports never see Sqlite/Pool/Join/Migration text from search." + - "Shared option-a corpus (M1-M5) + decision table duplicated verbatim at core, MCP and REST test layers so expectations cannot drift." + +requirements-completed: [SEARCH-04] + +coverage: + - id: D1 + description: "Natural-language / punctuated queries (what vector store do we use?, don't retry, auth, tokens, C++, \"unbalanced) succeed in hybrid AND keyword mode with the target memory first (VERIFICATION truth #15, CR-02)" + requirement: "SEARCH-04" + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#queries_fts5_cannot_parse_succeed_identically_in_both_modes" + status: pass + - kind: integration + ref: "crates/agent-memory/src/mcp.rs#memory_search_answers_every_query_shape_in_both_modes" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#search_answers_every_query_shape_in_both_modes" + status: pass + - kind: e2e + ref: "crates/agent-memory/tests/tools.rs#memory_search_answers_hyphenated_and_punctuated_queries_over_stdio" + status: pass + human_judgment: false + - id: D2 + description: "Hyphenated / colon-bearing queries (sqlite-vec, auth: tokens) succeed in both modes, never a 500 (VERIFICATION truth #14, CR-01)" + requirement: "SEARCH-04" + verification: + - kind: unit + ref: "crates/agent-memory-core/src/store/sqlite.rs#every_bundled_fts5_parse_error_is_classified_as_match_input" + status: pass + - kind: e2e + ref: "crates/agent-memory/tests/rest.rs (spawned serve-rest, keyword mode: \" and sqlite-vec -> 200)" + status: pass + - kind: manual_procedural + ref: "live serve-rest against local Ollama nomic-embed-text and --ollama-url http://127.0.0.1:9 (2026-10-23): both queries 200, id 1 first, hybrid/keyword respectively" + status: pass + human_judgment: false + - id: D3 + description: "Valid structured FTS5 syntax (sqlite AND vec, \"exact phrase\", vec*) still runs exactly as written (raw first)" + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#structured_fts5_syntax_still_runs_raw" + status: pass + - kind: unit + ref: "crates/agent-memory-core/src/store/sqlite.rs#valid_structured_fts5_syntax_runs_raw_in_the_bundled_sqlite" + status: pass + human_judgment: false + - id: D4 + description: "Tokenless queries (empty, ?!,, *, and no-hit AND) are an empty keyword answer and a non-empty semantic-only hybrid answer, never an error" + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#queries_fts5_cannot_parse_succeed_identically_in_both_modes" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/fallback.rs#a_lone_double_quote_is_an_empty_keyword_answer_never_an_error" + status: pass + human_judgment: false + - id: D5 + description: "No search response body carries SQLite text; storage failures surface as SEARCH_STORAGE_FAILURE (REST 500 / MCP internal_error) (T-04-19)" + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#an_internal_error_in_either_leg_propagates_and_never_serves_the_other_leg" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/hybrid.rs#a_broken_fts_index_in_keyword_mode_is_the_generic_internal_error" + status: pass + - kind: integration + ref: "crates/agent-memory/src/mcp.rs#memory_search_storage_failure_is_internal_error_without_sqlite_text" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#search_storage_failure_is_500_without_sqlite_text" + status: pass + human_judgment: false + - id: D6 + description: "Tool description, SearchArgs schema text, README and STATE.md state the forgiving contract" + verification: + - kind: unit + ref: "crates/agent-memory/src/mcp.rs#memory_search_description_enumerates_exactly_hybrid_and_keyword" + status: pass + - kind: other + ref: "grep -n 'never rejected' README.md; grep -n '04-05: T-04-15 revised' .planning/STATE.md" + status: pass + human_judgment: false + +duration: 10min +completed: 2026-10-23 +status: complete +--- + +# Phase 04 Plan 05: Forgiving keyword leg Summary + +**Search now accepts natural-language queries with any punctuation in either mode. The raw query runs through FTS5 first. If FTS5 can't parse it, it is retried once as its quoted words OR-ed together, and a query with no words at all simply gets no keyword matches. This holds identically whether Ollama is up or down. Separately, every storage failure on a search path now reaches the client only as a generic `SEARCH_STORAGE_FAILURE` message, never raw SQLite text.** + +## Performance + +- **Duration:** ~10 min +- **Started:** 2026-10-23T19:38:07Z +- **Completed:** 2026-10-23T19:48Z +- **Tasks:** 3/3 +- **Files modified:** 12 + +## Accomplishments + +- `match_with_sanitized_retry` / `sanitize_fts_query` / `is_match_input_error` sit at the single store seam that `Store::search` (keyword degrade) and `Store::keyword_candidates` (hybrid leg) share. Both SQL strings are byte-identical, and prepare-time failures are never retried. +- `sqlite-vec`, `auth: tokens`, `what vector store do we use?`, `don't retry`, `auth, tokens`, `C++` and `"unbalanced` all return the target memory first in both modes, at the core, over MCP, over REST, and through the real stdio binary. VERIFICATION truths #14/#15 and REVIEW CR-01/CR-02 are closed. +- `redact_search_error` (exhaustive, no `_` arm) is the only exit for search errors. A broken FTS or vec index gives `internal error: search failed due to a storage error (details are in the server log)`, with the detail logged to stderr. +- The classifier is pinned to the bundled SQLite 3.51.3. Every one of the 9 candidate erroring probes is classified, the 5 valid-raw probes run Ok, and every sanitized fallback parses. +- The old 400-pinning tests are reversed. The tool description, the `SearchArgs` schema text, README (search bullet, REST error sentence, tools table, Behavioral changes item 3) and the STATE decision all describe the new contract. + +## Task Commits + +1. **Task 1 (tracer): sqlite-vec and a punctuated question answered end-to-end in both modes, redacted search errors.** `289edbe` (feat) +2. **Task 2: bundled-SQLite parse classes, two-mode decision table, raw-first proofs.** `a719dd1` (test) +3. **Task 3: transport tables, stdio e2e, tool description, README, STATE decision.** `4453aa1` (feat) + +**Plan metadata:** see the `docs(04-05)` commit that follows. + +## Files Created/Modified + +- `crates/agent-memory-core/src/store/sqlite.rs`: the three helpers plus `FTS5_QUERY_PARSE_MARKERS`. `search` and `keyword_candidates` are rewired to them, `map_fts_query_error` is removed, and 5 new unit tests are added. +- `crates/agent-memory-core/src/store/mod.rs`: `Store::search` / `keyword_candidates` docs updated: never `InvalidQuery`, only server faults are errors. +- `crates/agent-memory-core/src/service.rs`: adds `SEARCH_STORAGE_FAILURE` and `redact_search_error`; `search` now wraps `search_unredacted`; `SearchArgs`, `search` and `hybrid_candidates` docs updated; IN-04 comment fixed. +- `crates/agent-memory-core/src/domain.rs`: doc comment on `InvalidQuery` (kept, no longer produced by search). +- `crates/agent-memory-core/tests/hybrid.rs`: full M1–M5 two-mode table (7 targeted + 4 tokenless rows), `structured_fts5_syntax_still_runs_raw`, broken-leg tests assert redaction in both modes, degrade-table comment updated. +- `crates/agent-memory-core/tests/fallback.rs`: a lone `"` is now an empty keyword answer. +- `crates/agent-memory/src/mcp.rs`: description and schema text, a flipped hybrid test, the table test, the redaction test, and extended description assertions. +- `crates/agent-memory/src/rest/handlers.rs`: two flipped tests, the table test, and a 500 redaction test. +- `crates/agent-memory/tests/rest.rs`: the spawned-binary `"` / `sqlite-vec` POSTs now expect 200 (harness untouched). +- `crates/agent-memory/tests/tools.rs`: stdio end-to-end test. +- `README.md`, `.planning/STATE.md`: contract docs and the decision entry. + +## Decisions Made + +- I implemented user decision option-a exactly as locked. All 9 candidate probes error in bundled SQLite 3.51.3, as they did in the 3.46 CLI table, so **no probe moved to the valid-raw group** and no marker beyond the planned five (`fts5: `, `unterminated string`, `no such column`, `unknown special query`, `expected integer`) was needed. +- In `is_match_input_error` I used `let … else` instead of a `match` with a `_` arm. This keeps the "no catch-all" rule without matching every `rusqlite::Error` variant. + +## Deviations from Plan + +### Process + +**1. Tracer feedback gate run as an automated re-verify, not an interactive checkpoint** +- **Found during:** Task 1 (tracer) +- **Issue:** Auto mode was off (`_auto_chain_active=false`, `auto_advance=false`), so the executor protocol calls for a `checkpoint:human-verify` after the tracer. However, the orchestrator asked for all tasks plus the SUMMARY in one run, the plan is `autonomous: true`, and the tracer's `` is entirely automated (cargo tests, including the spawned REST binary, plus clippy). +- **Fix:** Before starting Task 2 I re-ran the tracer's full `` as the gate, which is the autonomous form of the gate. It was all green. +- **Committed in:** n/a (process only) + +### Additive test strengthening (no scope change) + +**2. [Rule 2 - Coverage] Redaction tests run in both modes on both transports** +- The plan's MCP/REST redaction rows name one fixture without fixing the mode. I ran each in both hybrid and keyword mode. The classifier-negative test also gained a "parse marker under SQLITE_BUSY" case (proving the result-code gate matters) and a positive control. +- **Files modified:** crates/agent-memory/src/mcp.rs, crates/agent-memory/src/rest/handlers.rs, crates/agent-memory-core/src/store/sqlite.rs +- **Committed in:** `a719dd1`, `4453aa1` + +--- + +**Total deviations:** 1 process, 1 additive (Rule 2). **Impact:** none on behaviour or scope. + +## TDD Gate Compliance + +All three tasks carry `tdd="true"`, but the plan is `type: execute`. Task 1 is an atomic contract flip that the plan explicitly requires to land in **one** commit (CLAUDE.md forbids committing failing tests), so there is no separate RED commit. Task 2 is test-only (`test(...)`), and Task 3 bundles tests with docs and the description. RED was observed in place of separate commits: the reversed tests contradict the pre-change behaviour by construction (they assert `Ok` where the old code returned `InvalidQuery` or `Sqlite`). + +## Issues Encountered + +None. + +## Verification + +- `cargo fmt --all --check`: clean +- `cargo clippy --workspace --all-targets -- -D warnings`: clean +- `cargo test --workspace`: **182 passed / 0 failed / 2 ignored** (this plan added no `#[ignore]`; both ignored tests are pre-existing) +- Live re-check (optional plan item) on 2026-10-23 against a real local Ollama running `nomic-embed-text`, using `serve-rest` on a temp DB with M1–M3: + - Ollama up: `sqlite-vec` and `what vector store do we use?` both returned 200 `hybrid` with id 1 first. + - `--ollama-url http://127.0.0.1:9`: both returned 200 `keyword` with `[1]`. + - No `sqlite error` in any body. + +## Known Stubs + +None. + +## User Setup Required + +None. No external service configuration required. + +## Next Phase Readiness + +- Ready for 04-06 (CR-03 stale-vector backfill race; WR-03/IN-05/IN-06; WR-04 doc correction) and 04-07 (WR-02, IN-01..03, WR-01). +- Still deferred, as the plan states: internal-tier redaction on NON-search surfaces, 03-REVIEW WR-01 (filters after the KNN cut), and 03-REVIEW WR-03 (spawned REST harness hang). + +## Self-Check: PASSED + +- All 12 modified files present on disk. +- Commits `289edbe`, `a719dd1`, `4453aa1` found in `git log`. +- Acceptance greps re-run: three helpers present, `match_with_sanitized_retry(` ×3, both SQL markers ×1, `fn redact_search_error` and `map_err(redact_search_error)` present, `never rejected` in README, `04-05: T-04-15 revised` in STATE.md. From 91c8d82d30bca077a15f89512fba3debae597deb Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 21:49:55 +0200 Subject: [PATCH 24/42] docs(04-05): update state and roadmap progress (5/7 phase-04 plans) - STATE: position Plan 5 of 7, executing gap closure (04-06 next), P05 metric, session; the option-a decision entry landed with the Task 3 commit - ROADMAP: 04-05 checked, phase 04 progress 5/7 (In Progress, not complete) - SEARCH-04 already complete from 04-04; REQUIREMENTS.md unchanged Co-Authored-By: Claude Opus 5.5 --- .planning/ROADMAP.md | 6 +++--- .planning/STATE.md | 25 +++++++++++++------------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 9f33a23..67b56cc 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -67,7 +67,7 @@ Plans: 4. Hybrid ranking stays decay-aware — a fresh memory outranks a stale one at equal fused rank — and access bumps apply only to post-truncation returned ids. 5. Upgrading a real v0.0.1 database applies migration 0003 only after a pre-migration backup; shipped 0001/0002 SQL is provably unchanged (fixture divergence test against a v0.0.1 database), and a newer-schema DB met by an older binary yields a friendly error, not a crash. -**Plans**: 7 plans (4 executed + 3 gap closure) +**Plans**: 5/7 plans executed (4 executed + 3 gap closure) Plans: **Wave 1** @@ -87,7 +87,7 @@ Plans: **Gap closure** *(from 04-VERIFICATION.md + 04-REVIEW.md; gap waves run in order)* -- [ ] 04-05-PLAN.md — Gap wave 1: forgiving keyword leg per user decision option-a (raw MATCH first, one sanitized OR-of-quoted-tokens retry, tokenless = empty leg, identical in both modes) + search-path error redaction (CR-01, CR-02, truths #14/#15, IN-04) (SEARCH-04) +- [x] 04-05-PLAN.md — Gap wave 1: forgiving keyword leg per user decision option-a (raw MATCH first, one sanitized OR-of-quoted-tokens retry, tokenless = empty leg, identical in both modes) + search-path error redaction (CR-01, CR-02, truths #14/#15, IN-04) (SEARCH-04) - [ ] 04-06-PLAN.md — Gap wave 2: conditional sweep backfill closing the update/forget/sweep races + symmetric relates_to + IN-05/IN-06 docs (CR-03, WR-03) (MCP-06, MCP-07) - [ ] 04-07-PLAN.md — Gap wave 3: deny_unknown_fields on update DTOs, uniform REST rejection bodies, race-safe verify-then-rename pre-migration backup + Upgrading docs (WR-02, IN-02, WR-01, IN-03, IN-01) (MCP-06, MCP-07, STORE-05) @@ -130,6 +130,6 @@ Phases execute in numeric order: 3 → 4 → 5 → 6 | 1. Core Memory Foundation | v1.0 | 3/3 | Complete | 2026-06-25 | | 2. Semantic Search, Interop & Release | v1.0 | 5/5 | Complete | 2026-07-12 | | 3. API Hardening & Toolchain Spikes | v1.1 | 2/2 | Complete | 2026-07-12 | -| 4. Memory Update, Relations & Hybrid Search | v1.1 | 4/4 | In Progress| | +| 4. Memory Update, Relations & Hybrid Search | v1.1 | 5/7 | In Progress| | | 5. Portable Export & Import | v1.1 | 0/? | Not started | - | | 6. Distribution & v0.1.0 Release | v1.1 | 0/? | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index 9b33d81..4c81128 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -4,17 +4,17 @@ milestone: v1.1 milestone_name: Hardening & Interop current_phase: 04 current_phase_name: Memory Update, Relations & Hybrid Search -status: verifying -stopped_at: Completed 04-04-PLAN.md -last_updated: "2026-10-23T18:40:45.705Z" +status: executing +stopped_at: Completed 04-05-PLAN.md +last_updated: "2026-10-23T19:49:24.593Z" last_activity: 2026-10-23 -last_activity_desc: Phase 04 execution started -state_head: fb9c4634bf807922d5bc73a2959dd6be257def80 +last_activity_desc: Completed 04-05 (gap wave 1, option-a forgiving keyword leg) +state_head: 42533282bbb1b9dac1228e0438b38ddc90c8fdff progress: total_phases: 4 completed_phases: 1 - total_plans: 6 - completed_plans: 6 + total_plans: 9 + completed_plans: 7 --- # Project State @@ -29,9 +29,9 @@ See: .planning/PROJECT.md (updated 2026-07-12) ## Current Position Phase: 04 (Memory Update, Relations & Hybrid Search) — EXECUTING -Plan: 4 of 4 -Status: Phase complete — ready for verification -Last activity: 2026-10-23 — Phase 04 execution started +Plan: 5 of 7 (gap closure: 04-05 done; 04-06, 04-07 next) +Status: Executing gap closure — 04-06 ready to execute +Last activity: 2026-10-23 — Completed 04-05 (option-a forgiving keyword leg + search error redaction) Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) @@ -72,6 +72,7 @@ Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) | Phase 04 P02 | 10min | 2 tasks | 11 files | | Phase 04 P03 | 10 min | 2 tasks | 9 files | | Phase 04 P04 | 11 min | 2 tasks | 11 files | +| Phase 04 P05 | 10 min | 3 tasks | 12 files | ## Accumulated Context @@ -153,8 +154,8 @@ Items acknowledged and carried forward: ## Session Continuity -Last session: 2026-10-23T18:40:36.856Z -Stopped at: Completed 04-04-PLAN.md +Last session: 2026-10-23T19:49:24.539Z +Stopped at: Completed 04-05-PLAN.md Resume file: None ## Operator Next Steps From f299f49d71c71e22336103df2aca2b30c9c2cf83 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 21:54:14 +0200 Subject: [PATCH 25/42] =?UTF-8?q?fix(04-06):=20conditional=20sweep=20backf?= =?UTF-8?q?ill=20=E2=80=94=20never=20overwrite=20an=20edited=20or=20forgot?= =?UTF-8?q?ten=20row=20(CR-03)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Store::insert_embedding(id, expected_content, vec) -> Result: a guarded claim (embedding_status = 0 AND content = ?2) runs first in the writer tx; no claim, no vector write - sweep passes the exact embedded content back and logs backfilled + skipped_stale; SweepReport shape unchanged - tests/backfill_race.rs: 5 store-level interleavings (happy path, embedded update, failed update, forget + rowid reuse, double sweep) and 2 gated service-level sweep tests (tokio Notify, no sleeps) - README Lifecycle: an edited/deleted memory is never overwritten by an in-flight backfill Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +- crates/agent-memory-core/src/service.rs | 42 +- crates/agent-memory-core/src/store/mod.rs | 34 +- crates/agent-memory-core/src/store/sqlite.rs | 29 +- .../agent-memory-core/tests/backfill_race.rs | 465 ++++++++++++++++++ 5 files changed, 553 insertions(+), 22 deletions(-) create mode 100644 crates/agent-memory-core/tests/backfill_race.rs diff --git a/README.md b/README.md index 1c7835f..c32fe22 100644 --- a/README.md +++ b/README.md @@ -454,7 +454,10 @@ A background task sweeps on an interval while the server runs: ranking stays cheap between reads. Pinned types (`DECISION`, `ARCHITECTURE`, `CONSTRAINT`) decay markedly slower. 3. **Embedding backfill** — memories stored while Ollama was unreachable are - embedded in batches once it comes back. + embedded in batches once it comes back. A memory that is edited or deleted + while its backfill is in flight is never overwritten with a vector of its + old text: that write is skipped, and the memory's current content is + embedded on a later sweep (or by the edit itself, when Ollama is up). **Decay never deletes.** A memory with no TTL stays retrievable forever, however low its decay score falls — it simply ranks lower. Removal happens only via TTL or diff --git a/crates/agent-memory-core/src/service.rs b/crates/agent-memory-core/src/service.rs index 6ced680..8a1b61a 100644 --- a/crates/agent-memory-core/src/service.rs +++ b/crates/agent-memory-core/src/service.rs @@ -653,7 +653,13 @@ impl MemoryService { /// After the decay pass, pending embeddings (`embedding_status = 0`) are /// backfilled best-effort in ONE batch embed call (RESEARCH Open Question 1) /// — this completes the "install Ollama later and old memories become - /// semantically searchable" story. Backfill counts are tracing-only: + /// semantically searchable" story. A backfill never overwrites a row that + /// changed or vanished while it was being embedded: each vector is written + /// only if the row is still pending with exactly the embedded content + /// ([`Store::insert_embedding`], REVIEW CR-03). A row edited mid-embed is + /// either already embedded by `update` itself, or still pending, in which + /// case its CURRENT content is embedded on a later sweep. Backfill counts + /// (`backfilled`, `skipped_stale`) are tracing-only: /// [`SweepReport`]'s shape is unchanged, so the TTL contract stays intact. /// /// Driven by the injected clock so it is deterministic under test; exposed for @@ -679,20 +685,32 @@ impl MemoryService { pending.iter().map(|(_, content)| content.clone()).collect(); match self.embedder.embed(&contents).await { Ok(vectors) if vectors.len() == pending.len() => { - let ids: Vec = pending.iter().map(|(id, _)| *id).collect(); let store = self.store.clone(); - let written = - tokio::task::spawn_blocking(move || -> Result { + let (written, skipped) = tokio::task::spawn_blocking( + move || -> Result<(usize, usize), MemoryError> { let mut written = 0usize; - for (id, vector) in ids.into_iter().zip(vectors) { - store.insert_embedding(id, vector)?; - written += 1; + let mut skipped = 0usize; + // Each vector is written only if its row is still + // pending with exactly the content embedded above + // (CR-03). A row that changed or vanished while the + // embed was in flight is skipped, never overwritten. + for ((id, content), vector) in pending.into_iter().zip(vectors) { + if store.insert_embedding(id, &content, vector)? { + written += 1; + } else { + skipped += 1; + } } - Ok(written) - }) - .await - .map_err(MemoryError::Join)??; - tracing::info!(backfilled = written, "embedded pending memories on sweep"); + Ok((written, skipped)) + }, + ) + .await + .map_err(MemoryError::Join)??; + tracing::info!( + backfilled = written, + skipped_stale = skipped, + "embedded pending memories on sweep" + ); } Ok(_) => tracing::warn!( "embedding backfill skipped (batch size mismatch); retrying next sweep" diff --git a/crates/agent-memory-core/src/store/mod.rs b/crates/agent-memory-core/src/store/mod.rs index 172f4d9..702ae6b 100644 --- a/crates/agent-memory-core/src/store/mod.rs +++ b/crates/agent-memory-core/src/store/mod.rs @@ -42,12 +42,40 @@ pub trait Store: Send + Sync { args: SearchArgs, ) -> Result, MemoryError>; - /// Attach an embedding to an existing memory (sweep backfill): writes the - /// vector row and flips `embedding_status` to 1 on the writer lane. - fn insert_embedding(&self, memory_id: i64, embedding: Vec) -> Result<(), MemoryError>; + /// Attach an embedding to an existing memory (sweep backfill), but ONLY + /// if the row is still the pending row whose content was embedded + /// (REVIEW CR-03 / T-04-23, T-04-24). + /// + /// The backfill embeds outside any lock, so by the time the vector comes + /// back the row may have changed. The write therefore starts with one + /// guarded claim in the writer transaction: + /// `UPDATE memories SET embedding_status = 1 WHERE id = ? AND + /// embedding_status = 0 AND content = ?`. Only if that claim changes a row + /// is the vector written (vec0 DELETE + INSERT, same transaction). + /// + /// - `Ok(true)`: the vector was written and `embedding_status` is now 1. + /// - `Ok(false)`: nothing was written. The row was updated since it was + /// read (its current content is embedded by `update` itself or by a + /// later sweep), was already embedded, or was forgotten. + /// + /// The claim closes three races: an `update` landing mid-embed (a stale + /// vector would otherwise overwrite the fresh one, permanently, since the + /// row reads as embedded), a `forget` landing mid-embed (a vector would + /// otherwise attach to a later memory reusing the rowid), and two sweeps + /// embedding the same row. SQLite takes its write lock at the claim, so + /// this holds across the several processes that share one DB file, not + /// just across threads. + fn insert_embedding( + &self, + memory_id: i64, + expected_content: &str, + embedding: Vec, + ) -> Result; /// The ids + contents of up to `limit` rows still awaiting an embedding /// (`embedding_status = 0`), oldest first — the sweep backfill work queue. + /// The returned content is exactly the text the caller must embed AND pass + /// back to [`Store::insert_embedding`] as `expected_content`. fn pending_embeddings(&self, limit: usize) -> Result, MemoryError>; /// The INTEROP-01 idempotency probe: does a row with exactly this diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index fced7b0..95991d4 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -519,12 +519,33 @@ impl Store for SqliteStore { Ok(out) } - fn insert_embedding(&self, memory_id: i64, embedding: Vec) -> Result<(), MemoryError> { + fn insert_embedding( + &self, + memory_id: i64, + expected_content: &str, + embedding: Vec, + ) -> Result { let mut conn = self .writer .lock() .map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; let tx = conn.transaction()?; + // CR-03: claim the row FIRST. It must still be pending AND still hold + // the exact text that was embedded. An update (content changed, or + // status already 1), a forget (row gone, or a reused rowid with other + // content), or a racing sweep (status already 1) all match nothing, + // and then nothing is written: the tx rolls back on drop. SQLite takes + // the write lock at this statement, so the claim is atomic across + // processes sharing the DB file, not only across this process's + // threads (the writer mutex is per process). + let claimed = tx.execute( + "UPDATE memories SET embedding_status = 1 \ + WHERE id = ?1 AND embedding_status = 0 AND content = ?2", + params![memory_id, expected_content], + )?; + if claimed == 0 { + return Ok(false); + } let blob: &[u8] = bytemuck::cast_slice(&embedding); // DELETE + INSERT instead of INSERT OR REPLACE: conflict-resolution // clauses are not reliably supported on virtual tables, and the two @@ -537,12 +558,8 @@ impl Store for SqliteStore { "INSERT INTO vec_memories(memory_id, embedding) VALUES (?1, ?2)", params![memory_id, blob], )?; - tx.execute( - "UPDATE memories SET embedding_status = 1 WHERE id = ?1", - params![memory_id], - )?; tx.commit()?; - Ok(()) + Ok(true) } fn pending_embeddings(&self, limit: usize) -> Result, MemoryError> { diff --git a/crates/agent-memory-core/tests/backfill_race.rs b/crates/agent-memory-core/tests/backfill_race.rs new file mode 100644 index 0000000..53d6037 --- /dev/null +++ b/crates/agent-memory-core/tests/backfill_race.rs @@ -0,0 +1,465 @@ +//! CR-03: the sweep's embedding backfill never writes a vector that disagrees +//! with the row's current content. +//! +//! The backfill reads `(id, content)` for pending rows, awaits ONE batch embed +//! outside any lock (seconds on a CPU Ollama), then writes the vectors. Before +//! the fix that write was unconditional, so anything landing between the read +//! and the write was silently overwritten: +//! +//! - **update (embed ok):** the fresh vector of the NEW text was replaced by a +//! vector of the OLD text, and the row read as embedded (status 1), so no +//! later sweep ever repaired it; +//! - **update (embed failed):** the pending row was "repaired" with the stale +//! vector instead of being re-embedded with its new text; +//! - **forget:** a vector was re-inserted for a deleted id, ready to attach to +//! a later memory that reuses the rowid; +//! - **a second sweep** (several processes share one DB file, each with its +//! own sweep) wrote the same row twice. +//! +//! `Store::insert_embedding` now claims the row first, in the writer +//! transaction: `embedding_status = 0 AND content = `. No +//! claim, no write. The store-level tests below drive each interleaving +//! directly through the public `Store` methods; the service-level tests pin +//! the real `MemoryService::sweep` mid-embed with a `GatedEmbedder` (tokio +//! `Notify` only, no sleeps decide ordering) and land an update in the gap. +//! +//! Real on-disk temp DBs and an injected `TestClock`, as in tests/hybrid.rs. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use agent_memory_core::clock::{Clock, TestClock}; +use agent_memory_core::decay::DecayConfig; +use agent_memory_core::domain::{MemoryPatch, MemoryType, NewMemory}; +use agent_memory_core::embed::{ + BoxFuture, EmbedError, Embedder, EmbedderHealth, FakeEmbedder, EMBEDDING_DIM, +}; +use agent_memory_core::service::{MemoryService, SearchArgs}; +use agent_memory_core::store::sqlite::SqliteStore; +use agent_memory_core::store::Store; +use tokio::sync::Notify; + +const OLD_TEXT: &str = "old text"; +const NEW_TEXT: &str = "new text"; +const EPS: f64 = 1e-5; + +// --------------------------------------------------------------------------- +// Harness +// --------------------------------------------------------------------------- + +/// A programmed [`FakeEmbedder`] behind an outage switch and a one-shot gate. +/// +/// While `armed`, the first batch containing [`OLD_TEXT`] disarms the gate, +/// signals `entered`, and parks on `release` before delegating, which holds the +/// sweep exactly between its `pending_embeddings` read and its write. The +/// outage switch is checked on ENTRY, so a gated call that is already parked +/// is unaffected by flipping `fail` afterwards. +struct GatedEmbedder { + inner: FakeEmbedder, + fail: AtomicBool, + armed: AtomicBool, + entered: Notify, + release: Notify, +} + +impl GatedEmbedder { + fn new(vectors: HashMap>) -> Self { + GatedEmbedder { + inner: FakeEmbedder::with_vectors(vectors), + fail: AtomicBool::new(false), + armed: AtomicBool::new(false), + entered: Notify::new(), + release: Notify::new(), + } + } + + fn set_fail(&self, fail: bool) { + self.fail.store(fail, Ordering::SeqCst); + } + + fn arm(&self) { + self.armed.store(true, Ordering::SeqCst); + } +} + +impl Embedder for GatedEmbedder { + fn embed<'a>( + &'a self, + inputs: &'a [String], + ) -> BoxFuture<'a, Result>, EmbedError>> { + Box::pin(async move { + if self.fail.load(Ordering::SeqCst) { + return Err(EmbedError::InvalidResponse( + "GatedEmbedder is down".to_string(), + )); + } + let gated = inputs.iter().any(|input| input == OLD_TEXT) + && self.armed.swap(false, Ordering::SeqCst); + if gated { + // notify_one stores a permit if the test is not waiting yet. + self.entered.notify_one(); + self.release.notified().await; + } + self.inner.embed(inputs).await + }) + } + + fn health<'a>(&'a self) -> BoxFuture<'a, Result> { + self.inner.health() + } +} + +struct Harness { + _dir: tempfile::TempDir, + db_path: PathBuf, + clock: Arc, + store: Arc, + embedder: Arc, + service: MemoryService, +} + +/// `OLD_TEXT → v_old`, `NEW_TEXT → v_new`: orthogonal unit vectors, so under +/// the schema's cosine metric the distance between them is exactly 1.0. +fn harness() -> Harness { + let dir = tempfile::tempdir().expect("tempdir"); + let db_path = dir.path().join("memory.db"); + let clock = Arc::new(TestClock::new(1_000)); + let store = Arc::new(SqliteStore::open(&db_path).expect("open store")); + let mut vectors = HashMap::new(); + vectors.insert(OLD_TEXT.to_string(), v_old()); + vectors.insert(NEW_TEXT.to_string(), v_new()); + let embedder = Arc::new(GatedEmbedder::new(vectors)); + let service = MemoryService::new( + store.clone(), + clock.clone() as Arc, + embedder.clone() as Arc, + DecayConfig::default(), + ); + Harness { + _dir: dir, + db_path, + clock, + store, + embedder, + service, + } +} + +/// A 768-dim unit vector along `axis`. +fn unit_axis(axis: usize) -> Vec { + let mut v = vec![0.0_f32; EMBEDDING_DIM]; + v[axis] = 1.0; + v +} + +fn v_old() -> Vec { + unit_axis(1) +} + +fn v_new() -> Vec { + unit_axis(2) +} + +fn new_memory(content: &str) -> NewMemory { + NewMemory { + content: content.to_string(), + mem_type: MemoryType::Pattern, + tags: vec![], + source: None, + scope: None, + ttl_secs: None, + } +} + +fn content_patch(content: &str) -> MemoryPatch { + MemoryPatch { + content: Some(Some(content.to_string())), + ..Default::default() + } +} + +/// A row stored straight through the store with no vector: pending (status 0). +fn pending_row(h: &Harness, content: &str) -> i64 { + h.store + .insert(new_memory(content), None, h.clock.now()) + .expect("insert pending row") +} + +fn raw_conn(db_path: &Path) -> rusqlite::Connection { + agent_memory_core::store::sqlite::register_vec_extension().expect("register vec0"); + rusqlite::Connection::open(db_path).expect("open db for assertion") +} + +fn vec_rows(h: &Harness, id: i64) -> i64 { + raw_conn(&h.db_path) + .query_row( + "SELECT COUNT(*) FROM vec_memories WHERE memory_id = ?1", + [id], + |row| row.get(0), + ) + .expect("count vector rows") +} + +fn status(h: &Harness, id: i64) -> i64 { + raw_conn(&h.db_path) + .query_row( + "SELECT embedding_status FROM memories WHERE id = ?1", + [id], + |row| row.get(0), + ) + .expect("read embedding_status") +} + +fn content(h: &Harness, id: i64) -> String { + raw_conn(&h.db_path) + .query_row("SELECT content FROM memories WHERE id = ?1", [id], |row| { + row.get(0) + }) + .expect("read content") +} + +/// The KNN cosine distance of `id`'s stored vector to `query`. +fn knn_distance(h: &Harness, id: i64, query: Vec) -> f64 { + let hits = h + .store + .knn_search( + query, + SearchArgs { + limit: Some(50), + ..Default::default() + }, + ) + .expect("knn search"); + hits.into_iter() + .find(|(view, _)| view.id == id) + .map(|(_, distance)| distance) + .expect("the id has a stored vector") +} + +fn assert_stores_new_vector(h: &Harness, id: i64) { + assert_eq!(status(h, id), 1, "the row reads as embedded"); + assert_eq!(vec_rows(h, id), 1, "exactly one vector row for the id"); + let to_new = knn_distance(h, id, v_new()); + let to_old = knn_distance(h, id, v_old()); + assert!(to_new.abs() < EPS, "vector is v_new (distance {to_new})"); + assert!( + (to_old - 1.0).abs() < EPS, + "vector is not v_old (distance {to_old})" + ); +} + +// --------------------------------------------------------------------------- +// Store-level interleavings +// --------------------------------------------------------------------------- + +#[test] +fn backfill_of_an_unchanged_pending_row_writes_its_vector() { + let h = harness(); + let id = pending_row(&h, OLD_TEXT); + assert_eq!(status(&h, id), 0); + + let pending = h.store.pending_embeddings(128).expect("pending"); + assert_eq!(pending, vec![(id, OLD_TEXT.to_string())]); + + let written = h + .store + .insert_embedding(id, OLD_TEXT, v_old()) + .expect("insert_embedding"); + assert!(written, "an unchanged pending row is claimed and written"); + assert_eq!(status(&h, id), 1); + assert_eq!(vec_rows(&h, id), 1); + assert!(knn_distance(&h, id, v_old()).abs() < EPS); + assert!(h.store.pending_embeddings(128).expect("pending").is_empty()); +} + +#[test] +fn backfill_after_an_embedded_update_writes_nothing() { + let h = harness(); + let id = pending_row(&h, OLD_TEXT); + let pending = h.store.pending_embeddings(128).expect("pending"); + assert_eq!(pending, vec![(id, OLD_TEXT.to_string())]); + + // The update lands between the backfill's read and its write. + h.store + .update(id, content_patch(NEW_TEXT), Some(v_new()), h.clock.now()) + .expect("update") + .expect("id exists"); + + let written = h + .store + .insert_embedding(id, OLD_TEXT, v_old()) + .expect("insert_embedding"); + assert!(!written, "the stale backfill is skipped"); + assert_stores_new_vector(&h, id); +} + +#[test] +fn backfill_after_an_unembedded_update_leaves_the_row_pending_for_its_new_text() { + let h = harness(); + let id = pending_row(&h, OLD_TEXT); + let _ = h.store.pending_embeddings(128).expect("pending"); + + // The update's own embed failed: the row is pending with the new text. + h.store + .update(id, content_patch(NEW_TEXT), None, h.clock.now()) + .expect("update") + .expect("id exists"); + + let written = h + .store + .insert_embedding(id, OLD_TEXT, v_old()) + .expect("insert_embedding"); + assert!(!written, "the stale text is never embedded into the row"); + assert_eq!(vec_rows(&h, id), 0, "no vector for the row"); + assert_eq!(status(&h, id), 0, "still pending"); + + // The next sweep sees the CURRENT content and embeds that. + let pending = h.store.pending_embeddings(128).expect("pending"); + assert_eq!(pending, vec![(id, NEW_TEXT.to_string())]); + let written = h + .store + .insert_embedding(id, NEW_TEXT, v_new()) + .expect("insert_embedding"); + assert!(written); + assert_stores_new_vector(&h, id); +} + +#[test] +fn backfill_after_a_forget_never_resurrects_a_vector() { + let h = harness(); + let id = pending_row(&h, OLD_TEXT); + let _ = h.store.pending_embeddings(128).expect("pending"); + + assert!(h.store.forget(id).expect("forget")); + + let written = h + .store + .insert_embedding(id, OLD_TEXT, v_old()) + .expect("insert_embedding on a forgotten id is not an error"); + assert!(!written, "a forgotten row is never claimed"); + assert_eq!(vec_rows(&h, id), 0, "no vector for the forgotten id"); + + // INTEGER PRIMARY KEY without AUTOINCREMENT: the next insert into the now + // empty table reuses the rowid. The stale backfill must not attach to it. + let reused = pending_row(&h, "a different memory"); + assert_eq!(reused, id, "the rowid was recycled"); + let written = h + .store + .insert_embedding(id, OLD_TEXT, v_old()) + .expect("insert_embedding"); + assert!( + !written, + "a reused rowid with other content is never claimed" + ); + assert_eq!(vec_rows(&h, reused), 0); + assert_eq!(status(&h, reused), 0); +} + +#[test] +fn a_second_backfill_of_the_same_row_writes_nothing() { + let h = harness(); + let id = pending_row(&h, OLD_TEXT); + let _ = h.store.pending_embeddings(128).expect("pending"); + + // Two sweeps (e.g. two processes on one DB file) both read the row. + assert!(h + .store + .insert_embedding(id, OLD_TEXT, v_old()) + .expect("first backfill")); + assert!( + !h.store + .insert_embedding(id, OLD_TEXT, v_old()) + .expect("second backfill"), + "the row is no longer pending, so the second write is skipped" + ); + assert_eq!(vec_rows(&h, id), 1, "exactly one vector row"); + assert_eq!(status(&h, id), 1); +} + +// --------------------------------------------------------------------------- +// Service-level: the real sweep, pinned mid-embed +// --------------------------------------------------------------------------- + +/// Store `OLD_TEXT` while the embedder is down, so it lands pending. +async fn store_pending_via_service(h: &Harness) -> i64 { + h.embedder.set_fail(true); + let id = h + .service + .store(new_memory(OLD_TEXT)) + .await + .expect("store during an outage still succeeds"); + assert_eq!(status(h, id), 0, "stored pending"); + h.embedder.set_fail(false); + id +} + +#[tokio::test] +async fn an_update_landing_mid_backfill_keeps_its_fresh_vector() { + let h = harness(); + let id = store_pending_via_service(&h).await; + + h.embedder.arm(); + let sweeper = h.service.clone(); + let now = h.clock.now(); + let sweep = tokio::spawn(async move { sweeper.sweep(now).await }); + + // The sweep has read (id, OLD_TEXT) and is parked inside its embed call. + h.embedder.entered.notified().await; + + // The update's own embed is not gated: it succeeds and stores v_new. + let updated = h + .service + .update(id, content_patch(NEW_TEXT)) + .await + .expect("update") + .expect("id exists"); + assert_eq!(updated.embedding_status, 1); + + h.embedder.release.notify_one(); + sweep + .await + .expect("sweep task joins") + .expect("sweep succeeds"); + + assert_eq!(content(&h, id), NEW_TEXT); + assert_stores_new_vector(&h, id); +} + +#[tokio::test] +async fn a_failed_update_landing_mid_backfill_is_embedded_with_its_new_text_next_sweep() { + let h = harness(); + let id = store_pending_via_service(&h).await; + + h.embedder.arm(); + let sweeper = h.service.clone(); + let now = h.clock.now(); + let sweep = tokio::spawn(async move { sweeper.sweep(now).await }); + h.embedder.entered.notified().await; + + // Ollama goes down only around the update: it lands pending. + h.embedder.set_fail(true); + let updated = h + .service + .update(id, content_patch(NEW_TEXT)) + .await + .expect("an embed outage never fails the update") + .expect("id exists"); + assert_eq!(updated.embedding_status, 0); + h.embedder.set_fail(false); + + h.embedder.release.notify_one(); + sweep + .await + .expect("sweep task joins") + .expect("sweep succeeds"); + + assert_eq!(content(&h, id), NEW_TEXT); + assert_eq!(status(&h, id), 0, "the stale vector was not written"); + assert_eq!(vec_rows(&h, id), 0); + + // An ordinary sweep now embeds the CURRENT content. + h.service.sweep(h.clock.now()).await.expect("second sweep"); + assert_stores_new_vector(&h, id); +} From 90f91fddb2e11498133ba5a398f4ecf8a2ed7e73 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 21:57:02 +0200 Subject: [PATCH 26/42] =?UTF-8?q?fix(04-06):=20relates=5Fto=20is=20one=20s?= =?UTF-8?q?ymmetric=20relation=20=E2=80=94=20canonical=20storage,=20unlink?= =?UTF-8?q?=20from=20either=20end=20(WR-03)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LinkKind::is_symmetric / canonical_endpoints (exhaustive matches) - SqliteStore::link keeps the request-order endpoint pre-check, then inserts the canonical pair: relates_to as (min, max); directed kinds unchanged - SqliteStore::unlink: one bound DELETE matching the requested orientation, plus the reversed one for symmetric kinds (also removes pre-fix dev rows) - Store link/unlink docs; IN-05 (update on an expired-but-unswept row) and IN-06 (snippet may split a grapheme cluster) documented, no behaviour change - MCP memory_link / memory_unlink descriptions and arg docs state symmetry; tests lock request-order echo on MCP and REST and the description text - tests/relations.rs: 8 WR-03 tests (unit, both-ways, canonical row, either-end unlink, pre-fix backstop, directed kinds, request-order not-found) Co-Authored-By: Claude Opus 5.5 --- crates/agent-memory-core/src/domain.rs | 45 ++++- crates/agent-memory-core/src/store/mod.rs | 39 +++- crates/agent-memory-core/src/store/sqlite.rs | 22 +- crates/agent-memory-core/tests/relations.rs | 200 +++++++++++++++++++ crates/agent-memory/src/mcp.rs | 87 +++++++- crates/agent-memory/src/rest/handlers.rs | 31 +++ 6 files changed, 399 insertions(+), 25 deletions(-) diff --git a/crates/agent-memory-core/src/domain.rs b/crates/agent-memory-core/src/domain.rs index ec2243a..408fd74 100644 --- a/crates/agent-memory-core/src/domain.rs +++ b/crates/agent-memory-core/src/domain.rs @@ -78,6 +78,14 @@ impl std::convert::TryFrom<&str> for MemoryType { /// - `supersedes` — `from` replaces `to` (the newer memory points at the older). /// - `caused_by` — `from` was caused by `to` (the effect points at its cause). /// +/// `relates_to` is ONE symmetric relation (REVIEW WR-03): it is stored with +/// canonical endpoints, (smaller id, larger id), whichever way round it was +/// linked ([`LinkKind::canonical_endpoints`]), so linking b→a after a→b is the +/// same relation, and it can be unlinked from either end. The `direction` that +/// 1-hop expansion reports for a `relates_to` neighbour therefore carries no +/// meaning. `supersedes` and `caused_by` keep D-03 directed identity: a→b and +/// b→a are two distinct relations. +/// /// The wire form is snake_case (`relates_to`, `supersedes`, `caused_by`). /// Parse untrusted input with [`LinkKind::try_from`], which rejects anything /// else with [`MemoryError::InvalidArgument`] naming the allowed set — so a @@ -112,6 +120,33 @@ impl LinkKind { LinkKind::CausedBy => "caused", } } + + /// Whether the relation reads the same from both ends. True only for + /// `relates_to`; a symmetric kind is stored canonically and unlinked + /// from either end (REVIEW WR-03). + pub fn is_symmetric(self) -> bool { + match self { + LinkKind::RelatesTo => true, + LinkKind::Supersedes => false, + LinkKind::CausedBy => false, + } + } + + /// The `(from_id, to_id)` pair this relation is STORED under. + /// + /// `relates_to` is canonicalised to `(min, max)`, so a→b and b→a land on + /// the same primary key and a duplicate in either order is the D-02 + /// idempotent no-op. `supersedes` and `caused_by` are returned unchanged: + /// their direction is part of their identity (D-03). + /// + /// Storage only: transports always echo the REQUEST's ids on the wire. + pub fn canonical_endpoints(self, from_id: i64, to_id: i64) -> (i64, i64) { + match self { + LinkKind::RelatesTo => (from_id.min(to_id), from_id.max(to_id)), + LinkKind::Supersedes => (from_id, to_id), + LinkKind::CausedBy => (from_id, to_id), + } + } } impl std::fmt::Display for LinkKind { @@ -184,11 +219,17 @@ pub struct RelatedLink { /// Maximum characters of neighbor content carried in a [`RelatedLink`] /// snippet (D-05). Counted in Unicode scalar values, so a cut never splits a -/// UTF-8 codepoint. +/// UTF-8 codepoint. It may, however, split a grapheme cluster — a combining +/// accent sequence or a ZWJ emoji — leaving a dangling modifier before the +/// `…` (REVIEW IN-06). That is cosmetic and accepted under the D-05 +/// discretion: grapheme-aware cutting would need a new dependency for no +/// functional gain. pub const RELATED_SNIPPET_CHARS: usize = 160; /// Cut `content` to [`RELATED_SNIPPET_CHARS`] characters on a char boundary, -/// appending `…` only when something was cut. +/// appending `…` only when something was cut. The cut counts Unicode scalars, +/// not grapheme clusters, so it can split a combined emoji or accent sequence +/// (IN-06, documented and accepted). pub fn snippet(content: &str) -> String { match content.char_indices().nth(RELATED_SNIPPET_CHARS) { Some((byte_idx, _)) => format!("{}…", &content[..byte_idx]), diff --git a/crates/agent-memory-core/src/store/mod.rs b/crates/agent-memory-core/src/store/mod.rs index 702ae6b..9db26b9 100644 --- a/crates/agent-memory-core/src/store/mod.rs +++ b/crates/agent-memory-core/src/store/mod.rs @@ -157,6 +157,12 @@ pub trait Store: Send + Sync { /// `embedding_status` becomes 1; `None` leaves the row pending (0) for the /// sweep backfill. The id never changes: an update is never a DELETE+INSERT. /// `patch` must already be validated by the service. + /// + /// A row whose TTL has passed but which the sweep has not removed yet is + /// still a row (REVIEW IN-05): the update succeeds, and a `ttl_secs` patch + /// can extend or clear its expiry. This is consistent with list and search, + /// which also show such rows until the sweep runs. Documented behaviour, + /// deliberately unchanged. fn update( &self, id: i64, @@ -165,13 +171,20 @@ pub trait Store: Send + Sync { now: i64, ) -> Result, MemoryError>; - /// Create the directed edge `from_id -[kind]-> to_id` (MCP-07), stamped + /// Create the relation `from_id -[kind]-> to_id` (MCP-07), stamped /// `created_at = now`. One writer transaction: both endpoints are checked - /// first, and a missing one returns [`LinkOutcome::NotFound`] naming it - /// (`from_id` first) — a clean not-found, never an FK-violation error - /// (D-02 / T-04-13). A duplicate `(from_id, to_id, kind)` is an idempotent - /// no-op that still returns [`LinkOutcome::Linked`]. The self-link check - /// belongs to the service seam and has already run. + /// first, in REQUEST order, and a missing one returns + /// [`LinkOutcome::NotFound`] naming it (`from_id` first) — a clean + /// not-found, never an FK-violation error (D-02 / T-04-13). + /// + /// The row is stored under [`LinkKind::canonical_endpoints`]: `relates_to` + /// as (smaller id, larger id), so linking b→a after a→b is the same + /// symmetric relation (REVIEW WR-03); `supersedes` / `caused_by` as given + /// (D-03 directed identity). A duplicate of the stored row is an idempotent + /// no-op that still returns [`LinkOutcome::Linked`]. Canonicalisation is + /// storage-only: callers echo the request's ids on the wire, and + /// [`LinkOutcome`] carries no stored ids. The self-link check belongs to + /// the service seam and has already run. fn link( &self, from_id: i64, @@ -180,10 +193,16 @@ pub trait Store: Send + Sync { now: i64, ) -> Result; - /// Remove the directed edge `from_id -[kind]-> to_id` (MCP-07). Returns - /// `true` if an edge was removed, `false` if no such edge existed — the - /// clean not-found of [`Store::forget`], never an error (D-02). Direction - /// and kind are part of the edge's identity. + /// Remove the relation `from_id -[kind]-> to_id` (MCP-07). Returns `true` + /// if an edge was removed, `false` if no such edge existed — the clean + /// not-found of [`Store::forget`], never an error (D-02). + /// + /// For `supersedes` and `caused_by`, direction and kind are part of the + /// edge's identity: only the requested orientation is removed. For a + /// symmetric kind ([`LinkKind::is_symmetric`], i.e. `relates_to`) BOTH + /// orientations match, so the relation can be removed from either end + /// (REVIEW WR-03). That also removes any `relates_to` row a pre-fix dev + /// build stored in request order; no migration is involved. fn unlink(&self, from_id: i64, to_id: i64, kind: LinkKind) -> Result; /// The 1-hop neighbors of memory `id`, following its edges in BOTH diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index 95991d4..fb410e9 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -803,11 +803,17 @@ impl Store for SqliteStore { // The (from_id, to_id, kind) primary key makes a duplicate an // idempotent no-op: changes() == 0 is still Linked (D-02). The kind - // binds only the closed enum's wire form (T-04-10). + // binds only the closed enum's wire form (T-04-10). The endpoints are + // the STORED pair: relates_to is canonicalised to (min, max), so b->a + // after a->b hits the same key and stays one symmetric relation + // (REVIEW WR-03); directed kinds bind the request order unchanged. + // The pre-check above stays in request order, so NotFound still names + // the request's from_id first. + let (stored_from, stored_to) = kind.canonical_endpoints(from_id, to_id); tx.execute( "INSERT OR IGNORE INTO memory_links (from_id, to_id, kind, created_at) \ VALUES (?1, ?2, ?3, ?4)", - params![from_id, to_id, kind.as_wire_str(), now], + params![stored_from, stored_to, kind.as_wire_str(), now], )?; tx.commit()?; Ok(LinkOutcome::Linked) @@ -818,9 +824,17 @@ impl Store for SqliteStore { .writer .lock() .map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; + // The requested orientation always matches. For a symmetric kind + // (relates_to) the reversed orientation matches too (?4 = 1), so the + // relation can be removed from either end (REVIEW WR-03): that covers + // the canonical (min, max) row link() writes, and is also the backstop + // for any relates_to row a pre-fix dev build of the unreleased 0003 + // stored in request order. Every value is bound, never formatted. let removed = conn.execute( - "DELETE FROM memory_links WHERE from_id = ?1 AND to_id = ?2 AND kind = ?3", - params![from_id, to_id, kind.as_wire_str()], + "DELETE FROM memory_links WHERE kind = ?3 \ + AND ((from_id = ?1 AND to_id = ?2) \ + OR (?4 = 1 AND from_id = ?2 AND to_id = ?1))", + params![from_id, to_id, kind.as_wire_str(), kind.is_symmetric()], )?; Ok(removed > 0) } diff --git a/crates/agent-memory-core/tests/relations.rs b/crates/agent-memory-core/tests/relations.rs index 5e3731e..4b65c9c 100644 --- a/crates/agent-memory-core/tests/relations.rs +++ b/crates/agent-memory-core/tests/relations.rs @@ -735,3 +735,203 @@ async fn expansion_also_applies_on_the_semantic_path() { .expect("search"); assert!(plain.results.iter().all(|v| v.related.is_none())); } + +// --------------------------------------------------------------------------- +// 04-06 / REVIEW WR-03 — relates_to is ONE symmetric relation +// --------------------------------------------------------------------------- + +/// Every `(from_id, to_id)` row stored for `kind`, ordered by endpoints +/// (`memory_links` is `WITHOUT ROWID`). +fn stored_pairs(db_path: &Path, kind: &str) -> Vec<(i64, i64)> { + let conn = raw_conn(db_path); + let mut stmt = conn + .prepare("SELECT from_id, to_id FROM memory_links WHERE kind = ?1 ORDER BY from_id, to_id") + .expect("prepare"); + let rows = stmt + .query_map([kind], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("query pairs"); + rows.map(|row| row.expect("pair row")).collect() +} + +#[test] +fn canonical_endpoints_order_relates_to_and_leave_directed_kinds_alone() { + assert_eq!(LinkKind::RelatesTo.canonical_endpoints(5, 2), (2, 5)); + assert_eq!(LinkKind::RelatesTo.canonical_endpoints(2, 5), (2, 5)); + assert_eq!(LinkKind::Supersedes.canonical_endpoints(5, 2), (5, 2)); + assert_eq!(LinkKind::CausedBy.canonical_endpoints(5, 2), (5, 2)); +} + +#[test] +fn only_relates_to_is_symmetric() { + assert!(LinkKind::RelatesTo.is_symmetric()); + assert!(!LinkKind::Supersedes.is_symmetric()); + assert!(!LinkKind::CausedBy.is_symmetric()); +} + +#[tokio::test] +async fn relates_to_linked_both_ways_is_one_relation_listed_once_from_each_side() { + let h = harness(); + let a = store(&h, "see also one").await; + let b = store(&h, "see also two").await; + + let forward = h + .service + .link(a, b, LinkKind::RelatesTo) + .await + .expect("a -> b"); + let backward = h + .service + .link(b, a, LinkKind::RelatesTo) + .await + .expect("b -> a"); + assert_eq!(forward, LinkOutcome::Linked); + assert_eq!( + backward, + LinkOutcome::Linked, + "the reverse is the same relation: a D-02 no-op success" + ); + assert_eq!(edge_count(&h.db_path), 1, "one symmetric relation, one row"); + + let views = h.service.list(list_args(true)).await.expect("list"); + let from_a = related_of(&views, a); + let from_b = related_of(&views, b); + assert_eq!(from_a.len(), 1, "a lists b exactly once: {from_a:?}"); + assert_eq!(from_a[0].id, b); + assert_eq!(from_a[0].kind, "relates_to"); + assert_eq!(from_b.len(), 1, "b lists a exactly once: {from_b:?}"); + assert_eq!(from_b[0].id, a); + assert_eq!(from_b[0].kind, "relates_to"); +} + +#[tokio::test] +async fn relates_to_is_stored_with_the_smaller_id_first() { + let h = harness(); + let a = store(&h, "lower id").await; + let b = store(&h, "higher id").await; + assert!(b > a); + + h.service + .link(b, a, LinkKind::RelatesTo) + .await + .expect("link b -> a"); + assert_eq!(stored_pairs(&h.db_path, "relates_to"), vec![(a, b)]); +} + +#[tokio::test] +async fn relates_to_unlinks_from_either_end() { + let h = harness(); + let a = store(&h, "left").await; + let b = store(&h, "right").await; + h.service + .link(a, b, LinkKind::RelatesTo) + .await + .expect("link a -> b"); + + let removed = h + .service + .unlink(b, a, LinkKind::RelatesTo) + .await + .expect("unlink b -> a"); + assert!( + removed, + "the symmetric relation is removable from the far end" + ); + assert_eq!(edge_count(&h.db_path), 0); + + let again = h + .service + .unlink(b, a, LinkKind::RelatesTo) + .await + .expect("a second unlink is not an error"); + assert!(!again, "nothing left: the clean not-found"); +} + +#[tokio::test] +async fn unlink_removes_a_relates_to_row_a_pre_fix_build_stored_in_reverse() { + let h = harness(); + let a = store(&h, "older id").await; + let b = store(&h, "newer id").await; + assert!(b > a); + // A pre-fix dev build of the unreleased 0003 stored request order. + raw_conn(&h.db_path) + .execute( + "INSERT INTO memory_links (from_id, to_id, kind, created_at) \ + VALUES (?1, ?2, 'relates_to', 1000)", + [b, a], + ) + .expect("plant a reversed relates_to row"); + assert_eq!(stored_pairs(&h.db_path, "relates_to"), vec![(b, a)]); + + let removed = h + .service + .unlink(a, b, LinkKind::RelatesTo) + .await + .expect("unlink a -> b"); + assert!(removed); + assert_eq!(edge_count(&h.db_path), 0); +} + +#[tokio::test] +async fn directed_kinds_keep_two_distinct_relations_per_pair() { + let h = harness(); + let a = store(&h, "version a").await; + let b = store(&h, "version b").await; + h.service + .link(a, b, LinkKind::Supersedes) + .await + .expect("a supersedes b"); + h.service + .link(b, a, LinkKind::Supersedes) + .await + .expect("b supersedes a"); + assert_eq!(edge_count(&h.db_path), 2, "direction is identity (D-03)"); + + let removed = h + .service + .unlink(b, a, LinkKind::Supersedes) + .await + .expect("unlink b -> a"); + assert!(removed); + assert_eq!( + stored_pairs(&h.db_path, "supersedes"), + vec![(a, b)], + "only the requested direction went" + ); + + // The reverse of a directed edge is not that edge. + let wrong_way = h + .service + .unlink(b, a, LinkKind::Supersedes) + .await + .expect("unlink b -> a again"); + assert!(!wrong_way); + assert_eq!(edge_count(&h.db_path), 1); +} + +#[tokio::test] +async fn a_missing_relates_to_endpoint_is_named_in_request_order() { + let h = harness(); + let a = store(&h, "present").await; + + let cases = [ + (999_999, a, 999_999), + (a, 999_999, 999_999), + // Both missing: the request's from_id, even though it is the larger. + (999_998, 999_997, 999_998), + ]; + for (from_id, to_id, expected) in cases { + let outcome = h + .service + .link(from_id, to_id, LinkKind::RelatesTo) + .await + .expect("a missing endpoint is data, never an error"); + assert_eq!( + outcome, + LinkOutcome::NotFound { + missing_id: expected + }, + "link({from_id}, {to_id})" + ); + } + assert_eq!(edge_count(&h.db_path), 0); +} diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index 3676a37..f8e901e 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -205,28 +205,33 @@ impl UpdateArgs { } } -/// Arguments for `memory_link` (MCP-07): create the directed edge -/// `from_id -[kind]-> to_id`. +/// Arguments for `memory_link` (MCP-07): create the relation +/// `from_id -[kind]-> to_id`. `relates_to` is symmetric (one relation however +/// it is linked); `supersedes` and `caused_by` are directed. #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] pub struct LinkArgs { /// The id of the memory the edge starts at. pub from_id: i64, /// The id of the memory the edge points to. Must differ from `from_id`. pub to_id: i64, - /// One of: relates_to (symmetric "see also"), supersedes (from replaces - /// to), caused_by (from was caused by to). + /// One of: relates_to (symmetric "see also": linking b->a after a->b is + /// the same relation), supersedes (from replaces to), caused_by (from was + /// caused by to). pub kind: String, } -/// Arguments for `memory_unlink` (MCP-07): remove the directed edge -/// `from_id -[kind]-> to_id`. Direction and kind must match the stored edge. +/// Arguments for `memory_unlink` (MCP-07): remove the relation +/// `from_id -[kind]-> to_id`. For supersedes and caused_by, direction and kind +/// must match the stored edge; a symmetric relates_to can be removed from +/// either end. #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] pub struct UnlinkArgs { /// The id the edge starts at. pub from_id: i64, /// The id the edge points to. pub to_id: i64, - /// One of: relates_to, supersedes, caused_by. + /// One of: relates_to (symmetric: removable from either end), supersedes, + /// caused_by (direction must match). pub kind: String, } @@ -400,7 +405,7 @@ impl MemoryMcp { } #[tool( - description = "Link two memories with a typed, directed relation from_id -[kind]-> to_id. kind is one of: relates_to (symmetric see-also), supersedes (from_id replaces the older to_id), caused_by (from_id was caused by to_id). Linking an existing edge again is a no-op success; from_id and to_id must differ. Returns {from_id, to_id, kind, linked: true}, or {linked: false, reason: 'not_found', missing_id} when either memory does not exist." + description = "Link two memories with a typed relation from_id -[kind]-> to_id. kind is one of: relates_to (a symmetric see-also: linking b->a after a->b is the same relation), supersedes (from_id replaces the older to_id), caused_by (from_id was caused by to_id); supersedes and caused_by are directed. Linking an existing relation again is a no-op success; from_id and to_id must differ. The result always echoes the from_id and to_id you sent. Returns {from_id, to_id, kind, linked: true}, or {linked: false, reason: 'not_found', missing_id} when either memory does not exist." )] async fn memory_link( &self, @@ -438,7 +443,7 @@ impl MemoryMcp { } #[tool( - description = "Remove the directed relation from_id -[kind]-> to_id (direction and kind must match the stored edge; kind is relates_to, supersedes or caused_by). Returns {from_id, to_id, kind, unlinked: true}, or a clean {unlinked: false, reason: 'not_found'} result when no such edge exists." + description = "Remove the relation from_id -[kind]-> to_id; kind is relates_to, supersedes or caused_by. For supersedes and caused_by the direction must match the stored edge; relates_to is symmetric and can be removed from either end. Returns {from_id, to_id, kind, unlinked: true}, or a clean {unlinked: false, reason: 'not_found'} result when no such edge exists." )] async fn memory_unlink( &self, @@ -871,6 +876,70 @@ mod tests { ); } + #[tokio::test] + async fn relates_to_link_and_unlink_responses_echo_the_request_order() { + let dir = tempfile::tempdir().expect("tempdir"); + let server = test_server(&dir); + let a = store_one(&server, "first see-also").await; + let b = store_one(&server, "second see-also").await; + assert!(b > a, "b sorts after a, so canonical storage is (a, b)"); + // Stored canonically as (a, b), yet the wire reports what was sent. + let raw = format!(r#"{{"from_id": {b}, "to_id": {a}, "kind": "relates_to"}}"#); + + let linked = link_raw(&server, &raw).await.expect("link succeeds"); + let body = result_json(&linked); + assert_eq!(body["from_id"], b, "{body}"); + assert_eq!(body["to_id"], a, "{body}"); + assert_eq!(body["kind"], "relates_to", "{body}"); + assert_eq!(body["linked"], true, "{body}"); + + let removed = unlink_raw(&server, &raw).await.expect("unlink succeeds"); + let body = result_json(&removed); + assert_eq!(body["from_id"], b, "{body}"); + assert_eq!(body["to_id"], a, "{body}"); + assert_eq!(body["kind"], "relates_to", "{body}"); + assert_eq!(body["unlinked"], true, "{body}"); + + // A not-found body names the request's ids and its missing id. + let missing = link_raw( + &server, + &format!(r#"{{"from_id": 999999, "to_id": {a}, "kind": "relates_to"}}"#), + ) + .await + .expect("a missing endpoint is a successful tool call"); + assert_eq!( + result_json(&missing), + serde_json::json!({ + "from_id": 999_999, "to_id": a, "kind": "relates_to", + "linked": false, "reason": "not_found", "missing_id": 999_999 + }) + ); + } + + #[test] + fn link_and_unlink_descriptions_state_relates_to_symmetry() { + let tools = MemoryMcp::tool_router().list_all(); + let description_of = |name: &str| { + tools + .iter() + .find(|t| t.name == name) + .unwrap_or_else(|| panic!("{name} is registered")) + .description + .as_deref() + .unwrap_or_default() + .to_string() + }; + let link = description_of("memory_link"); + let unlink = description_of("memory_unlink"); + assert!(link.contains("symmetric"), "{link}"); + assert!(unlink.contains("symmetric"), "{unlink}"); + assert!(unlink.contains("either end"), "{unlink}"); + assert!( + unlink.contains("For supersedes and caused_by the direction must match"), + "{unlink}" + ); + } + #[tokio::test] async fn memory_link_to_a_missing_memory_is_a_clean_not_found_naming_it() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index 38ff150..24165f3 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -1331,6 +1331,37 @@ mod tests { ); } + #[tokio::test] + async fn relates_to_link_and_unlink_bodies_echo_the_request_order() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let a = stored_id(&state, "first see-also").await; + let b = stored_id(&state, "second see-also").await; + assert!(b > a, "b sorts after a, so canonical storage is (a, b)"); + + let resp = link_raw( + &state, + &format!(r#"{{"from_id": {b}, "to_id": {a}, "kind": "relates_to"}}"#), + ) + .await; + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + body_json(resp).await, + serde_json::json!({ + "from_id": b, "to_id": a, "kind": "relates_to", "linked": true + }) + ); + + let resp = unlink_path(&state, b, a, "relates_to").await; + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + body_json(resp).await, + serde_json::json!({ + "from_id": b, "to_id": a, "kind": "relates_to", "unlinked": true + }) + ); + } + #[tokio::test] async fn post_links_to_a_missing_memory_is_404_naming_it_never_500() { let dir = tempfile::tempdir().expect("tempdir"); From c21af3867000166086f6079617ab353ac0cca969 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 21:57:52 +0200 Subject: [PATCH 27/42] docs(04-06): relations, expiring-row updates, snippets and the real expanded-list bound - README Relations: relates_to symmetric (b->a after a->b is a no-op), unlink from either end, responses echo the request ids, relates_to direction carries no meaning, snippet may split a combined emoji/accent sequence, unlimited expanded memory_list note (pass a limit on large stores) - README memory_update: an expired-but-unswept memory can still be updated and its TTL extended or cleared (IN-05) - MAX_EXPANDED_LINKS doc: the 200 x 10 bound covers search and list-with-limit only; a list without a limit returns every row (WR-04 doc correction, behaviour change deferred) Co-Authored-By: Claude Opus 5.5 --- README.md | 32 +++++++++++++++++++------- crates/agent-memory-core/src/domain.rs | 13 +++++++++-- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c32fe22..e3924e9 100644 --- a/README.md +++ b/README.md @@ -242,14 +242,19 @@ returns — plus `embedding_status`: ranking (in hybrid and keyword mode alike) already sees the new content. - **An update counts as a use.** It refreshes `last_accessed` (and the access count), so an edited memory ranks as current rather than decayed. +- **Expired but not yet swept still counts as present.** An update to a memory + whose TTL has already passed, but which the background sweep + has not yet removed, still succeeds, just as `memory_list` and + `memory_search` still show it until the sweep runs. A `ttl_secs` patch can + extend or clear its expiry. For an unknown id the result is `{ "id": 42, "updated": false, "reason": "not_found" }` — a normal tool result, not an error. ### Relations (`memory_link` / `memory_unlink`) -Connect memories with flat, typed relations — no graph model, just one directed -edge `from_id -[kind]-> to_id` per link: +Connect memories with flat, typed relations — no graph model, just one edge +`from_id -[kind]-> to_id` per relation: | `kind` | Reads as | Seen from `to_id` (expansion) | |--------|----------|-------------------------------| @@ -265,7 +270,12 @@ edge `from_id -[kind]-> to_id` per link: - **Idempotent.** Linking the same `(from_id, to_id, kind)` again is a no-op success — safe to retry. A different `kind` between the same pair is a - separate relation. + separate relation. `relates_to` is symmetric, so linking `b → a` after + `a → b` is the same relation and also a no-op success. `supersedes` and + `caused_by` are directed: `a → b` and `b → a` are two relations. +- **Responses echo your request.** `memory_link` / `memory_unlink` (and the + REST link bodies) always report the `from_id` and `to_id` you sent, even + though a `relates_to` relation is stored as (smaller id, larger id). - **Rejected as `invalid_params`:** a self-link (`from_id and to_id must differ (got 57)`) and an unknown kind (`kind must be one of relates_to, supersedes, caused_by (got 'blocks')`). @@ -273,9 +283,10 @@ edge `from_id -[kind]-> to_id` per link: "not_found", "missing_id": 42 }` names the id that does not exist. Nothing is written. -`memory_unlink` takes the same three fields; direction and kind must match the -stored edge. It returns `"unlinked": true`, or `"unlinked": false, "reason": -"not_found"` when no such edge exists. +`memory_unlink` takes the same three fields. For `supersedes` and `caused_by`, +direction and kind must match the stored edge; a `relates_to` relation is +symmetric and can be removed from either end. It returns `"unlinked": true`, +or `"unlinked": false, "reason": "not_found"` when no such edge exists. **Relations never outlive their memories.** Forgetting a memory, or its TTL expiring, removes every relation touching it in the same transaction. @@ -294,11 +305,16 @@ summaries, following relations in both directions: - `direction` is `out` when the result is the relation's `from_id` and `in` when it is the `to_id`; `kind` reads naturally from the result's side (the - table above). + table above). For `relates_to` the reported `direction` carries no meaning: + the relation is symmetric and stored with the smaller id first. - `snippet` is the neighbor's content cut to 160 characters, with a trailing - `…` when cut. Fetch the neighbor by id if you need all of it. + `…` when cut. Fetch the neighbor by id if you need all of it. The cut counts + characters, not visual glyphs, so it may split a combined emoji or accent + sequence. - At most `MAX_EXPANDED_LINKS = 10` neighbors per result; when a memory has more, the most recently created relations win. +- `memory_list` with `expand_links` and no `limit` returns every memory, each + with up to 10 neighbors, so pass a `limit` on large stores. - Expansion is strictly one hop and a pure read: neighbors are **not** recency-bumped, so expanding never changes how anything decays. - A result with no relations gets `"related": []`. Without the flag there is no diff --git a/crates/agent-memory-core/src/domain.rs b/crates/agent-memory-core/src/domain.rs index 408fd74..5efbab4 100644 --- a/crates/agent-memory-core/src/domain.rs +++ b/crates/agent-memory-core/src/domain.rs @@ -357,8 +357,17 @@ pub const MAX_TTL_SECS: i64 = 3_155_760_000; /// Hard cap on the 1-hop neighbors attached to ONE search/list result when /// `expand_links` is set (MCP-07 / D-07). Overflow keeps the most recent /// edges (`memory_links.created_at` descending). Fixed — there is no -/// caller-tunable knob in v0.1.0. Together with [`MAX_LIMIT`] it bounds an -/// expanded response at `200 × 10` summaries (T-04-11). +/// caller-tunable knob in v0.1.0. +/// +/// What this bounds (REVIEW WR-04 doc correction): each result carries at most +/// `MAX_EXPANDED_LINKS` neighbours. Search is always bounded by its limit +/// (1–[`MAX_LIMIT`], default 50), so an expanded search returns at most +/// `200 × 10` summaries. A list without a limit returns every row (v1.0 +/// behaviour, RESEARCH Pitfall 6), and each row is expanded up to +/// `MAX_EXPANDED_LINKS`, so an unlimited expanded list grows with the corpus. +/// T-04-11's `200 × 10` bound therefore covers search and list-with-limit +/// only; changing unlimited expanded-list behaviour is a deferred API-contract +/// decision. pub const MAX_EXPANDED_LINKS: usize = 10; /// The library's typed error. `thiserror` per the ecosystem rule (libraries use From c72146bc6296170116a8ed40749576c6af5c6f87 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 21:59:34 +0200 Subject: [PATCH 28/42] docs(04-06): complete write-path integrity gap-closure plan (summary) Co-Authored-By: Claude Opus 5.5 --- .../04-06-SUMMARY.md | 287 ++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-06-SUMMARY.md diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-06-SUMMARY.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-06-SUMMARY.md new file mode 100644 index 0000000..db99705 --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-06-SUMMARY.md @@ -0,0 +1,287 @@ +--- +phase: 04-memory-update-relations-hybrid-search +plan: 06 +subsystem: store +tags: [sqlite, sqlite-vec, embedding-backfill, concurrency, relations, mcp, rest, gap-closure] + +# Dependency graph +requires: + - phase: 04-memory-update-relations-hybrid-search (04-05) + provides: forgiving keyword leg and search error redaction (the tree this wave builds on) + - phase: 04-memory-update-relations-hybrid-search (04-02, 04-03) + provides: in-place Store::update with in-tx vector swap; memory_links table, link/unlink, 1-hop expansion +provides: + - conditional Store::insert_embedding(id, expected_content, vec) -> Result (guarded claim in the writer tx) + - sweep backfill that skips stale rows and logs backfilled + skipped_stale + - LinkKind::is_symmetric / LinkKind::canonical_endpoints + - canonical (min, max) relates_to storage and both-orientation symmetric unlink + - docs for IN-05, IN-06 and the WR-04 expanded-list bound +affects: [04-07, phase-05, phase-06 release notes] + +actuals: + tokens: 10178 + tasks: 3 + commits: 4 + +tech-stack: + added: [] + patterns: + - "Guarded claim-then-write: one conditional UPDATE (status AND content) as the first statement of the writer tx; zero changes means roll back and report false" + - "Gated test embedder: tokio Notify entered/release around a one-shot armed flag to pin an async sweep between its read and its write, with no sleeps" + - "Canonical storage for symmetric relations, request-order echo on the wire" + +key-files: + created: + - crates/agent-memory-core/tests/backfill_race.rs + modified: + - crates/agent-memory-core/src/store/mod.rs + - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory-core/src/service.rs + - crates/agent-memory-core/src/domain.rs + - crates/agent-memory-core/tests/relations.rs + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - README.md + +key-decisions: + - "04-06: the sweep backfill writes a vector only through a guarded claim (UPDATE memories SET embedding_status = 1 WHERE id = ?1 AND embedding_status = 0 AND content = ?2) run first in the writer tx; Store::insert_embedding now returns bool (true = written, false = row changed, embedded or forgotten). Closes REVIEW CR-03 across processes, since SQLite takes the write lock at that statement." + - "04-06: relates_to is stored canonically as (min id, max id) and unlinked from either end; supersedes/caused_by keep D-03 directed identity. Storage-only: MCP and REST responses echo the request's from_id/to_id, and NotFound still names the request's from_id first. No migration; 0003 untouched (REVIEW WR-03)." + - "04-06: WR-04 is documented, not changed: MAX_EXPANDED_LINKS' 200 x 10 bound covers search and list-with-limit only; an unlimited expanded memory_list returns every row. The behaviour choice (reject / default limit / leave unbounded) stays an open API-contract question for Phase 5/6." + +patterns-established: + - "Any background write computed from a snapshot read outside the writer lock must re-assert the snapshot in its own write statement (compare-and-set), never write unconditionally." + +requirements-completed: [MCP-06, MCP-07] + +coverage: + - id: D1 + description: "An in-flight sweep backfill never overwrites an updated memory's vector with an embedding of its old text; the row keeps the vector of its current content (REVIEW CR-03, T-04-23)" + requirement: "MCP-06" + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/backfill_race.rs#backfill_after_an_embedded_update_writes_nothing" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/backfill_race.rs#an_update_landing_mid_backfill_keeps_its_fresh_vector" + status: pass + human_judgment: false + - id: D2 + description: "If the racing update's own embed failed, the row stays pending with no vector and the next sweep embeds its NEW content" + requirement: "MCP-06" + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/backfill_race.rs#backfill_after_an_unembedded_update_leaves_the_row_pending_for_its_new_text" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/backfill_race.rs#a_failed_update_landing_mid_backfill_is_embedded_with_its_new_text_next_sweep" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/update.rs#embed_outage_during_update_falls_back_to_pending_and_sweep_backfills (unchanged, green)" + status: pass + human_judgment: false + - id: D3 + description: "A memory forgotten mid-backfill never regains a vector, including when its rowid is recycled; two sweeps on one row write exactly one vector (T-04-24)" + requirement: "MCP-06" + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/backfill_race.rs#backfill_after_a_forget_never_resurrects_a_vector" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/backfill_race.rs#a_second_backfill_of_the_same_row_writes_nothing" + status: pass + human_judgment: false + - id: D4 + description: "relates_to is one symmetric relation: a->b then b->a stores one edge, expansion lists the neighbour once per side, unlink works from either end (incl. pre-fix reversed rows) (REVIEW WR-03, T-04-25)" + requirement: "MCP-07" + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#relates_to_linked_both_ways_is_one_relation_listed_once_from_each_side" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#relates_to_is_stored_with_the_smaller_id_first" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#relates_to_unlinks_from_either_end" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#unlink_removes_a_relates_to_row_a_pre_fix_build_stored_in_reverse" + status: pass + - kind: unit + ref: "crates/agent-memory-core/tests/relations.rs#canonical_endpoints_order_relates_to_and_leave_directed_kinds_alone" + status: pass + human_judgment: false + - id: D5 + description: "supersedes/caused_by keep directed identity, and a missing endpoint is named in request order" + requirement: "MCP-07" + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#directed_kinds_keep_two_distinct_relations_per_pair" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/relations.rs#a_missing_relates_to_endpoint_is_named_in_request_order" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#post_links_to_a_missing_memory_is_404_naming_it_never_500 (unchanged, green)" + status: pass + human_judgment: false + - id: D6 + description: "Canonical storage is invisible on the wire: MCP memory_link/memory_unlink results and REST link bodies echo the request's from_id/to_id; tool descriptions state relates_to symmetry" + requirement: "MCP-07" + verification: + - kind: integration + ref: "crates/agent-memory/src/mcp.rs#relates_to_link_and_unlink_responses_echo_the_request_order" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#relates_to_link_and_unlink_bodies_echo_the_request_order" + status: pass + - kind: unit + ref: "crates/agent-memory/src/mcp.rs#link_and_unlink_descriptions_state_relates_to_symmetry" + status: pass + human_judgment: false + - id: D7 + description: "README and doc comments describe symmetric relates_to, updates to expired-but-unswept rows (IN-05), grapheme-splitting snippets (IN-06) and the real expanded-list bound (WR-04 doc)" + verification: + - kind: other + ref: "grep -n 'either end' README.md; grep -n 'has not yet removed' README.md; grep -n 'carries no meaning' README.md; grep -n 'pass a .limit. on large stores' README.md; grep -n 'without a limit returns every row' crates/agent-memory-core/src/domain.rs" + status: pass + human_judgment: false + +duration: 7min +completed: 2026-10-23 +status: complete +--- + +# Phase 04 Plan 06: Write-path integrity (backfill race, symmetric relates_to) Summary + +**The sweep's embedding backfill can no longer overwrite an edited or deleted memory's vector with an embedding of its old text. The backfill used to write unconditionally; it now writes only through a guarded `UPDATE` that also checks the row is still pending with exactly the text that was embedded. Separately, `relates_to` is now one symmetric relation: it is stored as (smaller id, larger id), so linking it the other way round is a no-op and it can be unlinked from either end. MCP and REST responses still echo the ids exactly as the client sent them.** + +## Performance + +- **Duration:** ~7 min +- **Started:** 2026-10-23T19:51:05Z +- **Completed:** 2026-10-23T19:58Z +- **Tasks:** 3/3 +- **Files modified:** 9 (1 created, 8 modified) + +## Accomplishments + +- **CR-03 closed.** `Store::insert_embedding(id, expected_content, vec) -> Result` starts its writer transaction with `UPDATE memories SET embedding_status = 1 WHERE id = ?1 AND embedding_status = 0 AND content = ?2`. If that changes no row, the transaction rolls back and the call returns `Ok(false)`. Otherwise the existing vec0 DELETE + INSERT runs and commits. This closes four interleavings: + - update with a successful embed; + - update whose embed failed; + - forget, including when the rowid is recycled; + - sweep against sweep across processes. SQLite takes its write lock at the claim, so the per-process mutex is no longer the only guard. +- **Sweep wiring.** `MemoryService::sweep` zips each `(id, content)` with its vector, passes back the exact text it embedded, and logs both `backfilled` and `skipped_stale`. The `SweepReport` shape is unchanged. +- **Race tests.** `tests/backfill_race.rs` has 7 deterministic tests: + - 5 store-level interleavings driven through the public `Store` methods; + - 2 service-level tests that pin the real sweep mid-embed with a `GatedEmbedder` (tokio `Notify`, no sleeps) and land an update in the gap. In one the update embeds successfully; in the other the update's embed fails. +- **WR-03 closed.** + - `LinkKind::is_symmetric` and `LinkKind::canonical_endpoints` use exhaustive matches. + - `link` keeps the endpoint pre-check in request order, then inserts the canonical pair. + - `unlink` is one bound DELETE that also matches the reversed orientation when the kind is symmetric. The same DELETE removes relates_to rows a pre-fix dev build stored in request order. There is no migration. +- **Docs.** + - Store `link`, `unlink` and `update` (IN-05); `snippet` (IN-06); `MAX_EXPANDED_LINKS` (WR-04 correction). + - The memory_link and memory_unlink tool descriptions and argument docs. + - README sections: Relations, memory_update, Expansion, and the Lifecycle backfill bullet. + +## Task Commits + +1. **Task 1 (tracer): conditional sweep backfill (CR-03).** `f299f49` (fix) +2. **Task 2: symmetric relates_to (canonical storage, unlink from either end) (WR-03), plus IN-05/IN-06 doc comments.** `90f91fd` (fix) +3. **Task 3: README and the `MAX_EXPANDED_LINKS` doc (WR-03/IN-05/IN-06/WR-04).** `c21af38` (docs) + +**Plan metadata:** see the `docs(04-06)` commit that follows. + +## Files Created/Modified + +- `crates/agent-memory-core/tests/backfill_race.rs` (new, 7 tests): the CR-03 interleavings. +- `crates/agent-memory-core/src/store/mod.rs`: + - the new `insert_embedding` signature and contract; + - the `pending_embeddings` expected-content note; + - the `link` / `unlink` canonical and symmetric contract; + - the IN-05 note on `update`. +- `crates/agent-memory-core/src/store/sqlite.rs`: guarded `insert_embedding`, canonical `link` INSERT, symmetric `unlink` DELETE. +- `crates/agent-memory-core/src/service.rs`: the sweep write loop now passes `&content`, counts written and skipped rows, and logs `skipped_stale`; its doc comment is updated. +- `crates/agent-memory-core/src/domain.rs`: + - `is_symmetric` and `canonical_endpoints`; + - a symmetry note on the `LinkKind` docs; + - the IN-06 doc on `RELATED_SNIPPET_CHARS` / `snippet`; + - the WR-04 correction on `MAX_EXPANDED_LINKS`. +- `crates/agent-memory-core/tests/relations.rs`: 8 new WR-03 tests. All pre-existing tests are unchanged and green. +- `crates/agent-memory/src/mcp.rs`: the memory_link and memory_unlink descriptions, `LinkArgs`/`UnlinkArgs` docs, and 2 new tests. +- `crates/agent-memory/src/rest/handlers.rs`: 1 new test for the REST link and unlink bodies. +- `README.md`: Lifecycle (backfill), memory_update (IN-05), and Relations/Expansion (WR-03, IN-06, WR-04 note). The Relations table is unchanged. + +## Decisions Made + +- I bound the symmetric-unlink flag as an SQL parameter (`?4 = 1`) inside a single DELETE rather than branching to two statements. There is one statement and no formatted SQL, and it reports one `changes()` count for the clean not-found. +- In the forget-race test I asserted rowid recycling deterministically (`reused == id`), so the "a stale backfill never attaches to a recycled rowid" check is never vacuous. SQLite assigns max(rowid)+1, which is 1 in an empty table. + +## Deviations from Plan + +### Process + +**1. Tracer feedback gate run as an automated re-verify, not an interactive checkpoint** +- **Found during:** Task 1 (tracer) +- **Issue:** Auto mode is off (`_auto_chain_active=false`, `auto_advance=false`), so the protocol calls for a `checkpoint:human-verify` after the tracer. The orchestrator asked for the whole plan in one run, and the tracer's `` is fully automated (cargo tests plus clippy), as in 04-05. +- **Fix:** After committing `f299f49` I re-ran the full tracer `` on the committed tree as the gate. It was all green (7 + 5 + 4/1 ignored + 9, clippy clean), so I continued to Task 2. + +**2. RED proven by temporary mutation instead of separate RED commits** +- Each fix must land atomically with its tests, because CLAUDE.md forbids committing failing tests. I therefore proved RED in place, without committing it: + - Weakening the claim to `WHERE id = ?1` failed 6 of 7 backfill_race tests. The happy path, which is the positive control, still passed. + - Neutralising `canonical_endpoints` / `is_symmetric` failed 6 of the 8 new relations tests. The directed-kind and request-order tests are invariants and passed both ways. +- I restored both files from byte-identical backups (`cmp`-verified) before committing. + +### Additive test strengthening (no scope change) + +**3. [Rule 2 - Coverage] REST request-order echo test** +- **Found during:** Task 2 +- **Issue:** The plan's `` requires the REST `POST /api/links` body to echo the request order, but `` only adds the MCP test, and `handlers.rs` was not in `files_modified`. +- **Fix:** I added `relates_to_link_and_unlink_bodies_echo_the_request_order` to `crates/agent-memory/src/rest/handlers.rs`. It is test-only, and response construction is untouched. +- **Commit:** `90f91fd` + +**4. [Rule 2 - Coverage] Extra assertions** +- The MCP echo test also asserts the not-found body. The forget-race test also covers rowid reuse. The description test also pins the "direction must match" wording for supersedes and caused_by. + +--- + +**Total deviations:** 2 process, 2 additive (Rule 2). **Impact:** none on behaviour or scope. + +## TDD Gate Compliance + +Tasks 1 and 2 carry `tdd="true"` in a `type: execute` plan. Each lands as one `fix(...)` commit that bundles its tests, because a separate RED commit would mean committing failing tests, which CLAUDE.md forbids. RED was observed by temporary mutation (deviation 2), and GREEN is each task's commit. + +## Issues Encountered + +- `memory_links` is `WITHOUT ROWID`. My first draft of the `stored_pairs` test helper ordered by `rowid`, so I changed it to order by `from_id, to_id` before the first run. + +## Verification + +- `cargo fmt --all --check`: clean +- `cargo clippy --workspace --all-targets -- -D warnings`: clean +- `cargo test --workspace`: **200 passed / 0 failed / 2 ignored**. That is up from 182 after 04-05: +7 backfill_race, +8 relations, +3 binary-crate tests. Both ignored tests pre-date this plan. +- Plan acceptance greps: all present. These are `embedding_status = 0 AND content = ?2`, `expected_content: &str`, `skipped_stale`, `fn canonical_endpoints`, `fn is_symmetric`, `canonical_endpoints` / `is_symmetric` in sqlite.rs, `grapheme`, `sweep has not`, `either end`, `has not yet removed`, `carries no meaning`, `pass a .limit. on large stores` and `without a limit returns every row`. +- No `unwrap()` added to non-test code. No migration, no CI workflow change, no tag, release or push. + +## Known Stubs + +None. + +## Deferred (as the plan states) + +- **WR-04 behaviour change** (unbounded `memory_list` with `expand_links` and no `limit`) is an open API-contract question for Phase 5/6. The options are: reject, apply a default limit, or leave it unbounded and documented. Only the doc and README were corrected here. +- **IN-05 behaviour change** (treating expired-unswept rows as not-found in `update`): documented instead. +- **IN-06** grapheme-aware or word-boundary snippet cut: documented instead. +- **WR-01, IN-01, IN-02, IN-03, WR-02:** plan 04-07. + +## User Setup Required + +None. + +## Next Phase Readiness + +- Ready for 04-07 (WR-01, IN-01..03, WR-02). This is the last plan in phase 04, and the phase is not yet complete. + +## Self-Check: PASSED + +- `crates/agent-memory-core/tests/backfill_race.rs` exists, and all 8 modified files are present. +- Commits `f299f49`, `90f91fd` and `c21af38` were found in `git log`. From a95d91db9285d81226ed11bd1d52df4d1001be28 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 22:00:03 +0200 Subject: [PATCH 29/42] docs(04-06): update state and roadmap progress (6/7 phase-04 plans) Co-Authored-By: Claude Opus 5.5 --- .planning/ROADMAP.md | 6 +++--- .planning/STATE.md | 24 ++++++++++++++---------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 67b56cc..c725d19 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -67,7 +67,7 @@ Plans: 4. Hybrid ranking stays decay-aware — a fresh memory outranks a stale one at equal fused rank — and access bumps apply only to post-truncation returned ids. 5. Upgrading a real v0.0.1 database applies migration 0003 only after a pre-migration backup; shipped 0001/0002 SQL is provably unchanged (fixture divergence test against a v0.0.1 database), and a newer-schema DB met by an older binary yields a friendly error, not a crash. -**Plans**: 5/7 plans executed (4 executed + 3 gap closure) +**Plans**: 6/7 plans executed (4 executed + 3 gap closure) Plans: **Wave 1** @@ -88,7 +88,7 @@ Plans: **Gap closure** *(from 04-VERIFICATION.md + 04-REVIEW.md; gap waves run in order)* - [x] 04-05-PLAN.md — Gap wave 1: forgiving keyword leg per user decision option-a (raw MATCH first, one sanitized OR-of-quoted-tokens retry, tokenless = empty leg, identical in both modes) + search-path error redaction (CR-01, CR-02, truths #14/#15, IN-04) (SEARCH-04) -- [ ] 04-06-PLAN.md — Gap wave 2: conditional sweep backfill closing the update/forget/sweep races + symmetric relates_to + IN-05/IN-06 docs (CR-03, WR-03) (MCP-06, MCP-07) +- [x] 04-06-PLAN.md — Gap wave 2: conditional sweep backfill closing the update/forget/sweep races + symmetric relates_to + IN-05/IN-06 docs (CR-03, WR-03) (MCP-06, MCP-07) - [ ] 04-07-PLAN.md — Gap wave 3: deny_unknown_fields on update DTOs, uniform REST rejection bodies, race-safe verify-then-rename pre-migration backup + Upgrading docs (WR-02, IN-02, WR-01, IN-03, IN-01) (MCP-06, MCP-07, STORE-05) ### Phase 5: Portable Export & Import @@ -130,6 +130,6 @@ Phases execute in numeric order: 3 → 4 → 5 → 6 | 1. Core Memory Foundation | v1.0 | 3/3 | Complete | 2026-06-25 | | 2. Semantic Search, Interop & Release | v1.0 | 5/5 | Complete | 2026-07-12 | | 3. API Hardening & Toolchain Spikes | v1.1 | 2/2 | Complete | 2026-07-12 | -| 4. Memory Update, Relations & Hybrid Search | v1.1 | 5/7 | In Progress| | +| 4. Memory Update, Relations & Hybrid Search | v1.1 | 6/7 | In Progress| | | 5. Portable Export & Import | v1.1 | 0/? | Not started | - | | 6. Distribution & v0.1.0 Release | v1.1 | 0/? | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index 4c81128..0c83474 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -5,16 +5,16 @@ milestone_name: Hardening & Interop current_phase: 04 current_phase_name: Memory Update, Relations & Hybrid Search status: executing -stopped_at: Completed 04-05-PLAN.md -last_updated: "2026-10-23T19:49:24.593Z" +stopped_at: Completed 04-06-PLAN.md +last_updated: "2026-10-23T19:59:51.264Z" last_activity: 2026-10-23 -last_activity_desc: Completed 04-05 (gap wave 1, option-a forgiving keyword leg) -state_head: 42533282bbb1b9dac1228e0438b38ddc90c8fdff +last_activity_desc: Completed 04-06 (gap wave 2, conditional backfill + symmetric relates_to) +state_head: c72146bc6296170116a8ed40749576c6af5c6f87 progress: total_phases: 4 completed_phases: 1 total_plans: 9 - completed_plans: 7 + completed_plans: 8 --- # Project State @@ -29,9 +29,9 @@ See: .planning/PROJECT.md (updated 2026-07-12) ## Current Position Phase: 04 (Memory Update, Relations & Hybrid Search) — EXECUTING -Plan: 5 of 7 (gap closure: 04-05 done; 04-06, 04-07 next) -Status: Executing gap closure — 04-06 ready to execute -Last activity: 2026-10-23 — Completed 04-05 (option-a forgiving keyword leg + search error redaction) +Plan: 6 of 7 (gap closure: 04-05, 04-06 done; 04-07 next) +Status: Executing gap closure — 04-07 ready to execute +Last activity: 2026-10-23 — Completed 04-06 (CR-03 conditional sweep backfill, WR-03 symmetric relates_to, IN-05/IN-06/WR-04 docs) Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) @@ -73,6 +73,7 @@ Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) | Phase 04 P03 | 10 min | 2 tasks | 9 files | | Phase 04 P04 | 11 min | 2 tasks | 11 files | | Phase 04 P05 | 10 min | 3 tasks | 12 files | +| Phase 04 P06 | 7 min | 3 tasks | 9 files | ## Accumulated Context @@ -114,6 +115,9 @@ Recent decisions affecting current work: - [Phase 04]: 04-04: SearchMode::Semantic removed; embedder-up searches report search_mode "hybrid", embedder-down stays "keyword" byte-identical to v1.0 (README behavioral change 3) - [Phase 04]: 04-04: Keyword-leg errors propagate first, so a malformed MATCH (incl. bare ?/,/' punctuation) is InvalidQuery/400 in hybrid as in keyword mode (T-04-15); follow-up candidate: sanitize natural-language queries for the keyword leg - [Phase 04]: 04-05: T-04-15 revised by user decision option-a (2026-10-23) — search runs the raw FTS5 MATCH first, then one sanitized retry (word tokens quoted and OR-joined) on any FTS5 query-parse error, including the step-time `no such column` class from hyphens/colons; a tokenless query is an empty keyword leg (hybrid answers semantic-only, keyword mode returns empty); identical in both modes, so search never returns InvalidQuery; every search-path internal error is redacted to SEARCH_STORAGE_FAILURE (T-04-19). Supersedes the 04-04 decision that a malformed MATCH is InvalidQuery/400 in hybrid mode. +- [Phase 04]: 04-06: sweep backfill writes a vector only through a guarded claim (embedding_status = 0 AND content = embedded text) run first in the writer tx; Store::insert_embedding returns bool. Closes REVIEW CR-03, across processes too. +- [Phase 04]: 04-06: relates_to stored canonically as (min id, max id) and unlinked from either end; supersedes/caused_by stay directed (D-03). Storage-only: MCP/REST responses echo request ids; NotFound names the request's from_id first. No migration (REVIEW WR-03). +- [Phase 04]: 04-06: WR-04 documented only: the 200 x 10 expansion bound covers search and list-with-limit; unlimited expanded memory_list returns every row. Behaviour choice (reject / default limit / unbounded) is an open API-contract question for Phase 5/6. ### Pending Todos @@ -154,8 +158,8 @@ Items acknowledged and carried forward: ## Session Continuity -Last session: 2026-10-23T19:49:24.539Z -Stopped at: Completed 04-05-PLAN.md +Last session: 2026-10-23T19:59:51.214Z +Stopped at: Completed 04-06-PLAN.md Resume file: None ## Operator Next Steps From f09a69f8abe0560bb58443278f3ca0bc9a56fc70 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 22:04:52 +0200 Subject: [PATCH 30/42] fix(04-07): memory_update refuses unknown and immutable fields on MCP and REST (WR-02) - #[serde(deny_unknown_fields)] on mcp::UpdateArgs and rest::UpdateBody only; the [03-01] decision stands for the v0.0.1 DTOs - schema test now also pins additionalProperties == false - stdio e2e: type / ttl_sec are refused as an rmcp tool error result (isError, 'failed to deserialize parameters: unknown field'), row untouched - REST: PATCH with type/source/ttl_sec is a 400 naming the field, nothing written - README memory_update note and STATE decision Co-Authored-By: Claude Opus 5.5 --- .planning/STATE.md | 1 + README.md | 6 +++ crates/agent-memory/src/mcp.rs | 36 ++++++++++++- crates/agent-memory/src/rest/handlers.rs | 51 ++++++++++++++++++ crates/agent-memory/tests/tools.rs | 66 ++++++++++++++++++++++++ 5 files changed, 159 insertions(+), 1 deletion(-) diff --git a/.planning/STATE.md b/.planning/STATE.md index 0c83474..a020830 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -118,6 +118,7 @@ Recent decisions affecting current work: - [Phase 04]: 04-06: sweep backfill writes a vector only through a guarded claim (embedding_status = 0 AND content = embedded text) run first in the writer tx; Store::insert_embedding returns bool. Closes REVIEW CR-03, across processes too. - [Phase 04]: 04-06: relates_to stored canonically as (min id, max id) and unlinked from either end; supersedes/caused_by stay directed (D-03). Storage-only: MCP/REST responses echo request ids; NotFound names the request's from_id first. No migration (REVIEW WR-03). - [Phase 04]: 04-06: WR-04 documented only: the 200 x 10 expansion bound covers search and list-with-limit; unlimited expanded memory_list returns every row. Behaviour choice (reject / default limit / unbounded) is an open API-contract question for Phase 5/6. +- [Phase 04]: 04-07: deny_unknown_fields added to UpdateArgs/UpdateBody only (new v0.1.0 surfaces, REVIEW WR-02); the [03-01] decision stands for the v0.0.1 DTOs, so the v0.1.0 release-notes delta for existing surfaces is unchanged. Over MCP, rmcp 1.8 reports the rejection as a tool error result (isError: true, "failed to deserialize parameters: unknown field ..."), not a JSON-RPC -32602; REST is a 400 {"error"}. ### Pending Todos diff --git a/README.md b/README.md index e3924e9..efeef28 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,12 @@ forget the old. A patch with no field besides `id`, empty or whitespace-only `content`, or an out-of-bounds `ttl_secs` is rejected as `invalid_params` before anything is written. +Unknown fields are rejected too, never silently ignored: a patch carrying +`type`, `source` or a misspelled field such as `ttl_sec` fails as a whole and +writes nothing. Over MCP it is a tool error result (`isError: true`) whose text +reads `failed to deserialize parameters: unknown field ...` and names the +field; over REST it is HTTP 400 with the same message in the `error` body. + On success the result is the full updated memory — the same fields `memory_list` returns — plus `embedding_status`: diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index f8e901e..4bddaaa 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -152,11 +152,21 @@ pub struct ForgetArgs { /// can be replaced but never cleared or emptied. At least one field besides /// `id` is required. `type` and `source` cannot be changed. /// +/// Unknown keys are rejected (`deny_unknown_fields`, REVIEW WR-02): that covers +/// the immutable `type`/`source` and typos such as `ttl_sec`. In a merge-patch a +/// silently dropped member is a silent partial write, so the whole call fails +/// naming the field and nothing is written. rmcp deserializes the tool +/// `arguments` object straight into this type and reports the failure as a +/// tool error result (`isError: true`, "failed to deserialize parameters: +/// unknown field ...") before the handler runs. MCP `_meta` travels outside +/// `arguments`, so no legitimate key is lost. +/// /// `skip_serializing_if` on the patch fields is for the SCHEMA, not the wire: /// it stops schemars from advertising `"default": null`. A client that fills /// in schema defaults would otherwise send explicit nulls — and null means /// "clear" here, so every patch would wipe tags/scope/ttl. #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateArgs { /// The numeric id of the memory to update. pub id: i64, @@ -379,7 +389,7 @@ impl MemoryMcp { } #[tool( - description = "Update a memory in place by id (JSON merge-patch): omit a field to keep it, send null to clear it (tags -> [], scope -> global, ttl_secs -> no expiry), send a value to set it. content can be replaced but never cleared; a new content is re-indexed and re-embedded. Returns the full updated memory including embedding_status (0 = re-embed pending), or a clean not-found result for an unknown id." + description = "Update a memory in place by id (JSON merge-patch): omit a field to keep it, send null to clear it (tags -> [], scope -> global, ttl_secs -> no expiry), send a value to set it. content can be replaced but never cleared; a new content is re-indexed and re-embedded. Unknown or immutable fields (such as type and source, or a misspelled field) are rejected with an error naming the field and nothing is written. Returns the full updated memory including embedding_status (0 = re-embed pending), or a clean not-found result for an unknown id." )] async fn memory_update( &self, @@ -828,6 +838,30 @@ mod tests { "{field} must not advertise a default: {property}" ); } + // deny_unknown_fields is advertised, so a schema-driven client knows + // that type/source and misspelled keys are refused (REVIEW WR-02). + assert_eq!( + schema["additionalProperties"], + serde_json::json!(false), + "unknown fields are refused in the tool schema: {schema}" + ); + } + + #[test] + fn update_args_reject_unknown_and_immutable_fields() { + for (raw, field) in [ + (r#"{"id": 1, "content": "x", "type": "DECISION"}"#, "type"), + (r#"{"id": 1, "source": "y"}"#, "source"), + (r#"{"id": 1, "ttl_sec": 60}"#, "ttl_sec"), + ] { + let err = serde_json::from_str::(raw) + .expect_err("an unknown or immutable field is refused"); + let message = err.to_string(); + assert!( + message.contains("unknown field") && message.contains(field), + "{raw}: got {message:?}" + ); + } } async fn link_raw(server: &MemoryMcp, raw: &str) -> Result { diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index 24165f3..acc9cc6 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -98,7 +98,13 @@ pub struct ListQuery { /// `ttl_secs` → no expiry), a value sets it. `content` is never clearable. /// Uses the SAME explicit-null helper as the MCP DTO, so the two transports /// cannot drift on null-vs-absent. +/// +/// Unknown keys are rejected (`deny_unknown_fields`, REVIEW WR-02), exactly as +/// on the MCP DTO: the immutable `type`/`source` and typos such as `ttl_sec` +/// make the whole PATCH a 400 naming the field, and nothing is written. A +/// silently dropped merge-patch member would be a silent partial write. #[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] pub struct UpdateBody { /// New content (re-indexed and re-embedded). Never null or empty. #[serde( @@ -1249,6 +1255,51 @@ mod tests { assert_eq!(rows[0].tags, vec!["rest".to_string()]); } + #[tokio::test] + async fn patch_with_an_unknown_or_immutable_field_is_400_and_writes_nothing() { + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let id = store_with_ttl(&state, None).await; + + for (raw, field) in [ + (r#"{"content": "x", "type": "DECISION"}"#, "type"), + (r#"{"content": "x", "source": "elsewhere"}"#, "source"), + (r#"{"ttl_sec": 60}"#, "ttl_sec"), + ] { + let resp = patch_raw(&state, id, raw).await; + assert_eq!( + resp.status(), + StatusCode::BAD_REQUEST, + "{raw} must be a 400" + ); + let body = body_json(resp).await; + let message = body["error"] + .as_str() + .expect("400 body has an error string"); + assert!( + message.contains("unknown field") && message.contains(field), + "{raw}: error {message:?} should name the unknown field" + ); + } + assert_400_containing( + patch_raw(&state, id, r#"{"content": "x", "type": "DECISION"}"#).await, + "unknown field", + ) + .await; + + // Nothing was written: content, type, tags and scope are as stored. + let rows = state + .service + .list(ServiceListArgs::default()) + .await + .expect("list"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].content, "patch target"); + assert_eq!(rows[0].mem_type, MemoryType::Pattern); + assert_eq!(rows[0].tags, vec!["rest".to_string()]); + assert_eq!(rows[0].scope.as_deref(), Some("unit")); + } + // --- MCP-07 link / unlink ------------------------------------------------- async fn link_request( diff --git a/crates/agent-memory/tests/tools.rs b/crates/agent-memory/tests/tools.rs index 7e407d4..aa4ea55 100644 --- a/crates/agent-memory/tests/tools.rs +++ b/crates/agent-memory/tests/tools.rs @@ -413,3 +413,69 @@ fn memory_forget_deletes_then_search_and_list_omit_it_unknown_id_is_clean_not_fo server.shutdown(); } + +#[test] +fn memory_update_with_an_unknown_field_is_invalid_params_over_stdio() { + let dir = tempfile::tempdir().expect("tempdir"); + let db_path = dir.path().join("memory.db"); + + let mut server = Server::spawn(&db_path); + server.initialize(); + + let store_resp = server.call_tool( + 2, + "memory_store", + serde_json::json!({ "content": "original text", "type": "PATTERN" }), + ); + let id: i64 = tool_text(&store_resp) + .trim() + .parse() + .expect("memory_store returns a numeric id"); + + // rmcp deserializes `arguments` straight into UpdateArgs, so an unknown + // or immutable key is refused before the handler runs (REVIEW WR-02), + // never a success with the member silently dropped. rmcp 1.8 reports an + // argument-deserialization failure (its invalid_params error prefixed + // "failed to deserialize parameters:") as a tool error result + // (`isError: true`) rather than a JSON-RPC error member, per the MCP rule + // that input-validation errors are tool execution errors the model can + // correct (rmcp router/tool.rs `into_tool_argument_error`). + for (request_id, arguments, field) in [ + ( + 3, + serde_json::json!({ "id": id, "content": "changed", "type": "DECISION" }), + "type", + ), + ( + 4, + serde_json::json!({ "id": id, "content": "changed", "ttl_sec": 60 }), + "ttl_sec", + ), + ] { + let resp = server.call_tool(request_id, "memory_update", arguments); + assert!( + resp.get("error").is_none(), + "rmcp reports a bad argument as a tool error result: {resp}" + ); + assert_eq!( + resp.pointer("/result/isError"), + Some(&serde_json::json!(true)), + "an unknown field must be a tool error, never a success: {resp}" + ); + let message = tool_text(&resp); + assert!( + message.starts_with("failed to deserialize parameters:") + && message.contains("unknown field") + && message.contains(&format!("`{field}`")), + "the error names the field {field:?}: {resp}" + ); + } + + // Nothing was written: the memory still carries its original content and type. + let views = tool_views(&server.call_tool(5, "memory_list", serde_json::json!({}))); + assert_eq!(views.len(), 1, "exactly one memory: {views:?}"); + assert_eq!(views[0]["content"], "original text"); + assert_eq!(views[0]["type"], "PATTERN"); + + server.shutdown(); +} From cb24e84675f7fea3da24226830283ed2fec29024 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 22:06:52 +0200 Subject: [PATCH 31/42] fix(04-07): malformed REST path and query params get the JSON error body (IN-02) - forget/update/unlink take Result, list takes Result, mapped through extractor_rejection - status tier preserved: 4xx parse failures are 400 {"error"}, a 5xx routing misconfiguration stays a 500 - real-HTTP test: 5 malformed requests are 400 application/json with an error - README: every REST client error carries {"error": ...} Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- crates/agent-memory/src/rest/handlers.rs | 145 ++++++++++++++++++++--- 2 files changed, 129 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index efeef28..9a6bfef 100644 --- a/README.md +++ b/README.md @@ -393,7 +393,9 @@ never a silent clamp. The MCP transport rejects the same inputs identically as `invalid_params`, because validation lives once at the shared service seam. A `PATCH` body that is not valid JSON, has a wrongly typed field, or is sent without `Content-Type: application/json` is also a 400 with the same -`{"error": ...}` body. A storage failure during search reports a generic +`{"error": ...}` body. Every client error, including a malformed path segment +(`/api/memories/abc`) or query parameter (`?limit=abc`, `?expand_links=1`), +returns a JSON `{"error": "..."}` body. A storage failure during search reports a generic internal error (HTTP 500 / MCP `internal_error`, `search failed due to a storage error`); the detail goes only to the server's stderr log. diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index acc9cc6..3b19e53 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -10,7 +10,7 @@ use std::sync::Arc; -use axum::extract::rejection::JsonRejection; +use axum::extract::rejection::{JsonRejection, PathRejection, QueryRejection}; use axum::extract::{Path, Query, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; @@ -211,11 +211,29 @@ pub async fn store_handler( Ok((StatusCode::CREATED, Json(serde_json::json!({ "id": id }))).into_response()) } +/// Map an axum `Path`/`Query` extractor rejection to [`ApiError`], so a +/// malformed path segment or query parameter gets the same `{"error": ...}` +/// body as every other client error instead of axum's plain-text default +/// (REVIEW IN-02). The extractor's status tier is preserved: a parse failure +/// (4xx) is a 400, while a 5xx — an axum routing misconfiguration such as a +/// path-parameter count mismatch — stays a 500. +fn extractor_rejection(status: StatusCode, message: String) -> ApiError { + if status.is_server_error() { + ApiError::Internal(message) + } else { + ApiError::BadRequest(message) + } +} + /// `GET /api/memories?type=&tag=&scope=&limit=&expand_links=` → 200 `Vec`. +/// +/// A malformed query string (`limit=abc`, `expand_links=1`) is a 400 with the +/// `{"error"}` body via [`extractor_rejection`]. pub async fn list_handler( State(state): State>, - Query(q): Query, + query: Result, QueryRejection>, ) -> Result { + let Query(q) = query.map_err(|r| extractor_rejection(r.status(), r.body_text()))?; let mem_type = parse_type_filter(q.r#type.as_deref())?; let views = state @@ -260,11 +278,13 @@ pub async fn search_handler( /// `DELETE /api/memories/{id}` → 200 `{"id", "deleted": true}` or 404 /// `{"id", "deleted": false, "reason": "not_found"}` — the exact JSON bodies -/// the `mcp.rs` forget tool produces, with HTTP-native status codes. +/// the `mcp.rs` forget tool produces, with HTTP-native status codes. A +/// non-numeric id segment is a 400 `{"error"}` via [`extractor_rejection`]. pub async fn forget_handler( State(state): State>, - Path(id): Path, + path: Result, PathRejection>, ) -> Result { + let Path(id) = path.map_err(|r| extractor_rejection(r.status(), r.body_text()))?; let deleted = state.service.forget(id).await.map_err(map_memory_error)?; if deleted { @@ -285,16 +305,18 @@ pub async fn forget_handler( /// /// The body extractor's rejection is taken explicitly and reported as a 400 /// with the usual `{"error": ...}` body: a malformed or wrongly-typed patch is -/// client input, whatever axum's default status for it would be. Invalid +/// client input, whatever axum's default status for it would be. A +/// non-numeric id segment gets the same body via [`extractor_rejection`]. Invalid /// patch VALUES (null/empty content, out-of-bounds TTL, an empty patch) are /// rejected by the service seam and map to 400 through [`map_memory_error`]. /// /// [`UpdatedMemory`]: agent_memory_core::domain::UpdatedMemory pub async fn update_handler( State(state): State>, - Path(id): Path, + path: Result, PathRejection>, body: Result, JsonRejection>, ) -> Result { + let Path(id) = path.map_err(|r| extractor_rejection(r.status(), r.body_text()))?; let Json(body) = body.map_err(|rejection| ApiError::BadRequest(rejection.body_text()))?; let updated = state @@ -366,11 +388,14 @@ pub async fn link_handler( /// `DELETE /api/links/{from_id}/{to_id}/{kind}` → 200 `{"from_id", "to_id", /// "kind", "unlinked": true}`, or 404 `{..., "unlinked": false, "reason": /// "not_found"}` when no such edge exists — the `mcp.rs` unlink tool's JSON. -/// An unknown `kind` segment is a 400. +/// An unknown `kind` segment is a 400; a non-numeric id segment is a 400 +/// `{"error"}` via [`extractor_rejection`]. pub async fn unlink_handler( State(state): State>, - Path((from_id, to_id, kind)): Path<(i64, i64, String)>, + path: Result, PathRejection>, ) -> Result { + let Path((from_id, to_id, kind)) = + path.map_err(|r| extractor_rejection(r.status(), r.body_text()))?; let kind = parse_link_kind(&kind)?; let removed = state .service @@ -551,13 +576,13 @@ mod tests { let resp = list_handler( State(state.clone()), - Query(ListQuery { + Ok(Query(ListQuery { r#type: Some("TODO".to_string()), tag: None, scope: None, limit: None, expand_links: false, - }), + })), ) .await .expect("list succeeds") @@ -570,13 +595,13 @@ mod tests { let err = list_handler( State(state), - Query(ListQuery { + Ok(Query(ListQuery { r#type: Some("bogus".to_string()), tag: None, scope: None, limit: None, expand_links: false, - }), + })), ) .await .expect_err("bad type filter is rejected"); @@ -925,13 +950,13 @@ mod tests { let err = list_handler( State(state), - Query(ListQuery { + Ok(Query(ListQuery { r#type: None, tag: None, scope: None, limit: Some(-5), expand_links: false, - }), + })), ) .await .expect_err("limit=-5 is rejected"); @@ -959,7 +984,7 @@ mod tests { .into_response(); let id = body_json(created).await["id"].as_i64().expect("id"); - let resp = forget_handler(State(state.clone()), Path(id)) + let resp = forget_handler(State(state.clone()), Ok(Path(id))) .await .expect("forget succeeds") .into_response(); @@ -967,7 +992,7 @@ mod tests { assert_eq!(body_json(resp).await["deleted"], true); // Second delete of the same id: 404 with the exact mcp.rs not-found body. - let resp = forget_handler(State(state), Path(id)) + let resp = forget_handler(State(state), Ok(Path(id))) .await .expect("forget of a missing id is a clean handler result") .into_response(); @@ -1023,7 +1048,7 @@ mod tests { .body(axum::body::Body::from(raw.to_string())) .expect("request builds"); let body = Json::::from_request(req, &()).await; - match update_handler(State(state.clone()), Path(id), body).await { + match update_handler(State(state.clone()), Ok(Path(id)), body).await { Ok(resp) => resp, Err(err) => err.into_response(), } @@ -1328,7 +1353,7 @@ mod tests { } async fn unlink_path(state: &Arc, from: i64, to: i64, kind: &str) -> Response { - match unlink_handler(State(state.clone()), Path((from, to, kind.to_string()))).await { + match unlink_handler(State(state.clone()), Ok(Path((from, to, kind.to_string())))).await { Ok(resp) => resp, Err(err) => err.into_response(), } @@ -1506,7 +1531,7 @@ mod tests { .parse() .expect("uri parses"); let query = Query::::try_from_uri(&uri).expect("query parses"); - list_handler(State(state), query) + list_handler(State(state), Ok(query)) .await .expect("list succeeds") .into_response() @@ -1643,6 +1668,88 @@ mod tests { server.abort(); } + #[tokio::test] + async fn malformed_path_and_query_params_get_the_json_error_body() { + // REVIEW IN-02: path and query extractor rejections used to be axum's + // plain-text default; every client error now carries {"error"}. + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + let a = stored_id(&state, "target").await; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback"); + let addr = listener.local_addr().expect("local addr"); + let server = tokio::spawn(async move { + axum::serve(listener, crate::rest::build_router(state)) + .await + .expect("serve"); + }); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .build() + .expect("client"); + let base = format!("http://{addr}"); + + let requests = [ + ( + "PATCH /api/memories/abc", + client + .patch(format!("{base}/api/memories/abc")) + .json(&serde_json::json!({ "content": "valid body" })), + ), + ( + "DELETE /api/memories/abc", + client.delete(format!("{base}/api/memories/abc")), + ), + ( + "DELETE /api/links/x/2/relates_to", + client.delete(format!("{base}/api/links/x/{a}/relates_to")), + ), + ( + "GET /api/memories?expand_links=1", + client.get(format!("{base}/api/memories?expand_links=1")), + ), + ( + "GET /api/memories?limit=abc", + client.get(format!("{base}/api/memories?limit=abc")), + ), + ]; + for (label, request) in requests { + let resp = request.send().await.expect(label); + assert_eq!(resp.status().as_u16(), 400, "{label} is a client error"); + let content_type = resp + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_string(); + assert!( + content_type.starts_with("application/json"), + "{label}: JSON body, got content-type {content_type:?}" + ); + let body: serde_json::Value = resp.json().await.expect("JSON error body"); + let message = body["error"].as_str().unwrap_or_default(); + assert!( + !message.is_empty(), + "{label}: non-empty error string, got {body}" + ); + } + + // The malformed PATCH wrote nothing. + let rows: serde_json::Value = client + .get(format!("{base}/api/memories")) + .send() + .await + .expect("GET /api/memories") + .json() + .await + .expect("json rows"); + assert_eq!(rows[0]["content"], "target"); + + server.abort(); + } + #[test] fn map_memory_error_puts_internal_faults_in_the_500_tier() { // WR-02: a poisoned writer lock is a server fault — 500, never 404. From a9143115af9c814888bd5b2ea61266e7c5162ae4 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 22:10:29 +0200 Subject: [PATCH 32/42] fix(04-07): race-safe pre-migration backup and concurrent-upgrade-tolerant open (WR-01, IN-03, IN-01) - busy_timeout set first on the writer (before the D-13 probe) and in apply_pragmas - pre_migration_backup(src, path, expected_version) -> Option: copy to a pid-qualified temp outside the retention prefix, verify the copy's user_version, rename into place, only then prune older same-version backups; a copy taken after a concurrent migration is discarded, never kept as pre-migration - migrate_tolerating_concurrent_upgrade absorbs a to_latest failure only when a re-probe shows LATEST_SCHEMA_VERSION - seam tests (discard, replace-then-prune, failed temp write, failed rename, closure seam, real lost-race errors) and a 10-round two-thread first-open smoke test on a temp copy of the v0.0.1 fixture - README: Upgrading section (backup name, retention, restore, newer-schema refusal) Co-Authored-By: Claude Opus 5.5 --- README.md | 40 ++ crates/agent-memory-core/src/store/sqlite.rs | 503 ++++++++++++++++-- .../tests/migration_hygiene.rs | 82 +++ 3 files changed, 576 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 9a6bfef..97f10c8 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,46 @@ AGENT_MEMORY_DB=/path/to/memory.db agent-memory serve The server speaks JSON-RPC over **stdout**; all logs go to **stderr**, so stdout stays a pure protocol channel. +## Upgrading + +A new agent-memory version may need to migrate the database schema. It does +this automatically the first time it opens the file, and backs the database up +first. + +- **Pre-migration backup.** Before any schema migration, the database is copied + with SQLite's online backup to `memory.db.backup-pre-NNNN-YYYYMMDD`, next to + `memory.db`. `NNNN` is the schema version being migrated to and `YYYYMMDD` is + the UTC date. A brand-new database has nothing to back up. +- **One backup per target schema version.** Re-running the same upgrade + replaces that version's backup atomically. The new copy is written to a + temporary file, renamed into place, and only then is the older copy removed. + Backups made for other schema versions are never touched. +- **No backup, no migration.** If the backup cannot be written (for example a + full disk or a read-only directory), the server refuses to start and the + database is not migrated. +- **Starting several servers at once is safe.** If an MCP server and the REST + daemon open an older database at the same moment, both start. The schema is + migrated once, and the surviving backup is always a genuine pre-migration + copy. + +**Restoring a backup:** + +1. Stop every agent-memory process that uses the database: each MCP server and + the REST daemon. +2. Copy the backup over the database: + `cp memory.db.backup-pre-0003-20261023 memory.db`. +3. Delete `memory.db-wal` and `memory.db-shm` if they exist. They belong to the + database you replaced. +4. Start the older agent-memory binary that matches the backup's schema. + +**A newer database meets an older binary.** A database written by a newer +agent-memory is refused at startup, and the file is left untouched. The error +names both schema versions: + +```text +database schema is version 4 but this binary supports up to version 3 — upgrade agent-memory (brew upgrade unityinflow/tap/agent-memory) or point --db at a different file +``` + ## MCP configuration (`.mcp.json`) Point every runtime at **one** database via `AGENT_MEMORY_DB` and they share memory: diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index fb410e9..7047c06 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -81,11 +81,16 @@ pub fn register_vec_extension() -> Result<(), rusqlite::Error> { /// Apply the per-connection PRAGMAs that every connection (writer + pool) needs. fn apply_pragmas(conn: &Connection) -> Result<(), rusqlite::Error> { + // busy_timeout FIRST (REVIEW WR-01): several agent-memory processes open + // one DB at once by design (an MCP server per editor plus the REST + // daemon), and `journal_mode` and every later pragma can contend for a + // lock. With the timeout already set they wait out a concurrent opener + // instead of failing startup with SQLITE_BUSY. + conn.pragma_update(None, "busy_timeout", 5000)?; // journal_mode returns a row; the others do not. Use pragma_update where we can. conn.pragma_update(None, "journal_mode", "WAL")?; conn.pragma_update(None, "synchronous", "NORMAL")?; conn.pragma_update(None, "foreign_keys", "ON")?; - conn.pragma_update(None, "busy_timeout", 5000)?; Ok(()) } @@ -123,21 +128,115 @@ fn utc_yyyymmdd() -> String { format!("{:04}{:02}{:02}", today.year(), today.month(), today.day()) } -/// Write the pre-migration backup of `src` (the open DB at `db_path`) to the -/// sibling file `{db}.backup-pre-{NNNN}-{YYYYMMDD}`, where `NNNN` is the -/// zero-padded schema version about to be migrated to (D-11). Generic over the -/// target version, so every future migration inherits it (D-12). +/// Serializes [`pre_migration_backup`] within one process. The temp file is +/// pid-qualified, which keeps separate processes apart; two openers in the +/// SAME process (threads) would otherwise share one temp path. +static BACKUP_LOCK: Mutex<()> = Mutex::new(()); + +/// The in-flight temp file for this process's backup of `file_name`: +/// `{db}.backup-tmp-{NNNN}-{pid}`. Deliberately OUTSIDE the +/// `{db}.backup-pre-{NNNN}-` retention prefix, so no opener ever prunes +/// another's in-flight copy. +fn backup_temp_path(dir: &Path, file_name: &str) -> PathBuf { + dir.join(format!( + "{file_name}.backup-tmp-{LATEST_SCHEMA_VERSION:04}-{}", + std::process::id() + )) +} + +/// Best-effort removal of a backup temp file and any SQLite sidecar it left. +/// Never fails: a leftover temp is harmless (T-04-32) and must not mask the +/// error that led here. +fn discard_backup_temp(temp: &Path) { + for suffix in ["", "-journal", "-wal", "-shm"] { + let path = PathBuf::from(format!("{}{suffix}", temp.display())); + if let Err(e) = std::fs::remove_file(&path) { + if e.kind() != std::io::ErrorKind::NotFound { + tracing::warn!(temp = %path.display(), error = %e, "could not remove backup temp file"); + } + } + } +} + +/// Copy `src` into a NEW database at `temp` with the SQLite Online Backup API +/// and return the copy's `user_version`. The destination connection is +/// closed before returning, so the file can be renamed. +fn copy_into_temp(src: &Connection, temp: &Path) -> Result { + let mut dst = + Connection::open(temp).map_err(|e| format!("cannot create {}: {e}", temp.display()))?; + Backup::new(src, &mut dst) + .and_then(|backup| backup.run_to_completion(64, Duration::from_millis(50), None)) + .map_err(|e| format!("copy to {} failed: {e}", temp.display()))?; + let copied_version: i64 = dst + .pragma_query_value(None, "user_version", |row| row.get(0)) + .map_err(|e| format!("cannot read the schema version of {}: {e}", temp.display()))?; + dst.close() + .map_err(|(_, e)| format!("cannot close {}: {e}", temp.display()))?; + Ok(copied_version) +} + +/// Remove every `{prefix}*` entry in `dir` except `keep`. An entry that is +/// already gone (a concurrent opener pruned it first) is not an error. +fn prune_other_backups(dir: &Path, prefix: &str, keep: &Path) -> Result<(), String> { + let entries = std::fs::read_dir(dir) + .map_err(|e| format!("cannot read directory {}: {e}", dir.display()))?; + for entry in entries { + let entry = entry.map_err(|e| format!("cannot read directory entry: {e}"))?; + let matches = entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(prefix)); + let stale = entry.path(); + if !matches || stale == keep { + continue; + } + match std::fs::remove_file(&stale) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + return Err(format!( + "cannot remove previous backup {}: {e}", + stale.display() + )) + } + } + } + Ok(()) +} + +/// Write the pre-migration backup of `src` (the open DB at `db_path`, probed +/// at schema `expected_version`) to the sibling file +/// `{db}.backup-pre-{NNNN}-{YYYYMMDD}`, where `NNNN` is the zero-padded schema +/// version about to be migrated to (D-11). Generic over the target version, +/// so every future migration inherits it (D-12). +/// +/// Race-safe and replace-then-prune (REVIEW WR-01, IN-03): +/// +/// 1. The copy goes to a temp file outside the retention prefix +/// ([`backup_temp_path`]), via the SQLite Online Backup API, which is +/// WAL-correct by construction (a raw file copy of a live WAL database is +/// not). +/// 2. The copy's `user_version` must still equal `expected_version`. If it +/// does not, another process migrated the DB between our probe and our +/// copy: the copy is a POST-migration database, so it is discarded and +/// `Ok(None)` is returned. Existing backups are never touched. +/// 3. The temp file is renamed into place, an atomic replace of a same-day +/// stale backup. Only THEN are the other `{db}.backup-pre-{NNNN}-*` +/// entries pruned, so a failed write never costs the previous valid +/// backup. Other versions' backups are never touched. /// -/// Retention is one backup per target schema version: every existing -/// `{db}.backup-pre-{NNNN}-*` entry is removed first; other versions' backups -/// are never touched. The copy uses the SQLite Online Backup API, which is -/// WAL-correct by construction (a raw file copy of a live WAL database is not). /// The backup sits beside the DB, in the same directory and trust domain — /// never a shared temp dir (T-04-03). /// -/// Any failure is returned as an error and ABORTS the open: a machine that -/// cannot write a backup must not run an unattended migration (T-04-02). -fn pre_migration_backup(src: &Connection, db_path: &Path) -> Result { +/// Any failure to copy, rename or prune is returned as an error and ABORTS +/// the open: a machine that cannot write a backup must not run an unattended +/// migration (T-04-02). The temp file is removed (best-effort) on every +/// failure path. +fn pre_migration_backup( + src: &Connection, + db_path: &Path, + expected_version: i64, +) -> Result, MemoryError> { let fail = |what: String| { MemoryError::Internal(format!( "pre-migration backup failed ({what}); refusing to migrate {}", @@ -154,38 +253,84 @@ fn pre_migration_backup(src: &Connection, db_path: &Path) -> Result PathBuf::from("."), }; let prefix = format!("{file_name}.backup-pre-{LATEST_SCHEMA_VERSION:04}-"); - - let entries = std::fs::read_dir(&dir) - .map_err(|e| fail(format!("cannot read directory {}: {e}", dir.display())))?; - for entry in entries { - let entry = entry.map_err(|e| fail(format!("cannot read directory entry: {e}")))?; - let matches = entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with(&prefix)); - if matches { - let stale = entry.path(); - std::fs::remove_file(&stale).map_err(|e| { - fail(format!( - "cannot remove previous backup {}: {e}", - stale.display() - )) - })?; + let temp_path = backup_temp_path(&dir, file_name); + let final_path = dir.join(format!("{prefix}{}", utc_yyyymmdd())); + + // A poisoned lock only means another thread panicked mid-backup; the + // guard protects no data, so carry on. + let _serialized = BACKUP_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + let copied_version = match copy_into_temp(src, &temp_path) { + Ok(version) => version, + Err(what) => { + discard_backup_temp(&temp_path); + return Err(fail(what)); } + }; + + if copied_version != expected_version { + discard_backup_temp(&temp_path); + tracing::info!( + db = %db_path.display(), + expected_version, + copied_version, + "another process migrated the database first; discarding the post-migration copy" + ); + return Ok(None); } - let backup_path = dir.join(format!("{prefix}{}", utc_yyyymmdd())); - let mut dst = Connection::open(&backup_path) - .map_err(|e| fail(format!("cannot create {}: {e}", backup_path.display())))?; - Backup::new(src, &mut dst) - .and_then(|backup| backup.run_to_completion(64, Duration::from_millis(50), None)) - .map_err(|e| fail(format!("copy to {} failed: {e}", backup_path.display())))?; + if let Err(e) = std::fs::rename(&temp_path, &final_path) { + discard_backup_temp(&temp_path); + return Err(fail(format!( + "cannot move {} into place as {}: {e}", + temp_path.display(), + final_path.display() + ))); + } + + prune_other_backups(&dir, &prefix, &final_path).map_err(fail)?; + tracing::info!( - backup = %backup_path.display(), + backup = %final_path.display(), target_schema_version = LATEST_SCHEMA_VERSION, "pre-migration backup written" ); - Ok(backup_path) + Ok(Some(final_path)) +} + +/// Run `migrate` and absorb exactly one failure: another process completing +/// the same upgrade first (REVIEW WR-01). +/// +/// rusqlite_migration reads `user_version` OUTSIDE its own deferred +/// transaction, so a second opener that probed the old version can attempt +/// 0003's `CREATE TABLE` after the first one committed it, and fail with +/// `table memory_links already exists` or `SQLITE_BUSY_SNAPSHOT`. Its +/// transaction rolls back. On any failure the version is re-read on the same +/// connection: if it is now [`LATEST_SCHEMA_VERSION`], the schema this binary +/// needs is in place, so the open proceeds. Anything else returns the +/// original error. The FTS5 and vec0 smoke checks in [`SqliteStore::open`] +/// still run afterwards, so a half-built schema cannot slip through. +fn migrate_tolerating_concurrent_upgrade( + conn: &mut Connection, + migrate: impl FnOnce(&mut Connection) -> Result<(), MemoryError>, +) -> Result<(), MemoryError> { + let err = match migrate(conn) { + Ok(()) => return Ok(()), + Err(err) => err, + }; + match conn.pragma_query_value(None, "user_version", |row| row.get::<_, i64>(0)) { + Ok(version) if version == LATEST_SCHEMA_VERSION => { + tracing::info!( + schema_version = version, + error = %err, + "another process completed the schema upgrade first" + ); + Ok(()) + } + Ok(_) | Err(_) => Err(err), + } } /// A pool customizer that runs [`apply_pragmas`] on each pooled read connection. @@ -211,16 +356,24 @@ impl SqliteStore { /// Migration hygiene (STORE-05), in this order — the probe and the backup /// MUST precede `to_latest` (RESEARCH Pitfall 8): /// - /// 1. Probe `PRAGMA user_version`. A DB AHEAD of [`LATEST_SCHEMA_VERSION`] + /// 1. Register vec0, open the writer, and set its `busy_timeout`, so every + /// later step waits out a concurrent opener (REVIEW WR-01). The timeout + /// is connection-level state; it writes nothing to the file. + /// 2. Probe `PRAGMA user_version`. A DB AHEAD of [`LATEST_SCHEMA_VERSION`] /// is refused with [`MemoryError::SchemaTooNew`] (D-13). The probe runs - /// before even the per-connection PRAGMAs, because switching a non-WAL + /// before the other per-connection PRAGMAs, because switching a non-WAL /// file to WAL rewrites its header — a refused DB stays byte-for-byte /// untouched. - /// 2. If migrations are pending on a non-empty DB (`0 < user_version < - /// LATEST`, D-12), write a sibling online backup first (D-11). A backup - /// failure aborts the open; no migration runs. A brand-new file + /// 3. Apply the per-connection PRAGMAs and functions. + /// 4. If migrations are pending on a non-empty DB (`0 < user_version < + /// LATEST`, D-12), write a sibling online backup first (D-11), checked + /// against the probed version so a copy taken after a concurrent + /// migration is discarded, never kept under the pre-migration name. A + /// backup failure aborts the open; no migration runs. A brand-new file /// (`user_version == 0`) has nothing to back up. - /// 3. Migrate to latest. + /// 5. Migrate to latest, tolerating a concurrent process that completed + /// the same upgrade first ([`migrate_tolerating_concurrent_upgrade`]). + /// 6. Smoke-check FTS5 and vec0, then build the read pool. pub fn open(path: &Path) -> Result { // vec0 registration MUST precede Connection::open: auto-extensions only // apply to connections opened after registration (this also covers every @@ -228,8 +381,12 @@ impl SqliteStore { register_vec_extension()?; let mut writer = Connection::open(path)?; + // Connection-level state, set before anything that can contend for the + // DB (WR-01). It touches nothing on disk, so the D-13 probe below still + // leaves a refused file byte-for-byte untouched. + writer.busy_timeout(Duration::from_millis(5000))?; - // (1) D-13 newer-schema refusal — read-only, before any write. + // (2) D-13 newer-schema refusal — read-only, before any write. let db_version: i64 = writer.pragma_query_value(None, "user_version", |row| row.get(0))?; if db_version > LATEST_SCHEMA_VERSION { return Err(MemoryError::SchemaTooNew { @@ -238,18 +395,26 @@ impl SqliteStore { }); } + // (3) Per-connection PRAGMAs (busy_timeout first) and functions. prepare_connection(&writer)?; - // (2) D-11/D-12 pre-migration backup. open() runs before serving, so - // the writer is the only connection and can be the backup source. + // (4) D-11/D-12 pre-migration backup. open() runs before serving, so + // within this process the writer is the only connection and can be + // the backup source; another process may still be opening the same + // file, which the version check inside pre_migration_backup handles. if db_version > 0 && db_version < LATEST_SCHEMA_VERSION { - pre_migration_backup(&writer, path)?; + // Ok(None) = another process migrated in between, so our copy was + // post-migration and was discarded (logged inside); the genuine + // backup is the one that process wrote. Either way, migrate on. + pre_migration_backup(&writer, path, db_version)?; } - // (3) Migrate. rusqlite_migration's own DatabaseTooFarAhead check stays + // (5) Migrate. rusqlite_migration's own DatabaseTooFarAhead check stays // as a backstop should the probe above ever be bypassed. let migrations: Migrations<'static> = migrations(); - migrations.to_latest(&mut writer)?; + migrate_tolerating_concurrent_upgrade(&mut writer, |conn| { + migrations.to_latest(conn).map_err(MemoryError::from) + })?; // Smoke-check that FTS5 is available (bundled SQLite compiles it in). writer.query_row("SELECT count(*) FROM memories_fts", [], |row| { @@ -1100,6 +1265,246 @@ impl Store for SqliteStore { mod tests { use super::*; + // ----------------------------------------------------------------------- + // STORE-05 backup race and failure seams (REVIEW WR-01, IN-03). Real temp + // files; vec0 registered first because SqliteStore::open runs 0002. + // ----------------------------------------------------------------------- + + /// A raw database at schema version 2 with one table, standing in for a + /// v0.0.1 store whose upgrade is pending. + fn raw_v2_db(path: &Path) -> Connection { + register_vec_extension().expect("sqlite-vec registers"); + let conn = Connection::open(path).expect("create raw db"); + conn.execute_batch("CREATE TABLE pre_upgrade (x INTEGER); PRAGMA user_version = 2;") + .expect("build v2 db"); + conn + } + + fn backup_prefix(db: &Path) -> String { + let name = db.file_name().and_then(|n| n.to_str()).expect("db name"); + format!("{name}.backup-pre-{LATEST_SCHEMA_VERSION:04}-") + } + + fn dir_names(dir: &Path) -> Vec { + std::fs::read_dir(dir) + .expect("read dir") + .map(|e| { + e.expect("dir entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect() + } + + fn assert_no_backup_temp(dir: &Path) { + let names = dir_names(dir); + assert!( + !names.iter().any(|n| n.contains(".backup-tmp-")), + "no backup temp file may remain: {names:?}" + ); + } + + #[test] + fn a_backup_of_an_already_migrated_database_is_discarded_and_keeps_the_valid_one() { + register_vec_extension().expect("sqlite-vec registers"); + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("memory.db"); + // Another process already migrated: the DB is at LATEST. + drop(SqliteStore::open(&db).expect("create a latest-schema db")); + let valid = dir.path().join(format!("{}20000101", backup_prefix(&db))); + std::fs::write(&valid, b"valid").expect("plant the genuine backup"); + let conn = Connection::open(&db).expect("open raw"); + + // We probed 2 before the other process migrated. + let outcome = pre_migration_backup(&conn, &db, 2).expect("not an error"); + + assert!( + outcome.is_none(), + "a post-migration copy is never kept: {outcome:?}" + ); + assert_eq!( + std::fs::read(&valid).expect("the genuine backup survives"), + b"valid", + "an existing backup is never touched by a discarded copy" + ); + assert_no_backup_temp(dir.path()); + } + + #[test] + fn a_valid_backup_replaces_then_prunes_the_stale_one() { + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("memory.db"); + let conn = raw_v2_db(&db); + let stale = dir.path().join(format!("{}20000101", backup_prefix(&db))); + std::fs::write(&stale, b"stale").expect("plant stale backup"); + + let backup = pre_migration_backup(&conn, &db, 2) + .expect("backup succeeds") + .expect("a genuine pre-migration copy is kept"); + + let name = backup.file_name().and_then(|n| n.to_str()).expect("name"); + assert_eq!(name, format!("{}{}", backup_prefix(&db), utc_yyyymmdd())); + let copy = Connection::open(&backup).expect("backup opens"); + let version: i64 = copy + .pragma_query_value(None, "user_version", |row| row.get(0)) + .expect("read copy version"); + assert_eq!(version, 2, "the backup carries the pre-migration schema"); + assert!(!stale.exists(), "the stale same-version backup is pruned"); + assert_no_backup_temp(dir.path()); + } + + #[test] + fn a_failed_backup_write_never_removes_the_previous_backup() { + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("memory.db"); + let conn = raw_v2_db(&db); + let stale = dir.path().join(format!("{}20000101", backup_prefix(&db))); + std::fs::write(&stale, b"previous").expect("plant previous backup"); + // A non-empty directory occupying this process's temp path: the copy + // cannot even be created. + let blocker = backup_temp_path(dir.path(), "memory.db"); + std::fs::create_dir(&blocker).expect("create blocker dir"); + std::fs::write(blocker.join("keep"), b"x").expect("populate blocker dir"); + + let message = match pre_migration_backup(&conn, &db, 2) { + Ok(outcome) => panic!("the backup must fail, got {outcome:?}"), + Err(e) => e.to_string(), + }; + + assert!(message.contains("backup"), "names the backup: {message}"); + assert_eq!( + std::fs::read(&stale).expect("the previous backup survives"), + b"previous", + "IN-03: nothing is deleted before a valid replacement exists" + ); + } + + #[test] + fn a_backup_that_cannot_be_moved_into_place_keeps_the_previous_one() { + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("memory.db"); + let conn = raw_v2_db(&db); + let stale = dir.path().join(format!("{}20000101", backup_prefix(&db))); + std::fs::write(&stale, b"previous").expect("plant previous backup"); + // A non-empty directory at today's final name: the copy is written, + // but the rename into place fails. + let target = dir + .path() + .join(format!("{}{}", backup_prefix(&db), utc_yyyymmdd())); + std::fs::create_dir(&target).expect("create blocker dir"); + std::fs::write(target.join("keep"), b"x").expect("populate blocker dir"); + + let message = match pre_migration_backup(&conn, &db, 2) { + Ok(outcome) => panic!("the backup must fail, got {outcome:?}"), + Err(e) => e.to_string(), + }; + + assert!(message.contains("backup"), "names the backup: {message}"); + assert_eq!( + std::fs::read(&stale).expect("the previous backup survives"), + b"previous", + "IN-03: nothing is pruned before the new backup is in place" + ); + assert_no_backup_temp(dir.path()); + } + + fn set_user_version(conn: &Connection, version: i64) { + conn.pragma_update(None, "user_version", version) + .expect("set user_version"); + } + + #[test] + fn a_migration_failure_is_absorbed_only_when_another_process_reached_latest() { + let simulated = || { + Err::<(), _>(MemoryError::Internal( + "simulated: table memory_links already exists".to_string(), + )) + }; + let mut conn = Connection::open_in_memory().expect("in-memory db"); + + set_user_version(&conn, LATEST_SCHEMA_VERSION); + migrate_tolerating_concurrent_upgrade(&mut conn, |_| simulated()) + .expect("a concurrent completed upgrade is success"); + + set_user_version(&conn, 2); + let err = migrate_tolerating_concurrent_upgrade(&mut conn, |_| simulated()) + .expect_err("a migration that left the DB behind still fails"); + assert!( + err.to_string().contains("memory_links already exists"), + "the original error is returned: {err}" + ); + + migrate_tolerating_concurrent_upgrade(&mut conn, |_| Ok(())).expect("Ok stays Ok"); + } + + /// The two REAL ways a second opener loses the 0003 race, reproduced + /// deterministically with two connections on one WAL database: the loser + /// probed version 2, then the winner committed the whole upgrade before + /// the loser's 0003 statement ran (rusqlite_migration reads user_version + /// outside its transaction). Both are real SQLite errors, and both must be + /// absorbed because the re-probe sees the latest schema. + #[test] + fn a_real_lost_migration_race_is_absorbed() { + register_vec_extension().expect("sqlite-vec registers"); + let relations_sql = include_str!("../../sql/0003_relations.sql"); + + for stale_snapshot in [false, true] { + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("memory.db"); + { + let mut setup = Connection::open(&db).expect("create db"); + setup + .pragma_update(None, "journal_mode", "WAL") + .expect("WAL"); + migrations() + .to_version(&mut setup, 2) + .expect("schema 2 (v0.0.1)"); + } + let mut winner = Connection::open(&db).expect("winner"); + winner + .busy_timeout(Duration::from_millis(5000)) + .expect("winner busy_timeout"); + let mut loser = Connection::open(&db).expect("loser"); + loser + .busy_timeout(Duration::from_millis(5000)) + .expect("loser busy_timeout"); + + let mut observed = String::new(); + migrate_tolerating_concurrent_upgrade(&mut loser, |conn| { + let attempt = (|| -> Result<(), rusqlite::Error> { + let tx = conn.transaction()?; + if stale_snapshot { + // Read first: the loser's snapshot predates the winner's commit. + tx.query_row("SELECT count(*) FROM memories", [], |row| { + row.get::<_, i64>(0) + })?; + } + migrations() + .to_latest(&mut winner) + .expect("the other process wins the upgrade"); + tx.execute_batch(relations_sql)?; + tx.pragma_update(None, "user_version", LATEST_SCHEMA_VERSION)?; + tx.commit() + })(); + attempt.map_err(|e| { + observed = e.to_string(); + MemoryError::from(e) + }) + }) + .expect("a lost race against a completed upgrade is not a startup failure"); + + assert!( + !observed.is_empty(), + "stale_snapshot={stale_snapshot}: the loser really failed" + ); + let version: i64 = loser + .pragma_query_value(None, "user_version", |row| row.get(0)) + .expect("re-read version"); + assert_eq!(version, LATEST_SCHEMA_VERSION); + } + } + #[test] fn max_knn_k_keeps_four_x_oversample_at_max_limit() { // WR-01: the oversample must stay 4x even at the largest valid limit. diff --git a/crates/agent-memory-core/tests/migration_hygiene.rs b/crates/agent-memory-core/tests/migration_hygiene.rs index 4c02361..155a27d 100644 --- a/crates/agent-memory-core/tests/migration_hygiene.rs +++ b/crates/agent-memory-core/tests/migration_hygiene.rs @@ -98,6 +98,25 @@ fn backups_for_latest(db: &Path) -> Vec { out } +/// No in-flight backup temp file (`{db}.backup-tmp-*`) may be left beside the +/// DB after an open, successful or not (REVIEW WR-01 / IN-03). +fn assert_no_backup_temp(dir: &Path) { + let leftovers: Vec = fs::read_dir(dir) + .expect("read db dir") + .map(|e| { + e.expect("dir entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .filter(|n| n.contains(".backup-tmp-")) + .collect(); + assert!( + leftovers.is_empty(), + "no backup temp file may remain: {leftovers:?}" + ); +} + /// One-shot generator for the committed fixture. It applies ONLY migrations /// 0001+0002 (the schema v0.0.1 shipped — verified byte-identical to the /// `v0.0.1` tag before generation) and plants two rows. It is a no-op when the @@ -222,6 +241,7 @@ fn upgrading_writes_one_pre_migration_backup_with_the_old_schema() { // The live DB itself was migrated. assert_eq!(user_version(&db), LATEST_SCHEMA_VERSION); + assert_no_backup_temp(dir.path()); } #[test] @@ -266,6 +286,7 @@ fn re_running_the_upgrade_keeps_one_backup_per_schema_version() { b"other", "another schema version's backup is never touched" ); + assert_no_backup_temp(dir.path()); } #[test] @@ -355,3 +376,64 @@ fn a_failed_backup_aborts_the_open_before_migrating() { "no migration may run without a backup" ); } + +/// REVIEW WR-01 smoke test: an MCP server and the REST daemon starting at the +/// same moment after `brew upgrade`. Two openers race on a fresh copy of the +/// v0.0.1 fixture; both must start, the DB ends at the latest schema, and the +/// ONE surviving backup is a genuine pre-migration copy. The deterministic +/// proof lives in the seam tests in `store/sqlite.rs`; this exercises the real +/// race window, repeated so a real interleaving is likely to be hit. +#[test] +fn two_concurrent_first_opens_of_a_v0_0_1_database_both_succeed_with_one_genuine_backup() { + for round in 0..10 { + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("memory.db"); + copy_fixture(&db); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + + let openers: Vec<_> = (0..2) + .map(|_| { + let barrier = std::sync::Arc::clone(&barrier); + let db = db.clone(); + std::thread::spawn(move || { + barrier.wait(); + SqliteStore::open(&db).map(drop).map_err(|e| e.to_string()) + }) + }) + .collect(); + for (n, opener) in openers.into_iter().enumerate() { + let outcome = match opener.join() { + Ok(outcome) => outcome, + Err(payload) => { + let message = payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| (*s).to_string())) + .unwrap_or_else(|| "non-string panic payload".to_string()); + panic!("round {round}: opener {n} panicked: {message}"); + } + }; + if let Err(message) = outcome { + panic!("round {round}: opener {n} failed to open: {message}"); + } + } + + assert_eq!( + user_version(&db), + LATEST_SCHEMA_VERSION, + "round {round}: the DB was migrated" + ); + let backups = backups_for_latest(&db); + assert_eq!( + backups.len(), + 1, + "round {round}: exactly one backup, got {backups:?}" + ); + assert_eq!( + user_version(&backups[0]), + V0_0_1_SCHEMA_VERSION as i64, + "round {round}: the surviving backup is a genuine pre-migration copy" + ); + assert_no_backup_temp(dir.path()); + } +} From 9693c5fc8d881f1e3fce159415d5cd4ff2f8873b Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 22:12:35 +0200 Subject: [PATCH 33/42] docs(04-07): complete strict-payload, REST error-body and race-safe backup gap-closure plan (summary) Co-Authored-By: Claude Opus 5.5 --- .../04-07-SUMMARY.md | 331 ++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 .planning/phases/04-memory-update-relations-hybrid-search/04-07-SUMMARY.md diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-07-SUMMARY.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-07-SUMMARY.md new file mode 100644 index 0000000..45257ff --- /dev/null +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-07-SUMMARY.md @@ -0,0 +1,331 @@ +--- +phase: 04-memory-update-relations-hybrid-search +plan: 07 +subsystem: store +tags: [sqlite, migration, backup, concurrency, serde, axum, rmcp, rest, mcp, gap-closure] + +# Dependency graph +requires: + - phase: 04-memory-update-relations-hybrid-search (04-01) + provides: STORE-05 probe -> refuse -> backup -> migrate order, pre_migration_backup, v0.0.1 fixture + - phase: 04-memory-update-relations-hybrid-search (04-02) + provides: UpdateArgs / UpdateBody merge-patch DTOs and the "no default: null in schema" test + - phase: 04-memory-update-relations-hybrid-search (04-05, 04-06) + provides: the tree this last gap wave builds on +provides: + - deny_unknown_fields on mcp::UpdateArgs and rest::UpdateBody (only) + - extractor_rejection helper; Path/Query rejections on REST return the {"error"} body with the status tier preserved + - pre_migration_backup(src, path, expected_version) -> Result> (temp, verify, rename, prune) + - migrate_tolerating_concurrent_upgrade (re-probe absorbs a lost race against a completed upgrade) + - busy_timeout set first on the writer and in apply_pragmas + - README Upgrading section, memory_update unknown-field note, REST error-body sentence +affects: [phase-04 re-verification, phase-05, phase-06 release notes] + +actuals: + tokens: 10859 + tasks: 3 + commits: 4 + +tech-stack: + added: [] + patterns: + - "Verify-then-rename backups: copy to a temp outside the retention prefix, check the copy still has the probed version, rename into place, prune afterwards" + - "Tolerate-by-re-probe: a failed schema migration counts as success only when user_version already reads LATEST" + - "Result handler arguments mapped by status tier into the uniform ApiError body" + +key-files: + created: [] + modified: + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - crates/agent-memory/tests/tools.rs + - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory-core/tests/migration_hygiene.rs + - README.md + - .planning/STATE.md + +key-decisions: + - "04-07: deny_unknown_fields added to UpdateArgs/UpdateBody only (new v0.1.0 surfaces, REVIEW WR-02); the [03-01] decision stands for the v0.0.1 DTOs. Over MCP, rmcp 1.8 reports the rejection as a tool error result (isError: true, 'failed to deserialize parameters: unknown field ...'), not a JSON-RPC -32602; REST is a 400 {\"error\"}." + - "04-07: the backup temp file is pid-qualified ({db}.backup-tmp-NNNN-{pid}) and a process-wide BACKUP_LOCK serializes backups within one process, so two openers in the same process (threads) never share a temp path. Cross-process openers are kept apart by the pid." + - "04-07: WR-01 is closed with the review's cheaper alternative (verify-then-rename plus tolerate-by-re-probe), with no file lock and no new dependency. The exclusive cross-process lock stays deferred, as the plan states." + +patterns-established: + - "A backup step must never delete the previous valid backup before the new one is in place, and must never keep a copy whose schema version differs from the one it was taken for." + +requirements-completed: [MCP-06, MCP-07, STORE-05] + +coverage: + - id: D1 + description: "memory_update over MCP refuses unknown or immutable fields (type, source, ttl_sec) with an error naming the field, and writes nothing (REVIEW WR-02, T-04-27)" + requirement: "MCP-06" + verification: + - kind: e2e + ref: "crates/agent-memory/tests/tools.rs#memory_update_with_an_unknown_field_is_invalid_params_over_stdio" + status: pass + - kind: unit + ref: "crates/agent-memory/src/mcp.rs#update_args_reject_unknown_and_immutable_fields" + status: pass + - kind: unit + ref: "crates/agent-memory/src/mcp.rs#update_args_schema_makes_only_id_required (now also additionalProperties == false)" + status: pass + human_judgment: false + - id: D2 + description: "PATCH /api/memories/{id} with type, source or ttl_sec is a 400 naming the field, and the row is untouched" + requirement: "MCP-06" + verification: + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#patch_with_an_unknown_or_immutable_field_is_400_and_writes_nothing" + status: pass + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#malformed_or_empty_patches_are_400_never_500 (unchanged, green)" + status: pass + human_judgment: false + - id: D3 + description: "Malformed path or query parameters (PATCH/DELETE /api/memories/abc, DELETE /api/links/x/{id}/relates_to, ?expand_links=1, ?limit=abc) return 400 with an application/json {\"error\"} body (REVIEW IN-02, T-04-31)" + requirement: "MCP-07" + verification: + - kind: integration + ref: "crates/agent-memory/src/rest/handlers.rs#malformed_path_and_query_params_get_the_json_error_body" + status: pass + human_judgment: false + - id: D4 + description: "A backup taken after another process migrated is discarded and never replaces the genuine pre-migration backup; a valid backup replaces, then prunes (REVIEW WR-01, T-04-29)" + requirement: "STORE-05" + verification: + - kind: unit + ref: "crates/agent-memory-core/src/store/sqlite.rs#a_backup_of_an_already_migrated_database_is_discarded_and_keeps_the_valid_one" + status: pass + - kind: unit + ref: "crates/agent-memory-core/src/store/sqlite.rs#a_valid_backup_replaces_then_prunes_the_stale_one" + status: pass + human_judgment: false + - id: D5 + description: "A backup that fails to write, or fails to move into place, never removes the previous valid backup (REVIEW IN-03)" + requirement: "STORE-05" + verification: + - kind: unit + ref: "crates/agent-memory-core/src/store/sqlite.rs#a_failed_backup_write_never_removes_the_previous_backup" + status: pass + - kind: unit + ref: "crates/agent-memory-core/src/store/sqlite.rs#a_backup_that_cannot_be_moved_into_place_keeps_the_previous_one" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/migration_hygiene.rs#a_failed_backup_aborts_the_open_before_migrating (unchanged, green)" + status: pass + human_judgment: false + - id: D6 + description: "A to_latest failure is absorbed only when a re-probe shows LATEST_SCHEMA_VERSION, including the two real lost-race errors (table already exists, database is locked) (T-04-30)" + requirement: "STORE-05" + verification: + - kind: unit + ref: "crates/agent-memory-core/src/store/sqlite.rs#a_migration_failure_is_absorbed_only_when_another_process_reached_latest" + status: pass + - kind: unit + ref: "crates/agent-memory-core/src/store/sqlite.rs#a_real_lost_migration_race_is_absorbed" + status: pass + human_judgment: false + - id: D7 + description: "Two concurrent first opens of a temp copy of the v0.0.1 fixture both succeed; the DB ends at schema 3 with exactly one backup at schema 2 and no temp leftovers (10 rounds per run, 3 consecutive green runs)" + requirement: "STORE-05" + verification: + - kind: integration + ref: "crates/agent-memory-core/tests/migration_hygiene.rs#two_concurrent_first_opens_of_a_v0_0_1_database_both_succeed_with_one_genuine_backup" + status: pass + - kind: integration + ref: "crates/agent-memory-core/tests/migration_hygiene.rs#upgrading_writes_one_pre_migration_backup_with_the_old_schema / re_running_the_upgrade_keeps_one_backup_per_schema_version (+ no-temp-leftover asserts)" + status: pass + human_judgment: false + - id: D8 + description: "README documents the pre-migration backup, retention, restore steps (memory.db-wal / memory.db-shm) and the newer-schema refusal (REVIEW IN-01)" + verification: + - kind: other + ref: "grep -n '## Upgrading' README.md; grep -n 'memory.db-wal' README.md; grep -n 'brew upgrade unityinflow/tap/agent-memory' README.md" + status: pass + human_judgment: false + +duration: 12min +completed: 2026-10-23 +status: complete +--- + +# Phase 04 Plan 07: Strict update payloads, uniform REST error bodies, race-safe pre-migration backup Summary + +**`memory_update` and `PATCH` now reject unknown or immutable fields such as `type` or `ttl_sec`. Before, a patch carrying them reported success with the field silently dropped. Malformed REST path and query parameters get the same JSON `{"error"}` body as every other client error. When an MCP server and the REST daemon open an older database at the same time after an upgrade, both start, and the backup that survives is always a real pre-migration copy. The backup is written to a temp file, checked against the probed schema version, and renamed into place before any older copy is removed. A migration that loses the race to another process counts as success once the schema already reads 3.** + +## Performance + +- **Duration:** ~12 min +- **Started:** 2026-10-23T20:01:20Z +- **Completed:** 2026-10-23T20:13Z +- **Tasks:** 3/3 +- **Files modified:** 7 (0 created, 7 modified) + +## Accomplishments + +- **WR-02 closed.** `#[serde(deny_unknown_fields)]` is on `mcp::UpdateArgs` and `rest::handlers::UpdateBody` only. Every field attribute is unchanged, and the explicit-null deserializer still works alongside it, so the tags/scope/ttl absent/null/value matrices are unchanged and green. + - The 04-02 schema test still holds (`required == ["id"]`, no `default` on any patch field). It now also pins `additionalProperties == false`. + - Rejections are proven over the real stdio binary, through rmcp's actual parameter path, and over REST. In each case the row is unchanged afterwards. +- **IN-02 closed.** + - `forget_handler`, `update_handler` and `unlink_handler` take `Result, PathRejection>`, and `list_handler` takes `Result, QueryRejection>`. + - A private `extractor_rejection(status, message)` turns a 4xx rejection into `ApiError::BadRequest` and a 5xx into `ApiError::Internal`, so a routing misconfiguration still shows as a 500. + - A real-HTTP test sends 5 malformed requests. All return 400 with `application/json` and a non-empty `error`. +- **WR-01 and IN-03 closed.** + - `busy_timeout` is now set on the writer right after `Connection::open`, before the D-13 probe. It is connection-level, so a refused file stays byte-identical, and the existing test still passes. + - `busy_timeout` is also the first pragma in `apply_pragmas`. + - `pre_migration_backup(src, path, expected_version) -> Result>` follows the plan's contract: temp file, then verify, then rename, then prune. It cleans up the temp file on a best-effort basis on every failure path. + - `migrate_tolerating_concurrent_upgrade` wraps `to_latest`. The FTS5 and vec0 smoke checks still run after it. +- **IN-01 closed.** The README "Upgrading" section sits after "Run" and covers: + - the backup name; + - one backup per target version, replaced atomically; + - no backup means no migration; + - concurrent starts; + - the restore steps (stop all processes, copy over, delete `memory.db-wal` / `memory.db-shm`, start the older binary); + - the verbatim newer-schema error, which names `brew upgrade unityinflow/tap/agent-memory`. + +## Task Commits + +1. **Task 1 (tracer): memory_update refuses unknown and immutable fields on MCP and REST (WR-02).** `f09a69f` (fix) +2. **Task 2: malformed REST path and query params get the JSON error body (IN-02).** `cb24e84` (fix) +3. **Task 3: race-safe pre-migration backup, concurrent-upgrade-tolerant open, and README Upgrading (WR-01, IN-03, IN-01).** `a914311` (fix) + +**Plan metadata:** see the `docs(04-07)` commits that follow. + +## Files Created/Modified + +- `crates/agent-memory/src/mcp.rs`: `deny_unknown_fields` on `UpdateArgs`, plus doc and tool-description notes; 1 new test and 1 extended schema test. +- `crates/agent-memory/src/rest/handlers.rs`: + - `deny_unknown_fields` on `UpdateBody`; + - the `PathRejection` / `QueryRejection` imports; + - `extractor_rejection`; + - 4 handler signatures, with doc comments updated; + - the existing direct test calls wrapped in `Ok(..)`; + - 2 new tests. +- `crates/agent-memory/tests/tools.rs`: 1 new stdio end-to-end test. +- `crates/agent-memory-core/src/store/sqlite.rs`: + - the reordered `apply_pragmas`; + - `BACKUP_LOCK`, `backup_temp_path`, `discard_backup_temp`, `copy_into_temp`, `prune_other_backups`; + - the rewritten `pre_migration_backup` and the new `migrate_tolerating_concurrent_upgrade`; + - the `open()` changes (writer `busy_timeout`, `db_version` passed through, the wrapped migrate, the numbered doc); + - 6 new unit tests. +- `crates/agent-memory-core/tests/migration_hygiene.rs`: `assert_no_backup_temp`, leftover assertions in 2 existing tests, and the new concurrent first-open smoke test. +- `README.md`: the memory_update unknown-field note, the REST error-body sentence, and the new "## Upgrading" section. +- `.planning/STATE.md`: the `04-07: deny_unknown_fields` decision entry, added in Task 1. + +## Decisions Made + +- **Per-process backup lock alongside the pid-qualified temp name.** The plan's temp path `{db}.backup-tmp-{NNNN}-{pid}` keeps separate processes apart. The plan's own smoke test, however, races two threads in one process, which share a pid. A static `BACKUP_LOCK` serializes backups within a process, so the exact temp path from the plan, and its "failed write" seam test, are kept unchanged. A poisoned lock is recovered with `PoisonError::into_inner`, because the guard protects no data and no `unwrap` is needed. +- **Prune tolerates `NotFound`.** Two concurrent openers can both try to remove the same stale `{prefix}20000101`. The one that loses gets `NotFound`, which must not abort startup. Any other prune error still aborts, so the existing `a_failed_backup_aborts_the_open_before_migrating` blocker-directory test stays green. +- **The destination is closed explicitly** with `Connection::close()` before the rename, and the error is surfaced, rather than relying on drop. + +## Deviations from Plan + +### Auto-fixed Issues + +**1. [Rule 1 - Plan assumption] rmcp 1.8 returns an argument-deserialization failure as a tool error result, not a JSON-RPC -32602** +- **Found during:** Task 1 (the stdio end-to-end test failed on its first run) +- **Issue:** The plan's interface contract and `` expected a JSON-RPC `error` member with code -32602. In practice, rmcp-1.8.0 `src/handler/server/router/tool.rs` `into_tool_argument_error` deliberately turns an `INVALID_PARAMS` error whose message starts with `failed to deserialize parameters:` into `CallToolResult::error(..)`. That is, it returns `{"result": {"isError": true, "content": [{"text": "failed to deserialize parameters: unknown field `type`, expected one of ..."}]}}`, following the MCP rule that input-validation errors are tool execution errors the model can correct. +- **Fix:** The stdio test now asserts the real wire shape: no `error` member, `isError: true`, and text that starts with the rmcp prefix and names the field. The rejection still happens before the handler runs, names the field, and writes nothing, so WR-02's intent (no silent partial write) is fully met. The `UpdateArgs` doc, the tool description, the README and the STATE decision describe the actual shape. The test keeps the name the plan gave it. +- **Files modified:** `crates/agent-memory/tests/tools.rs`, `crates/agent-memory/src/mcp.rs`, `README.md`, `.planning/STATE.md` +- **Commit:** `f09a69f` + +**2. [Rule 1 - Bug in the plan's contract] Two openers in one process would share the pid-qualified temp path** +- **Found during:** Task 3 design +- **Issue:** The two-thread smoke test runs both openers in one process, so `{db}.backup-tmp-NNNN-{pid}` would collide: two Online Backups into one file, and one of them renaming it mid-write. +- **Fix:** A process-wide `BACKUP_LOCK` around the backup step (see Decisions). The temp name is exactly the plan's. +- **Commit:** `a914311` + +### Additive test strengthening (no scope change) + +**3. [Rule 2 - Coverage] Real lost-race test for the migration tolerance** +- **Found during:** Task 3 mutation check +- **Issue:** Removing the `migrate_tolerating_concurrent_upgrade` wrapper never turned the two-thread smoke test red in 30 rounds. With the in-process backup lock, B's backup reliably outlasts A's migration, so B's `to_latest` sees version 3 and does nothing. The closure seam test uses a simulated error only. +- **Fix:** Added `a_real_lost_migration_race_is_absorbed`. It uses two connections on one WAL database: the winner runs the real `to_latest`, and the loser then runs 0003 inside its own transaction, as rusqlite_migration does after its out-of-transaction version read. This reproduces both real failure modes, with and without a stale read snapshot: + - `table memory_links already exists`; + - `database is locked`, the SQLITE_BUSY_SNAPSHOT case. + + Both are absorbed. With the tolerance disabled, the test fails. +- **Commit:** `a914311` + +**4. [Rule 2 - Coverage] Rename-failure seam test** +- Added `a_backup_that_cannot_be_moved_into_place_keeps_the_previous_one`: a non-empty directory sits at today's final name, so the copy succeeds but the rename fails. The previous backup survives and no temp is left. This covers the IN-03 step after the copy, in addition to the plan's failed-temp-write case. + +### Process + +**5. Tracer feedback gate run as an automated re-verify, not an interactive checkpoint** +- Auto mode is off (`_auto_chain_active=false`, `auto_advance=false`), but the orchestrator asked for the whole plan in one run, and the tracer's `` is fully automated. Task 1's verify was: + - `cargo test -p agent-memory --bins`: 62 passed; + - the stdio test: 6/6 in `--test tools`; + - `clippy -p agent-memory -D warnings`: clean. + + It ran on exactly the tree that was committed, which `cargo fmt --check` confirmed was unchanged. That was the gate, as in 04-05 and 04-06. + +**6. RED proven by temporary mutation, not separate RED commits** +- CLAUDE.md forbids committing failing tests, so each fix lands together with its tests in one commit. RED was observed in place, and the files were then restored from `cmp`-verified copies: + - **Task 1:** removing both `deny_unknown_fields` attributes failed `update_args_reject_unknown_and_immutable_fields`, `update_args_schema_makes_only_id_required` and the stdio test. + - **Task 2:** the new real-HTTP test, run against the pre-change `handlers.rs`, failed with `content-type "text/plain; charset=utf-8"`. + - **Task 3:** disabling the copy-version check failed the concurrent smoke test (`round 5: the surviving backup is a genuine pre-migration copy`). Disabling the tolerance failed `a_real_lost_migration_race_is_absorbed`. + +--- + +**Total deviations:** 2 auto-fixed (Rule 1), 2 additive (Rule 2), 2 process. **Impact:** none on scope. The MCP wire shape of the WR-02 rejection is rmcp's, and it is documented. + +## TDD Gate Compliance + +All three tasks carry `tdd="true"` in a `type: execute` plan. Each lands as one `fix(...)` commit that bundles its tests, because CLAUDE.md forbids committing failing tests. RED was observed by temporary mutation (deviation 6), and GREEN is each task's commit. + +## Concurrent smoke test run count + +`cargo test -p agent-memory-core --test migration_hygiene two_concurrent` passed on **3 consecutive runs** (10 race rounds each, 0.17-0.18 s per run). It also passed in the full `--test migration_hygiene` run, in the plan verify, and in the final `cargo test --workspace`, so it has 5 green runs in total and 0 flakes. + +## Verification + +- `cargo fmt --all --check`: clean +- `cargo clippy --workspace --all-targets -- -D warnings`: clean +- `cargo test --workspace`: **211 passed / 0 failed / 2 ignored**. That is up from 200 after 04-06, an increase of 11: + - 1 MCP unit test; + - 2 REST tests; + - 1 stdio test; + - 6 sqlite seam tests; + - 1 migration_hygiene test. + + Both ignored tests pre-date this plan; one is the fixture generator. +- `cargo test -p agent-memory-core --test migration_hygiene`: 8 passed, 1 ignored. This includes the byte-for-byte too-new refusal and the failed-backup abort, both unchanged. +- `git diff --stat -- crates/agent-memory-core/sql/ crates/agent-memory-core/tests/fixtures/`: empty. The migrations and the committed v0.0.1 fixture are untouched, and every test works on temp copies. +- All plan acceptance greps are present: + - `deny_unknown_fields`: 3 in mcp.rs, 2 in handlers.rs; + - `PathRejection`, `QueryRejection`; + - `fn migrate_tolerating_concurrent_upgrade`; + - `pre_migration_backup(&writer, path, db_version)`; + - `backup-tmp-`, `writer.busy_timeout`; + - `## Upgrading`, `memory.db-wal`, `memory.db-shm`, `brew upgrade unityinflow/tap/agent-memory`; + - `04-07: deny_unknown_fields` in STATE.md. +- No `unwrap()` was added to non-test code. No new dependency, no migration, no CI workflow change, and no tag, release or push. + +## Known Stubs + +None. + +## Threat Flags + +None. No new network endpoint, auth path or schema change. The new file surface is the pid-qualified backup temp file beside the DB, which the plan already covers as T-04-32. + +## Deferred (as the plan states) + +- **WR-01 full fix with a cross-process exclusive lock** (a lock file, or `BEGIN IMMEDIATE` spanning both backup and migration). It needs a new dependency or a restructure of rusqlite_migration's transaction. Verify-then-rename plus re-probe closes both observable failure modes. +- **`deny_unknown_fields` on the v0.0.1 DTOs and on the link DTOs**, per the [03-01] decision. +- **Automatic cleanup of stale `.backup-tmp-*` files** left by crashed processes. That needs liveness detection of the owning pid (T-04-32). +- **WR-04 behaviour change** (unbounded expanded `memory_list`) remains an open API-contract question for Phase 5/6, carried over from 04-06. +- **Observation, not changed:** over MCP, a deserialization rejection (an unknown field) is a tool error result, while a seam validation error (for example empty content) is a JSON-RPC `invalid_params` error. That split is rmcp's design and pre-dates this plan. + +## User Setup Required + +None. + +## Next Phase Readiness + +- All 7 phase-04 plans are executed. MCP-06, MCP-07 and STORE-05 were already ticked in REQUIREMENTS.md, so mark-complete is idempotent here. The phase is ready for re-verification. It is not yet marked complete; the orchestrator does that after verification passes. + +## Self-Check: PASSED + +- All 7 modified files are present on disk. +- Commits `f09a69f`, `cb24e84` and `a914311` were found in `git log`. From 8b2c762967cb5bb6f7958b6ee35d807f23a23577 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 22:13:10 +0200 Subject: [PATCH 34/42] docs(04-07): update state and roadmap progress (7/7 phase-04 plans) Co-Authored-By: Claude Opus 5.5 --- .planning/ROADMAP.md | 6 +++--- .planning/STATE.md | 23 +++++++++++++---------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index c725d19..ba83660 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -67,7 +67,7 @@ Plans: 4. Hybrid ranking stays decay-aware — a fresh memory outranks a stale one at equal fused rank — and access bumps apply only to post-truncation returned ids. 5. Upgrading a real v0.0.1 database applies migration 0003 only after a pre-migration backup; shipped 0001/0002 SQL is provably unchanged (fixture divergence test against a v0.0.1 database), and a newer-schema DB met by an older binary yields a friendly error, not a crash. -**Plans**: 6/7 plans executed (4 executed + 3 gap closure) +**Plans**: 7/7 plans executed (4 executed + 3 gap closure) Plans: **Wave 1** @@ -89,7 +89,7 @@ Plans: - [x] 04-05-PLAN.md — Gap wave 1: forgiving keyword leg per user decision option-a (raw MATCH first, one sanitized OR-of-quoted-tokens retry, tokenless = empty leg, identical in both modes) + search-path error redaction (CR-01, CR-02, truths #14/#15, IN-04) (SEARCH-04) - [x] 04-06-PLAN.md — Gap wave 2: conditional sweep backfill closing the update/forget/sweep races + symmetric relates_to + IN-05/IN-06 docs (CR-03, WR-03) (MCP-06, MCP-07) -- [ ] 04-07-PLAN.md — Gap wave 3: deny_unknown_fields on update DTOs, uniform REST rejection bodies, race-safe verify-then-rename pre-migration backup + Upgrading docs (WR-02, IN-02, WR-01, IN-03, IN-01) (MCP-06, MCP-07, STORE-05) +- [x] 04-07-PLAN.md — Gap wave 3: deny_unknown_fields on update DTOs, uniform REST rejection bodies, race-safe verify-then-rename pre-migration backup + Upgrading docs (WR-02, IN-02, WR-01, IN-03, IN-01) (MCP-06, MCP-07, STORE-05) ### Phase 5: Portable Export & Import @@ -130,6 +130,6 @@ Phases execute in numeric order: 3 → 4 → 5 → 6 | 1. Core Memory Foundation | v1.0 | 3/3 | Complete | 2026-06-25 | | 2. Semantic Search, Interop & Release | v1.0 | 5/5 | Complete | 2026-07-12 | | 3. API Hardening & Toolchain Spikes | v1.1 | 2/2 | Complete | 2026-07-12 | -| 4. Memory Update, Relations & Hybrid Search | v1.1 | 6/7 | In Progress| | +| 4. Memory Update, Relations & Hybrid Search | v1.1 | 7/7 | In Progress| | | 5. Portable Export & Import | v1.1 | 0/? | Not started | - | | 6. Distribution & v0.1.0 Release | v1.1 | 0/? | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index a020830..745506d 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -5,16 +5,16 @@ milestone_name: Hardening & Interop current_phase: 04 current_phase_name: Memory Update, Relations & Hybrid Search status: executing -stopped_at: Completed 04-06-PLAN.md -last_updated: "2026-10-23T19:59:51.264Z" +stopped_at: Completed 04-07-PLAN.md +last_updated: "2026-10-23T20:12:50.775Z" last_activity: 2026-10-23 -last_activity_desc: Completed 04-06 (gap wave 2, conditional backfill + symmetric relates_to) -state_head: c72146bc6296170116a8ed40749576c6af5c6f87 +last_activity_desc: Completed 04-07 (gap wave 3, strict update DTOs + REST error bodies + race-safe backup) +state_head: 9693c5fc8d881f1e3fce159415d5cd4ff2f8873b progress: total_phases: 4 completed_phases: 1 total_plans: 9 - completed_plans: 8 + completed_plans: 9 --- # Project State @@ -29,9 +29,9 @@ See: .planning/PROJECT.md (updated 2026-07-12) ## Current Position Phase: 04 (Memory Update, Relations & Hybrid Search) — EXECUTING -Plan: 6 of 7 (gap closure: 04-05, 04-06 done; 04-07 next) -Status: Executing gap closure — 04-07 ready to execute -Last activity: 2026-10-23 — Completed 04-06 (CR-03 conditional sweep backfill, WR-03 symmetric relates_to, IN-05/IN-06/WR-04 docs) +Plan: 7 of 7 (gap closure: 04-05, 04-06, 04-07 done) +Status: All 7 plans executed — ready for phase 04 re-verification (not yet marked complete) +Last activity: 2026-10-23 — Completed 04-07 (WR-02 deny_unknown_fields on update DTOs, IN-02 uniform REST rejection bodies, WR-01/IN-03 race-safe pre-migration backup, IN-01 README Upgrading) Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) @@ -74,6 +74,7 @@ Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) | Phase 04 P04 | 11 min | 2 tasks | 11 files | | Phase 04 P05 | 10 min | 3 tasks | 12 files | | Phase 04 P06 | 7 min | 3 tasks | 9 files | +| Phase 04 P07 | 12 min | 3 tasks | 7 files | ## Accumulated Context @@ -119,6 +120,8 @@ Recent decisions affecting current work: - [Phase 04]: 04-06: relates_to stored canonically as (min id, max id) and unlinked from either end; supersedes/caused_by stay directed (D-03). Storage-only: MCP/REST responses echo request ids; NotFound names the request's from_id first. No migration (REVIEW WR-03). - [Phase 04]: 04-06: WR-04 documented only: the 200 x 10 expansion bound covers search and list-with-limit; unlimited expanded memory_list returns every row. Behaviour choice (reject / default limit / unbounded) is an open API-contract question for Phase 5/6. - [Phase 04]: 04-07: deny_unknown_fields added to UpdateArgs/UpdateBody only (new v0.1.0 surfaces, REVIEW WR-02); the [03-01] decision stands for the v0.0.1 DTOs, so the v0.1.0 release-notes delta for existing surfaces is unchanged. Over MCP, rmcp 1.8 reports the rejection as a tool error result (isError: true, "failed to deserialize parameters: unknown field ..."), not a JSON-RPC -32602; REST is a 400 {"error"}. +- [Phase 04]: 04-07: backup temp is pid-qualified and a process-wide BACKUP_LOCK serializes backups within one process, so same-process openers never share a temp; prune tolerates NotFound from a concurrent opener +- [Phase 04]: 04-07: WR-01 closed with verify-then-rename + tolerate-by-re-probe (no file lock, no new dep); cross-process exclusive lock stays deferred ### Pending Todos @@ -159,8 +162,8 @@ Items acknowledged and carried forward: ## Session Continuity -Last session: 2026-10-23T19:59:51.214Z -Stopped at: Completed 04-06-PLAN.md +Last session: 2026-10-23T20:12:50.720Z +Stopped at: Completed 04-07-PLAN.md Resume file: None ## Operator Next Steps From ce1fd1e5cee7ccebb8eb06ced8fc7ee80568c2d7 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 22:21:46 +0200 Subject: [PATCH 35/42] docs(04): re-verification (passed 28/28) and code re-review after gap closure Co-Authored-By: Claude Opus 5.5 --- .../04-REVIEW.md | 501 ++++++------------ .../04-VERIFICATION.md | 307 ++++++----- 2 files changed, 352 insertions(+), 456 deletions(-) diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md index 1c0e4d5..51c7957 100644 --- a/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md @@ -1,364 +1,211 @@ --- phase: 04-memory-update-relations-hybrid-search -reviewed: 2026-10-23T00:00:00Z +reviewed: 2026-10-23T21:00:00Z depth: standard -files_reviewed: 20 +review_kind: re-review (gap closure 04-05..04-07, diff d3a4580..HEAD) +files_reviewed: 14 files_reviewed_list: - - Cargo.toml - README.md - - crates/agent-memory-core/sql/0003_relations.sql - crates/agent-memory-core/src/domain.rs - crates/agent-memory-core/src/service.rs - - crates/agent-memory-core/src/store/migrations.rs - crates/agent-memory-core/src/store/mod.rs - crates/agent-memory-core/src/store/sqlite.rs + - crates/agent-memory-core/tests/backfill_race.rs + - crates/agent-memory-core/tests/fallback.rs - crates/agent-memory-core/tests/hybrid.rs - - crates/agent-memory-core/tests/import.rs - crates/agent-memory-core/tests/migration_hygiene.rs - crates/agent-memory-core/tests/relations.rs - - crates/agent-memory-core/tests/semantic.rs - - crates/agent-memory-core/tests/update.rs - - crates/agent-memory/Cargo.toml - - crates/agent-memory/src/main.rs - crates/agent-memory/src/mcp.rs - crates/agent-memory/src/rest/handlers.rs - - crates/agent-memory/src/rest/mod.rs - - crates/agent-memory/src/serde_util.rs + - crates/agent-memory/tests/rest.rs + - crates/agent-memory/tests/tools.rs findings: - critical: 3 - warning: 4 + critical: 0 + warning: 2 info: 6 - total: 13 + total: 8 status: issues_found --- -# Phase 4: Code Review Report +# Phase 4: Code Review Report (re-review after gap closure) **Reviewed:** 2026-10-23 -**Depth:** standard (diff `907161c..HEAD`) -**Files Reviewed:** 20 (binary fixture `tests/fixtures/v0.0.1.db` skipped) -**Status:** issues_found +**Depth:** standard. Scope is the gap-closure diff `d3a4580..HEAD`, excluding `.planning/`. +**Files Reviewed:** 14 +**Status:** issues_found (no blockers) ## Summary -Phase 4 adds `memory_update` (merge-patch with an in-transaction vector swap), flat typed -relations with 1-hop expansion, RRF hybrid search, and migration 0003 with a pre-migration -backup and a newer-schema refusal. The code is careful in many places. SQL is parameterized -everywhere. The `MemoryError` mappers are exhaustive. No `unwrap()` was added to production -code. Clippy (`-D warnings`, all targets) is clean and all `agent-memory-core` tests pass -(verified locally). - -The main problems come from making hybrid the default search path, and from content becoming -mutable: - -1. Hybrid sends the raw query to FTS5 on every search while Ollama is up. FTS5 reports - column-filter parse errors (`sqlite-vec`, `re-embed`, `note: x`) as `no such column`. The - classifier does not recognise that message, so these queries now return **HTTP 500 / - `internal_error`** on the primary path. v0.0.1 answered them. -2. Natural-language punctuation (`?`, `,`, `'`) now returns 400 whenever Ollama is up. The - brief asked for an explicit assessment of this. The verdict is a **regression, not an - acceptable trade-off** (CR-02). -3. Content can now change, so the sweep's embedding backfill can write a vector for *old* - content over a row that was updated meanwhile. That leaves a permanently wrong vector with - `embedding_status = 1`. - -The FTS5 behaviour behind CR-01/CR-02 was checked against the bundled SQLite (3.51.3, via -rusqlite 0.39): - -| query | FTS5 result | mapped to | tier | -|---|---|---|---| -| `what is it?` | `fts5: syntax error near "?"` | `InvalidQuery` | 400 / invalid_params | -| `sqlite-vec` | `no such column: vec` | `Sqlite` | **500 / internal_error** | -| `re-embed` | `no such column: embed` | `Sqlite` | **500 / internal_error** | -| `note: auth` | `no such column: note` | `Sqlite` | **500 / internal_error** | - -Known, already-tracked items (WR-01 filter-after-KNN-cut, WR-03 REST harness hang) are not -re-reported. - -## Narrative Findings (AI reviewer) - -## Critical Issues - -### CR-01: Hyphenated or colon-bearing queries are 500 / `internal_error` on the default (hybrid) search path - -**File:** `crates/agent-memory-core/src/store/sqlite.rs:333-340` (classifier), `crates/agent-memory-core/src/service.rs:427` (hybrid propagates the keyword leg's error), `crates/agent-memory/src/mcp.rs:310`, `README.md:165-170` -**Issue:** `map_fts_query_error` only treats `fts5: syntax error` and `unterminated string` as -client input. FTS5 parses `a-b` and `a: b` as column filters and fails at *step* time with -`no such column: b`. That error falls through to `MemoryError::Sqlite`, which both mappers send -to the internal tier. - -Before Phase 4, this only affected keyword-fallback mode (Ollama down). Now every search with -Ollama up runs `keyword_candidates`, and its error takes priority over the KNN leg -(service.rs:427). So an ordinary query such as `sqlite-vec`, `re-embed`, `pre-migration -backup`, `TODO: auth` or `x86-64`: - -- returned 200 (semantic) in v0.0.1; -- now returns HTTP 500 / MCP `internal_error`. - -This breaks the project's core contract that bad client input never surfaces as a 500 (02-05, -Phase 3 D-06). It also defeats agent error handling: an agent that sees `internal_error` assumes -a server fault and will not rephrase. The README (L165-170) and the `memory_search` tool -description promise `invalid_params` for queries FTS5 cannot parse, so both documents are wrong -for this class of input. No test covers it: `hybrid.rs:538-570` only exercises `"unbalanced`, -`AND` and `?`. - -**Fix:** Classify step-time column-filter failures as client input. The statement prepares -successfully, so a `no such column` at step time can only come from the MATCH expression. Then -add a regression row for `sqlite-vec` and `note: x` to the hybrid/keyword decision-table test. -```rust -fn map_fts_query_error(query: &str, e: rusqlite::Error) -> MemoryError { - let msg = e.to_string(); - if msg.contains("fts5: syntax error") - || msg.contains("unterminated string") - // FTS5 column filters (`a-b`, `col: x`) on the single-column - // memories_fts table: only reachable from the bound MATCH string, - // because the statement itself prepared successfully. - || msg.contains("no such column") - { - MemoryError::InvalidQuery(query.to_string()) - } else { - MemoryError::Sqlite(e) - } -} +Plans 04-05, 04-06 and 04-07 close every prior blocker: + +- **CR-01 / CR-02.** The keyword leg now runs the raw query first and retries once with a sanitized query. Natural-language and hyphenated queries are answered in both modes. +- **CR-03.** The backfill write is a guarded compare-and-set. +- **WR-01.** The backup is written to a temp file, its schema version is checked, and only then is it renamed into place. The migration tolerates a lost race by re-reading the schema version. +- **WR-02.** `deny_unknown_fields` is now on both patch DTOs. +- **WR-03.** `relates_to` is stored canonically. + +Production code adds no `unwrap()`/`expect()`. The new matches are exhaustive: `redact_search_error`, `is_symmetric`, `canonical_endpoints`, and the migrate re-probe. All SQL is still bound. + +I checked the classifier against the **bundled** SQLite with a scratch crate in `/tmp`. It calls `SqliteStore::search` and `Store::keyword_candidates` directly. 13 of 15 adversarial probes return `Ok`: `{a b}: c`, `^a`, `-a`, `NEAR(a b, 99999999999999999999)`, `NEAR(a b, -1)`, `a + b`, `a.b`, `rank: auth`, an embedded NUL, `(`×300 nesting (parser stack overflow is classified), and others. + +The other 2 probes fail. They are both cases of one FTS5 error class that the marker list misses (WR-01 below). That error is redacted, so no SQLite text leaks, but a client input still becomes a 500. + +The second warning is a README sentence added by 04-07. It promises a JSON error body for every REST client error, but two v0.0.1 routes still return axum's plain-text rejections. + +The workspace test suite was not re-run here, because a verifier runs in parallel. The 04-07 SUMMARY reports 211 passed / 0 failed / 2 ignored. + +## Prior findings — resolution + +| ID | Prior severity | Resolution | Evidence | +|----|----------------|------------|----------| +| CR-01 | BLOCKER | **Resolved** (one residual class, see new WR-01) | `sqlite.rs:498-504` adds `no such column`, `unknown special query` and `expected integer` to `FTS5_QUERY_PARSE_MARKERS`. `sqlite.rs:519-530` gates on primary code `SQLITE_ERROR`. `sqlite.rs:569-588` adds the raw-first retry, shared by `search` (`:866`) and `keyword_candidates` (`:908`). Tests: `tests/hybrid.rs` two-mode table, `tests/rest.rs` (`sqlite-vec` → 200), `tests/tools.rs` stdio e2e. | +| CR-02 | BLOCKER | **Resolved** (user decision option-a) | `sqlite.rs:541-552` `sanitize_fts_query`; a tokenless query returns an empty leg (`:582`). The decision is recorded in `.planning/STATE.md:118`. The MCP description (`mcp.rs:330`) and README (`README.md:203-212`, `:482-488`) describe the forgiving contract. | +| CR-03 | BLOCKER | **Resolved** | `sqlite.rs:706-713` adds a guarded claim (`embedding_status = 0 AND content = ?2`) as the first statement of the writer tx, with an `Ok(false)` rollback. `service.rs:697-703` passes the embedded content back. `tests/backfill_race.rs` has 7 interleavings, including a recycled rowid and a gated real sweep. | +| WR-01 | WARNING | **Resolved** for both observable failure modes. The cross-process exclusive lock is **deferred-as-documented** | `sqlite.rs:235-301`: temp file, then version check, then rename, then prune. A post-migration copy is discarded (`:273-282`). `sqlite.rs:315-334` absorbs a failed migrate only when the re-read version is `LATEST`. `busy_timeout` is set before the probe (`:387`) and first in `apply_pragmas` (`:89`). Tests: `sqlite.rs` seam tests and `a_real_lost_migration_race_is_absorbed`, plus `migration_hygiene.rs` two-opener smoke. | +| WR-02 | WARNING | **Resolved** | `mcp.rs:169` and `handlers.rs:107` add `#[serde(deny_unknown_fields)]`. The schema advertises `additionalProperties: false`. The MCP wire shape is rmcp's `isError: true` tool result, as documented at `README.md:265-269`. The rmcp interaction looks sound: `_meta` travels outside `arguments`, and the explicit-null deserializers still compose. | +| WR-03 | WARNING | **Resolved** (stale wording remains, see IN-03) | `domain.rs:127-149` adds `is_symmetric` / `canonical_endpoints`. `sqlite.rs:977-982` inserts the canonical pair. `sqlite.rs:998-1003` makes symmetric unlink a single bound DELETE. Tests: 8 new in `tests/relations.rs`. The MCP and REST echo tests pass. | +| WR-04 | WARNING | **Deferred-as-documented** (doc-only, by user instruction) | The `domain.rs:360-371` doc is corrected and `README.md:363` adds "pass a `limit` on large stores". Behaviour is unchanged (`service.rs:232-246`). Note: the `memory_list` **tool description** (`mcp.rs:297`), which is what agents read, still does not mention the unbounded case. | +| IN-01 | INFO | **Resolved** | `README.md:84-121` adds the Upgrading section: backup name, retention, restore steps including `-wal`/`-shm`, and the newer-schema error text. | +| IN-02 | INFO | **Resolved** for the `Path`/`Query` routes the finding named. README now overclaims, see new WR-02 | `handlers.rs:220-226` `extractor_rejection`. It is used at `:236`, `:287`, `:319` and `:397`, and keeps a 5xx for routing faults. | +| IN-03 | INFO | **Resolved** | `sqlite.rs:265-293`: no backup is deleted before the new copy has been renamed into place. Tests: `a_failed_backup_write_never_removes_the_previous_backup` and `a_backup_that_cannot_be_moved_into_place_keeps_the_previous_one`. | +| IN-04 | INFO | **Resolved** | `service.rs:345` now reads "Same order as the hybrid path". | +| IN-05 | INFO | **Deferred-as-documented** (by user instruction) | `store/mod.rs:161-165` and `README.md:291-295`. | +| IN-06 | INFO | **Deferred-as-documented** (by user instruction) | `domain.rs:220-232` doc on `RELATED_SNIPPET_CHARS` / `snippet`. | + +## New findings + +## Warnings + +### WR-01: A long FTS5 `NOT` chain still turns client input into a 500 / `internal_error` + +**File:** `crates/agent-memory-core/src/store/sqlite.rs:498-504` (markers), `:519-530` (classifier), `:586` (unclassified error becomes `Sqlite`), `crates/agent-memory-core/src/service.rs:302-310` (redacted to the internal tier) + +**Issue:** FTS5 enforces `SQLITE_FTS5_MAX_EXPR_DEPTH` (256). `NOT` nodes are never flattened (`fts5ExprAddChildren` skips `FTS5_NOT`), so `a NOT a NOT a …` with more than 256 `NOT`s builds a left-deep tree. That tree fails at step time with: + +``` +fts5 expression tree is too large (maximum depth 256) ``` -(CR-02's sanitising retry removes most of this input class altogether. This mapping is still -the required backstop.) - -### CR-02: Assessment of the requested behaviour change — natural-language queries now fail with the embedder up (regression) - -**File:** `crates/agent-memory-core/src/service.rs:244-265, 412-428`, `crates/agent-memory-core/src/store/sqlite.rs:606-647`, `crates/agent-memory-core/tests/hybrid.rs:538-570`, `README.md:165-170, 407-414` -**Issue:** With Ollama up, which is the recommended setup, every search now depends on the raw -query being a valid FTS5 expression. Queries like `what did we decide about auth?`, `don't -retry`, `auth, tokens`, `C++ build` and an empty string are rejected with 400 / -`invalid_params`. v0.0.1 answered all of them semantically. Including CR-01's inputs, this -covers most of what an LLM agent naturally sends: questions, sentences with commas or -apostrophes, and hyphenated technical terms. - -Severity: **BLOCKER**. The degradation is documented (README L407-414, the tool description), -but documenting it does not make it acceptable for an agent-facing tool. - -- **The semantic search value is lost.** The product's main promise is "semantic search via - local Ollama embeddings". In the configuration where that should work best, the query - embedding is now discarded whenever the query contains common punctuation. -- **The T-04-15 rationale does not hold.** The plan's justification (04-04-PLAN T-04-15) is - that answering semantically in one mode and with a 400 in the other would let a caller probe - which mode is active. But every response already reports `search_mode`, so there is no hidden - mode to probe. The second reason, the error taxonomy, only requires the *same* classification - in both modes. It does not require that classification to be an error. -- **Agents pay a retry cost.** An agent receiving `invalid_params` will rewrite and retry, which - costs tokens on every punctuated question. Agents rarely read the "use plain words" caveat in - the README. - -**Fix:** Keep exact FTS5 syntax available to power users, but stop rejecting natural language. -Apply the same logic in both modes so the T-04-15 "same classification everywhere" property -still holds: + +The message starts with `fts5 expression`, not `fts5: `. None of the five markers matches it, so `match_with_sanitized_retry` takes the `Err(e) => Err(MemoryError::Sqlite(e))` arm. `redact_search_error` then maps it to `SEARCH_STORAGE_FAILURE`, which means HTTP 500 / MCP `internal_error` plus an `ERROR` log line. + +I verified this against the bundled SQLite with the scratch probe. Both `NOT`×257 and `NOT`×300 return `Err("sqlite error: fts5 expression tree is too large (maximum depth 256)")` from `search` **and** from `keyword_candidates`. The parser-stack-overflow class (deep parentheses) is classified correctly and returns `Ok`. + +The input is contrived, but it breaks the contract this phase re-established in `README.md:203` ("a query is never rejected") and in D-06 (client input never surfaces as a 500). It also lets any client write `ERROR`-level log entries on demand. The sanitized form would parse: OR chains are flattened, and a 3000-term OR chain was verified to run. + +`every_bundled_fts5_parse_error_is_classified_as_match_input` does not cover this class, so the test suite cannot catch it. + +**Fix:** Add the marker, then add the probe to the bundled-SQLite classifier test and to the two-mode decision table. ```rust -// sqlite.rs: a fallback MATCH built from the query's word tokens, each -// phrase-quoted (inner quotes doubled) and OR-joined. It always parses. -fn sanitize_fts_query(raw: &str) -> Option { - let terms: Vec = raw - .split(|c: char| !c.is_alphanumeric() && c != '_') - .filter(|t| !t.is_empty()) - .map(|t| format!("\"{}\"", t.replace('"', "\"\""))) - .collect(); - (!terms.is_empty()).then(|| terms.join(" OR ")) -} +const FTS5_QUERY_PARSE_MARKERS: [&str; 6] = [ + "fts5: ", + "fts5 expression tree is too large", // SQLITE_FTS5_MAX_EXPR_DEPTH, e.g. 257+ chained NOTs + "unterminated string", + "no such column", + "unknown special query", + "expected integer", +]; +// test probe: format!("a {}", "NOT a ".repeat(300)) -> Ok in both modes ``` -In `keyword_candidates`, and in `search` for the fallback path: run the raw MATCH first. If it -fails with `InvalidQuery`, rerun once with `sanitize_fts_query(&args.query)`. If no tokens are -left, return an empty keyword leg so hybrid fuses KNN-only. Only report `InvalidQuery` when the -caller clearly asked for FTS syntax (for example an unbalanced `"`), or never. Then update -`a_malformed_match_is_invalid_query_in_hybrid_mode_exactly_as_in_keyword_mode`: `what is the -plan?` should succeed in both modes. If the owners still choose to keep the 400, record that as -an explicit decision in the CONTEXT decisions log, and fix CR-01 either way. - -### CR-03: The sweep backfill can overwrite an updated memory's vector with an embedding of its *old* content, permanently - -**File:** `crates/agent-memory-core/src/service.rs:612-637`, `crates/agent-memory-core/src/store/sqlite.rs:439-479`, `crates/agent-memory-core/src/store/sqlite.rs:812-836` -**Issue:** The backfill has three steps: - -1. It reads `(id, content)` for pending rows (`pending_embeddings`). -2. It `await`s a batch embed of up to 128 texts outside any lock. On a CPU Ollama that takes - seconds. -3. `insert_embedding` then runs an unconditional `DELETE` + `INSERT` of the vector and - `embedding_status = 1`. - -Before Phase 4, content was immutable, so this was safe. `memory_update` breaks that -assumption. Interleaving: - -1. Row 7 is pending (stored while Ollama was down). -2. Ollama recovers. The sweep reads `(7, "old text")` and starts embedding it. -3. A client PATCHes row 7 with `"new text"`. `update` embeds the new text successfully, swaps - the vector and sets status 1. -4. The sweep finishes and `insert_embedding(7, vec("old text"))` deletes the fresh vector, - writes the stale one, and sets status 1. - -Row 7 now matches its old meaning semantically, forever: status is 1, so no later sweep repairs -it. The same happens if the update's own embed fails (status 0, no vector), since the sweep -"repairs" it with the stale vector. The race is wider than it looks. `main.rs` documents that -several MCP stdio processes and the REST daemon share one DB file, each with its own sweep -task, while the writer `Mutex` is only per-process. - -Separately, a sweep racing `forget` re-inserts a vector for a deleted id. That vector can then -attach to a new memory that reuses the rowid (`INTEGER PRIMARY KEY` without `AUTOINCREMENT`). -That race already existed, but the fix below closes it too. - -**Fix:** Make the backfill write conditional on the row still being the pending row whose -content was embedded. Do it in the same writer transaction: + +### WR-02: README promises a JSON `{"error"}` body for every REST client error, but `POST /api/memories` and `POST /api/search` still return axum's plain-text rejections (and 422/415, not 400) + +**File:** `README.md:436-438`, `crates/agent-memory/src/rest/handlers.rs:194-197` (`store_handler`), `:257-260` (`search_handler`) + +**Issue:** 04-07 added this sentence: "Every client error, including a malformed path segment … or query parameter …, returns a JSON `{"error": "..."}` body." + +That is not true for the two POST routes that kept the bare `Json` extractor: + +- `POST /api/memories` with `{}` (missing `content`) returns axum's `JsonRejection`: **422** `text/plain` "Failed to deserialize the JSON body into the target type: missing field `content`". +- `POST /api/search` with a non-JSON body or no `Content-Type` returns **400/415** `text/plain`. + +A client that parses `error` from every non-2xx response, as the README now invites, fails on these routes. The status codes also contradict the README's 400-for-client-input story. + +The handlers are v0.0.1 code, but the README claim is new in this gap closure. It turns a known inconsistency into a documented contract that does not hold. + +**Fix:** Take the rejection explicitly, as `update_handler` / `link_handler` already do. Add one real-HTTP row per route to `malformed_path_and_query_params_get_the_json_error_body`. ```rust -fn insert_embedding(&self, memory_id: i64, expected_content: &str, embedding: Vec) - -> Result { - let mut conn = self.writer.lock().map_err(|_| MemoryError::Internal("writer lock poisoned".into()))?; - let tx = conn.transaction()?; - let claimed = tx.execute( - "UPDATE memories SET embedding_status = 1 \ - WHERE id = ?1 AND embedding_status = 0 AND content = ?2", - params![memory_id, expected_content], - )?; - if claimed == 0 { - return Ok(false); // updated, already embedded, or forgotten: skip; tx rolls back - } - let blob: &[u8] = bytemuck::cast_slice(&embedding); - tx.execute("DELETE FROM vec_memories WHERE memory_id = ?1", params![memory_id])?; - tx.execute("INSERT INTO vec_memories(memory_id, embedding) VALUES (?1, ?2)", params![memory_id, blob])?; - tx.commit()?; - Ok(true) +pub async fn store_handler( + State(state): State>, + body: Result, JsonRejection>, +) -> Result { + let Json(req) = body.map_err(|r| ApiError::BadRequest(r.body_text()))?; + // ... } +// same for search_handler ``` -Pass the content from `pending_embeddings` through `sweep()`. Add a test: pending row, then -the sweep reads it, then `update` with new content, then the backfill completes. Assert that -KNN on the new content's vector finds the row and KNN on the old content's vector does not. +Alternatively, narrow the README sentence to the routes that actually comply. -## Warnings +## Info -### WR-01: Concurrent first open after an upgrade can fail startup, or leave a "pre-0003" backup that is really schema v3 - -**File:** `crates/agent-memory-core/src/store/sqlite.rs:224-252`, `crates/agent-memory-core/src/store/sqlite.rs:140-189` -**Issue:** `open()` has three unlocked steps: probe `user_version`, write the backup, then -`to_latest()`. rusqlite_migration 2.5 reads `user_version` outside its deferred transaction -(`goto` → `goto_up`, lib.rs:713-651). 0003 is the first migration ever applied to deployed -v0.0.1 databases, and several processes share one DB by design (an MCP stdio server per editor -session plus the REST daemon). That makes simultaneous first opens after `brew upgrade` a -realistic scenario: - -- **Startup failure:** A and B both probe version 2. A migrates and commits. B's `CREATE TABLE - memory_links` then fails with `table memory_links already exists`, or `SQLITE_BUSY_SNAPSHOT`. - The transaction rolls back, but B's server fails to start. -- **Silently wrong backup:** B probes 2, A migrates to 3, and only then does B run - `pre_migration_backup`. B deletes A's valid `*.backup-pre-0003-*` file and writes a copy of - the *already-migrated* v3 database under the "pre-0003" name. B's `to_latest` is then a - no-op. There are no down-migrations (migrations.rs:6-8), so the backup is the only rollback - path, and it is now wrong. - -**Fix:** Run probe, backup and migrate under one exclusive lock, and re-read the version under -it. For example: `BEGIN IMMEDIATE` on the writer, re-read `user_version`, skip the backup if it -is no longer pending, then `COMMIT` and migrate. A cheaper alternative: write the backup to a -temp name, open it, check that its `user_version == db_version`, then rename it into place and -remove the stale backups. Treat a `to_latest` failure caused by the version having moved to -`LATEST` as success. - -### WR-02: Patch DTOs silently ignore non-patchable or misspelled fields - -**File:** `crates/agent-memory/src/mcp.rs:154-186`, `crates/agent-memory/src/rest/handlers.rs:101-127` -**Issue:** Neither `UpdateArgs` nor `UpdateBody` uses `deny_unknown_fields`. Two cases: - -- `{"content": "x", "type": "DECISION"}` returns 200 with the type unchanged. `type` and - `source` are documented as immutable, but the caller is told the update succeeded. -- A typo such as `{"content": "x", "ttl_sec": 60}` updates the content and silently drops the - TTL. - -With merge-patch semantics, a silently dropped member is a silent partial write. This violates -the seam's "bad input is always the client tier" rule. - -**Fix:** Add `#[serde(deny_unknown_fields)]` to both DTOs. On MCP, `id` is a declared field, -so it is fine. On REST, the explicit `JsonRejection` handling already turns this into a 400. -Add one test per transport for `{"content": "x", "type": "DECISION"}` returning -`invalid_params` / 400. - -### WR-03: `relates_to` is documented as symmetric but stored and deduplicated as directed - -**File:** `crates/agent-memory-core/src/store/sqlite.rs:710-714, 719-728, 856-905`, `crates/agent-memory-core/src/domain.rs:77` -**Issue:** The primary key is `(from_id, to_id, kind)`, so `link(a, b, relates_to)` and -`link(b, a, relates_to)` create two edges for one symmetric relation. Consequences: - -- Expansion lists the neighbor twice, once `out` and once `in`, and both copies count against - `MAX_EXPANDED_LINKS`. -- `unlink(a, b, relates_to)` reports `unlinked: true` while `b -> a` keeps the "symmetric" - relation alive. -- The D-02 idempotency promise ("duplicate link is a no-op") fails for the only kind whose - direction is meaningless. - -**Fix:** Canonicalise `relates_to` endpoints at the service seam: store `(min(a,b), max(a,b))` -for `link` and look it up the same way for `unlink`. Optionally, dedupe by neighbor id in -`related()`. Document that the direction shown for `relates_to` is not meaningful. - -### WR-04: `memory_list` with `expand_links` and no `limit` is unbounded, contradicting the documented T-04-11 bound - -**File:** `crates/agent-memory-core/src/service.rs:194-208`, `crates/agent-memory-core/src/domain.rs:316-321` -**Issue:** The `MAX_EXPANDED_LINKS` doc claims that, together with `MAX_LIMIT`, it "bounds an -expanded response at 200 × 10 summaries (T-04-11)". But `list` with an omitted `limit` stays -unlimited (Pitfall 6). An expanded list therefore returns every memory plus up to 10 neighbors -each, and runs one `related()` query per row inside a single `spawn_blocking`. So the threat -mitigation the plan relies on does not hold for `GET /api/memories?expand_links=true`. - -**Fix:** When `expand_links` is set and `limit` is `None`, either apply `DEFAULT_SEARCH_LIMIT` -(or `MAX_LIMIT`) or reject with `InvalidArgument` ("limit is required with expand_links"). -Correct the doc comment and the README accordingly. +### IN-01: Non-search surfaces still put raw SQLite text in client responses -## Info +**File:** `crates/agent-memory/src/rest/handlers.rs:165-180` (`ApiError::Internal(e.to_string())`), `crates/agent-memory/src/mcp.rs:35-50` + +**Issue:** `redact_search_error` (`service.rs:46-60`) covers `search` only. A storage fault in `store`, `list` (including `expand_links` → `related()`), `update`, `link`, `unlink`, `forget` or `/health` still sends `sqlite error: …` to the client. That text includes table and column names and engine detail. + +The README scopes its promise to search (`README.md:438-440`), and 04-05 lists this as deferred, so the code and docs agree. The risk is low: the server is loopback, single-user and unauthenticated by design. It is recorded here because this re-review was asked to look for remaining raw SQLite text. + +**Fix:** When the deferral is picked up, move the redaction into the two transport mappers. On the `Sqlite | Pool | Join | Migration` arm: log the error, then return a generic message. Keep `Internal` / `SchemaTooNew` as they are. + +### IN-02: Keyword mode: a punctuation-free question often finds nothing, while the same question with `?` finds results + +**File:** `crates/agent-memory-core/src/store/sqlite.rs:569-588` + +**Issue:** The retry only fires when FTS5 fails to *parse*. `what vector store do we use` is valid FTS5, an implicit AND of six words, so it usually matches nothing and returns `[]` in keyword mode. `what vector store do we use?` fails to parse, takes the OR fallback, and returns results. Adding a question mark makes the query broader. + +Hybrid mode hides this, because the semantic leg answers. With Ollama down, an agent that drops the `?` gets nothing. This follows from "valid FTS5 runs exactly as written", so it is a design consequence, not a regression from v0.0.1. It is still surprising, and the README example (`README.md:203-209`) implies the punctuation is incidental. + +**Fix:** Either document it ("an unpunctuated multi-word query is an AND of all its words"), or also run the sanitized OR form when the raw query returned zero rows and contains no explicit FTS5 operators or quotes. + +### IN-03: Stale "directed" wording after the WR-03 symmetric `relates_to` change + +**File:** `README.md:154`, `crates/agent-memory-core/src/service.rs:561`, `crates/agent-memory-core/src/service.rs:587-590`, `crates/agent-memory/src/rest/handlers.rs:135-144` + +**Issue:** These still describe every relation as a directed edge whose direction must match: + +- the tools table: "Link two memories with a typed, directed relation"; +- the `MemoryService::link` / `unlink` docs; +- the `LinkBody` doc and its `kind` field. + +The MCP descriptions, store-trait docs and README Relations section were updated, so the wording is now inconsistent. + +**Fix:** Reword them to "typed relation (`relates_to` is symmetric; `supersedes` / `caused_by` are directed)", matching `mcp.rs:418`. + +### IN-04: Backup edge cases: pid-only temp name, and a prune failure aborts startup after a valid backup is already in place + +**File:** `crates/agent-memory-core/src/store/sqlite.rs:140-145`, `crates/agent-memory-core/src/store/sqlite.rs:293` + +**Issue:** + +1. `backup_temp_path` makes the temp name unique by pid only. Two processes in different PID namespaces can both be pid 1. Example: the REST daemon in a container and another container sharing the same volume. Those two openers write Online Backups into the same temp file, and one renames it while the other is still writing. The result is a torn backup, or an `ENOENT` rename failure that aborts that opener. This case is rare for a Homebrew/binary tool. +2. When the rename succeeds but `prune_other_backups` cannot remove an older same-version backup (permissions, or an entry that is not a file), `open()` refuses to start, even though the new genuine backup is already in place. Refusing to migrate without a backup is the T-04-02 rule. Refusing to start because an *old* backup could not be deleted goes further than that rule requires. + +**Fix:** + +1. Add a random suffix to the temp name, for example `{pid}-{nanos}`, or create it with `O_EXCL` / `tempfile::Builder::new().tempfile_in(dir)`. +2. Log prune failures as warnings and return `Ok(Some(final_path))`. + +### IN-05: When a newer binary wins the upgrade race, the user gets a raw migration error instead of the `SchemaTooNew` message + +**File:** `crates/agent-memory-core/src/store/sqlite.rs:323-333` + +**Issue:** The scenario: a v3 binary probes 2 while a v4 binary migrates the same file to 4. + +- The v3 binary's backup copy is version 4, so it is discarded with `Ok(None)`. +- Its `to_latest` then fails with rusqlite_migration's `DatabaseTooFarAhead`. +- The re-probe reads 4, which is not `LATEST_SCHEMA_VERSION`, so the original migration error is returned. + +The user does not get the friendly D-13 text that names `brew upgrade`. The refusal itself is still correct. + +**Fix:** In the re-probe `match`, map `Ok(v) if v > LATEST_SCHEMA_VERSION` to `MemoryError::SchemaTooNew { db_version: v, binary_version: LATEST_SCHEMA_VERSION }`. + +### IN-06: Test name contradicts its assertion + +**File:** `crates/agent-memory/tests/tools.rs:418` (`memory_update_with_an_unknown_field_is_invalid_params_over_stdio`) + +**Issue:** The test asserts that there is **no** JSON-RPC `error` member and that `result.isError == true`. That is a tool error result, not `invalid_params`. 04-07 kept the plan's name on purpose, but anyone triaging a failure will be misled. -### IN-01: The pre-migration backup and the newer-schema refusal are not documented for users - -**File:** `README.md` (no section) -**Issue:** The README never mentions `backup`, `migration` or `upgrade`. Users will find -`memories.db.backup-pre-0003-YYYYMMDD` files next to their DB with no explanation, no -restore procedure, and no note that a newer DB refuses an older binary (D-11 to D-13). -**Fix:** Add a short "Upgrading" section covering the backup name pattern, the retention rule, -how to restore (stop the servers, copy the backup over `memories.db`, delete `-wal`/`-shm`), -and the newer-schema error message. - -### IN-02: Path and Query extractor rejections on the new routes return plain-text bodies, not `{"error": ...}` - -**File:** `crates/agent-memory/src/rest/handlers.rs:287-291, 364-367, 209-211` -**Issue:** `PATCH /api/memories/abc`, `DELETE /api/links/x/2/relates_to` and -`GET /api/memories?expand_links=1` hit axum's default `Path`/`Query` rejection, which returns a -plain-text body. The new handlers deliberately normalise `JsonRejection` to `{"error"}`, so the -error body shape is now inconsistent between body and path failures. -**Fix:** Take `Result, PathRejection>` / `Result, QueryRejection>` the same way -`update_handler` takes the JSON body, and map them to `ApiError::BadRequest`. - -### IN-03: The backup step deletes the previous same-version backup before the new one is written - -**File:** `crates/agent-memory-core/src/store/sqlite.rs:156-182` -**Issue:** If the new backup then fails (disk full, permissions), the previous pre-0003 backup -is already gone. The DB itself stays untouched because the open aborts, so this is low risk, -but the retention step is not atomic. -**Fix:** Write to `{prefix}{date}.tmp`, then rename it into place, and only then remove the -other `{prefix}*` entries. - -### IN-04: Stale comment refers to a "semantic leg" that no longer exists - -**File:** `crates/agent-memory-core/src/service.rs:289` -**Issue:** "Same order as the semantic leg" — the semantic-only path was removed. The fallback -warnings at L270-275 still say "semantic search unavailable", which is fine. The comment should -say "the hybrid path". -**Fix:** Reword the comment. - -### IN-05: `memory_update` can revive a row whose TTL has passed but which the sweep has not yet removed - -**File:** `crates/agent-memory-core/src/store/sqlite.rs:747-791` -**Issue:** `update` does not check `expires_at < now`. A patch with `ttl_secs: null` (or a new -TTL) on an expired row that has not been swept yet brings it back. This matches list/search, -which also show expired-but-unswept rows, but it should be a deliberate choice. -**Fix:** Either treat `expires_at < now` as not-found in `update`, or document the behaviour. - -### IN-06: Snippet truncation counts Unicode scalars, not grapheme clusters - -**File:** `crates/agent-memory-core/src/domain.rs:192-197` -**Issue:** The cut never splits a UTF-8 code point, but it can split a combining sequence or a -ZWJ emoji, leaving a dangling modifier before the `…`. This is cosmetic. -**Fix:** Accept and document it, or cut at the last whitespace before the limit (D-05 leaves -the choice open). +**Fix:** Rename it to `memory_update_with_an_unknown_field_is_a_tool_error_over_stdio`. --- diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-VERIFICATION.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-VERIFICATION.md index 246d42f..2161493 100644 --- a/.planning/phases/04-memory-update-relations-hybrid-search/04-VERIFICATION.md +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-VERIFICATION.md @@ -1,181 +1,230 @@ --- phase: 04-memory-update-relations-hybrid-search -verified: 2026-10-23T18:52:00Z -status: gaps_found -score: 13/15 must-haves verified +verified: 2026-10-23T20:19:46Z +status: passed +score: 28/28 must-haves verified behavior_unverified: 0 overrides_applied: 0 -gaps: - - truth: "A malformed MATCH query is a 400/invalid_params in hybrid mode exactly as in keyword mode — the keyword leg's InvalidQuery propagates (plan 04-04 must-have #3; Phase 3 taxonomy: client input is never a 500)" - status: failed - reason: >- - map_fts_query_error only recognises the substrings "fts5: syntax error" and "unterminated string". - FTS5 reports column-filter-shaped input (a hyphen or a colon: `sqlite-vec`, `auth: tokens`) as - "no such column: ", which falls through to MemoryError::Sqlite, so the result is MCP -32603 - internal_error / REST 500 and the raw SQLite message ("sqlite error: no such column: vec") leaks to the - client. Keyword mode has had this since v1.0, but this phase sends every Ollama-up search through the - keyword leg. v0.0.1 answered these queries semantically; HEAD returns a server fault. Observed live on - both v0.0.1 (/opt/homebrew/bin/agent-memory) and the HEAD binary with a real Ollama. - artifacts: - - path: "crates/agent-memory-core/src/store/sqlite.rs" - issue: "map_fts_query_error (line ~333) marker list misses the step-time 'no such column' FTS5 parse failure; it is used by keyword_candidates (hybrid leg) and search (keyword path)" - - path: "crates/agent-memory-core/tests/hybrid.rs" - issue: "a_malformed_match_is_invalid_query_in_hybrid_mode_exactly_as_in_keyword_mode covers only '\"unbalanced', 'AND', 'what is the plan?', with no hyphen or colon case" - missing: - - "Classify the step-time FTS5 'no such column' failure (the SQL is static, so at step time it can only come from the MATCH string) as MemoryError::InvalidQuery, or resolve it through the sanitisation option in the decision item below" - - "Add 'sqlite-vec' and 'auth: tokens' to the malformed-query table test for both hybrid and keyword modes, plus a REST and MCP assertion that the tier is 400 / invalid_params with no 'sqlite error' text in the body" -human_verification: - - test: "Decide the contract for natural-language queries when Ollama is up. Run memory_search with 'what vector store do we use?' (or any query containing ? , ' - :) against a store holding 'We chose sqlite-vec for local vector similarity search'" - expected: "Developer decision. Today it returns invalid_params (400). v0.0.1 returned the right memory semantically. This was a deliberate planner decision (T-04-15), documented in README 'Behavioral changes in v0.1.0' item 3, but it cuts against the phase outcome 'I find the right memory even when the phrasing differs'. Options: (a) accept it as is and record an override; (b) sanitise or quote the keyword leg's input (e.g. tokenise into quoted OR'd terms) so natural-language punctuation can never fail the leg, which also fixes the gap above; (c) serve the KNN leg alone on a keyword-leg parse error (the plan rejected this as mode-probing)" - why_human: "Two locked contracts conflict: the plan's T-04-15 classification parity against the roadmap goal's paraphrase-recall outcome. This is a product decision, not a code defect a grep can settle" +re_verification: + previous_status: gaps_found + previous_score: 13/15 + gaps_closed: + - "Truth #14: hyphen/colon queries (sqlite-vec, auth: tokens) no longer produce a 500 / internal_error with leaked SQLite text. They now return results in both modes. The original wording ('a malformed MATCH is 400 in hybrid mode') is superseded by user decision option-a, which is recorded in the 04-05 plan and in STATE.md [04-05]" + - "Truth #15: the human-decision item is resolved by option-a. Natural-language and punctuated queries succeed in hybrid and keyword mode, identically with Ollama up or down. Verified live over MCP and REST" + gaps_remaining: [] + regressions: [] + superseded_must_haves: + - must_have: "A malformed MATCH query is a 400/invalid_params in hybrid mode exactly as in keyword mode (04-04 must-have #3)" + superseded_by: "User decision option-a, 2026-10-23" + recorded_in: + - ".planning/phases/04-memory-update-relations-hybrid-search/04-05-PLAN.md section 'Locked user decision'" + - ".planning/STATE.md decision [Phase 04] 04-05 (line 118)" --- -# Phase 4: Memory Update, Relations & Hybrid Search — Verification Report +# Phase 4: Memory Update, Relations & Hybrid Search Verification Report -**Phase Goal:** As an agent, I want to update existing memories, link related ones, and get hybrid keyword+semantic recall, so that my memory stays current and connected and I find the right memory even when the phrasing differs. -**Verified:** 2026-10-23T18:52:00Z -**Status:** gaps_found -**Re-verification:** No, initial verification +**Phase goal:** As an agent, I want to update existing memories, link related ones, and get hybrid keyword+semantic recall, so that my memory stays current and connected and I find the right memory even when the phrasing differs. +**Verified:** 2026-10-23T20:19:46Z +**Status:** passed +**Re-verification:** Yes. This follows gap-closure plans 04-05, 04-06 and 04-07. The previous run was gaps_found at 13/15. -Evidence sources: -- Code read at HEAD `bff047f`, with a clean working tree under `crates/`. -- The phase test binaries (`update`, `relations`, `hybrid`, `migration_hygiene`, `import`): 56 passed, 0 failed, 1 ignored (the one-shot fixture generator). -- Orchestrator gates: fmt, clippy `-D warnings`, build, and `cargo test --workspace` (170 passed / 0 failed / 2 ignored). -- **Live sessions** of the HEAD binary over MCP stdio against a real local Ollama (`nomic-embed-text`) and against an unreachable URL. -- A **real v0.0.1 database** written by the installed Homebrew `agent-memory 0.0.1`, then upgraded by HEAD. +## Evidence -All scratch data lives in `/tmp/v04live-data` and `/tmp/v04real-data`. Nothing in the repo was modified. +- **Code:** read at HEAD `8b2c762`. The working tree under `crates/` and `README.md` is clean. +- **Build:** the binary was rebuilt from HEAD (`cargo build -p agent-memory`). +- **Tests I ran myself:** + - phase test binaries `hybrid` 21, `backfill_race` 7, `relations` 27, `migration_hygiene` 8 (+1 ignored generator), `update` 9: all pass; + - `agent-memory-core --lib store::sqlite`: 12 pass; + - the stdio e2e `tools`: 6 pass; + - the named MCP tests `memory_update_*` and `memory_search_storage_failure_is_internal_error_without_sqlite_text`: pass. +- **Orchestrator gates at HEAD:** fmt, clippy `-D warnings`, `cargo test --workspace` (211 passed / 0 failed / 2 ignored). +- **Live sessions:** the HEAD binary over MCP stdio and REST, against the real local Ollama (`nomic-embed-text`) and against an unreachable URL (`http://127.0.0.1:9`). +- **Scratch data:** all under `/tmp/v04re-data` and `/tmp/v04re-scripts`. Nothing in the repo was modified. -## Goal Achievement +## Goal Achievement — User Flow Coverage (MVP mode) + +| Story step | Expected | Evidence | Status | +|---|---|---|---| +| Update an existing memory | Patch changes it in place; the old meaning stops matching; unknown or immutable fields are refused | Live: `memory_update {id:4, content}` returned the full view with `embedding_status:1`. `{type}`, `{ttl_sec}` and `{source}` were refused, naming the field, and the row was unchanged. Unknown id 999 gave a clean not_found | ✓ | +| Link related memories | Typed link; symmetric `relates_to`; 1-hop expansion; cascade on forget | Live: link(5,2) followed by link(2,5) `relates_to` produced one edge. Expansion listed it once from each side. unlink(2,5) removed it, and the response echoed the request order. A pair of directed `supersedes` edges (6→1 and 1→6) stayed two edges. forget(6) left 0 edges | ✓ | +| Find the right memory even when phrasing differs | Hybrid recall with punctuation-tolerant queries, Ollama up or down | Live, 25-query table (below): every query succeeded in both modes, with 0 errors and no SQLite text in any body or log | ✓ | ### Observable Truths | # | Truth | Status | Evidence | |---|-------|--------|----------| -| 1 | SC1: `memory_update` partial patch (content/tags/scope/TTL) changes the memory in place; a content update re-mirrors FTS and invalidates/re-embeds the vector in the same writer tx; an outage gives `embedding_status=0` plus sweep backfill; the old meaning stops matching; an unknown id gets a clean not-found | ✓ VERIFIED | `SqliteStore::update` (sqlite.rs:731-854) does one lock and one tx. It runs `UPDATE memories SET ...` in place (never DELETE+INSERT); the `memories_au` trigger re-mirrors FTS; `DELETE FROM vec_memories` plus a re-insert, or status 0, happens in the same tx. Tests pass: `content_update_re_embeds_so_the_old_meaning_stops_matching` (it uses 4 distractors and also checks that the FTS old word is gone), `embed_outage_during_update_falls_back_to_pending_and_sweep_backfills`, `updating_an_unknown_id_is_a_clean_not_found`. Live: the update returned the full view with `embedding_status:1`. A paraphrase query that ranked id 1 first fell to rank 3 after its content changed. id 999 returned `{"id":999,"reason":"not_found","updated":false}` | -| 2 | SC2: `memory_link`/`memory_unlink` handle flat typed relations; search and list can expand 1-hop; forget and TTL expiry cascade links with no orphans; no graph model | ✓ VERIFIED | 0003 `memory_links` has a PK of (from,to,kind), FK `ON DELETE CASCADE` on both columns, `CHECK from!=to`, and `idx_links_to`. `related()` is a single non-recursive 1-hop SQL query. Explicit orphan deletes exist in both `forget` and `sweep_expired` (lines 669, 956). There are 19 relations tests, including pragma-less orphan healing. Live: a link of 1→2 `supersedes` expanded with `superseded_by`/`in` on id 2. After `forget(1)`, the DB held 0 orphan edges | -| 3 | SC3: with Ollama available, search returns RRF-fused results (k=60, duplicate ids sum) labelled `hybrid`; with Ollama unreachable, it degrades to keyword exactly as v1.0 with an unchanged envelope | ✓ VERIFIED (fusion and degrade mechanics; see #15 for the query-class caveat) | `RRF_K = 60.0` and `rrf_fuse` (service.rs:78-105) use 1-based ranks with summed contributions. The goldens pass (2/62 beats 1/61, duplicate ids sum, id tie-break). Public `fn search` is **byte-identical** to the pre-phase `907161c` (diffed), and the service keyword arm differs only by opt-in expansion. Live with Ollama up: `search_mode: hybrid`, keys `[results, search_mode]`. Live with Ollama down on the same DB: `search_mode: keyword` with the same keys and one fallback warning per search | -| 4 | SC4: hybrid ranking is decay-aware (fresh beats stale at equal fused rank); access bumps go only to post-truncation returned ids | ✓ VERIFIED | Decay is multiplicative and applied once after fusion (service.rs:437-455). `at_equal_fused_rank_the_fresh_memory_outranks_the_stale_one` uses a genuine cross-rank tie (1/61 + 1/62 each), in both orientations, so id order cannot win. `only_the_returned_ids_are_bumped_never_the_fusion_candidates` uses 20 candidates in both legs at limit 5 and finds exactly the 5 returned rows bumped. `spawn_bump` is called after `take(limit)` and before expansion | -| 5 | SC5: upgrading a real v0.0.1 DB applies 0003 only after a pre-migration backup; 0001/0002 SQL provably unchanged; a newer-schema DB gets a friendly error, not a crash | ✓ VERIFIED | Probe, then refuse, then pragmas, then backup, then migrate (sqlite.rs:224-252). `git diff v0.0.1` on 0001/0002 is empty. The fixture is committed, and the divergence test compares normalized `sqlite_master`. **Live with the real v0.0.1 binary:** the DB written by Homebrew 0.0.1 was opened by HEAD and produced `real.db.backup-pre-0003-20261023` (user_version=2, 2 rows, no memory_links); the live DB went to user_version=3 with data intact, and hybrid search, link and update all worked. **Live too-new:** `user_version=99` gave exit 1, 0 bytes on stdout, and the D-13 message on stderr naming 99, 3 and the brew command. The DB's shasum was unchanged | -| 6 | Re-importing a `source: None` draft is idempotent (CR-01) | ✓ VERIFIED | `source IS ?1` (sqlite.rs:496); `.as_deref()` in service import; test `reimporting_a_sourceless_draft_is_idempotent` passes | -| 7 | A poisoned writer lock is 500/internal_error, never 404 (WR-02) | ✓ VERIFIED | 9 `Internal("writer lock poisoned")` sites, including the new link/unlink/update writers. `MemoryError::NotFound`/`ApiError::NotFound` have 0 references. Mapper tier unit tests pass (full suite) | -| 8 | Merge-patch semantics: absent = unchanged, null = clear (ttl/tags/scope), content null or empty = 400 | ✓ VERIFIED | `validate_patch` runs before the embed. Matrix tests at core (`tags_and_scope_patch_matrix_absent_null_value`, `ttl_patch_matrix_...`) and on both transports from raw JSON (mcp.rs:669/701/753, handlers.rs PATCH tests). Patch DTOs set `skip_serializing_if`, so the schema carries no `default: null` | -| 9 | The update response is the full post-update view plus `embedding_status`, and the update bumps `last_accessed` | ✓ VERIFIED | `UpdatedMemory` flattens `MemoryView` and adds `embedding_status`. The UPDATE statement folds in `last_accessed = now, access_count + 1`. Tests: `update_bumps_freshness_by_exactly_one_access`, `update_returns_the_full_post_update_view_with_embedding_status`. Live response included `embedding_status` | -| 10 | Link edge cases: a duplicate is an idempotent no-op; a self-link is 400; an unknown kind is 400 listing the allowed values; a missing endpoint is a not-found naming the id | ✓ VERIFIED | `INSERT OR IGNORE`; endpoint pre-check inside the tx (no FK-violation 500); `validate_link_endpoints` at the seam. Relations tests and MCP/REST transport tests pass | -| 11 | Expansion covers both directions with natural reverse labels and a cap of `MAX_EXPANDED_LINKS = 10`, most recent first | ✓ VERIFIED | `related()` uses `ORDER BY l.created_at DESC ... LIMIT ?2`. Tests: `expansion_is_capped_and_the_most_recent_edges_win`, `expansion_follows_one_edge_both_ways_with_natural_labels`. Live: `supersedes`/`out` and `superseded_by`/`in` | -| 12 | With `expand_links` absent, the search and list JSON is byte-identical to v1.0 (no `related` key) | ✓ VERIFIED | `#[serde(skip_serializing_if = "Option::is_none")]` on `related`. Golden test `flag_less_envelopes_are_byte_identical_to_v1_and_never_mention_related`. Live: flag-less results carried no `related` | -| 13 | Expanded neighbors never receive access bumps | ✓ VERIFIED | Expansion runs after `spawn_bump(returned ids)`. Tests `expanded_neighbors_are_never_access_bumped` and `hybrid_expansion_runs_after_the_bump_and_never_bumps_neighbors` pass | -| 14 | A malformed MATCH is 400/invalid_params in hybrid mode exactly as in keyword mode (plan 04-04 must-have) | ✗ FAILED | Holds for the `fts5: syntax error` / `unterminated string` class only. **Live HEAD with Ollama up:** `memory_search {"query":"sqlite-vec"}` returned `-32603 internal_error "sqlite error: no such column: vec"`. The REST mapper sends `MemoryError::Sqlite` to 500. Classification parity with keyword mode holds (both modes give 500), but the 400 tier does not, and the SQL detail leaks. **Live v0.0.1 with Ollama up:** the same query returned the right memory (`search_mode: semantic`), so on the Ollama-up path this is a regression | -| 15 | Goal outcome: "I find the right memory even when the phrasing differs" (derived from the user story's so-that clause) | ? UNCERTAIN (decision needed) | Paraphrases without punctuation work live (for example, "which datastore keeps client information" finds "Postgres is our primary relational database for customer records" at rank 1 with no keyword overlap). Paraphrases with ordinary punctuation fail: `what vector store do we use?` gives invalid_params at HEAD, while v0.0.1 found the right memory. This is deliberate (T-04-15) and documented in the README, but it conflicts with the outcome clause. See Human Verification | - -**Score:** 13/15 truths verified (0 present-but-behavior-unverified, 1 failed, 1 uncertain) - -### Required Artifacts +| 1 | SC1: `memory_update` partial patch changes the memory in place. A content change re-mirrors FTS and re-embeds in the same writer tx. An embed outage gives status 0 plus backfill. The old meaning stops matching. An unknown id gets a clean not-found | ✓ VERIFIED (regression) | `update.rs` 9/9 pass. Live update returned `embedding_status:1`. Live id 999 returned `{"id":999,"reason":"not_found","updated":false}` on MCP, and 404 with the same body on REST | +| 2 | SC2: link/unlink handle flat typed relations; search/list can expand 1-hop; forget and TTL cascade with no orphans | ✓ VERIFIED (regression) | `relations.rs` 27/27 pass, including the forget/TTL cascade and orphan-healing tests. Live forget(6) left `memory_links` count = 0 | +| 3 | SC3: with Ollama up, RRF k=60 hybrid labelled `hybrid`; with Ollama down, keyword exactly as v1.0 with an unchanged envelope | ✓ VERIFIED (regression) | Live: up gave `search_mode:"hybrid"`, down gave `"keyword"`, both with keys `{search_mode, results}`. `hybrid.rs` RRF goldens pass | +| 4 | SC4: decay-aware hybrid ranking; bumps go only to post-truncation returned ids | ✓ VERIFIED (regression) | `hybrid.rs` passes (equal-fused-rank fresh-vs-stale, returned-ids-only bump). The service still bumps after `take(limit)` and before expansion | +| 5 | SC5: real v0.0.1 upgrade only after a pre-migration backup; 0001/0002 frozen; newer schema gives a friendly error | ✓ VERIFIED (regression) | `git diff v0.0.1 -- sql/0001* sql/0002*` is empty. The fixture upgrade live produced a backup with user_version=2 and 2 rows. Live `user_version=99`: exit 1, empty stdout, D-13 message on stderr naming 99/3 and the brew command, shasum unchanged | +| 6 | Re-importing a sourceless draft is idempotent (CR-01, old) | ✓ VERIFIED (regression) | `source IS ?1` unchanged; covered by the workspace suite | +| 7 | A poisoned writer lock is 500/internal, never 404 | ✓ VERIFIED (regression) | `Internal("writer lock poisoned")` at every writer, including the rewritten `insert_embedding`. Mapper tests pass | +| 8 | Merge-patch: absent = unchanged, null = clear, null or empty content = 400 | ✓ VERIFIED (regression) | MCP matrix tests (`memory_update_tags_and_scope_matrix_absent_null_value`, `..._ttl_matrix...`) pass | +| 9 | The update response is the full post-update view plus `embedding_status`; the update bumps freshness | ✓ VERIFIED (regression) | Live response carried `embedding_status` and a new `last_accessed` | +| 10 | Link edge cases: duplicate is a no-op, self-link is 400, unknown kind is 400, missing endpoint is a not-found naming the id | ✓ VERIFIED (regression) | Live: link(5,99) returned `missing_id:99`. REST link(9,1) returned 404 with `missing_id:9` (request's from_id first). REST unlink with `bogus` kind returned 400 listing the allowed kinds | +| 11 | Expansion goes both directions with natural reverse labels, a cap of 10, most recent first | ✓ VERIFIED (regression) | Live: `supersedes`/`out` and `superseded_by`/`in` | +| 12 | With the flag absent, the JSON is byte-identical to v1.0 (no `related` key) | ✓ VERIFIED (regression) | Live flag-less search results carried no `related` member | +| 13 | Expanded neighbours are never access-bumped | ✓ VERIFIED (regression) | relations/hybrid tests pass | +| 14 | **(previously FAILED)** Hyphenated or colon-bearing queries (`sqlite-vec`, `auth: tokens`, `re-embed`) succeed in both modes. Never a 500 / internal_error, and no SQLite text leaks. The original 400 wording is superseded by option-a | ✓ VERIFIED | `is_match_input_error` (sqlite.rs:519) now classifies the step-time `no such column` FTS5 class. `match_with_sanitized_retry` (sqlite.rs:569) is used by both `search` (:866) and `keyword_candidates` (:908). **Live MCP, Ollama up:** `sqlite-vec` → hybrid [1,…]; `auth: tokens` → hybrid [3,…]. **Ollama down:** `sqlite-vec` → keyword [1]; `auth: tokens` → keyword [3]. **Live REST:** 200 in both modes. `grep "sqlite error\|no such column"` over every serve log returned 0 | +| 15 | **(previously UNCERTAIN)** Goal outcome under option-a: natural-language queries with punctuation succeed in both modes and return the matching memory first | ✓ VERIFIED | **Live, both modes, correct memory at rank 1:** `what vector store do we use?` → 1, `don't retry` → 4, `auth, tokens` → 3, `C++` → 5, `how often do credentials get rotated?` → 3. Pure paraphrase with no keyword overlap: `which datastore keeps client information` → hybrid [2,…] (keyword mode returns [] as expected; the recall comes from the semantic leg) | +| 16 | Valid structured FTS5 syntax still runs exactly as written (raw first) | ✓ VERIFIED | `structured_fts5_syntax_still_runs_raw` passes. Live: `sqlite AND vec`, `"vector similarity"`, `vec*` → [1] in keyword mode. `content: sqlite` ran raw as a real column filter → [1] | +| 17 | A tokenless query: hybrid answers semantic-only, keyword mode returns an empty list, never an error | ✓ VERIFIED | `sanitize_fts_query` returns `None` → `Ok(vec![])`. Live: `""`, `"`, `?!`, `*`, `^` → hybrid returned 3 results (KNN only); keyword returned `[]` with HTTP 200 | +| 18 | Keyword-leg classification is identical with Ollama up or down (one shared helper) | ✓ VERIFIED | Both store paths call `match_with_sanitized_retry(&args.query, run)`. `queries_fts5_cannot_parse_succeed_identically_in_both_modes` passes. Live: no query errored in either mode | +| 19 | No MCP or REST search body ever carries raw SQLite text. A storage failure is a generic `SEARCH_STORAGE_FAILURE` (500 / internal_error), with the detail only in stderr | ✓ VERIFIED | `pub async fn search` → `.map_err(redact_search_error)` (service.rs:283). The redaction match has no catch-all arm and covers Sqlite/Pool/Join/Migration. Tests pass: `memory_search_storage_failure_is_internal_error_without_sqlite_text` (MCP, both modes, FTS table renamed away underneath a live server), the REST `search_storage_failure_is_500_without_sqlite_text`, and the hybrid.rs redaction rows. *A live induced-failure run was blocked by the project's DROP-statement hook, so the evidence here is test-based* | +| 20 | CR-03: a sweep backfill whose row changed, was forgotten, or was claimed by a racing sweep writes nothing. A failed racing update leaves the row pending for its NEW text | ✓ VERIFIED | `insert_embedding` (sqlite.rs:687) first runs a guarded `UPDATE … WHERE id=?1 AND embedding_status=0 AND content=?2` in the same tx as the vector write, and returns `false` with no write when 0 rows match. `service.sweep` passes the exact embedded `&content` (service.rs:698) and counts `skipped_stale`. `backfill_race.rs` 7/7 pass, deterministic: embedded update, unembedded update, forget, double backfill, gated-embedder mid-backfill update, and a failed update followed by the next sweep | +| 21 | `relates_to` is one symmetric relation: stored canonically, listed once per side, unlinkable from either end, with the request order echoed on the wire | ✓ VERIFIED | `canonical_endpoints`/`is_symmetric` (domain.rs:127-149). link stores `(min,max)`; unlink matches both orientations when `?4=1`. Live: link(5,2) and link(2,5) each echoed their own request order. Expansion showed id 2 once on 5's side. unlink(2,5) removed the edge (REST and MCP). Tests pass: `relates_to_linked_both_ways_is_one_relation_listed_once_from_each_side` and `relates_to_is_stored_with_the_smaller_id_first` | +| 22 | `supersedes`/`caused_by` keep directed identity; a missing endpoint is named in request order | ✓ VERIFIED | Live: 6→1 and 1→6 `supersedes` were both stored and both listed on id 6 (`supersedes` out, `superseded_by` in). unlink(1,6) removed only that one. REST link(9,1) named 9 | +| 23 | The `MAX_EXPANDED_LINKS` doc and the README state the real expanded bound (WR-04 doc correction) | ✓ VERIFIED | domain.rs:363-369 ("A list without a limit returns every row … deferred API-contract"), and a matching README note | +| 24 | `memory_update` / `PATCH` with an unknown or immutable field is rejected, naming the field, and nothing is written | ✓ VERIFIED (protocol-tier note) | `#[serde(deny_unknown_fields)]` on `UpdateArgs` (mcp.rs:169) and `UpdateBody` (handlers.rs:107). Live REST: `{type}`, `{ttl_sec}`, `{source,content}` → 400 `{"error":"… unknown field \`ttl_sec\`, expected one of …"}`, and row 2's content and type were unchanged. Live MCP: rejected as a tool error result (`isError:true`, "failed to deserialize parameters: unknown field `type` …") rather than a JSON-RPC -32602. That is rmcp 1.8's design; it is recorded in STATE.md [04-07] and the 04-07 SUMMARY. The intent (never silently drop a member, name the field, write nothing) holds | +| 25 | Malformed REST path or query params return a JSON `{"error"}` body with the extractor's status | ✓ VERIFIED | Live: `PATCH /api/memories/abc`, `DELETE /api/memories/abc`, `DELETE /api/links/x/2/relates_to`, `GET /api/memories?expand_links=1`, `?limit=abc` all returned 400 `application/json` `{"error":…}` | +| 26 | Two processes opening the same pre-0003 DB at once both start, and exactly one valid pre-migration backup with the OLD schema version survives | ✓ VERIFIED | `pre_migration_backup(&writer, path, db_version)` (sqlite.rs:409) writes a temp file, checks the version, renames, then prunes. `migrate_tolerating_concurrent_upgrade` re-probes the version on failure. **Live:** 3 rounds of two simultaneous `serve-rest` processes on a copy of the v0.0.1 fixture. Every round, both returned `/health` 200; the live DB reached uv=3; exactly one `m.db.backup-pre-0003-*` existed with uv=2, 2 rows and no `memory_links`. The log showed "another process completed the schema upgrade first". `two_concurrent_first_opens_of_a_v0_0_1_database_both_succeed_with_one_genuine_backup` and `a_real_lost_migration_race_is_absorbed` pass | +| 27 | A failed backup write never removes the previous valid backup (replace, then prune) | ✓ VERIFIED | Pruning happens only after a successful rename. Unit tests pass: `a_failed_backup_write_never_removes_the_previous_backup`, `a_backup_that_cannot_be_moved_into_place_keeps_the_previous_one`, `a_valid_backup_replaces_then_prunes_the_stale_one`, `a_backup_of_an_already_migrated_database_is_discarded_and_keeps_the_valid_one` | +| 28 | The README documents backup location, restore and the newer-schema refusal (IN-01) | ✓ VERIFIED | README `## Upgrading` (line 84): `memory.db.backup-pre-NNNN-YYYYMMDD`, the `cp` restore step, WAL/SHM notes, and the brew-upgrade message | + +**Score:** 28/28 truths verified. None are present-but-behavior-unverified: every behaviour-dependent truth (20, 26, 27) has a passing deterministic test, and 26 was also exercised live. + +### Live search decision table (MCP stdio, 25 queries, `limit: 3`) + +| Query | Ollama up | Ollama down | +|---|---|---| +| `sqlite-vec` | hybrid [1,2,3] | keyword [1] | +| `auth: tokens` | hybrid [3,4,2] | keyword [3] | +| `re-embed` | hybrid [2,1,4] | keyword [] | +| `what vector store do we use?` | hybrid [1,2,5] | keyword [1] | +| `don't retry` | hybrid [4,3,6] | keyword [4] | +| `auth, tokens` | hybrid [3,4,2] | keyword [3] | +| `C++` | hybrid [5,2,1] | keyword [5] | +| `"unbalanced` / `AND` / `foo OR` / `NEAR(` / `(sqlite` | hybrid (all ok) | keyword [] / [5] / [] / [] / [1] | +| `?!` / `*` / `` / `"` / `^` (tokenless) | hybrid, semantic-only | keyword [] | +| `sqlite AND vec` / `"vector similarity"` / `vec*` | hybrid [1,…] | keyword [1] | +| `which datastore keeps client information` | hybrid [2,1,3] | keyword [] | +| `how often do credentials get rotated?` | hybrid [3,2,4] | keyword [3] | + +There were 0 RPC errors and 0 tool errors. REST gave 200 for every query sampled in both modes. + +### Required Artifacts (gap-closure plans) | Artifact | Expected | Status | Details | -|----------|----------|--------|---------| -| `crates/agent-memory-core/sql/0003_relations.sql` | memory_links schema | ✓ VERIFIED | PK(from,to,kind), 2× ON DELETE CASCADE, CHECK, WITHOUT ROWID, idx_links_to | -| `crates/agent-memory-core/src/store/migrations.rs` | 0003 in chain + `LATEST_SCHEMA_VERSION` | ✓ VERIFIED | `= 3`, with a test that the const equals what `to_latest` writes | -| `crates/agent-memory-core/tests/migration_hygiene.rs` | divergence/backup/too-new tests | ✓ VERIFIED | 357 lines, 7 tests plus an ignored generator; the generator refuses to overwrite an existing fixture | -| `crates/agent-memory-core/tests/fixtures/v0.0.1.db` | committed fixture | ✓ VERIFIED | git-tracked, user_version=2, unmodified in the working tree | -| `crates/agent-memory-core/src/domain.rs` | Internal, SchemaTooNew, MemoryPatch, UpdatedMemory, LinkKind, RelatedLink, MAX_EXPANDED_LINKS | ✓ VERIFIED | all present and used | -| `crates/agent-memory-core/tests/update.rs` | MCP-06 tests | ✓ VERIFIED | 674 lines, 9 tests | -| `crates/agent-memory/src/serde_util.rs` | explicit-null helper | ✓ VERIFIED | used by UpdateArgs and UpdateBody | -| `crates/agent-memory-core/tests/relations.rs` | MCP-07 tests | ✓ VERIFIED | 737 lines, 19 tests | -| `crates/agent-memory-core/tests/hybrid.rs` | SEARCH-04/05 tests | ✓ VERIFIED (coverage hole: no hyphen or colon query) | 719 lines, 19 tests | -| `crates/agent-memory-core/src/store/sqlite.rs` | update / link / related / keyword_candidates / backup | ✓ VERIFIED (except the `map_fts_query_error` gap) | see truths 1, 2, 5, 14 | +|---|---|---|---| +| `crates/agent-memory-core/src/store/sqlite.rs` | `fn match_with_sanitized_retry`, `embedding_status = 0 AND content = ?2`, `fn migrate_tolerating_concurrent_upgrade` | ✓ VERIFIED | All present and called from the right sites | +| `crates/agent-memory-core/src/service.rs` | `fn redact_search_error`, `SEARCH_STORAGE_FAILURE` | ✓ VERIFIED | Wired at `pub async fn search` | +| `crates/agent-memory-core/src/domain.rs` | `fn canonical_endpoints`, `fn is_symmetric` | ✓ VERIFIED | Exhaustive matches, no `_` arm | +| `crates/agent-memory-core/tests/hybrid.rs` | `queries_fts5_cannot_parse_succeed_identically_in_both_modes` | ✓ VERIFIED | Line 587, passes | +| `crates/agent-memory-core/tests/backfill_race.rs` | ≥120 lines, deterministic race tests | ✓ VERIFIED | 7 tests, pass | +| `crates/agent-memory/src/mcp.rs` | `deny_unknown_fields` on UpdateArgs | ✓ VERIFIED | Line 169 | +| `crates/agent-memory/src/rest/handlers.rs` | `deny_unknown_fields` on UpdateBody; PathRejection/QueryRejection mapping | ✓ VERIFIED | Lines 107, 234, 285, 316, 395 | +| `README.md` | "never rejected", `## Upgrading` | ✓ VERIFIED | Lines 205, 84 | ### Key Link Verification -| From | To | Via | Status | Details | -|------|----|-----|--------|---------| -| migrations.rs | 0003_relations.sql | `M::up(include_str!)` | ✓ WIRED | line 24 | -| sqlite.rs open() | rusqlite Backup API | `run_to_completion` | ✓ WIRED | line 181; feature `backup` on the pinned 0.39 | -| sqlite.rs open() | PRAGMA user_version | probe before pragmas and migrations | ✓ WIRED | line 233 | -| service.rs import | Store::exists | `draft.source.as_deref()` | ✓ WIRED | line 338 | -| mcp.rs | MemoryService::update/link/unlink | tool arms | ✓ WIRED | mcp.rs:374/400/438; exercised live | -| rest/mod.rs | update/link/unlink handlers | PATCH `/api/memories/{id}`, POST `/api/links`, DELETE `/api/links/{from}/{to}/{kind}` | ✓ WIRED | rest/mod.rs:77-85 | -| service.rs | Store::related | `expand_related` after the bump | ✓ WIRED | search (both arms) and list | -| sqlite.rs forget/sweep | memory_links | belt-and-braces orphan DELETE | ✓ WIRED | lines 669, 956 | -| service.rs | Store::keyword_candidates + knn_search | `tokio::join!`, keyword error checked first | ✓ WIRED | lines 423-428 | -| service.rs | decay_score | post-fusion multiplicative blend | ✓ WIRED | line 443 | -| sqlite.rs keyword_candidates | map_fts_query_error | InvalidQuery propagation | ⚠️ PARTIAL | wired, but the mapper misclassifies the `no such column` FTS5 failure (gap 1) | +| From | To | Via | Status | +|---|---|---|---| +| sqlite.rs `fn search` | `match_with_sanitized_retry` | keyword-degrade path | ✓ WIRED (:866) | +| sqlite.rs `fn keyword_candidates` | `match_with_sanitized_retry` | hybrid keyword leg | ✓ WIRED (:908) | +| service.rs `pub async fn search` | `redact_search_error` | `map_err` over both arms and expansion | ✓ WIRED (:283) | +| service.rs `sweep` | `Store::insert_embedding` | `insert_embedding(id, &content, vector)` | ✓ WIRED (:698) | +| sqlite.rs link/unlink | `canonical_endpoints` / `is_symmetric` | canonical storage; both-orientation unlink | ✓ WIRED (:977, :1002) | +| sqlite.rs `SqliteStore::open` | `pre_migration_backup(&writer, path, db_version)` | probed version passed in | ✓ WIRED (:409) | +| sqlite.rs `SqliteStore::open` | `migrate_tolerating_concurrent_upgrade` | wraps `to_latest` | ✓ WIRED | ### Data-Flow Trace (Level 4) -| Artifact | Data Variable | Source | Produces Real Data | Status | -|----------|---------------|--------|--------------------|--------| -| `SearchOutcome.results` (hybrid) | fused views | `vec_memories` KNN plus `memories_fts` bm25, read pool | yes (live: real rows ranked) | ✓ FLOWING | -| `MemoryView.related` | RelatedLink list | `memory_links JOIN memories` | yes (live) | ✓ FLOWING | -| `UpdatedMemory` | post-update row | re-read inside the writer tx | yes (live) | ✓ FLOWING | +| Artifact | Data | Source | Real data | Status | +|---|---|---|---|---| +| Hybrid `SearchOutcome.results` | fused views | `vec_memories` KNN plus `memories_fts` bm25 (raw or sanitized) | yes (live) | ✓ FLOWING | +| Sweep backfill | vectors | Ollama batch embed, then guarded claim | yes (tests; live store gave status 1) | ✓ FLOWING | +| `MemoryView.related` | edges | `memory_links JOIN memories` | yes (live) | ✓ FLOWING | ### Behavioral Spot-Checks | Behavior | Command | Result | Status | -|----------|---------|--------|--------| -| Phase test binaries | `cargo test -p agent-memory-core --test update --test relations --test hybrid --test migration_hygiene --test import` | 56 passed, 0 failed, 1 ignored | ✓ PASS | -| Live hybrid with a real Ollama (executor human-check, part 1) | MCP stdio `serve` against `/tmp/v04live-data/live.db`, Ollama at localhost:11434 | paraphrase gives `search_mode: hybrid`, correct memory at rank 1 | ✓ PASS | -| Live degrade (executor human-check, part 2) | same DB, `--ollama-url http://127.0.0.1:9` | `search_mode: keyword`, envelope keys identical, one fallback WARN per search | ✓ PASS | -| Real v0.0.1 upgrade | Homebrew 0.0.1 writes 2 rows, then HEAD opens | backup user_version=2 with 2 rows; live DB user_version=3; link/update/hybrid OK | ✓ PASS | -| Too-new refusal through the binary | `--db future.db` (user_version=99) | exit 1, empty stdout, D-13 message on stderr, shasum unchanged | ✓ PASS | -| Hyphenated query, Ollama up | `memory_search {"query":"sqlite-vec"}` | `-32603 sqlite error: no such column: vec` (v0.0.1: correct semantic hit) | ✗ FAIL | -| `?` query, Ollama up | `memory_search {"query":"what vector store do we use?"}` | `-32602 invalid_params` (v0.0.1: correct semantic hit) | ? decision | -| Normal build excludes test-clock | `cargo tree -p agent-memory -e normal -i agent-memory-core -f '{p} {f}'` | no features | ✓ PASS | - -The executor's end-of-phase manual check (`` in plan 04-04 Task 2) was performed by the verifier against a real local Ollama. The hybrid and degrade observations above satisfy it. +|---|---|---|---| +| Phase test binaries | `cargo test -p agent-memory-core --test hybrid --test backfill_race --test relations --test migration_hygiene --test update` | 72 passed, 0 failed, 1 ignored | ✓ PASS | +| Store unit tests (backup, sanitizer, classifier) | `cargo test -p agent-memory-core --lib -- store::sqlite` | 12 passed | ✓ PASS | +| Stdio e2e | `cargo test -p agent-memory --test tools` | 6 passed | ✓ PASS | +| Live search table, Ollama up and down | MCP stdio driver, 25 queries × 2 modes | all succeed, no leaks | ✓ PASS | +| Live REST search and error bodies | `serve-rest` + curl | 200 on queries; 400 JSON on malformed path, query and unknown field | ✓ PASS | +| Live concurrent first open | 2 × `serve-rest` on a v0.0.1 fixture copy, 3 rounds | both up, one uv=2 backup | ✓ PASS | +| Live too-new refusal | `--db` with user_version=99 | exit 1, friendly message, file untouched | ✓ PASS | ### Probe Execution -None declared. No `scripts/*/tests/probe-*.sh` exists, and no phase artifact references a probe. +None declared. There is no `scripts/*/tests/probe-*.sh`, and no phase artifact references a probe. ### Requirements Coverage -| Requirement | Source Plan | Description | Status | Evidence | -|-------------|-------------|-------------|--------|----------| -| STORE-05 | 04-01 | 0003 with migration hygiene (frozen SQL, backup, newer-DB error) | ✓ SATISFIED | truth 5, live real-v0.0.1 upgrade | -| MCP-06 | 04-02 | memory_update, same-tx re-mirror and re-embed, outage fallback, not-found | ✓ SATISFIED | truths 1, 8, 9 | -| MCP-07 | 04-03 | link/unlink, 1-hop expansion, cascade, no graph model | ✓ SATISFIED | truths 2, 10-13 | -| SEARCH-04 | 04-04 | RRF k=60 hybrid, keyword degrade unchanged | ⚠️ SATISFIED WITH GAP | fusion and degrade verified (truth 3); hybrid now exposes the pre-existing 500 misclassification (truth 14) and the punctuation decision (truth 15) | -| SEARCH-05 | 04-04 | post-fusion decay, bumps on returned ids only | ✓ SATISFIED | truth 4 | +| Requirement | Source Plan | Status | Evidence | +|---|---|---|---| +| MCP-06 | 04-02, 04-06, 04-07 | ✓ SATISFIED | Truths 1, 8, 9, 20, 24 | +| MCP-07 | 04-03, 04-06, 04-07 | ✓ SATISFIED | Truths 2, 10-13, 21-23, 25 | +| STORE-05 | 04-01, 04-07 | ✓ SATISFIED | Truths 5, 26-28 | +| SEARCH-04 | 04-04, 04-05 | ✓ SATISFIED | Truths 3, 14-19. The previous gap is closed | +| SEARCH-05 | 04-04 | ✓ SATISFIED | Truth 4 | + +No requirements are orphaned. REQUIREMENTS.md maps exactly these 5 IDs to Phase 4. + +### Supersession Check (04-04 must-have #3) + +The 04-04 must-have ("a malformed MATCH query is a 400/invalid_params in hybrid mode") is superseded, and the supersession is recorded explicitly in two places: + +- `04-05-PLAN.md` § "Locked user decision": "…the 04-04 must-have 'A malformed MATCH query is a 400/invalid_params in hybrid mode' are **superseded** by this decision." +- `.planning/STATE.md` line 118 ([Phase 04] 04-05): "Supersedes the 04-04 decision that a malformed MATCH is InvalidQuery/400 in hybrid mode." The older line 117 is retained as history. + +The code matches the decision. The service doc states "Search never returns `MemoryError::InvalidQuery`", and the README "Behavioral changes in v0.1.0" item 3 is rewritten for the forgiving leg. + +### Deferred by Instruction (not gaps; confirmed recorded) -There are no orphaned requirements. REQUIREMENTS.md maps exactly these 5 IDs to Phase 4, and each is claimed by a plan. +| Item | Recorded in | +|---|---| +| WR-04: unbounded `memory_list` with `expand_links` and no `limit` (behaviour unchanged; doc corrected) | 04-06-PLAN Deferred table; 04-06-SUMMARY "Deferred"; 04-07-SUMMARY "Deferred"; STATE.md [04-06] line 121 ("open API-contract question for Phase 5/6"); domain.rs:363-369 doc | +| WR-01 full cross-process exclusive lock | 04-07-PLAN Deferred table; 04-07-SUMMARY "Deferred"; STATE.md [04-07] line 124 | +| 03-REVIEW WR-01: filters applied after the KNN cut | 04-05/06/07-PLAN Deferred tables ("still deferred (not re-opened)"); 04-05-SUMMARY line 236; STATE.md [04-01] line 103 | +| 03-REVIEW WR-03: spawned REST test-harness hang | 04-05/06/07-PLAN Deferred tables; 04-05-SUMMARY line 236 | + +The plans also record these deferrals, which are out of scope for this phase goal: + +- internal-tier redaction on non-search surfaces; +- `deny_unknown_fields` on the v0.0.1 and link DTOs, per [03-01]; +- stale `.backup-tmp-*` cleanup; +- IN-05 and IN-06 behaviour changes, which were documented instead. ### Anti-Patterns Found | File | Line | Pattern | Severity | Impact | -|------|------|---------|----------|--------| -| crates/agent-memory-core/src/store/sqlite.rs | ~333 | FTS5 error classifier matches message substrings and misses `no such column` | 🛑 Blocker | client input reaches the 500 tier with SQL detail on the now-default Ollama-up path (gap 1) | - -The scan of all 19 phase-modified files found no unreferenced TBD/FIXME/XXX markers, no TODO/HACK/placeholder text, and no `unwrap()`/`expect()` in non-test source. +|---|---|---|---|---| +| (gap-closure files, 14) | — | TBD/FIXME/XXX | none | Every "TODO" hit is the `TODO` memory-type domain value, not a debt marker | +| crates/agent-memory-core/src, crates/agent-memory/src | — | `unwrap()`/`expect()` before `#[cfg(test)]` | none | 0 hits | +| crates/agent-memory/src/mcp.rs, rest/handlers.rs | 40-44, 170-174 | Non-search internal errors still use `e.to_string()` | ℹ️ Info | Search paths are redacted (the decision's scope). Update/link/forget internal errors can still carry SQLite text on the loopback API. This deferral is recorded in the 04-05 plan | +| crates/agent-memory-core/src/store/sqlite.rs | 495-504 | FTS5 error classification matches message substrings | ℹ️ Info | Pinned by `every_bundled_fts5_parse_error_is_classified_as_match_input` against the bundled SQLite. A SQLite bump that rewords a message would fall to the redacted 500 tier, not leak, and the test would flag it | ### Human Verification Required -#### 1. Natural-language punctuation in hybrid queries (decision) - -**Test:** With Ollama up, run `memory_search` with `what vector store do we use?` against a store containing "We chose sqlite-vec for local vector similarity search". -**Expected:** A developer decision on the contract. Today the query returns invalid_params; v0.0.1 found the memory. Option (b), sanitising or quoting the keyword leg's input, would restore paraphrase recall and also close gap 1. Option (a) needs an override entry. -**Why human:** Plan decision T-04-15 (identical classification in every mode, so the mode cannot be probed) conflicts with the roadmap outcome clause. That is a product call. +None. The previous decision item was resolved by the user (option-a) and is verified live. ### Gaps Summary -The phase is substantively delivered. Update, relations, RRF fusion, decay blending, bump scoping and migration hygiene are all real, wired, test-locked, and confirmed live. That includes an upgrade of a database written by the real v0.0.1 binary and a live hybrid-to-keyword degrade against a real Ollama. +Both previous blockers are closed, and the code matches the user's option-a decision: + +- **Gap #14:** the hyphen/colon 500 with leaked SQLite text is fixed. +- **Item #15:** punctuated natural-language recall is resolved. + +How the fix works: + +- **Shared retry helper.** One store helper, used by both the keyword-degrade path and the hybrid keyword leg, runs the raw FTS5 query first. On any FTS5 parse-class error, including the `no such column` class, it retries once with OR-ed quoted words. A tokenless query becomes an empty keyword leg. +- **Single redaction point.** Every search error is tiered at one point, so storage failures surface only as `SEARCH_STORAGE_FAILURE`. +- **Live confirmation.** All of this was confirmed on MCP and REST, with Ollama up and down. -One gap blocks a clean pass, and it shares a root cause with the decision item. Hybrid search now always parses the query through FTS5, even when Ollama is up. Two consequences follow: +The other two gap-closure plans also hold: -1. **(Gap, fixable in one line plus tests.)** Hyphen- or colon-shaped queries (`sqlite-vec`, `auth: tokens`) produce FTS5's `no such column` error. `map_fts_query_error` does not recognise it, so it surfaces as a 500 / internal_error with the SQLite message. In v0.0.1 these queries worked whenever Ollama was up. -2. **(Decision.)** Queries with `?`, `,` or `'` are classified correctly as client errors, but that means ordinary paraphrased questions no longer return results with Ollama up. +- **04-06:** the CR-03 conditional backfill and the symmetric `relates_to` behave as specified. +- **04-07:** the strict update DTOs, the uniform REST rejection bodies and the race-safe backup with tolerant concurrent open behave as specified. The concurrent first open was confirmed live across three rounds. -Sanitising the keyword leg's input would resolve both at once. If the developer judges gap 1 pre-existing and out of scope, add this override to the frontmatter: +The regression checks on truths 1-13 found no regressions. The deferred items are each recorded explicitly. -```yaml -overrides: - - must_have: "A malformed MATCH query is a 400/invalid_params in hybrid mode exactly as in keyword mode" - reason: "no-such-column misclassification is pre-existing v1.0 keyword-path behavior; tracked for a follow-up" - accepted_by: "" - accepted_at: "" -``` +One protocol-tier note (truth 24): over MCP, an unknown update field is reported as a tool error result rather than JSON-RPC `invalid_params`. This is rmcp's deserialization path and is recorded in STATE.md. The field is named and nothing is written. --- -_Verified: 2026-10-23T18:52:00Z_ +_Verified: 2026-10-23T20:19:46Z_ _Verifier: Claude (gsd-verifier)_ From 36df9372b4bc253e6db00ac8f2b845112f09333e Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 22:24:14 +0200 Subject: [PATCH 36/42] fix(04): WR-01 classify the FTS5 expression-depth error as client input 257+ chained NOTs exceed SQLITE_FTS5_MAX_EXPR_DEPTH (256) and fail with "fts5 expression tree is too large", which carries no "fts5: " prefix and so fell through to a generic 500 / internal_error. Add it to FTS5_QUERY_PARSE_MARKERS so it takes the single sanitized retry like every other FTS5 parse error. The retry quotes each token and joins them with OR, a flat node, so it cannot hit the same limit. Tests, against the bundled SQLite: NOT x257 and x300 join the classifier and fallback-parses probes; a boundary test pins 256 as accepted, 257 as the classified depth error, and a 3000-NOT chain's fallback as parsing. A two-mode test (hybrid with FakeEmbedder, keyword) shows the chain is answered and the retry recalls the matching row. Co-Authored-By: Claude Opus 5.5 --- crates/agent-memory-core/src/store/sqlite.rs | 52 ++++++++++++++++++-- crates/agent-memory-core/tests/hybrid.rs | 30 +++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/crates/agent-memory-core/src/store/sqlite.rs b/crates/agent-memory-core/src/store/sqlite.rs index 7047c06..860b404 100644 --- a/crates/agent-memory-core/src/store/sqlite.rs +++ b/crates/agent-memory-core/src/store/sqlite.rs @@ -492,11 +492,16 @@ fn corrupt_column(what: &str, value: &str) -> rusqlite::Error { /// (`sqlite-vec` → column `vec`), which fails on the one-column table. /// - `unknown special query`: a leading `*`. /// - `expected integer`: a non-numeric `NEAR(…, x)` distance. +/// - `fts5 expression tree is too large`: the query exceeds +/// `SQLITE_FTS5_MAX_EXPR_DEPTH` (256), e.g. 257+ chained `NOT`s, which FTS5 +/// never flattens. The message has no `fts5: ` prefix. The sanitized retry +/// cannot hit it: quoted tokens joined by `OR` form one flat node. /// /// Only FTS5 query-parse messages belong here, never a generic SQLite message /// (T-04-22): a marker match is what licenses the sanitized retry. -const FTS5_QUERY_PARSE_MARKERS: [&str; 5] = [ +const FTS5_QUERY_PARSE_MARKERS: [&str; 6] = [ "fts5: ", + "fts5 expression tree is too large", "unterminated string", "no such column", "unknown special query", @@ -1541,9 +1546,19 @@ mod tests { format!("{}a{}", "(".repeat(3000), ")".repeat(3000)) } - /// Candidate probes from the planning table that FTS5 rejects. All nine - /// error in the bundled SQLite 3.51.3 exactly as in the sqlite3 3.46 CLI - /// probe (the empty string included), so none moved to the valid-raw group. + /// `a NOT a NOT a …` with `nots` chained `NOT`s. FTS5 never flattens `NOT` + /// nodes, so the chain is a left-deep tree; past + /// `SQLITE_FTS5_MAX_EXPR_DEPTH` (256) it fails at step time with + /// `fts5 expression tree is too large (maximum depth 256)` (re-review WR-01). + fn deep_not_chain(nots: usize) -> String { + format!("a{}", " NOT a".repeat(nots)) + } + + /// Candidate probes from the planning table that FTS5 rejects. The first + /// nine error in the bundled SQLite 3.51.3 exactly as in the sqlite3 3.46 + /// CLI probe (the empty string included), so none moved to the valid-raw + /// group. The two `NOT` chains (re-review WR-01) pin the expression-depth + /// limit at its boundary (257) and past it (300). fn erroring_probes() -> Vec { vec![ String::new(), @@ -1555,6 +1570,8 @@ mod tests { "*".to_string(), "NEAR(a b, x)".to_string(), deep_parens(), + deep_not_chain(257), + deep_not_chain(300), ] } @@ -1619,6 +1636,33 @@ mod tests { assert_eq!(probe(&mut stmt, &fallback).expect("parses"), 1); } + #[test] + fn the_depth_limit_is_exactly_256_nots_and_the_fallback_of_any_chain_parses() { + // re-review WR-01. 256 chained NOTs is the deepest tree FTS5 accepts; + // one more is the depth error, which must be classified as client input. + let conn = fts_probe_conn(); + let mut stmt = conn + .prepare("SELECT count(*) FROM t WHERE t MATCH ?1") + .expect("static statement prepares"); + probe(&mut stmt, &deep_not_chain(256)).expect("256 NOTs is within the depth limit"); + let err = probe(&mut stmt, &deep_not_chain(257)).expect_err("257 NOTs exceeds it"); + assert!( + err.to_string() + .contains("fts5 expression tree is too large"), + "unexpected bundled-SQLite message: {err}" + ); + assert!(is_match_input_error(&err), "got: {err}"); + // The retry cannot hit the same limit: it quotes every token (so `NOT` + // is a string, not an operator) and joins them with OR, which FTS5 + // flattens into one n-ary node. Even a 3000-NOT chain (6001 terms) + // parses once sanitized. + let fallback = sanitize_fts_query(&deep_not_chain(3000)).expect("has words"); + assert_eq!( + probe(&mut stmt, &fallback).expect("flat OR chain parses"), + 0 + ); + } + #[test] fn the_classifier_rejects_everything_that_is_not_an_fts5_parse_error() { let busy = rusqlite::Error::SqliteFailure( diff --git a/crates/agent-memory-core/tests/hybrid.rs b/crates/agent-memory-core/tests/hybrid.rs index 4e9c07f..3a4b20c 100644 --- a/crates/agent-memory-core/tests/hybrid.rs +++ b/crates/agent-memory-core/tests/hybrid.rs @@ -638,6 +638,36 @@ async fn queries_fts5_cannot_parse_succeed_identically_in_both_modes() { } } +#[tokio::test] +async fn a_not_chain_past_the_fts5_depth_limit_is_answered_in_both_modes() { + // Re-review WR-01. FTS5 never flattens NOT, so 257+ chained NOTs exceed + // SQLITE_FTS5_MAX_EXPR_DEPTH (256) and fail with "fts5 expression tree is + // too large". That is client input: it takes the single sanitized retry + // (quoted tokens OR-ed, a flat node) and is answered, never a 500. The + // retry's `"auth"` terms recall M2 in both modes, which proves it ran. + for nots in [257_usize, 300] { + let input = format!("auth{}", " NOT auth".repeat(nots)); + for (expected_mode, embedder) in both_modes() { + let h = harness(embedder); + let corpus = store_corpus(&h).await; + let outcome = h + .service + .search(query(&input, None)) + .await + .unwrap_or_else(|e| { + panic!("{expected_mode:?} mode: a {nots}-NOT chain must be answered, got {e:?}") + }); + assert_eq!(outcome.search_mode, expected_mode, "{nots} NOTs"); + assert_eq!( + ids(&outcome.results).first(), + Some(&corpus[1]), + "{expected_mode:?} mode: the {nots}-NOT chain's retry must recall M2 first, got {:?}", + ids(&outcome.results) + ); + } + } +} + #[tokio::test] async fn structured_fts5_syntax_still_runs_raw() { // The raw query always runs first, so valid FTS5 keeps its exact meaning. From 68b479c3404dd626b546d95597aaf7a30eb226da Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 22:24:19 +0200 Subject: [PATCH 37/42] docs(04): WR-02 scope the REST JSON error-body claim to the routes that honour it The README said every REST client error returns a JSON {"error"} body. POST /api/memories and POST /api/search still use axum's bare Json extractor, so a body that fails to deserialize gets a plain-text 422/400/415. List exactly which routes map malformed body, path and query input to a 400 {"error"}, and state that the two v0.0.1 body routes keep axum's default rejection. Doc only: no behaviour change. Co-Authored-By: Claude Opus 5.5 --- README.md | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 97f10c8..380fbea 100644 --- a/README.md +++ b/README.md @@ -430,12 +430,25 @@ Out-of-bounds `limit`/`ttl_secs` values return **HTTP 400** with a JSON body whose `error` field names the field, range, and offending value — shaped `invalid argument: limit must be between 1 and 200 (got 0)` — never a 500 and never a silent clamp. The MCP transport rejects the same inputs identically as -`invalid_params`, because validation lives once at the shared service seam. A -`PATCH` body that is not valid JSON, has a wrongly typed field, or is sent -without `Content-Type: application/json` is also a 400 with the same -`{"error": ...}` body. Every client error, including a malformed path segment -(`/api/memories/abc`) or query parameter (`?limit=abc`, `?expand_links=1`), -returns a JSON `{"error": "..."}` body. A storage failure during search reports a generic +`invalid_params`, because validation lives once at the shared service seam. + +Malformed input that never reaches the seam gets the same 400 `{"error": ...}` +body on these routes only: + +- **request body** on `PATCH /api/memories/{id}` and `POST /api/links`: not + valid JSON, a missing or wrongly typed field, or no + `Content-Type: application/json` (and, on `PATCH` only, an unknown field); +- **path segment** on `PATCH`/`DELETE /api/memories/{id}` and + `DELETE /api/links/{from_id}/{to_id}/{kind}`, e.g. `/api/memories/abc`; +- **query parameter** on `GET /api/memories`, e.g. `?limit=abc` or + `?expand_links=1`. + +The two v0.0.1 body routes, `POST /api/memories` and `POST /api/search`, keep +axum's default body rejection: a **plain-text** (not JSON) response with 422 +for a missing or wrongly typed field, 400 for malformed JSON, and 415 for a +missing `Content-Type`. A body that does deserialize is validated as above, so +an out-of-bounds `limit`/`ttl_secs` or an unknown `type` on those routes is +still a 400 `{"error"}`. A storage failure during search reports a generic internal error (HTTP 500 / MCP `internal_error`, `search failed due to a storage error`); the detail goes only to the server's stderr log. From 4441bdafab5d420a1fbcf3ba348c0589d72bdc11 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 22:24:27 +0200 Subject: [PATCH 38/42] docs(04): mark re-review WR-01 and WR-02 fixed in 04-REVIEW.md Co-Authored-By: Claude Opus 5.5 --- .../04-REVIEW.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md index 51c7957..06a88c0 100644 --- a/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md @@ -207,6 +207,15 @@ The user does not get the friendly D-13 text that names `brew upgrade`. The refu **Fix:** Rename it to `memory_update_with_an_unknown_field_is_a_tool_error_over_stdio`. +## Fixes applied (2026-10-23) + +| ID | Status | Commit | Change | +|----|--------|--------|--------| +| WR-01 (new) | Fixed | `36df937` | Added `fts5 expression tree is too large` to `FTS5_QUERY_PARSE_MARKERS`. The NOT×257 and NOT×300 probes are in the bundled-SQLite classifier and fallback tests. A boundary test pins 256 NOTs as accepted, 257 as the classified depth error, and a 3000-NOT chain's flat-OR fallback as parsing. The two-mode test `a_not_chain_past_the_fts5_depth_limit_is_answered_in_both_modes` covers hybrid and keyword mode. | +| WR-02 (new) | Fixed (doc only) | `68b479c` | The README now lists the exact routes that return a 400 `{"error"}` for a malformed body, path, or query. It also states that `POST /api/memories` and `POST /api/search` keep axum's plain-text 422/400/415 body rejection. No behaviour change. | + +Workspace after the fixes: 213 passed / 0 failed / 2 ignored. `cargo clippy --workspace --all-targets -- -D warnings` is clean. + --- _Reviewed: 2026-10-23_ From 63f240b8b8abae009596ce4110349ed8ecc58513 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Fri, 23 Oct 2026 22:25:06 +0200 Subject: [PATCH 39/42] docs(phase-04): complete phase execution Co-Authored-By: Claude Opus 5.5 --- .planning/ROADMAP.md | 4 ++-- .planning/STATE.md | 29 +++++++++++++++-------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index ba83660..cb3db38 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -27,7 +27,7 @@ Full phase details: [milestones/v1.0-ROADMAP.md](milestones/v1.0-ROADMAP.md) - Decimal phases (3.1, 3.2): Urgent insertions (marked INSERTED) - [x] **Phase 3: API Hardening & Toolchain Spikes** - Out-of-bounds limit/ttl rejected as client errors and exact tag matching at the shared service seam, plus early Windows/musl cross-compile spikes answering the milestone's only feasibility unknown (completed 2026-07-12) -- [ ] **Phase 4: Memory Update, Relations & Hybrid Search** - `memory_update` with same-transaction re-embed, flat typed links with cascade, migration 0003 hygiene, and decay-aware hybrid RRF as a parallel track +- [x] **Phase 4: Memory Update, Relations & Hybrid Search** - `memory_update` with same-transaction re-embed, flat typed links with cascade, migration 0003 hygiene, and decay-aware hybrid RRF as a parallel track (completed 2026-10-23) - [ ] **Phase 5: Portable Export & Import** - Versioned JSONL export (link edges included, embeddings excluded) and idempotent cross-machine import with sweep-backfill re-embedding - [ ] **Phase 6: Distribution & v0.1.0 Release** - musl + Windows binaries on the spike-chosen toolchains and the checksummed v0.1.0 release with Homebrew update and behavioral-change release notes @@ -130,6 +130,6 @@ Phases execute in numeric order: 3 → 4 → 5 → 6 | 1. Core Memory Foundation | v1.0 | 3/3 | Complete | 2026-06-25 | | 2. Semantic Search, Interop & Release | v1.0 | 5/5 | Complete | 2026-07-12 | | 3. API Hardening & Toolchain Spikes | v1.1 | 2/2 | Complete | 2026-07-12 | -| 4. Memory Update, Relations & Hybrid Search | v1.1 | 7/7 | In Progress| | +| 4. Memory Update, Relations & Hybrid Search | v1.1 | 7/7 | Complete | 2026-10-23 | | 5. Portable Export & Import | v1.1 | 0/? | Not started | - | | 6. Distribution & v0.1.0 Release | v1.1 | 0/? | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index 745506d..e4bd26b 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -2,17 +2,17 @@ gsd_state_version: 1.0 milestone: v1.1 milestone_name: Hardening & Interop -current_phase: 04 -current_phase_name: Memory Update, Relations & Hybrid Search -status: executing -stopped_at: Completed 04-07-PLAN.md -last_updated: "2026-10-23T20:12:50.775Z" +current_phase: 5 +current_phase_name: Portable Export & Import +status: planning +stopped_at: Phase 04 complete, ready to plan Phase 5 +last_updated: "2026-10-23T20:25:00.455Z" last_activity: 2026-10-23 -last_activity_desc: Completed 04-07 (gap wave 3, strict update DTOs + REST error bodies + race-safe backup) -state_head: 9693c5fc8d881f1e3fce159415d5cd4ff2f8873b +last_activity_desc: Phase 04 complete, transitioned to Phase 5 +state_head: 4441bdafab5d420a1fbcf3ba348c0589d72bdc11 progress: total_phases: 4 - completed_phases: 1 + completed_phases: 2 total_plans: 9 completed_plans: 9 --- @@ -28,10 +28,10 @@ See: .planning/PROJECT.md (updated 2026-07-12) ## Current Position -Phase: 04 (Memory Update, Relations & Hybrid Search) — EXECUTING -Plan: 7 of 7 (gap closure: 04-05, 04-06, 04-07 done) -Status: All 7 plans executed — ready for phase 04 re-verification (not yet marked complete) -Last activity: 2026-10-23 — Completed 04-07 (WR-02 deny_unknown_fields on update DTOs, IN-02 uniform REST rejection bodies, WR-01/IN-03 race-safe pre-migration backup, IN-01 README Upgrading) +Phase: 5 — Portable Export & Import +Plan: Not started +Status: Ready to plan +Last activity: 2026-10-23 — Phase 04 complete, transitioned to Phase 5 Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) @@ -39,7 +39,7 @@ Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) **Velocity:** -- Total plans completed: 13 +- Total plans completed: 17 - Average duration: ~28 min - Total execution time: ~1.4 hours @@ -50,6 +50,7 @@ Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) | 1 | 3 | - | - | | 02 | 5 | - | - | | 03 | 2 | - | - | +| 04 | 7 | - | - | **Recent Trend:** @@ -163,7 +164,7 @@ Items acknowledged and carried forward: ## Session Continuity Last session: 2026-10-23T20:12:50.720Z -Stopped at: Completed 04-07-PLAN.md +Stopped at: Phase 04 complete, ready to plan Phase 5 Resume file: None ## Operator Next Steps From 969e8916f02c6a3ebf3ff20be419182743f505ab Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Sat, 24 Oct 2026 01:37:03 +0200 Subject: [PATCH 40/42] fix(quick-261023-v6w): reject memory_list expand_links without a limit (WR-04) Implements user decision option-a (reject, 2026-10-23) for Phase 4 review finding WR-04. - MemoryService::list rejects expand_links=true with no limit as the existing MemoryError::InvalidArgument, before any store hop. Range validation runs first. No new error variant, and neither transport mapper changes. - MCP: invalid_params, which goes out over stdio as a JSON-RPC -32602 error. REST: 400 {"error": "invalid argument: limit is required when expand_links is true; pass a limit between 1 and 200"}. - expand_links with a limit, a list without expand_links, and search are unchanged, so the 200 x 10 expansion bound now covers every expanded response. - Tests at the service seam, over MCP stdio, over real REST HTTP, and an inline REST 400 test. Expansion fixtures that relied on unlimited expanded lists now pass a limit. - The rule is now stated in the tool description, the MCP/REST list DTO docs, the MAX_EXPANDED_LINKS doc and the README (the curl example now passes a limit). Co-Authored-By: Claude Opus 5.5 --- README.md | 25 +++-- crates/agent-memory-core/src/domain.rs | 18 ++-- crates/agent-memory-core/src/service.rs | 27 +++++- crates/agent-memory-core/tests/relations.rs | 6 +- crates/agent-memory-core/tests/validation.rs | 97 ++++++++++++++++++++ crates/agent-memory/src/mcp.rs | 27 +++++- crates/agent-memory/src/rest/handlers.rs | 50 ++++++++-- crates/agent-memory/tests/rest.rs | 91 ++++++++++++++++++ crates/agent-memory/tests/tools.rs | 88 ++++++++++++++++++ 9 files changed, 396 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 380fbea..37a052c 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ stored in one is immediately retrievable in another. |------|---------|----------------------| | `memory_store` | Persist a typed memory; returns its id. | `ttl_secs`: optional integer, valid range 1–3155760000 (~100 years); omitted = no expiry | | `memory_search` | Hybrid (keyword + semantic, rank-fused) search when Ollama is reachable, FTS5 keyword otherwise — weighted by decay, and only the returned results are recency-bumped. Accepts plain natural-language queries (any punctuation) as well as FTS5 syntax. | `limit`: optional integer, valid range 1–200; omitted defaults to 50. `tag`: exact tag match (case-sensitive). `expand_links`: optional boolean, default `false` | -| `memory_list` | List memories newest-first, with optional type/tag/scope filters. | `limit`: optional integer, valid range 1–200; omitted = unlimited. `tag`: exact tag match (case-sensitive). `expand_links`: optional boolean, default `false` | +| `memory_list` | List memories newest-first, with optional type/tag/scope filters. | `limit`: optional integer, valid range 1–200; omitted = unlimited, **required when `expand_links` is true**. `tag`: exact tag match (case-sensitive). `expand_links`: optional boolean, default `false` | | `memory_update` | Patch a memory in place by id — content, tags, scope, TTL (clean not-found for unknown ids). | `content`: never null or empty. `ttl_secs`: 1–3155760000 when set; `null` = no expiry. At least one field besides `id` | | `memory_link` | Link two memories with a typed, directed relation (idempotent; clean not-found for unknown ids). | `kind`: one of `relates_to`, `supersedes`, `caused_by`. `from_id` ≠ `to_id` | | `memory_unlink` | Remove a relation (clean not-found when no such edge exists). | `kind`: one of `relates_to`, `supersedes`, `caused_by` | @@ -226,8 +226,9 @@ case-sensitively. { "type": "TODO", "scope": "kore-runtime", "limit": 20 } ``` -A present `limit` must be between 1 and 200 (omitted returns everything); a -`tag` filter matches an exact tag (case-sensitive). +A present `limit` must be between 1 and 200 (omitted returns everything, +except that `expand_links: true` requires a `limit`); a `tag` filter matches +an exact tag (case-sensitive). Both `memory_search` and `memory_list` accept `"expand_links": true` to attach each result's linked memories — see [Relations](#relations-memory_link--memory_unlink). @@ -359,8 +360,12 @@ summaries, following relations in both directions: sequence. - At most `MAX_EXPANDED_LINKS = 10` neighbors per result; when a memory has more, the most recently created relations win. -- `memory_list` with `expand_links` and no `limit` returns every memory, each - with up to 10 neighbors, so pass a `limit` on large stores. +- `memory_list` with `expand_links` **requires a `limit`** (1–200). Without + one the call is rejected as a client error (HTTP 400 / MCP `invalid_params`) + whose message tells you to pass a `limit`, so an expanded response is always + bounded at 200 results × 10 neighbors. `memory_search` is bounded by its own + `limit` (default 50). Without `expand_links`, an omitted `limit` still lists + every memory. - Expansion is strictly one hop and a pure read: neighbors are **not** recency-bumped, so expanding never changes how anything decays. - A result with no relations gets `"related": []`. Without the flag there is no @@ -370,7 +375,7 @@ summaries, following relations in both directions: | Bound | Value | Applies to | |-------|-------|------------| -| `limit` | 1–200 (`MAX_LIMIT`) | `memory_search` (default 50), `memory_list` (default unlimited) | +| `limit` | 1–200 (`MAX_LIMIT`) | `memory_search` (default 50), `memory_list` (default unlimited; required with `expand_links`) | | `ttl_secs` | 1–3155760000 (~100 years) | `memory_store`, `memory_update` | | `MAX_EXPANDED_LINKS` | 10 neighbors per result, most recent first | `expand_links` on search and list | | snippet length | 160 characters (+ `…` when cut) | `related[].snippet` | @@ -418,7 +423,7 @@ agent-memory serve-rest --addr 127.0.0.1:8080 # custom loopback bind | Method | Path | Purpose | Validated parameters | |--------|------|---------|----------------------| | `POST` | `/api/memories` | Store a memory (same fields as `memory_store`) | `ttl_secs`: 1–3155760000 when present; omitted = no expiry | -| `GET` | `/api/memories` | List, with optional `type`/`tag`/`scope`/`limit`/`expand_links` query params | `limit`: 1–200 when present; omitted = unlimited. `tag`: exact tag (case-sensitive). `expand_links=true` attaches `related` summaries | +| `GET` | `/api/memories` | List, with optional `type`/`tag`/`scope`/`limit`/`expand_links` query params | `limit`: 1–200 when present; omitted = unlimited, but required with `expand_links=true` (400 without one). `tag`: exact tag (case-sensitive). `expand_links=true` attaches `related` summaries | | `POST` | `/api/search` | Search — returns the same `{search_mode, results}` envelope | `limit`: 1–200 when present; omitted defaults to 50. `tag`: exact tag (case-sensitive). `"expand_links": true` attaches `related` summaries | | `PATCH` | `/api/memories/{id}` | Update in place — JSON merge-patch body with the same fields and semantics as `memory_update`; 200 returns the updated memory with `embedding_status` | as `memory_update`; 404 `{"id", "updated": false, "reason": "not_found"}` for unknown ids | | `DELETE` | `/api/memories/{id}` | Forget by id (404 with a structured body for unknown ids) | — | @@ -431,6 +436,10 @@ whose `error` field names the field, range, and offending value — shaped `invalid argument: limit must be between 1 and 200 (got 0)` — never a 500 and never a silent clamp. The MCP transport rejects the same inputs identically as `invalid_params`, because validation lives once at the shared service seam. +`GET /api/memories?expand_links=true` without a `limit` is also a 400 +(`invalid argument: limit is required when expand_links is true; pass a limit +between 1 and 200`), and MCP `memory_list` rejects the same call as +`invalid_params`. Malformed input that never reaches the seam gets the same 400 `{"error": ...}` body on these routes only: @@ -461,7 +470,7 @@ curl -X POST localhost:7437/api/links \ -H 'content-type: application/json' \ -d '{"from_id": 57, "to_id": 42, "kind": "supersedes"}' -curl 'localhost:7437/api/memories?type=DECISION&expand_links=true' +curl 'localhost:7437/api/memories?type=DECISION&expand_links=true&limit=20' ``` The REST daemon and the MCP stdio server can run concurrently against the same diff --git a/crates/agent-memory-core/src/domain.rs b/crates/agent-memory-core/src/domain.rs index 5efbab4..59ecc4e 100644 --- a/crates/agent-memory-core/src/domain.rs +++ b/crates/agent-memory-core/src/domain.rs @@ -359,15 +359,15 @@ pub const MAX_TTL_SECS: i64 = 3_155_760_000; /// edges (`memory_links.created_at` descending). Fixed — there is no /// caller-tunable knob in v0.1.0. /// -/// What this bounds (REVIEW WR-04 doc correction): each result carries at most -/// `MAX_EXPANDED_LINKS` neighbours. Search is always bounded by its limit -/// (1–[`MAX_LIMIT`], default 50), so an expanded search returns at most -/// `200 × 10` summaries. A list without a limit returns every row (v1.0 -/// behaviour, RESEARCH Pitfall 6), and each row is expanded up to -/// `MAX_EXPANDED_LINKS`, so an unlimited expanded list grows with the corpus. -/// T-04-11's `200 × 10` bound therefore covers search and list-with-limit -/// only; changing unlimited expanded-list behaviour is a deferred API-contract -/// decision. +/// What this bounds (REVIEW WR-04, user decision option-a, 2026-10-23): each +/// result carries at most `MAX_EXPANDED_LINKS` neighbours. Search is always +/// bounded by its limit (1–[`MAX_LIMIT`], default 50), and +/// [`crate::service::MemoryService::list`] rejects `expand_links` without a +/// `limit` as [`MemoryError::InvalidArgument`] (REST 400 / MCP +/// `invalid_params`) before any store hop. So every expanded search or list +/// returns at most `200 × 10` summaries, and T-04-11's bound covers every +/// expanded response. A list without `expand_links` still returns every row +/// when `limit` is omitted (v1.0 behaviour, RESEARCH Pitfall 6). pub const MAX_EXPANDED_LINKS: usize = 10; /// The library's typed error. `thiserror` per the ecosystem rule (libraries use diff --git a/crates/agent-memory-core/src/service.rs b/crates/agent-memory-core/src/service.rs index 8a1b61a..662319b 100644 --- a/crates/agent-memory-core/src/service.rs +++ b/crates/agent-memory-core/src/service.rs @@ -19,7 +19,7 @@ use crate::clock::Clock; use crate::decay::{apply_decay, decay_score, DecayConfig, DecayEngine, RankWeights, SweepReport}; use crate::domain::{ LinkKind, LinkOutcome, MemoryError, MemoryPatch, MemoryType, MemoryView, NewMemory, - UpdatedMemory, MAX_EXPANDED_LINKS, + UpdatedMemory, MAX_EXPANDED_LINKS, MAX_LIMIT, MIN_LIMIT, }; use crate::embed::Embedder; use crate::store::sqlite::DEFAULT_SEARCH_LIMIT; @@ -68,7 +68,10 @@ pub struct ListArgs { pub scope: Option, pub limit: Option, /// Attach 1-hop relation summaries to every returned row (MCP-07 / D-04). - /// `false` (the default) keeps the v1.0 response byte-identical. + /// `false` (the default) keeps the v1.0 response byte-identical. `true` + /// requires `limit` (REVIEW WR-04, user decision option-a): without one, + /// [`MemoryService::list`] rejects the call as + /// [`MemoryError::InvalidArgument`]. pub expand_links: bool, } @@ -229,10 +232,26 @@ impl MemoryService { } /// List stored memories newest-first with optional filters (MCP-03 / D-06). + /// + /// `expand_links` requires a `limit` (REVIEW WR-04, user decision + /// option-a): an expanded list with no limit is rejected as + /// [`MemoryError::InvalidArgument`] before any store hop, with a message + /// telling the caller to pass a limit. The rule is enforced here, at the + /// shared seam, so MCP and REST cannot drift, and the + /// `200 × MAX_EXPANDED_LINKS` bound therefore covers every expanded + /// response. A list without `expand_links` still returns every row when + /// `limit` is omitted. pub async fn list(&self, args: ListArgs) -> Result, MemoryError> { - // API-02 / D-01: a present limit must be 1..=200; omitted stays - // unlimited (v1.0 behavior, RESEARCH Pitfall 6). + // API-02 / D-01: a present limit must be 1..=200. An omitted limit + // stays unlimited (v1.0 behavior, RESEARCH Pitfall 6) only without + // `expand_links`; range validation runs first. crate::domain::validate_limit(args.limit)?; + if args.expand_links && args.limit.is_none() { + return Err(MemoryError::InvalidArgument(format!( + "limit is required when expand_links is true; \ + pass a limit between {MIN_LIMIT} and {MAX_LIMIT}" + ))); + } let expand_links = args.expand_links; let store = self.store.clone(); let now = self.clock.now(); diff --git a/crates/agent-memory-core/tests/relations.rs b/crates/agent-memory-core/tests/relations.rs index 4b65c9c..4e3a802 100644 --- a/crates/agent-memory-core/tests/relations.rs +++ b/crates/agent-memory-core/tests/relations.rs @@ -24,7 +24,7 @@ use agent_memory_core::clock::{Clock, TestClock}; use agent_memory_core::decay::DecayConfig; use agent_memory_core::domain::{ LinkDirection, LinkKind, LinkOutcome, MemoryError, MemoryType, MemoryView, NewMemory, - RelatedLink, MAX_EXPANDED_LINKS, + RelatedLink, MAX_EXPANDED_LINKS, MAX_LIMIT, }; use agent_memory_core::embed::{Embedder, FakeEmbedder}; use agent_memory_core::service::{ListArgs, MemoryService, SearchArgs, SearchMode}; @@ -405,9 +405,13 @@ fn search_args(query: &str, expand_links: bool) -> SearchArgs { } } +/// An expanded list must carry a limit (REVIEW WR-04, user decision +/// option-a); `MAX_LIMIT` keeps every row of these small fixtures. A +/// flag-less list keeps `limit: None`, so it still exercises the unlimited path. fn list_args(expand_links: bool) -> ListArgs { ListArgs { expand_links, + limit: expand_links.then_some(MAX_LIMIT), ..Default::default() } } diff --git a/crates/agent-memory-core/tests/validation.rs b/crates/agent-memory-core/tests/validation.rs index 542db8b..dbb6697 100644 --- a/crates/agent-memory-core/tests/validation.rs +++ b/crates/agent-memory-core/tests/validation.rs @@ -9,6 +9,9 @@ //! Per RESEARCH Pitfall 2 the matrix includes the VALID boundaries (1, 200 / //! 1, 3_155_760_000), not just the invalid values — an off-by-one in the range //! check must fail these tests, not slip through. +//! +//! It also pins the REVIEW WR-04 option-a rule: a `list` with `expand_links` +//! and no `limit` is rejected as `InvalidArgument` at the same seam. use std::sync::Arc; @@ -119,6 +122,100 @@ async fn valid_limit_boundaries_succeed() { } } +/// REVIEW WR-04, user decision option-a (reject): an expanded list must carry +/// a limit, so every expanded response stays within `200 × MAX_EXPANDED_LINKS`. +#[tokio::test] +async fn expand_links_list_without_a_limit_is_rejected_at_the_service_seam() { + let dir = tempfile::tempdir().expect("tempdir"); + let service = dead_embedder_service(&dir); + + service + .store(new_memory("a searchable pipeline row", MemoryType::Pattern)) + .await + .expect("store succeeds under a dead embedder"); + + // Rejected: expand_links with no limit. + let err = service + .list(ListArgs { + expand_links: true, + ..Default::default() + }) + .await + .expect_err("expand_links without a limit must reject"); + assert!( + matches!(err, MemoryError::InvalidArgument(_)), + "expected InvalidArgument, got: {err:?}" + ); + let msg = err.to_string(); + assert!( + msg.contains("limit is required when expand_links is true"), + "message must name the rule, got: {msg}" + ); + assert!( + msg.contains("between 1 and 200"), + "message must name the valid range, got: {msg}" + ); + assert_eq!( + msg, + "invalid argument: limit is required when expand_links is true; \ + pass a limit between 1 and 200" + ); + + // Range validation runs first: an out-of-bounds limit still reports the range. + let err = service + .list(ListArgs { + expand_links: true, + limit: Some(0), + ..Default::default() + }) + .await + .expect_err("limit 0 must reject even with expand_links"); + assert!( + matches!(err, MemoryError::InvalidArgument(_)), + "expected InvalidArgument, got: {err:?}" + ); + assert!( + err.to_string().contains("limit must be between"), + "range error must win, got: {err}" + ); + + // Accepted: expand_links with a valid limit at both boundaries. + for good in [Some(1), Some(200)] { + let views = service + .list(ListArgs { + expand_links: true, + limit: good, + ..Default::default() + }) + .await + .unwrap_or_else(|e| panic!("expanded list with limit {good:?} must succeed: {e}")); + assert_eq!(views.len(), 1); + assert_eq!( + views[0].related.as_deref(), + Some(&[][..]), + "an expanded row with no relations carries an explicit empty list" + ); + } + + // Unchanged: no expand_links and no limit lists every row, with no related. + let views = service + .list(ListArgs::default()) + .await + .expect("plain unlimited list must succeed"); + assert_eq!(views.len(), 1); + assert!(views[0].related.is_none()); + + // Unchanged: expanded search is already bounded (default 50). + service + .search(SearchArgs { + query: "pipeline".to_string(), + expand_links: true, + ..Default::default() + }) + .await + .expect("expanded search with no limit must succeed"); +} + #[tokio::test] async fn out_of_bounds_ttl_rejected_on_store_and_import() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/agent-memory/src/mcp.rs b/crates/agent-memory/src/mcp.rs index 4bddaaa..b4e2d16 100644 --- a/crates/agent-memory/src/mcp.rs +++ b/crates/agent-memory/src/mcp.rs @@ -92,7 +92,8 @@ pub struct ListArgs { /// Filter by logical scope. Omit for all scopes. #[serde(default)] pub scope: Option, - /// Cap the number of returned rows. + /// Cap the number of returned rows (1-200). Omitted lists every row. + /// Required when `expand_links` is true. #[serde(default)] pub limit: Option, /// Attach up to 10 one-hop related memories to each result as @@ -100,7 +101,9 @@ pub struct ListArgs { /// `out` (this memory -> neighbor) or `in` (neighbor -> this memory); /// `kind` reads naturally from this memory's side (an incoming supersedes /// is `superseded_by`, an incoming caused_by is `caused`). Most recent - /// links first. Defaults to false (no `related` member at all). + /// links first. Defaults to false (no `related` member at all). Requires + /// `limit`: `expand_links=true` without a limit is rejected as + /// invalid_params. #[serde(default)] pub expand_links: bool, } @@ -294,7 +297,7 @@ impl MemoryMcp { } #[tool( - description = "List stored memories newest-first, with optional type/tag/scope/limit filters. Set expand_links=true to attach each memory's linked memories as trimmed related summaries." + description = "List stored memories newest-first, with optional type/tag/scope/limit filters. Set expand_links=true to attach each memory's linked memories as trimmed related summaries; expand_links requires a limit (1-200), and a call with expand_links=true and no limit is rejected as invalid_params. Without expand_links, an omitted limit lists every memory." )] async fn memory_list( &self, @@ -1094,8 +1097,24 @@ mod tests { .to_string(); assert!(!text.contains("related"), "{text}"); - // memory_list, flag on and off. + // memory_list without a limit: expand_links is rejected (REVIEW WR-04 + // option-a) as invalid_params, naming the fix. let args: ListArgs = serde_json::from_str(r#"{"expand_links": true}"#).expect("args"); + let err = server + .memory_list(Parameters(args)) + .await + .expect_err("expand_links without a limit must be rejected"); + assert_eq!(err.code, McpError::invalid_params("x", None).code); + assert!( + err.message + .contains("limit is required when expand_links is true"), + "{}", + err.message + ); + + // memory_list, flag on (with a limit) and off. + let args: ListArgs = + serde_json::from_str(r#"{"expand_links": true, "limit": 10}"#).expect("args"); let body = result_json(&server.memory_list(Parameters(args)).await.expect("list")); let rows = body.as_array().expect("array"); let anchor_row = rows.iter().find(|r| r["id"] == anchor).expect("anchor"); diff --git a/crates/agent-memory/src/rest/handlers.rs b/crates/agent-memory/src/rest/handlers.rs index 3b19e53..17d0600 100644 --- a/crates/agent-memory/src/rest/handlers.rs +++ b/crates/agent-memory/src/rest/handlers.rs @@ -83,11 +83,13 @@ pub struct ListQuery { /// Filter by logical scope. Omit for all scopes. #[serde(default)] pub scope: Option, - /// Cap the number of returned rows. + /// Cap the number of returned rows (1-200). Omitted lists every row. + /// Required when `expand_links` is true. #[serde(default)] pub limit: Option, /// Attach 1-hop `related` summaries to each row (MCP-07). Omitted = false - /// (the v1.0 response, byte-identical). + /// (the v1.0 response, byte-identical). Requires `limit`: + /// `expand_links=true` without a limit is rejected as a 400. #[serde(default)] pub expand_links: bool, } @@ -228,7 +230,9 @@ fn extractor_rejection(status: StatusCode, message: String) -> ApiError { /// `GET /api/memories?type=&tag=&scope=&limit=&expand_links=` → 200 `Vec`. /// /// A malformed query string (`limit=abc`, `expand_links=1`) is a 400 with the -/// `{"error"}` body via [`extractor_rejection`]. +/// `{"error"}` body via [`extractor_rejection`]. `expand_links=true` without a +/// `limit` is a 400 `{"error"}` raised at the service seam (REVIEW WR-04 +/// option-a). pub async fn list_handler( State(state): State>, query: Result, QueryRejection>, @@ -973,6 +977,38 @@ mod tests { ); } + #[tokio::test] + async fn list_with_expand_links_and_no_limit_returns_400_never_500() { + // REVIEW WR-04, user decision option-a: an expanded list must carry a + // limit — it is rejected as client input at the service seam. + let dir = tempfile::tempdir().expect("tempdir"); + let state = test_state(&dir); + + let err = list_handler( + State(state), + Ok(Query(ListQuery { + r#type: None, + tag: None, + scope: None, + limit: None, + expand_links: true, + })), + ) + .await + .expect_err("expand_links without a limit is rejected"); + + let resp = err.into_response(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let body = body_json(resp).await; + assert!( + body["error"] + .as_str() + .expect("400 body carries a string error field") + .contains("limit is required when expand_links is true"), + "the error must tell the caller to pass a limit, got: {body}" + ); + } + #[tokio::test] async fn forget_deletes_then_404s_with_not_found_body() { let dir = tempfile::tempdir().expect("tempdir"); @@ -1538,8 +1574,8 @@ mod tests { } }; - // GET /api/memories?expand_links=true - let rows = body_json(list_raw("?expand_links=true").await).await; + // GET /api/memories?expand_links=true&limit=10 + let rows = body_json(list_raw("?expand_links=true&limit=10").await).await; let rows = rows.as_array().expect("array"); let effect = rows.iter().find(|r| r["id"] == a).expect("effect row"); assert_eq!(effect["related"][0]["id"], b); @@ -1626,10 +1662,10 @@ mod tests { assert_eq!(resp.status().as_u16(), 200); let rows: serde_json::Value = client - .get(format!("{base}/api/memories?expand_links=true")) + .get(format!("{base}/api/memories?expand_links=true&limit=10")) .send() .await - .expect("GET /api/memories?expand_links=true") + .expect("GET /api/memories?expand_links=true&limit=10") .json() .await .expect("json rows"); diff --git a/crates/agent-memory/tests/rest.rs b/crates/agent-memory/tests/rest.rs index 7073212..4e7dd70 100644 --- a/crates/agent-memory/tests/rest.rs +++ b/crates/agent-memory/tests/rest.rs @@ -309,6 +309,97 @@ async fn rest_end_to_end_store_list_search_forget_health() { ); } +/// REVIEW WR-04, user decision option-a (reject): `GET /api/memories?expand_links=true` +/// without a `limit` is a 400 over real HTTP; with a limit, and without +/// `expand_links`, the route behaves exactly as before. +#[tokio::test] +async fn rest_list_with_expand_links_requires_a_limit() { + let server = RestServer::spawn(); + let base = server.base_url(); + let client = reqwest::Client::new(); + + let mut ids = Vec::new(); + for content in ["first linked decision", "second linked decision"] { + let resp = client + .post(format!("{base}/api/memories")) + .json(&serde_json::json!({ "content": content, "type": "DECISION" })) + .send() + .await + .expect("store request"); + assert_eq!(resp.status(), 201, "store must return 201 Created"); + let body: serde_json::Value = resp.json().await.expect("store body is JSON"); + let id = body + .get("id") + .and_then(|v| v.as_i64()) + .unwrap_or_else(|| panic!("store body must carry a numeric id, got: {body}")); + ids.push(id); + } + let (a, b) = (ids[0], ids[1]); + + let resp = client + .post(format!("{base}/api/links")) + .json(&serde_json::json!({ "from_id": a, "to_id": b, "kind": "relates_to" })) + .send() + .await + .expect("link request"); + assert_eq!(resp.status(), 200, "link must return 200"); + + // expand_links without a limit → 400 naming the rule. + let resp = client + .get(format!("{base}/api/memories?expand_links=true")) + .send() + .await + .expect("expanded list without a limit"); + assert_eq!( + resp.status(), + 400, + "expand_links without a limit must be a 400" + ); + let body: serde_json::Value = resp.json().await.expect("400 body is JSON"); + assert!( + body.get("error") + .and_then(|e| e.as_str()) + .is_some_and(|e| e.contains("limit is required when expand_links is true")), + "400 body must tell the caller to pass a limit, got: {body}" + ); + + // expand_links with a limit → 200, the neighbour is attached. + let resp = client + .get(format!("{base}/api/memories?expand_links=true&limit=10")) + .send() + .await + .expect("expanded list with a limit"); + assert_eq!(resp.status(), 200); + let body: serde_json::Value = resp.json().await.expect("list body is JSON"); + let arr = body.as_array().expect("list body is a JSON array"); + let row_a = arr + .iter() + .find(|v| v.get("id").and_then(|i| i.as_i64()) == Some(a)) + .unwrap_or_else(|| panic!("row {a} is listed, got: {body}")); + assert_eq!( + row_a.pointer("/related/0/id").and_then(|i| i.as_i64()), + Some(b), + "row {a} carries its neighbour, got: {row_a}" + ); + + // No expand_links (absent or false) and no limit → every row, no related. + for query in ["", "?expand_links=false"] { + let resp = client + .get(format!("{base}/api/memories{query}")) + .send() + .await + .expect("plain list"); + assert_eq!(resp.status(), 200, "plain list {query:?} must be 200"); + let body: serde_json::Value = resp.json().await.expect("list body is JSON"); + let arr = body.as_array().expect("list body is a JSON array"); + assert_eq!(arr.len(), 2, "every row for {query:?}, got: {body}"); + assert!( + arr.iter().all(|v| v.get("related").is_none()), + "no related member for {query:?}, got: {body}" + ); + } +} + /// Bind guard: a non-loopback `--addr` without `--allow-remote` must be /// refused with an error that names the flag (T-02-10). #[test] diff --git a/crates/agent-memory/tests/tools.rs b/crates/agent-memory/tests/tools.rs index aa4ea55..06ef482 100644 --- a/crates/agent-memory/tests/tools.rs +++ b/crates/agent-memory/tests/tools.rs @@ -479,3 +479,91 @@ fn memory_update_with_an_unknown_field_is_invalid_params_over_stdio() { server.shutdown(); } + +/// REVIEW WR-04, user decision option-a (reject): `memory_list` with +/// `expand_links` and no `limit` is invalid_params over stdio; with a limit, +/// and without `expand_links`, the tool behaves exactly as before. +#[test] +fn memory_list_with_expand_links_requires_a_limit_over_stdio() { + let dir = tempfile::tempdir().expect("tempdir"); + let db_path = dir.path().join("memory.db"); + + let mut server = Server::spawn(&db_path); + server.initialize(); + + let mut store = |request_id: i64, content: &str| -> i64 { + let resp = server.call_tool( + request_id, + "memory_store", + serde_json::json!({ "content": content, "type": "DECISION" }), + ); + tool_text(&resp) + .trim() + .parse() + .expect("memory_store returns a numeric id") + }; + let a = store(2, "first linked decision"); + let b = store(3, "second linked decision"); + + let link_resp = server.call_tool( + 4, + "memory_link", + serde_json::json!({ "from_id": b, "to_id": a, "kind": "relates_to" }), + ); + assert!( + link_resp.get("error").is_none() + && link_resp.pointer("/result/isError") != Some(&serde_json::json!(true)), + "memory_link must succeed: {link_resp}" + ); + + let resp = server.call_tool( + 5, + "memory_list", + serde_json::json!({ "expand_links": true }), + ); + // This is a JSON-RPC error member, NOT an `isError: true` tool result + // (unlike the memory_update unknown-field test above): rmcp 1.8 only turns + // invalid_params messages prefixed "failed to deserialize parameters:" into + // tool errors (router/tool.rs `into_tool_argument_error`), and this + // rejection is raised at the service seam after deserialization succeeded. + assert_eq!( + resp.pointer("/error/code"), + Some(&serde_json::json!(-32602)), + "an expanded list without a limit must be invalid_params: {resp}" + ); + assert!(resp.get("result").is_none(), "no result member: {resp}"); + let message = resp + .pointer("/error/message") + .and_then(|m| m.as_str()) + .unwrap_or_default(); + assert!( + message.contains("limit is required when expand_links is true"), + "the error tells the caller to pass a limit: {resp}" + ); + + // The server is still alive, and expand_links with a limit is unchanged. + let views = tool_views(&server.call_tool( + 6, + "memory_list", + serde_json::json!({ "expand_links": true, "limit": 10 }), + )); + let row_a = views + .iter() + .find(|v| v["id"] == serde_json::json!(a)) + .unwrap_or_else(|| panic!("row {a} is listed: {views:?}")); + assert_eq!( + row_a["related"][0]["id"], + serde_json::json!(b), + "row {a} carries its neighbour: {row_a}" + ); + + // A list without expand_links and without a limit returns every row, no related. + let views = tool_views(&server.call_tool(7, "memory_list", serde_json::json!({}))); + assert_eq!(views.len(), 2, "every row: {views:?}"); + assert!( + views.iter().all(|v| v.get("related").is_none()), + "no related member without expand_links: {views:?}" + ); + + server.shutdown(); +} From 766b8f07e22c6840ad38577fa5a584e9165a80f5 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Sat, 24 Oct 2026 01:55:41 +0200 Subject: [PATCH 41/42] docs(quick-261023-v6w): mark WR-04 resolved (option a, reject) in 04-REVIEW The WR-04 row now reads Resolved (user decision option-a, reject; 2026-10-23). Its evidence cites the service.rs guard, the MCP and REST mappers, the four new tests, the domain/README/tool-description docs, the STATE.md:122 pointer and code commit 969e891. STATE.md is committed separately by the orchestrator. Co-Authored-By: Claude Opus 5.5 --- .../04-memory-update-relations-hybrid-search/04-REVIEW.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md b/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md index 06a88c0..6808042 100644 --- a/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md +++ b/.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md @@ -64,7 +64,7 @@ The workspace test suite was not re-run here, because a verifier runs in paralle | WR-01 | WARNING | **Resolved** for both observable failure modes. The cross-process exclusive lock is **deferred-as-documented** | `sqlite.rs:235-301`: temp file, then version check, then rename, then prune. A post-migration copy is discarded (`:273-282`). `sqlite.rs:315-334` absorbs a failed migrate only when the re-read version is `LATEST`. `busy_timeout` is set before the probe (`:387`) and first in `apply_pragmas` (`:89`). Tests: `sqlite.rs` seam tests and `a_real_lost_migration_race_is_absorbed`, plus `migration_hygiene.rs` two-opener smoke. | | WR-02 | WARNING | **Resolved** | `mcp.rs:169` and `handlers.rs:107` add `#[serde(deny_unknown_fields)]`. The schema advertises `additionalProperties: false`. The MCP wire shape is rmcp's `isError: true` tool result, as documented at `README.md:265-269`. The rmcp interaction looks sound: `_meta` travels outside `arguments`, and the explicit-null deserializers still compose. | | WR-03 | WARNING | **Resolved** (stale wording remains, see IN-03) | `domain.rs:127-149` adds `is_symmetric` / `canonical_endpoints`. `sqlite.rs:977-982` inserts the canonical pair. `sqlite.rs:998-1003` makes symmetric unlink a single bound DELETE. Tests: 8 new in `tests/relations.rs`. The MCP and REST echo tests pass. | -| WR-04 | WARNING | **Deferred-as-documented** (doc-only, by user instruction) | The `domain.rs:360-371` doc is corrected and `README.md:363` adds "pass a `limit` on large stores". Behaviour is unchanged (`service.rs:232-246`). Note: the `memory_list` **tool description** (`mcp.rs:297`), which is what agents read, still does not mention the unbounded case. | +| WR-04 | WARNING | **Resolved** (user decision option-a: reject; quick task 261023-v6w, 2026-10-23) | `service.rs:249-254` rejects `expand_links` without a `limit` as `MemoryError::InvalidArgument` before any store hop, with a message that tells the caller to pass a `limit` (`limit is required when expand_links is true; pass a limit between 1 and 200`). It reaches MCP as `invalid_params` (`mcp.rs:39`; over stdio a JSON-RPC -32602 error) and REST as 400 `{"error"}` (`handlers.rs:171`). `expand_links` with a limit, a list without `expand_links`, and search are unchanged, so the `200 × 10` bound now covers every expanded response. Tests: `tests/validation.rs` `expand_links_list_without_a_limit_is_rejected_at_the_service_seam`, `tests/tools.rs` `memory_list_with_expand_links_requires_a_limit_over_stdio`, `tests/rest.rs` `rest_list_with_expand_links_requires_a_limit`, and the inline `list_with_expand_links_and_no_limit_returns_400_never_500` (`handlers.rs:981`). Docs: `domain.rs:362-371` (`MAX_EXPANDED_LINKS`), `README.md:363` (the advice is now the rule) and `:378` (Bounds row), and the `memory_list` tool description `mcp.rs:300`, which now states the rule. The decision is recorded in `.planning/STATE.md:122`. Commit `969e891`. | | IN-01 | INFO | **Resolved** | `README.md:84-121` adds the Upgrading section: backup name, retention, restore steps including `-wal`/`-shm`, and the newer-schema error text. | | IN-02 | INFO | **Resolved** for the `Path`/`Query` routes the finding named. README now overclaims, see new WR-02 | `handlers.rs:220-226` `extractor_rejection`. It is used at `:236`, `:287`, `:319` and `:397`, and keeps a 5xx for routing faults. | | IN-03 | INFO | **Resolved** | `sqlite.rs:265-293`: no backup is deleted before the new copy has been renamed into place. Tests: `a_failed_backup_write_never_removes_the_previous_backup` and `a_backup_that_cannot_be_moved_into_place_keeps_the_previous_one`. | From 0f1a9fdd8756d2d41cd2dc56b281b74072b291b9 Mon Sep 17 00:00:00 2001 From: hermanngeorge15 Date: Sat, 24 Oct 2026 07:47:22 +0200 Subject: [PATCH 42/42] docs(quick-261023-v6w): WR-04: reject memory_list expand_links without limit Records the quick-task plan and summary, the WR-04 decision (option a, reject) in STATE.md, and the Quick Tasks Completed row. Co-Authored-By: Claude Opus 5.5 --- .planning/STATE.md | 5 +- .../261023-v6w-PLAN.md | 300 ++++++++++++++++++ .../261023-v6w-SUMMARY.md | 112 +++++++ 3 files changed, 415 insertions(+), 2 deletions(-) create mode 100644 .planning/quick/261023-v6w-wr-04-reject-memory-list-expand-links-wi/261023-v6w-PLAN.md create mode 100644 .planning/quick/261023-v6w-wr-04-reject-memory-list-expand-links-wi/261023-v6w-SUMMARY.md diff --git a/.planning/STATE.md b/.planning/STATE.md index e4bd26b..ddb4e59 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -31,7 +31,7 @@ See: .planning/PROJECT.md (updated 2026-07-12) Phase: 5 — Portable Export & Import Plan: Not started Status: Ready to plan -Last activity: 2026-10-23 — Phase 04 complete, transitioned to Phase 5 +Last activity: 2026-10-23 - Completed quick task 261023-v6w: WR-04: reject memory_list expand_links without limit Progress: [░░░░░░░░░░] 0% (0/4 v1.1 phases) @@ -119,7 +119,7 @@ Recent decisions affecting current work: - [Phase 04]: 04-05: T-04-15 revised by user decision option-a (2026-10-23) — search runs the raw FTS5 MATCH first, then one sanitized retry (word tokens quoted and OR-joined) on any FTS5 query-parse error, including the step-time `no such column` class from hyphens/colons; a tokenless query is an empty keyword leg (hybrid answers semantic-only, keyword mode returns empty); identical in both modes, so search never returns InvalidQuery; every search-path internal error is redacted to SEARCH_STORAGE_FAILURE (T-04-19). Supersedes the 04-04 decision that a malformed MATCH is InvalidQuery/400 in hybrid mode. - [Phase 04]: 04-06: sweep backfill writes a vector only through a guarded claim (embedding_status = 0 AND content = embedded text) run first in the writer tx; Store::insert_embedding returns bool. Closes REVIEW CR-03, across processes too. - [Phase 04]: 04-06: relates_to stored canonically as (min id, max id) and unlinked from either end; supersedes/caused_by stay directed (D-03). Storage-only: MCP/REST responses echo request ids; NotFound names the request's from_id first. No migration (REVIEW WR-03). -- [Phase 04]: 04-06: WR-04 documented only: the 200 x 10 expansion bound covers search and list-with-limit; unlimited expanded memory_list returns every row. Behaviour choice (reject / default limit / unbounded) is an open API-contract question for Phase 5/6. +- [Phase 04]: 04-06: WR-04 resolved by user decision option-a (reject, 2026-10-23, quick task 261023-v6w) — memory_list with expand_links and no limit is a client error (MemoryError::InvalidArgument → REST 400 {error} / MCP invalid_params) whose message tells the caller to pass a limit; expand_links with a limit, a list without expand_links, and search (already bounded) are unchanged, so the 200 x 10 expansion bound now covers every expanded response. Supersedes the 04-06 doc-only deferral; closed, nothing left for Phase 5/6. - [Phase 04]: 04-07: deny_unknown_fields added to UpdateArgs/UpdateBody only (new v0.1.0 surfaces, REVIEW WR-02); the [03-01] decision stands for the v0.0.1 DTOs, so the v0.1.0 release-notes delta for existing surfaces is unchanged. Over MCP, rmcp 1.8 reports the rejection as a tool error result (isError: true, "failed to deserialize parameters: unknown field ..."), not a JSON-RPC -32602; REST is a 400 {"error"}. - [Phase 04]: 04-07: backup temp is pid-qualified and a process-wide BACKUP_LOCK serializes backups within one process, so same-process openers never share a temp; prune tolerates NotFound from a concurrent opener - [Phase 04]: 04-07: WR-01 closed with verify-then-rename + tolerate-by-re-probe (no file lock, no new dep); cross-process exclusive lock stays deferred @@ -147,6 +147,7 @@ Watch items: | # | Description | Date | Commit | Directory | |---|-------------|------|--------|-----------| | 261009-hc1 | Release pipeline on GitHub-hosted runners with SLSA provenance, semver-only trigger, all actions SHA-pinned (closes #1); resolves the release-runner decision Phase 3 D-12 deferred to Phase 6 | 2026-10-09 | 9810e31 | [261009-hc1-move-release-yml-to-github-hosted-runner](./quick/261009-hc1-move-release-yml-to-github-hosted-runner/) | +| 261023-v6w | WR-04: reject `memory_list` with `expand_links` and no `limit` as a client error (MCP `invalid_params`, REST 400); user decision option (a) | 2026-10-23 | 969e891 | [261023-v6w-wr-04-reject-memory-list-expand-links-wi](./quick/261023-v6w-wr-04-reject-memory-list-expand-links-wi/) | ## Deferred Items diff --git a/.planning/quick/261023-v6w-wr-04-reject-memory-list-expand-links-wi/261023-v6w-PLAN.md b/.planning/quick/261023-v6w-wr-04-reject-memory-list-expand-links-wi/261023-v6w-PLAN.md new file mode 100644 index 0000000..fb80565 --- /dev/null +++ b/.planning/quick/261023-v6w-wr-04-reject-memory-list-expand-links-wi/261023-v6w-PLAN.md @@ -0,0 +1,300 @@ +--- +phase: quick-261023-v6w +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - crates/agent-memory-core/src/service.rs + - crates/agent-memory-core/src/domain.rs + - crates/agent-memory-core/tests/validation.rs + - crates/agent-memory-core/tests/relations.rs + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - crates/agent-memory/tests/tools.rs + - crates/agent-memory/tests/rest.rs + - README.md + - .planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md + - .planning/STATE.md +autonomous: true +requirements: [WR-04] +user_setup: [] + +estimate: + tokens: 45000 + raw_tokens: 45000 + tasks: 3 + confidence: low + +must_haves: + truths: + - "Over MCP stdio, memory_list with {\"expand_links\": true} and no limit returns a JSON-RPC error with code -32602 (invalid_params) whose message contains 'limit is required when expand_links is true' and 'pass a limit between 1 and 200'; the server stays alive and answers the next call" + - "Over REST, GET /api/memories?expand_links=true with no limit returns HTTP 400 with the JSON body {\"error\": \"invalid argument: limit is required when expand_links is true; pass a limit between 1 and 200\"}" + - "memory_list / GET /api/memories with expand_links=true AND an explicit limit in 1..=200 is unchanged: 200, every row carries a related list (an explicit [] when it has no relations)" + - "memory_list / GET /api/memories WITHOUT expand_links (absent or false) and without a limit is unchanged: every row, no related member" + - "memory_search with expand_links=true and no limit is unchanged (default 50); an out-of-bounds limit combined with expand_links still reports the range error ('limit must be between'), because range validation runs first" + - "The rule lives once, in MemoryService::list, and raises the existing MemoryError::InvalidArgument before any store hop; no new MemoryError variant exists and both transport mappers are untouched" + - "Agents and humans can read the rule: the memory_list tool description and ListArgs schema field docs (mcp.rs), the REST ListQuery docs (handlers.rs), the MAX_EXPANDED_LINKS doc (domain.rs) and README.md all state that expand_links on a list requires a limit" + - "04-REVIEW.md's WR-04 row reads Resolved with file:line + test-name evidence, and STATE.md line 122 records user decision option-a (reject, 2026-10-23, quick task 261023-v6w) as closed, not as an open question" + artifacts: + - path: "crates/agent-memory-core/src/service.rs" + provides: "MemoryService::list rejects expand_links without a limit as MemoryError::InvalidArgument" + contains: "limit is required when expand_links is true" + - path: "crates/agent-memory-core/tests/validation.rs" + provides: "Service-seam proof: rejected without a limit, accepted with 1 and 200, no-expand list and expanded search unaffected" + contains: "expand_links_list_without_a_limit_is_rejected_at_the_service_seam" + - path: "crates/agent-memory/tests/tools.rs" + provides: "MCP stdio end-to-end proof of the rejection and the two unchanged paths" + contains: "memory_list_with_expand_links_requires_a_limit_over_stdio" + - path: "crates/agent-memory/tests/rest.rs" + provides: "REST real-HTTP end-to-end proof of the 400 and the two unchanged paths" + contains: "rest_list_with_expand_links_requires_a_limit" + - path: "crates/agent-memory/src/mcp.rs" + provides: "memory_list tool description that tells agents the rule" + contains: "expand_links requires a limit" + key_links: + - from: "crates/agent-memory-core/src/service.rs MemoryService::list" + to: "crates/agent-memory/src/mcp.rs map_mcp_error (InvalidArgument arm, ~line 38)" + via: "MemoryError::InvalidArgument -> McpError::invalid_params" + pattern: "MemoryError::InvalidArgument\\(_\\)\\) => McpError::invalid_params" + - from: "crates/agent-memory-core/src/service.rs MemoryService::list" + to: "crates/agent-memory/src/rest/handlers.rs map_memory_error (InvalidArgument arm, ~line 169)" + via: "MemoryError::InvalidArgument -> ApiError::BadRequest -> 400 {error}" + pattern: "MemoryError::InvalidArgument\\(_\\) => ApiError::BadRequest" + - from: "crates/agent-memory-core/tests/relations.rs list_args helper" + to: "MemoryService::list" + via: "expanded fixture lists pass limit MAX_LIMIT so the five expansion tests keep their meaning" + pattern: "then_some\\(MAX_LIMIT\\)" +--- + + +Close Phase 4 review finding WR-04 using **user decision option-a (reject, 2026-10-23)**. A `memory_list` call with `expand_links: true` and no `limit` becomes a client error whose message tells the caller to pass a `limit`. That is MCP `invalid_params` and REST 400 `{"error"}`. `expand_links` with an explicit limit is unchanged. A list without `expand_links` is unchanged. Search is unaffected, because it is already bounded (default 50). + +Purpose: T-04-11's `200 x 10` expansion bound then holds for every expanded response, not only for search and list-with-limit. The rule is enforced once at the shared service seam, so the two transports cannot drift. + +Output: one guard in `MemoryService::list` that reuses the existing `MemoryError::InvalidArgument`. Tests at the service seam, over MCP stdio and over real REST HTTP. Existing expansion tests that relied on an unlimited expanded list are refitted to pass a limit. The agent-facing docs (tool description, schema field docs, README, domain doc) state the rule. WR-04 is marked resolved in `04-REVIEW.md` and the decision is recorded in `STATE.md`. + +Decision on error class (planner, verified in code): no new error variant is needed. `MemoryError::InvalidArgument(String)` (`domain.rs:389-390`, Display `invalid argument: {0}`) is already the client-tier variant that `validate_limit` uses. Both transport mappers already route it to the client tier: `map_mcp_error` at `mcp.rs:35-38` to `invalid_params`, and `map_memory_error` at `handlers.rs:167-169` to `ApiError::BadRequest`, which is a 400 `{"error"}`. Neither mapper changes. + + + +@$HOME/.claude/gsd-core/workflows/execute-plan.md +@$HOME/.claude/gsd-core/templates/summary.md + + + +@.planning/STATE.md +@CLAUDE.md +@.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md + + +Extracted from the code at HEAD 63f240b. The line numbers are pre-change, so re-grep after editing. + +crates/agent-memory-core/src/service.rs +- `pub struct ListArgs { mem_type: Option, tag: Option, scope: Option, limit: Option, expand_links: bool }` (lines 62-73, derives Default). +- `pub async fn list(&self, args: ListArgs) -> Result, MemoryError>` (lines 231-246). It calls `crate::domain::validate_limit(args.limit)?`, then `spawn_blocking(store.list)`, then `self.expand_related(views)` when `expand_links` is set. +- The `use crate::domain::{...}` block is at lines 20-23 and does not yet import `MIN_LIMIT` / `MAX_LIMIT`. + +crates/agent-memory-core/src/domain.rs +- `pub const MIN_LIMIT: i64 = 1;` (342), `pub const MAX_LIMIT: i64 = 200;` (346). +- The `MAX_EXPANDED_LINKS` doc is at 357-370. The "REVIEW WR-04 doc correction" paragraph is lines 362-370, and the const is at 371. +- `MemoryError::InvalidArgument(String)` is at 389-390 (`#[error("invalid argument: {0}")]`). +- `pub(crate) fn validate_limit(limit: Option) -> Result<(), MemoryError>` is at 431. On out-of-bounds input it returns `InvalidArgument("limit must be between 1 and 200 (got N)")`. + +crates/agent-memory/src/mcp.rs +- `fn map_mcp_error(e: MemoryError) -> McpError` is at 35. Its InvalidArgument arm goes to `McpError::invalid_params`. No change. +- `pub struct ListArgs` (MCP DTO, schemars) is at 84-106. The `limit` doc is "Cap the number of returned rows." (~95). The `expand_links` doc block is at ~98-104. These doc comments become the JSON-schema descriptions that agents read. +- The `memory_list` `#[tool(description = ...)]` is at 296-298. +- The inline test `expand_links_from_raw_args_attaches_related_on_search_and_list` is at 1053-1115. Its list part parses `{"expand_links": true}` with no limit at ~1098, so the new rule breaks it. +- Inline test helpers: `test_server(&dir)` (521), `result_json` (548), `store_one` (557), `link_raw` (867). `McpError::invalid_params("x", None).code` is the comparison idiom used at 1329. + +crates/agent-memory/src/rest/handlers.rs +- `pub struct ListQuery` is at 75-93. Its `limit` doc is "Cap the number of returned rows." and `expand_links` has a doc too. +- `map_memory_error` is at ~165-179. No change. +- The `list_handler` doc is at 228-231 and the handler at 232-250. +- The inline test `expand_links_from_raw_query_and_body_on_list_and_search` (1515) calls `list_raw("?expand_links=true")` at ~1542 with no limit, so it breaks. +- The inline test `links_routes_work_over_real_http` (~1597) does `GET {base}/api/memories?expand_links=true` at ~1629 with no limit, so it breaks too. +- `list_with_negative_limit_returns_400_never_500` (944-974) is the style model for a new inline 400 test. It builds a `ListQuery { r#type: None, tag: None, scope: None, limit: ..., expand_links: ... }`, calls `list_handler(State(state), Ok(Query(...)))`, then `.expect_err`, `err.into_response()`, `StatusCode::BAD_REQUEST`, and `body_json(resp)["error"]`. +- The `?expand_links=1` case at ~1710 is an extractor rejection, so it is still a 400 and is unaffected. + +rmcp 1.8.0 wire shape (verified in the registry source, `handler/server/tool.rs:100-113` and `router/tool.rs:146-156`). A handler that returns `Err(McpError::invalid_params(msg))` is turned into an `isError: true` tool result ONLY when `msg` starts with "failed to deserialize parameters:". Every other `invalid_params` goes out as a JSON-RPC `error` member with `code: -32602` and no `result` member. The new rejection therefore arrives over stdio as `{"error": {"code": -32602, "message": "invalid argument: limit is required when expand_links is true; pass a limit between 1 and 200"}}`. That is a different shape from the `memory_update` unknown-field test at `tests/tools.rs:418-483`, and the new test's comment must say why. + +Test harness facts: +- `crates/agent-memory/tests/tools.rs` has `Server::spawn(&db_path)`, `initialize()`, `call_tool(id, name, json)`, `tool_text(&resp)`, `tool_views(&resp)` and `shutdown()`. Spawns use the dead Ollama URL. +- `crates/agent-memory/tests/rest.rs` has `RestServer::spawn()` and `base_url()`; the child is killed on Drop. It uses `reqwest::Client` with `#[tokio::test]`. `POST /api/links` takes `{from_id, to_id, kind}` and returns 200. +- `crates/agent-memory-core/tests/validation.rs` has `dead_embedder_service(&dir)` and `new_memory(content, MemoryType)` and already imports `MemoryError`, `ListArgs` and `SearchArgs`. +- `crates/agent-memory-core/tests/relations.rs:408-413` is `fn list_args(expand_links: bool) -> ListArgs`. `list_args(true)` is called at 496, 569, 600, 646 and 795. Every one of those fixtures stores well under 200 rows. Its `use agent_memory_core::domain::{...}` block (25-28) imports `MAX_EXPANDED_LINKS` but not `MAX_LIMIT`. + +Other `MemoryService::list` callers (no change needed): `health_handler` (handlers.rs:436, `limit: Some(1)`, no expand), the MCP `memory_list` tool (mcp.rs:314), and several inline tests using `ServiceListArgs::default()`. There is no CLI list subcommand. + + + + + + + Task 1: Reject expand_links list without a limit at the service seam, proven end-to-end over MCP stdio and REST HTTP + crates/agent-memory-core/src/service.rs, crates/agent-memory-core/tests/validation.rs, crates/agent-memory-core/tests/relations.rs, crates/agent-memory/tests/tools.rs, crates/agent-memory/tests/rest.rs + `git -C /Users/jirihermann/Documents/workspace-1-ideas/unity-in-flow-ai/10-agent-memory merge-base --is-ancestor 63f240b HEAD` succeeds and `grep -n "if expand_links" crates/agent-memory-core/src/service.rs` matches the list path at ~242 (the code this plan was read against). + + - Service seam: `list(ListArgs { expand_links: true, ..Default::default() })` returns `Err(MemoryError::InvalidArgument(_))`. Its Display is exactly `invalid argument: limit is required when expand_links is true; pass a limit between 1 and 200`. + - Service seam: `expand_links: true` with `limit: Some(0)` still returns the range error containing `limit must be between`, because range validation runs first. + - Service seam: `expand_links: true` with `limit: Some(1)` and `Some(200)` returns Ok. The one stored row carries `related` as `Some` and empty. + - Service seam: `ListArgs::default()` (no expand, no limit) returns Ok with every row and `related` None. `search(SearchArgs { query: "pipeline", expand_links: true, ..Default::default() })` returns Ok. + - MCP stdio: `memory_list` with `{"expand_links": true}` returns a response with `/error/code == -32602`, no `result` member, and an `/error/message` containing `limit is required when expand_links is true`. With `{"expand_links": true, "limit": 10}` the linked row's `related[0].id` is its neighbour. With `{}` both rows come back and neither has a `related` member. + - REST HTTP: `GET /api/memories?expand_links=true` returns 400 with `error` containing `limit is required when expand_links is true`. `?expand_links=true&limit=10` returns 200 with the neighbour in `related`. A plain `GET /api/memories` and `?expand_links=false` each return 200 with every row and no `related`. + + +Implements user decision option-a (reject) for REVIEW WR-04. + +RED first. Write the three new tests below and run them; the seam and transport rejection assertions must fail against the current code. Do not commit at the end of this task. The agent-memory library's inline tests in `mcp.rs` and `handlers.rs` stay broken until Task 2 refits them, and CLAUDE.md forbids committing failing tests. Task 2 makes the single code commit. + +1. `crates/agent-memory-core/tests/validation.rs`: add `#[tokio::test] async fn expand_links_list_without_a_limit_is_rejected_at_the_service_seam()` after `valid_limit_boundaries_succeed`, matching that file's style (`dead_embedder_service`, `new_memory`, `expect_err`, `matches!` plus a message `contains`). Store one memory, then assert each service-seam bullet of the behavior block in order: rejection without a limit, checking both the `InvalidArgument` variant and the substrings `limit is required when expand_links is true` and `between 1 and 200`; the range error for `Some(0)` with expand on; Ok for `Some(1)` and `Some(200)` with `related` Some and empty; Ok for `ListArgs::default()` with one row and `related` None; Ok for the expanded search with no limit. Add one sentence to the module doc comment saying this file also pins the WR-04 option-a rule. + +2. `crates/agent-memory/tests/tools.rs`: add `#[test] fn memory_list_with_expand_links_requires_a_limit_over_stdio()` in the style of `memory_update_with_an_unknown_field_is_invalid_params_over_stdio`. Spawn and initialize. Store two memories with `memory_store` and parse their ids from `tool_text`. Link them with `memory_link` arguments `{"from_id": b, "to_id": a, "kind": "relates_to"}` and assert that response is not an error. Then make three `memory_list` calls with distinct request ids: + - `{"expand_links": true}`: assert `resp.pointer("/error/code") == Some(&serde_json::json!(-32602))`, assert `resp.get("result")` is None, and assert the `/error/message` string contains `limit is required when expand_links is true`. + - `{"expand_links": true, "limit": 10}`: through `tool_views`, find row `a` and assert `related[0]["id"] == b`. + - `{}`: two rows, and every row has `get("related")` None. + + Put a comment above the first assertion: this is a JSON-RPC error and not an `isError` tool result, because rmcp 1.8 only converts invalid_params messages prefixed "failed to deserialize parameters:" into tool errors (router/tool.rs `into_tool_argument_error`), while this rejection comes from the service seam. The two later calls prove the server is still alive. Call `shutdown()` at the end. + +3. `crates/agent-memory/tests/rest.rs`: add `#[tokio::test] async fn rest_list_with_expand_links_requires_a_limit()` with its own `RestServer::spawn()` and a `///` doc line, matching the e2e test's assertion style (`assert_eq!(resp.status(), ...)` and `body.get("error").and_then(|e| e.as_str()).is_some_and(|e| e.contains(...))`). POST two memories (201) and POST `/api/links` with `relates_to` (200). Then: + - `GET /api/memories?expand_links=true`: 400 with the error substring. + - `GET /api/memories?expand_links=true&limit=10`: 200, and the first memory's row has the neighbour at `related[0].id`. + - Both `GET /api/memories` and `GET /api/memories?expand_links=false`: 200 with two rows and no `related` member. + +4. GREEN, in `crates/agent-memory-core/src/service.rs` `MemoryService::list`. Keep `crate::domain::validate_limit(args.limit)?` first. Immediately after it, and before `spawn_blocking`, so a rejected call never touches the store, add a guard: when `args.expand_links` is true and `args.limit` is None, return `Err(MemoryError::InvalidArgument(...))` with the message built by `format!` from `MIN_LIMIT` and `MAX_LIMIT`. The message must read exactly `limit is required when expand_links is true; pass a limit between {MIN_LIMIT} and {MAX_LIMIT}`. Add `MIN_LIMIT` and `MAX_LIMIT` to the existing `use crate::domain::{...}` block. Use no `unwrap()`, no new error variant, and no change to either transport mapper. + + Update three comments in the same file: + - The inline comment above `validate_limit` (lines 233-234): an omitted limit stays unlimited only without `expand_links`. + - The `list` doc comment: state the rule, naming REVIEW WR-04 and user decision option-a. Note that it is enforced here so MCP and REST cannot drift, and that the `200 x MAX_EXPANDED_LINKS` bound therefore covers every expanded response. + - The `ListArgs.expand_links` field doc (lines 70-72): it requires `limit`. + +5. Fallout, in `crates/agent-memory-core/tests/relations.rs`: change the `list_args` helper so that `limit` is `expand_links.then_some(MAX_LIMIT)`, and add `MAX_LIMIT` to its `agent_memory_core::domain` import. Add a short `///` on the helper: an expanded list must carry a limit (WR-04 option-a), and `MAX_LIMIT` keeps every row of these small fixtures. `list_args(false)` keeps `limit: None`, so the flag-less byte-identical test at ~530 still exercises the unlimited path. + +6. Run `cargo fmt --all` and then the verify command. Every test in `validation`, `relations`, `tools` and `rest` must pass. + + + cd /Users/jirihermann/Documents/workspace-1-ideas/unity-in-flow-ai/10-agent-memory && cargo test -p agent-memory-core --locked --test validation --test relations && cargo test -p agent-memory --locked --test tools --test rest && test "$(grep -c 'limit is required when expand_links is true' crates/agent-memory-core/src/service.rs)" -ge 1 + + The rejection is live through every layer. `MemoryService::list` returns `InvalidArgument` for an expanded list with no limit. MCP stdio reports it as JSON-RPC -32602 and REST as 400 `{"error"}`, with the same message on both. The expanded-with-limit, no-expand-unlimited and expanded-search paths are proven unchanged. The `validation`, `relations`, `tools` and `rest` test targets are green. Nothing is committed yet. + + + + Task 2: Refit the transport inline tests, state the rule in every agent- and human-facing doc, gate the workspace, and make the code commit + crates/agent-memory/src/mcp.rs, crates/agent-memory/src/rest/handlers.rs, crates/agent-memory-core/src/domain.rs, README.md + +Docs for user decision option-a (reject), REVIEW WR-04. The rule wording is the same everywhere: on `memory_list`, `expand_links` requires a `limit` (1-200); without one the call is a client error that tells you to pass a `limit`; a list without `expand_links` still returns every row when `limit` is omitted; search is bounded by its own limit (default 50). + +1. `crates/agent-memory/src/mcp.rs` covers the tool description, the schema docs and an inline test. + - Replace the `memory_list` `#[tool(description = ...)]` string (line ~297) with exactly: `List stored memories newest-first, with optional type/tag/scope/limit filters. Set expand_links=true to attach each memory's linked memories as trimmed related summaries; expand_links requires a limit (1-200), and a call with expand_links=true and no limit is rejected as invalid_params. Without expand_links, an omitted limit lists every memory.` + - In the MCP `ListArgs` struct only (not `SearchArgs`), change the `limit` doc to say rows are capped at 1-200, that omitted lists every row, and that it is required when `expand_links` is true. Append one sentence to its `expand_links` doc: it requires `limit`, and `expand_links=true` without a limit is rejected as invalid_params. + - In the inline test `expand_links_from_raw_args_attaches_related_on_search_and_list`, change the list-flag-on args to `{"expand_links": true, "limit": 10}`. Directly before that, add an in-process assertion: parsing `{"expand_links": true}` and calling `server.memory_list(Parameters(args)).await.expect_err(...)` yields an error whose `.code` equals `McpError::invalid_params("x", None).code` and whose `.message` contains `limit is required when expand_links is true`. + - The schema test `expand_links_is_an_optional_boolean_in_both_tool_schemas` needs no change. `expand_links` stays an optional boolean with default false; only its description text changes. + +2. `crates/agent-memory/src/rest/handlers.rs` covers the docs and the inline tests. + - `ListQuery` gets the same `limit` and `expand_links` doc sentences as the MCP DTO, with "400" in place of "invalid_params". + - The `list_handler` doc gets one sentence: `expand_links=true` without a `limit` is a 400 `{"error"}` raised at the service seam (REVIEW WR-04 option-a). + - In `expand_links_from_raw_query_and_body_on_list_and_search`, change `list_raw("?expand_links=true")` to `list_raw("?expand_links=true&limit=10")`. + - In `links_routes_work_over_real_http`, change the GET URL and its `expect` label to `?expand_links=true&limit=10`. + - Add an inline `#[tokio::test] async fn list_with_expand_links_and_no_limit_returns_400_never_500()`, mirroring `list_with_negative_limit_returns_400_never_500` exactly, with `limit: None, expand_links: true`. It asserts `StatusCode::BAD_REQUEST` and that `body["error"]` contains `limit is required when expand_links is true`. + +3. `crates/agent-memory-core/src/domain.rs`, the `MAX_EXPANDED_LINKS` doc. Rewrite the "What this bounds (REVIEW WR-04 doc correction)" paragraph (lines 362-370) so it is no longer a pending decision. New heading phrase: "What this bounds (REVIEW WR-04, user decision option-a, 2026-10-23)". The paragraph must say: + - each result carries at most `MAX_EXPANDED_LINKS` neighbours; + - search is always bounded by its limit (1–[`MAX_LIMIT`], default 50); + - [`crate::service::MemoryService::list`] rejects `expand_links` without a `limit` as [`MemoryError::InvalidArgument`] (REST 400 / MCP `invalid_params`) before any store hop; + - so every expanded search or list returns at most `200 × 10` summaries, and T-04-11's bound covers every expanded response; + - a list without `expand_links` still returns every row when `limit` is omitted (v1.0 behaviour, RESEARCH Pitfall 6). + + Drop the old closing sentence that left the unlimited-expanded-list behaviour as an unsettled contract question. Rustdoc intra-doc links must resolve; if the `MemoryService::list` path link fails `cargo doc`/clippy, use plain backticks instead. + +4. `README.md` (pre-change line numbers; re-grep): + - `:152`, `memory_list` row of the MCP tools table: `limit` becomes "optional integer, valid range 1–200; omitted = unlimited, **required when `expand_links` is true**". + - `:229`: "(omitted returns everything)" becomes "(omitted returns everything, except that `expand_links: true` requires a `limit`)". + - `:362-363`: replace the advisory bullet about passing a `limit` with the rule. `memory_list` with `expand_links` **requires a `limit`** (1–200). Without one the call is rejected as a client error (HTTP 400 / MCP `invalid_params`) whose message tells you to pass a `limit`, so an expanded response is always bounded at 200 results × 10 neighbors. `memory_search` is bounded by its own `limit` (default 50). Without `expand_links`, an omitted `limit` still lists every memory. + - `:373`, Bounds table `limit` row: the `memory_list` cell becomes "`memory_list` (default unlimited; required with `expand_links`)". + - `:421`, REST table `GET /api/memories` row: "omitted = unlimited" becomes "omitted = unlimited, but required with `expand_links=true` (400 without one)". + - `:432-436`, the out-of-bounds paragraph: append one sentence. `GET /api/memories?expand_links=true` without a `limit` is also a 400 (`invalid argument: limit is required when expand_links is true; pass a limit between 1 and 200`), and MCP `memory_list` rejects the same call as `invalid_params`. + - `:464`, curl example: add `&limit=20`, giving `curl 'localhost:7437/api/memories?type=DECISION&expand_links=true&limit=20'`. As written it would now return 400. + - Do NOT add a fourth item to "Behavioral changes in v0.1.0". That list is the v0.0.1→v0.1.0 delta, and `expand_links` did not exist in v0.0.1, so "Three deliberate contract changes" stays accurate. Item 2 ("omitted values behave exactly as before") stays true for every v0.0.1 call shape. + +5. Gate, then make the single code commit. + - Run, in the background because they are long: `cargo fmt --all -- --check`, `cargo clippy --workspace --all-targets --locked -- -D warnings`, `cargo test --workspace --locked`. All three must be green, with 0 failed. Fix anything red before committing; do not commit red. + - Stage only these explicit paths: `crates/agent-memory-core/src/service.rs`, `crates/agent-memory-core/src/domain.rs`, `crates/agent-memory-core/tests/validation.rs`, `crates/agent-memory-core/tests/relations.rs`, `crates/agent-memory/src/mcp.rs`, `crates/agent-memory/src/rest/handlers.rs`, `crates/agent-memory/tests/tools.rs`, `crates/agent-memory/tests/rest.rs`, `README.md`. Never stage `CLAUDE.md`, `.claude/`, `.mcp.json`, `.gitignore`, `10-agent-memory.md`, `claude-code-harness-engineering-guide-v2.md`, `.planning/research/.cache/` or `.planning/config.json`, and never use `git add -A` or `git add .`. + - Run `git diff --cached --name-only` and confirm exactly those 9 paths. + - Commit on local `main` as `fix(quick-261023-v6w): reject memory_list expand_links without a limit (WR-04)`. The body names user decision option-a, the reused `InvalidArgument` variant, and the MCP -32602 / REST 400 shapes. End it with the session's commit attribution trailer. + - Do not push, tag or release. Record the short SHA for Task 3. + + + cd /Users/jirihermann/Documents/workspace-1-ideas/unity-in-flow-ai/10-agent-memory && cargo fmt --all -- --check && cargo clippy --workspace --all-targets --locked -- -D warnings && cargo test --workspace --locked && test "$(grep -c 'expand_links requires a limit' crates/agent-memory/src/mcp.rs)" -ge 1 && test "$(awk '/Hard cap on the 1-hop neighbors/,/pub const MAX_EXPANDED_LINKS/' crates/agent-memory-core/src/domain.rs | grep -c 'InvalidArgument')" -ge 1 && test "$(awk '/Hard cap on the 1-hop neighbors/,/pub const MAX_EXPANDED_LINKS/' crates/agent-memory-core/src/domain.rs | grep -c 'deferred API-contract')" -eq 0 && test "$(awk '/^\*\*Expansion\./,/^#### Bounds/' README.md | grep -c 'requires a')" -ge 1 && test "$(awk '/^\*\*Expansion\./,/^#### Bounds/' README.md | grep -c 'large stores')" -eq 0 && test "$(grep -c 'expand_links=true&limit=20' README.md)" -ge 1 && test "$(grep '^| `limit` |' README.md | grep -c 'expand_links')" -ge 1 && git log -1 --format=%s | grep -q 'quick-261023-v6w' && test -z "$(git diff --name-only HEAD -- crates README.md)" + + Workspace fmt, clippy (-D warnings) and test are all green. The tool description, the MCP and REST list DTO docs, the `MAX_EXPANDED_LINKS` doc and README.md all state the rule, and the README curl example passes a limit. One code commit on local `main` contains exactly the 9 code and README paths. Nothing is pushed or tagged, and no local-only file is staged. + + + + Task 3: Mark WR-04 resolved in 04-REVIEW.md and record the decision as closed in STATE.md + .planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md, .planning/STATE.md + +Records user decision option-a (reject, 2026-10-23, quick task 261023-v6w) for REVIEW WR-04. + +1. Collect evidence with `grep -n` against the committed code, not the pre-change line numbers in this plan: + - the guard line(s) in `crates/agent-memory-core/src/service.rs`; + - the `MAX_EXPANDED_LINKS` doc range in `crates/agent-memory-core/src/domain.rs`; + - the `memory_list` tool description line in `crates/agent-memory/src/mcp.rs`; + - the README rule bullet and the Bounds row; + - the InvalidArgument arms of `map_mcp_error` and `map_memory_error`; + - the Task 2 commit short SHA. + +2. In `04-REVIEW.md`, edit only the `| WR-04 | WARNING | ... |` row in the "Prior findings — resolution" table (line 67). Leave the frontmatter counts and every other row untouched. + - Resolution cell: `**Resolved** (user decision option-a: reject; quick task 261023-v6w, 2026-10-23)`. + - Evidence cell: written in the CR-02 row's style, with backticked `file:line` references, test names and a STATE.md pointer. It must say that `service.rs:` rejects `expand_links` without a `limit` as `MemoryError::InvalidArgument` before any store hop, with the message telling the caller to pass a `limit`. That reaches MCP as `invalid_params` (`mcp.rs:`; over stdio a JSON-RPC -32602 error) and REST as 400 `{"error"}` (`handlers.rs:`). `expand_links` with a limit, a list without `expand_links` and search are unchanged, so the `200 × 10` bound now covers every expanded response. Name the tests: `tests/validation.rs` `expand_links_list_without_a_limit_is_rejected_at_the_service_seam`, `tests/tools.rs` `memory_list_with_expand_links_requires_a_limit_over_stdio`, `tests/rest.rs` `rest_list_with_expand_links_requires_a_limit`, and the inline `list_with_expand_links_and_no_limit_returns_400_never_500`. Name the docs: `domain.rs:`, `README.md:` (the advice is now the rule), and the `memory_list` tool description `mcp.rs:`, which now states the rule. End with: "The decision is recorded in `.planning/STATE.md:122`. Commit ``." + - The row stays a single table line. + +3. In `.planning/STATE.md`, replace only line 122, the `- [Phase 04]: 04-06: WR-04 ...` decision bullet. Keep the `- [Phase 04]: 04-06: WR-04` prefix so the entry stays in place and searchable. New text: `- [Phase 04]: 04-06: WR-04 resolved by user decision option-a (reject, 2026-10-23, quick task 261023-v6w) — memory_list with expand_links and no limit is a client error (MemoryError::InvalidArgument → REST 400 {error} / MCP invalid_params) whose message tells the caller to pass a limit; expand_links with a limit, a list without expand_links, and search (already bounded) are unchanged, so the 200 x 10 expansion bound now covers every expanded response. Supersedes the 04-06 doc-only deferral; closed, nothing left for Phase 5/6.` + - Do NOT touch the "Quick Tasks Completed" table, the last-activity line or any other line. The orchestrator writes those. + +4. Stage exactly these two paths and nothing else: `.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md` and `.planning/STATE.md`. Confirm with `git diff --cached --name-only`. Commit on local `main` as `docs(quick-261023-v6w): mark WR-04 resolved (option a, reject) in 04-REVIEW and STATE`, ending with the session's commit attribution trailer. Do not push or tag. + + + cd /Users/jirihermann/Documents/workspace-1-ideas/unity-in-flow-ai/10-agent-memory && R=.planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md && test "$(grep -c '^| WR-04 |' $R)" -eq 1 && test "$(grep '^| WR-04 |' $R | grep -c '\*\*Resolved\*\*')" -eq 1 && test "$(grep '^| WR-04 |' $R | grep -c '261023-v6w')" -eq 1 && test "$(grep '^| WR-04 |' $R | grep -c 'Deferred-as-documented')" -eq 0 && test "$(grep '^| WR-04 |' $R | grep -c 'memory_list_with_expand_links_requires_a_limit_over_stdio')" -eq 1 && test "$(grep -c '04-06: WR-04' .planning/STATE.md)" -eq 1 && test "$(grep '04-06: WR-04' .planning/STATE.md | grep -c 'option-a')" -eq 1 && test "$(grep '04-06: WR-04' .planning/STATE.md | grep -c '261023-v6w')" -eq 1 && test "$(grep '04-06: WR-04' .planning/STATE.md | grep -c 'open API-contract question')" -eq 0 && git log -1 --format=%s | grep -q 'docs(quick-261023-v6w)' && test "$(git diff-tree --no-commit-id --name-only -r HEAD | LC_ALL=C sort | tr '\n' ' ')" = ".planning/STATE.md $R " + + The WR-04 row in 04-REVIEW.md reads Resolved, with post-change file:line evidence, the four test names, the STATE.md pointer and the code commit SHA. STATE.md line 122 records option-a (reject, 2026-10-23, quick task 261023-v6w) as closed. A docs commit on local `main` touches exactly those two files. + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| MCP client → `memory_list` tool (stdio) | Untrusted tool arguments (`expand_links`, `limit`) from any agent | +| HTTP client → `GET /api/memories` (loopback REST) | Untrusted query string from any local process | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Severity | Disposition | Mitigation Plan | +|-----------|----------|-----------|----------|-------------|-----------------| +| T-261023-01 | Denial of Service | `MemoryService::list` with `expand_links` and no limit | medium | mitigate | The guard in `service.rs` rejects the unbounded expanded list before `spawn_blocking`, so no full-table read and no N×10 neighbour fan-out runs. Combined with `validate_limit` (1..=200) and `MAX_EXPANDED_LINKS = 10`, every expanded response is capped at 200 × 10 summaries (closes the T-04-11 gap the review named). | +| T-261023-02 | Information Disclosure | rejection error text on MCP/REST | low | mitigate | The message is a static, `format!`-built string from `MIN_LIMIT`/`MAX_LIMIT`, raised before any store hop, so no SQLite or path text can reach the client. It is classified client-tier (`InvalidArgument` → `invalid_params` / 400) so agents can self-correct instead of entering an internal-error retry loop. | +| T-261023-03 | Tampering | callers relying on unlimited expanded lists | low | accept | `expand_links` is new in unreleased v0.1.0 (absent in v0.0.1), so no released consumer depends on the unbounded shape. The README "Behavioral changes in v0.1.0" delta is unaffected, and the rejection message names the fix. | + +No package installs in this plan, so no supply-chain row applies. + + + +- `cargo fmt --all -- --check`, `cargo clippy --workspace --all-targets --locked -- -D warnings` and `cargo test --workspace --locked` are all green, with 0 failed, before the code commit. +- The new tests pass: `expand_links_list_without_a_limit_is_rejected_at_the_service_seam` (core), `memory_list_with_expand_links_requires_a_limit_over_stdio` (MCP stdio), `rest_list_with_expand_links_requires_a_limit` (REST HTTP) and `list_with_expand_links_and_no_limit_returns_400_never_500` (REST inline). The refitted expansion tests in `relations.rs`, `mcp.rs` and `handlers.rs` still pass. +- `git log -2 --format=%s` shows the `fix(quick-261023-v6w)` code commit followed by the `docs(quick-261023-v6w)` planning commit. `git status` shows the user's local files (`CLAUDE.md`, `.claude/`, `.mcp.json`, `.gitignore`, `10-agent-memory.md`, `claude-code-harness-engineering-guide-v2.md`, `.planning/research/.cache/`, `.planning/config.json`) still unstaged and uncommitted. Nothing is pushed or tagged. + + + +- An expanded `memory_list` with no limit is rejected with the same message on both transports: MCP `invalid_params` (JSON-RPC -32602 over stdio) and REST 400 `{"error"}`. The message tells the caller to pass a `limit` between 1 and 200. +- Expanded list with a limit, list without expand (limited or unlimited), and expanded search behave exactly as before. +- No new `MemoryError` variant, no change to `map_mcp_error` or `map_memory_error`, and no `unwrap()` in production code. +- The tool description, schema field docs, `domain.rs` doc and README state the rule, and the README curl example is valid under it. +- WR-04 is resolved in 04-REVIEW.md with evidence, and STATE.md records the decision as closed. + + + +Create `.planning/quick/261023-v6w-wr-04-reject-memory-list-expand-links-wi/261023-v6w-SUMMARY.md` when done. Include the two commit SHAs, the final test count from `cargo test --workspace --locked` (as reported, not estimated), and the post-change file:line evidence used in the 04-REVIEW.md row. + diff --git a/.planning/quick/261023-v6w-wr-04-reject-memory-list-expand-links-wi/261023-v6w-SUMMARY.md b/.planning/quick/261023-v6w-wr-04-reject-memory-list-expand-links-wi/261023-v6w-SUMMARY.md new file mode 100644 index 0000000..da4aa5a --- /dev/null +++ b/.planning/quick/261023-v6w-wr-04-reject-memory-list-expand-links-wi/261023-v6w-SUMMARY.md @@ -0,0 +1,112 @@ +--- +phase: quick-261023-v6w +plan: 01 +subsystem: memory-service / MCP + REST list transport +status: complete +tags: [WR-04, expand_links, limit, validation, review-closure] +requires: [Phase 04 (expand_links, validate_limit, InvalidArgument tiering)] +provides: [memory_list expand_links requires a limit; 200 x 10 bound on every expanded response] +affects: [MemoryService::list, memory_list MCP tool, GET /api/memories, README] +tech-stack: + added: [] + patterns: [single validation rule at the shared service seam, reuse of the client-tier InvalidArgument variant] +key-files: + created: [] + modified: + - crates/agent-memory-core/src/service.rs + - crates/agent-memory-core/src/domain.rs + - crates/agent-memory-core/tests/validation.rs + - crates/agent-memory-core/tests/relations.rs + - crates/agent-memory/src/mcp.rs + - crates/agent-memory/src/rest/handlers.rs + - crates/agent-memory/tests/tools.rs + - crates/agent-memory/tests/rest.rs + - README.md + - .planning/phases/04-memory-update-relations-hybrid-search/04-REVIEW.md + - .planning/STATE.md +key-decisions: + - "WR-04 resolved by user decision option-a (reject): memory_list with expand_links and no limit is MemoryError::InvalidArgument (MCP invalid_params / REST 400), enforced once in MemoryService::list" +requirements: [WR-04] +metrics: + completed: 2026-10-24 + duration: ~40 min +actuals: + tokens: 9600 + tasks: 3 + commits: 2 +--- + +# Quick Task 261023-v6w: Reject memory_list expand_links without a limit (WR-04) Summary + +`MemoryService::list` now rejects `expand_links: true` with no `limit` using the existing `MemoryError::InvalidArgument`. The check runs before any store hop, and the message reads `invalid argument: limit is required when expand_links is true; pass a limit between 1 and 200`. Over MCP stdio that is a JSON-RPC -32602 error, and over REST it is a 400 `{"error"}`. As a result, the T-04-11 `200 x 10` expansion bound now covers every expanded response. + +## Commits + +| Task | Commit | Subject | +|------|--------|---------| +| 1+2 | `969e891` | fix(quick-261023-v6w): reject memory_list expand_links without a limit (WR-04) | +| 3 | `766b8f0` | docs(quick-261023-v6w): mark WR-04 resolved (option a, reject) in 04-REVIEW | + +Task 1 had no commit of its own, as the plan required: until Task 2 refitted the inline tests, the library tests stayed red. `.planning/STATE.md` line 122 is edited but left uncommitted, because the orchestrator commits it. + +## Gate results (before the code commit) + +- `cargo fmt --all -- --check`: **clean** +- `cargo clippy --workspace --all-targets --locked -- -D warnings`: **clean, 0 warnings** +- `cargo test --workspace --locked`: **217 passed, 0 failed, 2 ignored** across 18 test binaries (exit 0) + +TDD RED was confirmed before the fix. All three new tests failed on their rejection assertion against the old code: +- the service seam returned `Ok(rows)`; +- stdio returned a `result` with no `error`; +- REST returned 200 instead of 400. + +## Changes + +- **service.rs:249-254**: the guard, placed after `validate_limit`, so a range error still wins for a bad limit. `MIN_LIMIT` and `MAX_LIMIT` are imported for the `format!`. The `list` doc, the inline comment and the `ListArgs.expand_links` doc now state the rule. +- **domain.rs:362-371**: the `MAX_EXPANDED_LINKS` doc is rewritten as the settled option-a contract, and the "deferred API-contract" sentence is dropped. +- **mcp.rs:300**: the tool description now says "expand_links requires a limit (1-200) ... rejected as invalid_params". The MCP `ListArgs` `limit`/`expand_links` docs are updated (`SearchArgs` is untouched). The inline test `expand_links_from_raw_args_attaches_related_on_search_and_list` now asserts the in-process invalid_params error, then lists with `limit: 10`. +- **handlers.rs**: the `ListQuery` docs and the `list_handler` doc are updated (`SearchRequest` is untouched). Two inline tests now pass `&limit=10`. A new inline test, `list_with_expand_links_and_no_limit_returns_400_never_500` (`:981`), was added. +- **tests**: + - `validation.rs:128` `expand_links_list_without_a_limit_is_rejected_at_the_service_seam` + - `tools.rs:487` `memory_list_with_expand_links_requires_a_limit_over_stdio` + - `rest.rs:316` `rest_list_with_expand_links_requires_a_limit` + - The `relations.rs` `list_args` helper now uses `limit: expand_links.then_some(MAX_LIMIT)`. +- **README.md**: + - the `memory_list` row in the MCP tools table; + - the `memory_list` limit sentence; + - the Expansion rule bullet (`:363`); + - the Bounds row (`:378`); + - the REST `GET /api/memories` row; + - one sentence added to the out-of-bounds paragraph; + - `&limit=20` added to the curl example. + - "Behavioral changes in v0.1.0" is unchanged, as the plan specified. +- Both transport mappers are unchanged: `map_mcp_error` at `mcp.rs:39` and `map_memory_error` at `handlers.rs:171`. No new `MemoryError` variant was added, and no `unwrap()` was added to production code. + +## Evidence used in the 04-REVIEW.md WR-04 row + +- `service.rs:249-254`: the guard +- `mcp.rs:39` and `handlers.rs:171`: the InvalidArgument arms of the two mappers +- `mcp.rs:300`: the tool description +- `domain.rs:362-371`: the `MAX_EXPANDED_LINKS` doc +- `README.md:363`: the rule bullet; `README.md:378`: the Bounds row +- the four tests named above; `.planning/STATE.md:122`; commit `969e891` + +## Deviations from Plan + +1. **Task 3 commit excludes STATE.md (orchestrator constraint).** The plan has the Task 3 commit cover both `04-REVIEW.md` and `STATE.md`. The orchestrator's instructions say not to commit `.planning/STATE.md`. So `766b8f0` touches only `04-REVIEW.md`, and the line-122 edit is still in the working tree for the orchestrator's docs commit. Consequently, the last check in the plan's Task 3 verify, which expects both paths in HEAD, is satisfied only once the orchestrator commits. Every other Task 3 check passes. +2. **The tracer gate was not a human checkpoint.** `workflow.auto_advance` is false, which would normally mean a `checkpoint:human-verify` after the tracer task. The orchestrator's instruction was to execute every task and stop only for a genuine human decision. The tracer `` is fully automated and passed end-to-end (validation 5, relations 27, tools 7, rest 3). Also, the plan forbids a tracer commit, so there was nothing to checkpoint on. I continued to Task 2. +3. **Scoped README rewrap.** The edited `memory_list` limit sentence was re-wrapped to keep the paragraph's line width. The text is as the plan specified. + +## Known Stubs + +None. + +## Threat Flags + +None. This change shrinks the attack surface (T-261023-01 mitigated). The rejection message is static, built only from public bounds (T-261023-02). + +## Self-Check: PASSED + +- All 11 modified files exist, and the 9 code/README files are in `969e891`. +- `969e891` and `766b8f0` are both on local `main`. Nothing was pushed or tagged. +- None of the user's local files were staged: `CLAUDE.md`, `.claude/`, `.mcp.json`, `.gitignore`, `10-agent-memory.md`, the harness guide, `.planning/research/.cache/` and `.planning/config.json` are still untracked or unstaged.