Improve memory eval source reconciliation and depth reporting

crates/tinymemory-integrations/examples/memory_eval/llm.rs

@@ -17,10 +17,18 @@
 
 use serde_json::{Value, json};
 
+/// Changes when the answer instruction changes, so reports remain comparable.
+pub(crate) const PROMPT_VERSION: &str = "source-reconciliation-v2";
+
 /// What the model is told.
-const SYSTEM: &str = "You are an assistant with a long-term memory. The user's message \
-     starts with what your memory recalled, then their question. Answer the question in one \
-     short sentence, using only the memory. If the memory does not say, answer \"unknown\".";
+const SYSTEM: &str = "Answer the question from all memory sections. When two sources have \
+     incompatible values and one has no date, do not assume which is current: name BOTH values \
+     and their sources. Example: an undated policy says shipping takes four days; a dated message \
+     says operations now says six days. Answer: \"The policy says four days, while operations says \
+     six days.\" When a newer dated message explicitly replaces an older dated value, answer \
+     with ONLY the new value and omit the old value even if the question asks what changed. \
+     Example: May says Alice owns a job; June says the job moved to Bob. Answer: \"Bob owns the \
+     job.\" Keep answers short. If no evidence, say unknown.";
 
 /// One answer and what it cost.
 pub(crate) struct Answer {

crates/tinymemory-integrations/examples/memory_eval/main.rs

@@ -48,6 +48,8 @@
 //!   pre-turn hook with its 5000 ms deadline, policy and logged tool results.
 //! - `--layout v3`: use CortexDB's per-person scope tree and pooled chats.
 //! - `--team-limit <n>`: override the host's team-conversation budget.
+//! - `--brain-limit <n>`: override the number of ranked brain documents in a
+//!   pack, for accuracy-versus-context-budget experiments.
 //! - `--loop-guard`: replay a JSON-scripted 500-turn thread and inspect all
 //!   stored items for an injected `<memory-context>` tag.
 //!
@@ -137,6 +139,7 @@ struct Args {
     host: String,
     layout: String,
     team_limit: Option<usize>,
+    brain_limit: Option<usize>,
     date_hint: bool,
     scale_events: Option<usize>,
     scale_position: String,
@@ -148,6 +151,10 @@ struct Args {
 }
 
 fn args() -> Result<Args, Error> {
+    parse_args(std::env::args().skip(1))
+}
+
+fn parse_args(raw: impl IntoIterator<Item = String>) -> Result<Args, Error> {
     let mut parsed = Args {
         engine: if std::env::var("CORTEX_DB_URL").is_ok() {
             "cortex".into()
@@ -162,6 +169,7 @@ fn args() -> Result<Args, Error> {
         host: "scripted".into(),
         layout: "legacy".into(),
         team_limit: None,
+        brain_limit: None,
         date_hint: false,
         scale_events: None,
         scale_position: "middle".into(),
@@ -171,7 +179,7 @@ fn args() -> Result<Args, Error> {
         expect_derived: false,
         loop_guard: false,
     };
-    let mut raw = std::env::args().skip(1);
+    let mut raw = raw.into_iter();
     while let Some(flag) = raw.next() {
         let mut value = || raw.next().ok_or(format!("{flag} needs a value"));
         match flag.as_str() {
@@ -184,6 +192,7 @@ fn args() -> Result<Args, Error> {
             "--host" => parsed.host = value()?,
             "--layout" => parsed.layout = value()?,
             "--team-limit" => parsed.team_limit = Some(value()?.parse()?),
+            "--brain-limit" => parsed.brain_limit = Some(value()?.parse()?),
             "--date-hint" => parsed.date_hint = true,
             "--scale-events" => parsed.scale_events = Some(value()?.parse()?),
             "--scale-position" => parsed.scale_position = value()?,
@@ -372,6 +381,9 @@ async fn main() -> Result<(), Error> {
     if let Some(team_limit) = args.team_limit {
         policy.team_limit = team_limit;
     }
+    if let Some(brain_limit) = args.brain_limit {
+        policy.brain_limit = brain_limit;
+    }
     let eval = Eval {
         engine: engine.clone(),
         inspector,
@@ -487,11 +499,13 @@ async fn main() -> Result<(), Error> {
             "flags": flags,
             "models": models,
             "probe_answer_model": eval.llm.as_ref().map(|llm| llm.model.as_str()),
+            "probe_answer_prompt": eval.llm.as_ref().map(|_| llm::PROMPT_VERSION),
             "server": server,
             "engine": engine.descriptor().id,
             "host": args.host,
             "layout": args.layout,
             "team_limit": eval.policy.team_limit,
+            "brain_limit": eval.policy.brain_limit,
             "pre_turn_timeout_ms": PRE_TURN_TIMEOUT.as_millis() as u64,
             "date_hint": args.date_hint,
             "scale_events": args.scale_events,

crates/tinymemory-integrations/examples/memory_eval/main_tests.rs

@@ -2,6 +2,18 @@
 
 use super::*;
 
+#[test]
+fn brain_limit_override_is_parsed_without_changing_the_default() {
+    assert_eq!(parse_args(Vec::new()).unwrap().brain_limit, None);
+    assert_eq!(
+        parse_args(["--brain-limit".into(), "24".into()])
+            .unwrap()
+            .brain_limit,
+        Some(24)
+    );
+    assert!(parse_args(["--brain-limit".into(), "bad".into()]).is_err());
+}
+
 #[tokio::test]
 async fn a_probe_does_not_leave_its_question_in_the_measured_corpus() {
     let engine: Arc<dyn MemoryEngine> = Arc::new(ReferenceEngine::new());

docs/evals/README.md

@@ -72,8 +72,10 @@ file.
 
 Flags (after `--`): `--engine reference|cortex|tinyhumans`, `--only <scenario>`,
 `--enrich-wait <secs>`, `--json <path>`, `--label <name>`, `--llm`,
-`--host openhuman` and `--loop-guard`. With
-`CORTEX_DB_KEEP=1` the run's data is left in place for inspection.
+`--host openhuman`, `--brain-limit <n>`, and `--loop-guard`.
+`--brain-limit` overrides the host's brain-document count and records it in
+the JSON report for retrieval-depth sweeps. `CORTEX_DB_KEEP=1` leaves the
+run's data in place for inspection.
 
 ### How a scenario runs
 

docs/evals/openhuman-host.md

@@ -254,3 +254,48 @@ that the longer deadline removes the timeout for these two probes, while the
 paraphrase still needs a retrieval-quality improvement. It is a fresh
 collection and cannot be treated as a paired timeout comparison with the
 1500 ms runs above.
+
+## Accuracy follow-up after #255
+
+Fresh CortexDB v0.10.4/OpenRouter runs used the same 100-document middle
+needle fixture, seed 251, v3 layout, three team turns, 5000 ms deadline, and
+30-second ranked-readiness wait. Each run had a new collection. The literal
+owner query hit and the semantic paraphrase missed in all three settings:
+
+| Setting | Pack hits | Paraphrase | Mean pack tokens | Probe timeout |
+| --- | ---: | ---: | ---: | ---: |
+| Default server, six brain documents | 1/2 | miss | 130 | 0/2 |
+| Max Recall server flags, six brain documents | 1/2 | miss | 130 | 0/2 |
+| Default server, 24 brain documents | 1/2 | miss | 512 | 0/2 |
+
+The Max Recall profile enables three HyDE passages, entity-vector seeding,
+assistant excerpts, and enforced bi-temporal validity. Neither it nor a
+fourfold increase in the displayed brain documents recovered this fixture's
+paraphrase. The wider section spent almost four times the context tokens, so
+the production six-document default stays unchanged. `--brain-limit <n>` now
+allows another depth to be measured without editing that default; reports
+record the chosen value.
+
+The same seven-question `team_handoff,conflicts` live slice exposed an answer
+problem after retrieval: all seven packs held their expected evidence, but
+the old benchmark model prompt answered only 5/7 correctly. Both misses
+chose the undated five-day refund document and omitted a conversation saying
+Finance had changed refunds to ten days. The final prompt asks the model to
+distinguish an unresolved conflict (undated policy versus dated report) from
+an explicit update of an older dated value. Reports record its version as
+`source-reconciliation-v2`.
+
+| Fresh run | Pack hit | Model answer, recall | Model answer, synthesis | Timeouts |
+| --- | ---: | ---: | ---: | ---: |
+| Live slice, original prompt | 7/7 | 5/7 | 5/7 | 0 |
+| Live slice, source reconciliation | 7/7 | 7/7 | 6/7 | 0 |
+| Full mock, first conflict prompt | 51/58 | 43/58 | 44/58 | 0 |
+| Full mock, source reconciliation | 51/58 | 50/58 | 50/58 | 0 |
+
+The remaining live synthesis miss named Acme's current Team plan but also
+mentioned the superseded Enterprise plan, which the strict answer grade
+rejects. Mock extraction stayed 34/58 in both full runs. The full mock uses
+keyword-based model doubles, so its pack misses do not estimate live semantic
+retrieval. These are small fresh runs, not a statistical confidence interval.
+The prompt changes the eval answerer only; it does not change TinyMemory's
+stored memory or OpenHuman's production answer instructions.