diff --git a/crates/tinymemory-integrations/examples/memory_eval/llm.rs b/crates/tinymemory-integrations/examples/memory_eval/llm.rs index bcb14765..cdec53a7 100644 --- a/crates/tinymemory-integrations/examples/memory_eval/llm.rs +++ b/crates/tinymemory-integrations/examples/memory_eval/llm.rs @@ -17,10 +17,18 @@ use serde_json::{Value, json}; +/// Changes when the answer instruction changes, so reports remain comparable. +pub(crate) const PROMPT_VERSION: &str = "source-reconciliation-v2"; + /// What the model is told. -const SYSTEM: &str = "You are an assistant with a long-term memory. The user's message \ - starts with what your memory recalled, then their question. Answer the question in one \ - short sentence, using only the memory. If the memory does not say, answer \"unknown\"."; +const SYSTEM: &str = "Answer the question from all memory sections. When two sources have \ + incompatible values and one has no date, do not assume which is current: name BOTH values \ + and their sources. Example: an undated policy says shipping takes four days; a dated message \ + says operations now says six days. Answer: \"The policy says four days, while operations says \ + six days.\" When a newer dated message explicitly replaces an older dated value, answer \ + with ONLY the new value and omit the old value even if the question asks what changed. \ + Example: May says Alice owns a job; June says the job moved to Bob. Answer: \"Bob owns the \ + job.\" Keep answers short. If no evidence, say unknown."; /// One answer and what it cost. pub(crate) struct Answer { diff --git a/crates/tinymemory-integrations/examples/memory_eval/main.rs b/crates/tinymemory-integrations/examples/memory_eval/main.rs index 6d0f9d64..cd4e47ee 100644 --- a/crates/tinymemory-integrations/examples/memory_eval/main.rs +++ b/crates/tinymemory-integrations/examples/memory_eval/main.rs @@ -48,6 +48,8 @@ //! pre-turn hook with its 5000 ms deadline, policy and logged tool results. //! - `--layout v3`: use CortexDB's per-person scope tree and pooled chats. //! - `--team-limit `: override the host's team-conversation budget. +//! - `--brain-limit `: override the number of ranked brain documents in a +//! pack, for accuracy-versus-context-budget experiments. //! - `--loop-guard`: replay a JSON-scripted 500-turn thread and inspect all //! stored items for an injected `` tag. //! @@ -137,6 +139,7 @@ struct Args { host: String, layout: String, team_limit: Option, + brain_limit: Option, date_hint: bool, scale_events: Option, scale_position: String, @@ -148,6 +151,10 @@ struct Args { } fn args() -> Result { + parse_args(std::env::args().skip(1)) +} + +fn parse_args(raw: impl IntoIterator) -> Result { let mut parsed = Args { engine: if std::env::var("CORTEX_DB_URL").is_ok() { "cortex".into() @@ -162,6 +169,7 @@ fn args() -> Result { host: "scripted".into(), layout: "legacy".into(), team_limit: None, + brain_limit: None, date_hint: false, scale_events: None, scale_position: "middle".into(), @@ -171,7 +179,7 @@ fn args() -> Result { expect_derived: false, loop_guard: false, }; - let mut raw = std::env::args().skip(1); + let mut raw = raw.into_iter(); while let Some(flag) = raw.next() { let mut value = || raw.next().ok_or(format!("{flag} needs a value")); match flag.as_str() { @@ -184,6 +192,7 @@ fn args() -> Result { "--host" => parsed.host = value()?, "--layout" => parsed.layout = value()?, "--team-limit" => parsed.team_limit = Some(value()?.parse()?), + "--brain-limit" => parsed.brain_limit = Some(value()?.parse()?), "--date-hint" => parsed.date_hint = true, "--scale-events" => parsed.scale_events = Some(value()?.parse()?), "--scale-position" => parsed.scale_position = value()?, @@ -372,6 +381,9 @@ async fn main() -> Result<(), Error> { if let Some(team_limit) = args.team_limit { policy.team_limit = team_limit; } + if let Some(brain_limit) = args.brain_limit { + policy.brain_limit = brain_limit; + } let eval = Eval { engine: engine.clone(), inspector, @@ -487,11 +499,13 @@ async fn main() -> Result<(), Error> { "flags": flags, "models": models, "probe_answer_model": eval.llm.as_ref().map(|llm| llm.model.as_str()), + "probe_answer_prompt": eval.llm.as_ref().map(|_| llm::PROMPT_VERSION), "server": server, "engine": engine.descriptor().id, "host": args.host, "layout": args.layout, "team_limit": eval.policy.team_limit, + "brain_limit": eval.policy.brain_limit, "pre_turn_timeout_ms": PRE_TURN_TIMEOUT.as_millis() as u64, "date_hint": args.date_hint, "scale_events": args.scale_events, diff --git a/crates/tinymemory-integrations/examples/memory_eval/main_tests.rs b/crates/tinymemory-integrations/examples/memory_eval/main_tests.rs index 11d93525..40bc5043 100644 --- a/crates/tinymemory-integrations/examples/memory_eval/main_tests.rs +++ b/crates/tinymemory-integrations/examples/memory_eval/main_tests.rs @@ -2,6 +2,18 @@ use super::*; +#[test] +fn brain_limit_override_is_parsed_without_changing_the_default() { + assert_eq!(parse_args(Vec::new()).unwrap().brain_limit, None); + assert_eq!( + parse_args(["--brain-limit".into(), "24".into()]) + .unwrap() + .brain_limit, + Some(24) + ); + assert!(parse_args(["--brain-limit".into(), "bad".into()]).is_err()); +} + #[tokio::test] async fn a_probe_does_not_leave_its_question_in_the_measured_corpus() { let engine: Arc = Arc::new(ReferenceEngine::new()); diff --git a/docs/evals/README.md b/docs/evals/README.md index fae455c8..c46d7aa9 100644 --- a/docs/evals/README.md +++ b/docs/evals/README.md @@ -72,8 +72,10 @@ file. Flags (after `--`): `--engine reference|cortex|tinyhumans`, `--only `, `--enrich-wait `, `--json `, `--label `, `--llm`, -`--host openhuman` and `--loop-guard`. With -`CORTEX_DB_KEEP=1` the run's data is left in place for inspection. +`--host openhuman`, `--brain-limit `, and `--loop-guard`. +`--brain-limit` overrides the host's brain-document count and records it in +the JSON report for retrieval-depth sweeps. `CORTEX_DB_KEEP=1` leaves the +run's data in place for inspection. ### How a scenario runs diff --git a/docs/evals/openhuman-host.md b/docs/evals/openhuman-host.md index 8d17e5aa..79df0e8d 100644 --- a/docs/evals/openhuman-host.md +++ b/docs/evals/openhuman-host.md @@ -254,3 +254,48 @@ that the longer deadline removes the timeout for these two probes, while the paraphrase still needs a retrieval-quality improvement. It is a fresh collection and cannot be treated as a paired timeout comparison with the 1500 ms runs above. + +## Accuracy follow-up after #255 + +Fresh CortexDB v0.10.4/OpenRouter runs used the same 100-document middle +needle fixture, seed 251, v3 layout, three team turns, 5000 ms deadline, and +30-second ranked-readiness wait. Each run had a new collection. The literal +owner query hit and the semantic paraphrase missed in all three settings: + +| Setting | Pack hits | Paraphrase | Mean pack tokens | Probe timeout | +| --- | ---: | ---: | ---: | ---: | +| Default server, six brain documents | 1/2 | miss | 130 | 0/2 | +| Max Recall server flags, six brain documents | 1/2 | miss | 130 | 0/2 | +| Default server, 24 brain documents | 1/2 | miss | 512 | 0/2 | + +The Max Recall profile enables three HyDE passages, entity-vector seeding, +assistant excerpts, and enforced bi-temporal validity. Neither it nor a +fourfold increase in the displayed brain documents recovered this fixture's +paraphrase. The wider section spent almost four times the context tokens, so +the production six-document default stays unchanged. `--brain-limit ` now +allows another depth to be measured without editing that default; reports +record the chosen value. + +The same seven-question `team_handoff,conflicts` live slice exposed an answer +problem after retrieval: all seven packs held their expected evidence, but +the old benchmark model prompt answered only 5/7 correctly. Both misses +chose the undated five-day refund document and omitted a conversation saying +Finance had changed refunds to ten days. The final prompt asks the model to +distinguish an unresolved conflict (undated policy versus dated report) from +an explicit update of an older dated value. Reports record its version as +`source-reconciliation-v2`. + +| Fresh run | Pack hit | Model answer, recall | Model answer, synthesis | Timeouts | +| --- | ---: | ---: | ---: | ---: | +| Live slice, original prompt | 7/7 | 5/7 | 5/7 | 0 | +| Live slice, source reconciliation | 7/7 | 7/7 | 6/7 | 0 | +| Full mock, first conflict prompt | 51/58 | 43/58 | 44/58 | 0 | +| Full mock, source reconciliation | 51/58 | 50/58 | 50/58 | 0 | + +The remaining live synthesis miss named Acme's current Team plan but also +mentioned the superseded Enterprise plan, which the strict answer grade +rejects. Mock extraction stayed 34/58 in both full runs. The full mock uses +keyword-based model doubles, so its pack misses do not estimate live semantic +retrieval. These are small fresh runs, not a statistical confidence interval. +The prompt changes the eval answerer only; it does not change TinyMemory's +stored memory or OpenHuman's production answer instructions.