From cc72e95cb01d2922a8f4049eb9725ace70dbb307 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 00:28:53 +0000 Subject: [PATCH 1/2] =?UTF-8?q?wip(v7.13.6):=20=E8=AF=AD=E4=B9=89=E8=9E=8D?= =?UTF-8?q?=E5=90=88=E9=99=8D=E6=9D=83=20+=20=E9=97=A8=E6=A7=9B=20Z=3D3?= =?UTF-8?q?=EF=BC=9B=E8=AF=84=E6=B5=8B=E6=A1=86=E6=9E=B6=E6=8B=92=E7=BB=9D?= =?UTF-8?q?=E3=80=8C=E8=AF=AD=E4=B9=89=E5=BC=80=E4=BD=86=E6=B2=A1=E5=BB=BA?= =?UTF-8?q?=E6=88=90=E7=B4=A2=E5=BC=95=E3=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 进行中：LongMemEval-S 全量独立验证、真实代码重跑、回归演练尚未完成，先不开 PR。 - search_router：融合时语义一票权重 0.5（NYX_EMBED_WEIGHT），词法一票为 1 - semantic：明显高于背景的门槛默认 Z=3（以前 2） LoCoMo（真实本地模型、离线复现与实跑吻合）：纯词法 R@5 0.545，现行等权融合 0.524， 新方案 0.578；10 段对话 9 段变好；同义改写（零字面重叠）仍有 4.1% 能进前 10 （权重 0.3 时为 0，语义只剩重排，所以不再往下压） - 评测框架 check_semantic：后端开着、索引没建成就报错。此前本地模型加载失败时， LoCoMo 与 LongMemEval「语义开」两组分数其实都是纯词法，报告里却写着 local:模型名 - tests：语义一票轻于词法 / 语义认可能提上词法候选 / 候选少时同义改写仍进前 3； 权重改回 1 时第一项失败 - README：NYX_EMBED_Z 默认 3，新增 NYX_EMBED_WEIGHT、NYX_EMBED_CACHE Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019ZZeQFn5F6mEbmX8vPsf4g --- README.md | 4 +++- benchmarks/locomo_eval.py | 4 ++-- benchmarks/longmemeval_eval.py | 14 +++++++++++- nexsandglass/core/search_router.py | 15 ++++++++++++- nexsandglass/core/semantic.py | 8 ++++--- tests/test_semantic_recall.py | 34 ++++++++++++++++++++++++++++++ 6 files changed, 71 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7b52a8b..fc50cb3 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,9 @@ cp -r skills/nyx ~/.hermes/skills/memory/ | `NYX_EMBED` | `auto` | 语义检索后端：`auto` / `local` / `api` / `off` | | `NYX_EMBED_MODEL` | 本地 `paraphrase-multilingual-MiniLM-L12-v2`；API `text-embedding-3-small` | 嵌入模型 | | `EMBEDDING_API_URL` / `EMBEDDING_API_KEY` | 空 | OpenAI 兼容 `/embeddings` | -| `NYX_EMBED_Z` | `2` | 语义命中要比背景高出多少（稳健 z 分数）才算证据 | +| `NYX_EMBED_Z` | `3` | 语义命中要比背景高出多少（稳健 z 分数）才算证据 | +| `NYX_EMBED_WEIGHT` | `0.5` | 融合时语义这一票的权重（词法一票是 1） | +| `NYX_EMBED_CACHE` | `~/.cache/nexsandglass/models` | 本地嵌入模型的存放目录 | | `NYX_UNDERSTAND_LLM` | 关 | `1` = 事实抽取走 LLM（结果必须过校验；失败退回规则） | | `LLM_EXTRACT_API_URL` / `NYX_UNDERSTAND_MODEL` | 空 / `deepseek-v4-flash` | 抽取用的 OpenAI 兼容网关与模型 | | `NYX_FORGET_RETENTION_DAYS` | `30` | 遗忘隔离期 | diff --git a/benchmarks/locomo_eval.py b/benchmarks/locomo_eval.py index 5091c47..b4a9da3 100644 --- a/benchmarks/locomo_eval.py +++ b/benchmarks/locomo_eval.py @@ -53,7 +53,7 @@ _REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from longmemeval_eval import map_to_seqs # noqa: E402 +from longmemeval_eval import check_semantic, map_to_seqs # noqa: E402 CATEGORIES = {1: "multi-hop", 2: "temporal", 3: "open-domain", 4: "single-hop", 5: "adversarial"} SCORED = (1, 2, 3, 4) @@ -173,7 +173,7 @@ def run_conversation(sample: dict, *, budget: int) -> dict: sandglass_vault.rebuild_index() semantic.wait_idle(600) semantic.index_all() - sem = semantic.stats() + sem = check_semantic(semantic.stats()) # 后端开着、索引没建成 → 报错，不记成「语义开」 ingest_s = time.time() - t0 bm25 = BM25(texts) diff --git a/benchmarks/longmemeval_eval.py b/benchmarks/longmemeval_eval.py index c6e2200..cf6b351 100644 --- a/benchmarks/longmemeval_eval.py +++ b/benchmarks/longmemeval_eval.py @@ -81,6 +81,18 @@ def _pairs(turns: list) -> list: return out +def check_semantic(sem: dict) -> dict: + """语义后端开着、索引却没建成 → 这一题按出错处理，不许把纯词法的结果记成「语义开」。 + + v7.13.5 实测：本地模型加载失败（模型不在 NYX_EMBED_CACHE、又设了离线），LoCoMo 10 段、LongMemEval 500 题 + 全部「已索引 0、待补全部」，报告里的后端却写着 local:模型名 —— 两组「语义开」的分数其实都是纯词法。 + """ + if sem.get("enabled") and (sem.get("pending") or not sem.get("indexed")): + raise RuntimeError(f"语义后端 {sem.get('backend')} 已启用，但索引没建成（已索引 {sem.get('indexed')}，" + f"待补 {sem.get('pending')}，错误：{sem.get('last_error') or sem.get('error')}）") + return sem + + def map_to_seqs(conn, objects) -> list: """召回结果 → ID 中枢的 seq（按原顺序；对不上的给 None）。先按行号，再按正文（去掉「未经证实」标注）。""" by_line = dict(conn.execute("SELECT line_start, seq FROM memories WHERE deleted_at IS NULL")) @@ -138,7 +150,7 @@ def run_instance(inst: dict, *, budget: int, prefetch_tokens: int, want_context: from nexsandglass.core import semantic semantic.wait_idle(600) # 后台补索引跑完 semantic.index_all() # 兜底，保证全部入索引 - sem = semantic.stats() + sem = check_semantic(semantic.stats()) ingest_s = time.time() - t0 clock.set_global(parse_date(inst["question_date"])) diff --git a/nexsandglass/core/search_router.py b/nexsandglass/core/search_router.py index e1f3369..b53bd60 100644 --- a/nexsandglass/core/search_router.py +++ b/nexsandglass/core/search_router.py @@ -303,6 +303,18 @@ def _scan_file(self, query, tokens, limit, results, results_token) -> None: _AUTO = object() RRF_K = 60 PIN_MIN_COVERAGE = 0.6 +# 语义这一票的权重（v7.13.5；NYX_EMBED_WEIGHT 可调）。词法一票是 1。 +# 等权时，嵌入模型比词法弱的场景下它会把好的词法结果挤出去：LoCoMo（默认本地模型）recall@5 +# 词法单独 0.545，等权融合后反而 0.524；权重 0.5 + 门槛 Z=3 → 0.578，10 段对话里 9 段变好。 +# 不再往下压：权重 0.3 时分数相近，但与问句零字面重叠的证据（同义改写）一条也进不了前 10，语义只剩重排 +VECTOR_WEIGHT = 0.5 + + +def _vector_weight() -> float: + try: + return float(os.environ.get("NYX_EMBED_WEIGHT", VECTOR_WEIGHT)) + except ValueError: + return VECTOR_WEIGHT class SearchRouter: @@ -419,8 +431,9 @@ def _rrf(ranked: list, vector_lines: list, tokens=()) -> list: score = {} for r, item in enumerate(ranked): score[item[0]] = score.get(item[0], 0.0) + 1.0 / (RRF_K + r + 1) + w = _vector_weight() for r, ln in enumerate(vector_lines): - score[ln] = score.get(ln, 0.0) + 1.0 / (RRF_K + r + 1) + score[ln] = score.get(ln, 0.0) + w / (RRF_K + r + 1) lows = {item[0]: (item[2] if len(item) > 2 else "").lower() for item in ranked} # 只认在候选里真实出现过的词：中文 2-gram 会跨过被剔除的数字拼出「第条」这种 # 哪条记录里都没有的幻影词，带上它就永远没有记录能「完整命中」 diff --git a/nexsandglass/core/semantic.py b/nexsandglass/core/semantic.py index 5f86358..c977dca 100644 --- a/nexsandglass/core/semantic.py +++ b/nexsandglass/core/semantic.py @@ -638,7 +638,9 @@ def _standout_threshold(scores: List[float]) -> float: 向量检索永远会返回最像的几条，哪怕全都不相关；把这些背景噪声送进 RRF， 它们会凭两票把精确的字面命中挤出前几名（v7.13 事故演练实测）。 只有和背景拉开距离的命中才算语义证据。全部一样像（MAD=0）→ 只有严格高于中位数 - 一个最小间隔的才算。Z 由 NYX_EMBED_Z 配置，默认 2。 + 一个最小间隔的才算。Z 由 NYX_EMBED_Z 配置，默认 3（v7.13.5；以前 2）。 + 在 LoCoMo 上用真实本地模型测：Z=2 时放进来的「略高于背景」的命中多是闲聊句，等权融合后 + recall@5 比纯词法还低（0.524 vs 0.545）；Z=3 加上语义一票减半（search_router.VECTOR_WEIGHT）→ 0.578。 """ if not scores: return float("inf") @@ -653,9 +655,9 @@ def _standout_threshold(scores: List[float]) -> float: dev = sorted(abs(x - med) for x in bg) mad = dev[n // 2] if n % 2 else (dev[n // 2 - 1] + dev[n // 2]) / 2 try: - z = float(os.environ.get("NYX_EMBED_Z", "2") or 2) + z = float(os.environ.get("NYX_EMBED_Z", "3") or 3) except ValueError: - z = 2.0 + z = 3.0 return med + max(z * 1.4826 * mad, 0.02) diff --git a/tests/test_semantic_recall.py b/tests/test_semantic_recall.py index c4f62a0..804f922 100644 --- a/tests/test_semantic_recall.py +++ b/tests/test_semantic_recall.py @@ -187,3 +187,37 @@ def test_mmap_fallback_follows_current_journal(env, tmp_path): # noqa: F811 fb = MmapFallback() assert fb._path() == sandglass_vault._SANDGLASS == env.journal assert not any("橘猫" in h[2] for h in fb.search("橘猫", 5)) + + +def _lex(n): + return [(i, "2026-10-01 10:00", f"lexical {i}") for i in range(1, n + 1)] + + +def test_vector_vote_weighs_less_than_lexical(monkeypatch): + """v7.13.5：语义一票减半。等权时，一个比词法弱的嵌入模型会把词法第 1 名挤下去 + （LoCoMo 实测：等权融合 recall@5 0.524，比纯词法 0.545 还低）。""" + from nexsandglass.core.search_router import SearchRouter + monkeypatch.delenv("NYX_EMBED_WEIGHT", raising=False) + # 与 SearchRouter.search 一致：纯向量命中先并进候选（排在词法候选末尾），再融合。 + # 真实库里词法候选通常有 60–90 条（三路各 30 条去重） + ranked = _lex(90) + [(ln, "2026-10-01 10:00", f"vector only {ln}") for ln in (900, 901, 902)] + out = [it[0] for it in SearchRouter._rrf(ranked, [900, 901, 902])] + assert out[0] == 1, out[:5] # 词法第 1 名不被纯向量命中顶掉 + monkeypatch.setenv("NYX_EMBED_WEIGHT", "1") + assert [it[0] for it in SearchRouter._rrf(ranked, [900, 901, 902])][0] == 900 # 等权时会被顶掉（对照） + + +def test_vector_agreement_lifts_a_lexical_candidate(monkeypatch): + from nexsandglass.core.search_router import SearchRouter + monkeypatch.delenv("NYX_EMBED_WEIGHT", raising=False) + out = [it[0] for it in SearchRouter._rrf(_lex(30), [12])] + assert out.index(12) < 3, out[:5] # 词法排 12、语义第 1 → 提到前 3 + + +def test_paraphrase_still_reaches_top10_when_few_candidates(monkeypatch): + """权重不能压到让同义改写（与问句零字面重叠）完全进不来 —— 那就只剩重排了。""" + from nexsandglass.core.search_router import SearchRouter + monkeypatch.delenv("NYX_EMBED_WEIGHT", raising=False) + ranked = _lex(5) + [(900, "2026-10-01 10:00", "joined Contoso last spring")] # 纯向量命中排在候选末尾 + out = [it[0] for it in SearchRouter._rrf(ranked, [900])] + assert out.index(900) < 3, out From f49b194d6745d22986b5260a6446d150b4a9c790 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 02:25:57 +0000 Subject: [PATCH 2/2] =?UTF-8?q?release(v7.13.6):=20=E7=9C=9F=E5=AE=9E?= =?UTF-8?q?=E5=B5=8C=E5=85=A5=E6=A8=A1=E5=9E=8B=E4=B8=8B=E7=9A=84=E5=85=AC?= =?UTF-8?q?=E5=BC=80=E5=9F=BA=E5=87=86=E5=88=86=E6=95=B0=20+=20=E8=9E=8D?= =?UTF-8?q?=E5=90=88=E5=8F=82=E6=95=B0=E5=88=86=E5=8D=8A=E9=80=89=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在上一提交（语义一票减半、门槛 Z=3、评测框架拒绝空索引）基础上完成验证并发布： - LongMemEval-S 首次真实数据分数（500 题，检索，nyx 真实代码实跑，本地模型）： 前 5 命中 0.981、前 5 全中 0.862、前 10 全中 0.885（纯词法 0.949 / 0.774 / 0.819） - LoCoMo recall@5 0.575（纯词法 0.546，旧融合 0.527，BM25 0.498） - 参数选择：两个基准各分两半，A 半按事先定死的规则选出 Z=3、w=0.5，B 半留出验证： LoCoMo 0.539 / 0.510 / 0.566（纯词法 / 旧 / 新），LongMemEval 全中 0.753 / 0.872 / 0.843。 新融合两边都不低于纯词法；代价 LongMemEval 全中比旧融合低约 3 点，README 写明取舍与回退参数 - 纵向评测 35 种子三种模式逐项与 v7.13.3 相同；投毒 / 事故演练三种模式通过 - README Benchmark 改为真实分数；results 只存分数；更正开发中途「语义几乎无贡献」的错误结论 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019ZZeQFn5F6mEbmX8vPsf4g --- README.md | 114 ++- benchmarks/longmemeval_eval.py | 4 +- benchmarks/results/locomo_retrieval.json | 957 ++++++++++-------- .../results/longmemeval_s_retrieval.json | 246 +++++ nexsandglass/core/sandglass_paths.py | 2 +- nexsandglass_hermes/plugin.yaml | 2 +- 6 files changed, 871 insertions(+), 454 deletions(-) create mode 100644 benchmarks/results/longmemeval_s_retrieval.json diff --git a/README.md b/README.md index fc50cb3..f147d39 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > **Nyx — 把「检索失败」也当作一类信号的记忆系统** -![Python](https://img.shields.io/badge/Python-3.10%2B-3776AB?logo=python) ![License](https://img.shields.io/badge/License-MIT-green) ![Version](https://img.shields.io/badge/version-7.13.5-blue) ![Deps](https://img.shields.io/badge/runtime%20deps-0-brightgreen) +![Python](https://img.shields.io/badge/Python-3.10%2B-3776AB?logo=python) ![License](https://img.shields.io/badge/License-MIT-green) ![Version](https://img.shields.io/badge/version-7.13.6-blue) ![Deps](https://img.shields.io/badge/runtime%20deps-0-brightgreen) ## 别的记忆系统回答「找到了什么」，Nyx 还回答「我是不是见过」 @@ -168,43 +168,56 @@ scripts/nyx_quarantine.py # 隔离区命令行：list / restore / purge ## 📊 Benchmark -**公开基准 LoCoMo：检索证据召回**（`benchmarks/locomo_eval.py`，v7.13.5，结果 `benchmarks/results/locomo_retrieval.json`） +两个公开基准都只测**检索**（零 LLM、可复现），逐轮走 Hermes 插件真实的写入路径。 +语义检索用默认的本地模型 `paraphrase-multilingual-MiniLM-L12-v2`（`pip install "nyx-memory[vector]"`）。 -10 段两人长对话（共 5882 轮、272 个会话，跨数月），计分 1536 题（对抗题 446、无证据标注 4 题按惯例不计）。 -逐轮走 Hermes 插件真实的写入路径；指标是「前 k 条召回里，标注的证据句找到了多少」。零 LLM，可复现。 -参照是同一批句子上最朴素的 BM25（底线，不是对手）。语义检索**关闭**（开发环境没有嵌入后端）： +**LongMemEval-S**（`benchmarks/longmemeval_eval.py`，500 题、每题约 50 个会话 / 490 轮；按官方定义算会话级召回， +「该拒答」的 30 题不计；结果 `benchmarks/results/longmemeval_s_retrieval.json`） -| 类别 | 题数 | nyx R@5 | BM25 R@5 | nyx R@10 | BM25 R@10 | nyx 全部证据进前 10 | -|---|---|---|---|---|---|---| -| 多跳 | 282 | **0.289** | 0.180 | **0.383** | 0.264 | 0.170 | -| 时间 | 321 | **0.666** | 0.611 | **0.726** | 0.678 | 0.701 | -| 开放域推理 | 92 | **0.282** | 0.243 | **0.328** | 0.272 | 0.228 | -| 单跳 | 841 | **0.615** | 0.590 | **0.690** | 0.655 | 0.674 | -| **总体** | 1536 | **0.546** | 0.498 | **0.619** | 0.565 | 0.561 | - -怎么读这张表： -- 这是**检索**，不是别人报的 LoCoMo 分数。Mem0 92.5%、Zep 94.7% 这类数字是「大模型读召回结果作答、再由大模型裁判」的 - 问答准确率，口径不同，不能拿上表去比 -- 前 10 条里至少有一句证据的题是 69%：剩下 31% 的题，读答案的模型一句证据都看不到。短板在多跳（证据分散在多个会话） -- v7.13.4 及以前这张表的 nyx 一列 R@5 是 **0.046** —— 召回聚合的排序缺陷，见 v7.13.5 更新日志 +| 题型 | 题数 | 前 5 命中 | 前 5 全中 | 前 10 全中 | 纯词法 前 5 全中 | +|---|---|---|---|---|---| +| 信息更新 | 72 | 1.000 | 0.986 | 1.000 | 0.889 | +| 多会话汇总 | 121 | 0.984 | 0.785 | 0.802 | 0.603 | +| 助手说过的话 | 56 | 1.000 | 1.000 | 1.000 | 0.982 | +| 用户偏好 | 30 | 0.900 | 0.900 | 0.900 | 0.700 | +| 用户说过的话 | 64 | 1.000 | 1.000 | 1.000 | 0.984 | +| 时间推理 | 127 | 0.969 | 0.724 | 0.787 | 0.693 | +| **总体** | 470 | **0.981** | **0.862** | **0.885** | 0.774 | -```bash -# 数据 https://github.com/snap-research/locomo（CC BY-NC 4.0，仓库里不放） -python3 benchmarks/locomo_eval.py locomo10.json --workers 10 --json benchmarks/results/locomo_retrieval.json -``` +「前 5 命中」= 前 5 条召回里至少有一个答案所在会话；「前 5 全中」= 答案分散在几个会话时，前 5 条把它们全找到了。 -**公开基准 LongMemEval**（`benchmarks/longmemeval_eval.py`，v7.12）：每题独立目录，按会话日期冻结时钟， -逐轮走 Hermes 插件真实的写入路径，再按官方定义算会话级 recall_any@k / recall_all@k（零 LLM，可复现）； -`--qa` 生成官方格式的答案文件，交给官方 `evaluate_qa.py` 打分。 -**目前还没有真实数据上的分数** —— 数据在 HuggingFace，开发环境连不到。在部署环境运行 -（开 / 关语义各跑一次，差值就是语义检索的贡献）： +**LoCoMo**（`benchmarks/locomo_eval.py`，10 段两人长对话、5882 轮、跨数月；计分 1536 题，对抗题 446、无证据标注 4 题按惯例不计； +指标是标注的证据句在前 k 条里找到了多少；参照是同一批句子上最朴素的 BM25；结果 `benchmarks/results/locomo_retrieval.json`） + +| 类别 | 题数 | nyx R@5 | 纯词法 R@5 | BM25 R@5 | nyx R@10 | BM25 R@10 | +|---|---|---|---|---|---|---| +| 多跳 | 282 | **0.297** | 0.289 | 0.180 | **0.398** | 0.264 | +| 时间 | 321 | **0.677** | 0.666 | 0.611 | **0.738** | 0.678 | +| 开放域推理 | 92 | 0.273 | **0.282** | 0.243 | **0.352** | 0.272 | +| 单跳 | 841 | **0.661** | 0.615 | 0.590 | **0.728** | 0.655 | +| **总体** | 1536 | **0.575** | 0.546 | 0.498 | **0.647** | 0.565 | + +怎么读这两张表： +- 这是**检索**，不是别人报的分数。Mem0、Zep、Hindsight 报的 LongMemEval / LoCoMo 分数是「大模型读召回结果作答、 + 再由大模型裁判」的问答准确率，口径不同，不能拿上表去比；端到端问答还没跑（需要大模型 API） +- 两个基准差别很大：LongMemEval 里大部分会话是与问题无关的填充对话，语义很好区分；LoCoMo 是同样两个人 + 从头聊到尾，句句都像，靠人名与关键词。nyx 的融合（v7.13.6）按两个基准各取一半选参数、在另一半上验证， + 两边都不比纯词法差。代价：LongMemEval「前 5 全中」比旧融合低约 3 个点（主要是多会话汇总题）， + 记忆更像 LongMemEval 的用户可以设 `NYX_EMBED_Z=2 NYX_EMBED_WEIGHT=1` 回到旧融合 +- 短板：LoCoMo 多跳题证据全部进前 10 只有 18%；LongMemEval 时间推理「前 5 全中」0.72 +- v7.13.4 及以前，LoCoMo 的 nyx R@5 是 **0.046**（召回聚合的排序缺陷，见 v7.13.5 更新日志） ```bash -pip install "nyx-memory[vector]" +pip install "nyx-memory[vector]" # 首次运行把模型下载到 ~/.cache/nexsandglass/models（NYX_EMBED_CACHE 可改） +# LongMemEval（MIT）：https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned python3 benchmarks/longmemeval_eval.py longmemeval_s_cleaned.json --workers 8 --json benchmarks/results/longmemeval_s.json -python3 benchmarks/longmemeval_eval.py longmemeval_s_cleaned.json --workers 8 --no-semantic --json benchmarks/results/longmemeval_s_lexical.json +python3 benchmarks/longmemeval_eval.py longmemeval_s_cleaned.json --workers 8 --no-semantic --json longmemeval_s_lexical.json +# LoCoMo（CC BY-NC 4.0，仓库里不放）：https://github.com/snap-research/locomo +python3 benchmarks/locomo_eval.py locomo10.json --workers 10 --json benchmarks/results/locomo.json ``` +语义后端开着、索引却没建成时，两个评测框架都直接报错（v7.13.6）—— 不会把纯词法的结果记成「语义开」。 + **仓库里可复现的评测**（结果在 `benchmarks/results/`）： | 评测 | 测什么 | 版本 | @@ -374,6 +387,49 @@ Hermes 用户：数据目录在 `HERMES_HOME` 之外时，`hermes backup` 会自 ## 📝 更新日志 +### v7.13.6 (2026-10-13) — 第一次用真实嵌入模型测公开基准：语义融合重新定参 + +**LongMemEval-S 第一次有了真实数据上的分数**（500 题，检索，零 LLM）：答案所在会话进前 5 的题 **98.1%**， +答案分散在几个会话时前 5 条全找到 **86.2%**；纯词法时是 94.9% / 77.4%。用户偏好题 70% → 90%， +多会话汇总题全找到 60% → 79%。LoCoMo recall@5 0.546 → **0.575**。详见 Benchmark 一节。 + +**先更正一件事**：本版开发中途我报告过「语义检索几乎没有贡献」，那是错的。那几组「语义开」的跑分， +本地模型其实没加载成功（我把模型下到了 HuggingFace 的缓存，nyx 用的是自己的 `~/.cache/nexsandglass/models`， +又设了离线），10 段对话、500 道题全部「已索引 0」，报告里的后端却照样写着模型名 —— 分数全是纯词法。 +产品本身没问题（用真实模型实测：中文问「宠物猫叫什么」召回英文原话 "I adopted a cat named Mochi"）， +是评测框架没有检查。现在两个框架都会拒绝这种情况（`check_semantic`）：后端开着、索引没建成就按出错处理。 + +**真实向量进来之后，旧融合在 LoCoMo 上反而拖后腿**：纯词法 recall@5 0.546，语义打开后 0.527。 +原因不是门槛太严，而是语义与词法等权投票：默认的小型多语言模型在英文闲聊里单独用只有 0.424（比 BM25 还低）， +等权时它的票把好的词法结果挤出去。 + +**新融合**：门槛 Z 2 → 3（`NYX_EMBED_Z`）、语义一票权重 1 → 0.5（新增 `NYX_EMBED_WEIGHT`）。 +- 选法：两个基准各分两半，候选 Z∈{2,3} × 权重∈{1, 0.5, 0.3}，在 A 半上按事先定死的规则 + （LoCoMo recall@5 与 LongMemEval 前 5 全中的平均）选出 Z=3、权重 0.5；只在没碰过的 B 半上报告 +- B 半：LoCoMo recall@5 纯词法 0.539 / 旧融合 0.510 / 新融合 **0.566**；LongMemEval 前 5 命中 + 0.957 / 0.983 / **0.987**，前 5 全中 0.753 / **0.872** / 0.843 +- 新融合是唯一一个在两个基准上都不比纯词法差的；**代价**是 LongMemEval「前 5 全中」比旧融合低约 3 个点 + （多会话汇总题）。记忆更像 LongMemEval（和 agent 聊大量不相关话题）的用户可以设 + `NYX_EMBED_Z=2 NYX_EMBED_WEIGHT=1` 回到旧融合 +- 权重没有压到 0.3：分数相近，但与问句零字面重叠的证据（同义改写）一条也进不了前 10，语义只剩重排 +- 两个基准差别的来源：LongMemEval 大部分会话是与问题无关的填充对话，语义很好区分；LoCoMo 是同样两个人 + 从头聊到尾，句句都像 + +**其他评测不受影响**：纵向评测 35 个种子在「语义关 / 假模型 + 新融合 / 假模型 + 旧融合」三种模式下都与 v7.13.3 逐项相同； +投毒演练（16 条攻击 0 条无标注进 prompt、主人 0 误伤）与事故演练在三种模式下都通过。 + +**防复发** +- `tests/test_semantic_recall.py` 新增 3 项：语义一票轻于词法（权重改回 1 时失败）、语义认可能把排在后面的 + 词法候选提上来、候选少时同义改写仍进前 3 +- 两个评测框架的 `check_semantic`：用之前的错误配置重跑，直接报错退出 + +**还差什么**（如实说明） +- 只测了检索；端到端问答准确率（别人报的分数）需要大模型 API，还没跑 +- LoCoMo 开放域推理类（92 题）语义打开后比纯词法低 0.9 个点；多跳题证据全进前 10 只有 18% +- 默认本地模型偏小；换成更强的嵌入模型（API）时最优权重可能不同，两个参数都可以调 + +**验证**：全量 513 项通过；真实 Hermes 检查通过；LoCoMo 与 LongMemEval 都用 nyx 真实代码实跑（不是模拟）。 + ### v7.13.5 (2026-10-12) — 第一次在公开基准的真实数据上测：LoCoMo 查出召回排序缺陷 **新增 LoCoMo 检索评测** `benchmarks/locomo_eval.py`：10 段两人长对话、1536 道计分题，逐轮走 Hermes 插件真实的写入路径， diff --git a/benchmarks/longmemeval_eval.py b/benchmarks/longmemeval_eval.py index cf6b351..abeb2b2 100644 --- a/benchmarks/longmemeval_eval.py +++ b/benchmarks/longmemeval_eval.py @@ -30,7 +30,9 @@ python3 benchmarks/longmemeval_eval.py data/longmemeval_s_cleaned.json --qa --hyp hyp.jsonl 如实说明 - - 这个环境连不到 HuggingFace，仓库里没有真实数据上的分数；tests/ 里用的是同格式的小样本 + - 真实数据上的检索分数见 README 与 benchmarks/results/longmemeval_s_retrieval.json（v7.13.6）； + tests/ 里用的是同格式的小样本，只防回归 + - 语义后端开着、索引却没建成（如本地模型加载失败）时，这一题按出错处理（check_semantic） - nyx 的规则抽取以中文为主；LongMemEval 是英文。英文上召回主要靠全文检索与倒排索引 """ from __future__ import annotations diff --git a/benchmarks/results/locomo_retrieval.json b/benchmarks/results/locomo_retrieval.json index 6e2f293..e2556b1 100644 --- a/benchmarks/results/locomo_retrieval.json +++ b/benchmarks/results/locomo_retrieval.json @@ -1,457 +1,570 @@ { "benchmark": "LoCoMo (locomo10.json, snap-research/locomo) — 检索证据召回，零 LLM", "data_sha256": "79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4", - "settings": { - "semantic": "off（开发环境没有嵌入后端）", - "budget": 1500, - "scored": "类别 1–4、有证据标注的 1536 题；对抗题 446、无证据 4 题不计" - }, - "note": "缺陷在第 1 段对话上定位；第 2–10 段（held_out）没有参与诊断", - "v7.13.5": { - "1 multi-hop": { - "n": 282, - "nyx": { - "recall@1": 0.1119, - "hit@1": 0.2943, - "all@1": 0.0, - "recall@5": 0.2886, - "hit@5": 0.5461, - "all@5": 0.1099, - "recall@10": 0.3828, - "hit@10": 0.6525, - "all@10": 0.1702, - "recall@20": 0.3954, - "hit@20": 0.6738, - "all@20": 0.1738 - }, - "bm25": { - "recall@1": 0.0589, - "hit@1": 0.1596, - "all@1": 0.0, - "recall@5": 0.1803, - "hit@5": 0.3688, - "all@5": 0.0532, - "recall@10": 0.2641, - "hit@10": 0.4929, - "all@10": 0.1028, - "recall@20": 0.3485, - "hit@20": 0.6028, - "all@20": 0.1525 - }, - "nyx_returned_p50": 18, - "recall_ms_p50": 49.9 - }, - "2 temporal": { - "n": 321, - "nyx": { - "recall@1": 0.4925, - "hit@1": 0.5202, - "all@1": 0.4704, - "recall@5": 0.6664, - "hit@5": 0.6978, - "all@5": 0.6386, - "recall@10": 0.7261, - "hit@10": 0.757, - "all@10": 0.7009, - "recall@20": 0.7448, - "hit@20": 0.7757, - "all@20": 0.7196 - }, - "bm25": { - "recall@1": 0.3409, - "hit@1": 0.3645, - "all@1": 0.324, - "recall@5": 0.6114, - "hit@5": 0.6417, - "all@5": 0.5857, - "recall@10": 0.6783, - "hit@10": 0.7072, - "all@10": 0.648, - "recall@20": 0.7163, - "hit@20": 0.7445, - "all@20": 0.6822 - }, - "nyx_returned_p50": 18, - "recall_ms_p50": 48.8 - }, - "3 open-domain": { - "n": 92, - "nyx": { - "recall@1": 0.1232, - "hit@1": 0.1522, - "all@1": 0.0978, - "recall@5": 0.2817, - "hit@5": 0.3913, - "all@5": 0.1957, - "recall@10": 0.3276, - "hit@10": 0.4565, - "all@10": 0.2283, - "recall@20": 0.3364, - "hit@20": 0.4783, - "all@20": 0.2283 - }, - "bm25": { - "recall@1": 0.1123, - "hit@1": 0.1413, - "all@1": 0.087, - "recall@5": 0.2431, - "hit@5": 0.3261, - "all@5": 0.1739, - "recall@10": 0.2716, - "hit@10": 0.3696, - "all@10": 0.1848, - "recall@20": 0.3188, - "hit@20": 0.4348, - "all@20": 0.2283 - }, - "nyx_returned_p50": 18, - "recall_ms_p50": 55.4 - }, - "4 single-hop": { - "n": 841, - "nyx": { - "recall@1": 0.4039, - "hit@1": 0.4162, - "all@1": 0.3936, - "recall@5": 0.6151, - "hit@5": 0.629, - "all@5": 0.6005, - "recall@10": 0.6895, - "hit@10": 0.7039, - "all@10": 0.6742, - "recall@20": 0.6938, - "hit@20": 0.7075, - "all@20": 0.679 + "scored": "类别 1–4、有证据标注的 1536 题；对抗题 446、无证据 4 题不计", + "v7.13.6_semantic": { + "backend": "local:paraphrase-multilingual-MiniLM-L12-v2", + "fusion": "NYX_EMBED_Z=3, NYX_EMBED_WEIGHT=0.5", + "scores": { + "1 multi-hop": { + "n": 282, + "nyx": { + "recall@1": 0.0991, + "hit@1": 0.2589, + "all@1": 0.0, + "recall@5": 0.2967, + "hit@5": 0.5603, + "all@5": 0.1064, + "recall@10": 0.3978, + "hit@10": 0.6773, + "all@10": 0.1773, + "recall@20": 0.4146, + "hit@20": 0.7021, + "all@20": 0.1809 + }, + "bm25": { + "recall@1": 0.0589, + "hit@1": 0.1596, + "all@1": 0.0, + "recall@5": 0.1803, + "hit@5": 0.3688, + "all@5": 0.0532, + "recall@10": 0.2641, + "hit@10": 0.4929, + "all@10": 0.1028, + "recall@20": 0.3485, + "hit@20": 0.6028, + "all@20": 0.1525 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 74.5 }, - "bm25": { - "recall@1": 0.3543, - "hit@1": 0.3639, - "all@1": 0.346, - "recall@5": 0.5896, - "hit@5": 0.6017, - "all@5": 0.5767, - "recall@10": 0.655, - "hit@10": 0.6659, - "all@10": 0.6433, - "recall@20": 0.7297, - "hit@20": 0.742, - "all@20": 0.717 + "2 temporal": { + "n": 321, + "nyx": { + "recall@1": 0.4816, + "hit@1": 0.5109, + "all@1": 0.4579, + "recall@5": 0.6773, + "hit@5": 0.7072, + "all@5": 0.6511, + "recall@10": 0.7375, + "hit@10": 0.7632, + "all@10": 0.7134, + "recall@20": 0.7593, + "hit@20": 0.785, + "all@20": 0.7352 + }, + "bm25": { + "recall@1": 0.3409, + "hit@1": 0.3645, + "all@1": 0.324, + "recall@5": 0.6114, + "hit@5": 0.6417, + "all@5": 0.5857, + "recall@10": 0.6783, + "hit@10": 0.7072, + "all@10": 0.648, + "recall@20": 0.7163, + "hit@20": 0.7445, + "all@20": 0.6822 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 72.9 }, - "nyx_returned_p50": 18, - "recall_ms_p50": 39.6 - }, - "_overall": { - "n": 1536, - "nyx": { - "recall@1": 0.352, - "hit@1": 0.3997, - "all@1": 0.3197, - "recall@5": 0.5459, - "hit@5": 0.6139, - "all@5": 0.4941, - "recall@10": 0.6192, - "hit@10": 0.6908, - "all@10": 0.5605, - "recall@20": 0.6283, - "hit@20": 0.7018, - "all@20": 0.5677 + "3 open-domain": { + "n": 92, + "nyx": { + "recall@1": 0.154, + "hit@1": 0.2174, + "all@1": 0.0978, + "recall@5": 0.2726, + "hit@5": 0.3696, + "all@5": 0.1848, + "recall@10": 0.3521, + "hit@10": 0.5, + "all@10": 0.2391, + "recall@20": 0.3609, + "hit@20": 0.5217, + "all@20": 0.2391 + }, + "bm25": { + "recall@1": 0.1123, + "hit@1": 0.1413, + "all@1": 0.087, + "recall@5": 0.2431, + "hit@5": 0.3261, + "all@5": 0.1739, + "recall@10": 0.2716, + "hit@10": 0.3696, + "all@10": 0.1848, + "recall@20": 0.3188, + "hit@20": 0.4348, + "all@20": 0.2283 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 74.4 }, - "bm25": { - "recall@1": 0.2828, - "hit@1": 0.3132, - "all@1": 0.2624, - "recall@5": 0.4982, - "hit@5": 0.5508, - "all@5": 0.4583, - "recall@10": 0.5651, - "hit@10": 0.625, - "all@10": 0.5176, - "recall@20": 0.6323, - "hit@20": 0.6986, - "all@20": 0.5768 + "4 single-hop": { + "n": 841, + "nyx": { + "recall@1": 0.3795, + "hit@1": 0.3924, + "all@1": 0.3686, + "recall@5": 0.6615, + "hit@5": 0.6766, + "all@5": 0.6457, + "recall@10": 0.7281, + "hit@10": 0.742, + "all@10": 0.7134, + "recall@20": 0.7313, + "hit@20": 0.7455, + "all@20": 0.7158 + }, + "bm25": { + "recall@1": 0.3543, + "hit@1": 0.3639, + "all@1": 0.346, + "recall@5": 0.5896, + "hit@5": 0.6017, + "all@5": 0.5767, + "recall@10": 0.655, + "hit@10": 0.6659, + "all@10": 0.6433, + "recall@20": 0.7297, + "hit@20": 0.742, + "all@20": 0.717 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 73.7 }, - "nyx_returned_p50": 18, - "recall_ms_p50": 43.4 - }, - "_meta": { - "conversations": 10, - "errors": 0, - "questions_scored": 1536, - "skipped": { - "no evidence": 4, - "adversarial": 446 + "_overall": { + "n": 1536, + "nyx": { + "recall@1": 0.3358, + "hit@1": 0.3822, + "all@1": 0.3034, + "recall@5": 0.5745, + "hit@5": 0.6432, + "all@5": 0.5202, + "recall@10": 0.6469, + "hit@10": 0.7201, + "all@10": 0.5866, + "recall@20": 0.6568, + "hit@20": 0.7324, + "all@20": 0.5931 + }, + "bm25": { + "recall@1": 0.2828, + "hit@1": 0.3132, + "all@1": 0.2624, + "recall@5": 0.4982, + "hit@5": 0.5508, + "all@5": 0.4583, + "recall@10": 0.5651, + "hit@10": 0.625, + "all@10": 0.5176, + "recall@20": 0.6323, + "hit@20": 0.6986, + "all@20": 0.5768 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 73.8 }, - "turns": 5882, - "ingest_s_total": 36.2, - "unmapped_objects": 0, - "semantic_backend": [ - "off" - ], - "data": "locomo10.json", - "wall_s": 15.7, - "budget": 1500 + "_meta": { + "conversations": 10, + "errors": 0, + "questions_scored": 1536, + "skipped": { + "no evidence": 4, + "adversarial": 446 + }, + "turns": 5882, + "ingest_s_total": 484.7, + "unmapped_objects": 0, + "semantic_backend": [ + "local:paraphrase-multilingual-MiniLM-L12-v2" + ], + "data": "locomo10.json", + "wall_s": 192.5, + "budget": 1500 + } } }, - "v7.13.4": { - "1 multi-hop": { - "n": 282, - "nyx": { - "recall@1": 0.0059, - "hit@1": 0.0142, - "all@1": 0.0, - "recall@5": 0.033, - "hit@5": 0.0851, - "all@5": 0.0, - "recall@10": 0.2177, - "hit@10": 0.4574, - "all@10": 0.0532, - "recall@20": 0.3966, - "hit@20": 0.6738, - "all@20": 0.1738 - }, - "bm25": { - "recall@1": 0.0589, - "hit@1": 0.1596, - "all@1": 0.0, - "recall@5": 0.1803, - "hit@5": 0.3688, - "all@5": 0.0532, - "recall@10": 0.2641, - "hit@10": 0.4929, - "all@10": 0.1028, - "recall@20": 0.3485, - "hit@20": 0.6028, - "all@20": 0.1525 - }, - "nyx_returned_p50": 18, - "recall_ms_p50": 46.1 - }, - "2 temporal": { - "n": 321, - "nyx": { - "recall@1": 0.0, - "hit@1": 0.0, - "all@1": 0.0, - "recall@5": 0.0717, - "hit@5": 0.0779, - "all@5": 0.0654, - "recall@10": 0.589, - "hit@10": 0.6137, - "all@10": 0.567, - "recall@20": 0.7479, - "hit@20": 0.7788, - "all@20": 0.7227 - }, - "bm25": { - "recall@1": 0.3409, - "hit@1": 0.3645, - "all@1": 0.324, - "recall@5": 0.6114, - "hit@5": 0.6417, - "all@5": 0.5857, - "recall@10": 0.6783, - "hit@10": 0.7072, - "all@10": 0.648, - "recall@20": 0.7163, - "hit@20": 0.7445, - "all@20": 0.6822 + "v7.13.5_semantic_old_fusion": { + "backend": "local:paraphrase-multilingual-MiniLM-L12-v2", + "fusion": "Z=2, 等权", + "scores": { + "1 multi-hop": { + "n": 282, + "nyx": { + "recall@1": 0.0997, + "hit@1": 0.2589, + "all@1": 0.0035, + "recall@5": 0.2708, + "hit@5": 0.5319, + "all@5": 0.0887, + "recall@10": 0.3809, + "hit@10": 0.6454, + "all@10": 0.1596, + "recall@20": 0.3959, + "hit@20": 0.6773, + "all@20": 0.1631 + }, + "bm25": { + "recall@1": 0.0589, + "hit@1": 0.1596, + "all@1": 0.0, + "recall@5": 0.1803, + "hit@5": 0.3688, + "all@5": 0.0532, + "recall@10": 0.2641, + "hit@10": 0.4929, + "all@10": 0.1028, + "recall@20": 0.3485, + "hit@20": 0.6028, + "all@20": 0.1525 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 73.1 }, - "nyx_returned_p50": 18, - "recall_ms_p50": 46.9 - }, - "3 open-domain": { - "n": 92, - "nyx": { - "recall@1": 0.0109, - "hit@1": 0.0109, - "all@1": 0.0109, - "recall@5": 0.0408, - "hit@5": 0.0543, - "all@5": 0.0326, - "recall@10": 0.2035, - "hit@10": 0.2826, - "all@10": 0.1522, - "recall@20": 0.3328, - "hit@20": 0.4783, - "all@20": 0.2283 + "2 temporal": { + "n": 321, + "nyx": { + "recall@1": 0.4847, + "hit@1": 0.514, + "all@1": 0.4611, + "recall@5": 0.6503, + "hit@5": 0.6822, + "all@5": 0.6199, + "recall@10": 0.7069, + "hit@10": 0.7352, + "all@10": 0.676, + "recall@20": 0.7334, + "hit@20": 0.7601, + "all@20": 0.704 + }, + "bm25": { + "recall@1": 0.3409, + "hit@1": 0.3645, + "all@1": 0.324, + "recall@5": 0.6114, + "hit@5": 0.6417, + "all@5": 0.5857, + "recall@10": 0.6783, + "hit@10": 0.7072, + "all@10": 0.648, + "recall@20": 0.7163, + "hit@20": 0.7445, + "all@20": 0.6822 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 72.2 }, - "bm25": { - "recall@1": 0.1123, - "hit@1": 0.1413, - "all@1": 0.087, - "recall@5": 0.2431, - "hit@5": 0.3261, - "all@5": 0.1739, - "recall@10": 0.2716, - "hit@10": 0.3696, - "all@10": 0.1848, - "recall@20": 0.3188, - "hit@20": 0.4348, - "all@20": 0.2283 + "3 open-domain": { + "n": 92, + "nyx": { + "recall@1": 0.1395, + "hit@1": 0.1848, + "all@1": 0.0978, + "recall@5": 0.2518, + "hit@5": 0.3261, + "all@5": 0.1848, + "recall@10": 0.3017, + "hit@10": 0.4022, + "all@10": 0.2283, + "recall@20": 0.3105, + "hit@20": 0.4348, + "all@20": 0.2283 + }, + "bm25": { + "recall@1": 0.1123, + "hit@1": 0.1413, + "all@1": 0.087, + "recall@5": 0.2431, + "hit@5": 0.3261, + "all@5": 0.1739, + "recall@10": 0.2716, + "hit@10": 0.3696, + "all@10": 0.1848, + "recall@20": 0.3188, + "hit@20": 0.4348, + "all@20": 0.2283 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 73.6 }, - "nyx_returned_p50": 18, - "recall_ms_p50": 50.5 - }, - "4 single-hop": { - "n": 841, - "nyx": { - "recall@1": 0.0012, - "hit@1": 0.0012, - "all@1": 0.0012, - "recall@5": 0.0406, - "hit@5": 0.0428, - "all@5": 0.038, - "recall@10": 0.519, - "hit@10": 0.5303, - "all@10": 0.5065, - "recall@20": 0.6962, - "hit@20": 0.7087, - "all@20": 0.6825 + "4 single-hop": { + "n": 841, + "nyx": { + "recall@1": 0.3623, + "hit@1": 0.3746, + "all@1": 0.352, + "recall@5": 0.5955, + "hit@5": 0.6088, + "all@5": 0.5815, + "recall@10": 0.6764, + "hit@10": 0.6861, + "all@10": 0.6659, + "recall@20": 0.6831, + "hit@20": 0.6932, + "all@20": 0.6718 + }, + "bm25": { + "recall@1": 0.3543, + "hit@1": 0.3639, + "all@1": 0.346, + "recall@5": 0.5896, + "hit@5": 0.6017, + "all@5": 0.5767, + "recall@10": 0.655, + "hit@10": 0.6659, + "all@10": 0.6433, + "recall@20": 0.7297, + "hit@20": 0.742, + "all@20": 0.717 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 71.9 }, - "bm25": { - "recall@1": 0.3543, - "hit@1": 0.3639, - "all@1": 0.346, - "recall@5": 0.5896, - "hit@5": 0.6017, - "all@5": 0.5767, - "recall@10": 0.655, - "hit@10": 0.6659, - "all@10": 0.6433, - "recall@20": 0.7297, - "hit@20": 0.742, - "all@20": 0.717 + "_overall": { + "n": 1536, + "nyx": { + "recall@1": 0.3263, + "hit@1": 0.3711, + "all@1": 0.2956, + "recall@5": 0.5268, + "hit@5": 0.5931, + "all@5": 0.4753, + "recall@10": 0.6061, + "hit@10": 0.6719, + "all@10": 0.5488, + "recall@20": 0.6186, + "hit@20": 0.6888, + "all@20": 0.5586 + }, + "bm25": { + "recall@1": 0.2828, + "hit@1": 0.3132, + "all@1": 0.2624, + "recall@5": 0.4982, + "hit@5": 0.5508, + "all@5": 0.4583, + "recall@10": 0.5651, + "hit@10": 0.625, + "all@10": 0.5176, + "recall@20": 0.6323, + "hit@20": 0.6986, + "all@20": 0.5768 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 72.3 }, - "nyx_returned_p50": 18, - "recall_ms_p50": 39.8 - }, - "_overall": { - "n": 1536, - "nyx": { - "recall@1": 0.0024, - "hit@1": 0.0039, - "all@1": 0.0013, - "recall@5": 0.0457, - "hit@5": 0.0586, - "all@5": 0.0365, - "recall@10": 0.4594, - "hit@10": 0.5195, - "all@10": 0.4147, - "recall@20": 0.6302, - "hit@20": 0.7031, - "all@20": 0.5703 + "_meta": { + "conversations": 10, + "errors": 0, + "questions_scored": 1536, + "skipped": { + "no evidence": 4, + "adversarial": 446 + }, + "turns": 5882, + "ingest_s_total": 451.8, + "unmapped_objects": 0, + "semantic_backend": [ + "local:paraphrase-multilingual-MiniLM-L12-v2" + ], + "data": "locomo10.json", + "wall_s": 179.5, + "budget": 1500 + } + } + }, + "lexical_only": { + "backend": "off", + "scores": { + "1 multi-hop": { + "n": 282, + "nyx": { + "recall@1": 0.1119, + "hit@1": 0.2943, + "all@1": 0.0, + "recall@5": 0.2886, + "hit@5": 0.5461, + "all@5": 0.1099, + "recall@10": 0.3828, + "hit@10": 0.6525, + "all@10": 0.1702, + "recall@20": 0.3954, + "hit@20": 0.6738, + "all@20": 0.1738 + }, + "bm25": { + "recall@1": 0.0589, + "hit@1": 0.1596, + "all@1": 0.0, + "recall@5": 0.1803, + "hit@5": 0.3688, + "all@5": 0.0532, + "recall@10": 0.2641, + "hit@10": 0.4929, + "all@10": 0.1028, + "recall@20": 0.3485, + "hit@20": 0.6028, + "all@20": 0.1525 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 49.9 }, - "bm25": { - "recall@1": 0.2828, - "hit@1": 0.3132, - "all@1": 0.2624, - "recall@5": 0.4982, - "hit@5": 0.5508, - "all@5": 0.4583, - "recall@10": 0.5651, - "hit@10": 0.625, - "all@10": 0.5176, - "recall@20": 0.6323, - "hit@20": 0.6986, - "all@20": 0.5768 + "2 temporal": { + "n": 321, + "nyx": { + "recall@1": 0.4925, + "hit@1": 0.5202, + "all@1": 0.4704, + "recall@5": 0.6664, + "hit@5": 0.6978, + "all@5": 0.6386, + "recall@10": 0.7261, + "hit@10": 0.757, + "all@10": 0.7009, + "recall@20": 0.7448, + "hit@20": 0.7757, + "all@20": 0.7196 + }, + "bm25": { + "recall@1": 0.3409, + "hit@1": 0.3645, + "all@1": 0.324, + "recall@5": 0.6114, + "hit@5": 0.6417, + "all@5": 0.5857, + "recall@10": 0.6783, + "hit@10": 0.7072, + "all@10": 0.648, + "recall@20": 0.7163, + "hit@20": 0.7445, + "all@20": 0.6822 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 48.8 }, - "nyx_returned_p50": 18, - "recall_ms_p50": 42.4 - }, - "_meta": { - "conversations": 10, - "errors": 0, - "questions_scored": 1536, - "skipped": { - "no evidence": 4, - "adversarial": 446 + "3 open-domain": { + "n": 92, + "nyx": { + "recall@1": 0.1232, + "hit@1": 0.1522, + "all@1": 0.0978, + "recall@5": 0.2817, + "hit@5": 0.3913, + "all@5": 0.1957, + "recall@10": 0.3276, + "hit@10": 0.4565, + "all@10": 0.2283, + "recall@20": 0.3364, + "hit@20": 0.4783, + "all@20": 0.2283 + }, + "bm25": { + "recall@1": 0.1123, + "hit@1": 0.1413, + "all@1": 0.087, + "recall@5": 0.2431, + "hit@5": 0.3261, + "all@5": 0.1739, + "recall@10": 0.2716, + "hit@10": 0.3696, + "all@10": 0.1848, + "recall@20": 0.3188, + "hit@20": 0.4348, + "all@20": 0.2283 + }, + "nyx_returned_p50": 18, + "recall_ms_p50": 55.4 }, - "turns": 5882, - "ingest_s_total": 40.3, - "unmapped_objects": 0, - "semantic_backend": [ - "off" - ], - "data": "locomo10.json", - "wall_s": 16.1, - "budget": 1500 - } - }, - "held_out_conversations_2_10": { - "v7.13.4": { - "_overall": { - "n": 1386, + "4 single-hop": { + "n": 841, "nyx": { - "recall@1": 0.0026, - "hit@1": 0.0043, - "all@1": 0.0014, - "recall@5": 0.0371, - "hit@5": 0.0505, - "all@5": 0.0274, - "recall@10": 0.4579, - "hit@10": 0.518, - "all@10": 0.4134, - "recall@20": 0.6314, - "hit@20": 0.7049, - "all@20": 0.5714 + "recall@1": 0.4039, + "hit@1": 0.4162, + "all@1": 0.3936, + "recall@5": 0.6151, + "hit@5": 0.629, + "all@5": 0.6005, + "recall@10": 0.6895, + "hit@10": 0.7039, + "all@10": 0.6742, + "recall@20": 0.6938, + "hit@20": 0.7075, + "all@20": 0.679 }, "bm25": { - "recall@1": 0.2858, - "hit@1": 0.3182, - "all@1": 0.2641, - "recall@5": 0.4983, - "hit@5": 0.5512, - "all@5": 0.4589, - "recall@10": 0.5657, - "hit@10": 0.6263, - "all@10": 0.518, - "recall@20": 0.6345, - "hit@20": 0.702, - "all@20": 0.5786 + "recall@1": 0.3543, + "hit@1": 0.3639, + "all@1": 0.346, + "recall@5": 0.5896, + "hit@5": 0.6017, + "all@5": 0.5767, + "recall@10": 0.655, + "hit@10": 0.6659, + "all@10": 0.6433, + "recall@20": 0.7297, + "hit@20": 0.742, + "all@20": 0.717 }, "nyx_returned_p50": 18, - "recall_ms_p50": 42.3 + "recall_ms_p50": 39.6 }, - "_meta": { - "questions_scored": 1386 - } - }, - "v7.13.5": { "_overall": { - "n": 1386, + "n": 1536, "nyx": { - "recall@1": 0.3545, - "hit@1": 0.404, - "all@1": 0.3218, - "recall@5": 0.546, - "hit@5": 0.6147, - "all@5": 0.4942, - "recall@10": 0.6191, - "hit@10": 0.6912, - "all@10": 0.5606, - "recall@20": 0.6292, - "hit@20": 0.7035, - "all@20": 0.5685 + "recall@1": 0.352, + "hit@1": 0.3997, + "all@1": 0.3197, + "recall@5": 0.5459, + "hit@5": 0.6139, + "all@5": 0.4941, + "recall@10": 0.6192, + "hit@10": 0.6908, + "all@10": 0.5605, + "recall@20": 0.6283, + "hit@20": 0.7018, + "all@20": 0.5677 }, "bm25": { - "recall@1": 0.2858, - "hit@1": 0.3182, - "all@1": 0.2641, - "recall@5": 0.4983, - "hit@5": 0.5512, - "all@5": 0.4589, - "recall@10": 0.5657, - "hit@10": 0.6263, - "all@10": 0.518, - "recall@20": 0.6345, - "hit@20": 0.702, - "all@20": 0.5786 + "recall@1": 0.2828, + "hit@1": 0.3132, + "all@1": 0.2624, + "recall@5": 0.4982, + "hit@5": 0.5508, + "all@5": 0.4583, + "recall@10": 0.5651, + "hit@10": 0.625, + "all@10": 0.5176, + "recall@20": 0.6323, + "hit@20": 0.6986, + "all@20": 0.5768 }, "nyx_returned_p50": 18, - "recall_ms_p50": 43.5 + "recall_ms_p50": 43.4 }, "_meta": { - "questions_scored": 1386 + "conversations": 10, + "errors": 0, + "questions_scored": 1536, + "skipped": { + "no evidence": 4, + "adversarial": 446 + }, + "turns": 5882, + "ingest_s_total": 36.2, + "unmapped_objects": 0, + "semantic_backend": [ + "off" + ], + "data": "locomo10.json", + "wall_s": 15.7, + "budget": 1500 } } } diff --git a/benchmarks/results/longmemeval_s_retrieval.json b/benchmarks/results/longmemeval_s_retrieval.json new file mode 100644 index 0000000..855de65 --- /dev/null +++ b/benchmarks/results/longmemeval_s_retrieval.json @@ -0,0 +1,246 @@ +{ + "benchmark": "LongMemEval-S (longmemeval_s_cleaned.json) — 会话级 recall_any@k / recall_all@k，官方定义，abstention 不计", + "data_sha256": "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442", + "v7.13.6_semantic": { + "backend": "local:paraphrase-multilingual-MiniLM-L12-v2", + "fusion": "NYX_EMBED_Z=3, NYX_EMBED_WEIGHT=0.5", + "scores": { + "_overall": { + "n": 470, + "recall_any@1": 0.9128, + "recall_all@1": 0.3191, + "recall_any@3": 0.9745, + "recall_all@3": 0.8064, + "recall_any@5": 0.9809, + "recall_all@5": 0.8617, + "recall_any@10": 0.9851, + "recall_all@10": 0.8851, + "turn_hit@10": 0.9064, + "recall_ms_p50": 138.8 + }, + "knowledge-update": { + "n": 72, + "recall_any@1": 0.9861, + "recall_all@1": 0.0, + "recall_any@3": 1.0, + "recall_all@3": 0.9583, + "recall_any@5": 1.0, + "recall_all@5": 0.9861, + "recall_any@10": 1.0, + "recall_all@10": 1.0, + "turn_hit@10": 0.9861, + "recall_ms_p50": 127.0 + }, + "multi-session": { + "n": 121, + "recall_any@1": 0.9256, + "recall_all@1": 0.0, + "recall_any@3": 0.9752, + "recall_all@3": 0.6364, + "recall_any@5": 0.9835, + "recall_all@5": 0.7851, + "recall_any@10": 0.9835, + "recall_all@10": 0.8017, + "turn_hit@10": 0.9256, + "recall_ms_p50": 134.4 + }, + "single-session-assistant": { + "n": 56, + "recall_any@1": 0.9643, + "recall_all@1": 0.9643, + "recall_any@3": 1.0, + "recall_all@3": 1.0, + "recall_any@5": 1.0, + "recall_all@5": 1.0, + "recall_any@10": 1.0, + "recall_all@10": 1.0, + "turn_hit@10": 0.9464, + "recall_ms_p50": 163.1 + }, + "single-session-preference": { + "n": 30, + "recall_any@1": 0.6667, + "recall_all@1": 0.6667, + "recall_any@3": 0.8667, + "recall_all@3": 0.8667, + "recall_any@5": 0.9, + "recall_all@5": 0.9, + "recall_any@10": 0.9, + "recall_all@10": 0.9, + "turn_hit@10": 0.7667, + "recall_ms_p50": 136.4 + }, + "single-session-user": { + "n": 64, + "recall_any@1": 0.9531, + "recall_all@1": 0.9531, + "recall_any@3": 1.0, + "recall_all@3": 1.0, + "recall_any@5": 1.0, + "recall_all@5": 1.0, + "recall_any@10": 1.0, + "recall_all@10": 1.0, + "turn_hit@10": 0.9688, + "recall_ms_p50": 132.2 + }, + "temporal-reasoning": { + "n": 127, + "recall_any@1": 0.874, + "recall_all@1": 0.1181, + "recall_any@3": 0.9606, + "recall_all@3": 0.685, + "recall_any@5": 0.9685, + "recall_all@5": 0.7244, + "recall_any@10": 0.9843, + "recall_all@10": 0.7874, + "turn_hit@10": 0.8268, + "recall_ms_p50": 140.0 + }, + "_meta": { + "questions": 500, + "errors": 0, + "abstention_skipped": 30, + "ingest_s_total": 5958.4, + "unmapped_objects": 0, + "data": "longmemeval_s_cleaned.json", + "wall_s": 1935.7, + "budget": 1500, + "workers": 4, + "semantic_backend": [ + "local:paraphrase-multilingual-MiniLM-L12-v2" + ] + } + } + }, + "lexical_only": { + "backend": "off", + "scores": { + "_overall": { + "n": 470, + "recall_any@1": 0.7766, + "recall_all@1": 0.2787, + "recall_any@3": 0.9085, + "recall_all@3": 0.6936, + "recall_any@5": 0.9489, + "recall_all@5": 0.7745, + "recall_any@10": 0.9681, + "recall_all@10": 0.8191, + "turn_hit@10": 0.783, + "recall_ms_p50": 111.8 + }, + "knowledge-update": { + "n": 72, + "recall_any@1": 0.9028, + "recall_all@1": 0.0, + "recall_any@3": 0.9722, + "recall_all@3": 0.8472, + "recall_any@5": 0.9861, + "recall_all@5": 0.8889, + "recall_any@10": 1.0, + "recall_all@10": 0.9167, + "turn_hit@10": 0.9167, + "recall_ms_p50": 109.6 + }, + "multi-session": { + "n": 121, + "recall_any@1": 0.719, + "recall_all@1": 0.0, + "recall_any@3": 0.9256, + "recall_all@3": 0.4545, + "recall_any@5": 0.9752, + "recall_all@5": 0.6033, + "recall_any@10": 0.9917, + "recall_all@10": 0.6777, + "turn_hit@10": 0.7438, + "recall_ms_p50": 103.6 + }, + "single-session-assistant": { + "n": 56, + "recall_any@1": 0.9107, + "recall_all@1": 0.9107, + "recall_any@3": 0.9821, + "recall_all@3": 0.9821, + "recall_any@5": 0.9821, + "recall_all@5": 0.9821, + "recall_any@10": 0.9821, + "recall_all@10": 0.9821, + "turn_hit@10": 0.8571, + "recall_ms_p50": 136.0 + }, + "single-session-preference": { + "n": 30, + "recall_any@1": 0.2667, + "recall_all@1": 0.2667, + "recall_any@3": 0.5333, + "recall_all@3": 0.5333, + "recall_any@5": 0.7, + "recall_all@5": 0.7, + "recall_any@10": 0.8, + "recall_all@10": 0.8, + "turn_hit@10": 0.1667, + "recall_ms_p50": 104.6 + }, + "single-session-user": { + "n": 64, + "recall_any@1": 0.9219, + "recall_all@1": 0.9219, + "recall_any@3": 0.9531, + "recall_all@3": 0.9531, + "recall_any@5": 0.9844, + "recall_all@5": 0.9844, + "recall_any@10": 0.9844, + "recall_all@10": 0.9844, + "turn_hit@10": 0.9219, + "recall_ms_p50": 107.0 + }, + "temporal-reasoning": { + "n": 127, + "recall_any@1": 0.748, + "recall_all@1": 0.1024, + "recall_any@3": 0.8898, + "recall_all@3": 0.6142, + "recall_any@5": 0.9291, + "recall_all@5": 0.6929, + "recall_any@10": 0.9528, + "recall_all@10": 0.748, + "turn_hit@10": 0.7874, + "recall_ms_p50": 116.8 + }, + "_meta": { + "questions": 500, + "errors": 0, + "abstention_skipped": 30, + "ingest_s_total": 1812.2, + "unmapped_objects": 0, + "data": "longmemeval_s_cleaned.json", + "wall_s": 502.4, + "budget": 1500, + "workers": 4, + "semantic_backend": [ + "off" + ] + } + } + }, + "fusion_selection": { + "protocol": "候选 Z∈{2,3}×w∈{1,0.5,0.3}；A 半（LoCoMo 第 1–5 段、LongMemEval 偶数题）上取 mean(LoCoMo recall@5, LongMemEval all@5) 最高者；B 半（另一半）只用来报告。检索结果由离线复现 SearchRouter 融合得到（与实跑对照：LoCoMo 0.578 vs 0.575，LongMemEval 纯词法 0.951 vs 0.949）", + "selected": "Z=3, w=0.5", + "held_out_B": { + "lexical": { + "locomo_recall@5": 0.539, + "lme_any@5": 0.957, + "lme_all@5": 0.753 + }, + "old_fusion_Z2_w1": { + "locomo_recall@5": 0.51, + "lme_any@5": 0.983, + "lme_all@5": 0.872 + }, + "selected_Z3_w0.5": { + "locomo_recall@5": 0.566, + "lme_any@5": 0.987, + "lme_all@5": 0.843 + } + } + } +} \ No newline at end of file diff --git a/nexsandglass/core/sandglass_paths.py b/nexsandglass/core/sandglass_paths.py index 454ac5d..bd33f36 100644 --- a/nexsandglass/core/sandglass_paths.py +++ b/nexsandglass/core/sandglass_paths.py @@ -8,7 +8,7 @@ import os _NB = os.environ.get("NEXSANDBASE_HOME") or os.path.join(os.path.expanduser("~"), ".neurobase") -__version__ = '7.13.5' +__version__ = '7.13.6' _SCRIPTS = os.path.join(_NB, "scripts") _PERSONA = os.path.join(_NB, "persona") _ARCHIVE = os.path.join(_NB, "archive") diff --git a/nexsandglass_hermes/plugin.yaml b/nexsandglass_hermes/plugin.yaml index ce24911..9e00603 100644 --- a/nexsandglass_hermes/plugin.yaml +++ b/nexsandglass_hermes/plugin.yaml @@ -1,5 +1,5 @@ name: nyx -version: 7.13.5 +version: 7.13.6 description: >- Nyx 夜神 — 单用户、本地、零 API Key 的 agent 记忆。双时态事实（现在信什么 / 当时以为什么）、 写入时绑定来源与信任（防记忆投毒）、两段式遗忘（隔离区可还原）、Déjà Vu 模糊感知。