From f322f200b0eed3f988516028a8e9a05a494b4624 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Thu, 20 Aug 2026 19:31:40 +0530 Subject: [PATCH] Add Recall@5 regression metric with Multihop30 dataset - New recall_evaluator.py: computes Recall@K by querying GraphRAG and matching retrieved chunks against ground-truth chunks using embedding cosine similarity (threshold=0.70) or LLM judge - New run_recall.sh: Docker wrapper to run recall evaluation - New Multihop30 dataset: 30 multi-hop questions (10 each from HotpotQA, 2WikiMultiHopQA, MuSiQue) with raw text corpus, answers, and ground_truth_chunks.csv for retrieval scoring - Updated evaluator.py, run_eval.sh: unified --mode parameter replacing --agent + --search-type - Updated run_setup.sh, setup_graph.py: support for Multihop30 graph setup Co-authored-by: Cursor --- graphrag/tests/regression/evaluator.py | 64 +- graphrag/tests/regression/recall_evaluator.py | 936 ++++++++++++++++++ graphrag/tests/regression/run_eval.sh | 28 +- graphrag/tests/regression/run_recall.sh | 66 ++ graphrag/tests/regression/run_setup.sh | 2 +- graphrag/tests/regression/setup_graph.py | 30 +- .../tests/test_questions/Multihop30/README.md | 81 ++ .../test_questions/Multihop30/answers.csv | 31 + .../data/2wikimultihopqa_corpus.txt | 289 ++++++ .../Multihop30/data/hotpotqa_corpus.txt | 300 ++++++ .../Multihop30/data/musique_corpus.txt | 453 +++++++++ .../Multihop30/ground_truth_chunks.csv | 91 ++ .../test_questions/Multihop30/questions.csv | 31 + 13 files changed, 2361 insertions(+), 41 deletions(-) create mode 100644 graphrag/tests/regression/recall_evaluator.py create mode 100755 graphrag/tests/regression/run_recall.sh mode change 100644 => 100755 graphrag/tests/regression/run_setup.sh create mode 100644 graphrag/tests/test_questions/Multihop30/README.md create mode 100644 graphrag/tests/test_questions/Multihop30/answers.csv create mode 100644 graphrag/tests/test_questions/Multihop30/data/2wikimultihopqa_corpus.txt create mode 100644 graphrag/tests/test_questions/Multihop30/data/hotpotqa_corpus.txt create mode 100644 graphrag/tests/test_questions/Multihop30/data/musique_corpus.txt create mode 100644 graphrag/tests/test_questions/Multihop30/ground_truth_chunks.csv create mode 100644 graphrag/tests/test_questions/Multihop30/questions.csv diff --git a/graphrag/tests/regression/evaluator.py b/graphrag/tests/regression/evaluator.py index 3638fd44..df2ae257 100644 --- a/graphrag/tests/regression/evaluator.py +++ b/graphrag/tests/regression/evaluator.py @@ -595,6 +595,44 @@ def _print_detailed(r: EvalResult, total: int) -> None: print(f" reason: {r.hallucination_reason}", flush=True) +def _parse_mode(mode_str: str) -> tuple: + """Parse --mode into (api_mode, rag_pattern). + + Agentic styles (mode=agentic, rag_pattern=