From 95ff41101a820653218d47ba69146933d716e14b Mon Sep 17 00:00:00 2001 From: Andy Tsai Date: Wed, 2 Sep 2026 20:20:10 +0800 Subject: [PATCH] fix(query): score the `rationale` attribute as its own tier (#2293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The extraction spec stores the WHY of a concept as a `rationale` attribute on the node — deliberately not as a node of its own. `_score_query` matched a question against `norm_label`, `label_tokens`, `source_file` and the node id only, and `_node_search_text` indexed the same fields, so for a "why does X …" question the node holding the answer was neither a trigram candidate nor a seed unless the asker already knew its label (#2293: 0/5 answers reached on a 5.5k-node corpus although every answer node existed and stated the answer in its rationale). Add a rationale tier, following the fix the issue proposes: - `_RATIONALE_MATCH_BONUS = 0.75`: below the label substring tier (1.0), above the source-path tier (0.5). A term found in the rationale adds to the score and to the per-term singleton used for seed seating, but — like the source tier — never to term coverage, so a long rationale adds recall without winning back an exact-label tier it did not earn. - `_node_rationale_text`: one normalizer (diacritics folded, lower-cased), tolerant of the list an extractor may emit; "" when absent. - `_node_search_text` appends the same text as a trailing field, only when present, so the trigram prefilter stays a complete candidate generator and every existing field position (#2467) is unchanged. Nodes without a rationale score exactly as before. Design credit: @nuboxworld-byte (issue #2293 and #2294). #2294 implements the same idea but its diff carries the whole repository (+252k lines, 766 files) from a base mismatch and cannot be reviewed or merged as-is; this is a fresh minimal implementation against v8. Tests: rationale-only match ranks; tier order label-substring > rationale > source; no coverage credit; list-valued attribute; search text carries the field (and is unchanged without it); end to end a "why" question phrased from the rationale seats the node as a seed. Co-Authored-By: Claude Fable 5.1 --- graphify/serve.py | 38 ++++++++++++++++++++++- tests/test_serve.py | 73 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/graphify/serve.py b/graphify/serve.py index a9ecd3540..771322221 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -295,6 +295,14 @@ def _query_terms(question: str) -> list[str]: _PREFIX_MATCH_BONUS = 100.0 _SUBSTRING_MATCH_BONUS = 1.0 _SOURCE_MATCH_BONUS = 0.5 +# The extraction spec stores the WHY of a concept as a `rationale` attribute +# on the node, not as a node of its own, so for a "why does X …" question that +# prose is often the only place the question's words occur (#2293). Score it +# as its own tier: below a label substring hit (the label still names the +# thing), above a source-path hit, and — like the source tier — never counted +# toward term coverage, so a long rationale adds recall without winning back +# an exact-label tier it did not earn. +_RATIONALE_MATCH_BONUS = 0.75 def _compute_idf(G: nx.Graph, terms: list[str]) -> dict[str, float]: @@ -329,10 +337,28 @@ def _trigrams(text: str) -> set[str]: return {text[i:i + 3] for i in range(len(text) - 2)} +def _node_rationale_text(data: dict) -> str: + """The node's `rationale` attribute normalized like a label (diacritics + folded, lower-cased) for substring matching. Semantic cleanup writes it as + one string (several sources joined with blank lines); an extractor may hand + over a list — join it. Missing or empty -> "" so callers can `if rationale`. + """ + raw = data.get("rationale") + if not raw: + return "" + if isinstance(raw, (list, tuple)): + raw = " ".join(str(part) for part in raw if part) + return _strip_diacritics(str(raw)).lower() + + def _node_search_text(data: dict, nid: str) -> str: """Concatenate every field _score_nodes / _find_node match a query against, so one trigram index over this text is a complete candidate generator for both. + - `rationale` (normalized via `_node_rationale_text`) feeds _score_nodes' + rationale tier (#2293); appended last, and only when present, so every + other field position is unchanged. + - `norm_label` and `source_file` feed _score_nodes' per-term substring tiers. - `label_tokens` (the space-joined token form) feeds _find_node's `term in label_tokens` branch, where a multi-word `term` can span a token @@ -363,6 +389,9 @@ class 0 and therefore survive the combining-character filter. The field is nid_folded = _strip_diacritics(str(nid)).lower() if nid_folded != nid_text: fields += (nid_folded,) + rationale = _node_rationale_text(data) + if rationale: + fields += (rationale,) return "\x00".join(fields) @@ -539,6 +568,7 @@ def _score_query( # driver". label_tokens = " ".join(_search_tokens(data.get("label") or "")) source = (data.get("source_file") or "").lower() + rationale = _node_rationale_text(data) # `nid_lower` is needed both by the full-query tier (`if joined`) and by # the per-token singleton tier (joined-singlet exact-match check). When # neither runs (`joined` empty AND not collecting seeds) skip the call; @@ -598,6 +628,12 @@ def _score_query( if t in source: source_value = _SOURCE_MATCH_BONUS * w score += source_value + # Rationale tier (#2293): recall for "why" questions whose words + # live only in the attribute. Adds to the score, not to `matched`. + rationale_value = 0.0 + if rationale and t in rationale: + rationale_value = _RATIONALE_MATCH_BONUS * w + score += rationale_value tiered += tier_value if collect_per_term_seeds and best_by_term is not None: # Singleton score for [t] on this node, mirroring @@ -616,7 +652,7 @@ def _score_query( singleton = _PREFIX_MATCH_BONUS * 10 * w else: singleton = 0.0 - singleton += tier_value + substr_value + source_value + singleton += tier_value + substr_value + source_value + rationale_value if singleton > 0: # Tie-break key mirrors the legacy sort+max(degree): # (-singleton, -degree, label_len, nid) — the minimum diff --git a/tests/test_serve.py b/tests/test_serve.py index 87e71f821..3d4a943b0 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -1735,3 +1735,76 @@ def test_resolve_single_node_shared_by_get_node_and_get_neighbors(): nid, err = _resolve_single_node(G, "nonexistent") assert nid is None assert "No node matching" in err + + +# --- rationale attribute scoring (#2293) --- + +_FAB_RATIONALE = ( + "Hidden when the mini card is dismissed and while the geolocation popover " + "is open, because that popover opens upward into the DirectionsFAB's space." +) + + +def _rationale_graph(): + """A doc-derived rule node whose LABEL shares no token with the question + while its `rationale` attribute states the answer in plain words — the + #2293 shape. Neighbors carry the identifier-ish labels a codebase would.""" + G = nx.Graph() + G.add_node("rule", label="FAB visibility rule", source_file="docs/fab.md", rationale=_FAB_RATIONALE) + G.add_node("fab", label="DirectionsFAB", source_file="src/DirectionsFAB.tsx") + G.add_node("geo", label="GeolocationButton", source_file="src/GeolocationButton.tsx") + G.add_node("card", label="MiniCard", source_file="src/MiniCard.tsx") + for u, v in [("rule", "fab"), ("fab", "geo"), ("rule", "card")]: + G.add_edge(u, v, relation="references", confidence="EXTRACTED") + return G + + +def test_score_nodes_reads_rationale_when_label_does_not_match(): + G = _rationale_graph() + assert [nid for _, nid in _score_nodes(G, ["popover"])] == ["rule"] + + +def test_score_nodes_rationale_tier_sits_below_label_substring_and_above_source(): + G = nx.Graph() + G.add_node("lbl", label="popover-anchor", source_file="ui/a.py") + G.add_node("rat", label="Sheet drag", source_file="ui/b.py", rationale="starts only once the popover is closed") + G.add_node("src", label="Thing", source_file="ui/popover/thing.py") + assert [nid for _, nid in _score_nodes(G, ["popover"])] == ["lbl", "rat", "src"] + + +def test_score_nodes_rationale_does_not_count_toward_term_coverage(): + """Like the source tier, a rationale hit adds recall but must not restore + the coverage-scaled exact tier: if it counted, `a` would gain roughly three + quarters of an exact-match bonus over `b`, not a sub-unit nudge.""" + from graphify.serve import _EXACT_MATCH_BONUS + G = nx.Graph() + G.add_node("a", label="cache", source_file="x.py", rationale="pinned because of drift") + G.add_node("b", label="cache", source_file="y.py") + score = {nid: s for s, nid in _score_nodes(G, ["cache", "pinned"])} + assert score["a"] > score["b"] + assert score["a"] - score["b"] < _EXACT_MATCH_BONUS * 0.5 + + +def test_score_nodes_tolerates_list_valued_rationale(): + G = nx.Graph() + G.add_node("n", label="X", source_file="x.py", rationale=["first reason", "popover second"]) + assert [nid for _, nid in _score_nodes(G, ["popover"])] == ["n"] + + +def test_node_search_text_includes_rationale_so_trigram_prefilter_stays_complete(): + parts = _node_search_text( + {"label": "Foo", "source_file": "a.py", "rationale": "Because the Popover opens upward"}, "foo" + ).split("\x00") + assert "because the popover opens upward" in parts + # No rationale: field layout unchanged (the #2467 positions still hold). + assert len(_node_search_text({"label": "Foo", "source_file": "a.py"}, "foo").split("\x00")) == 5 + + +def test_query_graph_text_seeds_the_node_whose_rationale_answers_a_why_question(): + G = _rationale_graph() + text = _query_graph_text( + G, "why is the directions button hidden when the geolocation popover opens", + mode="bfs", depth=2, token_budget=2000, + ) + header = text.split("\n\n", 1)[0] + assert "FAB visibility rule" in header, header