From 83a4dfd84c30263d8bd8bd6cdfe1f696bf0cda22 Mon Sep 17 00:00:00 2001 From: Shuxin Lin Date: Wed, 14 Oct 2026 15:36:27 -0400 Subject: [PATCH 1/3] fix(evaluation): score CAR on groundtruth_eval.json concept coverage When a CAR scenario ships groundtruth_eval.json with required_terms, the static_json scorer now uses it. The answer earns 0 unless it is a single-key object whose key is the groundtruth.txt mode. Otherwise its credit is the fraction of required concepts covered, a concept being covered when any one of its any_of forms appears, on word boundaries, in the value under the key. The scenario passes at credit >= 0.5 (CAR_PASS_THRESHOLD). Precision, recall, f1 and the new car_score field all report the credit: with one matching key the model side holds exactly the gold entries, so an F1 reduces to coverage. CAR scenarios without required_terms keep the extracted-term F1 and the strict pass rule. Term matching also read structured values through json.dumps, which kept upper-case letters out of the a-z0-9 filter and turned a newline into "\n", gluing an "n" onto the next word. It now joins the structure's keys and scalars and lowercases the result. Signed-off-by: Shuxin Lin --- docs/static-json-evaluation.md | 90 ++++++++--- src/evaluation/scorers/static_json.py | 148 ++++++++++++++---- .../tests/test_static_json_scorer.py | 141 +++++++++++++++++ 3 files changed, 325 insertions(+), 54 deletions(-) diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index daf2bd23..9f20a467 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -100,18 +100,36 @@ The evaluator then joins trajectories and ground truth by scenario id. Scenario folders may also include `groundtruth_eval.json`, which carries scorer-specific metadata alongside the canonical answer in `groundtruth.txt`. -The `static_json` scorer does not read it. For clarification-abstain-response -(CAR) scenarios the expected mode is the single key of `groundtruth.txt`, and -the required terms are extracted from the value under that key. +The loader exposes it as `Scenario.evaluation_metadata`. `static_json` reads +it only for clarification-abstain-response (CAR) scenarios, whose expected +mode is the single key of `groundtruth.txt`. -Example `groundtruth.txt`: +Example CAR `groundtruth.txt`: + +```json +{ + "clarification": "Which asset and which time window do you mean?" +} +``` + +Example CAR `groundtruth_eval.json`: ```json { - "clarification": "Which asset do you mean by 'the main unit'?" + "mode": "clarification", + "required_terms": [ + {"concept": "asset", "any_of": ["asset", "equipment"]}, + {"concept": "time window", "any_of": ["time window", "date range"]} + ], + "optional_terms": [] } ``` +Each required concept lists the surface forms (`any_of`) that count as +covering it. A bare string in `required_terms` is a concept with that one +form. `mode` and `optional_terms` are not scored; the mode comes from +`groundtruth.txt`. + --- ## How Matching Works @@ -232,20 +250,42 @@ exactly one of these top-level keys: response | clarification | abstain ``` -The required terms are extracted from the gold value: quoted phrases, asset or -fault identifiers (e.g. `PMP42144`), a leading `yes` / `no` stance, and a small -allowlist of domain terms. Matching is case-insensitive on normalised text. +Matching is case-insensitive on normalised text and on word boundaries, so +`asset` does not match inside `assetid`. + +#### With `groundtruth_eval.json` + +When the scenario's `groundtruth_eval.json` has `required_terms`, the score is +a credit in [0, 1]: -A CAR scenario passes when: +1. If the answer is not a JSON object with exactly one top-level key, or that + key is not the expected mode, the credit is 0. +2. Otherwise the credit is the fraction of required concepts covered. A + concept is covered when any one of its `any_of` forms appears in the value + under the key. -1. the model returns a JSON object with exactly one top-level key, -2. that key matches the expected mode (case-insensitive), and -3. every required term appears in the value under that key. +`precision`, `recall`, `f1` and `car_score` all report this credit: with one +matching key the model side holds exactly the gold entries, so precision +equals recall. The scenario passes when the credit is at least 0.5 +(`CAR_PASS_THRESHOLD`). For the example above (two concepts): -The score is the F1 over the mode key plus each required term. Gold has -`1 + len(required_terms)` entries; the model side has the same plus any extra -top-level keys. For gold `{"clarification": "Which asset do you mean by 'the -main unit'?"}` (one required term, `main unit`): +| Model answer | Passed | Score | +| ------------------------------------------------------------- | ------ | ----- | +| `{"clarification": "Which equipment, over what date range?"}` | yes | 1.0 | +| `{"clarification": "Which asset?"}` | yes | 0.5 | +| `{"clarification": "Can you say more?"}` | no | 0.0 | +| `{"abstain": "Which asset and time window?"}` | no | 0.0 | + +#### Without `groundtruth_eval.json` + +The required terms are extracted from the gold value: quoted phrases, asset or +fault identifiers (e.g. `PMP42144`), a leading `yes` / `no` stance, and a small +allowlist of domain terms. The scenario passes when the answer has exactly one +top-level key, that key is the expected mode, and every required term appears +in the value under it. The score is the F1 over the mode key plus each required +term. Gold has `1 + len(required_terms)` entries; the model side has the same +plus any extra top-level keys. For gold `{"clarification": "Which asset do you +mean by 'the main unit'?"}` (one required term, `main unit`): | Model answer | Passed | Score | | ---------------------------------------------------- | ------ | ----- | @@ -255,13 +295,14 @@ main unit'?"}` (one required term, `main unit`): Additional CAR report fields: -| Metric | Meaning | -| ---------------------- | ---------------------------------------------------------- | -| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | -| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | -| `mode_required_terms` | Required terms extracted from the gold answer value | -| `mode_matched_terms` | Required terms found in the model answer value | -| `mode_term_coverage` | Fraction of required terms found | +| Metric | Meaning | +| ---------------------- | -------------------------------------------------------------------- | +| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | +| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | +| `mode_required_terms` | Required concepts from `groundtruth_eval.json`, or extracted terms | +| `mode_matched_terms` | Required concepts or terms found in the model answer value | +| `mode_term_coverage` | Fraction of required concepts or terms found, regardless of the key | +| `car_score` | The gated credit; set only when `groundtruth_eval.json` is used | --- @@ -421,13 +462,14 @@ The scorer itself only needs: ```text gold_answer model_answer +eval_metadata # groundtruth_eval.json, CAR only ``` The evaluation pipeline is responsible for: ```text trajectory file → final answer + scenario_id/run_id -scenario folder → groundtruth.txt +scenario folder → groundtruth.txt (+ groundtruth_eval.json) join trajectory and scenario run scorer write report diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index a70b5d65..a26d8d32 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -78,6 +78,7 @@ class StaticJsonScore: mode_required_terms: list[str] = field(default_factory=list) mode_matched_terms: list[str] = field(default_factory=list) mode_term_coverage: float | None = None + car_score: float | None = None def to_dict(self) -> dict[str, Any]: """Return a JSON-serializable dictionary.""" @@ -426,6 +427,9 @@ def similarity_score(gold_value: str, model_value: str) -> float: _MODE_KEYS = frozenset({"response", "clarification", "abstain"}) +# A CAR answer scored against groundtruth_eval.json concepts passes at this +# credit. +CAR_PASS_THRESHOLD = 0.5 _IMPORTANT_MODE_TERMS = ( "air conditioner", "cannot determine", @@ -473,14 +477,25 @@ def similarity_score(gold_value: str, model_value: str) -> float: ) -def _normalize_text_for_terms(value: Any) -> str: +def _joined_text(value: Any) -> str: + """Join the keys and scalars of a structure without JSON escapes. + + json.dumps would turn a newline into ``\\n``, gluing the ``n`` onto the + next word. + """ if value is None: - text = "" - elif isinstance(value, (dict, list, tuple)): - text = json.dumps(value, sort_keys=True) - else: - text = str(value).strip().lower() - text = re.sub(r"[^a-z0-9]+", " ", text) + return "" + if isinstance(value, dict): + return " ".join( + f"{key} {_joined_text(item)}" for key, item in value.items() + ) + if isinstance(value, (list, tuple)): + return " ".join(_joined_text(item) for item in value) + return str(value) + + +def _normalize_text_for_terms(value: Any) -> str: + text = re.sub(r"[^a-z0-9]+", " ", _joined_text(value).lower()) return re.sub(r"\s+", " ", text).strip() @@ -540,6 +555,38 @@ def _extract_required_mode_terms(value: Any) -> list[str]: return _dedupe_terms(terms) +def _metadata_concepts( + eval_metadata: dict[str, Any] | None, +) -> list[tuple[str, list[str]]] | None: + """Read required concepts from a CAR scenario's groundtruth_eval.json. + + Returns ``(concept, any_of forms)`` pairs, or None when the metadata has + no ``required_terms`` list. A bare string entry is a concept whose only + form is itself. + """ + if not isinstance(eval_metadata, dict): + return None + required = eval_metadata.get("required_terms") + if not isinstance(required, list): + return None + + concepts: list[tuple[str, list[str]]] = [] + for entry in required: + if isinstance(entry, dict): + raw_forms = entry.get("any_of") or [] + name = str(entry.get("concept") or (raw_forms[0] if raw_forms else "")) + else: + raw_forms = [entry] + name = str(entry) + forms = [ + form + for form in (_normalize_text_for_terms(item) for item in raw_forms) + if form + ] + concepts.append((name, forms)) + return concepts + + def _is_mode_gold_answer(value: Any) -> bool: parsed = parse_structured_answer(value) if not isinstance(parsed, dict) or len(parsed) != 1: @@ -548,7 +595,21 @@ def _is_mode_gold_answer(value: Any) -> bool: return key in _MODE_KEYS -def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: +def _evaluate_mode_json( + gold_answer: Any, + model_answer: Any, + eval_metadata: dict[str, Any] | None = None, +) -> StaticJsonScore: + """Score a CAR answer: the mode key, then the terms under it. + + With groundtruth_eval.json concepts the answer earns nothing unless it is + a single-key object whose key is the gold mode; its credit is then the + fraction of required concepts covered, a concept being covered when any + one of its forms appears in the value. Precision, recall and f1 all + equal that credit, since a single matching key leaves the model side with + exactly the gold entries. Without concepts, terms are extracted from the + gold value and the score is the F1 over the mode key plus those terms. + """ gold = parse_structured_answer(gold_answer) model = parse_structured_answer(model_answer) @@ -562,10 +623,18 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: model_value = next(iter(model.values())) if model_is_dict and model_exactly_one_key else "" key_match = model_exactly_one_key and model_key == gold_key - required_terms = _extract_required_mode_terms(gold_value) + concepts = _metadata_concepts(eval_metadata) + gated = concepts is not None + if concepts is None: + concepts = [(term, [term]) for term in _extract_required_mode_terms(gold_value)] + required_terms = [name for name, _ in concepts] model_text = _normalize_text_for_terms(model_value) + matched_forms = [ + next((form for form in forms if f" {form} " in f" {model_text} "), None) + for _, forms in concepts + ] matched_terms = [ - term for term in required_terms if f" {term} " in f" {model_text} " + name for (name, _), form in zip(concepts, matched_forms) if form ] term_coverage = ( len(matched_terms) / len(required_terms) if required_terms else 1.0 @@ -583,13 +652,13 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: ) ] - for term in required_terms: - matched = term in matched_terms + for (term, forms), form in zip(concepts, matched_forms): + matched = form is not None details.append( KeyComparison( key=f"answer.required_term.{term}", - gold_value=term, - model_value=term if matched else "MISSING", + gold_value=" | ".join(forms), + model_value=form or "MISSING", exact=matched, match_type="term_present" if matched else "term_missing", similarity=1.0 if matched else 0.0, @@ -612,13 +681,16 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: total_model_keys = 1 + len(required_terms) + len(extra_keys) exact_matches = (1 if key_match else 0) + len(matched_terms) - precision = exact_matches / total_model_keys if total_model_keys else 0.0 - recall = exact_matches / total_gold_keys if total_gold_keys else 0.0 - f1 = ( - 2 * precision * recall / (precision + recall) - if precision + recall > 0 - else 0.0 - ) + if gated: + precision = recall = f1 = term_coverage if key_match else 0.0 + else: + precision = exact_matches / total_model_keys if total_model_keys else 0.0 + recall = exact_matches / total_gold_keys if total_gold_keys else 0.0 + f1 = ( + 2 * precision * recall / (precision + recall) + if precision + recall > 0 + else 0.0 + ) strict_exact = 1.0 if key_match and term_coverage == 1.0 and not extra_keys else 0.0 return StaticJsonScore( @@ -652,6 +724,7 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: mode_required_terms=required_terms, mode_matched_terms=matched_terms, mode_term_coverage=term_coverage, + car_score=f1 if gated else None, ) @@ -838,10 +911,15 @@ def evaluate_static_json( model_answer: Any, *, similarity_threshold: float = 0.0, + eval_metadata: dict[str, Any] | None = None, ) -> StaticJsonScore: - """Evaluate one structured gold answer against one model answer.""" + """Evaluate one structured gold answer against one model answer. + + ``eval_metadata`` is the scenario's groundtruth_eval.json; only CAR + answers read it. + """ if _is_mode_gold_answer(gold_answer): - return _evaluate_mode_json(gold_answer, model_answer) + return _evaluate_mode_json(gold_answer, model_answer, eval_metadata) if _is_choice_scalar(gold_answer): model_answer = _normalize_choice_answer(model_answer) @@ -1067,18 +1145,28 @@ def __call__( ), ) - static_score = evaluate_static_json(gold_answer, answer) - passed = static_score.strict_exact_match_accuracy == 1.0 + static_score = evaluate_static_json( + gold_answer, answer, eval_metadata=scenario.evaluation_metadata + ) + if static_score.car_score is not None: + passed = static_score.car_score >= CAR_PASS_THRESHOLD + rationale = ( + f"CAR credit {static_score.car_score:.3f} " + f"{'>=' if passed else '<'} {CAR_PASS_THRESHOLD}" + ) + else: + passed = static_score.strict_exact_match_accuracy == 1.0 + rationale = ( + "strict structured match" + if passed + else "structured answer differs from ground truth" + ) return ScorerResult( scorer=self.name, passed=passed, score=round(static_score.f1, 3), - rationale=( - "strict structured match" - if passed - else "structured answer differs from ground truth" - ), + rationale=rationale, details=static_score.to_dict(), ) diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index 2ab9db7e..65381884 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -1,3 +1,5 @@ +import json + import pytest from evaluation.scorers.static_json import ( @@ -534,6 +536,145 @@ def test_static_json_scorer_scores_car_on_mode_key_and_required_terms(): assert wrong_key.score == 0.4 +_CAR_GOLD = {"clarification": "Which asset and which time window do you mean?"} +_CAR_METADATA = { + "mode": "clarification", + "required_terms": [ + {"concept": "asset", "any_of": ["asset", "equipment"]}, + {"concept": "time window", "any_of": ["time window", "date range"]}, + ], +} + + +def test_car_concepts_give_zero_credit_on_a_wrong_key(): + score = evaluate_static_json( + _CAR_GOLD, + {"abstain": "Which asset and which time window?"}, + eval_metadata=_CAR_METADATA, + ) + + assert score.mode_key_match == 0.0 + assert score.mode_term_coverage == 1.0 + assert score.car_score == 0.0 + assert score.f1 == 0.0 + + +def test_car_concepts_give_zero_credit_to_a_multi_key_answer(): + score = evaluate_static_json( + _CAR_GOLD, + {"clarification": "Which asset, which time window?", "abstain": "n/a"}, + eval_metadata=_CAR_METADATA, + ) + + assert score.mode_exactly_one_key == 0.0 + assert score.car_score == 0.0 + + +def test_car_concepts_credit_is_the_covered_fraction(): + half = evaluate_static_json( + _CAR_GOLD, + {"clarification": "Which equipment do you mean?"}, + eval_metadata=_CAR_METADATA, + ) + none = evaluate_static_json( + _CAR_GOLD, + {"clarification": "Can you say more?"}, + eval_metadata=_CAR_METADATA, + ) + + assert half.mode_required_terms == ["asset", "time window"] + assert half.mode_matched_terms == ["asset"] + assert half.car_score == 0.5 + assert half.precision == half.recall == half.f1 == 0.5 + assert none.car_score == 0.0 + + +def test_car_concepts_match_any_form_on_word_boundaries(): + score = evaluate_static_json( + _CAR_GOLD, + {"clarification": "Which equipment, and over what date range?"}, + eval_metadata=_CAR_METADATA, + ) + partial_word = evaluate_static_json( + _CAR_GOLD, + {"clarification": "Which assetid?"}, + eval_metadata=_CAR_METADATA, + ) + + assert score.car_score == 1.0 + assert score.details[1].model_value == "equipment" + assert score.details[2].model_value == "date range" + assert partial_word.car_score == 0.0 + + +def test_car_concepts_read_a_structured_value_case_insensitively(): + score = evaluate_static_json( + _CAR_GOLD, + {"clarification": {"question": "Which Asset, over which Time Window?"}}, + eval_metadata=_CAR_METADATA, + ) + + assert score.car_score == 1.0 + + +def test_car_concepts_read_a_structured_value_across_newlines(): + score = evaluate_static_json( + _CAR_GOLD, + {"clarification": {"questions": ["Which asset?\ntime window?"]}}, + eval_metadata=_CAR_METADATA, + ) + + assert score.car_score == 1.0 + + +def test_car_concepts_accept_bare_string_terms(): + score = evaluate_static_json( + _CAR_GOLD, + {"clarification": "Which asset?"}, + eval_metadata={"required_terms": ["asset", "time window"]}, + ) + + assert score.mode_required_terms == ["asset", "time window"] + assert score.car_score == 0.5 + + +def test_car_without_concepts_keeps_extracted_term_scoring(): + score = evaluate_static_json( + {"clarification": "Which asset do you mean by 'the main unit'?"}, + {"clarification": "Which pump?"}, + eval_metadata={"optional_terms": []}, + ) + + assert score.car_score is None + assert score.mode_required_terms == ["main unit"] + assert score.f1 == 0.5 + + +def test_static_json_scorer_passes_car_at_half_concept_credit(): + scenario = Scenario.from_raw( + { + "id": "1", + "text": "Synthetic CAR scenario.", + "expected_answer": json.dumps(_CAR_GOLD), + "evaluation_metadata": _CAR_METADATA, + "scoring_method": "static_json", + } + ) + + scorer = StaticJsonScorer() + half = scorer(scenario, '{"clarification": "Which asset?"}', "") + none = scorer(scenario, '{"clarification": "Can you say more?"}', "") + wrong_key = scorer(scenario, '{"abstain": "Which asset and time window?"}', "") + + assert half.passed is True + assert half.score == 0.5 + assert half.details["car_score"] == 0.5 + assert none.passed is False + assert none.score == 0.0 + assert wrong_key.passed is False + assert wrong_key.score == 0.0 + + def test_identifier_with_a_separator_is_not_read_as_a_count(): """An asset id is a label, not a number. From d9268d8a88fbdb1d4cec6d7bf666f4f4760e4b9c Mon Sep 17 00:00:00 2001 From: Shuxin Lin Date: Wed, 14 Oct 2026 15:44:04 -0400 Subject: [PATCH 2/3] fix(evaluation): match regular plurals of CAR concept forms A groundtruth_eval.json form now also matches its regular plural: an added s, es after s/x/z/ch/sh, or ies for a consonant + y. Before, "Which assets?" missed the form "asset". Plurals apply to the last word, so "failure type" matches "failure types". Terms extracted from the gold value, used when a scenario has no groundtruth_eval.json, still match exactly. Signed-off-by: Shuxin Lin --- docs/static-json-evaluation.md | 6 ++-- src/evaluation/scorers/static_json.py | 27 +++++++++++++++-- .../tests/test_static_json_scorer.py | 30 +++++++++++++++++++ 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index 9f20a467..b89c4f3f 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -261,8 +261,10 @@ a credit in [0, 1]: 1. If the answer is not a JSON object with exactly one top-level key, or that key is not the expected mode, the credit is 0. 2. Otherwise the credit is the fraction of required concepts covered. A - concept is covered when any one of its `any_of` forms appears in the value - under the key. + concept is covered when any one of its `any_of` forms, or that form's + regular plural (`asset` → `assets`, `switch` → `switches`, `battery` → + `batteries`), appears in the value under the key. Irregular forms such as + `criteria` / `criterion` still need listing. `precision`, `recall`, `f1` and `car_score` all report this credit: with one matching key the model side holds exactly the gold entries, so precision diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index a26d8d32..5a9c8120 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -555,6 +555,21 @@ def _extract_required_mode_terms(value: Any) -> list[str]: return _dedupe_terms(terms) +def _plural_form(form: str) -> str: + """Regular English plural of a form's last word: asset → assets.""" + if form.endswith(("s", "x", "z", "ch", "sh")): + return f"{form}es" + if len(form) > 1 and form.endswith("y") and form[-2] not in "aeiou": + return f"{form[:-1]}ies" + return f"{form}s" + + +def _contains_form(text: str, form: str, *, plural: bool) -> bool: + """Whether normalised ``text`` holds ``form``, or its plural, as whole words.""" + padded = f" {text} " + return f" {form} " in padded or (plural and f" {_plural_form(form)} " in padded) + + def _metadata_concepts( eval_metadata: dict[str, Any] | None, ) -> list[tuple[str, list[str]]] | None: @@ -605,7 +620,8 @@ def _evaluate_mode_json( With groundtruth_eval.json concepts the answer earns nothing unless it is a single-key object whose key is the gold mode; its credit is then the fraction of required concepts covered, a concept being covered when any - one of its forms appears in the value. Precision, recall and f1 all + one of its forms, or that form's plural, appears in the value. Precision, + recall and f1 all equal that credit, since a single matching key leaves the model side with exactly the gold entries. Without concepts, terms are extracted from the gold value and the score is the F1 over the mode key plus those terms. @@ -630,7 +646,14 @@ def _evaluate_mode_json( required_terms = [name for name, _ in concepts] model_text = _normalize_text_for_terms(model_value) matched_forms = [ - next((form for form in forms if f" {form} " in f" {model_text} "), None) + next( + ( + form + for form in forms + if _contains_form(model_text, form, plural=gated) + ), + None, + ) for _, forms in concepts ] matched_terms = [ diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index 65381884..b34ccf68 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -607,6 +607,36 @@ def test_car_concepts_match_any_form_on_word_boundaries(): assert partial_word.car_score == 0.0 +@pytest.mark.parametrize( + ("form", "text"), + [ + ("asset", "Which assets?"), + ("time window", "Which time windows?"), + ("switch", "Which switches?"), + ("battery", "Which batteries?"), + ("relay", "Which relays?"), + ], +) +def test_car_concepts_match_the_plural_of_a_form(form, text): + score = evaluate_static_json( + _CAR_GOLD, + {"clarification": text}, + eval_metadata={"required_terms": [{"concept": form, "any_of": [form]}]}, + ) + + assert score.car_score == 1.0 + assert score.details[1].model_value == form + + +def test_car_extracted_terms_do_not_match_plurals(): + score = evaluate_static_json( + {"clarification": "Which asset do you mean by 'the main unit'?"}, + {"clarification": "Which of the main units?"}, + ) + + assert score.mode_matched_terms == [] + + def test_car_concepts_read_a_structured_value_case_insensitively(): score = evaluate_static_json( _CAR_GOLD, From 767a8b3f9b03409b5d8bff4698ca5493115ed8e2 Mon Sep 17 00:00:00 2001 From: Shuxin Lin Date: Wed, 14 Oct 2026 15:52:41 -0400 Subject: [PATCH 3/3] refactor(evaluation): score CAR concept answers on recall alone A CAR answer scored on groundtruth_eval.json concepts now reports recall as its score and pass criterion (recall >= 0.5). Precision and f1 are None: a free-text value gives no count of what the model asserted, so precision could only repeat recall and the f1 built from them was recall under another name. car_score duplicated recall and is removed, along with its report aggregate. Signed-off-by: Shuxin Lin --- docs/static-json-evaluation.md | 29 +++++++------- src/evaluation/report.py | 2 - src/evaluation/scorers/static_json.py | 40 ++++++++++--------- .../tests/test_static_json_scorer.py | 38 +++++++++--------- 4 files changed, 56 insertions(+), 53 deletions(-) diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index b89c4f3f..368ea7aa 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -238,7 +238,7 @@ object. | `extra_keys` | Keys present in model output but not in gold | | `details` | Per-key comparison records | -The scorer marks a scenario as passed only when the structured answer is a strict exact match. Partial correctness is still available through `score`, `f1`, `partial_exact_match_accuracy`, and detailed key-level results. +The scorer marks a scenario as passed only when the structured answer is a strict exact match, and reports `f1` as `score`. Partial correctness is still available through `score`, `f1`, `partial_exact_match_accuracy`, and detailed key-level results. CAR scenarios with `groundtruth_eval.json` are the exception: see below. ### CAR metrics and pass criteria @@ -256,27 +256,27 @@ Matching is case-insensitive on normalised text and on word boundaries, so #### With `groundtruth_eval.json` When the scenario's `groundtruth_eval.json` has `required_terms`, the score is -a credit in [0, 1]: +`recall`, in [0, 1]: 1. If the answer is not a JSON object with exactly one top-level key, or that - key is not the expected mode, the credit is 0. -2. Otherwise the credit is the fraction of required concepts covered. A + key is not the expected mode, recall is 0. +2. Otherwise recall is the fraction of required concepts covered. A concept is covered when any one of its `any_of` forms, or that form's regular plural (`asset` → `assets`, `switch` → `switches`, `battery` → `batteries`), appears in the value under the key. Irregular forms such as `criteria` / `criterion` still need listing. -`precision`, `recall`, `f1` and `car_score` all report this credit: with one -matching key the model side holds exactly the gold entries, so precision -equals recall. The scenario passes when the credit is at least 0.5 -(`CAR_PASS_THRESHOLD`). For the example above (two concepts): +The scenario passes when recall is at least 0.5 (`CAR_PASS_THRESHOLD`). +`precision` and `f1` are `null`: precision needs a count of what the model +asserted, and a free-text value gives none, so it could only repeat recall. +For the example above (two concepts): -| Model answer | Passed | Score | -| ------------------------------------------------------------- | ------ | ----- | -| `{"clarification": "Which equipment, over what date range?"}` | yes | 1.0 | -| `{"clarification": "Which asset?"}` | yes | 0.5 | -| `{"clarification": "Can you say more?"}` | no | 0.0 | -| `{"abstain": "Which asset and time window?"}` | no | 0.0 | +| Model answer | Passed | Score (recall) | +| ------------------------------------------------------------- | ------ | -------------- | +| `{"clarification": "Which equipment, over what date range?"}` | yes | 1.0 | +| `{"clarification": "Which assets?"}` | yes | 0.5 | +| `{"clarification": "Can you say more?"}` | no | 0.0 | +| `{"abstain": "Which asset and time window?"}` | no | 0.0 | #### Without `groundtruth_eval.json` @@ -304,7 +304,6 @@ Additional CAR report fields: | `mode_required_terms` | Required concepts from `groundtruth_eval.json`, or extracted terms | | `mode_matched_terms` | Required concepts or terms found in the model answer value | | `mode_term_coverage` | Fraction of required concepts or terms found, regardless of the key | -| `car_score` | The gated credit; set only when `groundtruth_eval.json` is used | --- diff --git a/src/evaluation/report.py b/src/evaluation/report.py index 0be6769d..31bbd3ed 100644 --- a/src/evaluation/report.py +++ b/src/evaluation/report.py @@ -58,7 +58,6 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]: "mode_exactly_one_key", "mode_term_coverage", "mode_optional_term_coverage", - "car_score", ] score_values: dict[str, list[float]] = {name: [] for name in metric_names} @@ -151,7 +150,6 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]: "mode_optional_term_coverage_avg": _avg( score_values["mode_optional_term_coverage"] ), - "car_score_avg": _avg(score_values["car_score"]), "missing_keys_total": missing_keys_total, "extra_keys_total": extra_keys_total, "detail_entries_total": detail_entries_total, diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index 5a9c8120..228df2af 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -56,9 +56,11 @@ class StaticJsonScore: partial_numeric_match_accuracy: float range_match_accuracy: float delta_1_match_accuracy: float - precision: float + # None for a CAR answer scored on groundtruth_eval.json concepts, which + # is scored on recall alone. + precision: float | None recall: float - f1: float + f1: float | None total_gold_keys: int total_model_keys: int matched_keys: int @@ -78,7 +80,6 @@ class StaticJsonScore: mode_required_terms: list[str] = field(default_factory=list) mode_matched_terms: list[str] = field(default_factory=list) mode_term_coverage: float | None = None - car_score: float | None = None def to_dict(self) -> dict[str, Any]: """Return a JSON-serializable dictionary.""" @@ -428,7 +429,7 @@ def similarity_score(gold_value: str, model_value: str) -> float: _MODE_KEYS = frozenset({"response", "clarification", "abstain"}) # A CAR answer scored against groundtruth_eval.json concepts passes at this -# credit. +# recall. CAR_PASS_THRESHOLD = 0.5 _IMPORTANT_MODE_TERMS = ( "air conditioner", @@ -617,14 +618,14 @@ def _evaluate_mode_json( ) -> StaticJsonScore: """Score a CAR answer: the mode key, then the terms under it. - With groundtruth_eval.json concepts the answer earns nothing unless it is - a single-key object whose key is the gold mode; its credit is then the - fraction of required concepts covered, a concept being covered when any - one of its forms, or that form's plural, appears in the value. Precision, - recall and f1 all - equal that credit, since a single matching key leaves the model side with - exactly the gold entries. Without concepts, terms are extracted from the - gold value and the score is the F1 over the mode key plus those terms. + With groundtruth_eval.json concepts the score is recall: 0 unless the + answer is a single-key object whose key is the gold mode, and otherwise + the fraction of required concepts covered, a concept being covered when + any one of its forms, or that form's plural, appears in the value. + Precision and f1 are None, since free text gives no count of the model's + claims to take precision over. Without concepts, terms are extracted + from the gold value and the score is the F1 over the mode key plus those + terms. """ gold = parse_structured_answer(gold_answer) model = parse_structured_answer(model_answer) @@ -705,7 +706,8 @@ def _evaluate_mode_json( exact_matches = (1 if key_match else 0) + len(matched_terms) if gated: - precision = recall = f1 = term_coverage if key_match else 0.0 + precision = f1 = None + recall = term_coverage if key_match else 0.0 else: precision = exact_matches / total_model_keys if total_model_keys else 0.0 recall = exact_matches / total_gold_keys if total_gold_keys else 0.0 @@ -747,7 +749,6 @@ def _evaluate_mode_json( mode_required_terms=required_terms, mode_matched_terms=matched_terms, mode_term_coverage=term_coverage, - car_score=f1 if gated else None, ) @@ -1171,13 +1172,16 @@ def __call__( static_score = evaluate_static_json( gold_answer, answer, eval_metadata=scenario.evaluation_metadata ) - if static_score.car_score is not None: - passed = static_score.car_score >= CAR_PASS_THRESHOLD + if static_score.f1 is None: + # CAR scored on groundtruth_eval.json concepts: recall only. + score = static_score.recall + passed = score >= CAR_PASS_THRESHOLD rationale = ( - f"CAR credit {static_score.car_score:.3f} " + f"CAR recall {score:.3f} " f"{'>=' if passed else '<'} {CAR_PASS_THRESHOLD}" ) else: + score = static_score.f1 passed = static_score.strict_exact_match_accuracy == 1.0 rationale = ( "strict structured match" @@ -1188,7 +1192,7 @@ def __call__( return ScorerResult( scorer=self.name, passed=passed, - score=round(static_score.f1, 3), + score=round(score, 3), rationale=rationale, details=static_score.to_dict(), ) diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index b34ccf68..5accea7d 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -546,7 +546,7 @@ def test_static_json_scorer_scores_car_on_mode_key_and_required_terms(): } -def test_car_concepts_give_zero_credit_on_a_wrong_key(): +def test_car_concepts_give_zero_recall_on_a_wrong_key(): score = evaluate_static_json( _CAR_GOLD, {"abstain": "Which asset and which time window?"}, @@ -555,11 +555,10 @@ def test_car_concepts_give_zero_credit_on_a_wrong_key(): assert score.mode_key_match == 0.0 assert score.mode_term_coverage == 1.0 - assert score.car_score == 0.0 - assert score.f1 == 0.0 + assert score.recall == 0.0 -def test_car_concepts_give_zero_credit_to_a_multi_key_answer(): +def test_car_concepts_give_zero_recall_to_a_multi_key_answer(): score = evaluate_static_json( _CAR_GOLD, {"clarification": "Which asset, which time window?", "abstain": "n/a"}, @@ -567,10 +566,10 @@ def test_car_concepts_give_zero_credit_to_a_multi_key_answer(): ) assert score.mode_exactly_one_key == 0.0 - assert score.car_score == 0.0 + assert score.recall == 0.0 -def test_car_concepts_credit_is_the_covered_fraction(): +def test_car_concepts_recall_is_the_covered_fraction(): half = evaluate_static_json( _CAR_GOLD, {"clarification": "Which equipment do you mean?"}, @@ -584,9 +583,10 @@ def test_car_concepts_credit_is_the_covered_fraction(): assert half.mode_required_terms == ["asset", "time window"] assert half.mode_matched_terms == ["asset"] - assert half.car_score == 0.5 - assert half.precision == half.recall == half.f1 == 0.5 - assert none.car_score == 0.0 + assert half.recall == 0.5 + assert half.precision is None + assert half.f1 is None + assert none.recall == 0.0 def test_car_concepts_match_any_form_on_word_boundaries(): @@ -601,10 +601,10 @@ def test_car_concepts_match_any_form_on_word_boundaries(): eval_metadata=_CAR_METADATA, ) - assert score.car_score == 1.0 + assert score.recall == 1.0 assert score.details[1].model_value == "equipment" assert score.details[2].model_value == "date range" - assert partial_word.car_score == 0.0 + assert partial_word.recall == 0.0 @pytest.mark.parametrize( @@ -624,7 +624,7 @@ def test_car_concepts_match_the_plural_of_a_form(form, text): eval_metadata={"required_terms": [{"concept": form, "any_of": [form]}]}, ) - assert score.car_score == 1.0 + assert score.recall == 1.0 assert score.details[1].model_value == form @@ -644,7 +644,7 @@ def test_car_concepts_read_a_structured_value_case_insensitively(): eval_metadata=_CAR_METADATA, ) - assert score.car_score == 1.0 + assert score.recall == 1.0 def test_car_concepts_read_a_structured_value_across_newlines(): @@ -654,7 +654,7 @@ def test_car_concepts_read_a_structured_value_across_newlines(): eval_metadata=_CAR_METADATA, ) - assert score.car_score == 1.0 + assert score.recall == 1.0 def test_car_concepts_accept_bare_string_terms(): @@ -665,7 +665,7 @@ def test_car_concepts_accept_bare_string_terms(): ) assert score.mode_required_terms == ["asset", "time window"] - assert score.car_score == 0.5 + assert score.recall == 0.5 def test_car_without_concepts_keeps_extracted_term_scoring(): @@ -675,12 +675,12 @@ def test_car_without_concepts_keeps_extracted_term_scoring(): eval_metadata={"optional_terms": []}, ) - assert score.car_score is None assert score.mode_required_terms == ["main unit"] + assert score.precision == 0.5 assert score.f1 == 0.5 -def test_static_json_scorer_passes_car_at_half_concept_credit(): +def test_static_json_scorer_passes_car_at_half_recall(): scenario = Scenario.from_raw( { "id": "1", @@ -698,7 +698,9 @@ def test_static_json_scorer_passes_car_at_half_concept_credit(): assert half.passed is True assert half.score == 0.5 - assert half.details["car_score"] == 0.5 + assert half.details["recall"] == 0.5 + assert half.details["f1"] is None + assert half.rationale == "CAR recall 0.500 >= 0.5" assert none.passed is False assert none.score == 0.0 assert wrong_key.passed is False