Fix four scoring defects surfaced by the DAT-3459 SIG rescore (spotting, KIE syntax, grounding bbox, boxed JSON)

datbench/datasets_scoring/cc_ocr_kie.py

@@ -69,6 +69,10 @@ def _extract_json(text: str) -> Dict[str, Any]:
                     if brace_count == 0:
                         text = text[start_idx:i].strip()
                         break
+            # Models often merge the boxed braces with the JSON braces
+            # (\boxed{"k": "v"}), leaving a brace-less body — restore it.
+            if boxed_start != -1 and text and not text.startswith('{'):
+                text = '{' + text + '}'
 
     # Remove markdown code blocks
     text = re.sub(r'```json\s*', '', text, flags=re.IGNORECASE)

datbench/datasets_scoring/ocrbench_v2.py

@@ -465,6 +465,15 @@ def _kie_f1_evaluation(predict: str, answers: List[str], task_type: str) -> floa
     if not isinstance(pred_dict, dict):
         return 0.0
 
+    # _generate_combinations unpacks GT list values to scalars; mirror it on
+    # the prediction side so an answer that keeps the GT's own
+    # {'key': ['value']} syntax is not compared as "['value']" vs "value"
+    # (the GT fed back as a prediction used to score 0).
+    pred_dict = {
+        k: v[0] if isinstance(v, list) and len(v) == 1 else v
+        for k, v in pred_dict.items()
+    }
+
     # Parse ground truth
     try:
         if isinstance(answers[0], str):
@@ -679,24 +688,29 @@ def _vqa_with_position_evaluation(predict: str, sample: Dict[str, Any]) -> float
     score_content = 0.0
     score_bbox = 0.0
 
+    # Grounding rows in exported eval sets (e.g. db-document) carry the GT
+    # box only as four numeric answer strings with a null bbox field —
+    # rebuild it so the IoU half of the score stays live.
+    gt_bbox = sample.get("bbox") or _bbox_from_answers(sample.get("answers", []))
+
     # Try to parse as structured dict with "answer" and "bbox"
     pred_dict = _try_parse_as_dict(predict)
 
     if isinstance(pred_dict, dict) and "answer" in pred_dict:
         # Structured output
         score_content = _vqa_evaluation(pred_dict["answer"], sample.get("answers", []))
 
-        if "bbox" in pred_dict and sample.get("bbox"):
+        if "bbox" in pred_dict and gt_bbox:
             try:
                 pred_bbox = ast.literal_eval(str(pred_dict["bbox"]))
-                score_bbox = _calculate_iou(pred_bbox, sample["bbox"])
+                score_bbox = _calculate_iou(pred_bbox, gt_bbox)
             except:
                 score_bbox = 0.0
     else:
         # Free-form text - extract coordinates
         coords = _extract_coordinates(predict)
-        if coords and sample.get("bbox"):
-            score_bbox = _calculate_iou(coords, sample["bbox"])
+        if coords and gt_bbox:
+            score_bbox = _calculate_iou(coords, gt_bbox)
 
         # Score the answer content
         score_content = _vqa_evaluation(predict, sample.get("answers", []))
@@ -705,10 +719,188 @@ def _vqa_with_position_evaluation(predict: str, sample: Dict[str, Any]) -> float
     return 0.5 * score_content + 0.5 * score_bbox
 
 
+def _bbox_from_answers(answers) -> Any:
+    """Rebuild a GT box from answers shaped like ['630', '443', '828', '496']."""
+    if len(answers) != 4:
+        return None
+    try:
+        return [int(float(a)) for a in answers]
+    except (TypeError, ValueError):
+        return None
+
+
 def _text_spotting_evaluation(predict: str, sample: Dict[str, Any]) -> float:
-    """Text spotting - simplified to VQA."""
-    answers = sample.get("answers", [])
-    return _vqa_evaluation(predict, answers)
+    """Evaluate text spotting with the official IoU/text h-mean semantics.
+
+    OCRBench v2's reference script shells out through the RRC zip-file
+    interface. This is its single-image equivalent: axis-align the reference
+    polygons, ignore ``###`` regions, greedily match boxes at IoU > 0.5, and
+    return recognition precision/recall h-mean.
+    """
+    pred_bboxes = _extract_bounding_boxes_robust(predict)
+    gt_bboxes = sample.get("bbox_list") or sample.get("bbox")
+    gt_content = sample.get("content")
+    if not pred_bboxes or not gt_bboxes or not gt_content:
+        return 0.0
+    if len(gt_bboxes) != len(gt_content):
+        return 0.0
+
+    ground_truth = []
+    for gt_bbox, gt_text in zip(gt_bboxes, gt_content):
+        try:
+            x_coords = gt_bbox[0::2]
+            y_coords = gt_bbox[1::2]
+            gt_box = [min(x_coords), min(y_coords), max(x_coords), max(y_coords)]
+        except (TypeError, ValueError):
+            continue
+        ground_truth.append((gt_box, str(gt_text)))
+
+    predictions = [
+        (item[:4], str(item[4]))
+        for item in pred_bboxes
+        if len(item) >= 5 and item[0] < item[2] and item[1] < item[3]
+    ]
+    if not predictions or not ground_truth:
+        return 0.0
+
+    care_ground_truth = [text != "###" for _, text in ground_truth]
+    care_predictions = [True] * len(predictions)
+    for prediction_index, (pred_box, _) in enumerate(predictions):
+        for (gt_box, _), is_care in zip(ground_truth, care_ground_truth):
+            if is_care:
+                continue
+            intersection = _intersection_area(pred_box, gt_box)
+            pred_area = _box_area(pred_box)
+            if pred_area and intersection / pred_area > 0.5:
+                care_predictions[prediction_index] = False
+                break
+
+    matched_ground_truth = set()
+    matched_predictions = set()
+    correct = 0
+    for gt_index, ((gt_box, gt_text), gt_is_care) in enumerate(
+        zip(ground_truth, care_ground_truth)
+    ):
+        if not gt_is_care:
+            continue
+        for pred_index, ((pred_box, pred_text), pred_is_care) in enumerate(
+            zip(predictions, care_predictions)
+        ):
+            if (
+                not pred_is_care
+                or gt_index in matched_ground_truth
+                or pred_index in matched_predictions
+            ):
+                continue
+            if _calculate_iou(pred_box, gt_box) <= 0.5:
+                continue
+
+            # The official evaluator locks a geometric pair even when its
+            # transcription is wrong, then counts only correct recognition.
+            matched_ground_truth.add(gt_index)
+            matched_predictions.add(pred_index)
+            if _spotting_text_matches(gt_text, pred_text):
+                correct += 1
+
+    num_gt_care = sum(care_ground_truth)
+    num_pred_care = sum(care_predictions)
+    if num_gt_care == 0:
+        recall = 1.0
+        precision = 0.0 if num_pred_care else 1.0
+    else:
+        recall = correct / num_gt_care
+        precision = correct / num_pred_care if num_pred_care else 0.0
+    return (
+        0.0
+        if precision + recall == 0
+        else 2.0 * precision * recall / (precision + recall)
+    )
+
+
+def _box_area(box) -> float:
+    return max(0, box[2] - box[0]) * max(0, box[3] - box[1])
+
+
+def _intersection_area(box1, box2) -> float:
+    return max(0, min(box1[2], box2[2]) - max(box1[0], box2[0])) * max(
+        0, min(box1[3], box2[3]) - max(box1[1], box2[1])
+    )
+
+
+def _spotting_text_matches(ground_truth: str, prediction: str) -> bool:
+    """Match OCRBench/RRC transcriptions, ignoring case and GT edge punctuation."""
+    gt = ground_truth.upper()
+    pred = prediction.upper()
+    if gt == pred:
+        return True
+    special_characters = "!?.:,*\"()·[]/'"
+    candidates = {gt}
+    if gt and gt[0] in special_characters:
+        candidates.add(gt[1:])
+    if gt and gt[-1] in special_characters:
+        candidates.add(gt[:-1])
+    if len(gt) > 1 and gt[0] in special_characters and gt[-1] in special_characters:
+        candidates.add(gt[1:-1])
+    return pred in candidates
+
+
+def _extract_bounding_boxes_robust(predict_str: str) -> List[List]:
+    """Extract ``[x1, y1, x2, y2, text]`` items from prediction text.
+
+    Ported from the official spotting_metric.py extractor, plus unwrapping the
+    ``\\boxed{...}`` wrapper our prompts request — a boxed list is otherwise a
+    literal_eval syntax error and the whole answer is lost.
+    """
+    results = []
+    seen = set()
+
+    literal_candidate = predict_str.strip()
+    boxed_match = re.search(
+        r"\\boxed\s*\{(.*)\}\s*$", literal_candidate, flags=re.DOTALL
+    )
+    if boxed_match:
+        literal_candidate = boxed_match.group(1).strip()
+
+    try:
+        data = ast.literal_eval(literal_candidate)
+        if isinstance(data, (list, tuple)):
+            for item in data:
+                if isinstance(item, (list, tuple)) and len(item) >= 5:
+                    x1, y1, x2, y2 = item[:4]
+                    text_content = item[4]
+                    try:
+                        x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
+                        text_content = (
+                            str(text_content)
+                            .replace("\n", "")
+                            .strip()
+                            .strip('"')
+                            .strip("'")
+                        )
+                        if all(0 <= n <= 1000 for n in [x1, y1, x2, y2]):
+                            key = (x1, y1, x2, y2, text_content)
+                            if key not in seen:
+                                seen.add(key)
+                                results.append([x1, y1, x2, y2, text_content])
+                    except (TypeError, ValueError):
+                        continue
+    except (SyntaxError, TypeError, ValueError):
+        items = re.findall(r"[\[\(]\s*([^\[\]\(\)]*?)\s*[\]\)]", predict_str)
+        for item in items:
+            parts = item.split(",", 4)
+            if len(parts) >= 5:
+                try:
+                    x1, y1, x2, y2 = map(int, [p.strip() for p in parts[:4]])
+                    text = parts[4].strip().strip('"').strip("'")
+                    if all(0 <= n <= 1000 for n in [x1, y1, x2, y2]):
+                        key = (x1, y1, x2, y2, text)
+                        if key not in seen:
+                            seen.add(key)
+                            results.append([x1, y1, x2, y2, text])
+                except (TypeError, ValueError):
+                    continue
+
+    return results
 
 
 def _full_page_ocr_evaluation(predict: str, answers: List[str]) -> float:

tests/test_sig_scoring_fixes.py

@@ -0,0 +1,122 @@
+"""Format-robustness fixes for extraction/grounding/spotting scorers.
+
+Four defects surfaced by rescoring the DAT-3459 arms from saved predictions:
+
+1. Text spotting was "simplified to VQA": the box list was string-matched
+   against the flattened polygon answer, which can essentially never match,
+   so every run scored a hard 0. Replaced with the official RRC-style
+   single-image metric (greedy IoU > 0.5 match + recognition h-mean).
+2. KIE F1 zeroed predictions that keep the GT's own single-element list
+   syntax ({'key': ['value']}) — the ground truth fed back as a prediction
+   scored 0.0.
+3. VQA-with-position/grounding rows in exported eval sets carry the GT box
+   only as four numeric answer strings (bbox field is null), so the IoU half
+   of the 0.5*content + 0.5*IoU score could never be earned.
+4. CC-OCR KIE returned an empty dict when a model merged the \\boxed braces
+   with the JSON braces (\\boxed{"k": "v"}).
+"""
+
+from datbench.datasets_scoring import cc_ocr_kie
+from datbench.datasets_scoring import ocrbench_v2 as ob
+
+SPOTTING_GT = {
+    "bbox_list": [
+        [10, 10, 110, 10, 110, 60, 10, 60],
+        [200, 200, 300, 200, 300, 260, 200, 260],
+    ],
+    "content": ["HELLO", "WORLD"],
+}
+
+
+def test_spotting_perfect_prediction_scores_one():
+    pred = "[(10, 10, 110, 60, 'HELLO'), (200, 200, 300, 260, 'WORLD')]"
+    assert ob._text_spotting_evaluation(pred, SPOTTING_GT) == 1.0
+
+
+def test_spotting_boxed_prediction_scores_one():
+    pred = "\\boxed{[(10, 10, 110, 60, 'HELLO'), (200, 200, 300, 260, 'WORLD')]}"
+    assert ob._text_spotting_evaluation(pred, SPOTTING_GT) == 1.0
+
+
+def test_spotting_fabricated_boxes_score_zero():
+    pred = "[(500, 500, 600, 600, 'HELLO'), (700, 700, 800, 800, 'WORLD')]"
+    assert ob._text_spotting_evaluation(pred, SPOTTING_GT) == 0.0
+
+
+def test_spotting_wrong_text_scores_zero():
+    pred = "[(10, 10, 110, 60, 'GOODBYE'), (200, 200, 300, 260, 'MOON')]"
+    assert ob._text_spotting_evaluation(pred, SPOTTING_GT) == 0.0
+
+
+def test_spotting_ignores_hashhash_regions():
+    gt = {"bbox_list": SPOTTING_GT["bbox_list"], "content": ["HELLO", "###"]}
+    pred = "[(10, 10, 110, 60, 'HELLO')]"
+    assert ob._text_spotting_evaluation(pred, gt) == 1.0
+
+
+GT_LIST_SYNTAX = "{'CE-P1': ['1700kJ/410kcal'], 'TF-P1': ['35g'], 'PRO-P1': ['24g']}"
+
+
+def test_kie_f1_gt_fed_back_as_prediction_scores_one():
+    score = ob._kie_f1_evaluation(
+        GT_LIST_SYNTAX, [GT_LIST_SYNTAX], "key information extraction en"
+    )
+    assert score == 1.0
+
+
+def test_kie_f1_scalar_syntax_still_scores_one():
+    scalar = "{'CE-P1': '1700kJ/410kcal', 'TF-P1': '35g', 'PRO-P1': '24g'}"
+    score = ob._kie_f1_evaluation(
+        scalar, [GT_LIST_SYNTAX], "key information extraction en"
+    )
+    assert score == 1.0
+
+
+def test_kie_f1_wrong_value_still_scores_zero():
+    score = ob._kie_f1_evaluation(
+        "{'CE-P1': ['9999']}",
+        ["{'CE-P1': ['1700kJ/410kcal']}"],
+        "key information extraction en",
+    )
+    assert score == 0.0
+
+
+def test_bbox_from_answers_rebuilds_box():
+    assert ob._bbox_from_answers(["630", "443", "828", "496"]) == [630, 443, 828, 496]
+    assert ob._bbox_from_answers(["Walter Crickmer"]) is None
+    assert ob._bbox_from_answers(["a", "b", "c", "d"]) is None
+
+
+def test_vqa_with_position_scores_iou_when_bbox_field_is_null():
+    sample = {"bbox": None, "answers": ["630", "443", "828", "496"]}
+    score = ob._vqa_with_position_evaluation("(630, 443, 828, 496)", sample)
+    assert score >= 0.5
+
+
+def test_vqa_with_position_prefers_explicit_bbox_field():
+    pred = "(630, 443, 828, 496)"
+    answers = ["630", "443", "828", "496"]
+    hit = ob._vqa_with_position_evaluation(
+        pred, {"bbox": [630, 443, 828, 496], "answers": answers}
+    )
+    miss = ob._vqa_with_position_evaluation(
+        pred, {"bbox": [0, 0, 10, 10], "answers": answers}
+    )
+    # if the answers fallback overrode the explicit bbox, both samples would
+    # score identically; the explicit box must decide the IoU half instead
+    assert hit - miss == 0.5
+
+
+def test_cc_ocr_extract_json_boxed_merged_braces():
+    assert cc_ocr_kie._extract_json('\\boxed{"CE-PS": "110", "TF-PS": "1g"}') == {
+        "CE-PS": "110",
+        "TF-PS": "1g",
+    }
+
+
+def test_cc_ocr_extract_json_boxed_with_proper_braces_unchanged():
+    assert cc_ocr_kie._extract_json('\\boxed{{"CE-PS": "110"}}') == {"CE-PS": "110"}
+
+
+def test_cc_ocr_extract_json_bare_json_unchanged():
+    assert cc_ocr_kie._extract_json('{"CE-PS": "110"}') == {"CE-PS": "110"}