diff --git a/datbench/datasets_scoring/cc_ocr_kie.py b/datbench/datasets_scoring/cc_ocr_kie.py index 7eb9b6d..35fbc5b 100644 --- a/datbench/datasets_scoring/cc_ocr_kie.py +++ b/datbench/datasets_scoring/cc_ocr_kie.py @@ -69,6 +69,10 @@ def _extract_json(text: str) -> Dict[str, Any]: if brace_count == 0: text = text[start_idx:i].strip() break + # Models often merge the boxed braces with the JSON braces + # (\boxed{"k": "v"}), leaving a brace-less body — restore it. + if boxed_start != -1 and text and not text.startswith('{'): + text = '{' + text + '}' # Remove markdown code blocks text = re.sub(r'```json\s*', '', text, flags=re.IGNORECASE) diff --git a/datbench/datasets_scoring/ocrbench_v2.py b/datbench/datasets_scoring/ocrbench_v2.py index 308858d..5827c77 100644 --- a/datbench/datasets_scoring/ocrbench_v2.py +++ b/datbench/datasets_scoring/ocrbench_v2.py @@ -465,6 +465,15 @@ def _kie_f1_evaluation(predict: str, answers: List[str], task_type: str) -> floa if not isinstance(pred_dict, dict): return 0.0 + # _generate_combinations unpacks GT list values to scalars; mirror it on + # the prediction side so an answer that keeps the GT's own + # {'key': ['value']} syntax is not compared as "['value']" vs "value" + # (the GT fed back as a prediction used to score 0). + pred_dict = { + k: v[0] if isinstance(v, list) and len(v) == 1 else v + for k, v in pred_dict.items() + } + # Parse ground truth try: if isinstance(answers[0], str): @@ -679,6 +688,11 @@ def _vqa_with_position_evaluation(predict: str, sample: Dict[str, Any]) -> float score_content = 0.0 score_bbox = 0.0 + # Grounding rows in exported eval sets (e.g. db-document) carry the GT + # box only as four numeric answer strings with a null bbox field — + # rebuild it so the IoU half of the score stays live. + gt_bbox = sample.get("bbox") or _bbox_from_answers(sample.get("answers", [])) + # Try to parse as structured dict with "answer" and "bbox" pred_dict = _try_parse_as_dict(predict) @@ -686,17 +700,17 @@ def _vqa_with_position_evaluation(predict: str, sample: Dict[str, Any]) -> float # Structured output score_content = _vqa_evaluation(pred_dict["answer"], sample.get("answers", [])) - if "bbox" in pred_dict and sample.get("bbox"): + if "bbox" in pred_dict and gt_bbox: try: pred_bbox = ast.literal_eval(str(pred_dict["bbox"])) - score_bbox = _calculate_iou(pred_bbox, sample["bbox"]) + score_bbox = _calculate_iou(pred_bbox, gt_bbox) except: score_bbox = 0.0 else: # Free-form text - extract coordinates coords = _extract_coordinates(predict) - if coords and sample.get("bbox"): - score_bbox = _calculate_iou(coords, sample["bbox"]) + if coords and gt_bbox: + score_bbox = _calculate_iou(coords, gt_bbox) # Score the answer content score_content = _vqa_evaluation(predict, sample.get("answers", [])) @@ -705,10 +719,188 @@ def _vqa_with_position_evaluation(predict: str, sample: Dict[str, Any]) -> float return 0.5 * score_content + 0.5 * score_bbox +def _bbox_from_answers(answers) -> Any: + """Rebuild a GT box from answers shaped like ['630', '443', '828', '496'].""" + if len(answers) != 4: + return None + try: + return [int(float(a)) for a in answers] + except (TypeError, ValueError): + return None + + def _text_spotting_evaluation(predict: str, sample: Dict[str, Any]) -> float: - """Text spotting - simplified to VQA.""" - answers = sample.get("answers", []) - return _vqa_evaluation(predict, answers) + """Evaluate text spotting with the official IoU/text h-mean semantics. + + OCRBench v2's reference script shells out through the RRC zip-file + interface. This is its single-image equivalent: axis-align the reference + polygons, ignore ``###`` regions, greedily match boxes at IoU > 0.5, and + return recognition precision/recall h-mean. + """ + pred_bboxes = _extract_bounding_boxes_robust(predict) + gt_bboxes = sample.get("bbox_list") or sample.get("bbox") + gt_content = sample.get("content") + if not pred_bboxes or not gt_bboxes or not gt_content: + return 0.0 + if len(gt_bboxes) != len(gt_content): + return 0.0 + + ground_truth = [] + for gt_bbox, gt_text in zip(gt_bboxes, gt_content): + try: + x_coords = gt_bbox[0::2] + y_coords = gt_bbox[1::2] + gt_box = [min(x_coords), min(y_coords), max(x_coords), max(y_coords)] + except (TypeError, ValueError): + continue + ground_truth.append((gt_box, str(gt_text))) + + predictions = [ + (item[:4], str(item[4])) + for item in pred_bboxes + if len(item) >= 5 and item[0] < item[2] and item[1] < item[3] + ] + if not predictions or not ground_truth: + return 0.0 + + care_ground_truth = [text != "###" for _, text in ground_truth] + care_predictions = [True] * len(predictions) + for prediction_index, (pred_box, _) in enumerate(predictions): + for (gt_box, _), is_care in zip(ground_truth, care_ground_truth): + if is_care: + continue + intersection = _intersection_area(pred_box, gt_box) + pred_area = _box_area(pred_box) + if pred_area and intersection / pred_area > 0.5: + care_predictions[prediction_index] = False + break + + matched_ground_truth = set() + matched_predictions = set() + correct = 0 + for gt_index, ((gt_box, gt_text), gt_is_care) in enumerate( + zip(ground_truth, care_ground_truth) + ): + if not gt_is_care: + continue + for pred_index, ((pred_box, pred_text), pred_is_care) in enumerate( + zip(predictions, care_predictions) + ): + if ( + not pred_is_care + or gt_index in matched_ground_truth + or pred_index in matched_predictions + ): + continue + if _calculate_iou(pred_box, gt_box) <= 0.5: + continue + + # The official evaluator locks a geometric pair even when its + # transcription is wrong, then counts only correct recognition. + matched_ground_truth.add(gt_index) + matched_predictions.add(pred_index) + if _spotting_text_matches(gt_text, pred_text): + correct += 1 + + num_gt_care = sum(care_ground_truth) + num_pred_care = sum(care_predictions) + if num_gt_care == 0: + recall = 1.0 + precision = 0.0 if num_pred_care else 1.0 + else: + recall = correct / num_gt_care + precision = correct / num_pred_care if num_pred_care else 0.0 + return ( + 0.0 + if precision + recall == 0 + else 2.0 * precision * recall / (precision + recall) + ) + + +def _box_area(box) -> float: + return max(0, box[2] - box[0]) * max(0, box[3] - box[1]) + + +def _intersection_area(box1, box2) -> float: + return max(0, min(box1[2], box2[2]) - max(box1[0], box2[0])) * max( + 0, min(box1[3], box2[3]) - max(box1[1], box2[1]) + ) + + +def _spotting_text_matches(ground_truth: str, prediction: str) -> bool: + """Match OCRBench/RRC transcriptions, ignoring case and GT edge punctuation.""" + gt = ground_truth.upper() + pred = prediction.upper() + if gt == pred: + return True + special_characters = "!?.:,*\"()·[]/'" + candidates = {gt} + if gt and gt[0] in special_characters: + candidates.add(gt[1:]) + if gt and gt[-1] in special_characters: + candidates.add(gt[:-1]) + if len(gt) > 1 and gt[0] in special_characters and gt[-1] in special_characters: + candidates.add(gt[1:-1]) + return pred in candidates + + +def _extract_bounding_boxes_robust(predict_str: str) -> List[List]: + """Extract ``[x1, y1, x2, y2, text]`` items from prediction text. + + Ported from the official spotting_metric.py extractor, plus unwrapping the + ``\\boxed{...}`` wrapper our prompts request — a boxed list is otherwise a + literal_eval syntax error and the whole answer is lost. + """ + results = [] + seen = set() + + literal_candidate = predict_str.strip() + boxed_match = re.search( + r"\\boxed\s*\{(.*)\}\s*$", literal_candidate, flags=re.DOTALL + ) + if boxed_match: + literal_candidate = boxed_match.group(1).strip() + + try: + data = ast.literal_eval(literal_candidate) + if isinstance(data, (list, tuple)): + for item in data: + if isinstance(item, (list, tuple)) and len(item) >= 5: + x1, y1, x2, y2 = item[:4] + text_content = item[4] + try: + x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2) + text_content = ( + str(text_content) + .replace("\n", "") + .strip() + .strip('"') + .strip("'") + ) + if all(0 <= n <= 1000 for n in [x1, y1, x2, y2]): + key = (x1, y1, x2, y2, text_content) + if key not in seen: + seen.add(key) + results.append([x1, y1, x2, y2, text_content]) + except (TypeError, ValueError): + continue + except (SyntaxError, TypeError, ValueError): + items = re.findall(r"[\[\(]\s*([^\[\]\(\)]*?)\s*[\]\)]", predict_str) + for item in items: + parts = item.split(",", 4) + if len(parts) >= 5: + try: + x1, y1, x2, y2 = map(int, [p.strip() for p in parts[:4]]) + text = parts[4].strip().strip('"').strip("'") + if all(0 <= n <= 1000 for n in [x1, y1, x2, y2]): + key = (x1, y1, x2, y2, text) + if key not in seen: + seen.add(key) + results.append([x1, y1, x2, y2, text]) + except (TypeError, ValueError): + continue + + return results def _full_page_ocr_evaluation(predict: str, answers: List[str]) -> float: diff --git a/tests/test_sig_scoring_fixes.py b/tests/test_sig_scoring_fixes.py new file mode 100644 index 0000000..3f82f78 --- /dev/null +++ b/tests/test_sig_scoring_fixes.py @@ -0,0 +1,122 @@ +"""Format-robustness fixes for extraction/grounding/spotting scorers. + +Four defects surfaced by rescoring the DAT-3459 arms from saved predictions: + +1. Text spotting was "simplified to VQA": the box list was string-matched + against the flattened polygon answer, which can essentially never match, + so every run scored a hard 0. Replaced with the official RRC-style + single-image metric (greedy IoU > 0.5 match + recognition h-mean). +2. KIE F1 zeroed predictions that keep the GT's own single-element list + syntax ({'key': ['value']}) — the ground truth fed back as a prediction + scored 0.0. +3. VQA-with-position/grounding rows in exported eval sets carry the GT box + only as four numeric answer strings (bbox field is null), so the IoU half + of the 0.5*content + 0.5*IoU score could never be earned. +4. CC-OCR KIE returned an empty dict when a model merged the \\boxed braces + with the JSON braces (\\boxed{"k": "v"}). +""" + +from datbench.datasets_scoring import cc_ocr_kie +from datbench.datasets_scoring import ocrbench_v2 as ob + +SPOTTING_GT = { + "bbox_list": [ + [10, 10, 110, 10, 110, 60, 10, 60], + [200, 200, 300, 200, 300, 260, 200, 260], + ], + "content": ["HELLO", "WORLD"], +} + + +def test_spotting_perfect_prediction_scores_one(): + pred = "[(10, 10, 110, 60, 'HELLO'), (200, 200, 300, 260, 'WORLD')]" + assert ob._text_spotting_evaluation(pred, SPOTTING_GT) == 1.0 + + +def test_spotting_boxed_prediction_scores_one(): + pred = "\\boxed{[(10, 10, 110, 60, 'HELLO'), (200, 200, 300, 260, 'WORLD')]}" + assert ob._text_spotting_evaluation(pred, SPOTTING_GT) == 1.0 + + +def test_spotting_fabricated_boxes_score_zero(): + pred = "[(500, 500, 600, 600, 'HELLO'), (700, 700, 800, 800, 'WORLD')]" + assert ob._text_spotting_evaluation(pred, SPOTTING_GT) == 0.0 + + +def test_spotting_wrong_text_scores_zero(): + pred = "[(10, 10, 110, 60, 'GOODBYE'), (200, 200, 300, 260, 'MOON')]" + assert ob._text_spotting_evaluation(pred, SPOTTING_GT) == 0.0 + + +def test_spotting_ignores_hashhash_regions(): + gt = {"bbox_list": SPOTTING_GT["bbox_list"], "content": ["HELLO", "###"]} + pred = "[(10, 10, 110, 60, 'HELLO')]" + assert ob._text_spotting_evaluation(pred, gt) == 1.0 + + +GT_LIST_SYNTAX = "{'CE-P1': ['1700kJ/410kcal'], 'TF-P1': ['35g'], 'PRO-P1': ['24g']}" + + +def test_kie_f1_gt_fed_back_as_prediction_scores_one(): + score = ob._kie_f1_evaluation( + GT_LIST_SYNTAX, [GT_LIST_SYNTAX], "key information extraction en" + ) + assert score == 1.0 + + +def test_kie_f1_scalar_syntax_still_scores_one(): + scalar = "{'CE-P1': '1700kJ/410kcal', 'TF-P1': '35g', 'PRO-P1': '24g'}" + score = ob._kie_f1_evaluation( + scalar, [GT_LIST_SYNTAX], "key information extraction en" + ) + assert score == 1.0 + + +def test_kie_f1_wrong_value_still_scores_zero(): + score = ob._kie_f1_evaluation( + "{'CE-P1': ['9999']}", + ["{'CE-P1': ['1700kJ/410kcal']}"], + "key information extraction en", + ) + assert score == 0.0 + + +def test_bbox_from_answers_rebuilds_box(): + assert ob._bbox_from_answers(["630", "443", "828", "496"]) == [630, 443, 828, 496] + assert ob._bbox_from_answers(["Walter Crickmer"]) is None + assert ob._bbox_from_answers(["a", "b", "c", "d"]) is None + + +def test_vqa_with_position_scores_iou_when_bbox_field_is_null(): + sample = {"bbox": None, "answers": ["630", "443", "828", "496"]} + score = ob._vqa_with_position_evaluation("(630, 443, 828, 496)", sample) + assert score >= 0.5 + + +def test_vqa_with_position_prefers_explicit_bbox_field(): + pred = "(630, 443, 828, 496)" + answers = ["630", "443", "828", "496"] + hit = ob._vqa_with_position_evaluation( + pred, {"bbox": [630, 443, 828, 496], "answers": answers} + ) + miss = ob._vqa_with_position_evaluation( + pred, {"bbox": [0, 0, 10, 10], "answers": answers} + ) + # if the answers fallback overrode the explicit bbox, both samples would + # score identically; the explicit box must decide the IoU half instead + assert hit - miss == 0.5 + + +def test_cc_ocr_extract_json_boxed_merged_braces(): + assert cc_ocr_kie._extract_json('\\boxed{"CE-PS": "110", "TF-PS": "1g"}') == { + "CE-PS": "110", + "TF-PS": "1g", + } + + +def test_cc_ocr_extract_json_boxed_with_proper_braces_unchanged(): + assert cc_ocr_kie._extract_json('\\boxed{{"CE-PS": "110"}}') == {"CE-PS": "110"} + + +def test_cc_ocr_extract_json_bare_json_unchanged(): + assert cc_ocr_kie._extract_json('{"CE-PS": "110"}') == {"CE-PS": "110"}