diff --git a/dingo/io/output/result_info.py b/dingo/io/output/result_info.py index 2935efe2..8a03a4ee 100644 --- a/dingo/io/output/result_info.py +++ b/dingo/io/output/result_info.py @@ -62,7 +62,7 @@ def _parse_container_string(cls, value: str) -> Any: ): return value try: - parsed = json.loads(text) + parsed = json.loads(text, strict=False) except json.JSONDecodeError: try: parsed = ast.literal_eval(text) diff --git a/dingo/model/rule/scibase/rule_patent.py b/dingo/model/rule/scibase/rule_patent.py new file mode 100644 index 00000000..7056c81b --- /dev/null +++ b/dingo/model/rule/scibase/rule_patent.py @@ -0,0 +1,128 @@ +import json +import re +from typing import Any, List + +from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io.input import Data +from dingo.io.output.eval_detail import EvalDetail, QualityLabel +from dingo.model.model import Model +from dingo.model.rule.base import BaseRule + +MAX_CLAIMS_LENGTH = 1_000_000 +IPC_CODE_RE = re.compile( + r"^(?P
[A-H])" + r"(?P0[1-9]|[1-9][0-9])" + r"(?P[A-Z]) " + r"(?P[1-9][0-9]{0,2})/" + r"(?P[0-9]{2,6})$" +) +ValidationResult = tuple[bool, List[str], List[str]] + + +def check_claims(claims: Any) -> ValidationResult: + claims_length = len(str(claims)) + if claims_length > MAX_CLAIMS_LENGTH: + return ( + True, + ["too_long"], + [ + f"string length {claims_length} exceeds the maximum " + f"of {MAX_CLAIMS_LENGTH}" + ], + ) + return False, [], [] + + +def check_ipc_unified(ipc_unified: Any) -> ValidationResult: + if ipc_unified is None: + return True, ["null"], ["value is null"] + + if isinstance(ipc_unified, str): + try: + ipc_unified = json.loads(ipc_unified, strict=False) + except json.JSONDecodeError: + return True, ["invalid_json"], ["value must be a JSON array"] + + if not isinstance(ipc_unified, list): + return True, ["wrong_type"], ["value must be a list"] + + for index, ipc_code in enumerate(ipc_unified): + if not isinstance(ipc_code, str): + return ( + True, + ["wrong_type"], + [f"item[{index}] must be a string"], + ) + + match = IPC_CODE_RE.fullmatch(ipc_code) + if match is None: + return ( + True, + ["invalid_format"], + [ + f"item[{index}] must match IPC format " + "'<01-99> <1-999>/<2-6 digits>'" + ], + ) + + subgroup = match.group("subgroup") + if subgroup != "00" and int(subgroup) == 0: + return ( + True, + ["invalid_format"], + [f"item[{index}] subgroup may be zero only when written as '00'"], + ) + + return False, [], [] + + +FIELD_VALIDATORS = { + "claims": lambda record: check_claims(record.get("claims")), + "ipc_unified": lambda record: check_ipc_unified(record.get("ipc_unified")), +} + + +@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", ["xinghe", "quanliang"]) +class RulePatentFieldValidation(BaseRule): + _metric_info = { + "category": "Rule-Based Metadata Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "RulePatentFieldValidation", + "description": "Validate patent metadata fields and report invalid fields", + "paper_title": "", + "paper_url": "", + "paper_authors": "", + "evaluation_results": "", + } + + _required_fields = [] + dynamic_config = EvaluatorRuleArgs(key_list=list(FIELD_VALIDATORS.keys())) + + def eval(self, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=self.__class__.__name__) + record = input_data.to_dict() + selected_fields = self.dynamic_config.key_list or [] + bad_fields: List[str] = [] + reasons: List[str] = [] + + for field in selected_fields: + if field not in FIELD_VALIDATORS: + bad_fields.append(f"{field}.unsupported_field") + reasons.append(f"{field}: unsupported field") + continue + if field not in record: + bad_fields.append(f"{field}.missing_field") + reasons.append(f"{field}: missing field") + continue + invalid, error_labels, detail_reasons = FIELD_VALIDATORS[field](record) + if invalid: + bad_fields.extend(f"{field}.{error_label}" for error_label in error_labels) + reasons.extend(f"{field}: {reason}" for reason in detail_reasons) + + if bad_fields: + res.status = True + res.label = bad_fields + res.reason = reasons + else: + res.label = [QualityLabel.QUALITY_GOOD] + return res diff --git a/docs/metrics.md b/docs/metrics.md index 35719a42..dfa304be 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -156,7 +156,7 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleAuthorFieldValidation, RuleQuanliangFieldValidation, RuleSourceFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate OpenAlex author fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleAuthorFieldValidation, RulePatentFieldValidation, RuleQuanliangFieldValidation, RuleSourceFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate OpenAlex author fields and report invalid fields; Validate patent metadata fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A | ### Rule-Based RESUME Quality Metrics diff --git a/docs/rules.md b/docs/rules.md index 8bd72f6d..0b40a64f 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -64,6 +64,7 @@ The specific rules for each quality metric are as follows: | RuleOnlyUrl | EFFECTIVENESS | Check whether content consists only of a URL. | | | RulePatternSearch | RELEVANCE | Search content using a user-provided pattern. | | | RulePIIDetection | SECURITY | Detect personally identifiable information in text. | | +| RulePatentFieldValidation | EFFECTIVENESS | Validate patent claims length and IPC classification-code format. | | | RuleQuanliangFieldValidation | EFFECTIVENESS | Validate full-volume scientific metadata fields. | | | RuleResumeDateFormat | RESUME_DATE | Check whether a resume uses inconsistent date formats. | | | RuleResumeDetailedAddress | RESUME_PRIVACY | Check whether a resume contains a detailed address. | | diff --git a/test/scripts/io/output/test_result_info.py b/test/scripts/io/output/test_result_info.py index 9530c54c..c0345053 100644 --- a/test/scripts/io/output/test_result_info.py +++ b/test/scripts/io/output/test_result_info.py @@ -73,6 +73,20 @@ def test_to_raw_dict_normalizes_container_and_scalar_types(self): assert output["day"] == "2026-06-01" assert output["ts"] == "2026-06-01T14:00:00" + def test_to_raw_dict_parses_container_string_with_unescaped_newline(self): + result_info = ResultInfo( + dingo_id="dingo-1", + raw_data={ + "claims": '[{"claim_text":"first line\nsecond line"}]', + }, + eval_status=False, + ) + + output = result_info.to_raw_dict() + assert output["claims"] == [ + {"claim_text": "first line\nsecond line"}, + ] + def test_to_raw_dict_keeps_original_raw_data_unchanged(self): original_raw_data = { "dingo_id": "user-id", diff --git a/test/scripts/model/rule/test_rule_patent.py b/test/scripts/model/rule/test_rule_patent.py new file mode 100644 index 00000000..b3e1cb6d --- /dev/null +++ b/test/scripts/model/rule/test_rule_patent.py @@ -0,0 +1,100 @@ +import pytest + +from dingo.io.input import Data +from dingo.model.rule.scibase.rule_patent import MAX_CLAIMS_LENGTH, RulePatentFieldValidation, check_ipc_unified + + +def _claims_with_string_length(length: int) -> list[dict]: + empty_claims = [{"text": ""}] + return [{"text": "x" * (length - len(str(empty_claims)))}] + + +class TestRulePatentFieldValidation: + def test_claims_at_maximum_length_is_valid(self): + claims = _claims_with_string_length(MAX_CLAIMS_LENGTH) + assert len(str(claims)) == MAX_CLAIMS_LENGTH + + result = RulePatentFieldValidation().eval(Data(claims=claims, ipc_unified=[])) + + assert result.status is False + assert result.label == ["QUALITY_GOOD"] + + def test_claims_over_maximum_length_is_invalid(self): + claims = _claims_with_string_length(MAX_CLAIMS_LENGTH + 1) + assert len(str(claims)) == MAX_CLAIMS_LENGTH + 1 + + result = RulePatentFieldValidation().eval(Data(claims=claims, ipc_unified=[])) + + assert result.status is True + assert result.label == ["claims.too_long"] + assert result.reason == [ + "claims: string length 1000001 exceeds the maximum of 1000000" + ] + + def test_missing_claims_is_invalid(self): + result = RulePatentFieldValidation().eval(Data(ipc_unified=[])) + + assert result.status is True + assert result.label == ["claims.missing_field"] + + @pytest.mark.parametrize( + "ipc_unified", + [ + [], + ["A01A 1/00"], + ["B60T 8/48", "H99Z 999/999999"], + '["B60T 8/48", "B60T 8/58"]', + ], + ) + def test_valid_ipc_unified(self, ipc_unified): + assert check_ipc_unified(ipc_unified) == (False, [], []) + + @pytest.mark.parametrize( + "ipc_code", + [ + "I01A 1/00", # Section must be A-H. + "A00A 1/00", # Class must be 01-99. + "A1A 1/00", # Class must contain exactly two digits. + "A01a 1/00", # Subclass must be uppercase A-Z. + "A01A\u00a01/00", # Separator must be one ASCII space. + "A01A 1/00", # Only one separator space is allowed. + "A01A 01/00", # Main group must not be zero-padded. + "A01A 1000/00", # Main group must be at most 999. + "A01A 1/0", # Subgroup must contain at least two digits. + "A01A 1/000", # A zero subgroup must be written as 00. + "A01A 1/1234567", # Subgroup must contain at most six digits. + "A01A 1-00", # Groups must be separated by a slash. + "A01A 1/00 A", # Extra suffixes are not allowed. + ], + ) + def test_invalid_ipc_format(self, ipc_code): + invalid, labels, reasons = check_ipc_unified([ipc_code]) + + assert invalid is True + assert labels == ["invalid_format"] + assert reasons + + @pytest.mark.parametrize( + ("ipc_unified", "expected_label"), + [ + (None, "null"), + ({"code": "A01A 1/00"}, "wrong_type"), + ([1], "wrong_type"), + ("not-json", "invalid_json"), + ], + ) + def test_invalid_ipc_unified_value(self, ipc_unified, expected_label): + invalid, labels, reasons = check_ipc_unified(ipc_unified) + + assert invalid is True + assert labels == [expected_label] + assert reasons + + def test_rule_reports_invalid_ipc_unified(self): + result = RulePatentFieldValidation().eval( + Data(claims=[], ipc_unified=["A01A 01/00"]) + ) + + assert result.status is True + assert result.label == ["ipc_unified.invalid_format"] + assert result.reason[0].startswith("ipc_unified: item[0]")