From 31dc5b66ba4f721267fe3cab64faa1ddd5c4b364 Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 22 Sep 2026 19:47:44 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20=E4=B8=93=E5=88=A9=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=EF=BC=8Cclaims=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/io/output/result_info.py | 2 +- dingo/model/rule/scibase/rule_patent.py | 76 +++++++++++++++++++++ docs/metrics.md | 2 +- docs/rules.md | 1 + test/scripts/io/output/test_result_info.py | 14 ++++ test/scripts/model/rule/test_rule_patent.py | 39 +++++++++++ 6 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 dingo/model/rule/scibase/rule_patent.py create mode 100644 test/scripts/model/rule/test_rule_patent.py diff --git a/dingo/io/output/result_info.py b/dingo/io/output/result_info.py index 2935efe2..8a03a4ee 100644 --- a/dingo/io/output/result_info.py +++ b/dingo/io/output/result_info.py @@ -62,7 +62,7 @@ def _parse_container_string(cls, value: str) -> Any: ): return value try: - parsed = json.loads(text) + parsed = json.loads(text, strict=False) except json.JSONDecodeError: try: parsed = ast.literal_eval(text) diff --git a/dingo/model/rule/scibase/rule_patent.py b/dingo/model/rule/scibase/rule_patent.py new file mode 100644 index 00000000..316b0694 --- /dev/null +++ b/dingo/model/rule/scibase/rule_patent.py @@ -0,0 +1,76 @@ +from typing import Any, List + +from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io.input import Data +from dingo.io.output.eval_detail import EvalDetail, QualityLabel +from dingo.model.model import Model +from dingo.model.rule.base import BaseRule + + +MAX_CLAIMS_LENGTH = 1_000 +ValidationResult = tuple[bool, List[str], List[str]] + + +def check_claims(claims: Any) -> ValidationResult: + claims_length = len(str(claims)) + if claims_length > MAX_CLAIMS_LENGTH: + return ( + True, + ["too_long"], + [ + f"string length {claims_length} exceeds the maximum " + f"of {MAX_CLAIMS_LENGTH}" + ], + ) + return False, [], [] + + +FIELD_VALIDATORS = { + "claims": lambda record: check_claims(record.get("claims")), +} + + +@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", ["xinghe", "quanliang"]) +class RulePatentFieldValidation(BaseRule): + _metric_info = { + "category": "Rule-Based Metadata Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "RulePatentFieldValidation", + "description": "Validate patent metadata fields and report invalid fields", + "paper_title": "", + "paper_url": "", + "paper_authors": "", + "evaluation_results": "", + } + + _required_fields = [] + dynamic_config = EvaluatorRuleArgs(key_list=list(FIELD_VALIDATORS.keys())) + + def eval(self, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=self.__class__.__name__) + record = input_data.to_dict() + selected_fields = self.dynamic_config.key_list or [] + bad_fields: List[str] = [] + reasons: List[str] = [] + + for field in selected_fields: + if field not in FIELD_VALIDATORS: + bad_fields.append(f"{field}.unsupported_field") + reasons.append(f"{field}: unsupported field") + continue + if field not in record: + bad_fields.append(f"{field}.missing_field") + reasons.append(f"{field}: missing field") + continue + invalid, error_labels, detail_reasons = FIELD_VALIDATORS[field](record) + if invalid: + bad_fields.extend(f"{field}.{error_label}" for error_label in error_labels) + reasons.extend(f"{field}: {reason}" for reason in detail_reasons) + + if bad_fields: + res.status = True + res.label = bad_fields + res.reason = reasons + else: + res.label = [QualityLabel.QUALITY_GOOD] + return res diff --git a/docs/metrics.md b/docs/metrics.md index d5326a31..cdd03142 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -148,7 +148,7 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleAuthorFieldValidation, RuleQuanliangFieldValidation, RuleSourceFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate OpenAlex author fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleAuthorFieldValidation, RulePatentFieldValidation, RuleQuanliangFieldValidation, RuleSourceFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate OpenAlex author fields and report invalid fields; Validate patent metadata fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A | ### Rule-Based RESUME Quality Metrics diff --git a/docs/rules.md b/docs/rules.md index 8bd72f6d..4c1ae569 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -64,6 +64,7 @@ The specific rules for each quality metric are as follows: | RuleOnlyUrl | EFFECTIVENESS | Check whether content consists only of a URL. | | | RulePatternSearch | RELEVANCE | Search content using a user-provided pattern. | | | RulePIIDetection | SECURITY | Detect personally identifiable information in text. | | +| RulePatentFieldValidation | EFFECTIVENESS | Validate patent claims and flag serialized content over 1,000,000 characters. | | | RuleQuanliangFieldValidation | EFFECTIVENESS | Validate full-volume scientific metadata fields. | | | RuleResumeDateFormat | RESUME_DATE | Check whether a resume uses inconsistent date formats. | | | RuleResumeDetailedAddress | RESUME_PRIVACY | Check whether a resume contains a detailed address. | | diff --git a/test/scripts/io/output/test_result_info.py b/test/scripts/io/output/test_result_info.py index 9530c54c..c0345053 100644 --- a/test/scripts/io/output/test_result_info.py +++ b/test/scripts/io/output/test_result_info.py @@ -73,6 +73,20 @@ def test_to_raw_dict_normalizes_container_and_scalar_types(self): assert output["day"] == "2026-06-01" assert output["ts"] == "2026-06-01T14:00:00" + def test_to_raw_dict_parses_container_string_with_unescaped_newline(self): + result_info = ResultInfo( + dingo_id="dingo-1", + raw_data={ + "claims": '[{"claim_text":"first line\nsecond line"}]', + }, + eval_status=False, + ) + + output = result_info.to_raw_dict() + assert output["claims"] == [ + {"claim_text": "first line\nsecond line"}, + ] + def test_to_raw_dict_keeps_original_raw_data_unchanged(self): original_raw_data = { "dingo_id": "user-id", diff --git a/test/scripts/model/rule/test_rule_patent.py b/test/scripts/model/rule/test_rule_patent.py new file mode 100644 index 00000000..f7cabf78 --- /dev/null +++ b/test/scripts/model/rule/test_rule_patent.py @@ -0,0 +1,39 @@ +from dingo.io.input import Data +from dingo.model.rule.scibase.rule_patent import ( + MAX_CLAIMS_LENGTH, + RulePatentFieldValidation, +) + + +def _claims_with_string_length(length: int) -> list[dict]: + empty_claims = [{"text": ""}] + return [{"text": "x" * (length - len(str(empty_claims)))}] + + +class TestRulePatentFieldValidation: + def test_claims_at_maximum_length_is_valid(self): + claims = _claims_with_string_length(MAX_CLAIMS_LENGTH) + assert len(str(claims)) == MAX_CLAIMS_LENGTH + + result = RulePatentFieldValidation().eval(Data(claims=claims)) + + assert result.status is False + assert result.label == ["QUALITY_GOOD"] + + def test_claims_over_maximum_length_is_invalid(self): + claims = _claims_with_string_length(MAX_CLAIMS_LENGTH + 1) + assert len(str(claims)) == MAX_CLAIMS_LENGTH + 1 + + result = RulePatentFieldValidation().eval(Data(claims=claims)) + + assert result.status is True + assert result.label == ["claims.too_long"] + assert result.reason == [ + "claims: string length 1000001 exceeds the maximum of 1000000" + ] + + def test_missing_claims_is_invalid(self): + result = RulePatentFieldValidation().eval(Data()) + + assert result.status is True + assert result.label == ["claims.missing_field"] From 7e2a6854fe379bd77b1a6b1f43e5d148d614b943 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 28 Sep 2026 14:14:31 +0800 Subject: [PATCH 2/4] =?UTF-8?q?feat:=20=E4=B8=93=E5=88=A9=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=EF=BC=8Cipc=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/rule/scibase/rule_patent.py | 55 +++++++++++++++- docs/rules.md | 2 +- test/scripts/model/rule/test_rule_patent.py | 71 ++++++++++++++++++++- 3 files changed, 123 insertions(+), 5 deletions(-) diff --git a/dingo/model/rule/scibase/rule_patent.py b/dingo/model/rule/scibase/rule_patent.py index 316b0694..65e63d68 100644 --- a/dingo/model/rule/scibase/rule_patent.py +++ b/dingo/model/rule/scibase/rule_patent.py @@ -1,3 +1,5 @@ +import json +import re from typing import Any, List from dingo.config.input_args import EvaluatorRuleArgs @@ -7,7 +9,14 @@ from dingo.model.rule.base import BaseRule -MAX_CLAIMS_LENGTH = 1_000 +MAX_CLAIMS_LENGTH = 1_000_000 +IPC_CODE_RE = re.compile( + r"^(?P
[A-H])" + r"(?P0[1-9]|[1-9][0-9])" + r"(?P[A-Z]) " + r"(?P[1-9][0-9]{0,2})/" + r"(?P[0-9]{2,6})$" +) ValidationResult = tuple[bool, List[str], List[str]] @@ -25,8 +34,52 @@ def check_claims(claims: Any) -> ValidationResult: return False, [], [] +def check_ipc(ipc: Any) -> ValidationResult: + if ipc is None: + return True, ["null"], ["value is null"] + + if isinstance(ipc, str): + try: + ipc = json.loads(ipc, strict=False) + except json.JSONDecodeError: + return True, ["invalid_json"], ["value must be a JSON array"] + + if not isinstance(ipc, list): + return True, ["wrong_type"], ["value must be a list"] + + for index, ipc_code in enumerate(ipc): + if not isinstance(ipc_code, str): + return ( + True, + ["wrong_type"], + [f"item[{index}] must be a string"], + ) + + match = IPC_CODE_RE.fullmatch(ipc_code) + if match is None: + return ( + True, + ["invalid_format"], + [ + f"item[{index}] must match IPC format " + "'<01-99> <1-999>/<2-6 digits>'" + ], + ) + + subgroup = match.group("subgroup") + if subgroup != "00" and int(subgroup) == 0: + return ( + True, + ["invalid_format"], + [f"item[{index}] subgroup may be zero only when written as '00'"], + ) + + return False, [], [] + + FIELD_VALIDATORS = { "claims": lambda record: check_claims(record.get("claims")), + "ipc": lambda record: check_ipc(record.get("ipc")), } diff --git a/docs/rules.md b/docs/rules.md index 4c1ae569..0b40a64f 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -64,7 +64,7 @@ The specific rules for each quality metric are as follows: | RuleOnlyUrl | EFFECTIVENESS | Check whether content consists only of a URL. | | | RulePatternSearch | RELEVANCE | Search content using a user-provided pattern. | | | RulePIIDetection | SECURITY | Detect personally identifiable information in text. | | -| RulePatentFieldValidation | EFFECTIVENESS | Validate patent claims and flag serialized content over 1,000,000 characters. | | +| RulePatentFieldValidation | EFFECTIVENESS | Validate patent claims length and IPC classification-code format. | | | RuleQuanliangFieldValidation | EFFECTIVENESS | Validate full-volume scientific metadata fields. | | | RuleResumeDateFormat | RESUME_DATE | Check whether a resume uses inconsistent date formats. | | | RuleResumeDetailedAddress | RESUME_PRIVACY | Check whether a resume contains a detailed address. | | diff --git a/test/scripts/model/rule/test_rule_patent.py b/test/scripts/model/rule/test_rule_patent.py index f7cabf78..cc424399 100644 --- a/test/scripts/model/rule/test_rule_patent.py +++ b/test/scripts/model/rule/test_rule_patent.py @@ -1,7 +1,10 @@ +import pytest + from dingo.io.input import Data from dingo.model.rule.scibase.rule_patent import ( MAX_CLAIMS_LENGTH, RulePatentFieldValidation, + check_ipc, ) @@ -15,7 +18,7 @@ def test_claims_at_maximum_length_is_valid(self): claims = _claims_with_string_length(MAX_CLAIMS_LENGTH) assert len(str(claims)) == MAX_CLAIMS_LENGTH - result = RulePatentFieldValidation().eval(Data(claims=claims)) + result = RulePatentFieldValidation().eval(Data(claims=claims, ipc=[])) assert result.status is False assert result.label == ["QUALITY_GOOD"] @@ -24,7 +27,7 @@ def test_claims_over_maximum_length_is_invalid(self): claims = _claims_with_string_length(MAX_CLAIMS_LENGTH + 1) assert len(str(claims)) == MAX_CLAIMS_LENGTH + 1 - result = RulePatentFieldValidation().eval(Data(claims=claims)) + result = RulePatentFieldValidation().eval(Data(claims=claims, ipc=[])) assert result.status is True assert result.label == ["claims.too_long"] @@ -33,7 +36,69 @@ def test_claims_over_maximum_length_is_invalid(self): ] def test_missing_claims_is_invalid(self): - result = RulePatentFieldValidation().eval(Data()) + result = RulePatentFieldValidation().eval(Data(ipc=[])) assert result.status is True assert result.label == ["claims.missing_field"] + + @pytest.mark.parametrize( + "ipc", + [ + [], + ["A01A 1/00"], + ["B60T 8/48", "H99Z 999/999999"], + '["B60T 8/48", "B60T 8/58"]', + ], + ) + def test_valid_ipc(self, ipc): + assert check_ipc(ipc) == (False, [], []) + + @pytest.mark.parametrize( + "ipc_code", + [ + "I01A 1/00", # Section must be A-H. + "A00A 1/00", # Class must be 01-99. + "A1A 1/00", # Class must contain exactly two digits. + "A01a 1/00", # Subclass must be uppercase A-Z. + "A01A\u00a01/00", # Separator must be one ASCII space. + "A01A 1/00", # Only one separator space is allowed. + "A01A 01/00", # Main group must not be zero-padded. + "A01A 1000/00", # Main group must be at most 999. + "A01A 1/0", # Subgroup must contain at least two digits. + "A01A 1/000", # A zero subgroup must be written as 00. + "A01A 1/1234567", # Subgroup must contain at most six digits. + "A01A 1-00", # Groups must be separated by a slash. + "A01A 1/00 A", # Extra suffixes are not allowed. + ], + ) + def test_invalid_ipc_format(self, ipc_code): + invalid, labels, reasons = check_ipc([ipc_code]) + + assert invalid is True + assert labels == ["invalid_format"] + assert reasons + + @pytest.mark.parametrize( + ("ipc", "expected_label"), + [ + (None, "null"), + ({"code": "A01A 1/00"}, "wrong_type"), + ([1], "wrong_type"), + ("not-json", "invalid_json"), + ], + ) + def test_invalid_ipc_value(self, ipc, expected_label): + invalid, labels, reasons = check_ipc(ipc) + + assert invalid is True + assert labels == [expected_label] + assert reasons + + def test_rule_reports_invalid_ipc(self): + result = RulePatentFieldValidation().eval( + Data(claims=[], ipc=["A01A 01/00"]) + ) + + assert result.status is True + assert result.label == ["ipc.invalid_format"] + assert result.reason[0].startswith("ipc: item[0]") From a6cec2964aa1b7413f1b3a01c9705cce833215a7 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 28 Sep 2026 14:35:30 +0800 Subject: [PATCH 3/4] =?UTF-8?q?feat:=20=E6=94=B9=E5=90=8D=20ipc=5Funified?= =?UTF-8?q?=20=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/rule/scibase/rule_patent.py | 14 +++++----- test/scripts/model/rule/test_rule_patent.py | 30 ++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/dingo/model/rule/scibase/rule_patent.py b/dingo/model/rule/scibase/rule_patent.py index 65e63d68..53ccddb2 100644 --- a/dingo/model/rule/scibase/rule_patent.py +++ b/dingo/model/rule/scibase/rule_patent.py @@ -34,20 +34,20 @@ def check_claims(claims: Any) -> ValidationResult: return False, [], [] -def check_ipc(ipc: Any) -> ValidationResult: - if ipc is None: +def check_ipc_unified(ipc_unified: Any) -> ValidationResult: + if ipc_unified is None: return True, ["null"], ["value is null"] - if isinstance(ipc, str): + if isinstance(ipc_unified, str): try: - ipc = json.loads(ipc, strict=False) + ipc_unified = json.loads(ipc_unified, strict=False) except json.JSONDecodeError: return True, ["invalid_json"], ["value must be a JSON array"] - if not isinstance(ipc, list): + if not isinstance(ipc_unified, list): return True, ["wrong_type"], ["value must be a list"] - for index, ipc_code in enumerate(ipc): + for index, ipc_code in enumerate(ipc_unified): if not isinstance(ipc_code, str): return ( True, @@ -79,7 +79,7 @@ def check_ipc(ipc: Any) -> ValidationResult: FIELD_VALIDATORS = { "claims": lambda record: check_claims(record.get("claims")), - "ipc": lambda record: check_ipc(record.get("ipc")), + "ipc_unified": lambda record: check_ipc_unified(record.get("ipc_unified")), } diff --git a/test/scripts/model/rule/test_rule_patent.py b/test/scripts/model/rule/test_rule_patent.py index cc424399..434b467e 100644 --- a/test/scripts/model/rule/test_rule_patent.py +++ b/test/scripts/model/rule/test_rule_patent.py @@ -4,7 +4,7 @@ from dingo.model.rule.scibase.rule_patent import ( MAX_CLAIMS_LENGTH, RulePatentFieldValidation, - check_ipc, + check_ipc_unified, ) @@ -18,7 +18,7 @@ def test_claims_at_maximum_length_is_valid(self): claims = _claims_with_string_length(MAX_CLAIMS_LENGTH) assert len(str(claims)) == MAX_CLAIMS_LENGTH - result = RulePatentFieldValidation().eval(Data(claims=claims, ipc=[])) + result = RulePatentFieldValidation().eval(Data(claims=claims, ipc_unified=[])) assert result.status is False assert result.label == ["QUALITY_GOOD"] @@ -27,7 +27,7 @@ def test_claims_over_maximum_length_is_invalid(self): claims = _claims_with_string_length(MAX_CLAIMS_LENGTH + 1) assert len(str(claims)) == MAX_CLAIMS_LENGTH + 1 - result = RulePatentFieldValidation().eval(Data(claims=claims, ipc=[])) + result = RulePatentFieldValidation().eval(Data(claims=claims, ipc_unified=[])) assert result.status is True assert result.label == ["claims.too_long"] @@ -36,13 +36,13 @@ def test_claims_over_maximum_length_is_invalid(self): ] def test_missing_claims_is_invalid(self): - result = RulePatentFieldValidation().eval(Data(ipc=[])) + result = RulePatentFieldValidation().eval(Data(ipc_unified=[])) assert result.status is True assert result.label == ["claims.missing_field"] @pytest.mark.parametrize( - "ipc", + "ipc_unified", [ [], ["A01A 1/00"], @@ -50,8 +50,8 @@ def test_missing_claims_is_invalid(self): '["B60T 8/48", "B60T 8/58"]', ], ) - def test_valid_ipc(self, ipc): - assert check_ipc(ipc) == (False, [], []) + def test_valid_ipc_unified(self, ipc_unified): + assert check_ipc_unified(ipc_unified) == (False, [], []) @pytest.mark.parametrize( "ipc_code", @@ -72,14 +72,14 @@ def test_valid_ipc(self, ipc): ], ) def test_invalid_ipc_format(self, ipc_code): - invalid, labels, reasons = check_ipc([ipc_code]) + invalid, labels, reasons = check_ipc_unified([ipc_code]) assert invalid is True assert labels == ["invalid_format"] assert reasons @pytest.mark.parametrize( - ("ipc", "expected_label"), + ("ipc_unified", "expected_label"), [ (None, "null"), ({"code": "A01A 1/00"}, "wrong_type"), @@ -87,18 +87,18 @@ def test_invalid_ipc_format(self, ipc_code): ("not-json", "invalid_json"), ], ) - def test_invalid_ipc_value(self, ipc, expected_label): - invalid, labels, reasons = check_ipc(ipc) + def test_invalid_ipc_unified_value(self, ipc_unified, expected_label): + invalid, labels, reasons = check_ipc_unified(ipc_unified) assert invalid is True assert labels == [expected_label] assert reasons - def test_rule_reports_invalid_ipc(self): + def test_rule_reports_invalid_ipc_unified(self): result = RulePatentFieldValidation().eval( - Data(claims=[], ipc=["A01A 01/00"]) + Data(claims=[], ipc_unified=["A01A 01/00"]) ) assert result.status is True - assert result.label == ["ipc.invalid_format"] - assert result.reason[0].startswith("ipc: item[0]") + assert result.label == ["ipc_unified.invalid_format"] + assert result.reason[0].startswith("ipc_unified: item[0]") From f15d807b0287c5db80aba23529bb4daeb3f48b4e Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 28 Sep 2026 14:45:20 +0800 Subject: [PATCH 4/4] feat: lint --- dingo/model/rule/scibase/rule_patent.py | 1 - test/scripts/model/rule/test_rule_patent.py | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/dingo/model/rule/scibase/rule_patent.py b/dingo/model/rule/scibase/rule_patent.py index 53ccddb2..7056c81b 100644 --- a/dingo/model/rule/scibase/rule_patent.py +++ b/dingo/model/rule/scibase/rule_patent.py @@ -8,7 +8,6 @@ from dingo.model.model import Model from dingo.model.rule.base import BaseRule - MAX_CLAIMS_LENGTH = 1_000_000 IPC_CODE_RE = re.compile( r"^(?P
[A-H])" diff --git a/test/scripts/model/rule/test_rule_patent.py b/test/scripts/model/rule/test_rule_patent.py index 434b467e..b3e1cb6d 100644 --- a/test/scripts/model/rule/test_rule_patent.py +++ b/test/scripts/model/rule/test_rule_patent.py @@ -1,11 +1,7 @@ import pytest from dingo.io.input import Data -from dingo.model.rule.scibase.rule_patent import ( - MAX_CLAIMS_LENGTH, - RulePatentFieldValidation, - check_ipc_unified, -) +from dingo.model.rule.scibase.rule_patent import MAX_CLAIMS_LENGTH, RulePatentFieldValidation, check_ipc_unified def _claims_with_string_length(length: int) -> list[dict]: