Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dingo/io/output/result_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ def _parse_container_string(cls, value: str) -> Any:
):
return value
try:
parsed = json.loads(text)
parsed = json.loads(text, strict=False)
except json.JSONDecodeError:
try:
parsed = ast.literal_eval(text)
Expand Down
128 changes: 128 additions & 0 deletions dingo/model/rule/scibase/rule_patent.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
import json
import re
from typing import Any, List

from dingo.config.input_args import EvaluatorRuleArgs
from dingo.io.input import Data
from dingo.io.output.eval_detail import EvalDetail, QualityLabel
from dingo.model.model import Model
from dingo.model.rule.base import BaseRule

MAX_CLAIMS_LENGTH = 1_000_000
IPC_CODE_RE = re.compile(
r"^(?P<section>[A-H])"
r"(?P<class>0[1-9]|[1-9][0-9])"
r"(?P<subclass>[A-Z]) "
r"(?P<main_group>[1-9][0-9]{0,2})/"
r"(?P<subgroup>[0-9]{2,6})$"
)
ValidationResult = tuple[bool, List[str], List[str]]


def check_claims(claims: Any) -> ValidationResult:
claims_length = len(str(claims))
if claims_length > MAX_CLAIMS_LENGTH:
return (
True,
["too_long"],
[
f"string length {claims_length} exceeds the maximum "
f"of {MAX_CLAIMS_LENGTH}"
],
)
return False, [], []


def check_ipc_unified(ipc_unified: Any) -> ValidationResult:
if ipc_unified is None:
return True, ["null"], ["value is null"]

if isinstance(ipc_unified, str):
try:
ipc_unified = json.loads(ipc_unified, strict=False)
except json.JSONDecodeError:
return True, ["invalid_json"], ["value must be a JSON array"]

if not isinstance(ipc_unified, list):
return True, ["wrong_type"], ["value must be a list"]

for index, ipc_code in enumerate(ipc_unified):
if not isinstance(ipc_code, str):
return (
True,
["wrong_type"],
[f"item[{index}] must be a string"],
)

match = IPC_CODE_RE.fullmatch(ipc_code)
if match is None:
return (
True,
["invalid_format"],
[
f"item[{index}] must match IPC format "
"'<A-H><01-99><A-Z> <1-999>/<2-6 digits>'"
],
)

subgroup = match.group("subgroup")
if subgroup != "00" and int(subgroup) == 0:
return (
True,
["invalid_format"],
[f"item[{index}] subgroup may be zero only when written as '00'"],
)

return False, [], []


FIELD_VALIDATORS = {
"claims": lambda record: check_claims(record.get("claims")),
"ipc_unified": lambda record: check_ipc_unified(record.get("ipc_unified")),
}


@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", ["xinghe", "quanliang"])
class RulePatentFieldValidation(BaseRule):
_metric_info = {
"category": "Rule-Based Metadata Quality Metrics",
"quality_dimension": "EFFECTIVENESS",
"metric_name": "RulePatentFieldValidation",
"description": "Validate patent metadata fields and report invalid fields",
"paper_title": "",
"paper_url": "",
"paper_authors": "",
"evaluation_results": "",
}

_required_fields = []
dynamic_config = EvaluatorRuleArgs(key_list=list(FIELD_VALIDATORS.keys()))

def eval(self, input_data: Data) -> EvalDetail:
res = EvalDetail(metric=self.__class__.__name__)
record = input_data.to_dict()
selected_fields = self.dynamic_config.key_list or []
bad_fields: List[str] = []
reasons: List[str] = []

for field in selected_fields:
if field not in FIELD_VALIDATORS:
bad_fields.append(f"{field}.unsupported_field")
reasons.append(f"{field}: unsupported field")
continue
if field not in record:
bad_fields.append(f"{field}.missing_field")
reasons.append(f"{field}: missing field")
continue
invalid, error_labels, detail_reasons = FIELD_VALIDATORS[field](record)
if invalid:
bad_fields.extend(f"{field}.{error_label}" for error_label in error_labels)
reasons.extend(f"{field}: {reason}" for reason in detail_reasons)

if bad_fields:
res.status = True
res.label = bad_fields
res.reason = reasons
else:
res.label = [QualityLabel.QUALITY_GOOD]
return res
2 changes: 1 addition & 1 deletion docs/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ This document provides comprehensive information about all quality metrics used

| Type | Metric | Description | Paper Source | Evaluation Results | Examples |
|------|--------|-------------|--------------|-------------------|----------|
| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleAuthorFieldValidation, RuleQuanliangFieldValidation, RuleSourceFieldValidation | 检查元数据字段与基准数据的相似度匹配,阈值默认为0.6; Validate OpenAlex author fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A |
| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleAuthorFieldValidation, RulePatentFieldValidation, RuleQuanliangFieldValidation, RuleSourceFieldValidation | 检查元数据字段与基准数据的相似度匹配,阈值默认为0.6; Validate OpenAlex author fields and report invalid fields; Validate patent metadata fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A |

### Rule-Based RESUME Quality Metrics

Expand Down
1 change: 1 addition & 0 deletions docs/rules.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ The specific rules for each quality metric are as follows:
| RuleOnlyUrl | EFFECTIVENESS | Check whether content consists only of a URL. | |
| RulePatternSearch | RELEVANCE | Search content using a user-provided pattern. | |
| RulePIIDetection | SECURITY | Detect personally identifiable information in text. | |
| RulePatentFieldValidation | EFFECTIVENESS | Validate patent claims length and IPC classification-code format. | |
| RuleQuanliangFieldValidation | EFFECTIVENESS | Validate full-volume scientific metadata fields. | |
| RuleResumeDateFormat | RESUME_DATE | Check whether a resume uses inconsistent date formats. | |
| RuleResumeDetailedAddress | RESUME_PRIVACY | Check whether a resume contains a detailed address. | |
Expand Down
14 changes: 14 additions & 0 deletions test/scripts/io/output/test_result_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,20 @@ def test_to_raw_dict_normalizes_container_and_scalar_types(self):
assert output["day"] == "2026-06-01"
assert output["ts"] == "2026-06-01T14:00:00"

def test_to_raw_dict_parses_container_string_with_unescaped_newline(self):
result_info = ResultInfo(
dingo_id="dingo-1",
raw_data={
"claims": '[{"claim_text":"first line\nsecond line"}]',
},
eval_status=False,
)

output = result_info.to_raw_dict()
assert output["claims"] == [
{"claim_text": "first line\nsecond line"},
]

def test_to_raw_dict_keeps_original_raw_data_unchanged(self):
original_raw_data = {
"dingo_id": "user-id",
Expand Down
100 changes: 100 additions & 0 deletions test/scripts/model/rule/test_rule_patent.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import pytest

from dingo.io.input import Data
from dingo.model.rule.scibase.rule_patent import MAX_CLAIMS_LENGTH, RulePatentFieldValidation, check_ipc_unified


def _claims_with_string_length(length: int) -> list[dict]:
empty_claims = [{"text": ""}]
return [{"text": "x" * (length - len(str(empty_claims)))}]


class TestRulePatentFieldValidation:
def test_claims_at_maximum_length_is_valid(self):
claims = _claims_with_string_length(MAX_CLAIMS_LENGTH)
assert len(str(claims)) == MAX_CLAIMS_LENGTH

result = RulePatentFieldValidation().eval(Data(claims=claims, ipc_unified=[]))

assert result.status is False
assert result.label == ["QUALITY_GOOD"]

def test_claims_over_maximum_length_is_invalid(self):
claims = _claims_with_string_length(MAX_CLAIMS_LENGTH + 1)
assert len(str(claims)) == MAX_CLAIMS_LENGTH + 1

result = RulePatentFieldValidation().eval(Data(claims=claims, ipc_unified=[]))

assert result.status is True
assert result.label == ["claims.too_long"]
assert result.reason == [
"claims: string length 1000001 exceeds the maximum of 1000000"
]

def test_missing_claims_is_invalid(self):
result = RulePatentFieldValidation().eval(Data(ipc_unified=[]))

assert result.status is True
assert result.label == ["claims.missing_field"]

@pytest.mark.parametrize(
"ipc_unified",
[
[],
["A01A 1/00"],
["B60T 8/48", "H99Z 999/999999"],
'["B60T 8/48", "B60T 8/58"]',
],
)
def test_valid_ipc_unified(self, ipc_unified):
assert check_ipc_unified(ipc_unified) == (False, [], [])

@pytest.mark.parametrize(
"ipc_code",
[
"I01A 1/00", # Section must be A-H.
"A00A 1/00", # Class must be 01-99.
"A1A 1/00", # Class must contain exactly two digits.
"A01a 1/00", # Subclass must be uppercase A-Z.
"A01A\u00a01/00", # Separator must be one ASCII space.
"A01A 1/00", # Only one separator space is allowed.
"A01A 01/00", # Main group must not be zero-padded.
"A01A 1000/00", # Main group must be at most 999.
"A01A 1/0", # Subgroup must contain at least two digits.
"A01A 1/000", # A zero subgroup must be written as 00.
"A01A 1/1234567", # Subgroup must contain at most six digits.
"A01A 1-00", # Groups must be separated by a slash.
"A01A 1/00 A", # Extra suffixes are not allowed.
],
)
def test_invalid_ipc_format(self, ipc_code):
invalid, labels, reasons = check_ipc_unified([ipc_code])

assert invalid is True
assert labels == ["invalid_format"]
assert reasons

@pytest.mark.parametrize(
("ipc_unified", "expected_label"),
[
(None, "null"),
({"code": "A01A 1/00"}, "wrong_type"),
([1], "wrong_type"),
("not-json", "invalid_json"),
],
)
def test_invalid_ipc_unified_value(self, ipc_unified, expected_label):
invalid, labels, reasons = check_ipc_unified(ipc_unified)

assert invalid is True
assert labels == [expected_label]
assert reasons

def test_rule_reports_invalid_ipc_unified(self):
result = RulePatentFieldValidation().eval(
Data(claims=[], ipc_unified=["A01A 01/00"])
)

assert result.status is True
assert result.label == ["ipc_unified.invalid_format"]
assert result.reason[0].startswith("ipc_unified: item[0]")
Loading