diff --git a/site/cds_rdm/inspire_harvester/load/matcher.py b/site/cds_rdm/inspire_harvester/load/matcher.py index 12ac2e15..394a9ff6 100644 --- a/site/cds_rdm/inspire_harvester/load/matcher.py +++ b/site/cds_rdm/inspire_harvester/load/matcher.py @@ -104,17 +104,53 @@ def query(self): @dataclass(frozen=True) class ReportNumberMatchFilter(FilterCandidate): - """Match a CDS report number.""" + """Match a CDS report number in metadata.identifiers.""" @property def query(self): """Build the CDS report number query.""" + return [ + dsl.Q("term", **{"metadata.identifiers.scheme": "cdsrn"}), + dsl.Q( + "term", + **{"metadata.identifiers.identifier": self.value}, + ), + ] + + +@dataclass(frozen=True) +class RelatedReportNumberMatchFilter(FilterCandidate): + """Match a CDS report number in metadata.related_identifiers.""" + + @property + def query(self): + """Build the related CDS report number query.""" return [ dsl.Q("term", **{"metadata.related_identifiers.scheme": "cdsrn"}), dsl.Q( "term", **{"metadata.related_identifiers.identifier": self.value}, ), + dsl.Q( + "term", + **{"metadata.related_identifiers.relation_type.id": "isvariantformof"}, + ), + ] + + +@dataclass(frozen=True) +class ApprovalReportNumberMatchFilter(FilterCandidate): + """Match an EP/approval report number (apprn) in metadata.identifiers.""" + + @property + def query(self): + """Build the approval report number query.""" + return [ + dsl.Q("term", **{"metadata.identifiers.scheme": "apprn"}), + dsl.Q( + "term", + **{"metadata.identifiers.identifier": self.value}, + ), ] @@ -163,11 +199,14 @@ def _retrieve_identifier(self, identifiers, scheme) -> Optional[str]: def _build_filter_priority(self, entry, inspire_id, cdsrdm_id): """Build the priority-ordered record match candidates.""" doi = entry.get("pids", {}).get("doi", {}).get("identifier") + identifiers = entry["metadata"].get("identifiers", []) related_identifiers = entry["metadata"].get("related_identifiers", []) cds_id = self._retrieve_identifier(related_identifiers, "cds") arxiv_id = self._retrieve_identifier(related_identifiers, "arxiv") - report_number = self._retrieve_identifier(related_identifiers, "cdsrn") + report_number = self._retrieve_identifier(identifiers, "cdsrn") + related_report_number = self._retrieve_identifier(related_identifiers, "cdsrn") + approval_report_number = self._retrieve_identifier(identifiers, "apprn") return [ ParentMatchFilter(value=cdsrdm_id), CDSIdentifierMatchFilter(value=cds_id), @@ -175,4 +214,6 @@ def _build_filter_priority(self, entry, inspire_id, cdsrdm_id): InspireIdentifierMatchFilter(value=inspire_id), ArxivIdentifierMatchFilter(value=arxiv_id), ReportNumberMatchFilter(value=report_number), + ApprovalReportNumberMatchFilter(value=approval_report_number), + RelatedReportNumberMatchFilter(value=related_report_number), ] diff --git a/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py b/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py index f0f40919..91e309a3 100644 --- a/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py +++ b/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py @@ -8,12 +8,14 @@ """INSPIRE to CDS harvester module.""" import json +import re from dataclasses import dataclass from flask import current_app from idutils.normalizers import normalize_isbn, normalize_urn from idutils.validators import is_doi, is_urn +from cds_rdm import schemes from cds_rdm.inspire_harvester.transform.mappers.mapper import MapperBase @@ -60,6 +62,34 @@ def _related_identifier(schema, value, ctx): return related +def _committee_approval_prefixes(): + """Return configured committee approval report-number prefixes.""" + communities = current_app.config.get("CDS_COMMITTEE_APPROVAL_COMMUNITIES", {}) + return { + cfg.get("report_number", {}).get("prefix") + for cfg in communities.values() + if cfg.get("report_number", {}).get("prefix") + } + + +def _is_approval_report_number(value): + """Return True if value is a valid EP/approval report number. + + Requires both a configured committee prefix and a value accepted by the + ``apprn`` scheme validator, so prefix look-alikes fall back to ``cdsrn`` + instead of failing record validation. + """ + if not value: + return False + if not schemes.is_approval_report_number(value): + return False + # Prefix then a digit (the year), not another word like DRAFT. + return any( + re.match(rf"^{re.escape(prefix)}-\d", value) + for prefix in _committee_approval_prefixes() + ) + + @dataclass(frozen=True) class DOIMapper(MapperBase): """Mapper for DOI identifiers.""" @@ -147,6 +177,19 @@ def map_value(self, src_record, ctx, logger): "Unexpected schema in external_system_identifiers. " f"| details: schema={schema}, value={value}" ) + + # Report numbers on the record itself: + # - EP/approval numbers (configured prefixes) → apprn + # - other CERN- report numbers → cdsrn + for rn in src_metadata.get("report_numbers", []): + report_number = rn.get("value") + if not report_number: + continue + if _is_approval_report_number(report_number): + identifiers.append({"identifier": report_number, "scheme": "apprn"}) + elif report_number.startswith("CERN-"): + identifiers.append({"identifier": report_number, "scheme": "cdsrn"}) + unique_ids = [dict(t) for t in {tuple(sorted(d.items())) for d in identifiers}] return unique_ids @@ -240,12 +283,21 @@ def map_value(self, src_record, ctx, logger): } ) + # Non-CERN- / non-approval report numbers stay related (scheme cdsrn). + # CERN- cdsrn and apprn values are handled by IdentifiersMapper. report_numbers = src_metadata.get("report_numbers", []) for rn in report_numbers: + report_number = rn.get("value") + if ( + not report_number + or report_number.startswith("CERN-") + or _is_approval_report_number(report_number) + ): + continue identifiers.append( { "scheme": "cdsrn", - "identifier": f"{rn['value']}", + "identifier": report_number, "relation_type": {"id": "isvariantformof"}, "resource_type": {"id": ctx.resource_type.value}, } diff --git a/site/tests/inspire_harvester/test_transformer.py b/site/tests/inspire_harvester/test_transformer.py index c941c5db..bb8642e4 100644 --- a/site/tests/inspire_harvester/test_transformer.py +++ b/site/tests/inspire_harvester/test_transformer.py @@ -270,6 +270,91 @@ def test_transform_funding_missing_award_errors(): assert "Award not found in vocabulary" in ctx.errors[0] +def test_transform_report_numbers_as_identifiers(running_app): + """CERN- report numbers go to identifiers; EP prefixes use apprn.""" + from flask import current_app + + current_app.config["CDS_COMMITTEE_APPROVAL_COMMUNITIES"] = { + "ep-community": { + "report_number": {"prefix": "CERN-EP"}, + } + } + src_metadata = { + "report_numbers": [ + {"value": "CERN-EP-2026-001"}, + {"value": "CERN-EP-DRAFT-2026-001"}, + {"value": "CERN-THESIS-2010-364"}, + {"value": "DESY-24-001"}, + ] + } + ctx = MetadataSerializationContext( + resource_type=ResourceType.OTHER, inspire_id="12345" + ) + logger = Logger(inspire_id="12345") + + identifiers = IdentifiersMapper().map_value( + {"metadata": src_metadata, "created": "2023-01-01"}, ctx, logger + ) + related = RelatedIdentifiersMapper().map_value( + {"metadata": src_metadata, "created": "2023-01-01"}, ctx, logger + ) + + assert {"identifier": "CERN-EP-2026-001", "scheme": "apprn"} in identifiers + assert {"identifier": "CERN-EP-DRAFT-2026-001", "scheme": "cdsrn"} in identifiers + assert {"identifier": "CERN-THESIS-2010-364", "scheme": "cdsrn"} in identifiers + assert not any(i.get("scheme") == "cdsrn" and i.get("identifier") == "CERN-EP-2026-001" for i in identifiers) + assert any( + i.get("scheme") == "cdsrn" and i.get("identifier") == "DESY-24-001" + for i in related + ) + assert not any(i.get("identifier") == "CERN-EP-2026-001" for i in related) + assert not any(i.get("identifier") == "CERN-THESIS-2010-364" for i in related) + + +def test_matcher_includes_approval_report_number(running_app): + """Matcher searches by apprn as well as cdsrn.""" + from cds_rdm.inspire_harvester.load.matcher import ( + ApprovalReportNumberMatchFilter, + RecordMatcher, + RelatedReportNumberMatchFilter, + ReportNumberMatchFilter, + ) + + entry = { + "pids": {}, + "metadata": { + "identifiers": [ + {"scheme": "cdsrn", "identifier": "CERN-THESIS-2010-364"}, + {"scheme": "apprn", "identifier": "CERN-EP-2026-001"}, + ], + "related_identifiers": [ + {"scheme": "cdsrn", "identifier": "DESY-24-001"}, + {"scheme": "inspire", "identifier": "12345"}, + ], + }, + } + candidates = RecordMatcher()._build_filter_priority(entry, "12345", None) + by_type = {type(c): c for c in candidates if c.value} + + assert by_type[ReportNumberMatchFilter].value == "CERN-THESIS-2010-364" + assert by_type[RelatedReportNumberMatchFilter].value == "DESY-24-001" + assert by_type[ApprovalReportNumberMatchFilter].value == "CERN-EP-2026-001" + assert any( + q.to_dict() == {"term": {"metadata.identifiers.scheme": "apprn"}} + for q in by_type[ApprovalReportNumberMatchFilter].query + ) + assert any( + q.to_dict() + == {"term": {"metadata.related_identifiers.relation_type.id": "isvariantformof"}} + for q in by_type[RelatedReportNumberMatchFilter].query + ) + + valued = [type(c) for c in candidates if c.value] + assert valued.index(ApprovalReportNumberMatchFilter) < valued.index( + RelatedReportNumberMatchFilter + ) + + @patch("cds_rdm.inspire_harvester.transform.mappers.identifiers.is_doi") def test_transform_dois_valid_external(mock_is_doi, running_app): """Test DOIMapper with valid external DOI."""