From 7f4ed7817bc40a2ef6067e62668de2cfe0f2e5dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Z=C3=BCbeyde=20Civelek?= Date: Thu, 6 Aug 2026 14:02:07 +0200 Subject: [PATCH] add(ship): new models and transformation rules --- cds_migrator_kit/rdm/migration_config.py | 12 +++- .../rdm/records/transform/models/ship.py | 70 +++++++++++++++++++ .../xml_processing/rules/research.py | 1 + .../transform/xml_processing/rules/ship.py | 54 ++++++++++++++ cds_migrator_kit/rdm/streams.yaml | 34 +++++++++ setup.cfg | 6 ++ 6 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 cds_migrator_kit/rdm/records/transform/models/ship.py create mode 100644 cds_migrator_kit/rdm/records/transform/xml_processing/rules/ship.py diff --git a/cds_migrator_kit/rdm/migration_config.py b/cds_migrator_kit/rdm/migration_config.py index 51b226f7..55982a8a 100644 --- a/cds_migrator_kit/rdm/migration_config.py +++ b/cds_migrator_kit/rdm/migration_config.py @@ -551,7 +551,7 @@ def resolve_record_pid(pid): "label": "EP approval", # shown in UI buttons/headings "referee_group": "cds-ph-ep-publication", # CERN e-group slug "report_number": { - "prefix": "CERN-TH-EP", # literal prefix, e.g. "CERN-EP" + "prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP" "include_year": True, # append the current year after prefix "counter_digits": 3, # zero-padding width, e.g. 3 → "001" }, @@ -596,4 +596,14 @@ def resolve_record_pid(pid): "counter_digits": 3, # zero-padding width, e.g. 3 → "001" }, }, + "7277793b-5fce-458a-a3c4-e05a6cc43c69":{ + # ship + "label": "EP approval", # shown in UI buttons/headings + "referee_group": "cds-ph-ep-publication", # CERN e-group slug + "report_number": { + "prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP" + "include_year": True, # append the current year after prefix + "counter_digits": 3, # zero-padding width, e.g. 3 → "001" + }, + }, } diff --git a/cds_migrator_kit/rdm/records/transform/models/ship.py b/cds_migrator_kit/rdm/records/transform/models/ship.py new file mode 100644 index 00000000..144f34ed --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/models/ship.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM SHIP research model.""" + +from cds_migrator_kit.rdm.records.transform.models.research import ( + ResearchModel, + research_model, +) + + +class SHIPResearchModel(ResearchModel): + """Translation model for SHIP records.""" + + __query__ = ( + "((980__.a:NOTE OR 980__.a:Note OR 980__.a:ConferencePaper) AND 690C_.a:SHiP) OR " + "(980__:SCICOMMPUBLSPSC SHiP) OR " + "980__.a:SHiPPUBDRAFTFINAL OR 980__.a:SHiP_Papers OR " + "(980__.a:INTNOTE AND 693__.e:SHiP) " + "-980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY" + ) + + __ignore_keys__ = { + "100__v", # complete affiliation + "100__m", # email of contributor + "0247_9", # provenance of the DOI + "0248_a", + "0248_p", + "035__h", # oai identifiers in 2281295, 2802785 + "035__d", # oai identifiers in 2281295, 2802785 + "035__t", # oai identifiers in 2281295, 2802785 + "035__u", # oai identifiers in 2281295, 2802785 + "035__m", # oai identifiers in 2281295, 2802785 + "037__c", # arxiv subject + "110__u", # TODO: remove + "245__9", # title source + "270__m", # document contact email + "300__a", # number of pages + "500__9", # provenance of the note + "520__9", # provenance of the description + "542__3", # copyright material + "8564_8", # file id + "8564_s", # bibdoc id + "8564_x", # icon thumbnails sizes + "8564_y", # file description - done by files dump, sometimes these are used for open access calculation + "700__m", # email of contributor + "700__v", # complete affiliation + "773__o", # Duplicate meeting title + "78002r", # Report number of the related record + "78502r", # Report number of the related record + "8564_z", # automatic process with EP value:Stamped by WebSubmit + "903__s", # public + "905__q", # TODO: removespokesperson ... + "905__k", # TODO: remove spokesperson ... + "905__l", # TODO: remove spokesperson ... + "937__c", # last modified by + "937__s", # last modification date + "960__a", # base number + "981__a", # duplicate record id + } + + +ship_research_model = SHIPResearchModel( + bases=(research_model,), + entry_point_group="cds_migrator_kit.migrator.rdm.rules.ship", +) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index 3cb4794f..9cae8576 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -784,6 +784,7 @@ def resource_type(self, key, value): "slide": {"id": "presentation"}, "faser_papers": {"id": "publication-article"}, "demsuppliers": {"id": "other"}, + "ship_papers": {"id": "publication-article"}, } try: diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/ship.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/ship.py new file mode 100644 index 00000000..3aa568c3 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/ship.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM SHIP research rules.""" + +from dojson.errors import IgnoreKey +from dojson.utils import for_each_value + +from cds_migrator_kit.errors import UnexpectedValue +from cds_migrator_kit.transform.xml_processing.quality.parsers import StringValue + +from ...models.ship import ship_research_model as model +from .faser_publication import spokesperson + +model.over("request_reviewers", "(^905__|^906__)", override=True)(spokesperson) + + +@model.over("document_type", "^594__") +def document_type(self, key, value): + """Translates document type. TODO: can we ignore this?""" + document_type = value.get("a").strip().lower() + if document_type and document_type not in ["int", "conferencepaper", "note", ]: + raise UnexpectedValue(f"Invalid document type: {document_type}") + raise IgnoreKey("document_type") + + +@model.over("related_identifiers", "(^78502|^78002)") +@for_each_value +def related_works(self, key, value): + """Translates related identifiers.""" + description = StringValue(value.get("i")).parse().strip().lower() + recid = value.get("w") + rel_ids = self.get("related_identifiers", []) + if "superseded by" == description: + relation_type = "isobsoletedby" + elif "supersedes" == description: + relation_type = "obsoletes" + else: + raise UnexpectedValue(f"Invalid relation type: {description}") + new_id = { + "identifier": recid, + "scheme": "cds", + "relation_type": {"id": relation_type}, + "resource_type": {"id": "other"}, + } + if new_id not in rel_ids: + return new_id + + raise IgnoreKey("related_identifiers") + diff --git a/cds_migrator_kit/rdm/streams.yaml b/cds_migrator_kit/rdm/streams.yaml index eefe4a0c..49a9a065 100644 --- a/cds_migrator_kit/rdm/streams.yaml +++ b/cds_migrator_kit/rdm/streams.yaml @@ -575,3 +575,37 @@ records: missing_users: cds_migrator_kit/rdm/data/users communities_ids: - "d9c138bd-d39c-4797-a06c-c6f8db45f3ce" + ship: + data_dir: cds_migrator_kit/rdm/data/ship + create_inclusion_request: true + plots: true + extract: + dirpath: cds_migrator_kit/rdm/data/ship/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/ship/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "7277793b-5fce-458a-a3c4-e05a6cc43c69" + ship-restricted: + data_dir: cds_migrator_kit/rdm/data/ship-restricted + restricted: "True" + plots: true + access_grants_view: + - ship-collaboration-dynamic + extract: + dirpath: cds_migrator_kit/rdm/data/ship-restricted/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/ship-restricted/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "7277793b-5fce-458a-a3c4-e05a6cc43c69" + ship-ep: + plots: true + data_dir: cds_migrator_kit/rdm/data/ship + extract: + dirpath: cds_migrator_kit/rdm/data/ship-ep/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/ship/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "7277793b-5fce-458a-a3c4-e05a6cc43c69" diff --git a/setup.cfg b/setup.cfg index 41117bce..d4626314 100644 --- a/setup.cfg +++ b/setup.cfg @@ -73,6 +73,7 @@ cds_migrator_kit.migrator.models = lep_research = cds_migrator_kit.rdm.records.transform.models.lep:lep_research_model antares = cds_migrator_kit.rdm.records.transform.models.antares:antares_research_model lcd = cds_migrator_kit.rdm.records.transform.models.lcd:lcd_research_model + ship = cds_migrator_kit.rdm.records.transform.models.ship:ship_research_model research_comm_model = cds_migrator_kit.rdm.records.transform.models.research_committee:research_comm_model fap = cds_migrator_kit.rdm.records.transform.models.fap:fap_model ssn = cds_migrator_kit.rdm.records.transform.models.summer_student_report:sspn_model @@ -117,6 +118,11 @@ cds_migrator_kit.migrator.rdm.rules.research = base = cds_migrator_kit.transform.xml_processing.rules.base base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base research = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research +cds_migrator_kit.migrator.rdm.rules.ship = + base = cds_migrator_kit.transform.xml_processing.rules.base + base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base + research = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research + ship = cds_migrator_kit.rdm.records.transform.xml_processing.rules.ship cds_migrator_kit.migrator.rdm.rules.small_exp = base = cds_migrator_kit.transform.xml_processing.rules.base base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base