Ab department

cds_migrator_kit/rdm/records/transform/transform.py

@@ -14,18 +14,22 @@
 
 import arrow
 import yaml
+from cds_dojson.marc21.utils import create_record
 from cds_rdm.legacy.models import CDSMigrationAffiliationMapping
+from cds_rdm.legacy.resolver import get_pid_by_legacy_recid
 from dateutil.parser import ParserError, parse
 from flask import current_app
 from idutils import normalize_ror
 from idutils.validators import is_doi, is_ror
+from invenio_access.permissions import system_identity
 from invenio_accounts.models import User, UserIdentity
 from invenio_db import db
 from invenio_pidstore.models import PersistentIdentifier, PIDStatus
 from invenio_rdm_migrator.streams.records.transform import (
     RDMRecordEntry,
     RDMRecordTransform,
 )
+from invenio_rdm_records.proxies import current_rdm_records_service, current_record_communities_service
 from invenio_vocabularies.contrib.affiliations.models import AffiliationsMetadata
 from invenio_vocabularies.contrib.names.models import NamesMetadata
 from sqlalchemy.exc import NoResultFound
@@ -622,7 +626,7 @@ def field_departments(record_json, custom_fields_dict):
                         value=department,
                         field="department",
                         message=f"Department {department} not found. "
-                        f"Added as unit and subject",
+                                f"Added as unit and subject",
                         stage="vocabulary match",
                     )
 
@@ -907,12 +911,12 @@ def _transform(self, entry):
                     "_clc_sync": clc_sync,
                 }
         except (
-            LossyConversion,
-            RestrictedFileDetected,
-            UnexpectedValue,
-            ManualImportRequired,
-            MissingRequiredField,
-            MultipleModelsMatched,
+                LossyConversion,
+                RestrictedFileDetected,
+                UnexpectedValue,
+                ManualImportRequired,
+                MissingRequiredField,
+                MultipleModelsMatched,
         ) as e:
             migration_logger.add_log(e, record=entry)
 
@@ -1014,7 +1018,7 @@ def compute_files(file_dump, versions_dict):
                 {
                     file_dump["full_name"]: {
                         "eos_tmp_path": tmp_eos_root
-                        / full_path.relative_to(legacy_path_root),
+                                        / full_path.relative_to(legacy_path_root),
                         "id_bibdoc": file_dump["bibdocid"],
                         "key": file_dump["full_name"],
                         "metadata": {
@@ -1100,17 +1104,77 @@ def _load_migrated_recids(self):
     def should_skip(self, entry):
         return str(entry["recid"]) in self._migrated_recids
 
+    def _existing_record_is_restricted(self, record_id):
+        """Check the current access state of an already-migrated RDM record.
+
+        Reads live from the record service (DB) rather than the legacy
+        MARCXML, since access restrictions may have been changed in RDM
+        after migration and the legacy record data would be stale.
+        """
+        record = current_rdm_records_service.read_latest(
+            system_identity, id_=record_id
+        )
+        access = record.data.get("access", {})
+        return access.get("record") != "public" or access.get("files") != "public"
+
     def run(self, entries):
         """Run transformation step."""
         self._migrated_recids = self._load_migrated_recids()
-
         for entry in entries:
             if self.should_skip(entry):
                 recid = entry["recid"]
+                try:
+                    parent_pid = get_pid_by_legacy_recid(str(recid))
+                    # we don't check here if the record has 980:MIGRATED
+                    # because this does not add anything and it should not be deciding factor.
+                    # if the legacy recid has been minted - we know already the record has been migrated
+                    if self._existing_record_is_restricted(parent_pid.pid_value):
+                        # checking if we need to be more careful while assigning access
+                        # to various communities
+                        raise ManualImportRequired(
+                            message=(
+                                "Existing record is restricted or has "
+                                "restricted files; not adding to "
+                                "communities automatically"
+                            ),
+                            field="access",
+                            stage="transform",
+                            recid=recid,
+                            priority="warning",
+                        )
+                    # bulk_add resolves record_ids via RDMRecord.pid.resolve(),
+                    # which -- unlike read_latest -- has no fallback for a
+                    # parent-level recid (what get_pid_by_legacy_recid returns).
+                    # We must pass the record's own recid instead.
+                    record_item = current_rdm_records_service.read_latest(
+                        system_identity, id_=parent_pid.pid_value
+                    )
+                    for community_id in self.communities_ids:
+                        current_record_communities_service.bulk_add(
+                            system_identity,
+                            community_id,
+                            [record_item.id],
+                        )
+                except NoResultFound:
+                    self.migration_logger.add_information(
+                        recid,
+                        {
+                            "message": (
+                                "Problem with PIDs - minted legacy recid but"
+                                "no corresponding parent record found."
+                            ),
+                            "value": recid,
+                        },
+                    )
+                except ManualImportRequired as exc:
+                    self.migration_logger.add_log(exc, record=entry)
+
                 self.migration_logger.add_information(
                     recid,
                     {
-                        "message": "Record already migrated, skipping",
+                        "message": "Record already migrated, skipping,"
+                                   " added existing {} to communities {}".format(
+                            recid, self.communities_ids),
                         "value": recid,
                     },
                 )

cds_migrator_kit/rdm/streams.yaml

@@ -458,6 +458,160 @@ records:
         - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
     load:
       legacy_pids_to_redirect: cds_migrator_kit/rdm/data/dep_ab/duplicated_pids.json
+  ac_dep:
+    data_dir: cds_migrator_kit/rdm/data/dep_ac
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/dep_ac/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/dep_ac/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "dfebd855-d156-4428-9b08-fe8ab8171799"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
+    load:
+      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/dep_ab/duplicated_pids.json
+  ar_dep:
+    data_dir: cds_migrator_kit/rdm/data/dep_ar
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/dep_ar/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/dep_ar/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "2bf2998b-b310-4f36-9344-71e47560ae16"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
+    load:
+      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/dep_ar/duplicated_pids.json
+  isr_dep:
+    data_dir: cds_migrator_kit/rdm/data/dep_isr
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/dep_isr/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/dep_isr/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "c2fdb4ec-201c-4ec7-a9f0-23e4380a3fce"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
+    load:
+      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/dep_ab/duplicated_pids.json
+  lep_dep:
+    data_dir: cds_migrator_kit/rdm/data/dep_lep
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/dep_lep/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/dep_lep/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "f49a960d-94bf-432c-9ce6-c2fb9115ff9a"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
+    load:
+      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/dep_lep/duplicated_pids.json
+  ps_dep:
+    data_dir: cds_migrator_kit/rdm/data/dep_ps
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/dep_ps/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/dep_ps/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "5b634108-c93c-4740-af5c-31fb32e548eb"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
+    load:
+      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/dep_ps/duplicated_pids.json
+  sc_dep:
+    data_dir: cds_migrator_kit/rdm/data/dep_sc
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/dep_sc/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/dep_sc/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "369f4c3d-10fc-41e3-8cce-681fad5fcafe"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
+    load:
+      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/dep_sc/duplicated_pids.json
+  si_dep:
+    data_dir: cds_migrator_kit/rdm/data/dep_si
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/dep_si/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/dep_si/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "db44a69b-babc-48e8-9c03-29436b5bff07"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
+    load:
+      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/dep_si/duplicated_pids.json
+  sl_dep:
+    data_dir: cds_migrator_kit/rdm/data/dep_sl
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/dep_sl/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/dep_sl/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "bcc34ae6-3a02-44cd-b08d-316a9d3e27e6"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
+    load:
+      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/dep_sl/duplicated_pids.json
+  sps_dep:
+    data_dir: cds_migrator_kit/rdm/data/dep_sps
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/dep_sps/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/dep_sps/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "4bdb0cb3-b5f0-4383-a621-44b6e287438a"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
+    load:
+      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/dep_sps/duplicated_pids.json
+  mps_div:
+    data_dir: cds_migrator_kit/rdm/data/div_mps
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/div_mps/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/div_mps/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "510206fb-2f80-44ee-8944-57f5dcfae5f0"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
+    load:
+      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/div_mps/duplicated_pids.json
+  msc_div:
+    data_dir: cds_migrator_kit/rdm/data/div_msc
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/div_msc/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/div_msc/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "7bd98561-4985-434a-aed1-74301f865c24"
+        - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f"
+    load:
+      legacy_pids_to_redirect: cds_migrator_kit/rdm/data/div_msc/duplicated_pids.json
 comments:
   faser-drafts:
     dir_path: /migration/faser-drafts/comments/

setup.cfg

@@ -17,7 +17,7 @@ zip_safe = False
 install_requires =
     sentry-sdk>=1.45,<2.0.0
     cds-dojson @ git+https://github.com/CERNDocumentServer/cds-dojson@master#egg=cds-dojson
-    invenio-rdm-migrator>=5.0.0
+    invenio-rdm-migrator @ git+https://github.com/CERNDocumentServer/invenio-rdm-migrator@master#egg=invenio-rdm-migrator
     lxml>=4.6.5
     ipython!=8.1.0
     uwsgi>=2.0
@@ -32,11 +32,10 @@ install_requires =
 
 [options.extras_require]
 rdm =
-    invenio-app-rdm[opensearch2]==14.0.0b10.dev4
+    invenio-app-rdm[opensearch2]>=v15.0.0b2.dev0
     cds-rdm @ git+https://github.com/CERNDocumentServer/cds-rdm@master#egg=cds-rdm&subdirectory=site
-    invenio-preservation-sync==0.3.0
-    invenio-cern-sync @ git+https://github.com/CERNDocumentServer/invenio-cern-sync@v0.6.0#egg=invenio-cern-sync
-    invenio-rdm-migrator @ git+https://github.com/CERNDocumentServer/invenio-rdm-migrator@master#egg=invenio-rdm-migrator
+    invenio-preservation-sync==0.5.0
+    invenio-cern-sync @ git+https://github.com/CERNDocumentServer/invenio-cern-sync@main#egg=invenio-cern-sync
     invenio-query-parser @ git+https://github.com/CERNDocumentServer/invenio-query-parser@master#egg=invenio-query-parser
 
 videos =