bug #16846 fix: binaires manquants à l'export alors que les métadonnées sont là

sedalib/src/main/java/fr/gouv/vitam/tools/sedalib/core/BinaryDataObject.java

@@ -370,6 +370,29 @@ private String undefined(String a) {
 
     // SEDA XML exporter
 
+    /**
+     * The length of the "content" package directory name.
+     */
+    private static final int CONTENT_PREFIX_LENGTH = 7;
+
+    /**
+     * Normalizes the package directory of an Uri declared in a manifest.
+     * <p>
+     * A manifest may name the binary directory in any case, "Content/ID13.txt" as well as
+     * "content/ID13.txt", and the SIP and DIP importers extract all of them into a lowercase
+     * "content" directory. The on disk path built from the Uri has to be normalized the same way,
+     * otherwise it points at a directory that doesn't exist on a case sensitive file system and the
+     * binary is unreachable while its metadata is there.
+     *
+     * @param uri the uri as declared in the manifest
+     * @return the uri with a lowercase content package directory
+     */
+    public static String normalizePackageUri(String uri) {
+        if (uri == null) return null;
+        if (uri.toLowerCase().startsWith("content")) return "content" + uri.substring(CONTENT_PREFIX_LENGTH);
+        return uri;
+    }
+
     private void finalizeUri() throws SEDALibException {
         FileInfo fileInfo = getMetadataFileInfo();
         String tmpUri = "content/" + inDataPackageObjectId;

sedalib/src/main/java/fr/gouv/vitam/tools/sedalib/core/DataObjectGroup.java

@@ -291,7 +291,9 @@ public static String idFromSedaXml(
                                 SedaContext.getVersion()
                             );
                             StringType bdoUri = (StringType) bdo.getFirstNamedMetadata("Uri");
-                            bdo.setOnDiskPathFromString(rootDir + File.separator + bdoUri.getValue());
+                            bdo.setOnDiskPathFromString(
+                                rootDir + File.separator + BinaryDataObject.normalizePackageUri(bdoUri.getValue())
+                            );
                             dog.addDataObject(bdo);
                             break;
                         case "PhysicalDataObject":

sedalib/src/main/java/fr/gouv/vitam/tools/sedalib/core/DataObjectPackage.java

@@ -38,6 +38,7 @@
 package fr.gouv.vitam.tools.sedalib.core;
 
 import fr.gouv.vitam.tools.sedalib.metadata.content.Content;
+import fr.gouv.vitam.tools.sedalib.metadata.data.FileInfo;
 import fr.gouv.vitam.tools.sedalib.metadata.namedtype.IntegerType;
 import fr.gouv.vitam.tools.sedalib.utils.SEDALibException;
 import fr.gouv.vitam.tools.sedalib.utils.SEDALibProgressLogger;
@@ -46,6 +47,8 @@
 
 import javax.xml.stream.XMLStreamException;
 import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.text.DecimalFormat;
 import java.util.*;
 import java.util.Map.Entry;
@@ -1216,7 +1219,11 @@ public static void importDataObjectPackageObjects(
                         break;
                     case "BinaryDataObject":
                         bdo = BinaryDataObject.fromSedaXml(xmlReader, dataObjectPackage, sedaLibProgressLogger);
-                        bdo.setOnDiskPathFromString(rootDir + File.separator + bdo.getMetadataUri().getValue());
+                        bdo.setOnDiskPathFromString(
+                            rootDir +
+                            File.separator +
+                            BinaryDataObject.normalizePackageUri(bdo.getMetadataUri().getValue())
+                        );
                         doProgressLog(
                             sedaLibProgressLogger,
                             SEDALibProgressLogger.OBJECTS,
@@ -1551,6 +1558,75 @@ public void setManagementMetadataXmlData(String managementMetadataXmlData) {
         this.managementMetadataXmlData = managementMetadataXmlData;
     }
 
+    /**
+     * Lists the BinaryDataObjects whose binary file can't be read on disk.
+     * <p>
+     * A BinaryDataObject carries its metadata and, apart from them, the path of the file holding its
+     * content. Nothing guarantees that both agree: a SIP import sets that path from the Uri declared
+     * in the manifest without verifying that the file was indeed in the package, and a work can be
+     * saved and reloaded long after the imported files have moved. The descriptive metadata is then
+     * complete while the binary is nowhere to be found, which is only discovered when the export
+     * tries to open it.
+     *
+     * @return one description per unreadable BinaryDataObject, empty if all of them can be read
+     */
+    private static final int MAX_LISTED_UNREADABLE = 20;
+
+    public List<String> getUnreadableBinaryDataObjectDescriptions() {
+        List<String> descriptions = new ArrayList<>();
+        for (DataObjectGroup dog : dogInDataObjectPackageIdMap.values()) {
+            if (dog.getBinaryDataObjectList() == null) continue;
+            for (BinaryDataObject bdo : dog.getBinaryDataObjectList()) {
+                String problem = getUnreadableDescription(bdo);
+                if (problem != null) descriptions.add(problem);
+            }
+        }
+        return descriptions;
+    }
+
+    /**
+     * Verifies that every BinaryDataObject file can be read, and tells which ones can't if any.
+     *
+     * @throws SEDALibException if at least one BinaryDataObject file is missing or unreadable, with
+     *                          all of them listed in the message
+     */
+    public void verifyBinaryDataObjectFilesAreReadable() throws SEDALibException {
+        List<String> problems = getUnreadableBinaryDataObjectDescriptions();
+        if (problems.isEmpty()) return;
+
+        StringBuilder message = new StringBuilder(
+            problems.size() +
+            " fichier(s) binaire(s) sont introuvables ou illisibles alors que leurs métadonnées sont présentes:"
+        );
+        for (String problem : problems.subList(0, Math.min(problems.size(), MAX_LISTED_UNREADABLE))) {
+            message.append("\n  - ").append(problem);
+        }
+        if (problems.size() > MAX_LISTED_UNREADABLE) {
+            message
+                .append("\n  - ... et ")
+                .append(problems.size() - MAX_LISTED_UNREADABLE)
+                .append(" autre(s), voir le journal pour la liste complète");
+        }
+        throw new SEDALibException(message.toString());
+    }
+
+    private static String getUnreadableDescription(BinaryDataObject bdo) {
+        String filename = null;
+        FileInfo fileInfo = bdo.getMetadataFileInfo();
+        if (fileInfo != null) filename = fileInfo.getSimpleMetadata("Filename");
+        String identification =
+            "BinaryDataObject [" +
+            bdo.getInDataObjectPackageId() +
+            "]" +
+            (filename == null ? "" : " [" + filename + "]");
+
+        Path path = bdo.getOnDiskPath();
+        if (path == null) return identification + " n'a pas de fichier associé";
+        if (!Files.exists(path)) return identification + " a pour fichier [" + path + "] qui n'existe pas";
+        if (!Files.isReadable(path)) return identification + " a pour fichier [" + path + "] qui n'est pas lisible";
+        return null;
+    }
+
     /**
      * Gets export metadata list.
      *

sedalib/src/main/java/fr/gouv/vitam/tools/sedalib/inout/exporter/ArchiveTransferToSIPExporter.java

@@ -191,6 +191,24 @@ public String getSEDAXMLManifest(boolean hierarchicalFlag, boolean indentedFlag)
         return result;
     }
 
+    /**
+     * Verifies that every BinaryDataObject file can be read before writing anything.
+     * <p>
+     * Without it the export stops on the first unreadable file, leaving a truncated SIP behind and
+     * naming only that one file, so a package missing many binaries has to be exported as many times
+     * to discover them all. All of them are listed here in one pass, and the SIP is not started at
+     * all when one is missing.
+     *
+     * @throws SEDALibException if at least one BinaryDataObject file is missing or unreadable
+     */
+    private void verifyAllBinaryDataObjectFilesAreReadable() throws SEDALibException {
+        try {
+            archiveTransfer.getDataObjectPackage().verifyBinaryDataObjectFilesAreReadable();
+        } catch (SEDALibException e) {
+            throw new SEDALibException("Export du SIP impossible, " + e.getMessage());
+        }
+    }
+
     /**
      * Do export the ArchiveTransfer to SEDA Submission Information Packet (SIP).
      *
@@ -215,6 +233,8 @@ public void doExportToSEDASIP(String fileName, boolean hierarchicalFlag, boolean
         this.indentedFlag = indentedFlag;
         this.manifestOnly = false;
 
+        verifyAllBinaryDataObjectFilesAreReadable();
+
         try {
             Files.createDirectories(Paths.get(fileName).toAbsolutePath().getParent());
         } catch (IOException e1) {

sedalib/src/main/java/fr/gouv/vitam/tools/sedalib/inout/exporter/DataObjectPackageToDiskExporter.java

@@ -596,6 +596,14 @@ private void exportArchiveUnit(ArchiveUnit au, Path containerPath) throws SEDALi
      * @throws InterruptedException if export process is interrupted
      */
     public void doExport(String directoryName) throws SEDALibException, InterruptedException {
+        // all the unreadable binaries are named at once, and nothing is written when there is one,
+        // rather than stopping on the first one with a half written hierarchy left behind
+        try {
+            dataObjectPackage.verifyBinaryDataObjectFilesAreReadable();
+        } catch (SEDALibException e) {
+            throw new SEDALibException("Export sur disque impossible, " + e.getMessage());
+        }
+
         Path exportPath = Paths.get(directoryName);
         try {
             Files.createDirectories(exportPath);

sedalib/src/main/java/fr/gouv/vitam/tools/sedalib/inout/importer/DIPToArchiveDeliveryRequestReplyImporter.java

@@ -38,6 +38,7 @@
 package fr.gouv.vitam.tools.sedalib.inout.importer;
 
 import fr.gouv.vitam.tools.sedalib.core.ArchiveDeliveryRequestReply;
+import fr.gouv.vitam.tools.sedalib.core.BinaryDataObject;
 import fr.gouv.vitam.tools.sedalib.utils.SEDALibException;
 import fr.gouv.vitam.tools.sedalib.utils.SEDALibProgressLogger;
 import fr.gouv.vitam.tools.sedalib.xml.SEDAXMLEventReader;
@@ -122,9 +123,9 @@ public String unZipDip(String zipFile, String outputFolder) throws SEDALibExcept
             ArchiveEntry ze;
             while ((ze = zais.getNextEntry()) != null) {
                 String fileName = ze.getName().trim();
-                // change any case ConTenT to lowercase content on import as in fromSEDA in
-                // BinaryDataObject
-                if (fileName.toLowerCase().startsWith("content")) fileName = "content" + fileName.substring(7);
+                // change any case ConTenT to lowercase content on import, the on disk path built
+                // from the manifest Uri is normalized the same way
+                fileName = BinaryDataObject.normalizePackageUri(fileName);
 
                 Path newPath = Paths.get(outputFolder + File.separator + fileName);
 

sedalib/src/main/java/fr/gouv/vitam/tools/sedalib/inout/importer/DiskToArchiveTransferImporter.java

@@ -346,6 +346,9 @@ public String getSummary() {
                 result += "encodé selon un modèle hybride V1/V2 de la structure\n";
                 break;
         }
+        if (diskToDataObjectPackageImporter.getIgnoredFileCount() > 0) result +=
+        diskToDataObjectPackageImporter.getIgnoredFileCount() +
+        " fichier(s) ignoré(s) car correspondant à un motif d'exclusion, voir le journal\n";
         if ((start != null) && (end != null)) result +=
         "chargé en " + Duration.between(start, end).toString().substring(2) + "\n";
         return result;

sedalib/src/main/java/fr/gouv/vitam/tools/sedalib/inout/importer/DiskToDataObjectPackageImporter.java

@@ -202,6 +202,20 @@ public class DiskToDataObjectPackageImporter {
      */
     private int inCounter;
 
+    /**
+     * The number of files skipped because they match one of the ignore patterns.
+     */
+    private int ignoredFileCount;
+
+    /**
+     * Gets the number of files skipped because they match one of the ignore patterns.
+     *
+     * @return the ignored file count
+     */
+    public int getIgnoredFileCount() {
+        return ignoredFileCount;
+    }
+
     /**
      * The start and end instants, for duration computation.
      */
@@ -245,6 +259,7 @@ private DiskToDataObjectPackageImporter(
         else this.extractTitleFromFileNameFunction = simpleCopy;
 
         this.inCounter = 0;
+        this.ignoredFileCount = 0;
         this.sedaLibProgressLogger = sedaLibProgressLogger;
     }
 
@@ -698,6 +713,15 @@ private ArchiveUnit processDirectory(Path path) throws SEDALibException, Interru
                 fileName = curPath.getFileName().toString();
 
                 if (!Files.isDirectory(curPath) && mustBeIgnored(fileName)) {
+                    // an ignored file used to disappear without a word, so a package silently missing
+                    // binaries gave the user nothing to go on
+                    ignoredFileCount++;
+                    doProgressLog(
+                        sedaLibProgressLogger,
+                        SEDALibProgressLogger.OBJECTS,
+                        "sedalib: fichier [" + curPathString + "] ignoré, il correspond à un motif d'exclusion",
+                        null
+                    );
                     continue;
                 } else if (analyzeLink(curPath)) {
                     if (noLinkFlag) continue;

sedalib/src/main/java/fr/gouv/vitam/tools/sedalib/inout/importer/SIPToArchiveTransferImporter.java

@@ -38,6 +38,7 @@
 package fr.gouv.vitam.tools.sedalib.inout.importer;
 
 import fr.gouv.vitam.tools.sedalib.core.ArchiveTransfer;
+import fr.gouv.vitam.tools.sedalib.core.BinaryDataObject;
 import fr.gouv.vitam.tools.sedalib.utils.SEDALibException;
 import fr.gouv.vitam.tools.sedalib.utils.SEDALibProgressLogger;
 import fr.gouv.vitam.tools.sedalib.xml.SEDAXMLEventReader;
@@ -57,6 +58,7 @@
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Date;
+import java.util.List;
 
 import static fr.gouv.vitam.tools.sedalib.utils.SEDALibProgressLogger.doProgressLog;
 import static fr.gouv.vitam.tools.sedalib.utils.SEDALibProgressLogger.doProgressLogIfStep;
@@ -84,6 +86,11 @@ public class SIPToArchiveTransferImporter {
      */
     private ArchiveTransfer archiveTransfer;
 
+    /**
+     * The number of BinaryDataObjects of the manifest whose file was not in the package.
+     */
+    private int unreadableBinaryDataObjectCount;
+
     /**
      * The end.
      */
@@ -121,9 +128,9 @@ public String unZipSip(String zipFile, String outputFolder) throws SEDALibExcept
             ArchiveEntry ze;
             while ((ze = zais.getNextEntry()) != null) {
                 String fileName = ze.getName().trim();
-                // change any case ConTenT to lowercase content on import as in fromSEDA in
-                // BinaryDataObject
-                if (fileName.toLowerCase().startsWith("content")) fileName = "content" + fileName.substring(7);
+                // change any case ConTenT to lowercase content on import, the on disk path built
+                // from the manifest Uri is normalized the same way
+                fileName = BinaryDataObject.normalizePackageUri(fileName);
 
                 Path newPath = Paths.get(outputFolder + File.separator + fileName);
 
@@ -244,10 +251,40 @@ public void doImport() throws SEDALibException, InterruptedException {
             throw new SEDALibException("Impossible d'importer le fichier [" + manifest + "] comme manifest du SIP", e);
         }
 
+        logUnreadableBinaryDataObjects();
+
         end = Instant.now();
         doProgressLog(sedaLibProgressLogger, SEDALibProgressLogger.GLOBAL, "sedalib: import du SIP terminé", null);
     }
 
+    /**
+     * Warns about the BinaryDataObjects of the manifest whose file is not in the package.
+     * <p>
+     * The on disk path of a BinaryDataObject is built from the Uri declared in the manifest, without
+     * checking that the file was really there. An incomplete SIP is thus imported without a word, and
+     * the missing binaries are only discovered much later, when the export fails on them. Saying it at
+     * import time is what lets the discrepancy be traced back to the source package.
+     */
+    private void logUnreadableBinaryDataObjects() throws InterruptedException {
+        List<String> problems = archiveTransfer.getDataObjectPackage().getUnreadableBinaryDataObjectDescriptions();
+        if (problems.isEmpty()) return;
+
+        unreadableBinaryDataObjectCount = problems.size();
+        doProgressLog(
+            sedaLibProgressLogger,
+            SEDALibProgressLogger.GLOBAL,
+            "sedalib: attention, " +
+            problems.size() +
+            " fichier(s) binaire(s) déclaré(s) dans le manifest sont absents du SIP [" +
+            zipFile +
+            "], les métadonnées sont importées mais l'export échouera tant qu'ils manqueront",
+            null
+        );
+        for (String problem : problems) {
+            doProgressLog(sedaLibProgressLogger, SEDALibProgressLogger.STEP, "sedalib: " + problem, null);
+        }
+    }
+
     /**
      * Gets the archive transfer.
      *
@@ -266,6 +303,10 @@ public String getSummary() {
         String result;
 
         result = archiveTransfer.getDescription() + "\n";
+        if (unreadableBinaryDataObjectCount > 0) result +=
+        "attention, " +
+        unreadableBinaryDataObjectCount +
+        " fichier(s) binaire(s) déclaré(s) dans le manifest sont absents du SIP, voir le journal\n";
         if (start != null) result += "chargé en " + Duration.between(start, end).toString().substring(2) + "\n";
         return result;
     }