Skip to content

bug #16848 fix: accents cassés dans les exports csv des métadonnées #116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Regzox wants to merge 2 commits into
base: master
Choose a base branch
from
Open

bug #16848 fix: accents cassés dans les exports csv des métadonnées #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7418d2f
bugs #16842 #16843 #16845 fix: import interminable, une attente d'1ms…
Regzox Sep 10, 2026
b2085c6
bug #16848 fix: accents cassés dans les exports csv des métadonnées
Regzox Sep 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 28 additions & 3 deletions resip/src/main/java/fr/gouv/vitam/tools/resip/frame/ExportContextDialog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
package fr.gouv.vitam.tools.resip.frame;

import fr.gouv.vitam.tools.resip.app.ResipGraphicApp;
import fr.gouv.vitam.tools.resip.frame.preferences.PreferencesDialog;
import fr.gouv.vitam.tools.resip.parameters.ExportContext;
import fr.gouv.vitam.tools.resip.parameters.Preferences;
import fr.gouv.vitam.tools.sedalib.core.GlobalMetadata;
Expand Down Expand Up @@ -89,6 +90,7 @@ public class ExportContextDialog extends JDialog {
private JRadioButton allUsageButton;
private JTextField nameMaxSizeTextField;
private JCheckBox csvExtendedFormatChexBox;
private JComboBox<String> csvExportCharsetCombobox;
private JTextArea metadataFilterTextArea;
private JCheckBox metadataFilterCheckBox;

Expand Down Expand Up @@ -708,6 +710,28 @@ public ExportContextDialog(JFrame owner, ExportContext exportContext) {
exportParametersPanel.add(csvExtendedFormatChexBox, gbc);
csvExtendedFormatChexBox.setSelected(exportContext.isCsvExtendedFormat());

JLabel csvExportCharsetLabel = new JLabel("Encodage du csv exporté :");
gbc = new GridBagConstraints();
gbc.anchor = GridBagConstraints.EAST;
gbc.insets = new Insets(0, 5, 5, 5);
gbc.gridx = 0;
gbc.gridy = 8;
exportParametersPanel.add(csvExportCharsetLabel, gbc);

csvExportCharsetCombobox = new JComboBox<>(PreferencesDialog.CHARSET_STRINGS);
csvExportCharsetCombobox.setFont(MainWindow.LABEL_FONT);
csvExportCharsetCombobox.setToolTipText(
"Encodage des csv produits à l'export, indépendant de celui attendu à l'import. " +
"UTF-8 est écrit avec un BOM, ce qu'Excel sous Windows sait lire."
);
gbc = new GridBagConstraints();
gbc.insets = new Insets(0, 0, 5, 5);
gbc.fill = GridBagConstraints.HORIZONTAL;
gbc.gridx = 1;
gbc.gridy = 8;
exportParametersPanel.add(csvExportCharsetCombobox, gbc);
csvExportCharsetCombobox.setSelectedItem(exportContext.getCsvCharsetName());

JLabel metadataFilterLabel = new JLabel("Filtrage des métadonnées");
metadataFilterLabel.setFont(MainWindow.BOLD_LABEL_FONT);
gbc = new GridBagConstraints();
Expand All @@ -718,7 +742,7 @@ public ExportContextDialog(JFrame owner, ExportContext exportContext) {
gbc.anchor = GridBagConstraints.NORTHWEST;
gbc.fill = GridBagConstraints.BOTH;
gbc.gridx = 0;
gbc.gridy = 8;
gbc.gridy = 9;
exportParametersPanel.add(metadataFilterLabel, gbc);

scrollPane = new JScrollPane();
Expand All @@ -729,7 +753,7 @@ public ExportContextDialog(JFrame owner, ExportContext exportContext) {
gbc.fill = GridBagConstraints.BOTH;
gbc.insets = new Insets(0, 5, 5, 5);
gbc.gridx = 1;
gbc.gridy = 9;
gbc.gridy = 10;
exportParametersPanel.add(scrollPane, gbc);

metadataFilterTextArea = new JTextArea();
Expand All @@ -750,7 +774,7 @@ public ExportContextDialog(JFrame owner, ExportContext exportContext) {
gbc.anchor = GridBagConstraints.EAST;
gbc.insets = new Insets(0, 5, 5, 5);
gbc.gridx = 0;
gbc.gridy = 9;
gbc.gridy = 10;
gbc.weighty = 1.0;
exportParametersPanel.add(metadataFilterCheckBox, gbc);
metadataFilterCheckBox.addItemListener(this::metadataFilterEvent);
Expand Down Expand Up @@ -834,6 +858,7 @@ public boolean extractFromDialog() {
}
gmc.setMaxNameSize(tmp);
gmc.setCsvExtendedFormat(csvExtendedFormatChexBox.isSelected());
gmc.setCsvCharsetName((String) csvExportCharsetCombobox.getSelectedItem());
gmc.setManagementMetadataXmlData(managementMetadataTextArea.getText());
gmc.setMetadataFilterFlag(metadataFilterCheckBox.isSelected());
gmc.setKeptMetadataList(
Expand Down
38 changes: 31 additions & 7 deletions resip/src/main/java/fr/gouv/vitam/tools/resip/frame/preferences/PreferencesDialog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ public class PreferencesDialog extends JDialog {
private final JRadioButton allUsageButton;
private final JTextField nameMaxSizeTextField;
private final JCheckBox csvExtendedFormatChexBox;
private final JComboBox<String> csvExportCharsetCombobox;
private final JRadioButton reindexYesRadioButton;
private final JTextArea metadataFilterTextArea;
private final JCheckBox metadataFilterCheckBox;
Expand Down Expand Up @@ -172,9 +173,9 @@ public class PreferencesDialog extends JDialog {
private int returnValue;

/**
* The proposed charsets.
* The proposed charsets, shared with the export context dialog.
*/
private static final String[] charsetStrings = {
public static final String[] CHARSET_STRINGS = {
"windows-1252",
"ISO-8859-1",
"UTF-8",
Expand Down Expand Up @@ -825,6 +826,28 @@ public PreferencesDialog(JFrame owner) {
exportParametersPanel.add(csvExtendedFormatChexBox, gbc);
csvExtendedFormatChexBox.setSelected(gmc.isCsvExtendedFormat());

JLabel csvExportCharsetLabel = new JLabel("Encodage du csv exporté :");
gbc = new GridBagConstraints();
gbc.anchor = GridBagConstraints.EAST;
gbc.insets = new Insets(0, 5, 5, 5);
gbc.gridx = 0;
gbc.gridy = 10;
exportParametersPanel.add(csvExportCharsetLabel, gbc);

csvExportCharsetCombobox = new JComboBox<>(CHARSET_STRINGS);
csvExportCharsetCombobox.setFont(MainWindow.LABEL_FONT);
csvExportCharsetCombobox.setToolTipText(
"Encodage des csv produits à l'export, indépendant de celui attendu à l'import. " +
"UTF-8 est écrit avec un BOM, ce qu'Excel sous Windows sait lire."
);
gbc = new GridBagConstraints();
gbc.insets = new Insets(0, 0, 5, 5);
gbc.fill = GridBagConstraints.HORIZONTAL;
gbc.gridx = 1;
gbc.gridy = 10;
exportParametersPanel.add(csvExportCharsetCombobox, gbc);
csvExportCharsetCombobox.setSelectedItem(gmc.getCsvCharsetName());

JLabel metadataFilterLabel = new JLabel("Filtrage des métadonnées");
metadataFilterLabel.setFont(MainWindow.BOLD_LABEL_FONT);
gbc = new GridBagConstraints();
Expand All @@ -834,7 +857,7 @@ public PreferencesDialog(JFrame owner) {
gbc.weighty = 0.0;
gbc.anchor = GridBagConstraints.NORTHWEST;
gbc.gridx = 0;
gbc.gridy = 10;
gbc.gridy = 11;
exportParametersPanel.add(metadataFilterLabel, gbc);

scrollPane = new JScrollPane();
Expand All @@ -845,7 +868,7 @@ public PreferencesDialog(JFrame owner) {
gbc.fill = GridBagConstraints.BOTH;
gbc.insets = new Insets(0, 5, 5, 5);
gbc.gridx = 1;
gbc.gridy = 11;
gbc.gridy = 12;
exportParametersPanel.add(scrollPane, gbc);

metadataFilterTextArea = new JTextArea();
Expand All @@ -866,7 +889,7 @@ public PreferencesDialog(JFrame owner) {
gbc.anchor = GridBagConstraints.EAST;
gbc.insets = new Insets(0, 5, 5, 5);
gbc.gridx = 0;
gbc.gridy = 11;
gbc.gridy = 12;
gbc.weighty = 1.0;
gbc.fill = GridBagConstraints.BOTH;
exportParametersPanel.add(metadataFilterCheckBox, gbc);
Expand Down Expand Up @@ -904,7 +927,7 @@ public PreferencesDialog(JFrame owner) {
gbc.gridy = 1;
importParametersPanel.add(mailCharsetLabel, gbc);

defaultMailCharsetCombobox = new JComboBox<>(charsetStrings);
defaultMailCharsetCombobox = new JComboBox<>(CHARSET_STRINGS);
defaultMailCharsetCombobox.setFont(MainWindow.LABEL_FONT);
gbc = new GridBagConstraints();
gbc.insets = new Insets(0, 0, 5, 5);
Expand Down Expand Up @@ -1057,7 +1080,7 @@ public PreferencesDialog(JFrame owner) {
gbc.gridy = 9;
importParametersPanel.add(csvCharsetLabel, gbc);

csvCharsetCombobox = new JComboBox<>(charsetStrings);
csvCharsetCombobox = new JComboBox<>(CHARSET_STRINGS);
csvCharsetCombobox.setFont(MainWindow.LABEL_FONT);
gbc = new GridBagConstraints();
gbc.insets = new Insets(0, 0, 5, 5);
Expand Down Expand Up @@ -1646,6 +1669,7 @@ private boolean extractFromDialog() {
}
gmc.setMaxNameSize(tmp);
gmc.setCsvExtendedFormat(csvExtendedFormatChexBox.isSelected());
gmc.setCsvCharsetName((String) csvExportCharsetCombobox.getSelectedItem());
gmc.setManagementMetadataXmlData(managementMetadataTextArea.getText());
gmc.setMetadataFilterFlag(metadataFilterCheckBox.isSelected());
gmc.setKeptMetadataList(
Expand Down
45 changes: 45 additions & 0 deletions resip/src/main/java/fr/gouv/vitam/tools/resip/parameters/ExportContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ public class ExportContext {
/** The csv extended format flag. */
private boolean csvExtendedFormat;

/** The csv export charset name. */
private String csvCharsetName;

/** The ManagementMetadata. */
private String managementMetadataXmlData;

Expand All @@ -106,6 +109,7 @@ public ExportContext() {
this.usageVersionSelectionMode = LAST_DATAOBJECT;
this.maxNameSize = 32;
this.csvExtendedFormat = true;
this.csvCharsetName = getDefaultCsvCharsetName();
this.globalMetadata = new GlobalMetadata();
this.managementMetadataXmlData = null;
this.onDiskOutput = null;
Expand Down Expand Up @@ -146,6 +150,7 @@ public ExportContext(
this.usageVersionSelectionMode = usageVersionSelectionMode;
this.maxNameSize = maxNameSize;
this.csvExtendedFormat = csvExtendedFormat;
this.csvCharsetName = getDefaultCsvCharsetName();
this.globalMetadata = globalMetadata;
this.managementMetadataXmlData = managementMetadataXmlData;
this.setOnDiskOutput(onDiskOutput);
Expand Down Expand Up @@ -173,6 +178,7 @@ public ExportContext(String sipExportFileName) throws ResipException {
this.usageVersionSelectionMode = sec.usageVersionSelectionMode;
this.maxNameSize = sec.maxNameSize;
this.csvExtendedFormat = sec.csvExtendedFormat;
this.csvCharsetName = (sec.csvCharsetName == null ? getDefaultCsvCharsetName() : sec.csvCharsetName);
this.globalMetadata = sec.globalMetadata;
this.managementMetadataXmlData = sec.managementMetadataXmlData;
this.metadataFilterFlag = sec.metadataFilterFlag;
Expand Down Expand Up @@ -235,6 +241,8 @@ public ExportContext(Preferences preferences) {
csvExtendedFormat = Boolean.parseBoolean(
preferences.getPrefProperties().getProperty("exportContext.csvExport.csvExtendedFormat", "true")
);
csvCharsetName = preferences.getPrefProperties().getProperty("exportContext.csvExport.charsetName", "");
if (csvCharsetName.isEmpty()) csvCharsetName = getDefaultCsvCharsetName();
managementMetadataXmlData = nullIfEmpty(
preferences.getPrefProperties().getProperty("exportContext.general.managementMetadataXmlData", "")
);
Expand Down Expand Up @@ -320,6 +328,12 @@ public void toPrefs(Preferences preferences) {
preferences
.getPrefProperties()
.setProperty("exportContext.csvExport.csvExtendedFormat", Boolean.toString(csvExtendedFormat));
preferences
.getPrefProperties()
.setProperty(
"exportContext.csvExport.charsetName",
(csvCharsetName == null ? getDefaultCsvCharsetName() : csvCharsetName)
);
preferences
.getPrefProperties()
.setProperty(
Expand Down Expand Up @@ -417,6 +431,36 @@ public void toPrefs(Preferences preferences) {
);
}

/**
* Gets the csv export charset name.
*
* @return the csv export charset name
*/
public String getCsvCharsetName() {
return csvCharsetName;
}

/**
* Sets the csv export charset name.
*
* @param csvCharsetName the csv export charset name
*/
public void setCsvCharsetName(String csvCharsetName) {
this.csvCharsetName = csvCharsetName;
}

/**
* Gets the platform default charset name for csv export, windows-1252 on Windows where Excel
* expects it, UTF-8 elsewhere. Same default as the csv import one, so that the export keeps the
* behaviour it had when it was wrongly driven by the import preference.
*
* @return the default csv export charset name
*/
public static String getDefaultCsvCharsetName() {
if (System.getProperty("os.name").toLowerCase().contains("win")) return "windows-1252";
return "UTF-8";
}

/**
* Sets the default prefs.
*/
Expand All @@ -427,6 +471,7 @@ public void setDefaultPrefs() {
this.usageVersionSelectionMode = LAST_DATAOBJECT;
this.maxNameSize = 32;
this.csvExtendedFormat = true;
this.csvCharsetName = getDefaultCsvCharsetName();
this.managementMetadataXmlData = " <ManagementMetadata>\n" +
" <AcquisitionInformation>Acquisition Information</AcquisitionInformation>\n" +
" <LegalStatus>Public Archive</LegalStatus>\n" +
Expand Down
6 changes: 3 additions & 3 deletions resip/src/main/java/fr/gouv/vitam/tools/resip/threads/ExportThread.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ public String doInBackground() {
CSVImportContext cmic = new CSVImportContext(Preferences.getInstance());
DataObjectPackageToCSVMetadataExporter cme = new DataObjectPackageToCSVMetadataExporter(
archiveTransfer.getDataObjectPackage(),
cmic.getCsvCharsetName(),
work.getExportContext().getCsvCharsetName(),
cmic.getDelimiter(),
work.getExportContext().getUsageVersionSelectionMode(),
work.getExportContext().isCsvExtendedFormat(),
Expand All @@ -245,7 +245,7 @@ public String doInBackground() {
CSVImportContext cmicz = new CSVImportContext(Preferences.getInstance());
DataObjectPackageToCSVMetadataExporter cmez = new DataObjectPackageToCSVMetadataExporter(
archiveTransfer.getDataObjectPackage(),
cmicz.getCsvCharsetName(),
work.getExportContext().getCsvCharsetName(),
cmicz.getDelimiter(),
work.getExportContext().getUsageVersionSelectionMode(),
work.getExportContext().isCsvExtendedFormat(),
Expand All @@ -264,7 +264,7 @@ public String doInBackground() {
CSVImportContext cmicm = new CSVImportContext(Preferences.getInstance());
DataObjectPackageToCSVMetadataExporter cmem = new DataObjectPackageToCSVMetadataExporter(
archiveTransfer.getDataObjectPackage(),
cmicm.getCsvCharsetName(),
work.getExportContext().getCsvCharsetName(),
cmicm.getDelimiter(),
work.getExportContext().getUsageVersionSelectionMode(),
work.getExportContext().isCsvExtendedFormat(),
Expand Down
40 changes: 40 additions & 0 deletions resip/src/test/java/fr/gouv/vitam/tools/resip/parameters/ExportContextTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,51 @@ void test() throws Exception {
ExportContext reloadSipContext = mapper.readValue(ssc, ExportContext.class);
String dssc = mapper.writeValueAsString(reloadSipContext);

// the csv export charset default depends on the platform, so it's asserted apart and dropped
// from the comparison with the reference file
assertThat(reloadSipContext.getCsvCharsetName()).isEqualTo(ExportContext.getDefaultCsvCharsetName());
dssc = dssc.replaceAll("\\R? *\"csvCharsetName\" : \"[^\"]*\",?", "");

String fromfile = new String(
Files.readAllBytes(Paths.get("src/test/resources/PacketSamples/ExportContext.config")),
StandardCharsets.UTF_8
);

assertThat(TestUtilities.LineEndNormalize(dssc)).isEqualTo(TestUtilities.LineEndNormalize(fromfile));
}

/**
* The csv export charset used to be the one of the csv import preference, so setting the import
* one to UTF-8 silently turned every csv export into an UTF-8 one, unreadable by Excel on Windows.
* It's now a setting of its own, that has to survive the serialization of a saved work.
*/
@Test
void csvExportCharsetSurvivesJsonRoundTrip() throws Exception {
ExportContext gmc = new ExportContext();
gmc.setDefaultPrefs();
gmc.setCsvCharsetName("UTF-8");
ObjectMapper mapper = new ObjectMapper();

ExportContext reloaded = mapper.readValue(mapper.writeValueAsString(gmc), ExportContext.class);

assertThat(reloaded.getCsvCharsetName()).isEqualTo("UTF-8");
}

/**
* A work saved before the export charset existed carries no value for it, and has to fall back on
* the platform default rather than on a null charset name, which would break the export.
*/
@Test
void csvExportCharsetFallsBackOnDefaultWhenAbsentFromJson() throws Exception {
ObjectMapper mapper = new ObjectMapper();
ExportContext reference = new ExportContext();
reference.setDefaultPrefs();
String withoutCharset = mapper
.writeValueAsString(reference)
.replaceAll("\\R? *\"csvCharsetName\" : \"[^\"]*\",?", "");

ExportContext reloaded = mapper.readValue(withoutCharset, ExportContext.class);

assertThat(reloaded.getCsvCharsetName()).isEqualTo(ExportContext.getDefaultCsvCharsetName());
}
}
7 changes: 6 additions & 1 deletion ...va/fr/gouv/vitam/tools/sedalib/inout/exporter/DataObjectPackageToCSVMetadataExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.apache.commons.io.FileUtils;

import java.io.*;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
Expand Down Expand Up @@ -843,7 +844,11 @@ private void createCsvBAOSPrintScream() throws SEDALibException {
try {
csvBAOS = new ByteArrayOutputStream();
csvPrintStream = new PrintStream(csvBAOS, true, encoding);
} catch (UnsupportedEncodingException e) {
// Excel on Windows reads a BOM less UTF-8 csv as windows-1252 and shows the accented
// characters as mojibake, so the byte order mark has to be there. The csv importers
// drop it back on reading.
if (StandardCharsets.UTF_8.equals(Charset.forName(encoding))) csvPrintStream.print('\uFEFF');
} catch (UnsupportedEncodingException | IllegalArgumentException e) {
throw new SEDALibException("Encodage [" + encoding + "] inconnu", e);
}
}
Expand Down
Loading
Loading