Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/main/java/org/apache/commons/csv/CSVParser.java
Original file line number Diff line number Diff line change
Expand Up @@ -806,8 +806,10 @@ private String handleNull(final String input) {
final String nullString = format.getNullString();
final boolean strictQuoteMode = isStrictQuoteMode();
if (input.equals(nullString)) {
// An escaped value is the null string itself, not the null marker: printing null writes the null string
// verbatim, so "\\N" for nullString "\N" can only have come from a value that really is "\N".
// nullString = NULL(String), distinguish between "NULL" and NULL in ALL_NON_NULL or NON_NUMERIC quote mode
return strictQuoteMode && isQuoted ? input : null;
return reusableToken.isEscaped || strictQuoteMode && isQuoted ? input : null;
}
// don't set nullString, distinguish between "" and ,, (absent values) in All_NON_NULL or NON_NUMERIC quote mode
return strictQuoteMode && nullString == null && input.isEmpty() && !isQuoted ? null : input;
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/org/apache/commons/csv/Lexer.java
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,15 @@ final class Lexer implements Closeable {
private void appendNextEscapedCharacterToToken(final Token token) throws IOException {
if (isEscapeDelimiter()) {
token.content.append(delimiter);
token.isEscaped = true;
} else {
final int unescaped = readEscape();
if (unescaped == EOF) { // unexpected char after escape
// The escape character is kept verbatim, so nothing was translated.
token.content.append((char) escape).append((char) reader.getLastChar());
} else {
token.content.append((char) unescaped);
token.isEscaped = true;
}
}
}
Expand Down
4 changes: 4 additions & 0 deletions src/main/java/org/apache/commons/csv/Token.java
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,15 @@ enum Type {

boolean isQuoted;

/** True when an escape sequence in the input was translated while building {@link #content}. */
boolean isEscaped;

void reset() {
content.setLength(0);
type = INVALID;
isReady = false;
isQuoted = false;
isEscaped = false;
}

/**
Expand Down
17 changes: 17 additions & 0 deletions src/test/java/org/apache/commons/csv/CSVParserTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -584,6 +584,23 @@ void testEndOfFileBehaviorExcel() throws Exception {
}
}

@ParameterizedTest
@EnumSource(value = CSVFormat.Predefined.class, names = { "MySQL", "PostgreSQLCsv", "PostgreSQLText", "Oracle" })
void testEscapedNullStringIsAValue(final CSVFormat.Predefined predefined) throws Exception {
// For formats whose null string is "\\N" (e.g., MySQL, PostgreSQL Text, Oracle),
// a literal value "\\N" must be written as "\\\\N" so it is not read back as null.
final CSVFormat format = predefined.getFormat();
final StringWriter writer = new StringWriter();
try (CSVPrinter printer = new CSVPrinter(writer, format)) {
printer.printRecord("\\N", null);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Refactor the magic string \\N into a constant or a local variable with a name that explains its intent. Since there are 2 such instances in this method, it will also make it obvious that both magic strings are indeed intended to be the same.

}
try (CSVParser parser = CSVParser.parse(writer.toString(), format)) {
final CSVRecord record = parser.nextRecord();
assertEquals("\\N", record.get(0));
assertNull(record.get(1));
}
}

@Test
void testExcelFormat1() throws IOException {
final String code = "value1,value2,value3,value4\r\na,b,c,d\r\n x,,,\r\n\r\n\"\"\"hello\"\"\",\" \"\"world\"\"\",\"abc\ndef\",\r\n";
Expand Down
19 changes: 15 additions & 4 deletions src/test/java/org/apache/commons/csv/CSVPrinterTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -157,13 +157,17 @@ private void doRandom(final CSVFormat format, final int iter) throws Exception {
}

/**
* Converts an input CSV array into expected output values, including NULLs. NULL strings are converted to null values because the parser will convert
* these strings to null.
* Converts an input CSV array into expected output values, including NULLs. A value equal to the null string is converted to null only when the printer
* writes that value unchanged; when the printer escapes or quotes it, the parser reads it back as the value it is.
*/
private <T> T[] expectNulls(final T[] original, final CSVFormat csvFormat) {
private <T> T[] expectNulls(final T[] original, final CSVFormat csvFormat) throws IOException {
final T[] fixed = original.clone();
final String nullString = csvFormat.getNullString();
if (nullString == null || !printsVerbatim(csvFormat, nullString)) {
return fixed;
}
for (int i = 0; i < fixed.length; i++) {
if (Objects.equals(csvFormat.getNullString(), fixed[i])) {
if (Objects.equals(nullString, fixed[i])) {
fixed[i] = null;
}
}
Expand All @@ -187,6 +191,13 @@ private Connection getH2Connection() throws SQLException, ClassNotFoundException
return DriverManager.getConnection("jdbc:h2:mem:my_test;", "sa", "");
}

/** Tests whether the format prints the given value unchanged, in which case the parser cannot tell it from the null string. */
private boolean printsVerbatim(final CSVFormat csvFormat, final String value) throws IOException {
final StringBuilder sb = new StringBuilder();
csvFormat.print(value, sb, true);
return value.contentEquals(sb);
}

private CSVPrinter printWithHeaderComments(final StringWriter sw, final Date now, final CSVFormat baseFormat) throws IOException {
// Use withHeaderComments first to test CSV-145
// @formatter:off
Expand Down
Loading