diff --git a/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java b/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java index 2cb803396..27880a2c7 100644 --- a/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java +++ b/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java @@ -24,24 +24,41 @@ public abstract class AbstractJSqlParser

{ protected boolean errorRecovery = false; protected List parseErrors = new ArrayList<>(); + public enum AdjacentStringLiterals { + OFF, NEWLINE, WHITESPACE + } + public enum Dialect { - ANSI_SQL, ORACLE, MYSQL(Feature.allowBackslashEscapeCharacter, + ANSI_SQL(AdjacentStringLiterals.NEWLINE), ORACLE, MYSQL(AdjacentStringLiterals.OFF, + Feature.allowBackslashEscapeCharacter, Feature.allowHashLineComments, - Feature.allowDoubleQuotedStrings), MARIADB(Feature.allowBackslashEscapeCharacter, + Feature.allowDoubleQuotedStrings), MARIADB(AdjacentStringLiterals.OFF, + Feature.allowBackslashEscapeCharacter, Feature.allowHashLineComments, - Feature.allowDoubleQuotedStrings), SQLSERVER( - Feature.allowSquareBracketQuotation), POSTGRESQL, H2, EXASOL; + Feature.allowDoubleQuotedStrings), SQLSERVER(AdjacentStringLiterals.OFF, + Feature.allowSquareBracketQuotation), POSTGRESQL( + AdjacentStringLiterals.NEWLINE), H2, EXASOL; private final Set lexerFeatures; + private final AdjacentStringLiterals adjacentStringLiterals; - Dialect(Feature... lexerFeatures) { + Dialect(AdjacentStringLiterals adjacentStringLiterals, Feature... lexerFeatures) { + this.adjacentStringLiterals = adjacentStringLiterals; this.lexerFeatures = lexerFeatures.length == 0 ? EnumSet.noneOf(Feature.class) : EnumSet.copyOf(Arrays.asList(lexerFeatures)); } + Dialect(Feature... lexerFeatures) { + this(AdjacentStringLiterals.OFF, lexerFeatures); + } + public Set getLexerFeatures() { return lexerFeatures; } + + public AdjacentStringLiterals getAdjacentStringLiterals() { + return adjacentStringLiterals; + } } public P withSquareBracketQuotation() { @@ -82,12 +99,30 @@ public P withTimeOut(long timeOutMillSeconds) { public P withDialect(Dialect dialect) { withFeature(Feature.dialect, dialect.name()); + if (dialect.getAdjacentStringLiterals() != AdjacentStringLiterals.OFF) { + withAdjacentStringLiterals(dialect.getAdjacentStringLiterals()); + } for (Feature lexerFeature : dialect.getLexerFeatures()) { withFeature(lexerFeature, true); } return me(); } + public P withAdjacentStringLiterals() { + return withAdjacentStringLiterals(AdjacentStringLiterals.NEWLINE); + } + + public P withAdjacentStringLiterals(boolean adjacentStringLiterals) { + return withAdjacentStringLiterals( + adjacentStringLiterals ? AdjacentStringLiterals.NEWLINE + : AdjacentStringLiterals.OFF); + } + + public P withAdjacentStringLiterals(AdjacentStringLiterals adjacentStringLiterals) { + getConfiguration().setValue(Feature.adjacentStringLiterals, adjacentStringLiterals); + return me(); + } + public P withAllowedNestingDepth(int allowedNestingDepth) { return withFeature(Feature.allowedNestingDepth, allowedNestingDepth); } diff --git a/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java b/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java index de5a492f2..bb68fa387 100644 --- a/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java +++ b/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java @@ -15,6 +15,7 @@ import net.sf.jsqlparser.expression.OracleHierarchicalExpression; import net.sf.jsqlparser.expression.OracleHint; import net.sf.jsqlparser.expression.operators.relational.SupportsOldOracleJoinSyntax; +import net.sf.jsqlparser.parser.AbstractJSqlParser; import net.sf.jsqlparser.statement.Block; import net.sf.jsqlparser.statement.Commit; import net.sf.jsqlparser.statement.CreateFunctionalStatement; @@ -801,6 +802,14 @@ public enum Feature { */ allowDoubleQuotedStrings(false), + /** + * concatenates adjacent String Literals: NEWLINE when separated by whitespace with at least one + * newline (the SQL standard and PostgreSQL), WHITESPACE across any whitespace (GoogleSQL, + * Spark/Databricks); OFF by default, where the second literal stays an alias (MySQL, SQL + * Server) or fails (everywhere else) + */ + adjacentStringLiterals(AbstractJSqlParser.AdjacentStringLiterals.OFF), + /** * allows MySQL `#` line comments; disabled by default, where a lone `#` stays the binary * operator (#2507: PostgreSQL bitwise XOR / geometric intersection) diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 4533078c9..e617d7e6c 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -1127,6 +1127,24 @@ public class CCJSqlParser extends AbstractJSqlParser { } } + /** + * Semantic lookahead for the adjacent string literal concatenation in PrimaryExpression: + * true when Feature.adjacentStringLiterals is on and the next token is a string literal + * meeting the mode's separation rule (WHITESPACE: any, NEWLINE: a newline between the + * tokens, detected through the token line numbers). + */ + protected boolean isAdjacentStringConcat() { + String mode = getAsString(Feature.adjacentStringLiterals); + if (mode == null || AdjacentStringLiterals.OFF.name().equals(mode)) { + return false; + } + if (getToken(1).kind != S_CHAR_LITERAL) { + return false; + } + return AdjacentStringLiterals.WHITESPACE.name().equals(mode) + || getToken(1).beginLine > getToken(0).endLine; + } + /** * Checks if the next token can start a condition suffix * (comparison, IN, BETWEEN, LIKE, IS NULL, etc.) @@ -8319,6 +8337,7 @@ Expression PrimaryExpression() #PrimaryExpression: Expression timezoneRightExpr = null; Token token = null; Token sign = null; + Token adjacentToken = null; String tmp = ""; ColDataType type = null; boolean not = false; @@ -8419,6 +8438,13 @@ Expression PrimaryExpression() #PrimaryExpression: | LOOKAHEAD(2, {!interrupted}) (token= | token=) { retval = new BooleanValue(token.image); } | token= { retval = new StringValue(token.image); linkAST(retval,jjtThis); } + ( LOOKAHEAD({ isAdjacentStringConcat() }) + adjacentToken= + { + ((StringValue) retval) + .setValue(((StringValue) retval).getValue() + new StringValue(adjacentToken.image).getValue()); + } + )* | "{d" token= "}" { retval = new DateValue(token.image); } diff --git a/src/site/sphinx/usage.rst b/src/site/sphinx/usage.rst index 4a9696b17..3ad3190d9 100644 --- a/src/site/sphinx/usage.rst +++ b/src/site/sphinx/usage.rst @@ -279,7 +279,7 @@ Define the Parser Features JSQLParser interprets Squared Brackets ``[..]`` as Arrays, which does not work with MS SQL Server and T-SQL. Please use the Parser Features to instruct JSQLParser to read Squared Brackets as Quotes instead. -JSQLParser allows for standard compliant Single Quote ``'..`` Escaping. Additional Back-slash ``\..`` Escaping needs to be activated by setting the ``BackSlashEscapeCharacter`` parser feature. JSQLParser reads Double Quotes ``".."`` as quoted identifiers (ANSI SQL); reading them as String Literals (BigQuery, Spark/Databricks, MySQL default sql_mode) needs the ``DoubleQuotedStrings`` parser feature. +JSQLParser allows for standard compliant Single Quote ``'..`` Escaping. Additional Back-slash ``\..`` Escaping needs to be activated by setting the ``BackSlashEscapeCharacter`` parser feature. JSQLParser reads Double Quotes ``".."`` as quoted identifiers (ANSI SQL); reading them as String Literals (BigQuery, Spark/Databricks, MySQL default sql_mode) needs the ``DoubleQuotedStrings`` parser feature. Adjacent String Literals concatenate optionally: only across a newline (``NEWLINE``, the SQL standard and PostgreSQL) or across any whitespace (``WHITESPACE``, GoogleSQL and Spark/Databricks); ``withAdjacentStringLiterals(true)`` selects the standard ``NEWLINE`` mode, ``false`` switches it off. Additionally there are Features to control the Parser's effort at the cost of the performance. @@ -319,7 +319,7 @@ Additionally there are Features to control the Parser's effort at the cost of th .withBackslashEscapeCharacter(true) ); -Instead of turning the individual Parser Features on one by one, a ``Dialect`` preset selects the features of that database dialect: ``withDialect(Dialect.MYSQL)`` turns on ``withBackslashEscapeCharacter``, ``withHashLineComments`` and ``withDoubleQuotedStrings`` (MySQL and MariaDB syntax, the latter for the default sql_mode), ``withDialect(Dialect.SQLSERVER)`` turns on ``withSquareBracketQuotation``. Features set explicitly after the dialect preset win over the preset. +Instead of turning the individual Parser Features on one by one, a ``Dialect`` preset selects the features of that database dialect: ``withDialect(Dialect.MYSQL)`` turns on ``withBackslashEscapeCharacter``, ``withHashLineComments`` and ``withDoubleQuotedStrings`` (MySQL and MariaDB syntax, the latter for the default sql_mode), ``withDialect(Dialect.SQLSERVER)`` turns on ``withSquareBracketQuotation``. ``withDialect(Dialect.POSTGRESQL)`` and ``withDialect(Dialect.ANSI_SQL)`` turn on the newline rule for adjacent String Literals. Features set explicitly after the dialect preset win over the preset. .. code-block:: java diff --git a/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java b/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java index 54c5e4402..8add3b1bc 100644 --- a/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java +++ b/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java @@ -655,4 +655,82 @@ public void testDoubleQuotedStringsPreset() throws Exception { p -> p.withDialect(AbstractJSqlParser.Dialect.SQLSERVER))).getSelectBody(); assertTrue(select.getSelectItems().get(0).getExpression() instanceof Column); } + + @Test + public void testAdjacentStringLiteralsNewline() throws Exception { + // default (off): the MySQL / SQL Server alias reading, unchanged + assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a' 'b'").toString()); + assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'").toString()); + assertThrows(JSQLParserException.class, + () -> CCJSqlParserUtil.parse("SELECT * FROM t WHERE x = 'a'\n'b'")); + // newline mode: the standard / Postgres reading, "Two string constants + // that are only separated by whitespace with at least one newline are + // concatenated" + PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil.parse("SELECT 'a'\n'b'", + p -> p.withAdjacentStringLiterals( + AbstractJSqlParser.AdjacentStringLiterals.NEWLINE))) + .getSelectBody(); + Expression expression = select.getSelectItems().get(0).getExpression(); + assertTrue(expression instanceof StringValue); + assertEquals("ab", ((StringValue) expression).getValue()); + assertEquals("SELECT 'ab'", select.toString()); + // same line keeps the alias reading under newline mode + assertEquals("SELECT 'a' 'b'", + CCJSqlParserUtil.parse("SELECT 'a' 'b'", + p -> p.withAdjacentStringLiterals( + AbstractJSqlParser.AdjacentStringLiterals.NEWLINE)) + .toString()); + // expression positions concatenate too + assertEquals("SELECT * FROM t WHERE x = 'ab'", + CCJSqlParserUtil.parse("SELECT * FROM t WHERE x = 'a'\n'b'", + p -> p.withAdjacentStringLiterals( + AbstractJSqlParser.AdjacentStringLiterals.NEWLINE)) + .toString()); + // three parts merge into one literal + assertEquals("SELECT 'abc'", + CCJSqlParserUtil.parse("SELECT 'a'\n'b'\n'c'", + p -> p.withAdjacentStringLiterals( + AbstractJSqlParser.AdjacentStringLiterals.NEWLINE)) + .toString()); + // the ANSI SQL and Postgres presets carry it, the MySQL preset does not + for (AbstractJSqlParser.Dialect dialect : new AbstractJSqlParser.Dialect[] { + AbstractJSqlParser.Dialect.ANSI_SQL, AbstractJSqlParser.Dialect.POSTGRESQL}) { + assertEquals("SELECT 'ab'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'", + p -> p.withDialect(dialect)).toString()); + } + assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'", + p -> p.withDialect(AbstractJSqlParser.Dialect.MYSQL)).toString()); + } + + @Test + public void testAdjacentStringLiteralsWhitespace() throws Exception { + // whitespace mode: the GoogleSQL / Spark reading, same line + // concatenates (their literal chunking) + assertEquals("SELECT 'ab'", + CCJSqlParserUtil.parse("SELECT 'a' 'b'", + p -> p.withAdjacentStringLiterals( + AbstractJSqlParser.AdjacentStringLiterals.WHITESPACE)) + .toString()); + // combined with allowDoubleQuotedStrings: the BigQuery chunking shape + assertEquals("SELECT \"12\"", + CCJSqlParserUtil.parse("SELECT \"1\" \"2\"", + p -> p.withDoubleQuotedStrings(true).withAdjacentStringLiterals( + AbstractJSqlParser.AdjacentStringLiterals.WHITESPACE)) + .toString()); + } + + @Test + public void testAdjacentStringLiteralsBoolean() throws Exception { + // true: the standard (newline) mode, the same line keeps the alias reading + assertEquals("SELECT 'ab'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'", + p -> p.withAdjacentStringLiterals(true)).toString()); + assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a' 'b'", + p -> p.withAdjacentStringLiterals(true)).toString()); + // false: off, the alias reading also across newlines + assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'", + p -> p.withAdjacentStringLiterals(false)).toString()); + // the no-arg variant enables the standard mode + assertEquals("SELECT 'ab'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'", + p -> p.withAdjacentStringLiterals()).toString()); + } }