feat(parser): support MySQL # line comments behind allowHashLineComments

The second step agreed in #2502: with Feature.allowHashLineComments
(default off) a `#` runs to end of line as a comment, unconditional
like MySQL itself (no blank needed, `42#24` is a comment too); with
the flag off a lone `#` stays the binary operator introduced in #2507,
so neither reading silently replaces the other.

Mechanics: under the flag SimpleCharStream rewrites a token-start `#`
in the buffer to a character no other lexical rule starts with, so the
dedicated HASH_LINE_COMMENT production wins the match for every `#`
form while identifier and JSON-operator lexing of the default mode stay
untouched (rewriting the buffer keeps the matcher's backup / re-read
arithmetic intact, and GetImage() restores the `#` in the token image).
Unquoted identifiers (and @@variables) end at their first `#` via their
token actions, which re-lex the remainder as the comment. Quoted forms
("#", `#`, "a#b") keep their `#` in both modes.

Under the flag the statement semantics are MySQL's: `SELECT #temp
FROM t` comments out the rest of the line and fails, quoted "#temp"
still parses.

Closes #2499, supersedes #2502.

Signed-off-by: Fu Dian <fudianchn@gmail.com>

Author: fudianchn

src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java

@@ -77,6 +77,14 @@ public P withBackslashEscapeCharacter(boolean allowBackslashEscapeCharacter) {
         return withFeature(Feature.allowBackslashEscapeCharacter, allowBackslashEscapeCharacter);
     }
 
+    public P withHashLineComments() {
+        return withFeature(Feature.allowHashLineComments, true);
+    }
+
+    public P withHashLineComments(boolean allowHashLineComments) {
+        return withFeature(Feature.allowHashLineComments, allowHashLineComments);
+    }
+
     public P withUnparenthesizedSubSelects() {
         return withFeature(Feature.allowUnparenthesizedSubSelects, true);
     }

src/main/java/net/sf/jsqlparser/parser/SimpleCharStream.java

@@ -7,9 +7,12 @@
  * Dual licensed under GNU LGPL 2.1 or Apache License 2.0
  * #L%
  */
-/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 8.0.0 */
+/* Generated By:JavaCC: Do not edit this line by hand. SimpleCharStream.java Version 8.0.0 */
 package net.sf.jsqlparser.parser;
 
+import net.sf.jsqlparser.parser.feature.Feature;
+import net.sf.jsqlparser.parser.feature.FeatureConfiguration;
+
 /**
  * An implementation of interface CharStream, where the stream is assumed to contain only ASCII
  * characters (without unicode processing).
@@ -42,6 +45,17 @@ public class SimpleCharStream {
     int available;
     int tokenBegin;
 
+    // MySQL `#` line comments (#2499): under the flag a token-start `#` is
+    // rewritten in the buffer to HASH_SUBSTITUTION, a character no other
+    // lexical rule starts with, so the dedicated HASH_LINE_COMMENT production
+    // wins the match for every `#` form (`# c`, `#c`, `#>`, ...). Rewriting
+    // the buffer itself (instead of synthesizing reads) keeps the matcher's
+    // backup / re-read arithmetic intact; the production's action restores
+    // the `#` in the token image. Wired (with the token manager's
+    // configuration) before parsing, null keeps this inert.
+    static final char HASH_SUBSTITUTION = '\u0001';
+    FeatureConfiguration featureConfiguration;
+
     /**
      * Constructor.
      */
@@ -159,6 +173,11 @@ public final char BeginToken() throws java.io.IOException {
 
         absoluteTokenBegin = totalCharsRead;
 
+        if (c == '#' && featureConfiguration != null
+                && featureConfiguration.getAsBoolean(Feature.allowHashLineComments)) {
+            buffer[bufpos] = HASH_SUBSTITUTION;
+            c = HASH_SUBSTITUTION;
+        }
         return c;
     }
 
@@ -325,6 +344,23 @@ public void ReInit(Provider dstream) {
      * Get token literal value.
      */
     public String GetImage() {
+        // restore the `#` rewritten to HASH_SUBSTITUTION: only the comment
+        // token's own window starts with the sentinel (token-start rewrites
+        // only), so this is exact and covers every consumer of GetImage.
+        // The wiring check keeps parses that never opted in on the original
+        // cost (the stream is only wired through the feature consumers /
+        // withConfiguration)
+        if (featureConfiguration == null) {
+            return doGetImage();
+        }
+        String image = doGetImage();
+        if (!image.isEmpty() && image.charAt(0) == HASH_SUBSTITUTION) {
+            image = "#" + image.substring(1);
+        }
+        return image;
+    }
+
+    private String doGetImage() {
         if (bufpos >= tokenBegin) {
             return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
         } else {

src/main/java/net/sf/jsqlparser/parser/feature/Feature.java

@@ -795,6 +795,12 @@ public enum Feature {
      */
     allowBackslashEscapeCharacter(false),
 
+    /**
+     * allows MySQL `#` line comments; disabled by default, where a lone `#` stays the binary
+     * operator (#2507: PostgreSQL bitwise XOR / geometric intersection)
+     */
+    allowHashLineComments(false),
+
     /**
      * allows sub selects without parentheses, e.g. `select * from dual where 1 = select 1`
      */

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -91,10 +91,17 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
 
     public CCJSqlParser withConfiguration(FeatureConfiguration configuration) {
         token_source.configuration = configuration;
+        jj_input_stream.featureConfiguration = configuration;
         return this;
     }
 
     public FeatureConfiguration getConfiguration() {
+        // wire the stream before parsing starts: every feature mutation goes
+        // through here (the consumers run pre-parse), default-off parsing
+        // never wires the stream and stays on the zero-cost path
+        if (jj_input_stream.featureConfiguration != token_source.configuration) {
+            jj_input_stream.featureConfiguration = token_source.configuration;
+        }
         return token_source.configuration;
     }
 
@@ -1883,6 +1890,19 @@ SPECIAL_TOKEN:
    < LINE_COMMENT: ("--" | "//") (~["\r","\n"])*>
 }
 
+// MySQL `#` line comment (#2499), flag-gated.  Under
+// Feature.allowHashLineComments the stream rewrites a token-start `#` to
+// SimpleCharStream.HASH_SUBSTITUTION, a character no other rule starts with,
+// so this production wins the match for every `#` form (`# c`, `#c`, `#>`,
+// ...) without touching the identifier and JSON-operator lexing of the
+// default mode.  The action restores the `#` in the token image; unquoted
+// identifiers end at their first `#` via the S_IDENTIFIER action, which
+// re-lexes the remainder as this comment.
+SPECIAL_TOKEN:
+{
+   < HASH_LINE_COMMENT: "\u0001" (~["\r","\n"])*>
+}
+
 // Nested block comments:  /* ... /* ... */ ... */
 //
 // Uses a nesting counter (commentNesting in TOKEN_MGR_DECLS) and
@@ -1940,12 +1960,46 @@ TOKEN:
 <S_HASH_OPERATOR: "#">
 |
 <S_IDENTIFIER: (<LETTER> (<PART_LETTER>)*) | "$" | ("$" <PART_LETTER_NO_DOLLAR> (<PART_LETTER>)*)>
+{
+    // MySQL `#` line comments (#2499): under the flag an unquoted identifier
+    // ends at its first `#`, the rest of the line becomes a comment via the
+    // stream-level substitution (real MySQL reads `42#24` as `42` plus
+    // comment too).  Quoted identifiers and strings keep their `#`.
+    // the wiring check keeps the flag lookup off the hot path of parses
+    // that never opted in (the stream is only wired through the feature
+    // consumers / withConfiguration); getValue avoids the String-based
+    // getAsBoolean roundtrip
+    if (input_stream.featureConfiguration != null
+            && Boolean.TRUE.equals(configuration.getValue(Feature.allowHashLineComments))) {
+        int hashIndex = matchedToken.image.indexOf('#');
+        if (hashIndex > 0) {
+            input_stream.backup(matchedToken.image.length() - hashIndex);
+            matchedToken.image = matchedToken.image.substring(0, hashIndex);
+        }
+    }
+}
 |   <#LETTER: <UnicodeIdentifierStart>
                 | <Nd> | [ "#", "_" ] // Not SQL:2016 compliant!
                 >
 |   <#PART_LETTER_NO_DOLLAR: <UnicodeIdentifierStart> | <UnicodeIdentifierExtend> | [ "#", "_" , "@"  ] >
 |   <#PART_LETTER: <UnicodeIdentifierStart> | <UnicodeIdentifierExtend> | [ "$" , "#", "_" , "@"  ] >
 | <S_AT_IDENTIFIER: <K_AT_SIGN> (<K_AT_SIGN>)? <S_IDENTIFIER> >
+{
+    // same truncation as S_IDENTIFIER: `@@#name` ends at the `#` under
+    // allowHashLineComments, the rest of the line becomes the comment
+    // the wiring check keeps the flag lookup off the hot path of parses
+    // that never opted in (the stream is only wired through the feature
+    // consumers / withConfiguration); getValue avoids the String-based
+    // getAsBoolean roundtrip
+    if (input_stream.featureConfiguration != null
+            && Boolean.TRUE.equals(configuration.getValue(Feature.allowHashLineComments))) {
+        int hashIndex = matchedToken.image.indexOf('#');
+        if (hashIndex > 0) {
+            input_stream.backup(matchedToken.image.length() - hashIndex);
+            matchedToken.image = matchedToken.image.substring(0, hashIndex);
+        }
+    }
+}
 
 // Unicode characters and categories are defined here: https://www.unicode.org/Public/UNIDATA/UnicodeData.txt
 // SQL:2016 states:

src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java

@@ -514,4 +514,54 @@ void testSingleStatementWithEmptyLines() throws JSQLParserException {
                 + "def'\n"
                 + "where id=?", true);
     }
+
+    @Test
+    public void testHashLineCommentsFeature() throws Exception {
+        // with the flag: MySQL line comments, `#` to end of line,
+        // unconditional like MySQL itself (no blank needed, `42#24` is a
+        // comment there too); the statement continues on the next line
+        assertEquals("SELECT 42", CCJSqlParserUtil
+                .parse("SELECT 42 # 24", p -> p.withHashLineComments(true)).toString());
+        assertEquals("SELECT 1", CCJSqlParserUtil
+                .parse("SELECT 1 # comment, 2", p -> p.withHashLineComments(true)).toString());
+        assertEquals("SELECT 1", CCJSqlParserUtil
+                .parse("SELECT 1 #comment", p -> p.withHashLineComments(true)).toString());
+        assertEquals("SELECT 1", CCJSqlParserUtil
+                .parse("SELECT 1 #!bang", p -> p.withHashLineComments(true)).toString());
+        assertEquals("SELECT 42", CCJSqlParserUtil
+                .parse("SELECT 42#24", p -> p.withHashLineComments(true)).toString());
+        assertEquals("SELECT a", CCJSqlParserUtil
+                .parse("SELECT a#b FROM t", p -> p.withHashLineComments(true)).toString());
+        assertEquals("SELECT 1 FROM t", CCJSqlParserUtil
+                .parse("SELECT 1 # c\nFROM t", p -> p.withHashLineComments(true)).toString());
+        assertEquals("SELECT 1", CCJSqlParserUtil
+                .parse("# leading\nSELECT 1", p -> p.withHashLineComments(true)).toString());
+        assertEquals("SELECT 1", CCJSqlParserUtil
+                .parse("SELECT 1 #", p -> p.withHashLineComments(true)).toString());
+        assertEquals("SELECT 1", CCJSqlParserUtil
+                .parse("SELECT 1 #\n", p -> p.withHashLineComments(true)).toString());
+        // the `#>` family has no meaning in MySQL and comments out like any
+        // other `#`; quoted forms keep their `#`
+        assertEquals("SELECT data", CCJSqlParserUtil
+                .parse("SELECT data #> '{a}' FROM t", p -> p.withHashLineComments(true))
+                .toString());
+        TestUtils.assertSqlCanBeParsedAndDeparsed("SELECT '#' FROM t", true,
+                p -> p.withHashLineComments(true));
+        TestUtils.assertSqlCanBeParsedAndDeparsed("SELECT \"a#b\" FROM t", true,
+                p -> p.withHashLineComments(true));
+        // default (flag off): the #2507 binary operator, unchanged
+        assertEquals("SELECT 42 # 24", CCJSqlParserUtil.parse("SELECT 42 # 24").toString());
+        assertEquals("SELECT 42#24", CCJSqlParserUtil.parse("SELECT 42#24").toString());
+    }
+
+    @Test
+    public void testHashLineCommentsMySQLStatementSemantics() throws Exception {
+        // `#temp` is a comment under the flag, so `SELECT #temp FROM t`
+        // loses the rest of the line and fails, exactly like MySQL; with the
+        // flag off the same input parses as the #2507 operator expression
+        assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil
+                .parse("SELECT #temp FROM t", p -> p.withHashLineComments(true)));
+        assertEquals("SELECT 1 # comment",
+                CCJSqlParserUtil.parse("SELECT 1 # comment").toString());
+    }
 }

src/test/java/net/sf/jsqlparser/statement/select/SelectASTTest.java

@@ -14,6 +14,7 @@
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.parser.CCJSqlParserDefaultVisitor;
 import net.sf.jsqlparser.parser.CCJSqlParserTreeConstants;
+import net.sf.jsqlparser.parser.CCJSqlParser;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.parser.Node;
 import net.sf.jsqlparser.parser.Token;
@@ -212,4 +213,21 @@ public void testSelectASTExtractWithCommentsIssue1580_2() throws JSQLParserExcep
         assertThat(root.jjtGetFirstToken().specialToken.image)
                 .isEqualTo("/* I want this comment */\n");
     }
+
+    @Test
+    public void testSelectASTHashLineCommentImage() throws Exception {
+        // a `#` comment under allowHashLineComments is a normal special
+        // token carrying the original `# ...` text
+        CCJSqlParser parser = CCJSqlParserUtil.newParser("SELECT 1 # note\nFROM t");
+        parser.withHashLineComments(true);
+        parser.Statement();
+        List<Token> comments = new ArrayList<>();
+        for (Token t = parser.getASTRoot().jjtGetFirstToken(); t.next != null; t = t.next) {
+            for (Token sp = t.specialToken; sp != null; sp = sp.specialToken) {
+                comments.add(sp);
+            }
+        }
+
+        assertThat(comments).extracting(token -> token.image).containsExactly("# note");
+    }
 }