build: add reproducible lookahead auditing

.github/workflows/ci.yml

@@ -45,6 +45,20 @@ jobs:
       - name: Check for grammar ambiguities
         run: ./gradlew checkGrammarAmbiguity --no-daemon
 
+      - name: Report changed lookahead contexts
+        env:
+          LOOKAHEAD_BASE: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }}
+        run: ./gradlew lookaheadAudit --no-daemon -PlookaheadBase="$LOOKAHEAD_BASE"
+
+      - name: Upload lookahead review report
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: lookahead-review
+          path: build/reports/lookahead/
+          retention-days: 7
+          if-no-files-found: ignore
+
       - name: Upload grammar warning report
         if: failure()
         uses: actions/upload-artifact@v5
@@ -216,4 +230,4 @@ jobs:
 
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v5
+        uses: actions/deploy-pages@v5

build.gradle

@@ -846,3 +846,23 @@ tasks.register('checkGrammarAmbiguity') {
 tasks.named('check') {
     dependsOn('checkGrammarAmbiguity')
 }
+
+// Report base/working-tree context changes without making historical lookahead debt a new gate.
+// The analyzer lives in test sources: no audit classes or dependencies enter the library JAR.
+tasks.register('lookaheadAudit', JavaExec) {
+    group = 'verification'
+    description = 'Reports changed lookahead decisions and stale evidence against a Git base.'
+    dependsOn(testClasses)
+    classpath = sourceSets.test.runtimeClasspath
+    mainClass = 'net.sf.jsqlparser.parser.audit.LookaheadAudit'
+    javaLauncher = javaToolchains.launcherFor(java.toolchain)
+    jvmArgs '-Xss4m', '-Xmx1g', '-Dfile.encoding=UTF-8'
+    workingDir = projectDir
+    doFirst {
+        args = [projectDir.absolutePath,
+                providers.gradleProperty('lookaheadBase').getOrElse('HEAD'),
+                layout.buildDirectory.dir('reports/lookahead').get().asFile.absolutePath,
+                file('config/lookahead-evidence.tsv').absolutePath,
+                providers.gradleProperty('lookaheadFailOnFindings').getOrElse('false')]
+    }
+}

config/lookahead-evidence.tsv

@@ -0,0 +1,2 @@
+# key<TAB>context fingerprint<TAB>reason<TAB>regression test
+ConnectionOrCloudConnectionDefinition	f0d3e3f23a41c3cf239c643bae45e52bfadc80e117be4d15cb8af2b7f54931bb	AT conn and AT CLOUD share the first token; retain the second-token decision.	LookaheadCleanupTest.preservesCloudAndOrdinaryConnections

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -1265,6 +1265,17 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
         }
     }
 
+    /**
+     * Both named-parameter forms match only a single-token name (or K_OUTER) directly followed
+     * by their assignment token; the 2-token lookahead fails for anything else, so this first
+     * check is equivalent to running that scan.
+     */
+    private boolean isNamedParameterAssignmentAhead() {
+        int kind = getToken(2).kind;
+        return kind == K_ORACLE_NAMED_PARAMETER_ASSIGNMENT
+                || kind == K_POSTGRES_NAMED_PARAMETER_ASSIGNMENT;
+    }
+
     /** Distinguish EXISTS identifiers from predicates before suffixes and clause boundaries. */
     private boolean isExistsIdentifierAhead() {
         if (getToken(1).kind != K_EXISTS) {
@@ -4548,7 +4559,6 @@ DBMSDestination DBMSDestination() #DBMSDestination: {
     connectionDefinition = ConnectionDefinition() { dbmsDestination.setConnectionDefinition(connectionDefinition); }
 
     (
-          LOOKAHEAD(3)
           <K_TABLE> table = Table() { dbmsDestination.setTable(table); }
           [ LOOKAHEAD(2) columns = ParenthesedColumnList() { dbmsDestination.setColumns(columns); } ]
           [ LOOKAHEAD(2) dbmsTableDestinationOptions = DBMSTableDestinationOptionList() { dbmsDestination.setDBMSTableDestinationOptions(dbmsTableDestinationOptions); } ]
@@ -11048,7 +11058,11 @@ Expression Condition():
         LOOKAHEAD(2, { !isExistsIdentifierAhead() })
         result=ExistsExpression()
         |
-          [ LOOKAHEAD(2, { getToken(1).kind == K_PRIOR }) <K_PRIOR> { oraclePrior = EqualsTo.ORACLE_PRIOR_START; } ]
+          // last alternative of the choice: entering it and letting the productions
+          // themselves reject is equivalent to scanning first, so skip the deep
+          // lookahead simulation (this path runs for every plain condition)
+          LOOKAHEAD({ !interrupted })
+          [ LOOKAHEAD({ getToken(1).kind == K_PRIOR }) <K_PRIOR> { oraclePrior = EqualsTo.ORACLE_PRIOR_START; } ]
           left=SimpleExpression() { result = left; }
 
           // Consume Oracle (+) once, before dispatching
@@ -11198,10 +11212,13 @@ Expression SQLCondition():
         |
         LOOKAHEAD(2, { !isExistsIdentifierAhead() })
         result=ExistsExpression()
-        | left = SimpleExpression() { result = left; }
+        |
+          // last alternative of the choice: skip the redundant lookahead simulation
+          LOOKAHEAD({ !interrupted })
+          left = SimpleExpression() { result = left; }
         [
             LOOKAHEAD(2) (
-                LOOKAHEAD(2, <K_OVERLAPS>) result=OverlapsCondition(left)
+                LOOKAHEAD({ getToken(1).kind == K_OVERLAPS }) result=OverlapsCondition(left)
                 |
                 LOOKAHEAD(3, {!interrupted}) result=InExpression(left)
                 |
@@ -11663,9 +11680,9 @@ ExpressionList ComplexExpressionList():
 }
 {
     (
-        LOOKAHEAD(2) expr=OracleNamedFunctionParameter()
+        LOOKAHEAD(2, { isNamedParameterAssignmentAhead() }) expr=OracleNamedFunctionParameter()
         |
-        LOOKAHEAD(2) expr=PostgresNamedFunctionParameter()
+        LOOKAHEAD(2, { isNamedParameterAssignmentAhead() }) expr=PostgresNamedFunctionParameter()
         |
         expr=Expression()
     )
@@ -11676,9 +11693,9 @@ ExpressionList ComplexExpressionList():
     (
         LOOKAHEAD(2, {!interrupted}) ","
         (
-            LOOKAHEAD(2) expr=OracleNamedFunctionParameter()
+            LOOKAHEAD(2, { isNamedParameterAssignmentAhead() }) expr=OracleNamedFunctionParameter()
             |
-            LOOKAHEAD(2) expr=PostgresNamedFunctionParameter()
+            LOOKAHEAD(2, { isNamedParameterAssignmentAhead() }) expr=PostgresNamedFunctionParameter()
             |
             LOOKAHEAD( RelObjectName() "->", { !isJsonArrowDialect() } ) expr=LambdaExpression()
             |
@@ -12094,7 +12111,8 @@ Expression PrimaryExpression() #PrimaryExpression:
         //| LOOKAHEAD(FunctionAllColumns()) retval=FunctionAllColumns()
 
         // support timestamp expressions
-        | LOOKAHEAD(2, {!interrupted}) (token=<K_TIME_KEY_EXPR> | token=<K_CURRENT>) { retval = new TimeKeyExpression(token.image); }
+        | LOOKAHEAD({ !interrupted && (getToken(1).kind == K_TIME_KEY_EXPR || getToken(1).kind == K_CURRENT) })
+          (token=<K_TIME_KEY_EXPR> | token=<K_CURRENT>) { retval = new TimeKeyExpression(token.image); }
 
         | LOOKAHEAD(2, {!interrupted}) retval=DateTimeLiteralExpression()
 
@@ -12114,7 +12132,7 @@ Expression PrimaryExpression() #PrimaryExpression:
 
         | LOOKAHEAD(2, { !interrupted && getToken(1).kind == K_KEY && getToken(2).kind != OPENING_BRACKET }) retval=KeyExpression()
 
-        | LOOKAHEAD(2, {!interrupted}) <K_ALL> { retval = new AllValue(); }
+        | LOOKAHEAD({ !interrupted && getToken(1).kind == K_ALL }) <K_ALL> { retval = new AllValue(); }
 
         | LOOKAHEAD({ !interrupted && isCharacterSetIntroducedStringAhead() })
                   retval=CharacterSetIntroducedString()
@@ -12128,7 +12146,8 @@ Expression PrimaryExpression() #PrimaryExpression:
                 "(" "+" ")" { ((Column) retval).setOldOracleJoinSyntax(EqualsTo.ORACLE_JOIN_RIGHT); }
             ]
 
-        | LOOKAHEAD(2, {!interrupted}) (token=<K_TRUE> | token=<K_FALSE>) { retval = new BooleanValue(token.image); }
+        | LOOKAHEAD({ !interrupted && (getToken(1).kind == K_TRUE || getToken(1).kind == K_FALSE) })
+          (token=<K_TRUE> | token=<K_FALSE>) { retval = new BooleanValue(token.image); }
 
         | retval=CharacterLiteral() { linkAST(retval,jjtThis); }
 

src/site/sphinx/contribution.rst

@@ -120,6 +120,31 @@ When adding a new keyword token to the Grammar:
     2) If the keyword must be reserved (e. |_| g. core SQL syntax like ``SELECT``, ``FROM``, ``WHERE``), add it to the ``Reserved SQL Keywords`` TOKEN block **after** the ``MAX_NON_RESERVED_WORD`` sentinel.
     3) Verify that existing tests pass and that the keyword can be used as a ``Schema``, ``Table``, ``Column``, ``Function`` or ``Alias`` name where expected.
 
+Review Lookahead Changes
+------------------------------
+
+Run the reusable lookahead report against the commit used as the review baseline:
+
+.. code-block:: shell
+
+    ./gradlew lookaheadAudit -PlookaheadBase=HEAD
+
+The default baseline is ``HEAD``, so this command also examines uncommitted grammar changes. CI uses the pull request base commit. Results are written to ``build/reports/lookahead/report.json``. The analyzer runs with the project's existing JDK and JavaCC dependencies; it does not require Python or modify sources.
+
+The machine calculation is fresh for both versions. Review the changed decisions, dependency roots, and stale evidence in the summary instead of manually rechecking every unchanged lookahead. Structural anchors ignore source line numbers and unrelated lookahead insertions. Identical anchors remain ambiguous and cannot reuse an individual evidence record.
+
+The report computes lexical FIRST/nullable/FOLLOW facts and follows direct and indirect production calls. Nullable competing alternatives contribute FOLLOW to the decision; unknown actions on either side prevent a disjoint-prefix suggestion. A changed callee can invalidate evidence even when the local FIRST set stays the same; a caller's nullable suffix can change a callee's FOLLOW context. Semantic predicates, Java actions and production declarations are treated conservatively, rather than assumed equivalent.
+
+All checked-in production Java files, runtime resources, ``build.gradle``, ``settings.gradle``, ``gradle.properties``, ``pom.xml`` and Gradle wrapper properties are monitored as opaque inputs. The dedicated cleanup regression test, analyzer sources and resolved JavaCC core/Java generator JARs are also bound to the evidence. Unrecognized watched symlinks fail the check instead of being skipped. Generated parser behavior is bound to the grammar and generator JARs; commit-derived build/version metadata is excluded. Evidence covers tracked static inputs and tested behavior, not every possible runtime configuration.
+
+``config/lookahead-evidence.tsv`` stores a small number of investigated decisions, their exact context fingerprint, reason and regression reference. It is not an approval of all historical sites or permission to publish changes. The report writes current unambiguous bindings to ``binding-candidates.tsv``; review and rerun the referenced tests before updating a stale record. A production name is accepted only when it identifies one decision; otherwise use a unique structural anchor or disambiguate the case.
+
+Pure one-terminal probes can suggest an equivalent token-kind condition. Non-nullable branches with disjoint FIRST sets can suggest default dispatch instead of a deeper scan. These are reusable syntactic rules, not an equivalence theorem for imperative JavaCC code. Retain state/dialect/interruption checks, verify complete consumption and AST behavior, and test malformed neighbors before adopting a candidate.
+
+Error locations can change when a rejected branch is entered directly. In the ``DBMSDestination`` cleanup, ``EXA AT conn TABLE`` remains invalid, but now reports the missing table name after consuming ``TABLE`` instead of falling through to the ``STATEMENT`` alternative. The dedicated regression checks this consumption boundary; exact diagnostic text is not promised unchanged.
+
+The dedicated report step does not turn historical debt into a new mandatory gate. ``-PlookaheadFailOnFindings=true`` opts into failure when contexts need review; generation or analysis errors always fail. ``LookaheadAuditTest`` exercises changed dependencies, nullable/FOLLOW effects, stable anchors, opaque inputs and conservative handling; ordinary ``check`` runs these tests alongside the parser regressions. Full applicable checks and paired project JMH measurements are still required before publishing a cleanup.
+
 
 Commit a Pull Request
 ---------------------------------

src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java

@@ -0,0 +1,76 @@
+/*-
+ * #%L
+ * JSQLParser library
+ * %%
+ * Copyright (C) 2004 - 2019 JSQLParser
+ * %%
+ * Dual licensed under GNU LGPL 2.1 or Apache License 2.0
+ * #L%
+ */
+package net.sf.jsqlparser.parser;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed;
+import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect;
+import net.sf.jsqlparser.statement.ConnectionDefinition;
+import net.sf.jsqlparser.statement.CloudConnectionDefinition;
+import net.sf.jsqlparser.statement.export.DBMSDestination;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class LookaheadCleanupTest {
+    @ParameterizedTest
+    @ValueSource(strings = {"EXA AT conn TABLE t", "ORA AT conn TABLE schemaName.tableName",
+            "JDBC DRIVER = 'driverName' AT conn TABLE \"t\"",
+            "EXA AT conn TABLE t (c1, c2) REPLACE TRUNCATE"})
+    void dispatchesTableDestinationsWithCompleteConsumption(String sql) throws Exception {
+        CCJSqlParser parser = CCJSqlParserUtil.newParser(sql);
+        DBMSDestination destination = parser.DBMSDestination();
+        assertNotNull(destination.getTable());
+        assertNull(destination.getStatement());
+        assertNotNull(destination.getConnectionDefinition());
+        assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind);
+        assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO " + sql, true,
+                p -> p.withDialect(Dialect.EXASOL));
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"EXA AT conn STATEMENT 'insert into t values (?)'",
+            "ORA AT conn STATEMENT 'insert into t values (?)'"})
+    void preservesStatementDestinations(String sql) throws Exception {
+        CCJSqlParser parser = CCJSqlParserUtil.newParser(sql);
+        DBMSDestination destination = parser.DBMSDestination();
+        assertNull(destination.getTable());
+        assertEquals("insert into t values (?)", destination.getStatement().getValue());
+        assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind);
+        assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO " + sql, true,
+                p -> p.withDialect(Dialect.EXASOL));
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"EXA AT conn TABLE", "ORA AT conn STATEMENT", "EXA AT conn SELECT t"})
+    void rejectsIncompleteAndWrongDestinationPrefixes(String sql) {
+        CCJSqlParser parser = CCJSqlParserUtil.newParser(sql);
+        assertThrows(ParseException.class, parser::DBMSDestination);
+        if (sql.endsWith("TABLE")) {
+            assertEquals(CCJSqlParserConstants.K_TABLE, parser.getToken(0).kind);
+            assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind);
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"AT conn", "AT '127.0.0.1'", "AT CLOUD NONE conn",
+            "AT CLOUD AZURE BLOBSTORAGE conn"})
+    void preservesCloudAndOrdinaryConnections(String sql) throws Exception {
+        CCJSqlParser parser = CCJSqlParserUtil.newParser(sql);
+        ConnectionDefinition connection = parser.ConnectionOrCloudConnectionDefinition();
+        if (sql.startsWith("AT CLOUD")) {
+            assertInstanceOf(CloudConnectionDefinition.class, connection);
+        } else {
+            assertEquals(ConnectionDefinition.class, connection.getClass());
+        }
+        assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind);
+        assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO CSV " + sql + " FILE 'file.csv'",
+                true, p -> p.withDialect(Dialect.EXASOL));
+    }
+}