fix(parser): support MySQL index options and prefix-length key parts in index DDL (#2505)

* fix(parser): support index key parts with a prefix length and a direction

MySQL allows a key part to carry both a prefix length and a sort direction,
e.g. "CREATE INDEX i ON t (c1(20) DESC)". IndexColumnWithParams() accepted at
most one optional CreateParameter() per key part, so the prefix length consumed
it and the following ASC/DESC could not be matched.

Collect the parameters in a loop instead. IndexColumnsWithParamsList() is shared
by the CREATE INDEX and the ALTER TABLE ADD INDEX paths, so both are fixed.

Refs #2490

* fix(parser): support MySQL index options in CREATE INDEX and DROP INDEX

Several valid MySQL index DDL statements were rejected because their option
keywords are tokens of their own and were therefore not reachable from the
option lists that CREATE INDEX and DROP INDEX use:

  CREATE INDEX i ON t (c1) KEY_BLOCK_SIZE = 8
  CREATE INDEX i ON t (c1) ALGORITHM = INPLACE LOCK = NONE
  CREATE FULLTEXT INDEX i ON t (body) WITH PARSER ngram
  CREATE SPATIAL INDEX i ON t (g)
  DROP INDEX i ON t ALGORITHM = INPLACE LOCK = NONE

The keywords are added to the existing flat token lists of CreateParameter()
and Drop() rather than as new grammar alternatives, so no new choice is
introduced and the JavaCC warning count is unchanged.

LOCK is the one exception: it also starts a LOCK TABLE statement, so taking it
unconditionally as a DROP argument would be ambiguous with the next statement.
It is guarded by a semantic lookahead that only accepts it when it is not
followed by TABLE.

Refs #2490

* refactor(parser): clarify key part option variable names in IndexColumnWithParams

Rename the accumulator to columnParams and keep parameter for the result of a
single CreateParameter(), matching how the other CreateParameter() loops in the
grammar name them. Collect eagerly and pass null only when no option was parsed,
because ColumnParams renders a separating space for a non-null list.

Author: minleejae

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -10979,24 +10979,30 @@ List<Index.ColumnParams> ColumnNamesWithParamsList() : {
 
 Index.ColumnParams IndexColumnWithParams(): {
     String columnName = null;
+    // the options collected for this key part, and the result of a single CreateParameter()
+    List<String> columnParams = new ArrayList<String>();
     List<String> parameter = null;
     Expression expression = null;
     Index.ColumnParams column = null;
 }
 {
     (
         columnName=RelObjectName()
-        { parameter = null; }
-        [ parameter = CreateParameter() ]
+        // MySQL allows a key part to carry a prefix length and a direction, e.g. "c1(20) DESC",
+        // so more than one parameter has to be collected here.
+        ( LOOKAHEAD(2) parameter = CreateParameter() { columnParams.addAll(parameter); } )*
         {
-            column = new Index.ColumnParams(columnName, parameter);
+            // ColumnParams renders a separating space for a non-null list, so a key part
+            // without options has to be given null rather than an empty list.
+            column = new Index.ColumnParams(columnName,
+                    columnParams.isEmpty() ? null : columnParams);
         }
         |
         "(" expression=Expression() ")"
-        { parameter = null; }
-        [ LOOKAHEAD(2) parameter = CreateParameter() ]
+        ( LOOKAHEAD(2) parameter = CreateParameter() { columnParams.addAll(parameter); } )*
         {
-            column = new Index.ColumnParams(expression, parameter);
+            column = new Index.ColumnParams(expression,
+                    columnParams.isEmpty() ? null : columnParams);
         }
     )
     {
@@ -11846,6 +11852,11 @@ List<String> CreateParameter():
             | tk=<K_TIME_KEY_EXPR> | tk=<K_RAW> | tk=<K_HASH> | tk=<K_FIRST> | tk=<K_LAST> | tk = <K_SIGNED>  | tk = <K_UNSIGNED>
             | tk=<K_ENGINE> | tk=<K_IDENTITY> | tk=<K_MATERIALIZED> | tk=<K_SAMPLE> | tk=<K_ALWAYS>
             | tk=<K_VISIBLE> | tk=<K_INVISIBLE>
+            // MySQL index_option / algorithm_option / lock_option keywords, e.g. the trailing
+            // "KEY_BLOCK_SIZE = 8 ALGORITHM = INPLACE LOCK = NONE" of CREATE INDEX, the
+            // "WITH PARSER" option, and the FULLTEXT / SPATIAL index types of CREATE INDEX.
+            | tk=<K_KEY_BLOCK_SIZE> | tk=<K_ALGORITHM> | tk=<K_LOCK> | tk=<K_NONE>
+            | tk=<K_PARSER> | tk=<K_FULLTEXT> | tk=<K_SPATIAL>
             | tk="="
         )
         { param.add(tk.image); }
@@ -12021,11 +12032,19 @@ Drop Drop():
     (
         (
             tk=<S_IDENTIFIER> | tk=<K_CASCADE> | tk=<K_RESTRICT>
+            // MySQL DROP INDEX accepts a trailing algorithm_option / lock_option,
+            // e.g. "DROP INDEX i ON t ALGORITHM = INPLACE LOCK = NONE".
+            | tk=<K_ALGORITHM> | tk=<K_NONE> | tk="="
         ) { dropArgs.add(tk.image); }
         |
         (
             <K_ON> name = Table() { dropArgs.add("ON"); dropArgs.add(name.toString()); }
         )
+        |
+        // The lock_option of DROP INDEX. LOCK also starts a LOCK TABLE statement, so it is only
+        // taken as a DROP argument when it cannot be the beginning of the next statement.
+        LOOKAHEAD({ getToken(1).kind == K_LOCK && getToken(2).kind != K_TABLE })
+        tk=<K_LOCK> { dropArgs.add(tk.image); }
     )*
     {
         if (dropArgs.size() > 0) {

src/test/java/net/sf/jsqlparser/statement/alter/AlterTest.java

@@ -2406,4 +2406,12 @@ public void testAlterTableAddConstraintPrimaryKeyUsingIndexNameAndTablespace()
 
         assertSqlCanBeParsedAndDeparsed(sql);
     }
+
+    @Test
+    public void testAlterTableAddIndexKeyPartWithPrefixLengthAndDirectionIssue2490()
+            throws JSQLParserException {
+        assertSqlCanBeParsedAndDeparsed("ALTER TABLE t ADD INDEX i05 (c1 (20) DESC)");
+        assertSqlCanBeParsedAndDeparsed("ALTER TABLE t ADD INDEX i33 (c1 (20) ASC)");
+        assertSqlCanBeParsedAndDeparsed("ALTER TABLE t ADD UNIQUE INDEX i34 (c1 (10) DESC)");
+    }
 }

src/test/java/net/sf/jsqlparser/statement/create/CreateIndexTest.java

@@ -179,4 +179,59 @@ public void testCreateIndexVisibility() throws JSQLParserException {
     public void testCreateIndexIncludeIssue2459() throws JSQLParserException {
         assertSqlCanBeParsedAndDeparsed("CREATE INDEX idx_a ON t1 (a) INCLUDE (b, c)");
     }
+
+    @Test
+    public void testCreateIndexKeyPartWithPrefixLengthAndDirectionIssue2490()
+            throws JSQLParserException {
+        // MySQL writes the prefix length without a space, JSqlParser deparses it with one.
+        String statement = "CREATE INDEX i03 ON t (c1(20) DESC)";
+        CreateIndex createIndex = (CreateIndex) parserManager.parse(new StringReader(statement));
+
+        List<String> params = createIndex.getIndex().getColumns().get(0).getParams();
+        assertEquals(2, params.size());
+        assertEquals("(20)", params.get(0));
+        assertEquals("DESC", params.get(1));
+
+        assertSqlCanBeParsedAndDeparsed("CREATE INDEX i03 ON t (c1 (20) DESC)");
+        assertSqlCanBeParsedAndDeparsed("CREATE INDEX i04 ON t (c1 (20) ASC, c2 (10) DESC)");
+        assertSqlCanBeParsedAndDeparsed("CREATE UNIQUE INDEX i25 ON t (c1 (10) DESC)");
+    }
+
+    @Test
+    public void testCreateIndexKeyBlockSizeIssue2490() throws JSQLParserException {
+        assertSqlCanBeParsedAndDeparsed("CREATE INDEX i08 ON t (c1) KEY_BLOCK_SIZE = 8");
+        assertSqlCanBeParsedAndDeparsed("CREATE INDEX i09 ON t (c1) KEY_BLOCK_SIZE 8");
+        assertSqlCanBeParsedAndDeparsed(
+                "CREATE INDEX i14 ON t (c1) USING BTREE KEY_BLOCK_SIZE = 8 COMMENT 'combo' INVISIBLE");
+    }
+
+    @Test
+    public void testCreateIndexAlgorithmAndLockOptionsIssue2490() throws JSQLParserException {
+        assertSqlCanBeParsedAndDeparsed(
+                "CREATE INDEX i10 ON t (c1) ALGORITHM = INPLACE LOCK = NONE");
+        assertSqlCanBeParsedAndDeparsed("CREATE INDEX i11 ON t (c1) ALGORITHM INPLACE LOCK NONE");
+        assertSqlCanBeParsedAndDeparsed("CREATE INDEX i12 ON t (c1) ALGORITHM = INPLACE");
+        assertSqlCanBeParsedAndDeparsed("CREATE INDEX i13 ON t (c1) LOCK = NONE");
+
+        CreateIndex createIndex = (CreateIndex) parserManager
+                .parse(new StringReader("CREATE INDEX i10 ON t (c1) ALGORITHM=INPLACE LOCK=NONE"));
+        assertEquals(List.of("ALGORITHM", "=", "INPLACE", "LOCK", "=", "NONE"),
+                createIndex.getTailParameters());
+    }
+
+    @Test
+    public void testCreateFullTextAndSpatialIndexIssue2490() throws JSQLParserException {
+        // These used to fall back to UnsupportedStatement instead of producing a CreateIndex.
+        CreateIndex fullText = (CreateIndex) parserManager
+                .parse(new StringReader("CREATE FULLTEXT INDEX i17 ON t (body)"));
+        assertEquals("FULLTEXT", fullText.getIndex().getType());
+
+        CreateIndex spatial = (CreateIndex) parserManager
+                .parse(new StringReader("CREATE SPATIAL INDEX i19 ON t (g)"));
+        assertEquals("SPATIAL", spatial.getIndex().getType());
+
+        assertSqlCanBeParsedAndDeparsed("CREATE FULLTEXT INDEX i17 ON t (body)");
+        assertSqlCanBeParsedAndDeparsed("CREATE FULLTEXT INDEX i18 ON t (body) WITH PARSER ngram");
+        assertSqlCanBeParsedAndDeparsed("CREATE SPATIAL INDEX i19 ON t (g)");
+    }
 }

src/test/java/net/sf/jsqlparser/statement/drop/DropTest.java

@@ -12,8 +12,10 @@
 import java.io.StringReader;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.parser.CCJSqlParserManager;
+import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Table;
 import net.sf.jsqlparser.statement.Statement;
+import net.sf.jsqlparser.statement.Statements;
 import static net.sf.jsqlparser.test.TestUtils.*;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import org.junit.jupiter.api.Test;
@@ -152,4 +154,19 @@ void dropTemporaryTableTestIssue1712() throws JSQLParserException {
         String sqlStr = "drop temporary table if exists tmp_MwYT8N0z";
         assertSqlCanBeParsedAndDeparsed(sqlStr, true);
     }
+
+    @Test
+    public void testDropIndexAlgorithmAndLockOptionsIssue2490() throws JSQLParserException {
+        assertSqlCanBeParsedAndDeparsed("DROP INDEX i15 ON t ALGORITHM = INPLACE LOCK = NONE");
+        assertSqlCanBeParsedAndDeparsed("DROP INDEX i16 ON t ALGORITHM INPLACE");
+        assertSqlCanBeParsedAndDeparsed("DROP INDEX i17 ON t LOCK = NONE");
+    }
+
+    @Test
+    public void testDropTableFollowedByLockTableIssue2490() throws JSQLParserException {
+        // LOCK must not be swallowed as a DROP argument when it starts the next statement.
+        Statements statements = CCJSqlParserUtil.parseStatements(
+                "DROP TABLE t1; LOCK TABLE t2 IN SHARE MODE;");
+        assertEquals(2, statements.size());
+    }
 }