Name lexer tokens after the MySQL grammar and default to MySQL 8.4.10

packages/mysql-parser/README.md

@@ -14,7 +14,7 @@ It outputs a compact parse table and a list of MySQL tokens for the lexer:
 mysql-server sources
   sql/sql_yacc.yy ──▶ Bison 3.8.2 ──▶ automaton.xml ─┬─▶ generate-parse-table.php ──▶ mysql-parse-table.php
                                                      |
-  sql/lex.h ─────────────────────────────────────────┴─▶ generate-tokens.php ───────▶ class-wp-mysql-tokens.php
+  sql/lex.h ─────────────────────────────────────────┴─▶ generate-tokens.php ───────▶ class-wp-mysql-lexer.php
 ```
 
 The runtime ships only the compiled artifacts and a thin parser implementation.
@@ -37,6 +37,30 @@ $tokens = ( new WP_MySQL_Lexer( 'SELECT 1 + 2' ) )->remaining_tokens();
 $ast    = $parser->parse( $tokens );
 ```
 
+## Tokens and AST nodes
+AST nodes and tokens use the names from the official MySQL grammar (`sql/sql_yacc.yy`):
+
+- **Nodes** are named after grammar rules, such as `select_item` or `simple_expr`.
+- **Keywords** are looked up by their SQL spelling in `WP_MySQL_Lexer::KEYWORDS`,
+  which is generated from the MySQL keyword list (`sql/lex.h`). Like MySQL, the lexer
+  emits a few of them as other terminals: `||` as `OR2_SYM` (without `PIPES_AS_CONCAT`),
+  `NOT` as `NOT2_SYM` (with `HIGH_NOT_PRECEDENCE`), and `WITH ROLLUP` as `WITH_ROLLUP_SYM`.
+- **Other tokens** are `WP_MySQL_Lexer` constants named after their grammar terminals,
+  such as `IDENT`, `TEXT_STRING`, `NUM`, or `EQ` (`=`).
+- **Single-character terminals**, which the grammar writes as a quoted character
+  like `'('`, are named after the character, such as `LEFT_PAREN`, `COMMA`, or `ASTERISK`.
+
+```php
+$ast->get_descendant_nodes( 'select_item' );                               // A grammar rule.
+$ast->get_first_descendant_token( WP_MySQL_Lexer::KEYWORDS['DISTINCT'] ); // A keyword.
+$ast->get_first_descendant_token( WP_MySQL_Lexer::IDENT );                 // A grammar terminal.
+$ast->get_first_descendant_token( WP_MySQL_Lexer::ASTERISK );              // A single character.
+```
+
+Token IDs are the grammar's own token numbers. Like in MySQL, the token stream of
+valid input ends with `END_OF_INPUT` and `YYEOF`. For debugging, `$token->get_name()`
+returns a readable token name.
+
 ## Development
 This package includes the full grammar compilation pipeline. The generated artifacts
 are committed to the repository, and they only need to be regenerated on grammar

packages/mysql-parser/src/class-wp-mysql-token.php

@@ -62,11 +62,7 @@ public function get_name(): string {
 	 */
 	public function get_value(): string {
 		$value = $this->get_bytes();
-		if (
-			WP_MySQL_Lexer::SINGLE_QUOTED_TEXT === $this->id
-			|| WP_MySQL_Lexer::DOUBLE_QUOTED_TEXT === $this->id
-			|| WP_MySQL_Lexer::BACK_TICK_QUOTED_ID === $this->id
-		) {
+		if ( WP_MySQL_Lexer::TEXT_STRING === $this->id || WP_MySQL_Lexer::IDENT_QUOTED === $this->id ) {
 			// Remove bounding quotes.
 			$quote = $value[0];
 			$value = substr( $value, 1, -1 );
@@ -79,7 +75,7 @@ public function get_value(): string {
 			 * See: https://dev.mysql.com/doc/refman/8.4/en/identifiers.html
 			 */
 			if (
-				WP_MySQL_Lexer::BACK_TICK_QUOTED_ID === $this->id
+				WP_MySQL_Lexer::IDENT_QUOTED === $this->id
 				|| $this->sql_mode_no_backslash_escapes_enabled
 			) {
 				return str_replace( $quote . $quote, $quote, $value );

packages/mysql-parser/tests/WP_MySQL_Server_Suite_Parser_Tests.php

@@ -20,7 +20,7 @@ class WP_MySQL_Server_Suite_Parser_Tests extends TestCase {
 	 * legitimately changes.
 	 */
 	const EXPECTED_QUERIES  = 69158;
-	const EXPECTED_FAILURES = 170;
+	const EXPECTED_FAILURES = 167;
 
 	public function test_corpus_acceptance_rate(): void {
 		$parser = WP_MySQL_Parser_Factory::create_parser();
@@ -36,7 +36,7 @@ public function test_corpus_acceptance_rate(): void {
 			++$total;
 
 			// Pin the MySQL version. The failure set in FAILURES_PATH depends on it.
-			$tokens = ( new WP_MySQL_Lexer( $query, 80038 ) )->remaining_tokens();
+			$tokens = ( new WP_MySQL_Lexer( $query, 80410 ) )->remaining_tokens();
 			if ( null === $parser->parse( $tokens ) ) {
 				$failures[] = $query;
 			}

packages/mysql-parser/tests/WP_MySQL_Token_Data_Tests.php

@@ -29,6 +29,18 @@ public function test_every_emittable_token_has_a_derived_name(): void {
 		}
 	}
 
+	public function test_token_constants_have_distinct_numbers(): void {
+		// Each token constant names one grammar terminal; none is an alias.
+		$numbers = array();
+		foreach ( ( new ReflectionClass( WP_MySQL_Lexer::class ) )->getConstants() as $name => $value ) {
+			if ( is_int( $value ) && $value >= 0 && 0 !== strpos( $name, 'SQL_MODE_' ) ) {
+				$this->assertArrayNotHasKey( $value, $numbers, "Constant $name shares token $value with another constant" );
+				$numbers[ $value ] = $name;
+			}
+		}
+		$this->assertNotEmpty( $numbers );
+	}
+
 	public function test_function_keywords_are_a_subset_of_keywords(): void {
 		$this->assertNotEmpty( WP_MySQL_Lexer::FUNCTIONS );
 		foreach ( array_keys( WP_MySQL_Lexer::FUNCTIONS ) as $keyword ) {
@@ -64,12 +76,12 @@ public function test_derived_names_prefer_plain_keywords_and_constants(): void {
 		$this->assertSame( 'CURRENT_DATE', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::KEYWORDS['CURDATE'] ) );
 
 		// Constant name for non-keyword tokens.
-		$this->assertSame( 'IDENTIFIER', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::IDENTIFIER ) );
-		$this->assertSame( 'SINGLE_QUOTED_TEXT', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::SINGLE_QUOTED_TEXT ) );
-		$this->assertSame( 'AT_SIGN_SYMBOL', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::AT_SIGN_SYMBOL ) );
-		$this->assertSame( 'WITH_ROLLUP_SYMBOL', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::WITH_ROLLUP_SYMBOL ) );
+		$this->assertSame( 'IDENT', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::IDENT ) );
+		$this->assertSame( 'TEXT_STRING', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::TEXT_STRING ) );
+		$this->assertSame( 'AT_SIGN', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::AT_SIGN ) );
+		$this->assertSame( 'WITH_ROLLUP_SYM', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::WITH_ROLLUP_SYM ) );
 		$this->assertSame( 'END_OF_INPUT', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::END_OF_INPUT ) );
-		$this->assertSame( 'END_MARKER', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::END_MARKER ) );
+		$this->assertSame( 'YYEOF', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::YYEOF ) );
 	}
 
 	public function test_internal_scanner_sentinels_cannot_collide_with_grammar_tokens(): void {

packages/mysql-parser/tests/WP_MySQL_Token_Tests.php

@@ -15,7 +15,7 @@ class WP_MySQL_Token_Tests extends TestCase {
 	 * @return WP_MySQL_Token      The first matching token.
 	 */
 	private static function first_token( string $sql, string $name, array $sql_modes = array() ): WP_MySQL_Token {
-		foreach ( ( new WP_MySQL_Lexer( $sql, 80400, $sql_modes ) )->remaining_tokens() as $token ) {
+		foreach ( ( new WP_MySQL_Lexer( $sql, 80410, $sql_modes ) )->remaining_tokens() as $token ) {
 			if ( $token->get_name() === $name ) {
 				return $token;
 			}
@@ -24,13 +24,13 @@ private static function first_token( string $sql, string $name, array $sql_modes
 	}
 
 	public function test_get_value_unquotes_string_literals(): void {
-		$this->assertSame( "a'b", self::first_token( "SELECT 'a''b'", 'SINGLE_QUOTED_TEXT' )->get_value() );
-		$this->assertSame( 'a"b', self::first_token( 'SELECT "a""b"', 'SINGLE_QUOTED_TEXT' )->get_value() );
-		$this->assertSame( "new\nline", self::first_token( "SELECT 'new\\nline'", 'SINGLE_QUOTED_TEXT' )->get_value() );
+		$this->assertSame( "a'b", self::first_token( "SELECT 'a''b'", 'TEXT_STRING' )->get_value() );
+		$this->assertSame( 'a"b', self::first_token( 'SELECT "a""b"', 'TEXT_STRING' )->get_value() );
+		$this->assertSame( "new\nline", self::first_token( "SELECT 'new\\nline'", 'TEXT_STRING' )->get_value() );
 	}
 
 	public function test_get_value_honors_no_backslash_escapes_mode(): void {
-		$token = self::first_token( "SELECT 'a\\nb'", 'SINGLE_QUOTED_TEXT', array( 'NO_BACKSLASH_ESCAPES' ) );
+		$token = self::first_token( "SELECT 'a\\nb'", 'TEXT_STRING', array( 'NO_BACKSLASH_ESCAPES' ) );
 		$this->assertSame( 'a\\nb', $token->get_value() );
 	}
 
@@ -40,41 +40,41 @@ public function test_get_value_preserves_non_utf8_bytes_in_string_literals(): vo
 		$raw = chr( 0xFF ) . chr( 0xFE );
 		$this->assertSame(
 			$raw,
-			self::first_token( "SELECT '$raw'", 'SINGLE_QUOTED_TEXT' )->get_value()
+			self::first_token( "SELECT '$raw'", 'TEXT_STRING' )->get_value()
 		);
 
 		// A backslash escape strips the backslash and preserves the following non-UTF-8 byte.
 		$this->assertSame(
 			chr( 0xE9 ),
-			self::first_token( "SELECT '\\" . chr( 0xE9 ) . "'", 'SINGLE_QUOTED_TEXT' )->get_value()
+			self::first_token( "SELECT '\\" . chr( 0xE9 ) . "'", 'TEXT_STRING' )->get_value()
 		);
 
 		// Valid multibyte UTF-8 still round-trips.
 		$this->assertSame(
 			'café 🙂',
-			self::first_token( "SELECT 'café 🙂'", 'SINGLE_QUOTED_TEXT' )->get_value()
+			self::first_token( "SELECT 'café 🙂'", 'TEXT_STRING' )->get_value()
 		);
 	}
 
 	public function test_get_value_unescapes_newlines_and_multibyte_characters(): void {
 		$this->assertSame(
 			"\n",
-			self::first_token( "SELECT '\\\n'", 'SINGLE_QUOTED_TEXT' )->get_value()
+			self::first_token( "SELECT '\\\n'", 'TEXT_STRING' )->get_value()
 		);
 		$this->assertSame(
 			'🙂',
-			self::first_token( "SELECT '\\🙂'", 'SINGLE_QUOTED_TEXT' )->get_value()
+			self::first_token( "SELECT '\\🙂'", 'TEXT_STRING' )->get_value()
 		);
 	}
 
 	public function test_get_value_unquotes_backtick_identifiers(): void {
-		$this->assertSame( 'col name', self::first_token( 'SELECT `col name` FROM t', 'BACK_TICK_QUOTED_ID' )->get_value() );
+		$this->assertSame( 'col name', self::first_token( 'SELECT `col name` FROM t', 'IDENT_QUOTED' )->get_value() );
 	}
 
 	public function test_get_value_unquotes_ansi_identifiers(): void {
 		$this->assertSame(
 			'a"b',
-			self::first_token( 'SELECT "a""b"', 'BACK_TICK_QUOTED_ID', array( 'ANSI_QUOTES' ) )->get_value()
+			self::first_token( 'SELECT "a""b"', 'IDENT_QUOTED', array( 'ANSI_QUOTES' ) )->get_value()
 		);
 	}
 
@@ -84,7 +84,7 @@ public function test_get_value_does_not_unquote_unquoted_tokens(): void {
 		// by token ids and must return keyword bytes as-is.
 		$this->assertSame( 'SSL', self::first_token( 'CREATE USER u REQUIRE SSL', 'SSL' )->get_value() );
 		$this->assertSame( 'SELECT', self::first_token( 'SELECT 1', 'SELECT' )->get_value() );
-		$this->assertSame( '42', self::first_token( 'SELECT 42', 'INT_NUMBER' )->get_value() );
+		$this->assertSame( '42', self::first_token( 'SELECT 42', 'NUM' )->get_value() );
 	}
 
 	public function test_get_name_resolves_bison_terminal_names(): void {
@@ -94,7 +94,7 @@ public function test_get_name_resolves_bison_terminal_names(): void {
 			$names[] = $token->get_name();
 		}
 		$this->assertSame(
-			array( 'SELECT', 'SINGLE_QUOTED_TEXT', 'COMMA_SYMBOL', 'OPEN_PAR_SYMBOL', 'INT_NUMBER', 'CLOSE_PAR_SYMBOL', 'END_OF_INPUT', 'END_MARKER' ),
+			array( 'SELECT', 'TEXT_STRING', 'COMMA', 'LEFT_PAREN', 'NUM', 'RIGHT_PAREN', 'END_OF_INPUT', 'YYEOF' ),
 			$names
 		);
 	}

packages/mysql-parser/tests/data/corpus-failures.csv

@@ -653,10 +653,6 @@ UNLOCK TABLES"
 "SELECT 'a%' NOT LIKE 'a!%' ESCAPE '!', 'a%' NOT LIKE 'a!' || '%' ESCAPE '!'"
 "SELECT 'a%' NOT LIKE 'a!%' ESCAPE '$', 'a%' NOT LIKE 'a!' || '%' ESCAPE '$'"
 "SELECT * from ""full"""
-"SELECT 1 /*!080100	+1*/ AS should_return_2"
-"SELECT 1 /*!080100
-+1*/ AS should_return_2"
-"SELECT 1 /*!080100 +1*/ AS should_return_2"
 "SELECT 1 /*!99999 /* */ */"
 "SET sql_mode = 'NO_ENGINE_SUBSTITUTION';
 create procedure test_resignal()

packages/mysql-parser/tools/fetch-mysql-grammar.sh

@@ -42,5 +42,8 @@ else
 	sha256 "$build_dir/sql_yacc.yy" "$build_dir/lex.h" | sed 's/^/    /'
 fi
 
+# Record the tag for the generators, so the generated files name the right source.
+echo "$MYSQL_TAG" > "$build_dir/mysql-tag"
+
 echo "  -> $build_dir/sql_yacc.yy ($(wc -l < "$build_dir/sql_yacc.yy" | tr -d ' ') lines)"
 echo "  -> $build_dir/lex.h ($(wc -l < "$build_dir/lex.h" | tr -d ' ') lines)"

packages/mysql-parser/tools/generate-parse-table.php

@@ -45,9 +45,13 @@
 }
 $xml_path    = $argv[1];
 $output_path = $argv[2];
-$mysql_tag   = getenv( 'MYSQL_TAG' );
-if ( false === $mysql_tag || '' === $mysql_tag ) {
-	$mysql_tag = 'mysql-8.4.3';
+
+// The tag of the grammar sources, recorded next to the automaton by fetch-mysql-grammar.sh.
+$tag_path  = dirname( $xml_path ) . '/mysql-tag';
+$mysql_tag = is_file( $tag_path ) ? trim( file_get_contents( $tag_path ) ) : '';
+if ( '' === $mysql_tag ) {
+	fwrite( STDERR, "error: MySQL tag not found in $tag_path; run fetch-mysql-grammar.sh first.\n" );
+	exit( 1 );
 }
 
 ini_set( 'memory_limit', '3G' );