diff --git a/packages/mysql-parser/README.md b/packages/mysql-parser/README.md index b4f61f9b1..254e7f4fa 100644 --- a/packages/mysql-parser/README.md +++ b/packages/mysql-parser/README.md @@ -14,7 +14,7 @@ It outputs a compact parse table and a list of MySQL tokens for the lexer: mysql-server sources sql/sql_yacc.yy ──▶ Bison 3.8.2 ──▶ automaton.xml ─┬─▶ generate-parse-table.php ──▶ mysql-parse-table.php | - sql/lex.h ─────────────────────────────────────────┴─▶ generate-tokens.php ───────▶ class-wp-mysql-tokens.php + sql/lex.h ─────────────────────────────────────────┴─▶ generate-tokens.php ───────▶ class-wp-mysql-lexer.php ``` The runtime ships only the compiled artifacts and a thin parser implementation. @@ -37,6 +37,30 @@ $tokens = ( new WP_MySQL_Lexer( 'SELECT 1 + 2' ) )->remaining_tokens(); $ast = $parser->parse( $tokens ); ``` +## Tokens and AST nodes +AST nodes and tokens use the names from the official MySQL grammar (`sql/sql_yacc.yy`): + +- **Nodes** are named after grammar rules, such as `select_item` or `simple_expr`. +- **Keywords** are looked up by their SQL spelling in `WP_MySQL_Lexer::KEYWORDS`, + which is generated from the MySQL keyword list (`sql/lex.h`). Like MySQL, the lexer + emits a few of them as other terminals: `||` as `OR2_SYM` (without `PIPES_AS_CONCAT`), + `NOT` as `NOT2_SYM` (with `HIGH_NOT_PRECEDENCE`), and `WITH ROLLUP` as `WITH_ROLLUP_SYM`. +- **Other tokens** are `WP_MySQL_Lexer` constants named after their grammar terminals, + such as `IDENT`, `TEXT_STRING`, `NUM`, or `EQ` (`=`). +- **Single-character terminals**, which the grammar writes as a quoted character + like `'('`, are named after the character, such as `LEFT_PAREN`, `COMMA`, or `ASTERISK`. + +```php +$ast->get_descendant_nodes( 'select_item' ); // A grammar rule. +$ast->get_first_descendant_token( WP_MySQL_Lexer::KEYWORDS['DISTINCT'] ); // A keyword. +$ast->get_first_descendant_token( WP_MySQL_Lexer::IDENT ); // A grammar terminal. +$ast->get_first_descendant_token( WP_MySQL_Lexer::ASTERISK ); // A single character. +``` + +Token IDs are the grammar's own token numbers. Like in MySQL, the token stream of +valid input ends with `END_OF_INPUT` and `YYEOF`. For debugging, `$token->get_name()` +returns a readable token name. + ## Development This package includes the full grammar compilation pipeline. The generated artifacts are committed to the repository, and they only need to be regenerated on grammar diff --git a/packages/mysql-parser/src/class-wp-mysql-lexer.php b/packages/mysql-parser/src/class-wp-mysql-lexer.php index cb13dcddc..9f6c1660d 100644 --- a/packages/mysql-parser/src/class-wp-mysql-lexer.php +++ b/packages/mysql-parser/src/class-wp-mysql-lexer.php @@ -37,62 +37,61 @@ class WP_MySQL_Lexer { // BEGIN: AUTO-GENERATED MYSQL GRAMMAR SYMBOLS // This block is auto-generated from MySQL mysql-8.4.10. DO NOT EDIT! // phpcs:disable - const OPEN_PAR_SYMBOL = 40; - const CLOSE_PAR_SYMBOL = 41; - const OPEN_CURLY_SYMBOL = 123; - const CLOSE_CURLY_SYMBOL = 125; - const AT_SIGN_SYMBOL = 64; - const COLON_SYMBOL = 58; - const COMMA_SYMBOL = 44; - const DOT_SYMBOL = 46; - const SEMICOLON_SYMBOL = 59; - const BITWISE_AND_OPERATOR = 38; - const BITWISE_NOT_OPERATOR = 126; - const BITWISE_OR_OPERATOR = 124; - const BITWISE_XOR_OPERATOR = 94; - const DIV_OPERATOR = 47; - const MINUS_OPERATOR = 45; - const MOD_OPERATOR = 37; - const MULT_OPERATOR = 42; - const PLUS_OPERATOR = 43; - const LOGICAL_NOT_OPERATOR = 33; - const EQUAL_OPERATOR = 415; - const NULL_SAFE_EQUAL_OPERATOR = 416; - const GREATER_OR_EQUAL_OPERATOR = 456; - const GREATER_THAN_OPERATOR = 469; - const LESS_OR_EQUAL_OPERATOR = 522; - const LESS_THAN_OPERATOR = 549; - const NOT_EQUAL_OPERATOR = 614; - const ASSIGN_OPERATOR = 757; - const LOGICAL_AND_OPERATOR = 273; - const LOGICAL_OR_OPERATOR = 642; - const CONCAT_PIPES_SYMBOL = 644; - const SHIFT_LEFT_OPERATOR = 759; - const SHIFT_RIGHT_OPERATOR = 760; - const JSON_SEPARATOR_SYMBOL = 514; - const JSON_UNQUOTED_SEPARATOR_SYMBOL = 907; - const PARAM_MARKER = 652; - const IDENTIFIER = 482; - const BACK_TICK_QUOTED_ID = 484; - const SINGLE_QUOTED_TEXT = 827; - const DOUBLE_QUOTED_TEXT = 827; - const NCHAR_TEXT = 611; - const UNDERSCORE_CHARSET = 852; - const BIN_NUMBER = 292; - const HEX_NUMBER = 474; - const INT_NUMBER = 628; - const LONG_NUMBER = 545; - const ULONGLONG_NUMBER = 849; - const DECIMAL_NUMBER = 377; - const FLOAT_NUMBER = 443; - const NULL2_SYMBOL = 627; - const NOT_SYMBOL = 622; - const NOT2_SYMBOL = 621; - const WITH_SYMBOL = 892; - const ROLLUP_SYMBOL = 734; - const WITH_ROLLUP_SYMBOL = 894; - const END_OF_INPUT = 411; - const END_MARKER = 0; + const LEFT_PAREN = 40; + const RIGHT_PAREN = 41; + const LEFT_BRACE = 123; + const RIGHT_BRACE = 125; + const COMMA = 44; + const DOT = 46; + const COLON = 58; + const SEMICOLON = 59; + const AT_SIGN = 64; + const PLUS = 43; + const MINUS = 45; + const ASTERISK = 42; + const SLASH = 47; + const PERCENT = 37; + const AMPERSAND = 38; + const PIPE = 124; + const CARET = 94; + const TILDE = 126; + const EXCLAMATION = 33; + const EQ = 415; + const EQUAL_SYM = 416; + const NE = 614; + const GE = 456; + const GT_SYM = 469; + const LE = 522; + const LT = 549; + const SET_VAR = 757; + const AND_AND_SYM = 273; + const OR2_SYM = 642; + const OR_OR_SYM = 644; + const SHIFT_LEFT = 759; + const SHIFT_RIGHT = 760; + const JSON_SEPARATOR_SYM = 514; + const JSON_UNQUOTED_SEPARATOR_SYM = 907; + const PARAM_MARKER = 652; + const IDENT = 482; + const IDENT_QUOTED = 484; + const TEXT_STRING = 827; + const NCHAR_STRING = 611; + const UNDERSCORE_CHARSET = 852; + const BIN_NUM = 292; + const HEX_NUM = 474; + const NUM = 628; + const LONG_NUM = 545; + const ULONGLONG_NUM = 849; + const DECIMAL_NUM = 377; + const FLOAT_NUM = 443; + const NULL_SYM = 627; + const NOT_SYM = 622; + const NOT2_SYM = 621; + const WITH = 892; + const ROLLUP_SYM = 734; + const WITH_ROLLUP_SYM = 894; + const END_OF_INPUT = 411; + const YYEOF = 0; const KEYWORDS = array( '!=' => 614, '&&' => 273, '<' => 549, '<<' => 759, '<=' => 522, '<=>' => 416, '<>' => 614, '=' => 415, '>' => 469, '>=' => 456, '>>' => 760, 'ACCESSIBLE' => 259, 'ACCOUNT' => 260, 'ACTION' => 261, 'ACTIVE' => 973, 'ADD' => 262, 'ADDDATE' => 263, 'ADMIN' => 910, 'AFTER' => 264, 'AGAINST' => 265, 'AGGREGATE' => 266, 'ALGORITHM' => 267, 'ALL' => 268, 'ALTER' => 269, 'ALWAYS' => 270, 'ANALYZE' => 272, 'AND' => 274, 'ANY' => 275, 'ARRAY' => 976, 'AS' => 276, 'ASC' => 277, 'ASCII' => 278, 'ASENSITIVE' => 279, 'ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS' => 1161, 'AT' => 280, 'ATTRIBUTE' => 1153, 'AUTHENTICATION' => 1191, 'AUTO' => 1211, 'AUTOEXTEND_SIZE' => 281, 'AUTO_INCREMENT' => 282, 'AVG' => 284, 'AVG_ROW_LENGTH' => 283, 'BACKUP' => 285, 'BEFORE' => 286, 'BEGIN' => 287, 'BERNOULLI' => 1213, 'BETWEEN' => 288, 'BIGINT' => 289, 'BINARY' => 290, 'BINLOG' => 291, 'BIT' => 295, 'BIT_AND' => 293, 'BIT_OR' => 294, 'BIT_XOR' => 296, 'BLOB' => 297, 'BLOCK' => 298, 'BOOL' => 300, 'BOOLEAN' => 299, 'BOTH' => 301, 'BTREE' => 302, 'BUCKETS' => 929, 'BULK' => 1201, 'BY' => 303, 'BYTE' => 304, 'CACHE' => 305, 'CALL' => 306, 'CASCADE' => 307, 'CASCADED' => 308, 'CASE' => 309, 'CAST' => 310, 'CATALOG_NAME' => 311, 'CHAIN' => 312, 'CHALLENGE_RESPONSE' => 1198, 'CHANGE' => 313, 'CHANGED' => 314, 'CHANNEL' => 315, 'CHAR' => 317, 'CHARACTER' => 317, 'CHARSET' => 316, 'CHECK' => 319, 'CHECKSUM' => 318, 'CIPHER' => 320, 'CLASS_ORIGIN' => 321, 'CLIENT' => 322, 'CLONE' => 931, 'CLOSE' => 323, 'COALESCE' => 324, 'CODE' => 325, 'COLLATE' => 326, 'COLLATION' => 327, 'COLUMN' => 329, 'COLUMNS' => 328, 'COLUMN_FORMAT' => 330, 'COLUMN_NAME' => 331, 'COMMENT' => 332, 'COMMIT' => 334, 'COMMITTED' => 333, 'COMPACT' => 335, 'COMPLETION' => 336, 'COMPONENT' => 914, 'COMPRESSED' => 337, 'COMPRESSION' => 338, 'CONCURRENT' => 340, 'CONDITION' => 341, 'CONNECTION' => 342, 'CONSISTENT' => 343, 'CONSTRAINT' => 344, 'CONSTRAINT_CATALOG' => 345, 'CONSTRAINT_NAME' => 346, 'CONSTRAINT_SCHEMA' => 347, 'CONTAINS' => 348, 'CONTEXT' => 349, 'CONTINUE' => 350, 'CONVERT' => 351, 'COUNT' => 352, 'CPU' => 353, 'CREATE' => 354, 'CROSS' => 355, 'CUBE' => 356, 'CUME_DIST' => 932, 'CURDATE' => 357, 'CURRENT' => 358, 'CURRENT_DATE' => 357, 'CURRENT_TIME' => 362, 'CURRENT_TIMESTAMP' => 623, 'CURRENT_USER' => 359, 'CURSOR' => 360, 'CURSOR_NAME' => 361, 'CURTIME' => 362, 'DATA' => 366, 'DATABASE' => 363, 'DATABASES' => 364, 'DATAFILE' => 365, 'DATE' => 370, 'DATETIME' => 367, 'DATE_ADD' => 368, 'DATE_SUB' => 369, 'DAY' => 375, 'DAY_HOUR' => 371, 'DAY_MICROSECOND' => 372, 'DAY_MINUTE' => 373, 'DAY_SECOND' => 374, 'DEALLOCATE' => 376, 'DEC' => 378, 'DECIMAL' => 378, 'DECLARE' => 379, 'DEFAULT' => 380, 'DEFAULT_AUTH' => 381, 'DEFINER' => 382, 'DEFINITION' => 969, 'DELAYED' => 383, 'DELAY_KEY_WRITE' => 384, 'DELETE' => 385, 'DENSE_RANK' => 933, 'DESC' => 386, 'DESCRIBE' => 387, 'DESCRIPTION' => 970, 'DETERMINISTIC' => 389, 'DIAGNOSTICS' => 390, 'DIRECTORY' => 391, 'DISABLE' => 392, 'DISCARD' => 393, 'DISK' => 394, 'DISTINCT' => 395, 'DISTINCTROW' => 395, 'DIV' => 396, 'DO' => 398, 'DOUBLE' => 397, 'DROP' => 399, 'DUAL' => 400, 'DUMPFILE' => 401, 'DUPLICATE' => 402, 'DYNAMIC' => 403, 'EACH' => 404, 'ELSE' => 405, 'ELSEIF' => 406, 'EMPTY' => 954, 'ENABLE' => 407, 'ENCLOSED' => 408, 'ENCRYPTION' => 339, 'END' => 409, 'ENDS' => 410, 'ENFORCED' => 985, 'ENGINE' => 413, 'ENGINES' => 412, 'ENGINE_ATTRIBUTE' => 1154, 'ENUM' => 414, 'ERROR' => 417, 'ERRORS' => 418, 'ESCAPE' => 420, 'ESCAPED' => 419, 'EVENT' => 422, 'EVENTS' => 421, 'EVERY' => 423, 'EXCEPT' => 913, 'EXCHANGE' => 424, 'EXCLUDE' => 934, 'EXECUTE' => 425, 'EXISTS' => 426, 'EXIT' => 427, 'EXPANSION' => 428, 'EXPIRE' => 429, 'EXPLAIN' => 387, 'EXPORT' => 430, 'EXTENDED' => 431, 'EXTENT_SIZE' => 432, 'EXTRACT' => 433, 'FACTOR' => 1192, 'FAILED_LOGIN_ATTEMPTS' => 995, 'FALSE' => 434, 'FAST' => 435, 'FAULTS' => 436, 'FETCH' => 437, 'FIELDS' => 328, 'FILE' => 438, 'FILE_BLOCK_SIZE' => 439, 'FILTER' => 440, 'FINISH' => 1193, 'FIRST' => 441, 'FIRST_VALUE' => 935, 'FIXED' => 442, 'FLOAT' => 444, 'FLOAT4' => 444, 'FLOAT8' => 397, 'FLUSH' => 445, 'FOLLOWING' => 936, 'FOLLOWS' => 446, 'FOR' => 449, 'FORCE' => 447, 'FOREIGN' => 448, 'FORMAT' => 450, 'FOUND' => 451, 'FROM' => 452, 'FULL' => 453, 'FULLTEXT' => 454, 'FUNCTION' => 455, 'GENERAL' => 457, 'GENERATE' => 1203, 'GENERATED' => 458, 'GEOMCOLLECTION' => 460, 'GEOMETRY' => 461, 'GEOMETRYCOLLECTION' => 460, 'GET' => 463, 'GET_FORMAT' => 462, 'GET_SOURCE_PUBLIC_KEY' => 1162, 'GLOBAL' => 464, 'GRANT' => 465, 'GRANTS' => 466, 'GROUP' => 467, 'GROUPING' => 926, 'GROUPS' => 937, 'GROUP_CONCAT' => 468, 'GROUP_REPLICATION' => 459, 'GTIDS' => 1207, 'GTID_ONLY' => 1199, 'HANDLER' => 470, 'HASH' => 471, 'HAVING' => 472, 'HELP' => 473, 'HIGH_PRIORITY' => 475, 'HISTOGRAM' => 928, 'HISTORY' => 959, 'HOST' => 476, 'HOSTS' => 477, 'HOUR' => 481, 'HOUR_MICROSECOND' => 478, 'HOUR_MINUTE' => 479, 'HOUR_SECOND' => 480, 'IDENTIFIED' => 483, 'IF' => 485, 'IGNORE' => 486, 'IGNORE_SERVER_IDS' => 487, 'IMPORT' => 488, 'IN' => 504, 'INACTIVE' => 974, 'INDEX' => 490, 'INDEXES' => 489, 'INFILE' => 491, 'INITIAL' => 1197, 'INITIAL_SIZE' => 492, 'INITIATE' => 1194, 'INNER' => 493, 'INOUT' => 494, 'INSENSITIVE' => 495, 'INSERT' => 496, 'INSERT_METHOD' => 497, 'INSTALL' => 499, 'INSTANCE' => 498, 'INT' => 502, 'INT1' => 836, 'INT2' => 768, 'INT3' => 584, 'INT4' => 502, 'INT8' => 289, 'INTEGER' => 502, 'INTERSECT' => 1200, 'INTERVAL' => 500, 'INTO' => 501, 'INVISIBLE' => 911, 'INVOKER' => 503, 'IO' => 507, 'IO_AFTER_GTIDS' => 505, 'IO_BEFORE_GTIDS' => 506, 'IO_THREAD' => 702, 'IPC' => 508, 'IS' => 509, 'ISOLATION' => 510, 'ISSUER' => 511, 'ITERATE' => 512, 'JOIN' => 513, 'JSON' => 515, 'JSON_ARRAYAGG' => 921, 'JSON_OBJECTAGG' => 920, 'JSON_TABLE' => 955, 'JSON_VALUE' => 1151, 'KEY' => 518, 'KEYRING' => 1190, 'KEYS' => 516, 'KEY_BLOCK_SIZE' => 517, 'KILL' => 519, 'LAG' => 938, 'LANGUAGE' => 520, 'LAST' => 521, 'LAST_VALUE' => 939, 'LATERAL' => 975, 'LEAD' => 940, 'LEADING' => 523, 'LEAVE' => 525, 'LEAVES' => 524, 'LEFT' => 526, 'LESS' => 527, 'LEVEL' => 528, 'LIKE' => 530, 'LIMIT' => 531, 'LINEAR' => 532, 'LINES' => 533, 'LINESTRING' => 534, 'LIST' => 535, 'LOAD' => 536, 'LOCAL' => 537, 'LOCALTIME' => 623, 'LOCALTIMESTAMP' => 623, 'LOCK' => 540, 'LOCKED' => 924, 'LOCKS' => 539, 'LOG' => 1206, 'LOGFILE' => 541, 'LOGS' => 542, 'LONG' => 546, 'LONGBLOB' => 543, 'LONGTEXT' => 544, 'LOOP' => 547, 'LOW_PRIORITY' => 548, 'MANUAL' => 1212, 'MASTER' => 571, 'MATCH' => 574, 'MAX' => 579, 'MAXVALUE' => 582, 'MAX_CONNECTIONS_PER_HOUR' => 575, 'MAX_QUERIES_PER_HOUR' => 576, 'MAX_ROWS' => 577, 'MAX_SIZE' => 578, 'MAX_UPDATES_PER_HOUR' => 580, 'MAX_USER_CONNECTIONS' => 581, 'MEDIUM' => 586, 'MEDIUMBLOB' => 583, 'MEDIUMINT' => 584, 'MEDIUMTEXT' => 585, 'MEMBER' => 977, 'MEMORY' => 587, 'MERGE' => 588, 'MESSAGE_TEXT' => 589, 'MICROSECOND' => 590, 'MID' => 811, 'MIDDLEINT' => 584, 'MIGRATE' => 591, 'MIN' => 596, 'MINUTE' => 594, 'MINUTE_MICROSECOND' => 592, 'MINUTE_SECOND' => 593, 'MIN_ROWS' => 595, 'MOD' => 600, 'MODE' => 597, 'MODIFIES' => 598, 'MODIFY' => 599, 'MONTH' => 601, 'MULTILINESTRING' => 602, 'MULTIPOINT' => 603, 'MULTIPOLYGON' => 604, 'MUTEX' => 605, 'MYSQL_ERRNO' => 606, 'NAME' => 608, 'NAMES' => 607, 'NATIONAL' => 609, 'NATURAL' => 610, 'NCHAR' => 612, 'NDB' => 613, 'NDBCLUSTER' => 613, 'NESTED' => 956, 'NETWORK_NAMESPACE' => 987, 'NEVER' => 616, 'NEW' => 617, 'NEXT' => 618, 'NO' => 624, 'NODEGROUP' => 619, 'NONE' => 620, 'NOT' => 622, 'NOW' => 623, 'NOWAIT' => 925, 'NO_WAIT' => 625, 'NO_WRITE_TO_BINLOG' => 626, 'NTH_VALUE' => 941, 'NTILE' => 942, 'NULL' => 627, 'NULLS' => 943, 'NUMBER' => 629, 'NUMERIC' => 630, 'NVARCHAR' => 631, 'OF' => 922, 'OFF' => 998, 'OFFSET' => 632, 'OJ' => 986, 'OLD' => 984, 'ON' => 633, 'ONE' => 634, 'ONLY' => 635, 'OPEN' => 636, 'OPTIMIZE' => 637, 'OPTIMIZER_COSTS' => 638, 'OPTION' => 640, 'OPTIONAL' => 978, 'OPTIONALLY' => 641, 'OPTIONS' => 639, 'OR' => 645, 'ORDER' => 643, 'ORDINALITY' => 957, 'ORGANIZATION' => 971, 'OTHERS' => 944, 'OUT' => 648, 'OUTER' => 646, 'OUTFILE' => 647, 'OVER' => 945, 'OWNER' => 649, 'PACK_KEYS' => 650, 'PAGE' => 651, 'PARALLEL' => 1208, 'PARSER' => 653, 'PARSE_TREE' => 1205, 'PARTIAL' => 655, 'PARTITION' => 656, 'PARTITIONING' => 658, 'PARTITIONS' => 657, 'PASSWORD' => 659, 'PASSWORD_LOCK_TIME' => 994, 'PATH' => 958, 'PERCENT_RANK' => 946, 'PERSIST' => 908, 'PERSIST_ONLY' => 927, 'PHASE' => 660, 'PLUGIN' => 662, 'PLUGINS' => 663, 'PLUGIN_DIR' => 661, 'POINT' => 664, 'POLYGON' => 665, 'PORT' => 666, 'POSITION' => 667, 'PRECEDES' => 668, 'PRECEDING' => 947, 'PRECISION' => 669, 'PREPARE' => 670, 'PRESERVE' => 671, 'PREV' => 672, 'PRIMARY' => 673, 'PRIVILEGES' => 674, 'PRIVILEGE_CHECKS_USER' => 991, 'PROCEDURE' => 675, 'PROCESS' => 676, 'PROCESSLIST' => 677, 'PROFILE' => 678, 'PROFILES' => 679, 'PROXY' => 680, 'PURGE' => 681, 'QUALIFY' => 1210, 'QUARTER' => 682, 'QUERY' => 683, 'QUICK' => 684, 'RANDOM' => 988, 'RANGE' => 685, 'RANK' => 948, 'READ' => 688, 'READS' => 686, 'READ_ONLY' => 687, 'READ_WRITE' => 689, 'REAL' => 690, 'REBUILD' => 691, 'RECOVER' => 692, 'RECURSIVE' => 915, 'REDO_BUFFER_SIZE' => 694, 'REDUNDANT' => 695, 'REFERENCE' => 972, 'REFERENCES' => 696, 'REGEXP' => 697, 'REGISTRATION' => 1195, 'RELAY' => 698, 'RELAYLOG' => 699, 'RELAY_LOG_FILE' => 700, 'RELAY_LOG_POS' => 701, 'RELAY_THREAD' => 702, 'RELEASE' => 703, 'RELOAD' => 704, 'REMOVE' => 705, 'RENAME' => 706, 'REORGANIZE' => 707, 'REPAIR' => 708, 'REPEAT' => 710, 'REPEATABLE' => 709, 'REPLACE' => 711, 'REPLICA' => 1159, 'REPLICAS' => 1160, 'REPLICATE_DO_DB' => 713, 'REPLICATE_DO_TABLE' => 715, 'REPLICATE_IGNORE_DB' => 714, 'REPLICATE_IGNORE_TABLE' => 716, 'REPLICATE_REWRITE_DB' => 719, 'REPLICATE_WILD_DO_TABLE' => 717, 'REPLICATE_WILD_IGNORE_TABLE' => 718, 'REPLICATION' => 712, 'REQUIRE' => 720, 'REQUIRE_ROW_FORMAT' => 993, 'REQUIRE_TABLE_PRIMARY_KEY_CHECK' => 996, 'RESET' => 721, 'RESIGNAL' => 722, 'RESOURCE' => 963, 'RESPECT' => 949, 'RESTART' => 968, 'RESTORE' => 724, 'RESTRICT' => 725, 'RESUME' => 726, 'RETAIN' => 983, 'RETURN' => 729, 'RETURNED_SQLSTATE' => 727, 'RETURNING' => 999, 'RETURNS' => 728, 'REUSE' => 960, 'REVERSE' => 730, 'REVOKE' => 731, 'RIGHT' => 732, 'RLIKE' => 697, 'ROLE' => 909, 'ROLLBACK' => 733, 'ROLLUP' => 734, 'ROTATE' => 735, 'ROUTINE' => 736, 'ROW' => 739, 'ROWS' => 737, 'ROW_COUNT' => 740, 'ROW_FORMAT' => 738, 'ROW_NUMBER' => 950, 'RTREE' => 741, 'S3' => 1209, 'SAVEPOINT' => 742, 'SCHEDULE' => 743, 'SCHEMA' => 363, 'SCHEMAS' => 364, 'SCHEMA_NAME' => 744, 'SECOND' => 746, 'SECONDARY' => 979, 'SECONDARY_ENGINE' => 980, 'SECONDARY_ENGINE_ATTRIBUTE' => 1155, 'SECONDARY_LOAD' => 981, 'SECONDARY_UNLOAD' => 982, 'SECOND_MICROSECOND' => 745, 'SECURITY' => 747, 'SELECT' => 748, 'SENSITIVE' => 749, 'SEPARATOR' => 750, 'SERIAL' => 752, 'SERIALIZABLE' => 751, 'SERVER' => 754, 'SESSION' => 753, 'SESSION_USER' => 867, 'SET' => 756, 'SHARE' => 758, 'SHOW' => 761, 'SHUTDOWN' => 762, 'SIGNAL' => 763, 'SIGNED' => 764, 'SIMPLE' => 765, 'SKIP' => 923, 'SLAVE' => 766, 'SLOW' => 767, 'SMALLINT' => 768, 'SNAPSHOT' => 769, 'SOCKET' => 770, 'SOME' => 275, 'SONAME' => 771, 'SOUNDS' => 772, 'SOURCE' => 773, 'SOURCE_AUTO_POSITION' => 1163, 'SOURCE_BIND' => 1164, 'SOURCE_COMPRESSION_ALGORITHMS' => 1165, 'SOURCE_CONNECTION_AUTO_FAILOVER' => 1156, 'SOURCE_CONNECT_RETRY' => 1166, 'SOURCE_DELAY' => 1167, 'SOURCE_HEARTBEAT_PERIOD' => 1168, 'SOURCE_HOST' => 1169, 'SOURCE_LOG_FILE' => 1170, 'SOURCE_LOG_POS' => 1171, 'SOURCE_PASSWORD' => 1172, 'SOURCE_PORT' => 1173, 'SOURCE_PUBLIC_KEY_PATH' => 1174, 'SOURCE_RETRY_COUNT' => 1175, 'SOURCE_SSL' => 1176, 'SOURCE_SSL_CA' => 1177, 'SOURCE_SSL_CAPATH' => 1178, 'SOURCE_SSL_CERT' => 1179, 'SOURCE_SSL_CIPHER' => 1180, 'SOURCE_SSL_CRL' => 1181, 'SOURCE_SSL_CRLPATH' => 1182, 'SOURCE_SSL_KEY' => 1183, 'SOURCE_SSL_VERIFY_SERVER_CERT' => 1184, 'SOURCE_TLS_CIPHERSUITES' => 1185, 'SOURCE_TLS_VERSION' => 1186, 'SOURCE_USER' => 1187, 'SOURCE_ZSTD_COMPRESSION_LEVEL' => 1188, 'SPATIAL' => 774, 'SPECIFIC' => 775, 'SQL' => 788, 'SQLEXCEPTION' => 776, 'SQLSTATE' => 777, 'SQLWARNING' => 778, 'SQL_AFTER_GTIDS' => 779, 'SQL_AFTER_MTS_GAPS' => 780, 'SQL_BEFORE_GTIDS' => 781, 'SQL_BIG_RESULT' => 782, 'SQL_BUFFER_RESULT' => 783, 'SQL_CALC_FOUND_ROWS' => 785, 'SQL_NO_CACHE' => 786, 'SQL_SMALL_RESULT' => 787, 'SQL_THREAD' => 789, 'SQL_TSI_DAY' => 375, 'SQL_TSI_HOUR' => 481, 'SQL_TSI_MINUTE' => 594, 'SQL_TSI_MONTH' => 601, 'SQL_TSI_QUARTER' => 682, 'SQL_TSI_SECOND' => 746, 'SQL_TSI_WEEK' => 887, 'SQL_TSI_YEAR' => 905, 'SRID' => 961, 'SSL' => 790, 'STACKED' => 791, 'START' => 794, 'STARTING' => 792, 'STARTS' => 793, 'STATS_AUTO_RECALC' => 795, 'STATS_PERSISTENT' => 796, 'STATS_SAMPLE_PAGES' => 797, 'STATUS' => 798, 'STD' => 800, 'STDDEV' => 800, 'STDDEV_POP' => 800, 'STDDEV_SAMP' => 799, 'STOP' => 801, 'STORAGE' => 802, 'STORED' => 803, 'STRAIGHT_JOIN' => 804, 'STREAM' => 997, 'STRING' => 805, 'ST_COLLECT' => 1189, 'SUBCLASS_ORIGIN' => 806, 'SUBDATE' => 807, 'SUBJECT' => 808, 'SUBPARTITION' => 810, 'SUBPARTITIONS' => 809, 'SUBSTR' => 811, 'SUBSTRING' => 811, 'SUM' => 812, 'SUPER' => 813, 'SUSPEND' => 814, 'SWAPS' => 815, 'SWITCHES' => 816, 'SYSDATE' => 817, 'SYSTEM' => 964, 'SYSTEM_USER' => 867, 'TABLE' => 821, 'TABLES' => 818, 'TABLESAMPLE' => 1214, 'TABLESPACE' => 819, 'TABLE_CHECKSUM' => 822, 'TABLE_NAME' => 823, 'TEMPORARY' => 824, 'TEMPTABLE' => 825, 'TERMINATED' => 826, 'TEXT' => 828, 'THAN' => 829, 'THEN' => 830, 'THREAD_PRIORITY' => 962, 'TIES' => 951, 'TIME' => 834, 'TIMESTAMP' => 831, 'TIMESTAMPADD' => 832, 'TIMESTAMPDIFF' => 833, 'TINYBLOB' => 835, 'TINYINT' => 836, 'TINYTEXT' => 837, 'TLS' => 1152, 'TO' => 838, 'TRAILING' => 839, 'TRANSACTION' => 840, 'TRIGGER' => 842, 'TRIGGERS' => 841, 'TRIM' => 843, 'TRUE' => 844, 'TRUNCATE' => 845, 'TYPE' => 847, 'TYPES' => 846, 'UNBOUNDED' => 952, 'UNCOMMITTED' => 850, 'UNDEFINED' => 851, 'UNDO' => 855, 'UNDOFILE' => 853, 'UNDO_BUFFER_SIZE' => 854, 'UNICODE' => 856, 'UNINSTALL' => 857, 'UNION' => 858, 'UNIQUE' => 859, 'UNKNOWN' => 860, 'UNLOCK' => 861, 'UNREGISTER' => 1196, 'UNSIGNED' => 862, 'UNTIL' => 863, 'UPDATE' => 864, 'UPGRADE' => 865, 'URL' => 1202, 'USAGE' => 866, 'USE' => 869, 'USER' => 867, 'USER_RESOURCES' => 723, 'USE_FRM' => 868, 'USING' => 870, 'UTC_DATE' => 871, 'UTC_TIME' => 873, 'UTC_TIMESTAMP' => 872, 'VALIDATION' => 874, 'VALUE' => 876, 'VALUES' => 875, 'VARBINARY' => 877, 'VARCHAR' => 878, 'VARCHARACTER' => 878, 'VARIABLES' => 879, 'VARIANCE' => 880, 'VARYING' => 881, 'VAR_POP' => 880, 'VAR_SAMP' => 882, 'VCPU' => 965, 'VIEW' => 883, 'VIRTUAL' => 884, 'VISIBLE' => 912, 'WAIT' => 885, 'WARNINGS' => 886, 'WEEK' => 887, 'WEIGHT_STRING' => 888, 'WHEN' => 889, 'WHERE' => 890, 'WHILE' => 891, 'WINDOW' => 953, 'WITH' => 892, 'WITHOUT' => 895, 'WORK' => 896, 'WRAPPER' => 897, 'WRITE' => 898, 'X509' => 899, 'XA' => 900, 'XID' => 901, 'XML' => 902, 'XOR' => 903, 'YEAR' => 905, 'YEAR_MONTH' => 904, 'ZEROFILL' => 906, 'ZONE' => 1157, '||' => 644 ); @@ -364,7 +363,7 @@ class WP_MySQL_Lexer { */ public function __construct( string $sql, - int $mysql_version = 80038, + int $mysql_version = 80410, array $sql_modes = array() ) { $this->sql = $sql; @@ -503,8 +502,8 @@ public function remaining_tokens(): array { // Common single-token lexeme — the overwhelming majority. if ( self::EOF !== $type - && self::WITH_SYMBOL !== $type - && self::AT_SIGN_SYMBOL !== $type + && self::WITH !== $type + && self::AT_SIGN !== $type ) { $tokens[] = new WP_MySQL_Token( $type, $start, $length, $sql, $nbe ); continue; @@ -514,7 +513,7 @@ public function remaining_tokens(): array { if ( self::EOF === $type ) { $this->emit_end_markers( $tokens ); break; - } elseif ( self::WITH_SYMBOL === $type ) { + } elseif ( self::WITH === $type ) { $this->produce_with_or_rollup( $tokens, $start, $length ); } else { $this->produce_at_variable( $tokens, $start ); @@ -550,32 +549,14 @@ public function is_sql_mode_active( int $mode ): bool { return ( $this->sql_modes & $mode ) !== 0; } - /** - * Get the numeric token ID for a given token name. - * - * @param string $token_name The name of the token. - * @return int|null The token ID for the given token name; null when not found. - */ - public static function get_token_id( string $token_name ): ?int { - // Token constants share the class with the lexer's own constants, so - // resolve only the grammar tokens: non-negative ints, never an SQL mode. - $constant_name = self::class . '::' . $token_name; - if ( 0 === strpos( $token_name, 'SQL_MODE_' ) || ! defined( $constant_name ) ) { - return null; - } - $value = constant( $constant_name ); - return is_int( $value ) && $value >= 0 ? $value : null; - } - /** * Get the name of a token for a given token ID. * * Names are derived from the grammar data rather than stored: a keyword * token resolves to its keyword string via self::KEYWORDS, any other token - * to its token constant name via reflection. When several names share a - * token number (keyword synonyms, equal-valued constants), plain keywords - * win over paren-gated function keywords (USER over SESSION_USER), then the - * first name in table order wins. + * to its token constant name via reflection. When several keywords share a + * token number (synonyms), plain keywords win over paren-gated function + * keywords (USER over SESSION_USER), then the first name in table order wins. * * This method is intended to be used only for testing and debugging purposes, * when tokens need to be presented by their names in a human-readable form. @@ -634,9 +615,9 @@ private function produce( array &$out ): void { if ( self::EOF === $type ) { $this->emit_end_markers( $out ); - } elseif ( self::WITH_SYMBOL === $type ) { + } elseif ( self::WITH === $type ) { $this->produce_with_or_rollup( $out, $start, $length ); - } elseif ( self::AT_SIGN_SYMBOL === $type ) { + } elseif ( self::AT_SIGN === $type ) { $this->produce_at_variable( $out, $start ); } else { $out[] = $this->make_token( $type, $start, $length ); @@ -655,21 +636,21 @@ private function produce( array &$out ): void { private function produce_with_or_rollup( array &$out, int $with_start, int $with_length ): void { if ( ! $this->scan_lexeme() ) { // "WITH" was the last valid token; the trailing input is invalid. - $out[] = $this->make_token( self::WITH_SYMBOL, $with_start, $with_length ); + $out[] = $this->make_token( self::WITH, $with_start, $with_length ); $this->stream_ended = true; return; } - if ( self::ROLLUP_SYMBOL === $this->token_type ) { + if ( self::ROLLUP_SYM === $this->token_type ) { $out[] = $this->make_token( - self::WITH_ROLLUP_SYMBOL, + self::WITH_ROLLUP_SYM, $with_start, $this->bytes_already_read - $with_start ); return; } - $out[] = $this->make_token( self::WITH_SYMBOL, $with_start, $with_length ); + $out[] = $this->make_token( self::WITH, $with_start, $with_length ); if ( self::EOF === $this->token_type ) { $this->emit_end_markers( $out ); return; @@ -695,13 +676,13 @@ private function produce_with_or_rollup( array &$out, int $with_start, int $with * @param int $at_start Byte offset of the "@" token. */ private function produce_at_variable( array &$out, int $at_start ): void { - $out[] = $this->make_token( self::AT_SIGN_SYMBOL, $at_start, 1 ); + $out[] = $this->make_token( self::AT_SIGN, $at_start, 1 ); $position = $this->bytes_already_read; // Right after the "@". $next = $this->sql[ $position ] ?? ''; if ( '@' === $next ) { - $out[] = $this->make_token( self::AT_SIGN_SYMBOL, $position, 1 ); + $out[] = $this->make_token( self::AT_SIGN, $position, 1 ); $this->bytes_already_read = $position + 1; // The name after it may be a keyword ("session" in "@@session.x"). @@ -722,10 +703,10 @@ private function produce_at_variable( array &$out, int $at_start ): void { $position ); if ( $name_length > 0 ) { - $out[] = $this->make_token( self::IDENTIFIER, $position, $name_length ); + $out[] = $this->make_token( self::IDENT, $position, $name_length ); $this->bytes_already_read = $position + $name_length; } elseif ( "'" !== $next && '"' !== $next && '`' !== $next ) { - $out[] = $this->make_token( self::IDENTIFIER, $position, 0 ); + $out[] = $this->make_token( self::IDENT, $position, 0 ); } } @@ -736,7 +717,7 @@ private function produce_at_variable( array &$out, int $at_start ): void { private function emit_end_markers( array &$out ): void { $end = $this->bytes_already_read; $out[] = $this->make_token( self::END_OF_INPUT, $end, 0 ); - $out[] = $this->make_token( self::END_MARKER, $end, 0 ); + $out[] = $this->make_token( self::YYEOF, $end, 0 ); $this->stream_ended = true; } @@ -795,7 +776,7 @@ private function read_next_token(): ?int { } if ( self::STATE_IDENTIFIER_OR_KEYWORD === $state ) { $type = $this->read_identifier(); - return self::IDENTIFIER === $type + return self::IDENT === $type ? $this->determine_identifier_or_keyword_type( $this->get_current_token_bytes() ) : $type; } @@ -806,18 +787,18 @@ private function read_next_token(): ?int { // A map for a single-byte symbol fast path. static $single_byte_ops = array( - '(' => self::OPEN_PAR_SYMBOL, - ')' => self::CLOSE_PAR_SYMBOL, - ',' => self::COMMA_SYMBOL, - ';' => self::SEMICOLON_SYMBOL, - '+' => self::PLUS_OPERATOR, - '~' => self::BITWISE_NOT_OPERATOR, - '%' => self::MOD_OPERATOR, - '^' => self::BITWISE_XOR_OPERATOR, + '(' => self::LEFT_PAREN, + ')' => self::RIGHT_PAREN, + ',' => self::COMMA, + ';' => self::SEMICOLON, + '+' => self::PLUS, + '~' => self::TILDE, + '%' => self::PERCENT, + '^' => self::CARET, '?' => self::PARAM_MARKER, - '{' => self::OPEN_CURLY_SYMBOL, - '}' => self::CLOSE_CURLY_SYMBOL, - '=' => self::EQUAL_OPERATOR, + '{' => self::LEFT_BRACE, + '}' => self::RIGHT_BRACE, + '=' => self::EQ, ); // Fast path for keywords and identifiers. @@ -836,18 +817,18 @@ private function read_next_token(): ?int { ) { $started_at = $this->bytes_already_read; $type = $this->read_identifier(); - if ( self::IDENTIFIER === $type ) { + if ( self::IDENT === $type ) { // Inline the keyword lookup on the hot identifier path: most // identifiers are not keywords, so this avoids two method calls // (token-bytes extraction + keyword determination) per token. $word = strtoupper( substr( $this->sql, $started_at, $this->bytes_already_read - $started_at ) ); - $keyword = self::KEYWORDS[ $word ] ?? self::IDENTIFIER; + $keyword = self::KEYWORDS[ $word ] ?? self::IDENT; // Before "." and a name, a keyword is an identifier ("order.id"). // read_identifier() sets STATE_IDENTIFIER_SEPARATOR in that case. if ( - self::IDENTIFIER !== $keyword + self::IDENT !== $keyword && self::STATE_IDENTIFIER_SEPARATOR !== $this->next_state ) { $type = $this->resolve_keyword_type( $keyword, $word ); @@ -871,7 +852,7 @@ private function read_next_token(): ?int { $type = $this->read_number(); } else { $this->bytes_already_read += 1; - $type = self::DOT_SYMBOL; + $type = self::DOT; // A name character right after it starts a qualified name part. if ( null !== $next_byte && ( strspn( $next_byte, self::IDENTIFIER_MASK ) > 0 || $next_byte > "\x7F" ) ) { @@ -882,9 +863,9 @@ private function read_next_token(): ?int { $this->bytes_already_read += 1; if ( '=' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::ASSIGN_OPERATOR; + $type = self::SET_VAR; } else { - $type = self::COLON_SYMBOL; + $type = self::COLON; } } elseif ( '<' === $byte ) { $this->bytes_already_read += 1; @@ -892,37 +873,37 @@ private function read_next_token(): ?int { $this->bytes_already_read += 1; if ( '>' === ( $this->sql[ $this->bytes_already_read ] ?? null ) ) { $this->bytes_already_read += 1; - $type = self::NULL_SAFE_EQUAL_OPERATOR; + $type = self::EQUAL_SYM; } else { - $type = self::LESS_OR_EQUAL_OPERATOR; + $type = self::LE; } } elseif ( '>' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::NOT_EQUAL_OPERATOR; + $type = self::NE; } elseif ( '<' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::SHIFT_LEFT_OPERATOR; + $type = self::SHIFT_LEFT; } else { - $type = self::LESS_THAN_OPERATOR; + $type = self::LT; } } elseif ( '>' === $byte ) { $this->bytes_already_read += 1; if ( '=' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::GREATER_OR_EQUAL_OPERATOR; + $type = self::GE; } elseif ( '>' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::SHIFT_RIGHT_OPERATOR; + $type = self::SHIFT_RIGHT; } else { - $type = self::GREATER_THAN_OPERATOR; + $type = self::GT_SYM; } } elseif ( '!' === $byte ) { $this->bytes_already_read += 1; if ( '=' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::NOT_EQUAL_OPERATOR; + $type = self::NE; } else { - $type = self::LOGICAL_NOT_OPERATOR; + $type = self::EXCLAMATION; } } elseif ( '-' === $byte ) { if ( @@ -936,20 +917,20 @@ private function read_next_token(): ?int { if ( '>' === ( $this->sql[ $this->bytes_already_read ] ?? null ) ) { $this->bytes_already_read += 1; if ( $this->mysql_version >= 50713 ) { - $type = self::JSON_UNQUOTED_SEPARATOR_SYMBOL; + $type = self::JSON_UNQUOTED_SEPARATOR_SYM; } else { return null; // Invalid input. } } else { if ( $this->mysql_version >= 50708 ) { - $type = self::JSON_SEPARATOR_SYMBOL; + $type = self::JSON_SEPARATOR_SYM; } else { return null; // Invalid input. } } } else { $this->bytes_already_read += 1; - $type = self::MINUS_OPERATOR; + $type = self::MINUS; } } elseif ( '*' === $byte ) { $this->bytes_already_read += 1; @@ -958,7 +939,7 @@ private function read_next_token(): ?int { $type = self::MYSQL_COMMENT_END; $this->in_mysql_comment = false; } else { - $type = self::MULT_OPERATOR; + $type = self::ASTERISK; } } elseif ( '/' === $byte ) { if ( '*' === $next_byte ) { @@ -971,36 +952,36 @@ private function read_next_token(): ?int { } } else { $this->bytes_already_read += 1; - $type = self::DIV_OPERATOR; + $type = self::SLASH; } } elseif ( '&' === $byte ) { $this->bytes_already_read += 1; if ( '&' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::LOGICAL_AND_OPERATOR; + $type = self::AND_AND_SYM; } else { - $type = self::BITWISE_AND_OPERATOR; + $type = self::AMPERSAND; } } elseif ( '|' === $byte ) { $this->bytes_already_read += 1; if ( '|' === $next_byte ) { $this->bytes_already_read += 1; $type = $this->is_sql_mode_active( self::SQL_MODE_PIPES_AS_CONCAT ) - ? self::CONCAT_PIPES_SYMBOL - : self::LOGICAL_OR_OPERATOR; + ? self::OR_OR_SYM + : self::OR2_SYM; } else { - $type = self::BITWISE_OR_OPERATOR; + $type = self::PIPE; } } elseif ( '@' === $byte ) { // MySQL treats "@" as a standalone terminal; the variable name that // follows is emitted as its own token (see produce_at_variable()). $this->bytes_already_read += 1; - $type = self::AT_SIGN_SYMBOL; + $type = self::AT_SIGN; } elseif ( '\\' === $byte ) { $this->bytes_already_read += 1; if ( 'N' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::NULL2_SYMBOL; + $type = self::NULL_SYM; } else { return null; // Invalid input. } @@ -1013,15 +994,15 @@ private function read_next_token(): ?int { $type = $this->read_number(); } elseif ( ( 'n' === $byte || 'N' === $byte ) && "'" === $next_byte ) { $this->bytes_already_read += 1; - $type = $this->read_quoted_text( "'" ); - if ( self::SINGLE_QUOTED_TEXT === $type ) { - $type = self::NCHAR_TEXT; + $type = $this->read_quoted_text(); + if ( self::TEXT_STRING === $type ) { + $type = self::NCHAR_STRING; } } elseif ( null === $byte ) { $type = self::EOF; } else { $type = $this->read_identifier(); - if ( self::IDENTIFIER === $type ) { + if ( self::IDENT === $type ) { if ( '_' === $byte && isset( self::UNDERSCORE_CHARSETS[ strtolower( $this->get_current_token_bytes() ) ] ) ) { $type = self::UNDERSCORE_CHARSET; } else { @@ -1124,7 +1105,7 @@ private function read_identifier(): ?int { ) { $this->next_state = self::STATE_IDENTIFIER_SEPARATOR; } - return self::IDENTIFIER; + return self::IDENT; } private function read_number(): ?int { @@ -1155,7 +1136,7 @@ private function read_number(): ?int { } $this->bytes_already_read += 1; // Consume the "'". } - $type = self::HEX_NUMBER; + $type = self::HEX_NUM; } elseif ( // BIN number in the form of 0bN. ( @@ -1178,18 +1159,18 @@ private function read_number(): ?int { } $this->bytes_already_read += 1; // Consume the "'". } - $type = self::BIN_NUMBER; + $type = self::BIN_NUM; } else { // Here, we have a sequence starting with N or .N, where N is a digit. // 1. Try integer first. $this->bytes_already_read += strspn( $this->sql, self::DIGIT_MASK, $this->bytes_already_read ); - $type = self::INT_NUMBER; + $type = self::NUM; // 2. In case of N. or .N, it's a decimal or float number. if ( '.' === ( $this->sql[ $this->bytes_already_read ] ?? null ) ) { $this->bytes_already_read += 1; - $type = self::DECIMAL_NUMBER; + $type = self::DECIMAL_NUM; $this->bytes_already_read += strspn( $this->sql, self::DIGIT_MASK, $this->bytes_already_read ); } @@ -1211,7 +1192,7 @@ private function read_number(): ?int { $this->bytes_already_read += 1; // Consume the 'e' or 'E'. $this->bytes_already_read += 1; // Consume the '+', '-', or digit. $this->bytes_already_read += strspn( $this->sql, self::DIGIT_MASK, $this->bytes_already_read ); - $type = self::FLOAT_NUMBER; + $type = self::FLOAT_NUM; } } @@ -1230,7 +1211,7 @@ private function read_number(): ?int { */ $text = $this->get_current_token_bytes(); $possible_identifier_prefix = - self::INT_NUMBER === $type + self::NUM === $type || ( '0' === $text[0] && ( 'b' === $text[1] || 'x' === $text[1] ) ); /* @@ -1240,16 +1221,16 @@ private function read_number(): ?int { * here, as we can look only at subsequent bytes instead of backtracking * to the beginning of the number (for valid identifiers like 0b019). */ - if ( $possible_identifier_prefix && self::IDENTIFIER === $this->read_identifier() ) { - $type = self::IDENTIFIER; + if ( $possible_identifier_prefix && self::IDENT === $this->read_identifier() ) { + $type = self::IDENT; } // Determine integer type. - if ( self::INT_NUMBER === $type ) { + if ( self::NUM === $type ) { // Fast path for most integers. $bytes = $this->get_current_token_bytes(); if ( strlen( $bytes ) < 10 ) { - return self::INT_NUMBER; + return self::NUM; } // Remove leading zeros. @@ -1258,23 +1239,23 @@ private function read_number(): ?int { // Determine integer type based on its length and value. if ( $length < 10 ) { - return self::INT_NUMBER; + return self::NUM; } elseif ( 10 === $length ) { return strcmp( $bytes, '2147483647' ) > 0 - ? self::LONG_NUMBER - : self::INT_NUMBER; + ? self::LONG_NUM + : self::NUM; } elseif ( $length < 19 ) { - return self::LONG_NUMBER; + return self::LONG_NUM; } elseif ( 19 === $length ) { return strcmp( $bytes, '9223372036854775807' ) > 0 - ? self::ULONGLONG_NUMBER - : self::LONG_NUMBER; + ? self::ULONGLONG_NUM + : self::LONG_NUM; } elseif ( 20 === $length ) { return strcmp( $bytes, '18446744073709551615' ) > 0 - ? self::DECIMAL_NUMBER - : self::ULONGLONG_NUMBER; + ? self::DECIMAL_NUM + : self::ULONGLONG_NUM; } else { - return self::DECIMAL_NUMBER; + return self::DECIMAL_NUM; } } return $type; @@ -1358,13 +1339,7 @@ private function read_quoted_text(): ?int { $this->bytes_already_read = $at; - if ( $is_identifier_quote ) { - return self::BACK_TICK_QUOTED_ID; - } elseif ( '"' === $quote ) { - return self::DOUBLE_QUOTED_TEXT; - } else { - return self::SINGLE_QUOTED_TEXT; - } + return $is_identifier_quote ? self::IDENT_QUOTED : self::TEXT_STRING; } private function read_line_comment(): int { @@ -1419,9 +1394,9 @@ private function read_comment_content(): void { private function determine_identifier_or_keyword_type( string $value ): int { $word = strtoupper( $value ); - $type = self::KEYWORDS[ $word ] ?? self::IDENTIFIER; - if ( self::IDENTIFIER === $type ) { - return self::IDENTIFIER; + $type = self::KEYWORDS[ $word ] ?? self::IDENT; + if ( self::IDENT === $type ) { + return self::IDENT; } return $this->resolve_keyword_type( $type, $word ); } @@ -1443,13 +1418,13 @@ private function resolve_keyword_type( int $type, string $word ): int { $peek += strspn( $this->sql, self::WHITESPACE_MASK, $peek ); } if ( '(' !== ( $this->sql[ $peek ] ?? null ) ) { - return self::IDENTIFIER; + return self::IDENT; } } // With "SQL_MODE_HIGH_NOT_PRECEDENCE" enabled, "NOT" needs to be emitted as a higher priority NOT2 symbol. - if ( self::NOT_SYMBOL === $type && $this->is_sql_mode_active( self::SQL_MODE_HIGH_NOT_PRECEDENCE ) ) { - return self::NOT2_SYMBOL; + if ( self::NOT_SYM === $type && $this->is_sql_mode_active( self::SQL_MODE_HIGH_NOT_PRECEDENCE ) ) { + return self::NOT2_SYM; } return $type; diff --git a/packages/mysql-parser/src/class-wp-mysql-token.php b/packages/mysql-parser/src/class-wp-mysql-token.php index 123e74213..010e03c46 100644 --- a/packages/mysql-parser/src/class-wp-mysql-token.php +++ b/packages/mysql-parser/src/class-wp-mysql-token.php @@ -62,11 +62,7 @@ public function get_name(): string { */ public function get_value(): string { $value = $this->get_bytes(); - if ( - WP_MySQL_Lexer::SINGLE_QUOTED_TEXT === $this->id - || WP_MySQL_Lexer::DOUBLE_QUOTED_TEXT === $this->id - || WP_MySQL_Lexer::BACK_TICK_QUOTED_ID === $this->id - ) { + if ( WP_MySQL_Lexer::TEXT_STRING === $this->id || WP_MySQL_Lexer::IDENT_QUOTED === $this->id ) { // Remove bounding quotes. $quote = $value[0]; $value = substr( $value, 1, -1 ); @@ -79,7 +75,7 @@ public function get_value(): string { * See: https://dev.mysql.com/doc/refman/8.4/en/identifiers.html */ if ( - WP_MySQL_Lexer::BACK_TICK_QUOTED_ID === $this->id + WP_MySQL_Lexer::IDENT_QUOTED === $this->id || $this->sql_mode_no_backslash_escapes_enabled ) { return str_replace( $quote . $quote, $quote, $value ); diff --git a/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php b/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php index 47589b72f..4a96ce3b8 100644 --- a/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php +++ b/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php @@ -17,7 +17,7 @@ public function test_tokenize_valid_input(): void { // id $this->assertTrue( $lexer->next_token() ); - $this->assertSame( WP_MySQL_Lexer::IDENTIFIER, $lexer->get_token()->id ); + $this->assertSame( WP_MySQL_Lexer::IDENT, $lexer->get_token()->id ); // FROM $this->assertTrue( $lexer->next_token() ); @@ -25,13 +25,13 @@ public function test_tokenize_valid_input(): void { // users $this->assertTrue( $lexer->next_token() ); - $this->assertSame( WP_MySQL_Lexer::IDENTIFIER, $lexer->get_token()->id ); + $this->assertSame( WP_MySQL_Lexer::IDENT, $lexer->get_token()->id ); // The stream ends with END_OF_INPUT followed by Bison's end marker. $this->assertTrue( $lexer->next_token() ); $this->assertSame( WP_MySQL_Lexer::END_OF_INPUT, $lexer->get_token()->id ); $this->assertTrue( $lexer->next_token() ); - $this->assertSame( WP_MySQL_Lexer::END_MARKER, $lexer->get_token()->id ); + $this->assertSame( WP_MySQL_Lexer::YYEOF, $lexer->get_token()->id ); // No more tokens. $this->assertFalse( $lexer->next_token() ); @@ -70,7 +70,7 @@ public function test_tokenize_invalid_input(): void { * @return string[] Terminal names, in input order. */ private static function token_names( string $sql, array $sql_modes = array() ): array { - $tokens = ( new WP_MySQL_Lexer( $sql, 80400, $sql_modes ) )->remaining_tokens(); + $tokens = ( new WP_MySQL_Lexer( $sql, 80410, $sql_modes ) )->remaining_tokens(); $names = array(); foreach ( $tokens as $token ) { $names[] = $token->get_name(); @@ -80,7 +80,7 @@ private static function token_names( string $sql, array $sql_modes = array() ): public function test_emits_bison_terminals_with_end_markers(): void { $this->assertSame( - array( 'SELECT', 'IDENTIFIER', 'FROM', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'IDENT', 'FROM', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT id FROM users' ) ); } @@ -88,29 +88,29 @@ public function test_emits_bison_terminals_with_end_markers(): void { public function test_ignore_space_preserves_function_name_ranges(): void { // Without "(", COUNT is an identifier whose range excludes trailing whitespace. $sql = 'SELECT COUNT FROM t'; - $tokens = ( new WP_MySQL_Lexer( $sql, 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); - $this->assertSame( 'IDENTIFIER', $tokens[1]->get_name(), $sql ); + $tokens = ( new WP_MySQL_Lexer( $sql, 80410, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); + $this->assertSame( 'IDENT', $tokens[1]->get_name(), $sql ); $this->assertSame( 'COUNT', $tokens[1]->get_value(), $sql ); $this->assertSame( 5, $tokens[1]->length, $sql ); $this->assertSame( 13, $tokens[2]->start, $sql ); $sql = "SELECT COUNT\t\n FROM t"; - $tokens = ( new WP_MySQL_Lexer( $sql, 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); - $this->assertSame( 'IDENTIFIER', $tokens[1]->get_name(), $sql ); + $tokens = ( new WP_MySQL_Lexer( $sql, 80410, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); + $this->assertSame( 'IDENT', $tokens[1]->get_name(), $sql ); $this->assertSame( 'COUNT', $tokens[1]->get_value(), $sql ); $this->assertSame( 5, $tokens[1]->length, $sql ); $this->assertSame( 15, $tokens[2]->start, $sql ); // With "(", COUNT is a function whose range still excludes the whitespace. - $tokens = ( new WP_MySQL_Lexer( 'SELECT COUNT (1)', 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); + $tokens = ( new WP_MySQL_Lexer( 'SELECT COUNT (1)', 80410, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); $this->assertSame( 5, $tokens[1]->length ); $this->assertSame( 'COUNT', $tokens[1]->get_name() ); - $this->assertSame( 'OPEN_PAR_SYMBOL', $tokens[2]->get_name() ); + $this->assertSame( 'LEFT_PAREN', $tokens[2]->get_name() ); $this->assertSame( 13, $tokens[2]->start ); $sql = "SELECT COUNT \t\n"; - $tokens = ( new WP_MySQL_Lexer( $sql, 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); - $this->assertSame( 'IDENTIFIER', $tokens[1]->get_name(), $sql ); + $tokens = ( new WP_MySQL_Lexer( $sql, 80410, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); + $this->assertSame( 'IDENT', $tokens[1]->get_name(), $sql ); $this->assertSame( 'COUNT', $tokens[1]->get_value(), $sql ); $this->assertSame( 5, $tokens[1]->length, $sql ); $this->assertSame( 'END_OF_INPUT', $tokens[2]->get_name(), $sql ); @@ -122,7 +122,7 @@ public function test_ignore_space_preserves_function_name_ranges(): void { */ public function test_version_comments( string $sql, int $mysql_version, array $expected_bytes ): void { $tokens = ( new WP_MySQL_Lexer( $sql, $mysql_version ) )->remaining_tokens(); - $this->assertSame( 'END_MARKER', array_pop( $tokens )->get_name() ); + $this->assertSame( 'YYEOF', array_pop( $tokens )->get_name() ); $this->assertSame( 'END_OF_INPUT', array_pop( $tokens )->get_name() ); $this->assertSame( $expected_bytes, @@ -190,10 +190,26 @@ public function data_version_comments(): array { return $cases; } + public function test_mysql_version_defaults_to_mysql_8_4_10(): void { + $lexer = new WP_MySQL_Lexer( 'SELECT /*!080400 1 + */ 2' ); + $this->assertSame( 80410, $lexer->get_mysql_version() ); + + // A six-digit MySQL 8.4 version comment is executed by default. + $this->assertSame( + array( 'SELECT', 'NUM', 'PLUS', 'NUM', 'END_OF_INPUT', 'YYEOF' ), + array_map( + static function ( WP_MySQL_Token $token ): string { + return $token->get_name(); + }, + $lexer->remaining_tokens() + ) + ); + } + public function test_at_name_splits_into_at_and_ident(): void { $tokens = ( new WP_MySQL_Lexer( 'SELECT @var1' ) )->remaining_tokens(); - $this->assertSame( 'AT_SIGN_SYMBOL', $tokens[1]->get_name() ); - $this->assertSame( 'IDENTIFIER', $tokens[2]->get_name() ); + $this->assertSame( 'AT_SIGN', $tokens[1]->get_name() ); + $this->assertSame( 'IDENT', $tokens[2]->get_name() ); $this->assertSame( 'var1', $tokens[2]->get_value() ); $this->assertSame( 7, $tokens[1]->start ); $this->assertSame( 1, $tokens[1]->length ); @@ -203,7 +219,7 @@ public function test_at_name_splits_into_at_and_ident(): void { public function test_at_at_splits_into_two_at_signs(): void { $this->assertSame( - array( 'SELECT', 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'AT_SIGN', 'AT_SIGN', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT @@sql_mode' ) ); } @@ -212,13 +228,13 @@ public function test_bare_at_emits_empty_name(): void { // MySQL's lexer emits an empty LEX_HOSTNAME after a bare "@", making // "user1@" (an empty host part) and "SELECT @" valid. $tokens = ( new WP_MySQL_Lexer( 'SELECT @' ) )->remaining_tokens(); - $this->assertSame( 'AT_SIGN_SYMBOL', $tokens[1]->get_name() ); - $this->assertSame( 'IDENTIFIER', $tokens[2]->get_name() ); + $this->assertSame( 'AT_SIGN', $tokens[1]->get_name() ); + $this->assertSame( 'IDENT', $tokens[2]->get_name() ); $this->assertSame( 0, $tokens[2]->length ); $this->assertSame( '', $tokens[2]->get_value() ); $this->assertSame( - array( 'CREATE', 'USER', 'IDENTIFIER', 'AT_SIGN_SYMBOL', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'CREATE', 'USER', 'IDENT', 'AT_SIGN', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'CREATE USER user1@' ) ); } @@ -226,14 +242,14 @@ public function test_bare_at_emits_empty_name(): void { public function test_bare_at_before_quote_stands_alone(): void { // In "@'name'" the quoted text supplies the name itself. $this->assertSame( - array( 'SET', 'AT_SIGN_SYMBOL', 'SINGLE_QUOTED_TEXT', '=', 'INT_NUMBER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SET', 'AT_SIGN', 'TEXT_STRING', '=', 'NUM', 'END_OF_INPUT', 'YYEOF' ), self::token_names( "SET @'v' = 1" ) ); } public function test_with_rollup_is_contracted(): void { $names = self::token_names( 'SELECT 1 FROM t GROUP BY a WITH ROLLUP' ); - $this->assertContains( 'WITH_ROLLUP_SYMBOL', $names ); + $this->assertContains( 'WITH_ROLLUP_SYM', $names ); $this->assertNotContains( 'WITH', $names ); } @@ -241,7 +257,7 @@ public function test_with_rollup_contracts_across_comments(): void { $tokens = ( new WP_MySQL_Lexer( 'SELECT 1 FROM t GROUP BY a WITH /* c */ ROLLUP' ) )->remaining_tokens(); $rollup = null; foreach ( $tokens as $token ) { - if ( 'WITH_ROLLUP_SYMBOL' === $token->get_name() ) { + if ( 'WITH_ROLLUP_SYM' === $token->get_name() ) { $rollup = $token; } } @@ -251,40 +267,40 @@ public function test_with_rollup_contracts_across_comments(): void { public function test_lone_with_is_emitted(): void { $this->assertSame( - array( 'WITH', 'IDENTIFIER', 'AS', 'OPEN_PAR_SYMBOL', 'SELECT', 'INT_NUMBER', 'CLOSE_PAR_SYMBOL', 'SELECT', 'MULT_OPERATOR', 'FROM', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'WITH', 'IDENT', 'AS', 'LEFT_PAREN', 'SELECT', 'NUM', 'RIGHT_PAREN', 'SELECT', 'ASTERISK', 'FROM', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'WITH c AS (SELECT 1) SELECT * FROM c' ) ); // A statement ending on WITH still emits it before the end markers. $this->assertSame( - array( 'SELECT', 'INT_NUMBER', 'WITH', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'NUM', 'WITH', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT 1 WITH' ) ); } public function test_invalid_input_returns_partial_stream_without_end_markers(): void { $names = self::token_names( "SELECT 1 WITH \x01" ); - $this->assertSame( array( 'SELECT', 'INT_NUMBER', 'WITH' ), $names ); + $this->assertSame( array( 'SELECT', 'NUM', 'WITH' ), $names ); } public function test_system_variable_name_must_not_be_empty(): void { // As in MySQL, the name after "@@" can't be empty, and only backticks // can quote it. Otherwise, the stream ends after the "@@". - $at_at = array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL' ); + $at_at = array( 'AT_SIGN', 'AT_SIGN' ); $this->assertSame( $at_at, self::token_names( '@@' ) ); $this->assertSame( $at_at, self::token_names( '@@ version' ) ); $this->assertSame( $at_at, self::token_names( "@@'version'" ) ); $this->assertSame( $at_at, self::token_names( '@@"version"' ) ); $this->assertSame( $at_at, self::token_names( '@@"version"', array( 'ANSI_QUOTES' ) ) ); $this->assertSame( - array_merge( $at_at, array( 'BACK_TICK_QUOTED_ID', 'END_OF_INPUT', 'END_MARKER' ) ), + array_merge( $at_at, array( 'IDENT_QUOTED', 'END_OF_INPUT', 'YYEOF' ) ), self::token_names( '@@`version`' ) ); } public function test_high_not_precedence_emits_not2(): void { $names = self::token_names( 'SELECT NOT 1', array( 'HIGH_NOT_PRECEDENCE' ) ); - $this->assertContains( 'NOT2_SYMBOL', $names ); + $this->assertContains( 'NOT2_SYM', $names ); $names = self::token_names( 'SELECT NOT 1' ); $this->assertContains( 'NOT', $names ); @@ -293,7 +309,7 @@ public function test_high_not_precedence_emits_not2(): void { public function test_end_of_input_word_is_an_identifier(): void { // "end_of_input" is not a MySQL keyword; it must not truncate the stream. $this->assertSame( - array( 'SELECT', 'IDENTIFIER', 'FROM', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'IDENT', 'FROM', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT end_of_input FROM t' ) ); } @@ -302,18 +318,18 @@ public function test_current_date_is_a_keyword_without_parentheses(): void { // CURRENT_DATE/CURRENT_TIME are plain reserved keywords in MySQL 8.4 // (lex.h SYM), unlike CURDATE/CURTIME which require parentheses. $this->assertSame( - array( 'SELECT', 'CURRENT_DATE', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'CURRENT_DATE', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT CURRENT_DATE' ) ); $this->assertSame( - array( 'SELECT', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT curdate' ) ); } public function test_json_aggregates_are_keywords_only_before_parenthesis(): void { $this->assertSame( - array( 'SELECT', 'IDENTIFIER', 'FROM', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'IDENT', 'FROM', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT json_objectagg FROM t' ) ); $names = self::token_names( 'SELECT JSON_OBJECTAGG(a, b) FROM t' ); @@ -321,10 +337,10 @@ public function test_json_aggregates_are_keywords_only_before_parenthesis(): voi } public function test_number_tokens_follow_mysql_magnitude_classes(): void { - $this->assertSame( array( 'SELECT', 'INT_NUMBER', 'END_OF_INPUT', 'END_MARKER' ), self::token_names( 'SELECT 2147483647' ) ); - $this->assertSame( array( 'SELECT', 'LONG_NUMBER', 'END_OF_INPUT', 'END_MARKER' ), self::token_names( 'SELECT 2147483648' ) ); - $this->assertSame( array( 'SELECT', 'ULONGLONG_NUMBER', 'END_OF_INPUT', 'END_MARKER' ), self::token_names( 'SELECT 18446744073709551615' ) ); - $this->assertSame( array( 'SELECT', 'DECIMAL_NUMBER', 'END_OF_INPUT', 'END_MARKER' ), self::token_names( 'SELECT 18446744073709551616' ) ); + $this->assertSame( array( 'SELECT', 'NUM', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT 2147483647' ) ); + $this->assertSame( array( 'SELECT', 'LONG_NUM', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT 2147483648' ) ); + $this->assertSame( array( 'SELECT', 'ULONGLONG_NUM', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT 18446744073709551615' ) ); + $this->assertSame( array( 'SELECT', 'DECIMAL_NUM', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT 18446744073709551616' ) ); } /** @@ -341,7 +357,7 @@ public function test_identifier_utf8_range(): void { $type = $lexer->get_token()->id; $is_valid = preg_match( '/^[\x{0080}-\x{ffff}]$/u', $value ); if ( $is_valid ) { - $this->assertSame( WP_MySQL_Lexer::IDENTIFIER, $type ); + $this->assertSame( WP_MySQL_Lexer::IDENT, $type ); } else { // A surrogate codepoint renders as an empty string, so the lexer // emits no identifier — only the end-of-input terminal. @@ -369,7 +385,7 @@ public function test_identifier_utf8_two_byte_sequences(): void { $is_valid = preg_match( '/^[\x{0080}-\x{ffff}]$/u', $value ); if ( $is_valid ) { $this->assertTrue( $result ); - $this->assertSame( WP_MySQL_Lexer::IDENTIFIER, $token->id ); + $this->assertSame( WP_MySQL_Lexer::IDENT, $token->id ); } else { $this->assertFalse( $result ); $this->assertNull( $token ); @@ -399,7 +415,7 @@ public function test_identifier_utf8_three_byte_sequences(): void { $is_valid = preg_match( '/^[\x{0080}-\x{ffff}]$/u', $value ); if ( $is_valid ) { $this->assertTrue( $result ); - $this->assertSame( WP_MySQL_Lexer::IDENTIFIER, $token->id ); + $this->assertSame( WP_MySQL_Lexer::IDENT, $token->id ); } else { $this->assertFalse( $result ); $this->assertNull( $token ); @@ -431,12 +447,12 @@ public function test_underscore_charset_name_after_dot_is_identifier( string $sq public function data_underscore_charset_after_dot(): array { return array( // `t . _utf8` - the member name must be an identifier, not a charset. - 'charset name after dot is identifier' => array( 't._utf8', 2, WP_MySQL_Lexer::IDENTIFIER ), - 'other charset name after dot' => array( 'a._binary', 2, WP_MySQL_Lexer::IDENTIFIER ), + 'charset name after dot is identifier' => array( 't._utf8', 2, WP_MySQL_Lexer::IDENT ), + 'other charset name after dot' => array( 'a._binary', 2, WP_MySQL_Lexer::IDENT ), // A genuine charset introducer (before a string) stays a charset. 'charset introducer before string' => array( "_utf8'x'", 0, WP_MySQL_Lexer::UNDERSCORE_CHARSET ), // A non-charset underscore name after a dot stays an identifier. - 'non-charset underscore name after dot' => array( 't._foo', 2, WP_MySQL_Lexer::IDENTIFIER ), + 'non-charset underscore name after dot' => array( 't._foo', 2, WP_MySQL_Lexer::IDENT ), ); } @@ -448,7 +464,7 @@ public function data_underscore_charset_after_dot(): array { */ public function test_qualified_name_tokens( string $sql, array $expected ): void { $this->assertSame( - array_merge( $expected, array( 'END_OF_INPUT', 'END_MARKER' ) ), + array_merge( $expected, array( 'END_OF_INPUT', 'YYEOF' ) ), self::token_names( $sql ) ); } @@ -457,54 +473,54 @@ public function test_qualified_name_tokens( string $sql, array $expected ): void * @return array */ public function data_qualified_name_tokens(): array { - $qualified = array( 'IDENTIFIER', 'DOT_SYMBOL', 'IDENTIFIER' ); + $qualified = array( 'IDENT', 'DOT', 'IDENT' ); return array( // A qualified name part can start with digits. 'digits only' => array( 't.1', $qualified ), 'digit-led identifier' => array( 't.1st', $qualified ), 'float-like identifier' => array( 't.1e5', $qualified ), - 'part ends before a sign' => array( 't.1e+5', array_merge( $qualified, array( 'PLUS_OPERATOR', 'INT_NUMBER' ) ) ), + 'part ends before a sign' => array( 't.1e+5', array_merge( $qualified, array( 'PLUS', 'NUM' ) ) ), 'after a digit-led qualifier' => array( '1ea10.1', $qualified ), - 'three parts' => array( 'd.1.2', array_merge( $qualified, array( 'DOT_SYMBOL', 'IDENTIFIER' ) ) ), + 'three parts' => array( 'd.1.2', array_merge( $qualified, array( 'DOT', 'IDENT' ) ) ), // Elsewhere, ".N" is a number. - 'after a quoted identifier' => array( '`t`.1', array( 'BACK_TICK_QUOTED_ID', 'DECIMAL_NUMBER' ) ), - 'after whitespace' => array( 't .1', array( 'IDENTIFIER', 'DECIMAL_NUMBER' ) ), - 'after a separator dot' => array( 't..1', array( 'IDENTIFIER', 'DOT_SYMBOL', 'DECIMAL_NUMBER' ) ), - 'after a decimal' => array( '1.5.6', array( 'DECIMAL_NUMBER', 'DECIMAL_NUMBER' ) ), - 'after a float' => array( '1e5.1', array( 'FLOAT_NUMBER', 'DECIMAL_NUMBER' ) ), - 'after a hex number' => array( '0x1F.1', array( 'HEX_NUMBER', 'DECIMAL_NUMBER' ) ), - 'after a binary number' => array( '0b1.1', array( 'BIN_NUMBER', 'DECIMAL_NUMBER' ) ), - 'after "\N"' => array( '\N.1', array( 'NULL', 'DECIMAL_NUMBER' ) ), - 'after a version comment' => array( '/*!50000.5*/', array( 'DECIMAL_NUMBER' ) ), + 'after a quoted identifier' => array( '`t`.1', array( 'IDENT_QUOTED', 'DECIMAL_NUM' ) ), + 'after whitespace' => array( 't .1', array( 'IDENT', 'DECIMAL_NUM' ) ), + 'after a separator dot' => array( 't..1', array( 'IDENT', 'DOT', 'DECIMAL_NUM' ) ), + 'after a decimal' => array( '1.5.6', array( 'DECIMAL_NUM', 'DECIMAL_NUM' ) ), + 'after a float' => array( '1e5.1', array( 'FLOAT_NUM', 'DECIMAL_NUM' ) ), + 'after a hex number' => array( '0x1F.1', array( 'HEX_NUM', 'DECIMAL_NUM' ) ), + 'after a binary number' => array( '0b1.1', array( 'BIN_NUM', 'DECIMAL_NUM' ) ), + 'after "\N"' => array( '\N.1', array( 'NULL', 'DECIMAL_NUM' ) ), + 'after a version comment' => array( '/*!50000.5*/', array( 'DECIMAL_NUM' ) ), // A user variable name includes the dot. - 'in a user variable' => array( '@a.1', array( 'AT_SIGN_SYMBOL', 'IDENTIFIER' ) ), + 'in a user variable' => array( '@a.1', array( 'AT_SIGN', 'IDENT' ) ), // A keyword before a dot and a name is an identifier. 'keyword before a name' => array( 'order.id', $qualified ), 'keyword before digits' => array( 'select.1', $qualified ), 'keyword before non-ASCII' => array( 'order.ñ', $qualified ), 'keyword before "$"' => array( 'order.$x', $qualified ), - 'keyword in three parts' => array( 'order.t.id', array_merge( $qualified, array( 'DOT_SYMBOL', 'IDENTIFIER' ) ) ), + 'keyword in three parts' => array( 'order.t.id', array_merge( $qualified, array( 'DOT', 'IDENT' ) ) ), // A name right after a "." token is always an identifier. - 'keyword after a dot' => array( '`t`.order', array( 'BACK_TICK_QUOTED_ID', 'DOT_SYMBOL', 'IDENTIFIER' ) ), - 'literal prefix after a dot' => array( "t.x'41'", array_merge( $qualified, array( 'SINGLE_QUOTED_TEXT' ) ) ), + 'keyword after a dot' => array( '`t`.order', array( 'IDENT_QUOTED', 'DOT', 'IDENT' ) ), + 'literal prefix after a dot' => array( "t.x'41'", array_merge( $qualified, array( 'TEXT_STRING' ) ) ), // Otherwise, it stays a keyword or a charset. - 'keyword after a decimal' => array( '1.FROM', array( 'DECIMAL_NUMBER', 'FROM' ) ), - 'keyword before a backtick' => array( 'order.`id`', array( 'ORDER', 'DOT_SYMBOL', 'BACK_TICK_QUOTED_ID' ) ), - 'keyword before an asterisk' => array( 'order.*', array( 'ORDER', 'DOT_SYMBOL', 'MULT_OPERATOR' ) ), - 'keyword before whitespace' => array( 'order .id', array( 'ORDER', 'DOT_SYMBOL', 'IDENTIFIER' ) ), - 'system variable scope' => array( '@@session.sql_mode', array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'SESSION', 'DOT_SYMBOL', 'IDENTIFIER' ) ), - 'charset introducer' => array( '_utf8mb4.x', array( 'UNDERSCORE_CHARSET', 'DOT_SYMBOL', 'IDENTIFIER' ) ), + 'keyword after a decimal' => array( '1.FROM', array( 'DECIMAL_NUM', 'FROM' ) ), + 'keyword before a backtick' => array( 'order.`id`', array( 'ORDER', 'DOT', 'IDENT_QUOTED' ) ), + 'keyword before an asterisk' => array( 'order.*', array( 'ORDER', 'DOT', 'ASTERISK' ) ), + 'keyword before whitespace' => array( 'order .id', array( 'ORDER', 'DOT', 'IDENT' ) ), + 'system variable scope' => array( '@@session.sql_mode', array( 'AT_SIGN', 'AT_SIGN', 'SESSION', 'DOT', 'IDENT' ) ), + 'charset introducer' => array( '_utf8mb4.x', array( 'UNDERSCORE_CHARSET', 'DOT', 'IDENT' ) ), // After "@@", the name is never a number, a literal, or a charset. - 'system variable digits' => array( '@@123', array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'IDENTIFIER' ) ), - 'system variable float-like' => array( '@@1e5', array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'IDENTIFIER' ) ), - 'system variable before text' => array( "@@b'01'", array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'IDENTIFIER', 'SINGLE_QUOTED_TEXT' ) ), - 'system variable charset' => array( '@@_utf8mb4', array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'IDENTIFIER' ) ), + 'system variable digits' => array( '@@123', array( 'AT_SIGN', 'AT_SIGN', 'IDENT' ) ), + 'system variable float-like' => array( '@@1e5', array( 'AT_SIGN', 'AT_SIGN', 'IDENT' ) ), + 'system variable before text' => array( "@@b'01'", array( 'AT_SIGN', 'AT_SIGN', 'IDENT', 'TEXT_STRING' ) ), + 'system variable charset' => array( '@@_utf8mb4', array( 'AT_SIGN', 'AT_SIGN', 'IDENT' ) ), ); } @@ -519,29 +535,29 @@ public function test_integer_types( $input, $expected ): void { public function data_integer_types(): array { return array( - array( '0', WP_MySQL_Lexer::INT_NUMBER ), - array( '123', WP_MySQL_Lexer::INT_NUMBER ), - array( '2147483647', WP_MySQL_Lexer::INT_NUMBER ), - array( '00000000001', WP_MySQL_Lexer::INT_NUMBER ), - array( '00000000002147483647', WP_MySQL_Lexer::INT_NUMBER ), - - array( '2147483648', WP_MySQL_Lexer::LONG_NUMBER ), - array( '123456789123456789', WP_MySQL_Lexer::LONG_NUMBER ), - array( '9223372036854775807', WP_MySQL_Lexer::LONG_NUMBER ), - array( '00000000002147483648', WP_MySQL_Lexer::LONG_NUMBER ), - array( '00000000009223372036854775807', WP_MySQL_Lexer::LONG_NUMBER ), - - array( '9223372036854775808', WP_MySQL_Lexer::ULONGLONG_NUMBER ), - array( '12345678912345678912', WP_MySQL_Lexer::ULONGLONG_NUMBER ), - array( '18446744073709551615', WP_MySQL_Lexer::ULONGLONG_NUMBER ), - array( '00000000000000000009223372036854775808', WP_MySQL_Lexer::ULONGLONG_NUMBER ), - array( '000000000000000000018446744073709551615', WP_MySQL_Lexer::ULONGLONG_NUMBER ), - - array( '18446744073709551616', WP_MySQL_Lexer::DECIMAL_NUMBER ), - array( '23456789123456789123', WP_MySQL_Lexer::DECIMAL_NUMBER ), - array( '123456789123456789123456789', WP_MySQL_Lexer::DECIMAL_NUMBER ), - array( '0000000000000000000018446744073709551616', WP_MySQL_Lexer::DECIMAL_NUMBER ), - array( '00000000000000000000123456789123456789123456789', WP_MySQL_Lexer::DECIMAL_NUMBER ), + array( '0', WP_MySQL_Lexer::NUM ), + array( '123', WP_MySQL_Lexer::NUM ), + array( '2147483647', WP_MySQL_Lexer::NUM ), + array( '00000000001', WP_MySQL_Lexer::NUM ), + array( '00000000002147483647', WP_MySQL_Lexer::NUM ), + + array( '2147483648', WP_MySQL_Lexer::LONG_NUM ), + array( '123456789123456789', WP_MySQL_Lexer::LONG_NUM ), + array( '9223372036854775807', WP_MySQL_Lexer::LONG_NUM ), + array( '00000000002147483648', WP_MySQL_Lexer::LONG_NUM ), + array( '00000000009223372036854775807', WP_MySQL_Lexer::LONG_NUM ), + + array( '9223372036854775808', WP_MySQL_Lexer::ULONGLONG_NUM ), + array( '12345678912345678912', WP_MySQL_Lexer::ULONGLONG_NUM ), + array( '18446744073709551615', WP_MySQL_Lexer::ULONGLONG_NUM ), + array( '00000000000000000009223372036854775808', WP_MySQL_Lexer::ULONGLONG_NUM ), + array( '000000000000000000018446744073709551615', WP_MySQL_Lexer::ULONGLONG_NUM ), + + array( '18446744073709551616', WP_MySQL_Lexer::DECIMAL_NUM ), + array( '23456789123456789123', WP_MySQL_Lexer::DECIMAL_NUM ), + array( '123456789123456789123456789', WP_MySQL_Lexer::DECIMAL_NUM ), + array( '0000000000000000000018446744073709551616', WP_MySQL_Lexer::DECIMAL_NUM ), + array( '00000000000000000000123456789123456789123456789', WP_MySQL_Lexer::DECIMAL_NUM ), ); } @@ -571,64 +587,64 @@ function ( $token ) { } public function data_identifier_or_number(): array { - $end = array( WP_MySQL_Lexer::END_OF_INPUT, WP_MySQL_Lexer::END_MARKER ); + $end = array( WP_MySQL_Lexer::END_OF_INPUT, WP_MySQL_Lexer::YYEOF ); return array( // integer - array( '123', array_merge( array( WP_MySQL_Lexer::INT_NUMBER ), $end ) ), - array( '123abc', array_merge( array( WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // identifier + array( '123', array_merge( array( WP_MySQL_Lexer::NUM ), $end ) ), + array( '123abc', array_merge( array( WP_MySQL_Lexer::IDENT ), $end ) ), // identifier // binary - array( '0b01', array_merge( array( WP_MySQL_Lexer::BIN_NUMBER ), $end ) ), - array( '0b01xyz', array_merge( array( WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // identifier - array( '0b', array_merge( array( WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // identifier - array( "b'01'", array_merge( array( WP_MySQL_Lexer::BIN_NUMBER ), $end ) ), + array( '0b01', array_merge( array( WP_MySQL_Lexer::BIN_NUM ), $end ) ), + array( '0b01xyz', array_merge( array( WP_MySQL_Lexer::IDENT ), $end ) ), // identifier + array( '0b', array_merge( array( WP_MySQL_Lexer::IDENT ), $end ) ), // identifier + array( "b'01'", array_merge( array( WP_MySQL_Lexer::BIN_NUM ), $end ) ), array( "b'01xyz'", array() ), // invalid input - array( "b''", array_merge( array( WP_MySQL_Lexer::BIN_NUMBER ), $end ) ), + array( "b''", array_merge( array( WP_MySQL_Lexer::BIN_NUM ), $end ) ), array( "b'", array() ), // invalid input array( "b'01", array() ), // invalid input // hex - array( '0xab01', array_merge( array( WP_MySQL_Lexer::HEX_NUMBER ), $end ) ), - array( '0xab01xyz', array_merge( array( WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // identifier - array( '0x', array_merge( array( WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // identifier - array( "x'ab01'", array_merge( array( WP_MySQL_Lexer::HEX_NUMBER ), $end ) ), + array( '0xab01', array_merge( array( WP_MySQL_Lexer::HEX_NUM ), $end ) ), + array( '0xab01xyz', array_merge( array( WP_MySQL_Lexer::IDENT ), $end ) ), // identifier + array( '0x', array_merge( array( WP_MySQL_Lexer::IDENT ), $end ) ), // identifier + array( "x'ab01'", array_merge( array( WP_MySQL_Lexer::HEX_NUM ), $end ) ), array( "x'ab01xyz'", array() ), // invalid input - array( "x''", array_merge( array( WP_MySQL_Lexer::HEX_NUMBER ), $end ) ), + array( "x''", array_merge( array( WP_MySQL_Lexer::HEX_NUM ), $end ) ), array( "x'", array() ), // invalid input array( "x'ab", array() ), // invalid input // decimal - array( '123.456', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER ), $end ) ), - array( '.123', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER ), $end ) ), - array( '123.', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER ), $end ) ), - array( '123.456abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '.123abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '123.abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier + array( '123.456', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM ), $end ) ), + array( '.123', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM ), $end ) ), + array( '123.', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM ), $end ) ), + array( '123.456abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '.123abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '123.abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier // float - array( '1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '.1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '.1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '.1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1.1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1.1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1.1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier (this differs from INT/BIN/HEX numbers) - array( '1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '.1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '.1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '.1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '1.1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '1.1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '1.1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier + array( '1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '.1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '.1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '.1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1.1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1.1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1.1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier (this differs from INT/BIN/HEX numbers) + array( '1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '.1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '.1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '.1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '1.1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '1.1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '1.1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier // non-numbers - array( '.SELECT', array_merge( array( WP_MySQL_Lexer::DOT_SYMBOL, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not decimal or float - array( '1+e10', array_merge( array( WP_MySQL_Lexer::INT_NUMBER, WP_MySQL_Lexer::PLUS_OPERATOR, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not float - array( '1-e10', array_merge( array( WP_MySQL_Lexer::INT_NUMBER, WP_MySQL_Lexer::MINUS_OPERATOR, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not float + array( '.SELECT', array_merge( array( WP_MySQL_Lexer::DOT, WP_MySQL_Lexer::IDENT ), $end ) ), // not decimal or float + array( '1+e10', array_merge( array( WP_MySQL_Lexer::NUM, WP_MySQL_Lexer::PLUS, WP_MySQL_Lexer::IDENT ), $end ) ), // not float + array( '1-e10', array_merge( array( WP_MySQL_Lexer::NUM, WP_MySQL_Lexer::MINUS, WP_MySQL_Lexer::IDENT ), $end ) ), // not float ); } @@ -690,12 +706,12 @@ public function test_valid_escaped_string( string $sql, int $expected_token_id ) public function data_valid_escaped_strings(): array { return array( - 'escaped single quote' => array( "'it\\'s'", WP_MySQL_Lexer::SINGLE_QUOTED_TEXT ), - 'trailing escaped backslash' => array( "'path\\\\'", WP_MySQL_Lexer::SINGLE_QUOTED_TEXT ), - 'doubled single quote' => array( "'it''s'", WP_MySQL_Lexer::SINGLE_QUOTED_TEXT ), - 'empty single-quoted string' => array( "''", WP_MySQL_Lexer::SINGLE_QUOTED_TEXT ), - 'escaped double quote' => array( '"col\\"name"', WP_MySQL_Lexer::DOUBLE_QUOTED_TEXT ), - 'backtick identifier' => array( '`my_column`', WP_MySQL_Lexer::BACK_TICK_QUOTED_ID ), + 'escaped single quote' => array( "'it\\'s'", WP_MySQL_Lexer::TEXT_STRING ), + 'trailing escaped backslash' => array( "'path\\\\'", WP_MySQL_Lexer::TEXT_STRING ), + 'doubled single quote' => array( "'it''s'", WP_MySQL_Lexer::TEXT_STRING ), + 'empty single-quoted string' => array( "''", WP_MySQL_Lexer::TEXT_STRING ), + 'escaped double quote' => array( '"col\\"name"', WP_MySQL_Lexer::TEXT_STRING ), + 'backtick identifier' => array( '`my_column`', WP_MySQL_Lexer::IDENT_QUOTED ), ); } @@ -749,34 +765,50 @@ public function test_ansi_quotes_changes_double_quote_tokenization( array $sql_modes, string $expected_token_name ): void { - $tokens = ( new WP_MySQL_Lexer( $sql, 80400, $sql_modes ) )->remaining_tokens(); + $tokens = ( new WP_MySQL_Lexer( $sql, 80410, $sql_modes ) )->remaining_tokens(); $this->assertSame( $expected_token_name, $tokens[0]->get_name(), $sql ); } public function data_ansi_quotes_tokenization(): array { return array( - 'double quote is a string by default' => array( '"foo"', array(), 'SINGLE_QUOTED_TEXT' ), - 'double quote is an identifier in ANSI_QUOTES' => array( '"foo"', array( 'ANSI_QUOTES' ), 'BACK_TICK_QUOTED_ID' ), - 'ANSI_QUOTES is case-insensitive' => array( '"foo"', array( 'ansi_quotes' ), 'BACK_TICK_QUOTED_ID' ), - 'ANSI_QUOTES with other modes' => array( '"foo"', array( 'STRICT_ALL_TABLES', 'ANSI_QUOTES' ), 'BACK_TICK_QUOTED_ID' ), - 'single quote stays a string in ANSI_QUOTES' => array( "'foo'", array( 'ANSI_QUOTES' ), 'SINGLE_QUOTED_TEXT' ), - 'backtick stays an identifier in ANSI_QUOTES' => array( '`foo`', array( 'ANSI_QUOTES' ), 'BACK_TICK_QUOTED_ID' ), + 'double quote is a string by default' => array( '"foo"', array(), 'TEXT_STRING' ), + 'double quote is an identifier in ANSI_QUOTES' => array( '"foo"', array( 'ANSI_QUOTES' ), 'IDENT_QUOTED' ), + 'ANSI_QUOTES is case-insensitive' => array( '"foo"', array( 'ansi_quotes' ), 'IDENT_QUOTED' ), + 'ANSI_QUOTES with other modes' => array( '"foo"', array( 'STRICT_ALL_TABLES', 'ANSI_QUOTES' ), 'IDENT_QUOTED' ), + 'single quote stays a string in ANSI_QUOTES' => array( "'foo'", array( 'ANSI_QUOTES' ), 'TEXT_STRING' ), + 'backtick stays an identifier in ANSI_QUOTES' => array( '`foo`', array( 'ANSI_QUOTES' ), 'IDENT_QUOTED' ), + ); + } + + /** + * @dataProvider data_national_strings + */ + public function test_national_strings( string $sql, array $expected ): void { + $this->assertSame( array_merge( $expected, array( 'END_OF_INPUT', 'YYEOF' ) ), self::token_names( $sql ) ); + } + + public function data_national_strings(): array { + return array( + 'uppercase prefix' => array( "N'abc'", array( 'NCHAR_STRING' ) ), + 'lowercase prefix' => array( "n'abc'", array( 'NCHAR_STRING' ) ), + 'escaped quote' => array( "N'a''b'", array( 'NCHAR_STRING' ) ), + 'double quote is not a literal' => array( 'N"abc"', array( 'IDENT', 'TEXT_STRING' ) ), ); } public function test_composite_ansi_mode_expands_to_lexer_component_modes(): void { // ANSI_QUOTES: a double-quoted sequence is a quoted identifier. - $lexer = new WP_MySQL_Lexer( '"foo"', 80400, array( 'ANSI' ) ); + $lexer = new WP_MySQL_Lexer( '"foo"', 80410, array( 'ANSI' ) ); $this->assertTrue( $lexer->next_token() ); - $this->assertSame( WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, $lexer->get_token()->id ); + $this->assertSame( WP_MySQL_Lexer::IDENT_QUOTED, $lexer->get_token()->id ); // PIPES_AS_CONCAT: "||" is the string concatenation operator. - $lexer = new WP_MySQL_Lexer( '||', 80400, array( 'ANSI' ) ); + $lexer = new WP_MySQL_Lexer( '||', 80410, array( 'ANSI' ) ); $this->assertTrue( $lexer->next_token() ); - $this->assertSame( WP_MySQL_Lexer::CONCAT_PIPES_SYMBOL, $lexer->get_token()->id ); + $this->assertSame( WP_MySQL_Lexer::OR_OR_SYM, $lexer->get_token()->id ); // IGNORE_SPACE: whitespace is permitted between a function name and "(". - $lexer = new WP_MySQL_Lexer( 'COUNT (1)', 80400, array( 'ANSI' ) ); + $lexer = new WP_MySQL_Lexer( 'COUNT (1)', 80410, array( 'ANSI' ) ); $this->assertTrue( $lexer->next_token() ); $this->assertSame( WP_MySQL_Lexer::KEYWORDS['COUNT'], $lexer->get_token()->id ); } @@ -792,7 +824,7 @@ public function test_quoted_identifiers_treat_backslash_literally( int $expected_token_id, string $expected_value ): void { - $lexer = new WP_MySQL_Lexer( $sql, 80400, $sql_modes ); + $lexer = new WP_MySQL_Lexer( $sql, 80410, $sql_modes ); $this->assertTrue( $lexer->next_token() ); $token = $lexer->get_token(); @@ -807,16 +839,16 @@ public function data_quoted_identifier_backslash(): array { $sq = "'"; return array( // Backtick identifiers: backslash is a literal character. - 'backtick keeps backslash-n literal' => array( $bt . 'a' . $bs . 'nb' . $bt, array(), WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, 'a' . $bs . 'nb' ), - 'backtick allows trailing backslash' => array( $bt . 'a' . $bs . $bt, array(), WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, 'a' . $bs ), - 'backtick escapes only by doubling' => array( $bt . 'a' . $bt . $bt . 'b' . $bt, array(), WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, 'a' . $bt . 'b' ), + 'backtick keeps backslash-n literal' => array( $bt . 'a' . $bs . 'nb' . $bt, array(), WP_MySQL_Lexer::IDENT_QUOTED, 'a' . $bs . 'nb' ), + 'backtick allows trailing backslash' => array( $bt . 'a' . $bs . $bt, array(), WP_MySQL_Lexer::IDENT_QUOTED, 'a' . $bs ), + 'backtick escapes only by doubling' => array( $bt . 'a' . $bt . $bt . 'b' . $bt, array(), WP_MySQL_Lexer::IDENT_QUOTED, 'a' . $bt . 'b' ), // ANSI-quoted identifiers behave like backtick identifiers. - 'ansi-quoted keeps backslash literal' => array( $dq . 'a' . $bs . 'nb' . $dq, array( 'ANSI_QUOTES' ), WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, 'a' . $bs . 'nb' ), - 'ansi-quoted allows trailing backslash' => array( $dq . 'a' . $bs . $dq, array( 'ANSI_QUOTES' ), WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, 'a' . $bs ), + 'ansi-quoted keeps backslash literal' => array( $dq . 'a' . $bs . 'nb' . $dq, array( 'ANSI_QUOTES' ), WP_MySQL_Lexer::IDENT_QUOTED, 'a' . $bs . 'nb' ), + 'ansi-quoted allows trailing backslash' => array( $dq . 'a' . $bs . $dq, array( 'ANSI_QUOTES' ), WP_MySQL_Lexer::IDENT_QUOTED, 'a' . $bs ), // String literals still process backslash escapes. - 'single-quoted string escapes newline' => array( $sq . 'a' . $bs . 'nb' . $sq, array(), WP_MySQL_Lexer::SINGLE_QUOTED_TEXT, "a\nb" ), + 'single-quoted string escapes newline' => array( $sq . 'a' . $bs . 'nb' . $sq, array(), WP_MySQL_Lexer::TEXT_STRING, "a\nb" ), ); } diff --git a/packages/mysql-parser/tests/WP_MySQL_Server_Suite_Parser_Tests.php b/packages/mysql-parser/tests/WP_MySQL_Server_Suite_Parser_Tests.php index 7e3dab250..d50656cf5 100644 --- a/packages/mysql-parser/tests/WP_MySQL_Server_Suite_Parser_Tests.php +++ b/packages/mysql-parser/tests/WP_MySQL_Server_Suite_Parser_Tests.php @@ -20,7 +20,7 @@ class WP_MySQL_Server_Suite_Parser_Tests extends TestCase { * legitimately changes. */ const EXPECTED_QUERIES = 69158; - const EXPECTED_FAILURES = 170; + const EXPECTED_FAILURES = 167; public function test_corpus_acceptance_rate(): void { $parser = WP_MySQL_Parser_Factory::create_parser(); @@ -36,7 +36,7 @@ public function test_corpus_acceptance_rate(): void { ++$total; // Pin the MySQL version. The failure set in FAILURES_PATH depends on it. - $tokens = ( new WP_MySQL_Lexer( $query, 80038 ) )->remaining_tokens(); + $tokens = ( new WP_MySQL_Lexer( $query, 80410 ) )->remaining_tokens(); if ( null === $parser->parse( $tokens ) ) { $failures[] = $query; } diff --git a/packages/mysql-parser/tests/WP_MySQL_Token_Data_Tests.php b/packages/mysql-parser/tests/WP_MySQL_Token_Data_Tests.php index 4a1e12002..25763c4ce 100644 --- a/packages/mysql-parser/tests/WP_MySQL_Token_Data_Tests.php +++ b/packages/mysql-parser/tests/WP_MySQL_Token_Data_Tests.php @@ -29,6 +29,18 @@ public function test_every_emittable_token_has_a_derived_name(): void { } } + public function test_token_constants_have_distinct_numbers(): void { + // Each token constant names one grammar terminal; none is an alias. + $numbers = array(); + foreach ( ( new ReflectionClass( WP_MySQL_Lexer::class ) )->getConstants() as $name => $value ) { + if ( is_int( $value ) && $value >= 0 && 0 !== strpos( $name, 'SQL_MODE_' ) ) { + $this->assertArrayNotHasKey( $value, $numbers, "Constant $name shares token $value with another constant" ); + $numbers[ $value ] = $name; + } + } + $this->assertNotEmpty( $numbers ); + } + public function test_function_keywords_are_a_subset_of_keywords(): void { $this->assertNotEmpty( WP_MySQL_Lexer::FUNCTIONS ); foreach ( array_keys( WP_MySQL_Lexer::FUNCTIONS ) as $keyword ) { @@ -64,12 +76,12 @@ public function test_derived_names_prefer_plain_keywords_and_constants(): void { $this->assertSame( 'CURRENT_DATE', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::KEYWORDS['CURDATE'] ) ); // Constant name for non-keyword tokens. - $this->assertSame( 'IDENTIFIER', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::IDENTIFIER ) ); - $this->assertSame( 'SINGLE_QUOTED_TEXT', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::SINGLE_QUOTED_TEXT ) ); - $this->assertSame( 'AT_SIGN_SYMBOL', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::AT_SIGN_SYMBOL ) ); - $this->assertSame( 'WITH_ROLLUP_SYMBOL', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::WITH_ROLLUP_SYMBOL ) ); + $this->assertSame( 'IDENT', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::IDENT ) ); + $this->assertSame( 'TEXT_STRING', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::TEXT_STRING ) ); + $this->assertSame( 'AT_SIGN', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::AT_SIGN ) ); + $this->assertSame( 'WITH_ROLLUP_SYM', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::WITH_ROLLUP_SYM ) ); $this->assertSame( 'END_OF_INPUT', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::END_OF_INPUT ) ); - $this->assertSame( 'END_MARKER', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::END_MARKER ) ); + $this->assertSame( 'YYEOF', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::YYEOF ) ); } public function test_internal_scanner_sentinels_cannot_collide_with_grammar_tokens(): void { diff --git a/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php b/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php index 676f1cc00..a35b619b6 100644 --- a/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php +++ b/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php @@ -15,7 +15,7 @@ class WP_MySQL_Token_Tests extends TestCase { * @return WP_MySQL_Token The first matching token. */ private static function first_token( string $sql, string $name, array $sql_modes = array() ): WP_MySQL_Token { - foreach ( ( new WP_MySQL_Lexer( $sql, 80400, $sql_modes ) )->remaining_tokens() as $token ) { + foreach ( ( new WP_MySQL_Lexer( $sql, 80410, $sql_modes ) )->remaining_tokens() as $token ) { if ( $token->get_name() === $name ) { return $token; } @@ -24,13 +24,13 @@ private static function first_token( string $sql, string $name, array $sql_modes } public function test_get_value_unquotes_string_literals(): void { - $this->assertSame( "a'b", self::first_token( "SELECT 'a''b'", 'SINGLE_QUOTED_TEXT' )->get_value() ); - $this->assertSame( 'a"b', self::first_token( 'SELECT "a""b"', 'SINGLE_QUOTED_TEXT' )->get_value() ); - $this->assertSame( "new\nline", self::first_token( "SELECT 'new\\nline'", 'SINGLE_QUOTED_TEXT' )->get_value() ); + $this->assertSame( "a'b", self::first_token( "SELECT 'a''b'", 'TEXT_STRING' )->get_value() ); + $this->assertSame( 'a"b', self::first_token( 'SELECT "a""b"', 'TEXT_STRING' )->get_value() ); + $this->assertSame( "new\nline", self::first_token( "SELECT 'new\\nline'", 'TEXT_STRING' )->get_value() ); } public function test_get_value_honors_no_backslash_escapes_mode(): void { - $token = self::first_token( "SELECT 'a\\nb'", 'SINGLE_QUOTED_TEXT', array( 'NO_BACKSLASH_ESCAPES' ) ); + $token = self::first_token( "SELECT 'a\\nb'", 'TEXT_STRING', array( 'NO_BACKSLASH_ESCAPES' ) ); $this->assertSame( 'a\\nb', $token->get_value() ); } @@ -40,41 +40,41 @@ public function test_get_value_preserves_non_utf8_bytes_in_string_literals(): vo $raw = chr( 0xFF ) . chr( 0xFE ); $this->assertSame( $raw, - self::first_token( "SELECT '$raw'", 'SINGLE_QUOTED_TEXT' )->get_value() + self::first_token( "SELECT '$raw'", 'TEXT_STRING' )->get_value() ); // A backslash escape strips the backslash and preserves the following non-UTF-8 byte. $this->assertSame( chr( 0xE9 ), - self::first_token( "SELECT '\\" . chr( 0xE9 ) . "'", 'SINGLE_QUOTED_TEXT' )->get_value() + self::first_token( "SELECT '\\" . chr( 0xE9 ) . "'", 'TEXT_STRING' )->get_value() ); // Valid multibyte UTF-8 still round-trips. $this->assertSame( 'café 🙂', - self::first_token( "SELECT 'café 🙂'", 'SINGLE_QUOTED_TEXT' )->get_value() + self::first_token( "SELECT 'café 🙂'", 'TEXT_STRING' )->get_value() ); } public function test_get_value_unescapes_newlines_and_multibyte_characters(): void { $this->assertSame( "\n", - self::first_token( "SELECT '\\\n'", 'SINGLE_QUOTED_TEXT' )->get_value() + self::first_token( "SELECT '\\\n'", 'TEXT_STRING' )->get_value() ); $this->assertSame( '🙂', - self::first_token( "SELECT '\\🙂'", 'SINGLE_QUOTED_TEXT' )->get_value() + self::first_token( "SELECT '\\🙂'", 'TEXT_STRING' )->get_value() ); } public function test_get_value_unquotes_backtick_identifiers(): void { - $this->assertSame( 'col name', self::first_token( 'SELECT `col name` FROM t', 'BACK_TICK_QUOTED_ID' )->get_value() ); + $this->assertSame( 'col name', self::first_token( 'SELECT `col name` FROM t', 'IDENT_QUOTED' )->get_value() ); } public function test_get_value_unquotes_ansi_identifiers(): void { $this->assertSame( 'a"b', - self::first_token( 'SELECT "a""b"', 'BACK_TICK_QUOTED_ID', array( 'ANSI_QUOTES' ) )->get_value() + self::first_token( 'SELECT "a""b"', 'IDENT_QUOTED', array( 'ANSI_QUOTES' ) )->get_value() ); } @@ -84,7 +84,7 @@ public function test_get_value_does_not_unquote_unquoted_tokens(): void { // by token ids and must return keyword bytes as-is. $this->assertSame( 'SSL', self::first_token( 'CREATE USER u REQUIRE SSL', 'SSL' )->get_value() ); $this->assertSame( 'SELECT', self::first_token( 'SELECT 1', 'SELECT' )->get_value() ); - $this->assertSame( '42', self::first_token( 'SELECT 42', 'INT_NUMBER' )->get_value() ); + $this->assertSame( '42', self::first_token( 'SELECT 42', 'NUM' )->get_value() ); } public function test_get_name_resolves_bison_terminal_names(): void { @@ -94,7 +94,7 @@ public function test_get_name_resolves_bison_terminal_names(): void { $names[] = $token->get_name(); } $this->assertSame( - array( 'SELECT', 'SINGLE_QUOTED_TEXT', 'COMMA_SYMBOL', 'OPEN_PAR_SYMBOL', 'INT_NUMBER', 'CLOSE_PAR_SYMBOL', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'TEXT_STRING', 'COMMA', 'LEFT_PAREN', 'NUM', 'RIGHT_PAREN', 'END_OF_INPUT', 'YYEOF' ), $names ); } diff --git a/packages/mysql-parser/tests/data/corpus-failures.csv b/packages/mysql-parser/tests/data/corpus-failures.csv index 78e09b53d..563aedfe2 100644 --- a/packages/mysql-parser/tests/data/corpus-failures.csv +++ b/packages/mysql-parser/tests/data/corpus-failures.csv @@ -653,10 +653,6 @@ UNLOCK TABLES" "SELECT 'a%' NOT LIKE 'a!%' ESCAPE '!', 'a%' NOT LIKE 'a!' || '%' ESCAPE '!'" "SELECT 'a%' NOT LIKE 'a!%' ESCAPE '$', 'a%' NOT LIKE 'a!' || '%' ESCAPE '$'" "SELECT * from ""full""" -"SELECT 1 /*!080100 +1*/ AS should_return_2" -"SELECT 1 /*!080100 -+1*/ AS should_return_2" -"SELECT 1 /*!080100 +1*/ AS should_return_2" "SELECT 1 /*!99999 /* */ */" "SET sql_mode = 'NO_ENGINE_SUBSTITUTION'; create procedure test_resignal() diff --git a/packages/mysql-parser/tools/fetch-mysql-grammar.sh b/packages/mysql-parser/tools/fetch-mysql-grammar.sh index f6767be63..9e1141e86 100755 --- a/packages/mysql-parser/tools/fetch-mysql-grammar.sh +++ b/packages/mysql-parser/tools/fetch-mysql-grammar.sh @@ -42,5 +42,8 @@ else sha256 "$build_dir/sql_yacc.yy" "$build_dir/lex.h" | sed 's/^/ /' fi +# Record the tag for the generators, so the generated files name the right source. +echo "$MYSQL_TAG" > "$build_dir/mysql-tag" + echo " -> $build_dir/sql_yacc.yy ($(wc -l < "$build_dir/sql_yacc.yy" | tr -d ' ') lines)" echo " -> $build_dir/lex.h ($(wc -l < "$build_dir/lex.h" | tr -d ' ') lines)" diff --git a/packages/mysql-parser/tools/generate-parse-table.php b/packages/mysql-parser/tools/generate-parse-table.php index 3e39f8c11..c124e279c 100644 --- a/packages/mysql-parser/tools/generate-parse-table.php +++ b/packages/mysql-parser/tools/generate-parse-table.php @@ -45,9 +45,13 @@ } $xml_path = $argv[1]; $output_path = $argv[2]; -$mysql_tag = getenv( 'MYSQL_TAG' ); -if ( false === $mysql_tag || '' === $mysql_tag ) { - $mysql_tag = 'mysql-8.4.3'; + +// The tag of the grammar sources, recorded next to the automaton by fetch-mysql-grammar.sh. +$tag_path = dirname( $xml_path ) . '/mysql-tag'; +$mysql_tag = is_file( $tag_path ) ? trim( file_get_contents( $tag_path ) ) : ''; +if ( '' === $mysql_tag ) { + fwrite( STDERR, "error: MySQL tag not found in $tag_path; run fetch-mysql-grammar.sh first.\n" ); + exit( 1 ); } ini_set( 'memory_limit', '3G' ); diff --git a/packages/mysql-parser/tools/generate-tokens.php b/packages/mysql-parser/tools/generate-tokens.php index 3c807cd66..b072c7819 100644 --- a/packages/mysql-parser/tools/generate-tokens.php +++ b/packages/mysql-parser/tools/generate-tokens.php @@ -10,7 +10,7 @@ * * - Named constants for the structural tokens the scanner code references * (operators, punctuation, literals, and the few keywords with scanner - * logic of their own). + * logic of their own), named after their terminals in sql_yacc.yy. * - KEYWORDS — the keyword string => token number table, taken directly from * MySQL's lex.h. Words declared there with a terminal that is not part of * the grammar (hint-only keywords) are omitted and thus lex as identifiers, @@ -32,9 +32,13 @@ $xml_path = $argv[1]; $lexh_path = $argv[2]; $lexer_path = $argv[3]; -$mysql_tag = getenv( 'MYSQL_TAG' ); -if ( false === $mysql_tag || '' === $mysql_tag ) { - $mysql_tag = 'mysql-8.4.10'; + +// The tag of the grammar sources, recorded next to the automaton by fetch-mysql-grammar.sh. +$tag_path = dirname( $xml_path ) . '/mysql-tag'; +$mysql_tag = is_file( $tag_path ) ? trim( file_get_contents( $tag_path ) ) : ''; +if ( '' === $mysql_tag ) { + fwrite( STDERR, "error: MySQL tag not found in $tag_path; run fetch-mysql-grammar.sh first.\n" ); + exit( 1 ); } // Bison terminal name => token-number, read straight from the automaton. @@ -53,83 +57,91 @@ // would only surface at runtime, far from the cause — fail the build instead. $bison_number = function ( $name ) use ( $terminals ) { if ( ! isset( $terminals[ $name ] ) ) { - fwrite( STDERR, "error: Bison terminal '$name' not found in the automaton; the grammar bridge needs updating for this MySQL version.\n" ); + fwrite( STDERR, "error: Bison terminal '$name' not found in the automaton; the lexer token constants need updating for this MySQL version.\n" ); exit( 1 ); } return $terminals[ $name ]; }; /* - * Named constants: lexer constant name => Bison terminal name. Single-char - * tokens use Bison's quoted-character names (e.g. "'('"); the rest are the - * explicit %token names from sql_yacc.yy. These are the tokens the scanner - * code refers to by name: operators and literals it recognises structurally, - * and the few keywords with scanner logic of their own (the WITH ROLLUP - * contraction and the HIGH_NOT_PRECEDENCE mode), plus the end markers. + * Token constants for the terminals the scanner code refers to by name: + * operators and literals it recognises structurally, the few keywords with + * scanner logic of their own ("\N", the WITH ROLLUP contraction, and the + * HIGH_NOT_PRECEDENCE mode), and the end markers. + * + * Each constant is named after its %token in sql_yacc.yy. Single-character + * terminals, which Bison names by the quoted character (e.g. "'('"), are + * named after the character, and Bison's end-of-file terminal ("$end") is + * named YYEOF, as in the C code that Bison generates. */ -$constants = array( - 'OPEN_PAR_SYMBOL' => "'('", - 'CLOSE_PAR_SYMBOL' => "')'", - 'OPEN_CURLY_SYMBOL' => "'{'", - 'CLOSE_CURLY_SYMBOL' => "'}'", - 'AT_SIGN_SYMBOL' => "'@'", - 'COLON_SYMBOL' => "':'", - 'COMMA_SYMBOL' => "','", - 'DOT_SYMBOL' => "'.'", - 'SEMICOLON_SYMBOL' => "';'", - 'BITWISE_AND_OPERATOR' => "'&'", - 'BITWISE_NOT_OPERATOR' => "'~'", - 'BITWISE_OR_OPERATOR' => "'|'", - 'BITWISE_XOR_OPERATOR' => "'^'", - 'DIV_OPERATOR' => "'/'", - 'MINUS_OPERATOR' => "'-'", - 'MOD_OPERATOR' => "'%'", - 'MULT_OPERATOR' => "'*'", - 'PLUS_OPERATOR' => "'+'", - 'LOGICAL_NOT_OPERATOR' => "'!'", - 'EQUAL_OPERATOR' => 'EQ', - 'NULL_SAFE_EQUAL_OPERATOR' => 'EQUAL_SYM', - 'GREATER_OR_EQUAL_OPERATOR' => 'GE', - 'GREATER_THAN_OPERATOR' => 'GT_SYM', - 'LESS_OR_EQUAL_OPERATOR' => 'LE', - 'LESS_THAN_OPERATOR' => 'LT', - 'NOT_EQUAL_OPERATOR' => 'NE', - 'ASSIGN_OPERATOR' => 'SET_VAR', - 'LOGICAL_AND_OPERATOR' => 'AND_AND_SYM', - 'LOGICAL_OR_OPERATOR' => 'OR2_SYM', - 'CONCAT_PIPES_SYMBOL' => 'OR_OR_SYM', - 'SHIFT_LEFT_OPERATOR' => 'SHIFT_LEFT', - 'SHIFT_RIGHT_OPERATOR' => 'SHIFT_RIGHT', - 'JSON_SEPARATOR_SYMBOL' => 'JSON_SEPARATOR_SYM', - 'JSON_UNQUOTED_SEPARATOR_SYMBOL' => 'JSON_UNQUOTED_SEPARATOR_SYM', - 'PARAM_MARKER' => 'PARAM_MARKER', - 'IDENTIFIER' => 'IDENT', - 'BACK_TICK_QUOTED_ID' => 'IDENT_QUOTED', - 'SINGLE_QUOTED_TEXT' => 'TEXT_STRING', - 'DOUBLE_QUOTED_TEXT' => 'TEXT_STRING', - 'NCHAR_TEXT' => 'NCHAR_STRING', - 'UNDERSCORE_CHARSET' => 'UNDERSCORE_CHARSET', - 'BIN_NUMBER' => 'BIN_NUM', - 'HEX_NUMBER' => 'HEX_NUM', - 'INT_NUMBER' => 'NUM', - 'LONG_NUMBER' => 'LONG_NUM', - 'ULONGLONG_NUMBER' => 'ULONGLONG_NUM', - 'DECIMAL_NUMBER' => 'DECIMAL_NUM', - 'FLOAT_NUMBER' => 'FLOAT_NUM', - 'NULL2_SYMBOL' => 'NULL_SYM', - 'NOT_SYMBOL' => 'NOT_SYM', - 'NOT2_SYMBOL' => 'NOT2_SYM', - 'WITH_SYMBOL' => 'WITH', - 'ROLLUP_SYMBOL' => 'ROLLUP_SYM', - 'WITH_ROLLUP_SYMBOL' => 'WITH_ROLLUP_SYM', - 'END_OF_INPUT' => 'END_OF_INPUT', - 'END_MARKER' => '$end', +$character_names = array( + '(' => 'LEFT_PAREN', + ')' => 'RIGHT_PAREN', + '{' => 'LEFT_BRACE', + '}' => 'RIGHT_BRACE', + ',' => 'COMMA', + '.' => 'DOT', + ':' => 'COLON', + ';' => 'SEMICOLON', + '@' => 'AT_SIGN', + '+' => 'PLUS', + '-' => 'MINUS', + '*' => 'ASTERISK', + '/' => 'SLASH', + '%' => 'PERCENT', + '&' => 'AMPERSAND', + '|' => 'PIPE', + '^' => 'CARET', + '~' => 'TILDE', + '!' => 'EXCLAMATION', +); + +$terminal_names = array( + 'EQ', + 'EQUAL_SYM', + 'NE', + 'GE', + 'GT_SYM', + 'LE', + 'LT', + 'SET_VAR', + 'AND_AND_SYM', + 'OR2_SYM', + 'OR_OR_SYM', + 'SHIFT_LEFT', + 'SHIFT_RIGHT', + 'JSON_SEPARATOR_SYM', + 'JSON_UNQUOTED_SEPARATOR_SYM', + 'PARAM_MARKER', + 'IDENT', + 'IDENT_QUOTED', + 'TEXT_STRING', + 'NCHAR_STRING', + 'UNDERSCORE_CHARSET', + 'BIN_NUM', + 'HEX_NUM', + 'NUM', + 'LONG_NUM', + 'ULONGLONG_NUM', + 'DECIMAL_NUM', + 'FLOAT_NUM', + 'NULL_SYM', + 'NOT_SYM', + 'NOT2_SYM', + 'WITH', + 'ROLLUP_SYM', + 'WITH_ROLLUP_SYM', + 'END_OF_INPUT', ); $constant_values = array(); -foreach ( $constants as $constant_name => $terminal_name ) { - $constant_values[ $constant_name ] = $bison_number( $terminal_name ); +foreach ( $character_names as $character => $constant_name ) { + $constant_values[ $constant_name ] = $bison_number( "'$character'" ); +} +foreach ( $terminal_names as $terminal_name ) { + $constant_values[ $terminal_name ] = $bison_number( $terminal_name ); } +$constant_values['YYEOF'] = $bison_number( '$end' ); /* * Keyword table: lex.h declares each keyword string with its Bison terminal