From 3dc1869a954a2ba875204f904b055bf1fcfaa182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jake=C5=A1?= Date: Mon, 2 Nov 2026 13:30:12 +0100 Subject: [PATCH 1/7] Name lexer token constants after the MySQL grammar The token constants kept the names of the MySQL Workbench lexer the package was ported from (OPEN_PAR_SYMBOL, INT_NUMBER, BACK_TICK_QUOTED_ID, ...), and generate-tokens.php mapped them to grammar terminals. Name them after their sql_yacc.yy terminals instead (IDENT, NUM, TEXT_STRING, GT_SYM, ...), the same source as the AST rule names. Single-character terminals have no usable Bison name, so they are named after the character (LEFT_PAREN, ASTERISK, ...). The old role names were often wrong, e.g., "*" is also the SELECT wildcard. The end marker is YYEOF, as in the C code that Bison generates. SINGLE_QUOTED_TEXT and DOUBLE_QUOTED_TEXT were aliases of one terminal, so they become a single TEXT_STRING constant. A new test guards against such aliases. --- .../mysql-parser/src/class-wp-mysql-lexer.php | 292 +++++++++-------- .../mysql-parser/src/class-wp-mysql-token.php | 8 +- .../tests/WP_MySQL_Lexer_Tests.php | 296 +++++++++--------- .../tests/WP_MySQL_Token_Data_Tests.php | 22 +- .../tests/WP_MySQL_Token_Tests.php | 26 +- .../mysql-parser/tools/generate-tokens.php | 142 +++++---- 6 files changed, 397 insertions(+), 389 deletions(-) diff --git a/packages/mysql-parser/src/class-wp-mysql-lexer.php b/packages/mysql-parser/src/class-wp-mysql-lexer.php index cb13dcddc..83196faa0 100644 --- a/packages/mysql-parser/src/class-wp-mysql-lexer.php +++ b/packages/mysql-parser/src/class-wp-mysql-lexer.php @@ -37,62 +37,61 @@ class WP_MySQL_Lexer { // BEGIN: AUTO-GENERATED MYSQL GRAMMAR SYMBOLS // This block is auto-generated from MySQL mysql-8.4.10. DO NOT EDIT! // phpcs:disable - const OPEN_PAR_SYMBOL = 40; - const CLOSE_PAR_SYMBOL = 41; - const OPEN_CURLY_SYMBOL = 123; - const CLOSE_CURLY_SYMBOL = 125; - const AT_SIGN_SYMBOL = 64; - const COLON_SYMBOL = 58; - const COMMA_SYMBOL = 44; - const DOT_SYMBOL = 46; - const SEMICOLON_SYMBOL = 59; - const BITWISE_AND_OPERATOR = 38; - const BITWISE_NOT_OPERATOR = 126; - const BITWISE_OR_OPERATOR = 124; - const BITWISE_XOR_OPERATOR = 94; - const DIV_OPERATOR = 47; - const MINUS_OPERATOR = 45; - const MOD_OPERATOR = 37; - const MULT_OPERATOR = 42; - const PLUS_OPERATOR = 43; - const LOGICAL_NOT_OPERATOR = 33; - const EQUAL_OPERATOR = 415; - const NULL_SAFE_EQUAL_OPERATOR = 416; - const GREATER_OR_EQUAL_OPERATOR = 456; - const GREATER_THAN_OPERATOR = 469; - const LESS_OR_EQUAL_OPERATOR = 522; - const LESS_THAN_OPERATOR = 549; - const NOT_EQUAL_OPERATOR = 614; - const ASSIGN_OPERATOR = 757; - const LOGICAL_AND_OPERATOR = 273; - const LOGICAL_OR_OPERATOR = 642; - const CONCAT_PIPES_SYMBOL = 644; - const SHIFT_LEFT_OPERATOR = 759; - const SHIFT_RIGHT_OPERATOR = 760; - const JSON_SEPARATOR_SYMBOL = 514; - const JSON_UNQUOTED_SEPARATOR_SYMBOL = 907; - const PARAM_MARKER = 652; - const IDENTIFIER = 482; - const BACK_TICK_QUOTED_ID = 484; - const SINGLE_QUOTED_TEXT = 827; - const DOUBLE_QUOTED_TEXT = 827; - const NCHAR_TEXT = 611; - const UNDERSCORE_CHARSET = 852; - const BIN_NUMBER = 292; - const HEX_NUMBER = 474; - const INT_NUMBER = 628; - const LONG_NUMBER = 545; - const ULONGLONG_NUMBER = 849; - const DECIMAL_NUMBER = 377; - const FLOAT_NUMBER = 443; - const NULL2_SYMBOL = 627; - const NOT_SYMBOL = 622; - const NOT2_SYMBOL = 621; - const WITH_SYMBOL = 892; - const ROLLUP_SYMBOL = 734; - const WITH_ROLLUP_SYMBOL = 894; - const END_OF_INPUT = 411; - const END_MARKER = 0; + const LEFT_PAREN = 40; + const RIGHT_PAREN = 41; + const LEFT_BRACE = 123; + const RIGHT_BRACE = 125; + const COMMA = 44; + const DOT = 46; + const COLON = 58; + const SEMICOLON = 59; + const AT_SIGN = 64; + const PLUS = 43; + const MINUS = 45; + const ASTERISK = 42; + const SLASH = 47; + const PERCENT = 37; + const AMPERSAND = 38; + const PIPE = 124; + const CARET = 94; + const TILDE = 126; + const EXCLAMATION = 33; + const EQ = 415; + const EQUAL_SYM = 416; + const NE = 614; + const GE = 456; + const GT_SYM = 469; + const LE = 522; + const LT = 549; + const SET_VAR = 757; + const AND_AND_SYM = 273; + const OR2_SYM = 642; + const OR_OR_SYM = 644; + const SHIFT_LEFT = 759; + const SHIFT_RIGHT = 760; + const JSON_SEPARATOR_SYM = 514; + const JSON_UNQUOTED_SEPARATOR_SYM = 907; + const PARAM_MARKER = 652; + const IDENT = 482; + const IDENT_QUOTED = 484; + const TEXT_STRING = 827; + const NCHAR_STRING = 611; + const UNDERSCORE_CHARSET = 852; + const BIN_NUM = 292; + const HEX_NUM = 474; + const NUM = 628; + const LONG_NUM = 545; + const ULONGLONG_NUM = 849; + const DECIMAL_NUM = 377; + const FLOAT_NUM = 443; + const NULL_SYM = 627; + const NOT_SYM = 622; + const NOT2_SYM = 621; + const WITH = 892; + const ROLLUP_SYM = 734; + const WITH_ROLLUP_SYM = 894; + const END_OF_INPUT = 411; + const YYEOF = 0; const KEYWORDS = array( '!=' => 614, '&&' => 273, '<' => 549, '<<' => 759, '<=' => 522, '<=>' => 416, '<>' => 614, '=' => 415, '>' => 469, '>=' => 456, '>>' => 760, 'ACCESSIBLE' => 259, 'ACCOUNT' => 260, 'ACTION' => 261, 'ACTIVE' => 973, 'ADD' => 262, 'ADDDATE' => 263, 'ADMIN' => 910, 'AFTER' => 264, 'AGAINST' => 265, 'AGGREGATE' => 266, 'ALGORITHM' => 267, 'ALL' => 268, 'ALTER' => 269, 'ALWAYS' => 270, 'ANALYZE' => 272, 'AND' => 274, 'ANY' => 275, 'ARRAY' => 976, 'AS' => 276, 'ASC' => 277, 'ASCII' => 278, 'ASENSITIVE' => 279, 'ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS' => 1161, 'AT' => 280, 'ATTRIBUTE' => 1153, 'AUTHENTICATION' => 1191, 'AUTO' => 1211, 'AUTOEXTEND_SIZE' => 281, 'AUTO_INCREMENT' => 282, 'AVG' => 284, 'AVG_ROW_LENGTH' => 283, 'BACKUP' => 285, 'BEFORE' => 286, 'BEGIN' => 287, 'BERNOULLI' => 1213, 'BETWEEN' => 288, 'BIGINT' => 289, 'BINARY' => 290, 'BINLOG' => 291, 'BIT' => 295, 'BIT_AND' => 293, 'BIT_OR' => 294, 'BIT_XOR' => 296, 'BLOB' => 297, 'BLOCK' => 298, 'BOOL' => 300, 'BOOLEAN' => 299, 'BOTH' => 301, 'BTREE' => 302, 'BUCKETS' => 929, 'BULK' => 1201, 'BY' => 303, 'BYTE' => 304, 'CACHE' => 305, 'CALL' => 306, 'CASCADE' => 307, 'CASCADED' => 308, 'CASE' => 309, 'CAST' => 310, 'CATALOG_NAME' => 311, 'CHAIN' => 312, 'CHALLENGE_RESPONSE' => 1198, 'CHANGE' => 313, 'CHANGED' => 314, 'CHANNEL' => 315, 'CHAR' => 317, 'CHARACTER' => 317, 'CHARSET' => 316, 'CHECK' => 319, 'CHECKSUM' => 318, 'CIPHER' => 320, 'CLASS_ORIGIN' => 321, 'CLIENT' => 322, 'CLONE' => 931, 'CLOSE' => 323, 'COALESCE' => 324, 'CODE' => 325, 'COLLATE' => 326, 'COLLATION' => 327, 'COLUMN' => 329, 'COLUMNS' => 328, 'COLUMN_FORMAT' => 330, 'COLUMN_NAME' => 331, 'COMMENT' => 332, 'COMMIT' => 334, 'COMMITTED' => 333, 'COMPACT' => 335, 'COMPLETION' => 336, 'COMPONENT' => 914, 'COMPRESSED' => 337, 'COMPRESSION' => 338, 'CONCURRENT' => 340, 'CONDITION' => 341, 'CONNECTION' => 342, 'CONSISTENT' => 343, 'CONSTRAINT' => 344, 'CONSTRAINT_CATALOG' => 345, 'CONSTRAINT_NAME' => 346, 'CONSTRAINT_SCHEMA' => 347, 'CONTAINS' => 348, 'CONTEXT' => 349, 'CONTINUE' => 350, 'CONVERT' => 351, 'COUNT' => 352, 'CPU' => 353, 'CREATE' => 354, 'CROSS' => 355, 'CUBE' => 356, 'CUME_DIST' => 932, 'CURDATE' => 357, 'CURRENT' => 358, 'CURRENT_DATE' => 357, 'CURRENT_TIME' => 362, 'CURRENT_TIMESTAMP' => 623, 'CURRENT_USER' => 359, 'CURSOR' => 360, 'CURSOR_NAME' => 361, 'CURTIME' => 362, 'DATA' => 366, 'DATABASE' => 363, 'DATABASES' => 364, 'DATAFILE' => 365, 'DATE' => 370, 'DATETIME' => 367, 'DATE_ADD' => 368, 'DATE_SUB' => 369, 'DAY' => 375, 'DAY_HOUR' => 371, 'DAY_MICROSECOND' => 372, 'DAY_MINUTE' => 373, 'DAY_SECOND' => 374, 'DEALLOCATE' => 376, 'DEC' => 378, 'DECIMAL' => 378, 'DECLARE' => 379, 'DEFAULT' => 380, 'DEFAULT_AUTH' => 381, 'DEFINER' => 382, 'DEFINITION' => 969, 'DELAYED' => 383, 'DELAY_KEY_WRITE' => 384, 'DELETE' => 385, 'DENSE_RANK' => 933, 'DESC' => 386, 'DESCRIBE' => 387, 'DESCRIPTION' => 970, 'DETERMINISTIC' => 389, 'DIAGNOSTICS' => 390, 'DIRECTORY' => 391, 'DISABLE' => 392, 'DISCARD' => 393, 'DISK' => 394, 'DISTINCT' => 395, 'DISTINCTROW' => 395, 'DIV' => 396, 'DO' => 398, 'DOUBLE' => 397, 'DROP' => 399, 'DUAL' => 400, 'DUMPFILE' => 401, 'DUPLICATE' => 402, 'DYNAMIC' => 403, 'EACH' => 404, 'ELSE' => 405, 'ELSEIF' => 406, 'EMPTY' => 954, 'ENABLE' => 407, 'ENCLOSED' => 408, 'ENCRYPTION' => 339, 'END' => 409, 'ENDS' => 410, 'ENFORCED' => 985, 'ENGINE' => 413, 'ENGINES' => 412, 'ENGINE_ATTRIBUTE' => 1154, 'ENUM' => 414, 'ERROR' => 417, 'ERRORS' => 418, 'ESCAPE' => 420, 'ESCAPED' => 419, 'EVENT' => 422, 'EVENTS' => 421, 'EVERY' => 423, 'EXCEPT' => 913, 'EXCHANGE' => 424, 'EXCLUDE' => 934, 'EXECUTE' => 425, 'EXISTS' => 426, 'EXIT' => 427, 'EXPANSION' => 428, 'EXPIRE' => 429, 'EXPLAIN' => 387, 'EXPORT' => 430, 'EXTENDED' => 431, 'EXTENT_SIZE' => 432, 'EXTRACT' => 433, 'FACTOR' => 1192, 'FAILED_LOGIN_ATTEMPTS' => 995, 'FALSE' => 434, 'FAST' => 435, 'FAULTS' => 436, 'FETCH' => 437, 'FIELDS' => 328, 'FILE' => 438, 'FILE_BLOCK_SIZE' => 439, 'FILTER' => 440, 'FINISH' => 1193, 'FIRST' => 441, 'FIRST_VALUE' => 935, 'FIXED' => 442, 'FLOAT' => 444, 'FLOAT4' => 444, 'FLOAT8' => 397, 'FLUSH' => 445, 'FOLLOWING' => 936, 'FOLLOWS' => 446, 'FOR' => 449, 'FORCE' => 447, 'FOREIGN' => 448, 'FORMAT' => 450, 'FOUND' => 451, 'FROM' => 452, 'FULL' => 453, 'FULLTEXT' => 454, 'FUNCTION' => 455, 'GENERAL' => 457, 'GENERATE' => 1203, 'GENERATED' => 458, 'GEOMCOLLECTION' => 460, 'GEOMETRY' => 461, 'GEOMETRYCOLLECTION' => 460, 'GET' => 463, 'GET_FORMAT' => 462, 'GET_SOURCE_PUBLIC_KEY' => 1162, 'GLOBAL' => 464, 'GRANT' => 465, 'GRANTS' => 466, 'GROUP' => 467, 'GROUPING' => 926, 'GROUPS' => 937, 'GROUP_CONCAT' => 468, 'GROUP_REPLICATION' => 459, 'GTIDS' => 1207, 'GTID_ONLY' => 1199, 'HANDLER' => 470, 'HASH' => 471, 'HAVING' => 472, 'HELP' => 473, 'HIGH_PRIORITY' => 475, 'HISTOGRAM' => 928, 'HISTORY' => 959, 'HOST' => 476, 'HOSTS' => 477, 'HOUR' => 481, 'HOUR_MICROSECOND' => 478, 'HOUR_MINUTE' => 479, 'HOUR_SECOND' => 480, 'IDENTIFIED' => 483, 'IF' => 485, 'IGNORE' => 486, 'IGNORE_SERVER_IDS' => 487, 'IMPORT' => 488, 'IN' => 504, 'INACTIVE' => 974, 'INDEX' => 490, 'INDEXES' => 489, 'INFILE' => 491, 'INITIAL' => 1197, 'INITIAL_SIZE' => 492, 'INITIATE' => 1194, 'INNER' => 493, 'INOUT' => 494, 'INSENSITIVE' => 495, 'INSERT' => 496, 'INSERT_METHOD' => 497, 'INSTALL' => 499, 'INSTANCE' => 498, 'INT' => 502, 'INT1' => 836, 'INT2' => 768, 'INT3' => 584, 'INT4' => 502, 'INT8' => 289, 'INTEGER' => 502, 'INTERSECT' => 1200, 'INTERVAL' => 500, 'INTO' => 501, 'INVISIBLE' => 911, 'INVOKER' => 503, 'IO' => 507, 'IO_AFTER_GTIDS' => 505, 'IO_BEFORE_GTIDS' => 506, 'IO_THREAD' => 702, 'IPC' => 508, 'IS' => 509, 'ISOLATION' => 510, 'ISSUER' => 511, 'ITERATE' => 512, 'JOIN' => 513, 'JSON' => 515, 'JSON_ARRAYAGG' => 921, 'JSON_OBJECTAGG' => 920, 'JSON_TABLE' => 955, 'JSON_VALUE' => 1151, 'KEY' => 518, 'KEYRING' => 1190, 'KEYS' => 516, 'KEY_BLOCK_SIZE' => 517, 'KILL' => 519, 'LAG' => 938, 'LANGUAGE' => 520, 'LAST' => 521, 'LAST_VALUE' => 939, 'LATERAL' => 975, 'LEAD' => 940, 'LEADING' => 523, 'LEAVE' => 525, 'LEAVES' => 524, 'LEFT' => 526, 'LESS' => 527, 'LEVEL' => 528, 'LIKE' => 530, 'LIMIT' => 531, 'LINEAR' => 532, 'LINES' => 533, 'LINESTRING' => 534, 'LIST' => 535, 'LOAD' => 536, 'LOCAL' => 537, 'LOCALTIME' => 623, 'LOCALTIMESTAMP' => 623, 'LOCK' => 540, 'LOCKED' => 924, 'LOCKS' => 539, 'LOG' => 1206, 'LOGFILE' => 541, 'LOGS' => 542, 'LONG' => 546, 'LONGBLOB' => 543, 'LONGTEXT' => 544, 'LOOP' => 547, 'LOW_PRIORITY' => 548, 'MANUAL' => 1212, 'MASTER' => 571, 'MATCH' => 574, 'MAX' => 579, 'MAXVALUE' => 582, 'MAX_CONNECTIONS_PER_HOUR' => 575, 'MAX_QUERIES_PER_HOUR' => 576, 'MAX_ROWS' => 577, 'MAX_SIZE' => 578, 'MAX_UPDATES_PER_HOUR' => 580, 'MAX_USER_CONNECTIONS' => 581, 'MEDIUM' => 586, 'MEDIUMBLOB' => 583, 'MEDIUMINT' => 584, 'MEDIUMTEXT' => 585, 'MEMBER' => 977, 'MEMORY' => 587, 'MERGE' => 588, 'MESSAGE_TEXT' => 589, 'MICROSECOND' => 590, 'MID' => 811, 'MIDDLEINT' => 584, 'MIGRATE' => 591, 'MIN' => 596, 'MINUTE' => 594, 'MINUTE_MICROSECOND' => 592, 'MINUTE_SECOND' => 593, 'MIN_ROWS' => 595, 'MOD' => 600, 'MODE' => 597, 'MODIFIES' => 598, 'MODIFY' => 599, 'MONTH' => 601, 'MULTILINESTRING' => 602, 'MULTIPOINT' => 603, 'MULTIPOLYGON' => 604, 'MUTEX' => 605, 'MYSQL_ERRNO' => 606, 'NAME' => 608, 'NAMES' => 607, 'NATIONAL' => 609, 'NATURAL' => 610, 'NCHAR' => 612, 'NDB' => 613, 'NDBCLUSTER' => 613, 'NESTED' => 956, 'NETWORK_NAMESPACE' => 987, 'NEVER' => 616, 'NEW' => 617, 'NEXT' => 618, 'NO' => 624, 'NODEGROUP' => 619, 'NONE' => 620, 'NOT' => 622, 'NOW' => 623, 'NOWAIT' => 925, 'NO_WAIT' => 625, 'NO_WRITE_TO_BINLOG' => 626, 'NTH_VALUE' => 941, 'NTILE' => 942, 'NULL' => 627, 'NULLS' => 943, 'NUMBER' => 629, 'NUMERIC' => 630, 'NVARCHAR' => 631, 'OF' => 922, 'OFF' => 998, 'OFFSET' => 632, 'OJ' => 986, 'OLD' => 984, 'ON' => 633, 'ONE' => 634, 'ONLY' => 635, 'OPEN' => 636, 'OPTIMIZE' => 637, 'OPTIMIZER_COSTS' => 638, 'OPTION' => 640, 'OPTIONAL' => 978, 'OPTIONALLY' => 641, 'OPTIONS' => 639, 'OR' => 645, 'ORDER' => 643, 'ORDINALITY' => 957, 'ORGANIZATION' => 971, 'OTHERS' => 944, 'OUT' => 648, 'OUTER' => 646, 'OUTFILE' => 647, 'OVER' => 945, 'OWNER' => 649, 'PACK_KEYS' => 650, 'PAGE' => 651, 'PARALLEL' => 1208, 'PARSER' => 653, 'PARSE_TREE' => 1205, 'PARTIAL' => 655, 'PARTITION' => 656, 'PARTITIONING' => 658, 'PARTITIONS' => 657, 'PASSWORD' => 659, 'PASSWORD_LOCK_TIME' => 994, 'PATH' => 958, 'PERCENT_RANK' => 946, 'PERSIST' => 908, 'PERSIST_ONLY' => 927, 'PHASE' => 660, 'PLUGIN' => 662, 'PLUGINS' => 663, 'PLUGIN_DIR' => 661, 'POINT' => 664, 'POLYGON' => 665, 'PORT' => 666, 'POSITION' => 667, 'PRECEDES' => 668, 'PRECEDING' => 947, 'PRECISION' => 669, 'PREPARE' => 670, 'PRESERVE' => 671, 'PREV' => 672, 'PRIMARY' => 673, 'PRIVILEGES' => 674, 'PRIVILEGE_CHECKS_USER' => 991, 'PROCEDURE' => 675, 'PROCESS' => 676, 'PROCESSLIST' => 677, 'PROFILE' => 678, 'PROFILES' => 679, 'PROXY' => 680, 'PURGE' => 681, 'QUALIFY' => 1210, 'QUARTER' => 682, 'QUERY' => 683, 'QUICK' => 684, 'RANDOM' => 988, 'RANGE' => 685, 'RANK' => 948, 'READ' => 688, 'READS' => 686, 'READ_ONLY' => 687, 'READ_WRITE' => 689, 'REAL' => 690, 'REBUILD' => 691, 'RECOVER' => 692, 'RECURSIVE' => 915, 'REDO_BUFFER_SIZE' => 694, 'REDUNDANT' => 695, 'REFERENCE' => 972, 'REFERENCES' => 696, 'REGEXP' => 697, 'REGISTRATION' => 1195, 'RELAY' => 698, 'RELAYLOG' => 699, 'RELAY_LOG_FILE' => 700, 'RELAY_LOG_POS' => 701, 'RELAY_THREAD' => 702, 'RELEASE' => 703, 'RELOAD' => 704, 'REMOVE' => 705, 'RENAME' => 706, 'REORGANIZE' => 707, 'REPAIR' => 708, 'REPEAT' => 710, 'REPEATABLE' => 709, 'REPLACE' => 711, 'REPLICA' => 1159, 'REPLICAS' => 1160, 'REPLICATE_DO_DB' => 713, 'REPLICATE_DO_TABLE' => 715, 'REPLICATE_IGNORE_DB' => 714, 'REPLICATE_IGNORE_TABLE' => 716, 'REPLICATE_REWRITE_DB' => 719, 'REPLICATE_WILD_DO_TABLE' => 717, 'REPLICATE_WILD_IGNORE_TABLE' => 718, 'REPLICATION' => 712, 'REQUIRE' => 720, 'REQUIRE_ROW_FORMAT' => 993, 'REQUIRE_TABLE_PRIMARY_KEY_CHECK' => 996, 'RESET' => 721, 'RESIGNAL' => 722, 'RESOURCE' => 963, 'RESPECT' => 949, 'RESTART' => 968, 'RESTORE' => 724, 'RESTRICT' => 725, 'RESUME' => 726, 'RETAIN' => 983, 'RETURN' => 729, 'RETURNED_SQLSTATE' => 727, 'RETURNING' => 999, 'RETURNS' => 728, 'REUSE' => 960, 'REVERSE' => 730, 'REVOKE' => 731, 'RIGHT' => 732, 'RLIKE' => 697, 'ROLE' => 909, 'ROLLBACK' => 733, 'ROLLUP' => 734, 'ROTATE' => 735, 'ROUTINE' => 736, 'ROW' => 739, 'ROWS' => 737, 'ROW_COUNT' => 740, 'ROW_FORMAT' => 738, 'ROW_NUMBER' => 950, 'RTREE' => 741, 'S3' => 1209, 'SAVEPOINT' => 742, 'SCHEDULE' => 743, 'SCHEMA' => 363, 'SCHEMAS' => 364, 'SCHEMA_NAME' => 744, 'SECOND' => 746, 'SECONDARY' => 979, 'SECONDARY_ENGINE' => 980, 'SECONDARY_ENGINE_ATTRIBUTE' => 1155, 'SECONDARY_LOAD' => 981, 'SECONDARY_UNLOAD' => 982, 'SECOND_MICROSECOND' => 745, 'SECURITY' => 747, 'SELECT' => 748, 'SENSITIVE' => 749, 'SEPARATOR' => 750, 'SERIAL' => 752, 'SERIALIZABLE' => 751, 'SERVER' => 754, 'SESSION' => 753, 'SESSION_USER' => 867, 'SET' => 756, 'SHARE' => 758, 'SHOW' => 761, 'SHUTDOWN' => 762, 'SIGNAL' => 763, 'SIGNED' => 764, 'SIMPLE' => 765, 'SKIP' => 923, 'SLAVE' => 766, 'SLOW' => 767, 'SMALLINT' => 768, 'SNAPSHOT' => 769, 'SOCKET' => 770, 'SOME' => 275, 'SONAME' => 771, 'SOUNDS' => 772, 'SOURCE' => 773, 'SOURCE_AUTO_POSITION' => 1163, 'SOURCE_BIND' => 1164, 'SOURCE_COMPRESSION_ALGORITHMS' => 1165, 'SOURCE_CONNECTION_AUTO_FAILOVER' => 1156, 'SOURCE_CONNECT_RETRY' => 1166, 'SOURCE_DELAY' => 1167, 'SOURCE_HEARTBEAT_PERIOD' => 1168, 'SOURCE_HOST' => 1169, 'SOURCE_LOG_FILE' => 1170, 'SOURCE_LOG_POS' => 1171, 'SOURCE_PASSWORD' => 1172, 'SOURCE_PORT' => 1173, 'SOURCE_PUBLIC_KEY_PATH' => 1174, 'SOURCE_RETRY_COUNT' => 1175, 'SOURCE_SSL' => 1176, 'SOURCE_SSL_CA' => 1177, 'SOURCE_SSL_CAPATH' => 1178, 'SOURCE_SSL_CERT' => 1179, 'SOURCE_SSL_CIPHER' => 1180, 'SOURCE_SSL_CRL' => 1181, 'SOURCE_SSL_CRLPATH' => 1182, 'SOURCE_SSL_KEY' => 1183, 'SOURCE_SSL_VERIFY_SERVER_CERT' => 1184, 'SOURCE_TLS_CIPHERSUITES' => 1185, 'SOURCE_TLS_VERSION' => 1186, 'SOURCE_USER' => 1187, 'SOURCE_ZSTD_COMPRESSION_LEVEL' => 1188, 'SPATIAL' => 774, 'SPECIFIC' => 775, 'SQL' => 788, 'SQLEXCEPTION' => 776, 'SQLSTATE' => 777, 'SQLWARNING' => 778, 'SQL_AFTER_GTIDS' => 779, 'SQL_AFTER_MTS_GAPS' => 780, 'SQL_BEFORE_GTIDS' => 781, 'SQL_BIG_RESULT' => 782, 'SQL_BUFFER_RESULT' => 783, 'SQL_CALC_FOUND_ROWS' => 785, 'SQL_NO_CACHE' => 786, 'SQL_SMALL_RESULT' => 787, 'SQL_THREAD' => 789, 'SQL_TSI_DAY' => 375, 'SQL_TSI_HOUR' => 481, 'SQL_TSI_MINUTE' => 594, 'SQL_TSI_MONTH' => 601, 'SQL_TSI_QUARTER' => 682, 'SQL_TSI_SECOND' => 746, 'SQL_TSI_WEEK' => 887, 'SQL_TSI_YEAR' => 905, 'SRID' => 961, 'SSL' => 790, 'STACKED' => 791, 'START' => 794, 'STARTING' => 792, 'STARTS' => 793, 'STATS_AUTO_RECALC' => 795, 'STATS_PERSISTENT' => 796, 'STATS_SAMPLE_PAGES' => 797, 'STATUS' => 798, 'STD' => 800, 'STDDEV' => 800, 'STDDEV_POP' => 800, 'STDDEV_SAMP' => 799, 'STOP' => 801, 'STORAGE' => 802, 'STORED' => 803, 'STRAIGHT_JOIN' => 804, 'STREAM' => 997, 'STRING' => 805, 'ST_COLLECT' => 1189, 'SUBCLASS_ORIGIN' => 806, 'SUBDATE' => 807, 'SUBJECT' => 808, 'SUBPARTITION' => 810, 'SUBPARTITIONS' => 809, 'SUBSTR' => 811, 'SUBSTRING' => 811, 'SUM' => 812, 'SUPER' => 813, 'SUSPEND' => 814, 'SWAPS' => 815, 'SWITCHES' => 816, 'SYSDATE' => 817, 'SYSTEM' => 964, 'SYSTEM_USER' => 867, 'TABLE' => 821, 'TABLES' => 818, 'TABLESAMPLE' => 1214, 'TABLESPACE' => 819, 'TABLE_CHECKSUM' => 822, 'TABLE_NAME' => 823, 'TEMPORARY' => 824, 'TEMPTABLE' => 825, 'TERMINATED' => 826, 'TEXT' => 828, 'THAN' => 829, 'THEN' => 830, 'THREAD_PRIORITY' => 962, 'TIES' => 951, 'TIME' => 834, 'TIMESTAMP' => 831, 'TIMESTAMPADD' => 832, 'TIMESTAMPDIFF' => 833, 'TINYBLOB' => 835, 'TINYINT' => 836, 'TINYTEXT' => 837, 'TLS' => 1152, 'TO' => 838, 'TRAILING' => 839, 'TRANSACTION' => 840, 'TRIGGER' => 842, 'TRIGGERS' => 841, 'TRIM' => 843, 'TRUE' => 844, 'TRUNCATE' => 845, 'TYPE' => 847, 'TYPES' => 846, 'UNBOUNDED' => 952, 'UNCOMMITTED' => 850, 'UNDEFINED' => 851, 'UNDO' => 855, 'UNDOFILE' => 853, 'UNDO_BUFFER_SIZE' => 854, 'UNICODE' => 856, 'UNINSTALL' => 857, 'UNION' => 858, 'UNIQUE' => 859, 'UNKNOWN' => 860, 'UNLOCK' => 861, 'UNREGISTER' => 1196, 'UNSIGNED' => 862, 'UNTIL' => 863, 'UPDATE' => 864, 'UPGRADE' => 865, 'URL' => 1202, 'USAGE' => 866, 'USE' => 869, 'USER' => 867, 'USER_RESOURCES' => 723, 'USE_FRM' => 868, 'USING' => 870, 'UTC_DATE' => 871, 'UTC_TIME' => 873, 'UTC_TIMESTAMP' => 872, 'VALIDATION' => 874, 'VALUE' => 876, 'VALUES' => 875, 'VARBINARY' => 877, 'VARCHAR' => 878, 'VARCHARACTER' => 878, 'VARIABLES' => 879, 'VARIANCE' => 880, 'VARYING' => 881, 'VAR_POP' => 880, 'VAR_SAMP' => 882, 'VCPU' => 965, 'VIEW' => 883, 'VIRTUAL' => 884, 'VISIBLE' => 912, 'WAIT' => 885, 'WARNINGS' => 886, 'WEEK' => 887, 'WEIGHT_STRING' => 888, 'WHEN' => 889, 'WHERE' => 890, 'WHILE' => 891, 'WINDOW' => 953, 'WITH' => 892, 'WITHOUT' => 895, 'WORK' => 896, 'WRAPPER' => 897, 'WRITE' => 898, 'X509' => 899, 'XA' => 900, 'XID' => 901, 'XML' => 902, 'XOR' => 903, 'YEAR' => 905, 'YEAR_MONTH' => 904, 'ZEROFILL' => 906, 'ZONE' => 1157, '||' => 644 ); @@ -503,8 +502,8 @@ public function remaining_tokens(): array { // Common single-token lexeme — the overwhelming majority. if ( self::EOF !== $type - && self::WITH_SYMBOL !== $type - && self::AT_SIGN_SYMBOL !== $type + && self::WITH !== $type + && self::AT_SIGN !== $type ) { $tokens[] = new WP_MySQL_Token( $type, $start, $length, $sql, $nbe ); continue; @@ -514,7 +513,7 @@ public function remaining_tokens(): array { if ( self::EOF === $type ) { $this->emit_end_markers( $tokens ); break; - } elseif ( self::WITH_SYMBOL === $type ) { + } elseif ( self::WITH === $type ) { $this->produce_with_or_rollup( $tokens, $start, $length ); } else { $this->produce_at_variable( $tokens, $start ); @@ -572,10 +571,9 @@ public static function get_token_id( string $token_name ): ?int { * * Names are derived from the grammar data rather than stored: a keyword * token resolves to its keyword string via self::KEYWORDS, any other token - * to its token constant name via reflection. When several names share a - * token number (keyword synonyms, equal-valued constants), plain keywords - * win over paren-gated function keywords (USER over SESSION_USER), then the - * first name in table order wins. + * to its token constant name via reflection. When several keywords share a + * token number (synonyms), plain keywords win over paren-gated function + * keywords (USER over SESSION_USER), then the first name in table order wins. * * This method is intended to be used only for testing and debugging purposes, * when tokens need to be presented by their names in a human-readable form. @@ -634,9 +632,9 @@ private function produce( array &$out ): void { if ( self::EOF === $type ) { $this->emit_end_markers( $out ); - } elseif ( self::WITH_SYMBOL === $type ) { + } elseif ( self::WITH === $type ) { $this->produce_with_or_rollup( $out, $start, $length ); - } elseif ( self::AT_SIGN_SYMBOL === $type ) { + } elseif ( self::AT_SIGN === $type ) { $this->produce_at_variable( $out, $start ); } else { $out[] = $this->make_token( $type, $start, $length ); @@ -655,21 +653,21 @@ private function produce( array &$out ): void { private function produce_with_or_rollup( array &$out, int $with_start, int $with_length ): void { if ( ! $this->scan_lexeme() ) { // "WITH" was the last valid token; the trailing input is invalid. - $out[] = $this->make_token( self::WITH_SYMBOL, $with_start, $with_length ); + $out[] = $this->make_token( self::WITH, $with_start, $with_length ); $this->stream_ended = true; return; } - if ( self::ROLLUP_SYMBOL === $this->token_type ) { + if ( self::ROLLUP_SYM === $this->token_type ) { $out[] = $this->make_token( - self::WITH_ROLLUP_SYMBOL, + self::WITH_ROLLUP_SYM, $with_start, $this->bytes_already_read - $with_start ); return; } - $out[] = $this->make_token( self::WITH_SYMBOL, $with_start, $with_length ); + $out[] = $this->make_token( self::WITH, $with_start, $with_length ); if ( self::EOF === $this->token_type ) { $this->emit_end_markers( $out ); return; @@ -695,13 +693,13 @@ private function produce_with_or_rollup( array &$out, int $with_start, int $with * @param int $at_start Byte offset of the "@" token. */ private function produce_at_variable( array &$out, int $at_start ): void { - $out[] = $this->make_token( self::AT_SIGN_SYMBOL, $at_start, 1 ); + $out[] = $this->make_token( self::AT_SIGN, $at_start, 1 ); $position = $this->bytes_already_read; // Right after the "@". $next = $this->sql[ $position ] ?? ''; if ( '@' === $next ) { - $out[] = $this->make_token( self::AT_SIGN_SYMBOL, $position, 1 ); + $out[] = $this->make_token( self::AT_SIGN, $position, 1 ); $this->bytes_already_read = $position + 1; // The name after it may be a keyword ("session" in "@@session.x"). @@ -722,10 +720,10 @@ private function produce_at_variable( array &$out, int $at_start ): void { $position ); if ( $name_length > 0 ) { - $out[] = $this->make_token( self::IDENTIFIER, $position, $name_length ); + $out[] = $this->make_token( self::IDENT, $position, $name_length ); $this->bytes_already_read = $position + $name_length; } elseif ( "'" !== $next && '"' !== $next && '`' !== $next ) { - $out[] = $this->make_token( self::IDENTIFIER, $position, 0 ); + $out[] = $this->make_token( self::IDENT, $position, 0 ); } } @@ -736,7 +734,7 @@ private function produce_at_variable( array &$out, int $at_start ): void { private function emit_end_markers( array &$out ): void { $end = $this->bytes_already_read; $out[] = $this->make_token( self::END_OF_INPUT, $end, 0 ); - $out[] = $this->make_token( self::END_MARKER, $end, 0 ); + $out[] = $this->make_token( self::YYEOF, $end, 0 ); $this->stream_ended = true; } @@ -795,7 +793,7 @@ private function read_next_token(): ?int { } if ( self::STATE_IDENTIFIER_OR_KEYWORD === $state ) { $type = $this->read_identifier(); - return self::IDENTIFIER === $type + return self::IDENT === $type ? $this->determine_identifier_or_keyword_type( $this->get_current_token_bytes() ) : $type; } @@ -806,18 +804,18 @@ private function read_next_token(): ?int { // A map for a single-byte symbol fast path. static $single_byte_ops = array( - '(' => self::OPEN_PAR_SYMBOL, - ')' => self::CLOSE_PAR_SYMBOL, - ',' => self::COMMA_SYMBOL, - ';' => self::SEMICOLON_SYMBOL, - '+' => self::PLUS_OPERATOR, - '~' => self::BITWISE_NOT_OPERATOR, - '%' => self::MOD_OPERATOR, - '^' => self::BITWISE_XOR_OPERATOR, + '(' => self::LEFT_PAREN, + ')' => self::RIGHT_PAREN, + ',' => self::COMMA, + ';' => self::SEMICOLON, + '+' => self::PLUS, + '~' => self::TILDE, + '%' => self::PERCENT, + '^' => self::CARET, '?' => self::PARAM_MARKER, - '{' => self::OPEN_CURLY_SYMBOL, - '}' => self::CLOSE_CURLY_SYMBOL, - '=' => self::EQUAL_OPERATOR, + '{' => self::LEFT_BRACE, + '}' => self::RIGHT_BRACE, + '=' => self::EQ, ); // Fast path for keywords and identifiers. @@ -836,18 +834,18 @@ private function read_next_token(): ?int { ) { $started_at = $this->bytes_already_read; $type = $this->read_identifier(); - if ( self::IDENTIFIER === $type ) { + if ( self::IDENT === $type ) { // Inline the keyword lookup on the hot identifier path: most // identifiers are not keywords, so this avoids two method calls // (token-bytes extraction + keyword determination) per token. $word = strtoupper( substr( $this->sql, $started_at, $this->bytes_already_read - $started_at ) ); - $keyword = self::KEYWORDS[ $word ] ?? self::IDENTIFIER; + $keyword = self::KEYWORDS[ $word ] ?? self::IDENT; // Before "." and a name, a keyword is an identifier ("order.id"). // read_identifier() sets STATE_IDENTIFIER_SEPARATOR in that case. if ( - self::IDENTIFIER !== $keyword + self::IDENT !== $keyword && self::STATE_IDENTIFIER_SEPARATOR !== $this->next_state ) { $type = $this->resolve_keyword_type( $keyword, $word ); @@ -871,7 +869,7 @@ private function read_next_token(): ?int { $type = $this->read_number(); } else { $this->bytes_already_read += 1; - $type = self::DOT_SYMBOL; + $type = self::DOT; // A name character right after it starts a qualified name part. if ( null !== $next_byte && ( strspn( $next_byte, self::IDENTIFIER_MASK ) > 0 || $next_byte > "\x7F" ) ) { @@ -882,9 +880,9 @@ private function read_next_token(): ?int { $this->bytes_already_read += 1; if ( '=' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::ASSIGN_OPERATOR; + $type = self::SET_VAR; } else { - $type = self::COLON_SYMBOL; + $type = self::COLON; } } elseif ( '<' === $byte ) { $this->bytes_already_read += 1; @@ -892,37 +890,37 @@ private function read_next_token(): ?int { $this->bytes_already_read += 1; if ( '>' === ( $this->sql[ $this->bytes_already_read ] ?? null ) ) { $this->bytes_already_read += 1; - $type = self::NULL_SAFE_EQUAL_OPERATOR; + $type = self::EQUAL_SYM; } else { - $type = self::LESS_OR_EQUAL_OPERATOR; + $type = self::LE; } } elseif ( '>' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::NOT_EQUAL_OPERATOR; + $type = self::NE; } elseif ( '<' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::SHIFT_LEFT_OPERATOR; + $type = self::SHIFT_LEFT; } else { - $type = self::LESS_THAN_OPERATOR; + $type = self::LT; } } elseif ( '>' === $byte ) { $this->bytes_already_read += 1; if ( '=' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::GREATER_OR_EQUAL_OPERATOR; + $type = self::GE; } elseif ( '>' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::SHIFT_RIGHT_OPERATOR; + $type = self::SHIFT_RIGHT; } else { - $type = self::GREATER_THAN_OPERATOR; + $type = self::GT_SYM; } } elseif ( '!' === $byte ) { $this->bytes_already_read += 1; if ( '=' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::NOT_EQUAL_OPERATOR; + $type = self::NE; } else { - $type = self::LOGICAL_NOT_OPERATOR; + $type = self::EXCLAMATION; } } elseif ( '-' === $byte ) { if ( @@ -936,20 +934,20 @@ private function read_next_token(): ?int { if ( '>' === ( $this->sql[ $this->bytes_already_read ] ?? null ) ) { $this->bytes_already_read += 1; if ( $this->mysql_version >= 50713 ) { - $type = self::JSON_UNQUOTED_SEPARATOR_SYMBOL; + $type = self::JSON_UNQUOTED_SEPARATOR_SYM; } else { return null; // Invalid input. } } else { if ( $this->mysql_version >= 50708 ) { - $type = self::JSON_SEPARATOR_SYMBOL; + $type = self::JSON_SEPARATOR_SYM; } else { return null; // Invalid input. } } } else { $this->bytes_already_read += 1; - $type = self::MINUS_OPERATOR; + $type = self::MINUS; } } elseif ( '*' === $byte ) { $this->bytes_already_read += 1; @@ -958,7 +956,7 @@ private function read_next_token(): ?int { $type = self::MYSQL_COMMENT_END; $this->in_mysql_comment = false; } else { - $type = self::MULT_OPERATOR; + $type = self::ASTERISK; } } elseif ( '/' === $byte ) { if ( '*' === $next_byte ) { @@ -971,36 +969,36 @@ private function read_next_token(): ?int { } } else { $this->bytes_already_read += 1; - $type = self::DIV_OPERATOR; + $type = self::SLASH; } } elseif ( '&' === $byte ) { $this->bytes_already_read += 1; if ( '&' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::LOGICAL_AND_OPERATOR; + $type = self::AND_AND_SYM; } else { - $type = self::BITWISE_AND_OPERATOR; + $type = self::AMPERSAND; } } elseif ( '|' === $byte ) { $this->bytes_already_read += 1; if ( '|' === $next_byte ) { $this->bytes_already_read += 1; $type = $this->is_sql_mode_active( self::SQL_MODE_PIPES_AS_CONCAT ) - ? self::CONCAT_PIPES_SYMBOL - : self::LOGICAL_OR_OPERATOR; + ? self::OR_OR_SYM + : self::OR2_SYM; } else { - $type = self::BITWISE_OR_OPERATOR; + $type = self::PIPE; } } elseif ( '@' === $byte ) { // MySQL treats "@" as a standalone terminal; the variable name that // follows is emitted as its own token (see produce_at_variable()). $this->bytes_already_read += 1; - $type = self::AT_SIGN_SYMBOL; + $type = self::AT_SIGN; } elseif ( '\\' === $byte ) { $this->bytes_already_read += 1; if ( 'N' === $next_byte ) { $this->bytes_already_read += 1; - $type = self::NULL2_SYMBOL; + $type = self::NULL_SYM; } else { return null; // Invalid input. } @@ -1014,14 +1012,14 @@ private function read_next_token(): ?int { } elseif ( ( 'n' === $byte || 'N' === $byte ) && "'" === $next_byte ) { $this->bytes_already_read += 1; $type = $this->read_quoted_text( "'" ); - if ( self::SINGLE_QUOTED_TEXT === $type ) { - $type = self::NCHAR_TEXT; + if ( self::TEXT_STRING === $type ) { + $type = self::NCHAR_STRING; } } elseif ( null === $byte ) { $type = self::EOF; } else { $type = $this->read_identifier(); - if ( self::IDENTIFIER === $type ) { + if ( self::IDENT === $type ) { if ( '_' === $byte && isset( self::UNDERSCORE_CHARSETS[ strtolower( $this->get_current_token_bytes() ) ] ) ) { $type = self::UNDERSCORE_CHARSET; } else { @@ -1124,7 +1122,7 @@ private function read_identifier(): ?int { ) { $this->next_state = self::STATE_IDENTIFIER_SEPARATOR; } - return self::IDENTIFIER; + return self::IDENT; } private function read_number(): ?int { @@ -1155,7 +1153,7 @@ private function read_number(): ?int { } $this->bytes_already_read += 1; // Consume the "'". } - $type = self::HEX_NUMBER; + $type = self::HEX_NUM; } elseif ( // BIN number in the form of 0bN. ( @@ -1178,18 +1176,18 @@ private function read_number(): ?int { } $this->bytes_already_read += 1; // Consume the "'". } - $type = self::BIN_NUMBER; + $type = self::BIN_NUM; } else { // Here, we have a sequence starting with N or .N, where N is a digit. // 1. Try integer first. $this->bytes_already_read += strspn( $this->sql, self::DIGIT_MASK, $this->bytes_already_read ); - $type = self::INT_NUMBER; + $type = self::NUM; // 2. In case of N. or .N, it's a decimal or float number. if ( '.' === ( $this->sql[ $this->bytes_already_read ] ?? null ) ) { $this->bytes_already_read += 1; - $type = self::DECIMAL_NUMBER; + $type = self::DECIMAL_NUM; $this->bytes_already_read += strspn( $this->sql, self::DIGIT_MASK, $this->bytes_already_read ); } @@ -1211,7 +1209,7 @@ private function read_number(): ?int { $this->bytes_already_read += 1; // Consume the 'e' or 'E'. $this->bytes_already_read += 1; // Consume the '+', '-', or digit. $this->bytes_already_read += strspn( $this->sql, self::DIGIT_MASK, $this->bytes_already_read ); - $type = self::FLOAT_NUMBER; + $type = self::FLOAT_NUM; } } @@ -1230,7 +1228,7 @@ private function read_number(): ?int { */ $text = $this->get_current_token_bytes(); $possible_identifier_prefix = - self::INT_NUMBER === $type + self::NUM === $type || ( '0' === $text[0] && ( 'b' === $text[1] || 'x' === $text[1] ) ); /* @@ -1240,16 +1238,16 @@ private function read_number(): ?int { * here, as we can look only at subsequent bytes instead of backtracking * to the beginning of the number (for valid identifiers like 0b019). */ - if ( $possible_identifier_prefix && self::IDENTIFIER === $this->read_identifier() ) { - $type = self::IDENTIFIER; + if ( $possible_identifier_prefix && self::IDENT === $this->read_identifier() ) { + $type = self::IDENT; } // Determine integer type. - if ( self::INT_NUMBER === $type ) { + if ( self::NUM === $type ) { // Fast path for most integers. $bytes = $this->get_current_token_bytes(); if ( strlen( $bytes ) < 10 ) { - return self::INT_NUMBER; + return self::NUM; } // Remove leading zeros. @@ -1258,23 +1256,23 @@ private function read_number(): ?int { // Determine integer type based on its length and value. if ( $length < 10 ) { - return self::INT_NUMBER; + return self::NUM; } elseif ( 10 === $length ) { return strcmp( $bytes, '2147483647' ) > 0 - ? self::LONG_NUMBER - : self::INT_NUMBER; + ? self::LONG_NUM + : self::NUM; } elseif ( $length < 19 ) { - return self::LONG_NUMBER; + return self::LONG_NUM; } elseif ( 19 === $length ) { return strcmp( $bytes, '9223372036854775807' ) > 0 - ? self::ULONGLONG_NUMBER - : self::LONG_NUMBER; + ? self::ULONGLONG_NUM + : self::LONG_NUM; } elseif ( 20 === $length ) { return strcmp( $bytes, '18446744073709551615' ) > 0 - ? self::DECIMAL_NUMBER - : self::ULONGLONG_NUMBER; + ? self::DECIMAL_NUM + : self::ULONGLONG_NUM; } else { - return self::DECIMAL_NUMBER; + return self::DECIMAL_NUM; } } return $type; @@ -1358,13 +1356,7 @@ private function read_quoted_text(): ?int { $this->bytes_already_read = $at; - if ( $is_identifier_quote ) { - return self::BACK_TICK_QUOTED_ID; - } elseif ( '"' === $quote ) { - return self::DOUBLE_QUOTED_TEXT; - } else { - return self::SINGLE_QUOTED_TEXT; - } + return $is_identifier_quote ? self::IDENT_QUOTED : self::TEXT_STRING; } private function read_line_comment(): int { @@ -1419,9 +1411,9 @@ private function read_comment_content(): void { private function determine_identifier_or_keyword_type( string $value ): int { $word = strtoupper( $value ); - $type = self::KEYWORDS[ $word ] ?? self::IDENTIFIER; - if ( self::IDENTIFIER === $type ) { - return self::IDENTIFIER; + $type = self::KEYWORDS[ $word ] ?? self::IDENT; + if ( self::IDENT === $type ) { + return self::IDENT; } return $this->resolve_keyword_type( $type, $word ); } @@ -1443,13 +1435,13 @@ private function resolve_keyword_type( int $type, string $word ): int { $peek += strspn( $this->sql, self::WHITESPACE_MASK, $peek ); } if ( '(' !== ( $this->sql[ $peek ] ?? null ) ) { - return self::IDENTIFIER; + return self::IDENT; } } // With "SQL_MODE_HIGH_NOT_PRECEDENCE" enabled, "NOT" needs to be emitted as a higher priority NOT2 symbol. - if ( self::NOT_SYMBOL === $type && $this->is_sql_mode_active( self::SQL_MODE_HIGH_NOT_PRECEDENCE ) ) { - return self::NOT2_SYMBOL; + if ( self::NOT_SYM === $type && $this->is_sql_mode_active( self::SQL_MODE_HIGH_NOT_PRECEDENCE ) ) { + return self::NOT2_SYM; } return $type; diff --git a/packages/mysql-parser/src/class-wp-mysql-token.php b/packages/mysql-parser/src/class-wp-mysql-token.php index 123e74213..010e03c46 100644 --- a/packages/mysql-parser/src/class-wp-mysql-token.php +++ b/packages/mysql-parser/src/class-wp-mysql-token.php @@ -62,11 +62,7 @@ public function get_name(): string { */ public function get_value(): string { $value = $this->get_bytes(); - if ( - WP_MySQL_Lexer::SINGLE_QUOTED_TEXT === $this->id - || WP_MySQL_Lexer::DOUBLE_QUOTED_TEXT === $this->id - || WP_MySQL_Lexer::BACK_TICK_QUOTED_ID === $this->id - ) { + if ( WP_MySQL_Lexer::TEXT_STRING === $this->id || WP_MySQL_Lexer::IDENT_QUOTED === $this->id ) { // Remove bounding quotes. $quote = $value[0]; $value = substr( $value, 1, -1 ); @@ -79,7 +75,7 @@ public function get_value(): string { * See: https://dev.mysql.com/doc/refman/8.4/en/identifiers.html */ if ( - WP_MySQL_Lexer::BACK_TICK_QUOTED_ID === $this->id + WP_MySQL_Lexer::IDENT_QUOTED === $this->id || $this->sql_mode_no_backslash_escapes_enabled ) { return str_replace( $quote . $quote, $quote, $value ); diff --git a/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php b/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php index 47589b72f..97e47fcd9 100644 --- a/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php +++ b/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php @@ -17,7 +17,7 @@ public function test_tokenize_valid_input(): void { // id $this->assertTrue( $lexer->next_token() ); - $this->assertSame( WP_MySQL_Lexer::IDENTIFIER, $lexer->get_token()->id ); + $this->assertSame( WP_MySQL_Lexer::IDENT, $lexer->get_token()->id ); // FROM $this->assertTrue( $lexer->next_token() ); @@ -25,13 +25,13 @@ public function test_tokenize_valid_input(): void { // users $this->assertTrue( $lexer->next_token() ); - $this->assertSame( WP_MySQL_Lexer::IDENTIFIER, $lexer->get_token()->id ); + $this->assertSame( WP_MySQL_Lexer::IDENT, $lexer->get_token()->id ); // The stream ends with END_OF_INPUT followed by Bison's end marker. $this->assertTrue( $lexer->next_token() ); $this->assertSame( WP_MySQL_Lexer::END_OF_INPUT, $lexer->get_token()->id ); $this->assertTrue( $lexer->next_token() ); - $this->assertSame( WP_MySQL_Lexer::END_MARKER, $lexer->get_token()->id ); + $this->assertSame( WP_MySQL_Lexer::YYEOF, $lexer->get_token()->id ); // No more tokens. $this->assertFalse( $lexer->next_token() ); @@ -80,7 +80,7 @@ private static function token_names( string $sql, array $sql_modes = array() ): public function test_emits_bison_terminals_with_end_markers(): void { $this->assertSame( - array( 'SELECT', 'IDENTIFIER', 'FROM', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'IDENT', 'FROM', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT id FROM users' ) ); } @@ -89,14 +89,14 @@ public function test_ignore_space_preserves_function_name_ranges(): void { // Without "(", COUNT is an identifier whose range excludes trailing whitespace. $sql = 'SELECT COUNT FROM t'; $tokens = ( new WP_MySQL_Lexer( $sql, 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); - $this->assertSame( 'IDENTIFIER', $tokens[1]->get_name(), $sql ); + $this->assertSame( 'IDENT', $tokens[1]->get_name(), $sql ); $this->assertSame( 'COUNT', $tokens[1]->get_value(), $sql ); $this->assertSame( 5, $tokens[1]->length, $sql ); $this->assertSame( 13, $tokens[2]->start, $sql ); $sql = "SELECT COUNT\t\n FROM t"; $tokens = ( new WP_MySQL_Lexer( $sql, 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); - $this->assertSame( 'IDENTIFIER', $tokens[1]->get_name(), $sql ); + $this->assertSame( 'IDENT', $tokens[1]->get_name(), $sql ); $this->assertSame( 'COUNT', $tokens[1]->get_value(), $sql ); $this->assertSame( 5, $tokens[1]->length, $sql ); $this->assertSame( 15, $tokens[2]->start, $sql ); @@ -105,12 +105,12 @@ public function test_ignore_space_preserves_function_name_ranges(): void { $tokens = ( new WP_MySQL_Lexer( 'SELECT COUNT (1)', 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); $this->assertSame( 5, $tokens[1]->length ); $this->assertSame( 'COUNT', $tokens[1]->get_name() ); - $this->assertSame( 'OPEN_PAR_SYMBOL', $tokens[2]->get_name() ); + $this->assertSame( 'LEFT_PAREN', $tokens[2]->get_name() ); $this->assertSame( 13, $tokens[2]->start ); $sql = "SELECT COUNT \t\n"; $tokens = ( new WP_MySQL_Lexer( $sql, 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); - $this->assertSame( 'IDENTIFIER', $tokens[1]->get_name(), $sql ); + $this->assertSame( 'IDENT', $tokens[1]->get_name(), $sql ); $this->assertSame( 'COUNT', $tokens[1]->get_value(), $sql ); $this->assertSame( 5, $tokens[1]->length, $sql ); $this->assertSame( 'END_OF_INPUT', $tokens[2]->get_name(), $sql ); @@ -122,7 +122,7 @@ public function test_ignore_space_preserves_function_name_ranges(): void { */ public function test_version_comments( string $sql, int $mysql_version, array $expected_bytes ): void { $tokens = ( new WP_MySQL_Lexer( $sql, $mysql_version ) )->remaining_tokens(); - $this->assertSame( 'END_MARKER', array_pop( $tokens )->get_name() ); + $this->assertSame( 'YYEOF', array_pop( $tokens )->get_name() ); $this->assertSame( 'END_OF_INPUT', array_pop( $tokens )->get_name() ); $this->assertSame( $expected_bytes, @@ -192,8 +192,8 @@ public function data_version_comments(): array { public function test_at_name_splits_into_at_and_ident(): void { $tokens = ( new WP_MySQL_Lexer( 'SELECT @var1' ) )->remaining_tokens(); - $this->assertSame( 'AT_SIGN_SYMBOL', $tokens[1]->get_name() ); - $this->assertSame( 'IDENTIFIER', $tokens[2]->get_name() ); + $this->assertSame( 'AT_SIGN', $tokens[1]->get_name() ); + $this->assertSame( 'IDENT', $tokens[2]->get_name() ); $this->assertSame( 'var1', $tokens[2]->get_value() ); $this->assertSame( 7, $tokens[1]->start ); $this->assertSame( 1, $tokens[1]->length ); @@ -203,7 +203,7 @@ public function test_at_name_splits_into_at_and_ident(): void { public function test_at_at_splits_into_two_at_signs(): void { $this->assertSame( - array( 'SELECT', 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'AT_SIGN', 'AT_SIGN', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT @@sql_mode' ) ); } @@ -212,13 +212,13 @@ public function test_bare_at_emits_empty_name(): void { // MySQL's lexer emits an empty LEX_HOSTNAME after a bare "@", making // "user1@" (an empty host part) and "SELECT @" valid. $tokens = ( new WP_MySQL_Lexer( 'SELECT @' ) )->remaining_tokens(); - $this->assertSame( 'AT_SIGN_SYMBOL', $tokens[1]->get_name() ); - $this->assertSame( 'IDENTIFIER', $tokens[2]->get_name() ); + $this->assertSame( 'AT_SIGN', $tokens[1]->get_name() ); + $this->assertSame( 'IDENT', $tokens[2]->get_name() ); $this->assertSame( 0, $tokens[2]->length ); $this->assertSame( '', $tokens[2]->get_value() ); $this->assertSame( - array( 'CREATE', 'USER', 'IDENTIFIER', 'AT_SIGN_SYMBOL', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'CREATE', 'USER', 'IDENT', 'AT_SIGN', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'CREATE USER user1@' ) ); } @@ -226,14 +226,14 @@ public function test_bare_at_emits_empty_name(): void { public function test_bare_at_before_quote_stands_alone(): void { // In "@'name'" the quoted text supplies the name itself. $this->assertSame( - array( 'SET', 'AT_SIGN_SYMBOL', 'SINGLE_QUOTED_TEXT', '=', 'INT_NUMBER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SET', 'AT_SIGN', 'TEXT_STRING', '=', 'NUM', 'END_OF_INPUT', 'YYEOF' ), self::token_names( "SET @'v' = 1" ) ); } public function test_with_rollup_is_contracted(): void { $names = self::token_names( 'SELECT 1 FROM t GROUP BY a WITH ROLLUP' ); - $this->assertContains( 'WITH_ROLLUP_SYMBOL', $names ); + $this->assertContains( 'WITH_ROLLUP_SYM', $names ); $this->assertNotContains( 'WITH', $names ); } @@ -241,7 +241,7 @@ public function test_with_rollup_contracts_across_comments(): void { $tokens = ( new WP_MySQL_Lexer( 'SELECT 1 FROM t GROUP BY a WITH /* c */ ROLLUP' ) )->remaining_tokens(); $rollup = null; foreach ( $tokens as $token ) { - if ( 'WITH_ROLLUP_SYMBOL' === $token->get_name() ) { + if ( 'WITH_ROLLUP_SYM' === $token->get_name() ) { $rollup = $token; } } @@ -251,40 +251,40 @@ public function test_with_rollup_contracts_across_comments(): void { public function test_lone_with_is_emitted(): void { $this->assertSame( - array( 'WITH', 'IDENTIFIER', 'AS', 'OPEN_PAR_SYMBOL', 'SELECT', 'INT_NUMBER', 'CLOSE_PAR_SYMBOL', 'SELECT', 'MULT_OPERATOR', 'FROM', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'WITH', 'IDENT', 'AS', 'LEFT_PAREN', 'SELECT', 'NUM', 'RIGHT_PAREN', 'SELECT', 'ASTERISK', 'FROM', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'WITH c AS (SELECT 1) SELECT * FROM c' ) ); // A statement ending on WITH still emits it before the end markers. $this->assertSame( - array( 'SELECT', 'INT_NUMBER', 'WITH', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'NUM', 'WITH', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT 1 WITH' ) ); } public function test_invalid_input_returns_partial_stream_without_end_markers(): void { $names = self::token_names( "SELECT 1 WITH \x01" ); - $this->assertSame( array( 'SELECT', 'INT_NUMBER', 'WITH' ), $names ); + $this->assertSame( array( 'SELECT', 'NUM', 'WITH' ), $names ); } public function test_system_variable_name_must_not_be_empty(): void { // As in MySQL, the name after "@@" can't be empty, and only backticks // can quote it. Otherwise, the stream ends after the "@@". - $at_at = array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL' ); + $at_at = array( 'AT_SIGN', 'AT_SIGN' ); $this->assertSame( $at_at, self::token_names( '@@' ) ); $this->assertSame( $at_at, self::token_names( '@@ version' ) ); $this->assertSame( $at_at, self::token_names( "@@'version'" ) ); $this->assertSame( $at_at, self::token_names( '@@"version"' ) ); $this->assertSame( $at_at, self::token_names( '@@"version"', array( 'ANSI_QUOTES' ) ) ); $this->assertSame( - array_merge( $at_at, array( 'BACK_TICK_QUOTED_ID', 'END_OF_INPUT', 'END_MARKER' ) ), + array_merge( $at_at, array( 'IDENT_QUOTED', 'END_OF_INPUT', 'YYEOF' ) ), self::token_names( '@@`version`' ) ); } public function test_high_not_precedence_emits_not2(): void { $names = self::token_names( 'SELECT NOT 1', array( 'HIGH_NOT_PRECEDENCE' ) ); - $this->assertContains( 'NOT2_SYMBOL', $names ); + $this->assertContains( 'NOT2_SYM', $names ); $names = self::token_names( 'SELECT NOT 1' ); $this->assertContains( 'NOT', $names ); @@ -293,7 +293,7 @@ public function test_high_not_precedence_emits_not2(): void { public function test_end_of_input_word_is_an_identifier(): void { // "end_of_input" is not a MySQL keyword; it must not truncate the stream. $this->assertSame( - array( 'SELECT', 'IDENTIFIER', 'FROM', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'IDENT', 'FROM', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT end_of_input FROM t' ) ); } @@ -302,18 +302,18 @@ public function test_current_date_is_a_keyword_without_parentheses(): void { // CURRENT_DATE/CURRENT_TIME are plain reserved keywords in MySQL 8.4 // (lex.h SYM), unlike CURDATE/CURTIME which require parentheses. $this->assertSame( - array( 'SELECT', 'CURRENT_DATE', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'CURRENT_DATE', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT CURRENT_DATE' ) ); $this->assertSame( - array( 'SELECT', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT curdate' ) ); } public function test_json_aggregates_are_keywords_only_before_parenthesis(): void { $this->assertSame( - array( 'SELECT', 'IDENTIFIER', 'FROM', 'IDENTIFIER', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'IDENT', 'FROM', 'IDENT', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT json_objectagg FROM t' ) ); $names = self::token_names( 'SELECT JSON_OBJECTAGG(a, b) FROM t' ); @@ -321,10 +321,10 @@ public function test_json_aggregates_are_keywords_only_before_parenthesis(): voi } public function test_number_tokens_follow_mysql_magnitude_classes(): void { - $this->assertSame( array( 'SELECT', 'INT_NUMBER', 'END_OF_INPUT', 'END_MARKER' ), self::token_names( 'SELECT 2147483647' ) ); - $this->assertSame( array( 'SELECT', 'LONG_NUMBER', 'END_OF_INPUT', 'END_MARKER' ), self::token_names( 'SELECT 2147483648' ) ); - $this->assertSame( array( 'SELECT', 'ULONGLONG_NUMBER', 'END_OF_INPUT', 'END_MARKER' ), self::token_names( 'SELECT 18446744073709551615' ) ); - $this->assertSame( array( 'SELECT', 'DECIMAL_NUMBER', 'END_OF_INPUT', 'END_MARKER' ), self::token_names( 'SELECT 18446744073709551616' ) ); + $this->assertSame( array( 'SELECT', 'NUM', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT 2147483647' ) ); + $this->assertSame( array( 'SELECT', 'LONG_NUM', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT 2147483648' ) ); + $this->assertSame( array( 'SELECT', 'ULONGLONG_NUM', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT 18446744073709551615' ) ); + $this->assertSame( array( 'SELECT', 'DECIMAL_NUM', 'END_OF_INPUT', 'YYEOF' ), self::token_names( 'SELECT 18446744073709551616' ) ); } /** @@ -341,7 +341,7 @@ public function test_identifier_utf8_range(): void { $type = $lexer->get_token()->id; $is_valid = preg_match( '/^[\x{0080}-\x{ffff}]$/u', $value ); if ( $is_valid ) { - $this->assertSame( WP_MySQL_Lexer::IDENTIFIER, $type ); + $this->assertSame( WP_MySQL_Lexer::IDENT, $type ); } else { // A surrogate codepoint renders as an empty string, so the lexer // emits no identifier — only the end-of-input terminal. @@ -369,7 +369,7 @@ public function test_identifier_utf8_two_byte_sequences(): void { $is_valid = preg_match( '/^[\x{0080}-\x{ffff}]$/u', $value ); if ( $is_valid ) { $this->assertTrue( $result ); - $this->assertSame( WP_MySQL_Lexer::IDENTIFIER, $token->id ); + $this->assertSame( WP_MySQL_Lexer::IDENT, $token->id ); } else { $this->assertFalse( $result ); $this->assertNull( $token ); @@ -399,7 +399,7 @@ public function test_identifier_utf8_three_byte_sequences(): void { $is_valid = preg_match( '/^[\x{0080}-\x{ffff}]$/u', $value ); if ( $is_valid ) { $this->assertTrue( $result ); - $this->assertSame( WP_MySQL_Lexer::IDENTIFIER, $token->id ); + $this->assertSame( WP_MySQL_Lexer::IDENT, $token->id ); } else { $this->assertFalse( $result ); $this->assertNull( $token ); @@ -431,12 +431,12 @@ public function test_underscore_charset_name_after_dot_is_identifier( string $sq public function data_underscore_charset_after_dot(): array { return array( // `t . _utf8` - the member name must be an identifier, not a charset. - 'charset name after dot is identifier' => array( 't._utf8', 2, WP_MySQL_Lexer::IDENTIFIER ), - 'other charset name after dot' => array( 'a._binary', 2, WP_MySQL_Lexer::IDENTIFIER ), + 'charset name after dot is identifier' => array( 't._utf8', 2, WP_MySQL_Lexer::IDENT ), + 'other charset name after dot' => array( 'a._binary', 2, WP_MySQL_Lexer::IDENT ), // A genuine charset introducer (before a string) stays a charset. 'charset introducer before string' => array( "_utf8'x'", 0, WP_MySQL_Lexer::UNDERSCORE_CHARSET ), // A non-charset underscore name after a dot stays an identifier. - 'non-charset underscore name after dot' => array( 't._foo', 2, WP_MySQL_Lexer::IDENTIFIER ), + 'non-charset underscore name after dot' => array( 't._foo', 2, WP_MySQL_Lexer::IDENT ), ); } @@ -448,7 +448,7 @@ public function data_underscore_charset_after_dot(): array { */ public function test_qualified_name_tokens( string $sql, array $expected ): void { $this->assertSame( - array_merge( $expected, array( 'END_OF_INPUT', 'END_MARKER' ) ), + array_merge( $expected, array( 'END_OF_INPUT', 'YYEOF' ) ), self::token_names( $sql ) ); } @@ -457,54 +457,54 @@ public function test_qualified_name_tokens( string $sql, array $expected ): void * @return array */ public function data_qualified_name_tokens(): array { - $qualified = array( 'IDENTIFIER', 'DOT_SYMBOL', 'IDENTIFIER' ); + $qualified = array( 'IDENT', 'DOT', 'IDENT' ); return array( // A qualified name part can start with digits. 'digits only' => array( 't.1', $qualified ), 'digit-led identifier' => array( 't.1st', $qualified ), 'float-like identifier' => array( 't.1e5', $qualified ), - 'part ends before a sign' => array( 't.1e+5', array_merge( $qualified, array( 'PLUS_OPERATOR', 'INT_NUMBER' ) ) ), + 'part ends before a sign' => array( 't.1e+5', array_merge( $qualified, array( 'PLUS', 'NUM' ) ) ), 'after a digit-led qualifier' => array( '1ea10.1', $qualified ), - 'three parts' => array( 'd.1.2', array_merge( $qualified, array( 'DOT_SYMBOL', 'IDENTIFIER' ) ) ), + 'three parts' => array( 'd.1.2', array_merge( $qualified, array( 'DOT', 'IDENT' ) ) ), // Elsewhere, ".N" is a number. - 'after a quoted identifier' => array( '`t`.1', array( 'BACK_TICK_QUOTED_ID', 'DECIMAL_NUMBER' ) ), - 'after whitespace' => array( 't .1', array( 'IDENTIFIER', 'DECIMAL_NUMBER' ) ), - 'after a separator dot' => array( 't..1', array( 'IDENTIFIER', 'DOT_SYMBOL', 'DECIMAL_NUMBER' ) ), - 'after a decimal' => array( '1.5.6', array( 'DECIMAL_NUMBER', 'DECIMAL_NUMBER' ) ), - 'after a float' => array( '1e5.1', array( 'FLOAT_NUMBER', 'DECIMAL_NUMBER' ) ), - 'after a hex number' => array( '0x1F.1', array( 'HEX_NUMBER', 'DECIMAL_NUMBER' ) ), - 'after a binary number' => array( '0b1.1', array( 'BIN_NUMBER', 'DECIMAL_NUMBER' ) ), - 'after "\N"' => array( '\N.1', array( 'NULL', 'DECIMAL_NUMBER' ) ), - 'after a version comment' => array( '/*!50000.5*/', array( 'DECIMAL_NUMBER' ) ), + 'after a quoted identifier' => array( '`t`.1', array( 'IDENT_QUOTED', 'DECIMAL_NUM' ) ), + 'after whitespace' => array( 't .1', array( 'IDENT', 'DECIMAL_NUM' ) ), + 'after a separator dot' => array( 't..1', array( 'IDENT', 'DOT', 'DECIMAL_NUM' ) ), + 'after a decimal' => array( '1.5.6', array( 'DECIMAL_NUM', 'DECIMAL_NUM' ) ), + 'after a float' => array( '1e5.1', array( 'FLOAT_NUM', 'DECIMAL_NUM' ) ), + 'after a hex number' => array( '0x1F.1', array( 'HEX_NUM', 'DECIMAL_NUM' ) ), + 'after a binary number' => array( '0b1.1', array( 'BIN_NUM', 'DECIMAL_NUM' ) ), + 'after "\N"' => array( '\N.1', array( 'NULL', 'DECIMAL_NUM' ) ), + 'after a version comment' => array( '/*!50000.5*/', array( 'DECIMAL_NUM' ) ), // A user variable name includes the dot. - 'in a user variable' => array( '@a.1', array( 'AT_SIGN_SYMBOL', 'IDENTIFIER' ) ), + 'in a user variable' => array( '@a.1', array( 'AT_SIGN', 'IDENT' ) ), // A keyword before a dot and a name is an identifier. 'keyword before a name' => array( 'order.id', $qualified ), 'keyword before digits' => array( 'select.1', $qualified ), 'keyword before non-ASCII' => array( 'order.ñ', $qualified ), 'keyword before "$"' => array( 'order.$x', $qualified ), - 'keyword in three parts' => array( 'order.t.id', array_merge( $qualified, array( 'DOT_SYMBOL', 'IDENTIFIER' ) ) ), + 'keyword in three parts' => array( 'order.t.id', array_merge( $qualified, array( 'DOT', 'IDENT' ) ) ), // A name right after a "." token is always an identifier. - 'keyword after a dot' => array( '`t`.order', array( 'BACK_TICK_QUOTED_ID', 'DOT_SYMBOL', 'IDENTIFIER' ) ), - 'literal prefix after a dot' => array( "t.x'41'", array_merge( $qualified, array( 'SINGLE_QUOTED_TEXT' ) ) ), + 'keyword after a dot' => array( '`t`.order', array( 'IDENT_QUOTED', 'DOT', 'IDENT' ) ), + 'literal prefix after a dot' => array( "t.x'41'", array_merge( $qualified, array( 'TEXT_STRING' ) ) ), // Otherwise, it stays a keyword or a charset. - 'keyword after a decimal' => array( '1.FROM', array( 'DECIMAL_NUMBER', 'FROM' ) ), - 'keyword before a backtick' => array( 'order.`id`', array( 'ORDER', 'DOT_SYMBOL', 'BACK_TICK_QUOTED_ID' ) ), - 'keyword before an asterisk' => array( 'order.*', array( 'ORDER', 'DOT_SYMBOL', 'MULT_OPERATOR' ) ), - 'keyword before whitespace' => array( 'order .id', array( 'ORDER', 'DOT_SYMBOL', 'IDENTIFIER' ) ), - 'system variable scope' => array( '@@session.sql_mode', array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'SESSION', 'DOT_SYMBOL', 'IDENTIFIER' ) ), - 'charset introducer' => array( '_utf8mb4.x', array( 'UNDERSCORE_CHARSET', 'DOT_SYMBOL', 'IDENTIFIER' ) ), + 'keyword after a decimal' => array( '1.FROM', array( 'DECIMAL_NUM', 'FROM' ) ), + 'keyword before a backtick' => array( 'order.`id`', array( 'ORDER', 'DOT', 'IDENT_QUOTED' ) ), + 'keyword before an asterisk' => array( 'order.*', array( 'ORDER', 'DOT', 'ASTERISK' ) ), + 'keyword before whitespace' => array( 'order .id', array( 'ORDER', 'DOT', 'IDENT' ) ), + 'system variable scope' => array( '@@session.sql_mode', array( 'AT_SIGN', 'AT_SIGN', 'SESSION', 'DOT', 'IDENT' ) ), + 'charset introducer' => array( '_utf8mb4.x', array( 'UNDERSCORE_CHARSET', 'DOT', 'IDENT' ) ), // After "@@", the name is never a number, a literal, or a charset. - 'system variable digits' => array( '@@123', array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'IDENTIFIER' ) ), - 'system variable float-like' => array( '@@1e5', array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'IDENTIFIER' ) ), - 'system variable before text' => array( "@@b'01'", array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'IDENTIFIER', 'SINGLE_QUOTED_TEXT' ) ), - 'system variable charset' => array( '@@_utf8mb4', array( 'AT_SIGN_SYMBOL', 'AT_SIGN_SYMBOL', 'IDENTIFIER' ) ), + 'system variable digits' => array( '@@123', array( 'AT_SIGN', 'AT_SIGN', 'IDENT' ) ), + 'system variable float-like' => array( '@@1e5', array( 'AT_SIGN', 'AT_SIGN', 'IDENT' ) ), + 'system variable before text' => array( "@@b'01'", array( 'AT_SIGN', 'AT_SIGN', 'IDENT', 'TEXT_STRING' ) ), + 'system variable charset' => array( '@@_utf8mb4', array( 'AT_SIGN', 'AT_SIGN', 'IDENT' ) ), ); } @@ -519,29 +519,29 @@ public function test_integer_types( $input, $expected ): void { public function data_integer_types(): array { return array( - array( '0', WP_MySQL_Lexer::INT_NUMBER ), - array( '123', WP_MySQL_Lexer::INT_NUMBER ), - array( '2147483647', WP_MySQL_Lexer::INT_NUMBER ), - array( '00000000001', WP_MySQL_Lexer::INT_NUMBER ), - array( '00000000002147483647', WP_MySQL_Lexer::INT_NUMBER ), - - array( '2147483648', WP_MySQL_Lexer::LONG_NUMBER ), - array( '123456789123456789', WP_MySQL_Lexer::LONG_NUMBER ), - array( '9223372036854775807', WP_MySQL_Lexer::LONG_NUMBER ), - array( '00000000002147483648', WP_MySQL_Lexer::LONG_NUMBER ), - array( '00000000009223372036854775807', WP_MySQL_Lexer::LONG_NUMBER ), - - array( '9223372036854775808', WP_MySQL_Lexer::ULONGLONG_NUMBER ), - array( '12345678912345678912', WP_MySQL_Lexer::ULONGLONG_NUMBER ), - array( '18446744073709551615', WP_MySQL_Lexer::ULONGLONG_NUMBER ), - array( '00000000000000000009223372036854775808', WP_MySQL_Lexer::ULONGLONG_NUMBER ), - array( '000000000000000000018446744073709551615', WP_MySQL_Lexer::ULONGLONG_NUMBER ), - - array( '18446744073709551616', WP_MySQL_Lexer::DECIMAL_NUMBER ), - array( '23456789123456789123', WP_MySQL_Lexer::DECIMAL_NUMBER ), - array( '123456789123456789123456789', WP_MySQL_Lexer::DECIMAL_NUMBER ), - array( '0000000000000000000018446744073709551616', WP_MySQL_Lexer::DECIMAL_NUMBER ), - array( '00000000000000000000123456789123456789123456789', WP_MySQL_Lexer::DECIMAL_NUMBER ), + array( '0', WP_MySQL_Lexer::NUM ), + array( '123', WP_MySQL_Lexer::NUM ), + array( '2147483647', WP_MySQL_Lexer::NUM ), + array( '00000000001', WP_MySQL_Lexer::NUM ), + array( '00000000002147483647', WP_MySQL_Lexer::NUM ), + + array( '2147483648', WP_MySQL_Lexer::LONG_NUM ), + array( '123456789123456789', WP_MySQL_Lexer::LONG_NUM ), + array( '9223372036854775807', WP_MySQL_Lexer::LONG_NUM ), + array( '00000000002147483648', WP_MySQL_Lexer::LONG_NUM ), + array( '00000000009223372036854775807', WP_MySQL_Lexer::LONG_NUM ), + + array( '9223372036854775808', WP_MySQL_Lexer::ULONGLONG_NUM ), + array( '12345678912345678912', WP_MySQL_Lexer::ULONGLONG_NUM ), + array( '18446744073709551615', WP_MySQL_Lexer::ULONGLONG_NUM ), + array( '00000000000000000009223372036854775808', WP_MySQL_Lexer::ULONGLONG_NUM ), + array( '000000000000000000018446744073709551615', WP_MySQL_Lexer::ULONGLONG_NUM ), + + array( '18446744073709551616', WP_MySQL_Lexer::DECIMAL_NUM ), + array( '23456789123456789123', WP_MySQL_Lexer::DECIMAL_NUM ), + array( '123456789123456789123456789', WP_MySQL_Lexer::DECIMAL_NUM ), + array( '0000000000000000000018446744073709551616', WP_MySQL_Lexer::DECIMAL_NUM ), + array( '00000000000000000000123456789123456789123456789', WP_MySQL_Lexer::DECIMAL_NUM ), ); } @@ -571,64 +571,64 @@ function ( $token ) { } public function data_identifier_or_number(): array { - $end = array( WP_MySQL_Lexer::END_OF_INPUT, WP_MySQL_Lexer::END_MARKER ); + $end = array( WP_MySQL_Lexer::END_OF_INPUT, WP_MySQL_Lexer::YYEOF ); return array( // integer - array( '123', array_merge( array( WP_MySQL_Lexer::INT_NUMBER ), $end ) ), - array( '123abc', array_merge( array( WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // identifier + array( '123', array_merge( array( WP_MySQL_Lexer::NUM ), $end ) ), + array( '123abc', array_merge( array( WP_MySQL_Lexer::IDENT ), $end ) ), // identifier // binary - array( '0b01', array_merge( array( WP_MySQL_Lexer::BIN_NUMBER ), $end ) ), - array( '0b01xyz', array_merge( array( WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // identifier - array( '0b', array_merge( array( WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // identifier - array( "b'01'", array_merge( array( WP_MySQL_Lexer::BIN_NUMBER ), $end ) ), + array( '0b01', array_merge( array( WP_MySQL_Lexer::BIN_NUM ), $end ) ), + array( '0b01xyz', array_merge( array( WP_MySQL_Lexer::IDENT ), $end ) ), // identifier + array( '0b', array_merge( array( WP_MySQL_Lexer::IDENT ), $end ) ), // identifier + array( "b'01'", array_merge( array( WP_MySQL_Lexer::BIN_NUM ), $end ) ), array( "b'01xyz'", array() ), // invalid input - array( "b''", array_merge( array( WP_MySQL_Lexer::BIN_NUMBER ), $end ) ), + array( "b''", array_merge( array( WP_MySQL_Lexer::BIN_NUM ), $end ) ), array( "b'", array() ), // invalid input array( "b'01", array() ), // invalid input // hex - array( '0xab01', array_merge( array( WP_MySQL_Lexer::HEX_NUMBER ), $end ) ), - array( '0xab01xyz', array_merge( array( WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // identifier - array( '0x', array_merge( array( WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // identifier - array( "x'ab01'", array_merge( array( WP_MySQL_Lexer::HEX_NUMBER ), $end ) ), + array( '0xab01', array_merge( array( WP_MySQL_Lexer::HEX_NUM ), $end ) ), + array( '0xab01xyz', array_merge( array( WP_MySQL_Lexer::IDENT ), $end ) ), // identifier + array( '0x', array_merge( array( WP_MySQL_Lexer::IDENT ), $end ) ), // identifier + array( "x'ab01'", array_merge( array( WP_MySQL_Lexer::HEX_NUM ), $end ) ), array( "x'ab01xyz'", array() ), // invalid input - array( "x''", array_merge( array( WP_MySQL_Lexer::HEX_NUMBER ), $end ) ), + array( "x''", array_merge( array( WP_MySQL_Lexer::HEX_NUM ), $end ) ), array( "x'", array() ), // invalid input array( "x'ab", array() ), // invalid input // decimal - array( '123.456', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER ), $end ) ), - array( '.123', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER ), $end ) ), - array( '123.', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER ), $end ) ), - array( '123.456abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '.123abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '123.abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier + array( '123.456', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM ), $end ) ), + array( '.123', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM ), $end ) ), + array( '123.', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM ), $end ) ), + array( '123.456abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '.123abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '123.abc', array_merge( array( WP_MySQL_Lexer::DECIMAL_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier // float - array( '1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '.1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '.1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '.1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1.1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1.1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1.1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER ), $end ) ), - array( '1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier (this differs from INT/BIN/HEX numbers) - array( '1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '.1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '.1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '.1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '1.1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '1.1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier - array( '1.1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUMBER, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not identifier + array( '1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '.1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '.1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '.1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1.1e10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1.1e-10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1.1e+10', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM ), $end ) ), + array( '1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier (this differs from INT/BIN/HEX numbers) + array( '1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '.1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '.1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '.1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '1.1e10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '1.1e+10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier + array( '1.1e-10abc', array_merge( array( WP_MySQL_Lexer::FLOAT_NUM, WP_MySQL_Lexer::IDENT ), $end ) ), // not identifier // non-numbers - array( '.SELECT', array_merge( array( WP_MySQL_Lexer::DOT_SYMBOL, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not decimal or float - array( '1+e10', array_merge( array( WP_MySQL_Lexer::INT_NUMBER, WP_MySQL_Lexer::PLUS_OPERATOR, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not float - array( '1-e10', array_merge( array( WP_MySQL_Lexer::INT_NUMBER, WP_MySQL_Lexer::MINUS_OPERATOR, WP_MySQL_Lexer::IDENTIFIER ), $end ) ), // not float + array( '.SELECT', array_merge( array( WP_MySQL_Lexer::DOT, WP_MySQL_Lexer::IDENT ), $end ) ), // not decimal or float + array( '1+e10', array_merge( array( WP_MySQL_Lexer::NUM, WP_MySQL_Lexer::PLUS, WP_MySQL_Lexer::IDENT ), $end ) ), // not float + array( '1-e10', array_merge( array( WP_MySQL_Lexer::NUM, WP_MySQL_Lexer::MINUS, WP_MySQL_Lexer::IDENT ), $end ) ), // not float ); } @@ -690,12 +690,12 @@ public function test_valid_escaped_string( string $sql, int $expected_token_id ) public function data_valid_escaped_strings(): array { return array( - 'escaped single quote' => array( "'it\\'s'", WP_MySQL_Lexer::SINGLE_QUOTED_TEXT ), - 'trailing escaped backslash' => array( "'path\\\\'", WP_MySQL_Lexer::SINGLE_QUOTED_TEXT ), - 'doubled single quote' => array( "'it''s'", WP_MySQL_Lexer::SINGLE_QUOTED_TEXT ), - 'empty single-quoted string' => array( "''", WP_MySQL_Lexer::SINGLE_QUOTED_TEXT ), - 'escaped double quote' => array( '"col\\"name"', WP_MySQL_Lexer::DOUBLE_QUOTED_TEXT ), - 'backtick identifier' => array( '`my_column`', WP_MySQL_Lexer::BACK_TICK_QUOTED_ID ), + 'escaped single quote' => array( "'it\\'s'", WP_MySQL_Lexer::TEXT_STRING ), + 'trailing escaped backslash' => array( "'path\\\\'", WP_MySQL_Lexer::TEXT_STRING ), + 'doubled single quote' => array( "'it''s'", WP_MySQL_Lexer::TEXT_STRING ), + 'empty single-quoted string' => array( "''", WP_MySQL_Lexer::TEXT_STRING ), + 'escaped double quote' => array( '"col\\"name"', WP_MySQL_Lexer::TEXT_STRING ), + 'backtick identifier' => array( '`my_column`', WP_MySQL_Lexer::IDENT_QUOTED ), ); } @@ -755,12 +755,12 @@ public function test_ansi_quotes_changes_double_quote_tokenization( public function data_ansi_quotes_tokenization(): array { return array( - 'double quote is a string by default' => array( '"foo"', array(), 'SINGLE_QUOTED_TEXT' ), - 'double quote is an identifier in ANSI_QUOTES' => array( '"foo"', array( 'ANSI_QUOTES' ), 'BACK_TICK_QUOTED_ID' ), - 'ANSI_QUOTES is case-insensitive' => array( '"foo"', array( 'ansi_quotes' ), 'BACK_TICK_QUOTED_ID' ), - 'ANSI_QUOTES with other modes' => array( '"foo"', array( 'STRICT_ALL_TABLES', 'ANSI_QUOTES' ), 'BACK_TICK_QUOTED_ID' ), - 'single quote stays a string in ANSI_QUOTES' => array( "'foo'", array( 'ANSI_QUOTES' ), 'SINGLE_QUOTED_TEXT' ), - 'backtick stays an identifier in ANSI_QUOTES' => array( '`foo`', array( 'ANSI_QUOTES' ), 'BACK_TICK_QUOTED_ID' ), + 'double quote is a string by default' => array( '"foo"', array(), 'TEXT_STRING' ), + 'double quote is an identifier in ANSI_QUOTES' => array( '"foo"', array( 'ANSI_QUOTES' ), 'IDENT_QUOTED' ), + 'ANSI_QUOTES is case-insensitive' => array( '"foo"', array( 'ansi_quotes' ), 'IDENT_QUOTED' ), + 'ANSI_QUOTES with other modes' => array( '"foo"', array( 'STRICT_ALL_TABLES', 'ANSI_QUOTES' ), 'IDENT_QUOTED' ), + 'single quote stays a string in ANSI_QUOTES' => array( "'foo'", array( 'ANSI_QUOTES' ), 'TEXT_STRING' ), + 'backtick stays an identifier in ANSI_QUOTES' => array( '`foo`', array( 'ANSI_QUOTES' ), 'IDENT_QUOTED' ), ); } @@ -768,12 +768,12 @@ public function test_composite_ansi_mode_expands_to_lexer_component_modes(): voi // ANSI_QUOTES: a double-quoted sequence is a quoted identifier. $lexer = new WP_MySQL_Lexer( '"foo"', 80400, array( 'ANSI' ) ); $this->assertTrue( $lexer->next_token() ); - $this->assertSame( WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, $lexer->get_token()->id ); + $this->assertSame( WP_MySQL_Lexer::IDENT_QUOTED, $lexer->get_token()->id ); // PIPES_AS_CONCAT: "||" is the string concatenation operator. $lexer = new WP_MySQL_Lexer( '||', 80400, array( 'ANSI' ) ); $this->assertTrue( $lexer->next_token() ); - $this->assertSame( WP_MySQL_Lexer::CONCAT_PIPES_SYMBOL, $lexer->get_token()->id ); + $this->assertSame( WP_MySQL_Lexer::OR_OR_SYM, $lexer->get_token()->id ); // IGNORE_SPACE: whitespace is permitted between a function name and "(". $lexer = new WP_MySQL_Lexer( 'COUNT (1)', 80400, array( 'ANSI' ) ); @@ -807,16 +807,16 @@ public function data_quoted_identifier_backslash(): array { $sq = "'"; return array( // Backtick identifiers: backslash is a literal character. - 'backtick keeps backslash-n literal' => array( $bt . 'a' . $bs . 'nb' . $bt, array(), WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, 'a' . $bs . 'nb' ), - 'backtick allows trailing backslash' => array( $bt . 'a' . $bs . $bt, array(), WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, 'a' . $bs ), - 'backtick escapes only by doubling' => array( $bt . 'a' . $bt . $bt . 'b' . $bt, array(), WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, 'a' . $bt . 'b' ), + 'backtick keeps backslash-n literal' => array( $bt . 'a' . $bs . 'nb' . $bt, array(), WP_MySQL_Lexer::IDENT_QUOTED, 'a' . $bs . 'nb' ), + 'backtick allows trailing backslash' => array( $bt . 'a' . $bs . $bt, array(), WP_MySQL_Lexer::IDENT_QUOTED, 'a' . $bs ), + 'backtick escapes only by doubling' => array( $bt . 'a' . $bt . $bt . 'b' . $bt, array(), WP_MySQL_Lexer::IDENT_QUOTED, 'a' . $bt . 'b' ), // ANSI-quoted identifiers behave like backtick identifiers. - 'ansi-quoted keeps backslash literal' => array( $dq . 'a' . $bs . 'nb' . $dq, array( 'ANSI_QUOTES' ), WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, 'a' . $bs . 'nb' ), - 'ansi-quoted allows trailing backslash' => array( $dq . 'a' . $bs . $dq, array( 'ANSI_QUOTES' ), WP_MySQL_Lexer::BACK_TICK_QUOTED_ID, 'a' . $bs ), + 'ansi-quoted keeps backslash literal' => array( $dq . 'a' . $bs . 'nb' . $dq, array( 'ANSI_QUOTES' ), WP_MySQL_Lexer::IDENT_QUOTED, 'a' . $bs . 'nb' ), + 'ansi-quoted allows trailing backslash' => array( $dq . 'a' . $bs . $dq, array( 'ANSI_QUOTES' ), WP_MySQL_Lexer::IDENT_QUOTED, 'a' . $bs ), // String literals still process backslash escapes. - 'single-quoted string escapes newline' => array( $sq . 'a' . $bs . 'nb' . $sq, array(), WP_MySQL_Lexer::SINGLE_QUOTED_TEXT, "a\nb" ), + 'single-quoted string escapes newline' => array( $sq . 'a' . $bs . 'nb' . $sq, array(), WP_MySQL_Lexer::TEXT_STRING, "a\nb" ), ); } diff --git a/packages/mysql-parser/tests/WP_MySQL_Token_Data_Tests.php b/packages/mysql-parser/tests/WP_MySQL_Token_Data_Tests.php index 4a1e12002..25763c4ce 100644 --- a/packages/mysql-parser/tests/WP_MySQL_Token_Data_Tests.php +++ b/packages/mysql-parser/tests/WP_MySQL_Token_Data_Tests.php @@ -29,6 +29,18 @@ public function test_every_emittable_token_has_a_derived_name(): void { } } + public function test_token_constants_have_distinct_numbers(): void { + // Each token constant names one grammar terminal; none is an alias. + $numbers = array(); + foreach ( ( new ReflectionClass( WP_MySQL_Lexer::class ) )->getConstants() as $name => $value ) { + if ( is_int( $value ) && $value >= 0 && 0 !== strpos( $name, 'SQL_MODE_' ) ) { + $this->assertArrayNotHasKey( $value, $numbers, "Constant $name shares token $value with another constant" ); + $numbers[ $value ] = $name; + } + } + $this->assertNotEmpty( $numbers ); + } + public function test_function_keywords_are_a_subset_of_keywords(): void { $this->assertNotEmpty( WP_MySQL_Lexer::FUNCTIONS ); foreach ( array_keys( WP_MySQL_Lexer::FUNCTIONS ) as $keyword ) { @@ -64,12 +76,12 @@ public function test_derived_names_prefer_plain_keywords_and_constants(): void { $this->assertSame( 'CURRENT_DATE', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::KEYWORDS['CURDATE'] ) ); // Constant name for non-keyword tokens. - $this->assertSame( 'IDENTIFIER', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::IDENTIFIER ) ); - $this->assertSame( 'SINGLE_QUOTED_TEXT', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::SINGLE_QUOTED_TEXT ) ); - $this->assertSame( 'AT_SIGN_SYMBOL', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::AT_SIGN_SYMBOL ) ); - $this->assertSame( 'WITH_ROLLUP_SYMBOL', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::WITH_ROLLUP_SYMBOL ) ); + $this->assertSame( 'IDENT', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::IDENT ) ); + $this->assertSame( 'TEXT_STRING', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::TEXT_STRING ) ); + $this->assertSame( 'AT_SIGN', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::AT_SIGN ) ); + $this->assertSame( 'WITH_ROLLUP_SYM', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::WITH_ROLLUP_SYM ) ); $this->assertSame( 'END_OF_INPUT', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::END_OF_INPUT ) ); - $this->assertSame( 'END_MARKER', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::END_MARKER ) ); + $this->assertSame( 'YYEOF', WP_MySQL_Lexer::get_token_name( WP_MySQL_Lexer::YYEOF ) ); } public function test_internal_scanner_sentinels_cannot_collide_with_grammar_tokens(): void { diff --git a/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php b/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php index 676f1cc00..f49d3235d 100644 --- a/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php +++ b/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php @@ -24,13 +24,13 @@ private static function first_token( string $sql, string $name, array $sql_modes } public function test_get_value_unquotes_string_literals(): void { - $this->assertSame( "a'b", self::first_token( "SELECT 'a''b'", 'SINGLE_QUOTED_TEXT' )->get_value() ); - $this->assertSame( 'a"b', self::first_token( 'SELECT "a""b"', 'SINGLE_QUOTED_TEXT' )->get_value() ); - $this->assertSame( "new\nline", self::first_token( "SELECT 'new\\nline'", 'SINGLE_QUOTED_TEXT' )->get_value() ); + $this->assertSame( "a'b", self::first_token( "SELECT 'a''b'", 'TEXT_STRING' )->get_value() ); + $this->assertSame( 'a"b', self::first_token( 'SELECT "a""b"', 'TEXT_STRING' )->get_value() ); + $this->assertSame( "new\nline", self::first_token( "SELECT 'new\\nline'", 'TEXT_STRING' )->get_value() ); } public function test_get_value_honors_no_backslash_escapes_mode(): void { - $token = self::first_token( "SELECT 'a\\nb'", 'SINGLE_QUOTED_TEXT', array( 'NO_BACKSLASH_ESCAPES' ) ); + $token = self::first_token( "SELECT 'a\\nb'", 'TEXT_STRING', array( 'NO_BACKSLASH_ESCAPES' ) ); $this->assertSame( 'a\\nb', $token->get_value() ); } @@ -40,41 +40,41 @@ public function test_get_value_preserves_non_utf8_bytes_in_string_literals(): vo $raw = chr( 0xFF ) . chr( 0xFE ); $this->assertSame( $raw, - self::first_token( "SELECT '$raw'", 'SINGLE_QUOTED_TEXT' )->get_value() + self::first_token( "SELECT '$raw'", 'TEXT_STRING' )->get_value() ); // A backslash escape strips the backslash and preserves the following non-UTF-8 byte. $this->assertSame( chr( 0xE9 ), - self::first_token( "SELECT '\\" . chr( 0xE9 ) . "'", 'SINGLE_QUOTED_TEXT' )->get_value() + self::first_token( "SELECT '\\" . chr( 0xE9 ) . "'", 'TEXT_STRING' )->get_value() ); // Valid multibyte UTF-8 still round-trips. $this->assertSame( 'café 🙂', - self::first_token( "SELECT 'café 🙂'", 'SINGLE_QUOTED_TEXT' )->get_value() + self::first_token( "SELECT 'café 🙂'", 'TEXT_STRING' )->get_value() ); } public function test_get_value_unescapes_newlines_and_multibyte_characters(): void { $this->assertSame( "\n", - self::first_token( "SELECT '\\\n'", 'SINGLE_QUOTED_TEXT' )->get_value() + self::first_token( "SELECT '\\\n'", 'TEXT_STRING' )->get_value() ); $this->assertSame( '🙂', - self::first_token( "SELECT '\\🙂'", 'SINGLE_QUOTED_TEXT' )->get_value() + self::first_token( "SELECT '\\🙂'", 'TEXT_STRING' )->get_value() ); } public function test_get_value_unquotes_backtick_identifiers(): void { - $this->assertSame( 'col name', self::first_token( 'SELECT `col name` FROM t', 'BACK_TICK_QUOTED_ID' )->get_value() ); + $this->assertSame( 'col name', self::first_token( 'SELECT `col name` FROM t', 'IDENT_QUOTED' )->get_value() ); } public function test_get_value_unquotes_ansi_identifiers(): void { $this->assertSame( 'a"b', - self::first_token( 'SELECT "a""b"', 'BACK_TICK_QUOTED_ID', array( 'ANSI_QUOTES' ) )->get_value() + self::first_token( 'SELECT "a""b"', 'IDENT_QUOTED', array( 'ANSI_QUOTES' ) )->get_value() ); } @@ -84,7 +84,7 @@ public function test_get_value_does_not_unquote_unquoted_tokens(): void { // by token ids and must return keyword bytes as-is. $this->assertSame( 'SSL', self::first_token( 'CREATE USER u REQUIRE SSL', 'SSL' )->get_value() ); $this->assertSame( 'SELECT', self::first_token( 'SELECT 1', 'SELECT' )->get_value() ); - $this->assertSame( '42', self::first_token( 'SELECT 42', 'INT_NUMBER' )->get_value() ); + $this->assertSame( '42', self::first_token( 'SELECT 42', 'NUM' )->get_value() ); } public function test_get_name_resolves_bison_terminal_names(): void { @@ -94,7 +94,7 @@ public function test_get_name_resolves_bison_terminal_names(): void { $names[] = $token->get_name(); } $this->assertSame( - array( 'SELECT', 'SINGLE_QUOTED_TEXT', 'COMMA_SYMBOL', 'OPEN_PAR_SYMBOL', 'INT_NUMBER', 'CLOSE_PAR_SYMBOL', 'END_OF_INPUT', 'END_MARKER' ), + array( 'SELECT', 'TEXT_STRING', 'COMMA', 'LEFT_PAREN', 'NUM', 'RIGHT_PAREN', 'END_OF_INPUT', 'YYEOF' ), $names ); } diff --git a/packages/mysql-parser/tools/generate-tokens.php b/packages/mysql-parser/tools/generate-tokens.php index 3c807cd66..ba0ea206d 100644 --- a/packages/mysql-parser/tools/generate-tokens.php +++ b/packages/mysql-parser/tools/generate-tokens.php @@ -10,7 +10,7 @@ * * - Named constants for the structural tokens the scanner code references * (operators, punctuation, literals, and the few keywords with scanner - * logic of their own). + * logic of their own), named after their terminals in sql_yacc.yy. * - KEYWORDS — the keyword string => token number table, taken directly from * MySQL's lex.h. Words declared there with a terminal that is not part of * the grammar (hint-only keywords) are omitted and thus lex as identifiers, @@ -53,83 +53,91 @@ // would only surface at runtime, far from the cause — fail the build instead. $bison_number = function ( $name ) use ( $terminals ) { if ( ! isset( $terminals[ $name ] ) ) { - fwrite( STDERR, "error: Bison terminal '$name' not found in the automaton; the grammar bridge needs updating for this MySQL version.\n" ); + fwrite( STDERR, "error: Bison terminal '$name' not found in the automaton; the lexer token constants need updating for this MySQL version.\n" ); exit( 1 ); } return $terminals[ $name ]; }; /* - * Named constants: lexer constant name => Bison terminal name. Single-char - * tokens use Bison's quoted-character names (e.g. "'('"); the rest are the - * explicit %token names from sql_yacc.yy. These are the tokens the scanner - * code refers to by name: operators and literals it recognises structurally, - * and the few keywords with scanner logic of their own (the WITH ROLLUP - * contraction and the HIGH_NOT_PRECEDENCE mode), plus the end markers. + * Token constants for the terminals the scanner code refers to by name: + * operators and literals it recognises structurally, the few keywords with + * scanner logic of their own ("\N", the WITH ROLLUP contraction, and the + * HIGH_NOT_PRECEDENCE mode), and the end markers. + * + * Each constant is named after its %token in sql_yacc.yy. Single-character + * terminals, which Bison names by the quoted character (e.g. "'('"), are + * named after the character, and Bison's end-of-file terminal ("$end") is + * named YYEOF, as in the C code that Bison generates. */ -$constants = array( - 'OPEN_PAR_SYMBOL' => "'('", - 'CLOSE_PAR_SYMBOL' => "')'", - 'OPEN_CURLY_SYMBOL' => "'{'", - 'CLOSE_CURLY_SYMBOL' => "'}'", - 'AT_SIGN_SYMBOL' => "'@'", - 'COLON_SYMBOL' => "':'", - 'COMMA_SYMBOL' => "','", - 'DOT_SYMBOL' => "'.'", - 'SEMICOLON_SYMBOL' => "';'", - 'BITWISE_AND_OPERATOR' => "'&'", - 'BITWISE_NOT_OPERATOR' => "'~'", - 'BITWISE_OR_OPERATOR' => "'|'", - 'BITWISE_XOR_OPERATOR' => "'^'", - 'DIV_OPERATOR' => "'/'", - 'MINUS_OPERATOR' => "'-'", - 'MOD_OPERATOR' => "'%'", - 'MULT_OPERATOR' => "'*'", - 'PLUS_OPERATOR' => "'+'", - 'LOGICAL_NOT_OPERATOR' => "'!'", - 'EQUAL_OPERATOR' => 'EQ', - 'NULL_SAFE_EQUAL_OPERATOR' => 'EQUAL_SYM', - 'GREATER_OR_EQUAL_OPERATOR' => 'GE', - 'GREATER_THAN_OPERATOR' => 'GT_SYM', - 'LESS_OR_EQUAL_OPERATOR' => 'LE', - 'LESS_THAN_OPERATOR' => 'LT', - 'NOT_EQUAL_OPERATOR' => 'NE', - 'ASSIGN_OPERATOR' => 'SET_VAR', - 'LOGICAL_AND_OPERATOR' => 'AND_AND_SYM', - 'LOGICAL_OR_OPERATOR' => 'OR2_SYM', - 'CONCAT_PIPES_SYMBOL' => 'OR_OR_SYM', - 'SHIFT_LEFT_OPERATOR' => 'SHIFT_LEFT', - 'SHIFT_RIGHT_OPERATOR' => 'SHIFT_RIGHT', - 'JSON_SEPARATOR_SYMBOL' => 'JSON_SEPARATOR_SYM', - 'JSON_UNQUOTED_SEPARATOR_SYMBOL' => 'JSON_UNQUOTED_SEPARATOR_SYM', - 'PARAM_MARKER' => 'PARAM_MARKER', - 'IDENTIFIER' => 'IDENT', - 'BACK_TICK_QUOTED_ID' => 'IDENT_QUOTED', - 'SINGLE_QUOTED_TEXT' => 'TEXT_STRING', - 'DOUBLE_QUOTED_TEXT' => 'TEXT_STRING', - 'NCHAR_TEXT' => 'NCHAR_STRING', - 'UNDERSCORE_CHARSET' => 'UNDERSCORE_CHARSET', - 'BIN_NUMBER' => 'BIN_NUM', - 'HEX_NUMBER' => 'HEX_NUM', - 'INT_NUMBER' => 'NUM', - 'LONG_NUMBER' => 'LONG_NUM', - 'ULONGLONG_NUMBER' => 'ULONGLONG_NUM', - 'DECIMAL_NUMBER' => 'DECIMAL_NUM', - 'FLOAT_NUMBER' => 'FLOAT_NUM', - 'NULL2_SYMBOL' => 'NULL_SYM', - 'NOT_SYMBOL' => 'NOT_SYM', - 'NOT2_SYMBOL' => 'NOT2_SYM', - 'WITH_SYMBOL' => 'WITH', - 'ROLLUP_SYMBOL' => 'ROLLUP_SYM', - 'WITH_ROLLUP_SYMBOL' => 'WITH_ROLLUP_SYM', - 'END_OF_INPUT' => 'END_OF_INPUT', - 'END_MARKER' => '$end', +$character_names = array( + '(' => 'LEFT_PAREN', + ')' => 'RIGHT_PAREN', + '{' => 'LEFT_BRACE', + '}' => 'RIGHT_BRACE', + ',' => 'COMMA', + '.' => 'DOT', + ':' => 'COLON', + ';' => 'SEMICOLON', + '@' => 'AT_SIGN', + '+' => 'PLUS', + '-' => 'MINUS', + '*' => 'ASTERISK', + '/' => 'SLASH', + '%' => 'PERCENT', + '&' => 'AMPERSAND', + '|' => 'PIPE', + '^' => 'CARET', + '~' => 'TILDE', + '!' => 'EXCLAMATION', +); + +$terminal_names = array( + 'EQ', + 'EQUAL_SYM', + 'NE', + 'GE', + 'GT_SYM', + 'LE', + 'LT', + 'SET_VAR', + 'AND_AND_SYM', + 'OR2_SYM', + 'OR_OR_SYM', + 'SHIFT_LEFT', + 'SHIFT_RIGHT', + 'JSON_SEPARATOR_SYM', + 'JSON_UNQUOTED_SEPARATOR_SYM', + 'PARAM_MARKER', + 'IDENT', + 'IDENT_QUOTED', + 'TEXT_STRING', + 'NCHAR_STRING', + 'UNDERSCORE_CHARSET', + 'BIN_NUM', + 'HEX_NUM', + 'NUM', + 'LONG_NUM', + 'ULONGLONG_NUM', + 'DECIMAL_NUM', + 'FLOAT_NUM', + 'NULL_SYM', + 'NOT_SYM', + 'NOT2_SYM', + 'WITH', + 'ROLLUP_SYM', + 'WITH_ROLLUP_SYM', + 'END_OF_INPUT', ); $constant_values = array(); -foreach ( $constants as $constant_name => $terminal_name ) { - $constant_values[ $constant_name ] = $bison_number( $terminal_name ); +foreach ( $character_names as $character => $constant_name ) { + $constant_values[ $constant_name ] = $bison_number( "'$character'" ); +} +foreach ( $terminal_names as $terminal_name ) { + $constant_values[ $terminal_name ] = $bison_number( $terminal_name ); } +$constant_values['YYEOF'] = $bison_number( '$end' ); /* * Keyword table: lex.h declares each keyword string with its Bison terminal From f60fea19955549754f7c680fd4dd86a51a4033e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jake=C5=A1?= Date: Mon, 2 Nov 2026 13:31:07 +0100 Subject: [PATCH 2/7] Record the grammar's MySQL tag for the generators The other build scripts moved to mysql-8.4.10, but the parse table generator still defaulted to mysql-8.4.3. Running bin/build-grammar wrote the old tag into the parse table's source comment, although the table was built from 8.4.10. Both generators kept their own copy of the default tag. The fetch script now records the fetched tag in build/mysql-tag, and the generators read it from there, so the generated files always name the grammar they were built from. --- packages/mysql-parser/tools/fetch-mysql-grammar.sh | 3 +++ packages/mysql-parser/tools/generate-parse-table.php | 10 +++++++--- packages/mysql-parser/tools/generate-tokens.php | 10 +++++++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/packages/mysql-parser/tools/fetch-mysql-grammar.sh b/packages/mysql-parser/tools/fetch-mysql-grammar.sh index f6767be63..9e1141e86 100755 --- a/packages/mysql-parser/tools/fetch-mysql-grammar.sh +++ b/packages/mysql-parser/tools/fetch-mysql-grammar.sh @@ -42,5 +42,8 @@ else sha256 "$build_dir/sql_yacc.yy" "$build_dir/lex.h" | sed 's/^/ /' fi +# Record the tag for the generators, so the generated files name the right source. +echo "$MYSQL_TAG" > "$build_dir/mysql-tag" + echo " -> $build_dir/sql_yacc.yy ($(wc -l < "$build_dir/sql_yacc.yy" | tr -d ' ') lines)" echo " -> $build_dir/lex.h ($(wc -l < "$build_dir/lex.h" | tr -d ' ') lines)" diff --git a/packages/mysql-parser/tools/generate-parse-table.php b/packages/mysql-parser/tools/generate-parse-table.php index 3e39f8c11..c124e279c 100644 --- a/packages/mysql-parser/tools/generate-parse-table.php +++ b/packages/mysql-parser/tools/generate-parse-table.php @@ -45,9 +45,13 @@ } $xml_path = $argv[1]; $output_path = $argv[2]; -$mysql_tag = getenv( 'MYSQL_TAG' ); -if ( false === $mysql_tag || '' === $mysql_tag ) { - $mysql_tag = 'mysql-8.4.3'; + +// The tag of the grammar sources, recorded next to the automaton by fetch-mysql-grammar.sh. +$tag_path = dirname( $xml_path ) . '/mysql-tag'; +$mysql_tag = is_file( $tag_path ) ? trim( file_get_contents( $tag_path ) ) : ''; +if ( '' === $mysql_tag ) { + fwrite( STDERR, "error: MySQL tag not found in $tag_path; run fetch-mysql-grammar.sh first.\n" ); + exit( 1 ); } ini_set( 'memory_limit', '3G' ); diff --git a/packages/mysql-parser/tools/generate-tokens.php b/packages/mysql-parser/tools/generate-tokens.php index ba0ea206d..b072c7819 100644 --- a/packages/mysql-parser/tools/generate-tokens.php +++ b/packages/mysql-parser/tools/generate-tokens.php @@ -32,9 +32,13 @@ $xml_path = $argv[1]; $lexh_path = $argv[2]; $lexer_path = $argv[3]; -$mysql_tag = getenv( 'MYSQL_TAG' ); -if ( false === $mysql_tag || '' === $mysql_tag ) { - $mysql_tag = 'mysql-8.4.10'; + +// The tag of the grammar sources, recorded next to the automaton by fetch-mysql-grammar.sh. +$tag_path = dirname( $xml_path ) . '/mysql-tag'; +$mysql_tag = is_file( $tag_path ) ? trim( file_get_contents( $tag_path ) ) : ''; +if ( '' === $mysql_tag ) { + fwrite( STDERR, "error: MySQL tag not found in $tag_path; run fetch-mysql-grammar.sh first.\n" ); + exit( 1 ); } // Bison terminal name => token-number, read straight from the automaton. From 339da20cc3b2f5649764de52d480293c5a8d65c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jake=C5=A1?= Date: Mon, 2 Nov 2026 13:31:52 +0100 Subject: [PATCH 3/7] Drop an unused argument from a read_quoted_text() call read_quoted_text() reads the quote character from the input and takes no arguments, so the "'" passed for N'...' literals was silently ignored. Add tests for national string literals, which had no direct lexer coverage. --- .../mysql-parser/src/class-wp-mysql-lexer.php | 2 +- .../mysql-parser/tests/WP_MySQL_Lexer_Tests.php | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/mysql-parser/src/class-wp-mysql-lexer.php b/packages/mysql-parser/src/class-wp-mysql-lexer.php index 83196faa0..67ebd074d 100644 --- a/packages/mysql-parser/src/class-wp-mysql-lexer.php +++ b/packages/mysql-parser/src/class-wp-mysql-lexer.php @@ -1011,7 +1011,7 @@ private function read_next_token(): ?int { $type = $this->read_number(); } elseif ( ( 'n' === $byte || 'N' === $byte ) && "'" === $next_byte ) { $this->bytes_already_read += 1; - $type = $this->read_quoted_text( "'" ); + $type = $this->read_quoted_text(); if ( self::TEXT_STRING === $type ) { $type = self::NCHAR_STRING; } diff --git a/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php b/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php index 97e47fcd9..77bc00f7b 100644 --- a/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php +++ b/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php @@ -764,6 +764,22 @@ public function data_ansi_quotes_tokenization(): array { ); } + /** + * @dataProvider data_national_strings + */ + public function test_national_strings( string $sql, array $expected ): void { + $this->assertSame( array_merge( $expected, array( 'END_OF_INPUT', 'YYEOF' ) ), self::token_names( $sql ) ); + } + + public function data_national_strings(): array { + return array( + 'uppercase prefix' => array( "N'abc'", array( 'NCHAR_STRING' ) ), + 'lowercase prefix' => array( "n'abc'", array( 'NCHAR_STRING' ) ), + 'escaped quote' => array( "N'a''b'", array( 'NCHAR_STRING' ) ), + 'double quote is not a literal' => array( 'N"abc"', array( 'IDENT', 'TEXT_STRING' ) ), + ); + } + public function test_composite_ansi_mode_expands_to_lexer_component_modes(): void { // ANSI_QUOTES: a double-quoted sequence is a quoted identifier. $lexer = new WP_MySQL_Lexer( '"foo"', 80400, array( 'ANSI' ) ); From 4319ff46c075fbf723105ed125dadefc6e7cf8aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jake=C5=A1?= Date: Mon, 2 Nov 2026 13:55:20 +0100 Subject: [PATCH 4/7] Fix the token data file in the parser README The grammar build diagram still showed generate-tokens.php writing a separate class-wp-mysql-tokens.php. The token data is now generated into the lexer class. --- packages/mysql-parser/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mysql-parser/README.md b/packages/mysql-parser/README.md index b4f61f9b1..1e148d6a6 100644 --- a/packages/mysql-parser/README.md +++ b/packages/mysql-parser/README.md @@ -14,7 +14,7 @@ It outputs a compact parse table and a list of MySQL tokens for the lexer: mysql-server sources sql/sql_yacc.yy ──▶ Bison 3.8.2 ──▶ automaton.xml ─┬─▶ generate-parse-table.php ──▶ mysql-parse-table.php | - sql/lex.h ─────────────────────────────────────────┴─▶ generate-tokens.php ───────▶ class-wp-mysql-tokens.php + sql/lex.h ─────────────────────────────────────────┴─▶ generate-tokens.php ───────▶ class-wp-mysql-lexer.php ``` The runtime ships only the compiled artifacts and a thin parser implementation. From 71cffb824c9960c23836b4a0dc6ebfc2701460a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jake=C5=A1?= Date: Mon, 2 Nov 2026 13:55:34 +0100 Subject: [PATCH 5/7] Remove the unused WP_MySQL_Lexer::get_token_id() Nothing calls it, and it resolves only the named token constants, not keywords, so it is not the inverse of get_token_name(). Keyword ids are available via WP_MySQL_Lexer::KEYWORDS. --- .../mysql-parser/src/class-wp-mysql-lexer.php | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/packages/mysql-parser/src/class-wp-mysql-lexer.php b/packages/mysql-parser/src/class-wp-mysql-lexer.php index 67ebd074d..37c7d45c2 100644 --- a/packages/mysql-parser/src/class-wp-mysql-lexer.php +++ b/packages/mysql-parser/src/class-wp-mysql-lexer.php @@ -549,23 +549,6 @@ public function is_sql_mode_active( int $mode ): bool { return ( $this->sql_modes & $mode ) !== 0; } - /** - * Get the numeric token ID for a given token name. - * - * @param string $token_name The name of the token. - * @return int|null The token ID for the given token name; null when not found. - */ - public static function get_token_id( string $token_name ): ?int { - // Token constants share the class with the lexer's own constants, so - // resolve only the grammar tokens: non-negative ints, never an SQL mode. - $constant_name = self::class . '::' . $token_name; - if ( 0 === strpos( $token_name, 'SQL_MODE_' ) || ! defined( $constant_name ) ) { - return null; - } - $value = constant( $constant_name ); - return is_int( $value ) && $value >= 0 ? $value : null; - } - /** * Get the name of a token for a given token ID. * From 1da94453bdbc58c5f023a1515a8c7bb9f76a66cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jake=C5=A1?= Date: Mon, 2 Nov 2026 13:56:17 +0100 Subject: [PATCH 6/7] Document token and AST node names in the parser README Explain how to refer to grammar rules, keywords, other terminals, and single-character tokens, now that all of them follow the MySQL grammar. --- packages/mysql-parser/README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/mysql-parser/README.md b/packages/mysql-parser/README.md index 1e148d6a6..254e7f4fa 100644 --- a/packages/mysql-parser/README.md +++ b/packages/mysql-parser/README.md @@ -37,6 +37,30 @@ $tokens = ( new WP_MySQL_Lexer( 'SELECT 1 + 2' ) )->remaining_tokens(); $ast = $parser->parse( $tokens ); ``` +## Tokens and AST nodes +AST nodes and tokens use the names from the official MySQL grammar (`sql/sql_yacc.yy`): + +- **Nodes** are named after grammar rules, such as `select_item` or `simple_expr`. +- **Keywords** are looked up by their SQL spelling in `WP_MySQL_Lexer::KEYWORDS`, + which is generated from the MySQL keyword list (`sql/lex.h`). Like MySQL, the lexer + emits a few of them as other terminals: `||` as `OR2_SYM` (without `PIPES_AS_CONCAT`), + `NOT` as `NOT2_SYM` (with `HIGH_NOT_PRECEDENCE`), and `WITH ROLLUP` as `WITH_ROLLUP_SYM`. +- **Other tokens** are `WP_MySQL_Lexer` constants named after their grammar terminals, + such as `IDENT`, `TEXT_STRING`, `NUM`, or `EQ` (`=`). +- **Single-character terminals**, which the grammar writes as a quoted character + like `'('`, are named after the character, such as `LEFT_PAREN`, `COMMA`, or `ASTERISK`. + +```php +$ast->get_descendant_nodes( 'select_item' ); // A grammar rule. +$ast->get_first_descendant_token( WP_MySQL_Lexer::KEYWORDS['DISTINCT'] ); // A keyword. +$ast->get_first_descendant_token( WP_MySQL_Lexer::IDENT ); // A grammar terminal. +$ast->get_first_descendant_token( WP_MySQL_Lexer::ASTERISK ); // A single character. +``` + +Token IDs are the grammar's own token numbers. Like in MySQL, the token stream of +valid input ends with `END_OF_INPUT` and `YYEOF`. For debugging, `$token->get_name()` +returns a readable token name. + ## Development This package includes the full grammar compilation pipeline. The generated artifacts are committed to the repository, and they only need to be regenerated on grammar From 7a2cb7dfc398ba7b97dfe221058d819f02b436ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jake=C5=A1?= Date: Mon, 2 Nov 2026 14:24:06 +0100 Subject: [PATCH 7/7] Default the lexer to MySQL 8.4.10 The lexer defaulted to MySQL 8.0.38, while its grammar, keywords, and query corpus come from MySQL 8.4.10. Use 8.4.10 as the default, and in the corpus test and the lexer test helpers. Lexing the corpus as 8.4.10 fixes three queries from MySQL's own tests for six-digit version comments, which the lexer read the MySQL 8.0 way. --- .../mysql-parser/src/class-wp-mysql-lexer.php | 2 +- .../tests/WP_MySQL_Lexer_Tests.php | 36 +++++++++++++------ .../WP_MySQL_Server_Suite_Parser_Tests.php | 4 +-- .../tests/WP_MySQL_Token_Tests.php | 2 +- .../tests/data/corpus-failures.csv | 4 --- 5 files changed, 30 insertions(+), 18 deletions(-) diff --git a/packages/mysql-parser/src/class-wp-mysql-lexer.php b/packages/mysql-parser/src/class-wp-mysql-lexer.php index 37c7d45c2..9f6c1660d 100644 --- a/packages/mysql-parser/src/class-wp-mysql-lexer.php +++ b/packages/mysql-parser/src/class-wp-mysql-lexer.php @@ -363,7 +363,7 @@ class WP_MySQL_Lexer { */ public function __construct( string $sql, - int $mysql_version = 80038, + int $mysql_version = 80410, array $sql_modes = array() ) { $this->sql = $sql; diff --git a/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php b/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php index 77bc00f7b..4a96ce3b8 100644 --- a/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php +++ b/packages/mysql-parser/tests/WP_MySQL_Lexer_Tests.php @@ -70,7 +70,7 @@ public function test_tokenize_invalid_input(): void { * @return string[] Terminal names, in input order. */ private static function token_names( string $sql, array $sql_modes = array() ): array { - $tokens = ( new WP_MySQL_Lexer( $sql, 80400, $sql_modes ) )->remaining_tokens(); + $tokens = ( new WP_MySQL_Lexer( $sql, 80410, $sql_modes ) )->remaining_tokens(); $names = array(); foreach ( $tokens as $token ) { $names[] = $token->get_name(); @@ -88,28 +88,28 @@ public function test_emits_bison_terminals_with_end_markers(): void { public function test_ignore_space_preserves_function_name_ranges(): void { // Without "(", COUNT is an identifier whose range excludes trailing whitespace. $sql = 'SELECT COUNT FROM t'; - $tokens = ( new WP_MySQL_Lexer( $sql, 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); + $tokens = ( new WP_MySQL_Lexer( $sql, 80410, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); $this->assertSame( 'IDENT', $tokens[1]->get_name(), $sql ); $this->assertSame( 'COUNT', $tokens[1]->get_value(), $sql ); $this->assertSame( 5, $tokens[1]->length, $sql ); $this->assertSame( 13, $tokens[2]->start, $sql ); $sql = "SELECT COUNT\t\n FROM t"; - $tokens = ( new WP_MySQL_Lexer( $sql, 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); + $tokens = ( new WP_MySQL_Lexer( $sql, 80410, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); $this->assertSame( 'IDENT', $tokens[1]->get_name(), $sql ); $this->assertSame( 'COUNT', $tokens[1]->get_value(), $sql ); $this->assertSame( 5, $tokens[1]->length, $sql ); $this->assertSame( 15, $tokens[2]->start, $sql ); // With "(", COUNT is a function whose range still excludes the whitespace. - $tokens = ( new WP_MySQL_Lexer( 'SELECT COUNT (1)', 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); + $tokens = ( new WP_MySQL_Lexer( 'SELECT COUNT (1)', 80410, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); $this->assertSame( 5, $tokens[1]->length ); $this->assertSame( 'COUNT', $tokens[1]->get_name() ); $this->assertSame( 'LEFT_PAREN', $tokens[2]->get_name() ); $this->assertSame( 13, $tokens[2]->start ); $sql = "SELECT COUNT \t\n"; - $tokens = ( new WP_MySQL_Lexer( $sql, 80400, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); + $tokens = ( new WP_MySQL_Lexer( $sql, 80410, array( 'IGNORE_SPACE' ) ) )->remaining_tokens(); $this->assertSame( 'IDENT', $tokens[1]->get_name(), $sql ); $this->assertSame( 'COUNT', $tokens[1]->get_value(), $sql ); $this->assertSame( 5, $tokens[1]->length, $sql ); @@ -190,6 +190,22 @@ public function data_version_comments(): array { return $cases; } + public function test_mysql_version_defaults_to_mysql_8_4_10(): void { + $lexer = new WP_MySQL_Lexer( 'SELECT /*!080400 1 + */ 2' ); + $this->assertSame( 80410, $lexer->get_mysql_version() ); + + // A six-digit MySQL 8.4 version comment is executed by default. + $this->assertSame( + array( 'SELECT', 'NUM', 'PLUS', 'NUM', 'END_OF_INPUT', 'YYEOF' ), + array_map( + static function ( WP_MySQL_Token $token ): string { + return $token->get_name(); + }, + $lexer->remaining_tokens() + ) + ); + } + public function test_at_name_splits_into_at_and_ident(): void { $tokens = ( new WP_MySQL_Lexer( 'SELECT @var1' ) )->remaining_tokens(); $this->assertSame( 'AT_SIGN', $tokens[1]->get_name() ); @@ -749,7 +765,7 @@ public function test_ansi_quotes_changes_double_quote_tokenization( array $sql_modes, string $expected_token_name ): void { - $tokens = ( new WP_MySQL_Lexer( $sql, 80400, $sql_modes ) )->remaining_tokens(); + $tokens = ( new WP_MySQL_Lexer( $sql, 80410, $sql_modes ) )->remaining_tokens(); $this->assertSame( $expected_token_name, $tokens[0]->get_name(), $sql ); } @@ -782,17 +798,17 @@ public function data_national_strings(): array { public function test_composite_ansi_mode_expands_to_lexer_component_modes(): void { // ANSI_QUOTES: a double-quoted sequence is a quoted identifier. - $lexer = new WP_MySQL_Lexer( '"foo"', 80400, array( 'ANSI' ) ); + $lexer = new WP_MySQL_Lexer( '"foo"', 80410, array( 'ANSI' ) ); $this->assertTrue( $lexer->next_token() ); $this->assertSame( WP_MySQL_Lexer::IDENT_QUOTED, $lexer->get_token()->id ); // PIPES_AS_CONCAT: "||" is the string concatenation operator. - $lexer = new WP_MySQL_Lexer( '||', 80400, array( 'ANSI' ) ); + $lexer = new WP_MySQL_Lexer( '||', 80410, array( 'ANSI' ) ); $this->assertTrue( $lexer->next_token() ); $this->assertSame( WP_MySQL_Lexer::OR_OR_SYM, $lexer->get_token()->id ); // IGNORE_SPACE: whitespace is permitted between a function name and "(". - $lexer = new WP_MySQL_Lexer( 'COUNT (1)', 80400, array( 'ANSI' ) ); + $lexer = new WP_MySQL_Lexer( 'COUNT (1)', 80410, array( 'ANSI' ) ); $this->assertTrue( $lexer->next_token() ); $this->assertSame( WP_MySQL_Lexer::KEYWORDS['COUNT'], $lexer->get_token()->id ); } @@ -808,7 +824,7 @@ public function test_quoted_identifiers_treat_backslash_literally( int $expected_token_id, string $expected_value ): void { - $lexer = new WP_MySQL_Lexer( $sql, 80400, $sql_modes ); + $lexer = new WP_MySQL_Lexer( $sql, 80410, $sql_modes ); $this->assertTrue( $lexer->next_token() ); $token = $lexer->get_token(); diff --git a/packages/mysql-parser/tests/WP_MySQL_Server_Suite_Parser_Tests.php b/packages/mysql-parser/tests/WP_MySQL_Server_Suite_Parser_Tests.php index 7e3dab250..d50656cf5 100644 --- a/packages/mysql-parser/tests/WP_MySQL_Server_Suite_Parser_Tests.php +++ b/packages/mysql-parser/tests/WP_MySQL_Server_Suite_Parser_Tests.php @@ -20,7 +20,7 @@ class WP_MySQL_Server_Suite_Parser_Tests extends TestCase { * legitimately changes. */ const EXPECTED_QUERIES = 69158; - const EXPECTED_FAILURES = 170; + const EXPECTED_FAILURES = 167; public function test_corpus_acceptance_rate(): void { $parser = WP_MySQL_Parser_Factory::create_parser(); @@ -36,7 +36,7 @@ public function test_corpus_acceptance_rate(): void { ++$total; // Pin the MySQL version. The failure set in FAILURES_PATH depends on it. - $tokens = ( new WP_MySQL_Lexer( $query, 80038 ) )->remaining_tokens(); + $tokens = ( new WP_MySQL_Lexer( $query, 80410 ) )->remaining_tokens(); if ( null === $parser->parse( $tokens ) ) { $failures[] = $query; } diff --git a/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php b/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php index f49d3235d..a35b619b6 100644 --- a/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php +++ b/packages/mysql-parser/tests/WP_MySQL_Token_Tests.php @@ -15,7 +15,7 @@ class WP_MySQL_Token_Tests extends TestCase { * @return WP_MySQL_Token The first matching token. */ private static function first_token( string $sql, string $name, array $sql_modes = array() ): WP_MySQL_Token { - foreach ( ( new WP_MySQL_Lexer( $sql, 80400, $sql_modes ) )->remaining_tokens() as $token ) { + foreach ( ( new WP_MySQL_Lexer( $sql, 80410, $sql_modes ) )->remaining_tokens() as $token ) { if ( $token->get_name() === $name ) { return $token; } diff --git a/packages/mysql-parser/tests/data/corpus-failures.csv b/packages/mysql-parser/tests/data/corpus-failures.csv index 78e09b53d..563aedfe2 100644 --- a/packages/mysql-parser/tests/data/corpus-failures.csv +++ b/packages/mysql-parser/tests/data/corpus-failures.csv @@ -653,10 +653,6 @@ UNLOCK TABLES" "SELECT 'a%' NOT LIKE 'a!%' ESCAPE '!', 'a%' NOT LIKE 'a!' || '%' ESCAPE '!'" "SELECT 'a%' NOT LIKE 'a!%' ESCAPE '$', 'a%' NOT LIKE 'a!' || '%' ESCAPE '$'" "SELECT * from ""full""" -"SELECT 1 /*!080100 +1*/ AS should_return_2" -"SELECT 1 /*!080100 -+1*/ AS should_return_2" -"SELECT 1 /*!080100 +1*/ AS should_return_2" "SELECT 1 /*!99999 /* */ */" "SET sql_mode = 'NO_ENGINE_SUBSTITUTION'; create procedure test_resignal()