Run the lexer benchmark workflow on the MySQL parser package

.github/workflows/lexer-benchmark.yml

@@ -3,10 +3,10 @@ name: Lexer benchmark
 on:
   pull_request:
     paths:
-      - 'packages/mysql-on-sqlite/src/mysql/class-wp-mysql-lexer.php'
-      - 'packages/mysql-on-sqlite/src/mysql/class-wp-mysql-token.php'
-      - 'packages/mysql-on-sqlite/src/parser/class-wp-parser-token.php'
-      - 'packages/mysql-on-sqlite/tests/tools/run-lexer-benchmark.php'
+      - 'packages/mysql-parser/src/class-wp-mysql-lexer.php'
+      - 'packages/mysql-parser/src/class-wp-mysql-token.php'
+      - 'packages/mysql-parser/src/parser/class-wp-parser-token.php'
+      - 'packages/mysql-parser/tests/benchmark.php'
       - '.github/workflows/lexer-benchmark.yml'
 
 # A new push supersedes the previous run; the result comment is updated in place.
@@ -38,16 +38,29 @@ jobs:
           php-version: '8.4'
           coverage: none
 
+      - name: Generate the autoloader
+        working-directory: packages/mysql-parser
+        run: composer dump-autoload --no-interaction
+
       - name: Benchmark base vs PR
         env:
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          BENCH=packages/mysql-on-sqlite/tests/tools/run-lexer-benchmark.php
+          BENCH=packages/mysql-parser/tests/benchmark.php
 
-          # Best-pass QPS for a given PHP flag set.
+          # Best-pass QPS for a given PHP flag set. A lexer that throws looks fast, and a
+          # crashed run prints no result, so both fail the job instead of posting a number.
           best() {
-            php -d memory_limit=512M "$@" "$BENCH" --json \
-              | php -r '$j = json_decode( stream_get_contents( STDIN ), true ); echo (int) $j["qps_best"];'
+            php -d memory_limit=512M "$@" "$BENCH" --lexer-only --iterations=10 --warmup=3 --json \
+              | php -r '
+                $output = stream_get_contents( STDIN );
+                $result = json_decode( $output, true );
+                if ( ! $result || $result["exceptions"] > 0 ) {
+                  fwrite( STDERR, "The lexer benchmark failed:\n$output\n" );
+                  exit( 1 );
+                }
+                echo (int) $result["qps_best"];
+              '
           }
           jit_flags="-d opcache.enable_cli=1 -d opcache.jit_buffer_size=64M -d opcache.jit=tracing"
 
@@ -57,11 +70,13 @@ jobs:
 
           # Swap only the source tree to the base commit and re-measure with the
           # same (PR) benchmark tool, so both sides are timed identically. The
-          # benchmark tool itself (tests/tools/) is left at the PR version.
-          git checkout "$BASE_SHA" -- packages/mysql-on-sqlite/src
+          # benchmark tool itself (tests/) is left at the PR version. PR-only files
+          # are removed and the classmap is rebuilt, so the base run can't load PR code.
+          git checkout --no-overlay "$BASE_SHA" -- packages/mysql-parser/src
+          composer dump-autoload --no-interaction --working-dir=packages/mysql-parser
           base_nojit=$( best )
           base_jit=$( best $jit_flags )
-          git checkout HEAD -- packages/mysql-on-sqlite/src
+          git checkout HEAD -- packages/mysql-parser/src
 
           fmt() { php -r 'echo number_format( (int) $argv[1] );' "$1"; }
           ratio() { php -r 'printf( "%.2f", $argv[1] / max( 1, (int) $argv[2] ) );' "$1" "$2"; }
@@ -80,7 +95,7 @@ jobs:
             echo
             echo "To reproduce locally:"
             echo '```'
-            echo "cd packages/mysql-on-sqlite && composer run bench-lexer"
+            echo "cd packages/mysql-parser && composer dump-autoload && composer run benchmark -- --lexer-only"
             echo '```'
           } > "$RUNNER_TEMP/comment.md"
           echo "COMMENT_FILE=$RUNNER_TEMP/comment.md" >> "$GITHUB_ENV"

packages/mysql-parser/README.md

@@ -97,6 +97,7 @@ This requires `bash`, `git`, and `php`. It performs the following steps:
 To run lexer and parser tests and benchmarks, use:
 
 ```bash
-composer run test        # PHPUnit suite, including the query corpus tests
-composer run benchmark   # Query throughput with and without JIT on the whole corpus
+composer run test                       # PHPUnit suite, including the query corpus tests
+composer run benchmark                  # Query throughput with and without JIT on the whole corpus
+composer run benchmark -- --lexer-only  # The same, measuring only the lexer
 ```

packages/mysql-parser/tests/benchmark.php

@@ -2,7 +2,8 @@
 /**
  * Benchmark the MySQL parser over the MySQL server test corpus.
  *
- * Reports the corpus parse rate and end-to-end (lex + parse) throughput.
+ * Reports the corpus parse rate and end-to-end (lex + parse) throughput. With
+ * --lexer-only, it measures only the lexer and parses nothing.
  *
  * Methodology: a few warmup passes (discarded — they heat opcache, the tracing
  * JIT, and the CPU caches) followed by N timed passes over the whole corpus. The
@@ -13,6 +14,7 @@
  *
  * Options:
  *   --json          Machine-readable output.
+ *   --lexer-only    Only lex the queries, without parsing them.
  *   --limit=N       Only benchmark the first N queries.
  *   --iterations=N  Number of timed passes (default 5).
  *   --warmup=N      Number of discarded warmup passes (default 2).
@@ -26,6 +28,7 @@ function ( $severity, $message, $file, $line ) {
 );
 
 $json       = in_array( '--json', $argv, true );
+$lexer_only = in_array( '--lexer-only', $argv, true );
 $limit      = null;
 $iterations = 5;
 $warmup     = 2;
@@ -43,7 +46,7 @@ function ( $severity, $message, $file, $line ) {
 }
 
 require_once __DIR__ . '/../vendor/autoload.php';
-$parser = WP_MySQL_Parser_Factory::create_parser();
+$parser = $lexer_only ? null : WP_MySQL_Parser_Factory::create_parser();
 
 // Load the corpus before timing so file IO is excluded.
 if ( ! is_readable( $corpus ) ) {
@@ -65,17 +68,17 @@ function ( $severity, $message, $file, $line ) {
 fclose( $handle );
 $query_count = count( $queries );
 
-// One end-to-end pass over the corpus (lex + parse), recording failures and
-// exceptions (deterministic across passes, so the last pass's counts are kept).
-$failures     = 0;
-$exceptions   = 0;
-$parse_corpus = function () use ( $queries, $parser, &$failures, &$exceptions ) {
+// One pass over the corpus, recording failures and exceptions (deterministic
+// across passes, so the last pass's counts are kept).
+$failures   = 0;
+$exceptions = 0;
+$run_pass   = function () use ( $queries, $parser, &$failures, &$exceptions ) {
 	$failures   = 0;
 	$exceptions = 0;
 	foreach ( $queries as $query ) {
 		try {
 			$tokens = ( new WP_MySQL_Lexer( $query ) )->remaining_tokens();
-			if ( null === $parser->parse( $tokens ) ) {
+			if ( null !== $parser && null === $parser->parse( $tokens ) ) {
 				++$failures;
 			}
 		} catch ( Throwable $e ) {
@@ -85,13 +88,13 @@ function ( $severity, $message, $file, $line ) {
 };
 
 for ( $i = 0; $i < $warmup; $i++ ) {
-	$parse_corpus();
+	$run_pass();
 }
 
 $samples = array();
 for ( $i = 0; $i < $iterations; $i++ ) {
 	$start = microtime( true );
-	$parse_corpus();
+	$run_pass();
 	$samples[] = $query_count / ( microtime( true ) - $start );
 }
 sort( $samples );
@@ -112,7 +115,7 @@ function ( $severity, $message, $file, $line ) {
 if ( $json ) {
 	echo json_encode(
 		array(
-			'benchmark'   => 'mysql-parser',
+			'benchmark'   => $lexer_only ? 'mysql-lexer' : 'mysql-parser',
 			'opcache'     => $opcache_on,
 			'jit'         => $jit_on,
 			'queries'     => $query_count,
@@ -134,12 +137,18 @@ function ( $severity, $message, $file, $line ) {
 }
 
 $config = $jit_on ? 'opcache + tracing JIT' : ( $opcache_on ? 'opcache, no JIT' : 'no opcache' );
-printf( "MySQL parser (official grammar) — %s\n", $config );
+printf( "MySQL %s — %s\n", $lexer_only ? 'lexer' : 'parser (official grammar)', $config );
 $jit_requested = ! in_array( strtolower( (string) ini_get( 'opcache.jit' ) ), array( '', '0', 'off', 'disable' ), true );
 if ( $jit_requested && ! $jit_on ) {
 	printf( "  warning: opcache.jit is set but the JIT is NOT active here — check that opcache is enabled and jit_buffer_size > 0.\n" );
 }
-printf( "%s queries, %d warmup + %d timed passes (end-to-end lex+parse)\n", number_format( $query_count ), $warmup, $iterations );
+printf(
+	"%s queries, %d warmup + %d timed passes (%s)\n",
+	number_format( $query_count ),
+	$warmup,
+	$iterations,
+	$lexer_only ? 'lex only' : 'end-to-end lex+parse'
+);
 printf( "  best:   %s QPS\n", number_format( $best ) );
 printf( "  median: %s QPS\n", number_format( $median ) );
 printf( "  spread: %.1f%% (best vs worst)\n", $spread * 100 );