Fix BINARY and VARBINARY column defaults

packages/mysql-on-sqlite/src/mysql/class-wp-mysql-token.php

@@ -64,9 +64,14 @@ public function get_name(): string {
 	 */
 	public function get_value(): string {
 		$value = $this->get_bytes();
+		if ( WP_MySQL_Lexer::NCHAR_TEXT === $this->id ) {
+			// Remove the national string prefix.
+			$value = substr( $value, 1 );
+		}
 		if (
 			WP_MySQL_Lexer::SINGLE_QUOTED_TEXT === $this->id
 			|| WP_MySQL_Lexer::DOUBLE_QUOTED_TEXT === $this->id
+			|| WP_MySQL_Lexer::NCHAR_TEXT === $this->id
 			|| WP_MySQL_Lexer::BACK_TICK_QUOTED_ID === $this->id
 		) {
 			// Remove bounding quotes.

packages/mysql-on-sqlite/src/sqlite/class-wp-mysql-on-sqlite.php

@@ -4383,7 +4383,7 @@ private function translate( $node ): ?string {
 				}
 				return $value;
 			case 'textStringLiteral':
-				return $this->translate_string_literal( $node );
+				return $this->translate_string_literal( $node->get_first_child_token() );
 			case 'dataType':
 			case 'nchar':
 				$child = $node->get_first_child();
@@ -4607,6 +4607,9 @@ private function translate_token( WP_MySQL_Token $token ): ?string {
 					return sprintf( "x'%s'", $hex );
 				}
 				return $value;
+			case WP_MySQL_Lexer::NCHAR_TEXT:
+				// A national string, such as N'abc', is a regular string in SQLite.
+				return $this->translate_string_literal( $token );
 			case WP_MySQL_Lexer::AUTO_INCREMENT_SYMBOL:
 				return 'AUTOINCREMENT';
 			case WP_MySQL_Lexer::BINARY_SYMBOL:
@@ -4657,11 +4660,10 @@ private function translate_sequence( array $nodes, string $separator = ' ' ): ?s
 	/**
 	 * Translate a MySQL string literal to SQLite.
 	 *
-	 * @param  WP_Parser_Node $node The "textStringLiteral" AST node.
-	 * @return string               The translated value.
+	 * @param  WP_MySQL_Token $token The string literal token.
+	 * @return string                The translated value.
 	 */
-	private function translate_string_literal( WP_Parser_Node $node ): string {
-		$token = $node->get_first_child_token();
+	private function translate_string_literal( WP_MySQL_Token $token ): string {
 		$value = $token->get_value();
 
 		/*
@@ -5591,6 +5593,7 @@ private function translate_select_item( WP_Parser_Node $node ): string {
 		 *
 		 * Descend the AST until we reach a textStringLiteral. If at any level
 		 * we don't have a single child node, bail out; it's not a bare literal.
+		 * A national string, such as N'abc', is a single NCHAR_TEXT token.
 		 */
 		$current = $node;
 		while ( 'textStringLiteral' !== $current->rule_name ) {
@@ -5600,7 +5603,10 @@ private function translate_select_item( WP_Parser_Node $node ): string {
 			}
 			$current = $children[0];
 		}
-		if ( 'textStringLiteral' === $current->rule_name ) {
+		if (
+			'textStringLiteral' === $current->rule_name
+			|| ( 1 === count( $current->get_children() ) && $current->has_child_token( WP_MySQL_Lexer::NCHAR_TEXT ) )
+		) {
 			$alias = $current->get_first_child_token()->get_value();
 
 			// When the literal value contains a NULL byte, MySQL truncates the
@@ -7413,7 +7419,11 @@ private function get_sqlite_create_table_statement(
 					$ast            = $this->create_parser( 'SELECT ' . $column['COLUMN_DEFAULT'] )->parse();
 					$expr           = $ast->get_first_descendant_node( 'selectItem' )->get_first_child_node();
 					$default_clause = $this->translate( $expr );
-					$query         .= sprintf( ' DEFAULT (%s)', $default_clause );
+					if ( 'BLOB' === $type ) {
+						// STRICT tables reject a TEXT default in a BLOB column.
+						$default_clause = sprintf( 'CAST(%s AS BLOB)', $default_clause );
+					}
+					$query .= sprintf( ' DEFAULT (%s)', $default_clause );
 				} else {
 					$query .= ' DEFAULT ' . $this->format_sqlite_column_default( $column );
 				}
@@ -7899,6 +7909,12 @@ private function format_sqlite_column_default( array $column ): string {
 			return (string) bindec( substr( $column['COLUMN_DEFAULT'], 2, -1 ) );
 		}
 
+		// A BLOB column in a STRICT table only accepts a BLOB default.
+		$bytes = $this->get_binary_column_default_bytes( $column );
+		if ( null !== $bytes ) {
+			return sprintf( "x'%s'", bin2hex( $bytes ) );
+		}
+
 		return $this->quote_sqlite_value( $column['COLUMN_DEFAULT'] );
 	}
 
@@ -7914,9 +7930,38 @@ private function format_mysql_column_default( array $column ): string {
 			return $column['COLUMN_DEFAULT'];
 		}
 
+		// Like MySQL, keep the hex literal for bytes that aren't valid utf8mb3,
+		// i.e., UTF-8 with no characters above U+FFFF.
+		$bytes = $this->get_binary_column_default_bytes( $column );
+		if ( null !== $bytes ) {
+			return 1 === preg_match( '/\A[\x{0}-\x{FFFF}]*\z/u', $bytes )
+				? $this->quote_mysql_utf8_string_literal( $bytes )
+				: $column['COLUMN_DEFAULT'];
+		}
+
 		return $this->quote_mysql_utf8_string_literal( $column['COLUMN_DEFAULT'] );
 	}
 
+	/**
+	 * Get the bytes of a BINARY or VARBINARY column default.
+	 *
+	 * These defaults are stored as "0x..." hex literals, or as an empty string.
+	 * The forms that older driver versions stored are migrated to this format.
+	 *
+	 * @param  array       $column The information schema column data.
+	 * @return string|null         The default value bytes, or null when the column
+	 *                             is not binary or its default is not migrated.
+	 */
+	private function get_binary_column_default_bytes( array $column ): ?string {
+		if (
+			( 'binary' !== $column['DATA_TYPE'] && 'varbinary' !== $column['DATA_TYPE'] )
+			|| 1 !== preg_match( '/\A(?:0x((?:[0-9A-F]{2})+))?\z/', $column['COLUMN_DEFAULT'], $matches )
+		) {
+			return null;
+		}
+		return hex2bin( $matches[1] ?? '' );
+	}
+
 	/**
 	 * Unquote a quoted SQLite identifier.
 	 *

packages/mysql-on-sqlite/src/sqlite/class-wp-sqlite-configurator.php

@@ -345,6 +345,11 @@ private function run_migrations( string $saved_version ): void {
 			$this->migrate_datetime_precision();
 			$this->migrate_temporal_zero_defaults();
 		}
+
+		// Fix binary column defaults stored by versions <= 3.1.2.
+		if ( version_compare( $saved_version, '3.1.2', '<=' ) ) {
+			$this->migrate_binary_defaults();
+		}
 	}
 
 	/**
@@ -484,7 +489,7 @@ private function migrate_temporal_zero_defaults(): void {
 				);
 			}
 
-			$alter_list = array();
+			$defaults = array();
 			foreach ( $repaired_columns as $column ) {
 				// Preserve numeric and string types so comparisons apply the same
 				// SQLite conversion as when the defaults were stored.
@@ -502,26 +507,95 @@ private function migrate_temporal_zero_defaults(): void {
 						: array( $column['zero_value'] )
 				);
 
-				$alter_list[] = sprintf(
-					'ALTER COLUMN `%s` SET DEFAULT %s',
-					str_replace( '`', '``', $column['name'] ),
-					$this->driver->quote( $column['zero_value'] )
-				);
+				$defaults[ $column['name'] ] = $this->driver->quote( $column['zero_value'] );
 			}
+			$this->recreate_table( $table_name, $defaults );
+		}
+	}
 
-			// ALTER TABLE stores the new defaults, and it recreates the table to apply them.
-			try {
-				$this->driver->query(
-					sprintf( 'ALTER TABLE `%s` %s', str_replace( '`', '``', $table_name ), implode( ', ', $alter_list ) )
-				);
-			} catch ( WP_MySQL_On_SQLite_Exception $e ) {
-				throw new WP_MySQL_On_SQLite_Exception(
-					$this->driver,
-					sprintf( 'Failed to recreate table "%s": %s', $table_name, $e->getMessage() ),
-					$e->getCode(),
-					$e
-				);
+	/**
+	 * Migrate binary column defaults stored by older driver versions.
+	 *
+	 * Older driver versions stored BINARY and VARBINARY defaults as written, such
+	 * as x'00', while MySQL stores their bytes in hex, such as 0x00. They also gave
+	 * the SQLite tables TEXT defaults for these and for binary expression defaults,
+	 * which STRICT tables reject in BLOB columns, so inserts omitting them failed.
+	 */
+	private function migrate_binary_defaults(): void {
+		$columns_table = $this->driver->get_connection()->quote_identifier(
+			$this->schema_builder->get_table_name( false, 'columns' )
+		);
+
+		$columns = $this->driver->execute_sqlite_query(
+			sprintf(
+				"SELECT c.TABLE_NAME, c.COLUMN_NAME, c.DATA_TYPE, c.COLUMN_DEFAULT, c.EXTRA, p.dflt_value
+				FROM %s AS c
+				JOIN pragma_table_info(c.TABLE_NAME, 'main') AS p ON c.COLUMN_NAME = p.name
+				WHERE c.TABLE_SCHEMA = ?
+				AND c.DATA_TYPE IN ('binary', 'varbinary', 'tinyblob', 'blob', 'mediumblob', 'longblob')
+				AND c.COLUMN_DEFAULT IS NOT NULL",
+				$columns_table
+			),
+			array( WP_SQLite_Information_Schema_Builder::SAVED_DATABASE_NAME )
+		)->fetchAll( PDO::FETCH_ASSOC );
+
+		$table_defaults = array();
+		foreach ( $columns as $column ) {
+			$table_name     = $column['TABLE_NAME'];
+			$sqlite_default = (string) $column['dflt_value'];
+			if ( str_contains( $column['EXTRA'], 'DEFAULT_GENERATED' ) ) {
+				// Recreating the table casts the expression to BLOB.
+				if ( 1 !== preg_match( '/\ACAST\(.* AS BLOB\)\z/s', $sqlite_default ) ) {
+					$table_defaults[ $table_name ] = $table_defaults[ $table_name ] ?? array();
+				}
+			} elseif ( 'binary' === $column['DATA_TYPE'] || 'varbinary' === $column['DATA_TYPE'] ) {
+				// A literal default is stored as "0x..." and used as "x'...'" in SQLite.
+				if (
+					1 !== preg_match( '/\A(?:0x(?:[0-9A-F]{2})+)?\z/', $column['COLUMN_DEFAULT'] )
+					|| 1 !== preg_match( "/\A[xX]'[0-9a-fA-F]*'\z/", $sqlite_default )
+				) {
+					$bytes = WP_SQLite_Information_Schema_Builder::get_binary_default_bytes( $column['COLUMN_DEFAULT'] );
+
+					$table_defaults[ $table_name ][ $column['COLUMN_NAME'] ] = '' === $bytes ? "''" : '0x' . bin2hex( $bytes );
+				}
 			}
 		}
+
+		foreach ( $table_defaults as $table_name => $defaults ) {
+			$this->recreate_table( $table_name, $defaults );
+		}
+	}
+
+	/**
+	 * Recreate a table with new column defaults.
+	 *
+	 * ALTER TABLE stores the new defaults, and it recreates the table to apply them.
+	 * Without new defaults, ALTER TABLE ... FORCE only recreates the table.
+	 *
+	 * @param string $table_name The table name.
+	 * @param array  $defaults   The new defaults as MySQL literals, keyed by column name.
+	 */
+	private function recreate_table( string $table_name, array $defaults ): void {
+		$alter_list = array();
+		foreach ( $defaults as $column_name => $default ) {
+			$alter_list[] = sprintf( 'ALTER COLUMN `%s` SET DEFAULT %s', str_replace( '`', '``', $column_name ), $default );
+		}
+
+		try {
+			$this->driver->query(
+				sprintf(
+					'ALTER TABLE `%s` %s',
+					str_replace( '`', '``', $table_name ),
+					count( $alter_list ) > 0 ? implode( ', ', $alter_list ) : 'FORCE'
+				)
+			);
+		} catch ( WP_MySQL_On_SQLite_Exception $e ) {
+			throw new WP_MySQL_On_SQLite_Exception(
+				$this->driver,
+				sprintf( 'Failed to recreate table "%s": %s', $table_name, $e->getMessage() ),
+				$e->getCode(),
+				$e
+			);
+		}
 	}
 }

packages/mysql-on-sqlite/src/sqlite/class-wp-sqlite-information-schema-builder.php

@@ -878,6 +878,33 @@ public function record_drop_index( WP_Parser_Node $node ): void {
 		$this->record_drop_index_data( $table_is_temporary, $table_name, $index_name );
 	}
 
+	/**
+	 * Get the bytes of a BINARY or VARBINARY default from a hex or bit literal.
+	 *
+	 * Older driver versions stored these defaults as written, so this also reads
+	 * any other value as the bytes of a text default, for their migration.
+	 *
+	 * @param  string $default_value A hex or bit literal, or a default stored by an older version.
+	 * @return string                The default value bytes.
+	 */
+	public static function get_binary_default_bytes( string $default_value ): string {
+		if ( 1 === preg_match( "/\A(?|0x([0-9a-fA-F]+)|[xX]'([0-9a-fA-F]*)')\z/", $default_value, $matches ) ) {
+			// MySQL pads odd-length 0x literals with a leading zero.
+			return hex2bin( 1 === strlen( $matches[1] ) % 2 ? '0' . $matches[1] : $matches[1] );
+		}
+
+		if ( 1 === preg_match( "/\A(?|0b([01]+)|[bB]'([01]*)')\z/", $default_value, $matches ) ) {
+			$bits  = str_pad( $matches[1], (int) ceil( strlen( $matches[1] ) / 8 ) * 8, '0', STR_PAD_LEFT );
+			$bytes = '';
+			for ( $i = 0; $i < strlen( $bits ); $i += 8 ) {
+				$bytes .= chr( bindec( substr( $bits, $i, 8 ) ) );
+			}
+			return $bytes;
+		}
+
+		return $default_value;
+	}
+
 	/**
 	 * Analyze ADD COLUMN definition and record data in the information schema.
 	 *
@@ -2274,6 +2301,10 @@ private function get_literal_default( WP_Parser_Node $literal, string $data_type
 			return null;
 		}
 
+		if ( 'binary' === $data_type || 'varbinary' === $data_type ) {
+			return $this->get_binary_literal_default( $literal );
+		}
+
 		// DEFAULT TRUE or DEFAULT FALSE
 		if ( $literal->has_child_node( 'boolLiteral' ) ) {
 			$bool_literal = $literal->get_first_child_node( 'boolLiteral' );
@@ -2315,19 +2346,57 @@ private function normalize_literal_default( string $default_value, string $data_
 			return $bit_default;
 		}
 
+		if ( 'binary' === $data_type || 'varbinary' === $data_type ) {
+			return $this->format_binary_default( $default_value );
+		}
+
 		/*
-		 * @TODO: Non-BIT literal defaults, except for temporal zero values, are
+		 * @TODO: Other literal defaults, except for temporal zero values, are
 		 * currently stored verbatim. To match MySQL, they should be normalized.
 		 * For example:
-		 *  - BINARY/VARBINARY defaults stored as 0x hex literals.
 		 *  - Hex and bit literals coerced to raw bytes for text columns and to
 		 *    their decimal value for numeric columns.
-		 *  - The charset introducer dropped from text-literal defaults.
 		 *  - Numeric defaults normalized, e.g. 1.0 to 1 or 1e3 to 1000.
 		 */
 		return $default_value;
 	}
 
+	/**
+	 * Get the default of a BINARY or VARBINARY column from a literal.
+	 *
+	 * Hex and bit literals are byte strings. Other literals give the bytes of
+	 * their text, so the text '0x41' stays distinct from the hex literal 0x41.
+	 *
+	 * @param  WP_Parser_Node $literal The "literal" AST node.
+	 * @return string                  The default value as stored in information schema.
+	 */
+	private function get_binary_literal_default( WP_Parser_Node $literal ): string {
+		$byte_string = $literal->get_first_child_token( WP_MySQL_Lexer::HEX_NUMBER )
+			?? $literal->get_first_child_token( WP_MySQL_Lexer::BIN_NUMBER );
+		if ( null !== $byte_string ) {
+			return $this->format_binary_default( self::get_binary_default_bytes( $byte_string->get_value() ) );
+		}
+
+		$bool_literal = $literal->get_first_child_node( 'boolLiteral' );
+		if ( null !== $bool_literal ) {
+			return $this->format_binary_default( $bool_literal->has_child_token( WP_MySQL_Lexer::TRUE_SYMBOL ) ? '1' : '0' );
+		}
+
+		return $this->format_binary_default( $this->get_value( $literal ) );
+	}
+
+	/**
+	 * Format the bytes of a BINARY or VARBINARY default for the information schema.
+	 *
+	 * Like in MySQL, the bytes are shown in hex, except for an empty default.
+	 *
+	 * @param  string $bytes The default value bytes.
+	 * @return string        The default value as stored in information schema.
+	 */
+	private function format_binary_default( string $bytes ): string {
+		return '' === $bytes ? '' : '0x' . strtoupper( bin2hex( $bytes ) );
+	}
+
 	/**
 	 * Get the zero value of a temporal column when its default is zero.
 	 *
@@ -3310,6 +3379,9 @@ private function get_value( WP_Parser_Node $node ): string {
 						}
 					}
 				}
+			} elseif ( WP_MySQL_Lexer::UNDERSCORE_CHARSET === $child->id ) {
+				// A charset introducer isn't part of the value.
+				continue;
 			} else {
 				$value = $child->get_value();
 			}

packages/mysql-on-sqlite/src/sqlite/class-wp-sqlite-information-schema-reconstructor.php

@@ -600,8 +600,8 @@ private function generate_column_default( string $mysql_type, ?string $default_v
 		// BLOB literals (string). E.g.: x'1a2f', X'1A2F'
 		// Checking the prefix is enough as SQLite doesn't allow malformed values.
 		if ( str_starts_with( $uppercase_default_value, "X'" ) ) {
-			// Convert the hex string to ASCII bytes.
-			return "'" . pack( 'H*', substr( $default_value, 2, -1 ) ) . "'";
+			// Convert the hex string to bytes, and quote them as a string literal.
+			return $this->quote_mysql_utf8_string_literal( pack( 'H*', substr( $default_value, 2, -1 ) ) );
 		}
 
 		// Unquoted string literal. E.g.: abc