From 2f3c3b8d763cb916b233f2ed0d4e3e146cae83ce Mon Sep 17 00:00:00 2001 From: Bastian Bartmann Date: Mon, 28 Sep 2026 17:42:36 +0200 Subject: [PATCH 1/2] Checkpoint only fully applied binlog transactions Route ordered completion markers through the existing DML queues and persist a separate applied-transaction watermark only after preceding SQL commits. Preserve batching and reject acknowledgements after failed delivery or migration cancellation. Make reader operations and checkpoint SQL cancellation-aware. Cover multi-row events, MySQL/MariaDB GTID cloning, partial-apply replay, failed batches, and cancellation during both checkpoint polling and an in-flight INSERT. --- go/binlog/binlog_entry.go | 5 +- go/binlog/cancellation_test.go | 115 +++++++ go/binlog/gomysql_reader.go | 49 ++- go/binlog/transaction_marker_test.go | 74 +++++ go/binlog/transaction_test_helpers_test.go | 47 +++ go/logic/applier.go | 15 +- go/logic/applier_test.go | 9 +- go/logic/binlog_checkpoint_test.go | 311 ++++++++++++++++++ go/logic/checkpoint_cancellation_test.go | 191 +++++++++++ go/logic/migrator.go | 154 ++++++--- go/logic/migrator_move_tables_cutover_test.go | 39 ++- go/logic/streamer.go | 72 +++- go/logic/streamer_test.go | 45 +++ go/logic/transaction_marker_test.go | 184 +++++++++++ 14 files changed, 1231 insertions(+), 79 deletions(-) create mode 100644 go/binlog/cancellation_test.go create mode 100644 go/binlog/transaction_marker_test.go create mode 100644 go/binlog/transaction_test_helpers_test.go create mode 100644 go/logic/binlog_checkpoint_test.go create mode 100644 go/logic/checkpoint_cancellation_test.go create mode 100644 go/logic/transaction_marker_test.go diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index c1216054c..0255fde29 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -18,8 +18,9 @@ type BinlogEntry struct { // Timestamp is the wall-clock time recorded in the binlog event header of the // event that produced this entry. It is used in move-tables mode to measure // writer lag (now - last applied event timestamp). - Timestamp time.Time - DmlEvent *BinlogDMLEvent + Timestamp time.Time + DmlEvent *BinlogDMLEvent + TransactionComplete bool } // NewBinlogEntryAt creates an empty, ready to go BinlogEntry object diff --git a/go/binlog/cancellation_test.go b/go/binlog/cancellation_test.go new file mode 100644 index 000000000..c4c83a4e4 --- /dev/null +++ b/go/binlog/cancellation_test.go @@ -0,0 +1,115 @@ +package binlog + +import ( + "context" + "testing" + "time" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" + "github.com/go-mysql-org/go-mysql/replication" + "github.com/stretchr/testify/require" +) + +func TestTransactionMarkerSendCanceled(t *testing.T) { + assertTransactionMarkerSendCanceled(t, nil) +} + +func assertTransactionMarkerSendCanceled(t *testing.T, wrapEvents func([]*replication.BinlogEvent) []*replication.BinlogEvent) { + t.Helper() + reader := transactionTestReader(false) + defer reader.migrationContext.CancelContext() + previous := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 4) + reader.LastTrxCoords = previous.Clone() + row := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 200}, + Event: &replication.RowsEvent{ + Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")}, + Rows: [][]interface{}{{1}}, + }, + } + commit := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: 220}, + Event: &replication.XIDEvent{}, + } + events := []*replication.BinlogEvent{row, commit} + if wrapEvents != nil { + events = wrapEvents(events) + } + for _, event := range events { + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(event)) + } + entries := make(chan *BinlogEntry) + done := make(chan error, 1) + go func() { + err := reader.StreamEvents(func() bool { return false }, entries) + _ = base.SendWithContext(t.Context(), done, err) + }() + select { + case entry := <-entries: + require.NotNil(t, entry.DmlEvent) + case <-time.After(time.Second): + t.Fatal("row was not delivered") + } + // Wait until the commit position has been read, then stop consuming. + // The reader must propagate a canceled completion-marker send. + require.Eventually(t, func() bool { + return reader.GetCurrentBinlogCoordinates().(*mysql.FileBinlogCoordinates).LogPos == 220 + }, time.Second, time.Millisecond) + reader.migrationContext.CancelContext() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("completion marker send ignored cancellation") + } + require.True(t, reader.transactionHasRows, "failed marker delivery must retain pending state") + require.True(t, previous.Equals(reader.LastTrxCoords), "failed delivery must not advance reconnect coordinates") +} + +func TestRowsSendCanceled(t *testing.T) { + reader := transactionTestReader(false) + defer reader.migrationContext.CancelContext() + event := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2}, + Event: &replication.RowsEvent{ + Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")}, + Rows: [][]interface{}{{1}, {2}}, + }, + } + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(event)) + entries := make(chan *BinlogEntry) + done := make(chan error, 1) + go func() { + err := reader.StreamEvents(func() bool { return false }, entries) + _ = base.SendWithContext(t.Context(), done, err) + }() + select { + case <-entries: + case <-time.After(time.Second): + t.Fatal("first row was not delivered") + } + reader.migrationContext.CancelContext() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("blocked row send ignored cancellation") + } +} + +func TestIdleReaderCanceled(t *testing.T) { + reader := transactionTestReader(false) + reader.migrationContext.CancelContext() + done := make(chan error, 1) + go func() { + err := reader.StreamEvents(func() bool { return false }, make(chan *BinlogEntry)) + _ = base.SendWithContext(t.Context(), done, err) + }() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("idle reader ignored cancellation") + } +} diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index ee50b0d1a..5d5c6f000 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -15,8 +15,6 @@ import ( "time" - "context" - gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" ) @@ -47,8 +45,11 @@ type GoMySQLReader struct { currentCoordinates mysql.BinlogCoordinates currentCoordinatesMutex *sync.Mutex // LastTrxCoords are the coordinates of the last transaction completely read. - // If using the file coordinates it is binlog position of the transaction's XID event. + // They are for streamer reconnects, not durable applier checkpoints. + // For file coordinates, this is the end position of the transaction's XID event. LastTrxCoords mysql.BinlogCoordinates + + transactionHasRows bool } func NewGoMySQLReader(migrationContext *base.MigrationContext, rowsEventFilters ...RowsEventFilterFunc) *GoMySQLReader { @@ -148,15 +149,43 @@ func (gmr *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent // decides whether action is taken synchronously (meaning we wait before // next iteration) or asynchronously (we keep pushing more events) // In reality, reads will be synchronous - entriesChannel <- binlogEntry + if err := base.SendWithContext(gmr.migrationContext.GetContext(), entriesChannel, binlogEntry); err != nil { + return err + } + gmr.transactionHasRows = true } return nil } +func (gmr *GoMySQLReader) completeTransaction(event *replication.XIDEvent, entriesChannel chan<- *BinlogEntry) error { + var coords mysql.BinlogCoordinates + if gmr.migrationContext.UseGTIDs && event.GSet != nil { + coords = (&mysql.GTIDBinlogCoordinates{GTIDSet: event.GSet}).Clone() + } else { + // File positions use the current event's coordinates. For GTID + // events parsed without the syncer, fall back to the coordinates + // already updated by the preceding GTID event. + coords = gmr.GetCurrentBinlogCoordinates() + } + if gmr.transactionHasRows { + marker := &BinlogEntry{ + Coordinates: coords.Clone(), + TransactionComplete: true, + } + if err := base.SendWithContext(gmr.migrationContext.GetContext(), entriesChannel, marker); err != nil { + return err + } + gmr.transactionHasRows = false + } + // Reconnect only after all rows and their completion marker were delivered. + gmr.LastTrxCoords = coords + return nil +} + // StreamEvents func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChannel chan<- *BinlogEntry) error { for !canStopStreaming() { - ev, err := gmr.binlogStreamer.GetEvent(context.Background()) + ev, err := gmr.binlogStreamer.GetEvent(gmr.migrationContext.GetContext()) if err != nil { return err } @@ -215,14 +244,8 @@ func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChan gmr.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", coords.LogFile, int64(ev.Header.LogPos), event.NextLogName) gmr.currentCoordinatesMutex.Unlock() case *replication.XIDEvent: - if gmr.migrationContext.UseGTIDs { - // event.GSet is the full executed GTID set maintained by the - // syncer (MysqlGTIDSet or MariadbGTIDSet depending on flavor). - if event.GSet != nil { - gmr.LastTrxCoords = &mysql.GTIDBinlogCoordinates{GTIDSet: event.GSet} - } - } else { - gmr.LastTrxCoords = gmr.currentCoordinates.Clone() + if err := gmr.completeTransaction(event, entriesChannel); err != nil { + return err } case *replication.RowsEvent: if err := gmr.handleRowsEvent(ev, event, entriesChannel); err != nil { diff --git a/go/binlog/transaction_marker_test.go b/go/binlog/transaction_marker_test.go new file mode 100644 index 000000000..efc152a38 --- /dev/null +++ b/go/binlog/transaction_marker_test.go @@ -0,0 +1,74 @@ +package binlog + +import ( + "testing" + + "github.com/github/gh-ost/go/mysql" + "github.com/go-mysql-org/go-mysql/replication" + "github.com/stretchr/testify/require" +) + +func TestTransactionMarkerClonesGTIDSet(t *testing.T) { + for _, test := range []struct { + flavor, committed, next string + }{ + {mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1", "00000000-0000-0000-0000-000000000001:2"}, + {mysql.MariaDBFlavor, "0-1-1", "0-1-2"}, + } { + t.Run(test.flavor, func(t *testing.T) { + reader := transactionTestReader(true) + defer reader.migrationContext.CancelContext() + coords, err := mysql.NewGTIDBinlogCoordinates(test.flavor, test.committed) + require.NoError(t, err) + reader.currentCoordinates = coords.Clone() + rows := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2}, + Event: &replication.RowsEvent{ + Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")}, + Rows: [][]interface{}{{1}, {2}}, + }, + } + commit := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.XID_EVENT}, + Event: &replication.XIDEvent{GSet: coords.GTIDSet}, + } + entries := transactionTestStream(t, reader, rows, commit) + require.Len(t, entries, 3) + require.True(t, entries[2].TransactionComplete) + require.NoError(t, coords.GTIDSet.Update(test.next)) + require.Equal(t, test.committed, reader.LastTrxCoords.String()) + require.Equal(t, test.committed, entries[2].Coordinates.String()) + }) + } +} + +func TestMultiRowEventDoesNotCompleteTransaction(t *testing.T) { + reader := transactionTestReader(false) + rows := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 200, EventSize: 100}, + Event: &replication.RowsEvent{ + Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")}, + Rows: [][]interface{}{{1}, {2}, {3}}, + }, + } + // Stop before XID. All rows share the event end position, but none of + // them may independently acknowledge that position for checkpointing. + entries := transactionTestStream(t, reader, rows) + require.Len(t, entries, 3) + for _, entry := range entries { + require.Equal(t, int64(200), entry.Coordinates.(*mysql.FileBinlogCoordinates).LogPos) + require.False(t, entry.TransactionComplete) + } + require.Nil(t, reader.LastTrxCoords) + + commit := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: 220, EventSize: 20}, + Event: &replication.XIDEvent{}, + } + entries = transactionTestStream(t, reader, commit) + require.Len(t, entries, 1) + require.True(t, entries[0].TransactionComplete) + require.Nil(t, entries[0].DmlEvent) + require.Equal(t, int64(220), entries[0].Coordinates.(*mysql.FileBinlogCoordinates).LogPos) + require.Equal(t, reader.LastTrxCoords, entries[0].Coordinates) +} diff --git a/go/binlog/transaction_test_helpers_test.go b/go/binlog/transaction_test_helpers_test.go new file mode 100644 index 000000000..f916e26e9 --- /dev/null +++ b/go/binlog/transaction_test_helpers_test.go @@ -0,0 +1,47 @@ +package binlog + +import ( + "sync" + "testing" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" + "github.com/go-mysql-org/go-mysql/replication" + "github.com/stretchr/testify/require" +) + +func transactionTestReader(useGTIDs bool) *GoMySQLReader { + ctx := base.NewMigrationContext() + ctx.UseGTIDs = useGTIDs + ctx.DatabaseName = "testdb" + ctx.OriginalTableName = "testing" + var coords mysql.BinlogCoordinates = &mysql.FileBinlogCoordinates{LogFile: "mysql-bin.000001", LogPos: 4} + if useGTIDs { + coords = &mysql.GTIDBinlogCoordinates{} + } + return &GoMySQLReader{ + migrationContext: ctx, + binlogStreamer: replication.NewBinlogStreamer(), + currentCoordinates: coords, + currentCoordinatesMutex: &sync.Mutex{}, + } +} + +func transactionTestStream(t *testing.T, reader *GoMySQLReader, events ...*replication.BinlogEvent) []*BinlogEntry { + t.Helper() + for _, event := range events { + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(event)) + } + entries := make(chan *BinlogEntry, 32) + iterations := 0 + require.NoError(t, reader.StreamEvents(func() bool { + iterations++ + return iterations > len(events) + }, entries)) + close(entries) + var result []*BinlogEntry + for entry := range entries { + result = append(result, entry) + } + return result +} diff --git a/go/logic/applier.go b/go/logic/applier.go index e6d2682ba..a513cce8f 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -76,8 +76,9 @@ type Applier struct { finishedMigrating int64 name string - CurrentCoordinatesMutex sync.Mutex - CurrentCoordinates mysql.BinlogCoordinates + CurrentCoordinatesMutex sync.Mutex + CurrentCoordinates mysql.BinlogCoordinates + AppliedTransactionCoordinates mysql.BinlogCoordinates LastIterationRangeMutex sync.Mutex LastIterationRangeMinValues *sql.ColumnValues @@ -1272,7 +1273,7 @@ func (apl *Applier) WriteChangelogState(value string) (string, error) { // WriteCheckpoint writes a standard-mode checkpoint row to the _ghk table. In // move-tables mode use WriteMoveTableCheckpoints instead; calling this there is a // programmer error (the checkpoint schema and query builder are standard-only). -func (apl *Applier) WriteCheckpoint(chk *Checkpoint) (int64, error) { +func (apl *Applier) WriteCheckpoint(ctx context.Context, chk *Checkpoint) (int64, error) { if apl.migrationContext.IsMoveTablesMode() { panic("WriteCheckpoint() must not be called in move-tables mode; use WriteMoveTableCheckpoints") } @@ -1285,7 +1286,7 @@ func (apl *Applier) WriteCheckpoint(chk *Checkpoint) (int64, error) { } args := sqlutils.Args(chk.LastTrxCoords.String(), chk.Iteration, chk.RowsCopied, chk.DMLApplied, chk.IsCutover) args = append(args, uniqueKeyArgs...) - res, err := apl.checkpointDB().Exec(query, args...) + res, err := apl.checkpointDB().ExecContext(ctx, query, args...) if err != nil { return insertId, err } @@ -1314,6 +1315,10 @@ var moveTablesCheckpointColumns = []string{ // rows-copied, and serialized iteration range. The latest row therefore always // reflects the freshest run-wide state. func (apl *Applier) WriteMoveTableCheckpoints(rows []*Checkpoint) error { + return apl.writeMoveTableCheckpoints(apl.migrationContext.GetContext(), rows) +} + +func (apl *Applier) writeMoveTableCheckpoints(ctx context.Context, rows []*Checkpoint) error { if len(rows) == 0 { return nil } @@ -1347,7 +1352,7 @@ func (apl *Applier) WriteMoveTableCheckpoints(rows []*Checkpoint) error { serializeRangeValues(chk.IterationRangeMin), serializeRangeValues(chk.IterationRangeMax), ) - if _, err := apl.checkpointDB().Exec(query, args...); err != nil { + if _, err := apl.checkpointDB().ExecContext(ctx, query, args...); err != nil { return err } } diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index da4d9b8ec..465841cf5 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -1374,13 +1374,20 @@ func (suite *ApplierTestSuite) TestWriteCheckpoint() { DMLApplied: 200000, IsCutover: true, } - id, err := applier.WriteCheckpoint(chk) + id, err := applier.WriteCheckpoint(ctx, chk) suite.Require().NoError(err) suite.Require().Equal(int64(1), id) + // A canceled write must leave the last durable checkpoint untouched. + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + _, err = applier.WriteCheckpoint(canceledCtx, chk) + suite.Require().ErrorIs(err, context.Canceled) + gotChk, err := applier.ReadLastCheckpoint() suite.Require().NoError(err) + suite.Require().Equal(id, gotChk.Id) suite.Require().Equal(chk.Iteration, gotChk.Iteration) suite.Require().Equal(chk.LastTrxCoords.String(), gotChk.LastTrxCoords.String()) suite.Require().Equal(chk.IterationRangeMin.String(), gotChk.IterationRangeMin.String()) diff --git a/go/logic/binlog_checkpoint_test.go b/go/logic/binlog_checkpoint_test.go new file mode 100644 index 000000000..0d6030e50 --- /dev/null +++ b/go/logic/binlog_checkpoint_test.go @@ -0,0 +1,311 @@ +package logic + +import ( + "context" + gosql "database/sql" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/binlog" + "github.com/github/gh-ost/go/mysql" + "github.com/github/gh-ost/go/sql" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + testmysql "github.com/testcontainers/testcontainers-go/modules/mysql" +) + +// queueCheckpointTestEvents uses the production reader, dispatcher and listeners, +// but leaves consumption of the apply queue under the test's control. +// +//nolint:contextcheck // Streamer APIs use MigrationContext; the test deadline closes the reader. +func queueCheckpointTestEvents(t *testing.T, ctx context.Context, migrationContext *base.MigrationContext, migrator *Migrator, start, end mysql.BinlogCoordinates) { + t.Helper() + reader := binlog.NewGoMySQLReader(migrationContext, func(database, table string) bool { + return database == testMysqlDatabase && table == testMysqlTableName + }) + defer reader.Close() + require.NoError(t, reader.ConnectBinlogStreamer(start)) + streamer := NewEventsStreamer(migrationContext) + streamer.binlogReader = reader + migrator.eventsStreamer = streamer + require.NoError(t, migrator.addDMLEventsListener()) + + delivered := make(chan struct{}, 1) + streamer.AddTransactionCompleteListener(func(entry *binlog.BinlogEntry) error { + if entry.Coordinates.Equals(end) { + delivered <- struct{}{} + } + return nil + }) + done := make(chan error, 1) + go func() { + done <- streamer.StreamEvents(func() bool { + return reader.LastTrxCoords != nil && reader.LastTrxCoords.Equals(end) + }) + }() + select { + case err := <-done: + require.NoError(t, err) + case <-ctx.Done(): + reader.Close() + <-done + t.Fatal("timed out reading transaction") + } + // The reader has stopped sending. Let the dispatcher drain and exit. + close(streamer.eventsChannel) + select { + case <-delivered: + case <-ctx.Done(): + t.Fatal("timed out dispatching transaction completion") + } +} + +//nolint:contextcheck // Migrator APIs use MigrationContext's internal context; SQL/checkpoint calls use the test deadline. +func TestBinlogCheckpointWaitsForTransactionIntegration(t *testing.T) { + if testing.Short() { + t.Skip("requires MySQL 8.0") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + container, err := testmysql.Run(ctx, testMysqlContainerImage, + testmysql.WithDatabase(testMysqlDatabase), + testmysql.WithUsername(testMysqlUser), + testmysql.WithPassword(testMysqlPass), + testcontainers.WithCmdArgs("--server-id=1", "--gtid-mode=ON", "--enforce-gtid-consistency=ON"), + ) + require.NoError(t, err) + defer testcontainers.TerminateContainer(container) + dsn, err := container.ConnectionString(ctx) + require.NoError(t, err) + db, err := gosql.Open("mysql", dsn) + require.NoError(t, err) + defer db.Close() + conn, err := db.Conn(ctx) + require.NoError(t, err) + defer conn.Close() + for _, query := range []string{ + "SET SESSION time_zone='+00:00'", + "CREATE TABLE testing (id INT PRIMARY KEY, amount DECIMAL(5,2), created_at TIMESTAMP, padding TEXT)", + } { + _, err := conn.ExecContext(ctx, query) + require.NoError(t, err) + } + status := func() (*mysql.FileBinlogCoordinates, *mysql.GTIDBinlogCoordinates) { + var file, doDB, ignoreDB, gtid string + var pos int64 + require.NoError(t, conn.QueryRowContext(ctx, "SHOW MASTER STATUS").Scan(&file, &pos, &doDB, &ignoreDB, >id)) + coords, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, gtid) + require.NoError(t, err) + return mysql.NewFileBinlogCoordinates(file, pos), coords + } + type transaction struct { + name string + startFile, endFile *mysql.FileBinlogCoordinates + startGTID, endGTID *mysql.GTIDBinlogCoordinates + } + var transactions []transaction + for _, compression := range []string{"OFF"} { + _, err := conn.ExecContext(ctx, "SET SESSION binlog_transaction_compression="+compression) + require.NoError(t, err) + startFile, startGTID := status() + tx, err := conn.BeginTx(ctx, nil) + require.NoError(t, err) + for _, query := range []string{ + "INSERT INTO testing VALUES (1, 123.45, '2023-11-14 22:13:20', REPEAT('x', 10000))", + "UPDATE testing SET amount=456.78 WHERE id=1", + "DELETE FROM testing WHERE id=1", + } { + _, err := tx.ExecContext(ctx, query) + require.NoError(t, err) + } + require.NoError(t, tx.Commit()) + endFile, endGTID := status() + transactions = append(transactions, transaction{compression, startFile, endFile, startGTID, endGTID}) + } + config, err := getTestConnectionConfig(ctx, container) + require.NoError(t, err) + + for _, transaction := range transactions { + for _, mode := range []struct { + useGTIDs bool + cancelAt string + }{ + {false, "none"}, {true, "none"}, + {false, "apply"}, {true, "apply"}, + {false, "insert"}, {true, "insert"}, + } { + useGTIDs := mode.useGTIDs + t.Run(fmt.Sprintf("compression=%s/GTID=%t/cancel=%s", transaction.name, useGTIDs, mode.cancelAt), func(t *testing.T) { + migrationContext := newTestMigrationContext() + defer migrationContext.CancelContext() + migrationContext.Checkpoint = true + migrationContext.UseGTIDs = useGTIDs + // Replaying an already-applied INSERT uses INSERT IGNORE. Duplicate + // warnings are expected on resume; use the default production policy. + migrationContext.PanicOnWarnings = false + migrationContext.ApplierConnectionConfig = config + migrationContext.InspectorConnectionConfig = config + require.NoError(t, migrationContext.SetConnectionConfig("innodb")) + migrationContext.SetDMLBatchSize(1) + columns := sql.NewColumnList([]string{"id", "amount", "created_at", "padding"}) + migrationContext.OriginalTableColumns = columns + migrationContext.SharedColumns = columns + migrationContext.MappedSharedColumns = columns + uniqueKeyColumns := sql.NewColumnList([]string{"id"}) + uniqueKeyColumns.GetColumn("id").MySQLType = "int" + migrationContext.UniqueKey = &sql.UniqueKey{Name: "PRIMARY", NameInGhostTable: "PRIMARY", Columns: *uniqueKeyColumns} + _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s LIKE %s", getTestGhostTableName(), getTestTableName())) + require.NoError(t, err) + defer func() { + _, err := conn.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", getTestGhostTableName())) + require.NoError(t, err) + }() + + migrator := NewMigrator(migrationContext, "test") + migrator.applier = NewApplier(migrationContext) + require.NoError(t, migrator.applier.InitDBConnections()) + defer migrator.applier.Teardown() + migrator.applier.LastIterationRangeMinValues = sql.ToColumnValues([]interface{}{int64(1)}) + migrator.applier.LastIterationRangeMaxValues = sql.ToColumnValues([]interface{}{int64(1)}) + require.NoError(t, migrator.applier.prepareQueries()) + require.NoError(t, migrator.applier.CreateCheckpointTable()) + defer func() { require.NoError(t, migrator.applier.DropCheckpointTable()) }() + start, end := transaction.startFile.Clone(), transaction.endFile.Clone() + if useGTIDs { + start, end = transaction.startGTID.Clone(), transaction.endGTID.Clone() + } + seed := &Checkpoint{ + LastTrxCoords: start.Clone(), + IterationRangeMin: migrator.applier.LastIterationRangeMinValues.Clone(), + IterationRangeMax: migrator.applier.LastIterationRangeMaxValues.Clone(), + } + seedID, err := migrator.applier.WriteCheckpoint(ctx, seed) + require.NoError(t, err) + migrator.applier.AppliedTransactionCoordinates = start.Clone() + queueCheckpointTestEvents(t, ctx, migrationContext, migrator, start, end) + require.Len(t, migrator.applyEventsQueue, 4) + first := <-migrator.applyEventsQueue + require.NoError(t, migrator.onApplyEventStruct(first)) + require.Len(t, migrator.applyEventsQueue, 3, "two DMLs and the completion marker remain pending") + + assertCheckpointBlocked := func() { + checkpointCtx, cancelCheckpoint := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancelCheckpoint() + checkpoint, err := migrator.Checkpoint(checkpointCtx) + require.ErrorIs(t, err, context.DeadlineExceeded, "premature checkpoint: %+v", checkpoint) + stored, err := migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + require.Equal(t, seedID, stored.Id, "failed checkpoint must not replace the previous durable checkpoint") + require.True(t, stored.LastTrxCoords.Equals(start)) + } + assertCheckpointBlocked() + switch mode.cancelAt { + case "apply": + assertCanceledTransactionCheckpoint(t, ctx, migrationContext, migrator, seedID) + return + case "insert": + for len(migrator.applyEventsQueue) > 0 { + require.NoError(t, migrator.onApplyEventStruct(<-migrator.applyEventsQueue)) + } + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(end)) + assertCanceledCheckpointInsert(t, ctx, migrationContext, migrator, seedID) + return + } + + // Simulate a crash after the first applied row: discard volatile + // queue contents, then restart from the last persisted checkpoint. + for len(migrator.applyEventsQueue) > 0 { + <-migrator.applyEventsQueue + } + stored, err := migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + queueCheckpointTestEvents(t, ctx, migrationContext, migrator, stored.LastTrxCoords, end) + require.Len(t, migrator.applyEventsQueue, 4) + var transactionRows []*applyEventStruct + for len(migrator.applyEventsQueue) > 1 { + row := <-migrator.applyEventsQueue + transactionRows = append(transactionRows, row) + require.NoError(t, migrator.onApplyEventStruct(row)) + } + // Even the final row cannot independently acknowledge its commit. + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(start)) + if !useGTIDs && transaction.name == "OFF" { + // Sample a real row-event position P between the previous commit A + // and this transaction's XID B. A must block; B must be persisted + // rather than P once its marker is acknowledged. + sampled := first.coords.Clone() + require.True(t, start.SmallerThan(sampled)) + require.True(t, sampled.SmallerThan(end)) + reader := binlog.NewGoMySQLReader(migrationContext, nil) + defer reader.Close() + require.NoError(t, reader.ConnectBinlogStreamer(sampled)) + migrator.eventsStreamer.binlogReader = reader + } + assertCheckpointBlocked() + marker := <-migrator.applyEventsQueue + require.True(t, marker.transactionComplete) + require.NoError(t, migrator.onApplyEventStruct(marker)) + checkpoint, err := migrator.Checkpoint(ctx) + require.NoError(t, err) + require.True(t, checkpoint.LastTrxCoords.Equals(end)) + stored, err = migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + require.True(t, stored.LastTrxCoords.Equals(end)) + var remaining int + require.NoError(t, conn.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s", getTestGhostTableName())).Scan(&remaining)) + require.Zero(t, remaining, "resuming after the partial apply must not skip the DELETE") + + // Resume again from the completed checkpoint and apply later changes. + _, err = conn.ExecContext(ctx, "INSERT INTO testing VALUES (100, 12.34, '2023-11-14 22:13:20', 'after checkpoint') ON DUPLICATE KEY UPDATE amount=amount+1") + require.NoError(t, err) + resumeFile, resumeGTID := status() + resumeEnd := mysql.BinlogCoordinates(resumeFile) + if useGTIDs { + resumeEnd = resumeGTID + } + queueCheckpointTestEvents(t, ctx, migrationContext, migrator, stored.LastTrxCoords, resumeEnd) + for len(migrator.applyEventsQueue) > 0 { + require.NoError(t, migrator.onApplyEventStruct(<-migrator.applyEventsQueue)) + } + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(resumeEnd)) + var sourceRows, ghostRows string + query := "SELECT GROUP_CONCAT(CONCAT(id, ':', amount) ORDER BY id) FROM %s" + require.NoError(t, conn.QueryRowContext(ctx, fmt.Sprintf(query, getTestTableName())).Scan(&sourceRows)) + require.NoError(t, conn.QueryRowContext(ctx, fmt.Sprintf(query, getTestGhostTableName())).Scan(&ghostRows)) + require.Equal(t, sourceRows, ghostRows) + + // Batch across two source-transaction markers, ending inside the + // second transaction. Only the first may be acknowledged yet. + migrationContext.SetDMLBatchSize(4) + migrator.applier.AppliedTransactionCoordinates = start.Clone() + nextMarker := &applyEventStruct{coords: resumeEnd.Clone(), transactionComplete: true} + for _, event := range []*applyEventStruct{transactionRows[1], transactionRows[2], marker, transactionRows[0], transactionRows[1], transactionRows[2], nextMarker} { + migrator.applyEventsQueue <- event + } + appliedBefore := atomic.LoadInt64(&migrationContext.TotalDMLEventsApplied) + require.NoError(t, migrator.onApplyEventStruct(transactionRows[0])) + require.Equal(t, appliedBefore+4, atomic.LoadInt64(&migrationContext.TotalDMLEventsApplied)) + require.Len(t, migrator.applyEventsQueue, 3) + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(end)) + require.NoError(t, migrator.onApplyEventStruct(<-migrator.applyEventsQueue)) + require.Empty(t, migrator.applyEventsQueue) + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(resumeEnd)) + + // A marker collected while assembling a batch cannot acknowledge + // anything if the destination SQL transaction fails. + _, err = conn.ExecContext(ctx, fmt.Sprintf("DROP TABLE %s", getTestGhostTableName())) + require.NoError(t, err) + migrationContext.SetDefaultNumRetries(1) + migrationContext.PanicAbort = make(chan error, 1) + migrator.applier.AppliedTransactionCoordinates = end.Clone() + migrator.applyEventsQueue <- nextMarker + migrator.applyEventsQueue <- transactionRows[1] + require.Error(t, migrator.onApplyEventStruct(first)) + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(end)) + }) + } + } +} diff --git a/go/logic/checkpoint_cancellation_test.go b/go/logic/checkpoint_cancellation_test.go new file mode 100644 index 000000000..b5d719c8e --- /dev/null +++ b/go/logic/checkpoint_cancellation_test.go @@ -0,0 +1,191 @@ +package logic + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/binlog" + "github.com/github/gh-ost/go/metrics" + "github.com/github/gh-ost/go/sql" + drivermysql "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Signal when Checkpoint is waiting for its sampled position to be applied, +// without relying on sleeps to arrange the cancellation interleaving. +type checkpointWaitMetrics struct { + metrics.Emitter + waiting chan struct{} + once sync.Once +} + +func (m *checkpointWaitMetrics) Histogram(name string, _ float64, tags ...string) { + if name == "sleep.duration_milliseconds" && len(tags) == 1 && tags[0] == "stage:replica_wait" { + m.once.Do(func() { close(m.waiting) }) + } +} + +// The caller applied the INSERT and seeded an older durable checkpoint. The +// UPDATE, DELETE and completion marker are still pending. Replay their delivery +// under cancellation to exercise the failed-enqueue/marker interleaving. +// +//nolint:contextcheck // Migrator and dispatcher APIs use MigrationContext's internal context. +func assertCanceledTransactionCheckpoint(t *testing.T, ctx context.Context, migrationContext *base.MigrationContext, migrator *Migrator, seedID int64) { + t.Helper() + var pending []*applyEventStruct + for len(migrator.applyEventsQueue) > 0 { + pending = append(pending, <-migrator.applyEventsQueue) + } + require.Len(t, pending, 3) + marker := pending[2] + require.True(t, marker.transactionComplete) + previous := migrator.applier.AppliedTransactionCoordinates.Clone() + + wait := &checkpointWaitMetrics{Emitter: metrics.Noop, waiting: make(chan struct{})} + migrationContext.Metrics = wait + checkpointCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := migrator.Checkpoint(checkpointCtx) + _ = base.SendWithContext(ctx, done, err) + }() + select { + case <-wait.waiting: + case <-checkpointCtx.Done(): + t.Fatal("checkpoint never waited for the pending transaction") + } + + migrationContext.CancelContext() + for _, event := range pending[:2] { + err := migrator.eventsStreamer.notifyListeners(&binlog.BinlogEntry{Coordinates: event.coords, DmlEvent: event.dmlEvent}) + require.ErrorIs(t, err, context.Canceled) + } + require.ErrorIs(t, migrator.eventsStreamer.notifyListeners(&binlog.BinlogEntry{Coordinates: marker.coords, TransactionComplete: true}), context.Canceled) + require.Empty(t, migrator.applyEventsQueue) + // Also reject a marker that was already dequeued before cancellation, + // e.g. by a worker that passed its loop's abort check before throttling. + require.ErrorIs(t, migrator.onApplyEventStruct(marker), context.Canceled) + require.True(t, previous.Equals(migrator.applier.AppliedTransactionCoordinates)) + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-checkpointCtx.Done(): + t.Fatal("migration cancellation did not interrupt the pending checkpoint") + } + require.NoError(t, checkpointCtx.Err(), "checkpoint's caller has not canceled") + _, err := migrator.Checkpoint(checkpointCtx) + require.ErrorIs(t, err, context.Canceled, "new checkpoints must also reject an aborted migration") + stored, err := migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + require.Equal(t, seedID, stored.Id) + require.True(t, stored.LastTrxCoords.Equals(previous)) + var remaining int + require.NoError(t, migrator.applier.db.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s", getTestGhostTableName())).Scan(&remaining)) + require.Equal(t, 1, remaining, "the unapplied DELETE must still be replayed from the safe checkpoint") +} + +// Block the production checkpoint INSERT with a real MySQL table lock. Unlike +// cancellation during polling, this requires the migration cancellation bridge +// to reach an already-running ExecContext with an independent caller context. +func assertCanceledCheckpointInsert(t *testing.T, ctx context.Context, migrationContext *base.MigrationContext, migrator *Migrator, seedID int64) { + t.Helper() + previous, err := migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + require.Equal(t, seedID, previous.Id) + checkpointTable := fmt.Sprintf("%s.%s", sql.EscapeName(migrationContext.DatabaseName), sql.EscapeName(migrationContext.GetCheckpointTableName())) + blocker, err := migrator.applier.db.Conn(ctx) + require.NoError(t, err) + defer blocker.Close() + _, err = blocker.ExecContext(ctx, "LOCK TABLES "+checkpointTable+" READ") + require.NoError(t, err) + + checkpointCtx, cancelCheckpoint := context.WithTimeout(ctx, 30*time.Second) + var checkpointConnectionID int64 + done := make(chan error, 1) + workerDone := make(chan struct{}) + go func() { + defer close(workerDone) + _, err := migrator.Checkpoint(checkpointCtx) + _ = base.SendWithContext(ctx, done, err) + }() + defer func() { + // Release both waits on assertion failures, including mutation tests + // that remove cancellation propagation into the SQL operation. + cancelCheckpoint() + // Driver cancellation closes the client connection, but MySQL may + // retain a lock-blocked statement. Kill our test connection before + // unlocking so that cleanup cannot accidentally execute its INSERT. + if checkpointConnectionID != 0 { + _, err := migrator.applier.db.ExecContext(ctx, fmt.Sprintf("KILL CONNECTION %d", checkpointConnectionID)) + var mysqlErr *drivermysql.MySQLError + if !errors.As(err, &mysqlErr) || mysqlErr.Number != 1094 { // ER_NO_SUCH_THREAD: already disconnected + assert.NoError(t, err) + } + } + _, err := blocker.ExecContext(ctx, "UNLOCK TABLES") + assert.NoError(t, err) + select { + case <-workerDone: + case <-ctx.Done(): + t.Error("checkpoint worker did not exit during cleanup") + } + }() + + require.EventuallyWithT(t, func(collect *assert.CollectT) { + // Observing the server-side lock wait proves that Checkpoint passed + // its abort/polling checks and the INSERT is actually executing. + err := migrator.applier.db.QueryRowContext(ctx, ` + SELECT ID FROM information_schema.PROCESSLIST + WHERE DB = ? AND INFO LIKE '%insert /* gh-ost */%' + AND LOCATE(?, INFO) > 0 AND STATE = 'Waiting for table metadata lock'`, + migrationContext.DatabaseName, checkpointTable).Scan(&checkpointConnectionID) + assert.NoError(collect, err) + }, 5*time.Second, 10*time.Millisecond, "checkpoint INSERT never entered the table-lock wait") + select { + case err := <-done: + t.Fatalf("checkpoint returned before cancellation: %v", err) + default: + } + + migrationContext.CancelContext() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("migration cancellation did not interrupt the blocked checkpoint INSERT") + } + require.NoError(t, checkpointCtx.Err(), "checkpoint's caller must remain live") + + // READ permits this SELECT while still preventing the pending INSERT. + // Check before cleanup: neither the lock nor the caller has been canceled + // to make the checkpoint call return. + stored, err := migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + require.Equal(t, previous.Id, stored.Id) + require.True(t, previous.LastTrxCoords.Equals(stored.LastTrxCoords)) +} + +func TestCheckpointLoopStopsOnCancellation(t *testing.T) { + migrationContext := newTestMigrationContext() + defer migrationContext.CancelContext() + migrationContext.CheckpointIntervalSeconds = 3600 + migrator := NewMigrator(migrationContext, "test") + done := make(chan struct{}) + go func() { + defer close(done) + migrator.checkpointLoop() + }() + migrationContext.CancelContext() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("checkpoint loop waited for its ticker after cancellation") + } +} diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 2e72149e6..44152d3ed 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -56,10 +56,11 @@ type lockProcessedStruct struct { } type applyEventStruct struct { - writeFunc *tableWriteFunc - dmlEvent *binlog.BinlogDMLEvent - coords mysql.BinlogCoordinates - eventTimestamp time.Time + writeFunc *tableWriteFunc + dmlEvent *binlog.BinlogDMLEvent + coords mysql.BinlogCoordinates + eventTimestamp time.Time + transactionComplete bool } func newApplyEventStructByFunc(writeFunc *tableWriteFunc) *applyEventStruct { @@ -2578,6 +2579,13 @@ func (mgtr *Migrator) initiateStreaming() error { if err := mgtr.eventsStreamer.InitDBConnections(); err != nil { return err } + // The starting position is a safe boundary before any streamed DML. In + // move-tables mode there is no heartbeat to advance it on a quiet source. + if mgtr.migrationContext.IsMoveTablesMode() { + mgtr.applier.CurrentCoordinatesMutex.Lock() + mgtr.applier.AppliedTransactionCoordinates = mgtr.eventsStreamer.initialBinlogCoordinates.Clone() + mgtr.applier.CurrentCoordinatesMutex.Unlock() + } if mgtr.migrationContext.IsMoveTablesMode() { mgtr.migrationContext.Log.Info("Skipping stream of the changelog table") @@ -2640,6 +2648,8 @@ func (mgtr *Migrator) initiateStreaming() error { // by table name. The streamer already dispatches per (database, table), and the // applier routes DML to the right per-table query builders by table name. func (mgtr *Migrator) addDMLEventsListener() error { + // Register before the DML listeners to preserve row/commit order. + mgtr.eventsStreamer.AddTransactionCompleteListener(mgtr.onTransactionComplete) enqueue := func(dmlEntry *binlog.BinlogEntry) error { // Record that the streamer just delivered an event for a moved table, so // the idle-bump rule can tell "falling behind" from "source is quiet". @@ -2673,6 +2683,14 @@ func (mgtr *Migrator) addDMLEventsListener() error { ) } +// onTransactionComplete enqueues an acknowledgement, not another SQL write. +// The applier may batch across these markers, but cannot acknowledge them +// until the batch containing their preceding DML has successfully committed. +func (mgtr *Migrator) onTransactionComplete(entry *binlog.BinlogEntry) error { + marker := &applyEventStruct{coords: entry.Coordinates.Clone(), transactionComplete: true} + return base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.applyEventsQueue, marker) +} + // initiateThrottler kicks in the throttling collection and the throttling checks. func (mgtr *Migrator) initiateThrottler() { mgtr.throttler = NewThrottler(mgtr.migrationContext, mgtr.applier, mgtr.inspector, mgtr.appVersion) @@ -3017,7 +3035,17 @@ func (mgtr *Migrator) iterateChunksMoveTables() error { func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { atomic.AddInt64(&mgtr.applyEventsInFlight, 1) defer atomic.AddInt64(&mgtr.applyEventsInFlight, -1) + // Cancellation may happen after the worker's loop check, while throttled. + // A standalone marker must not bypass the abort checks used by SQL tasks. + if err := mgtr.checkAbort(); err != nil { + return err + } handleNonDMLEventStruct := func(eventStruct *applyEventStruct) error { + if eventStruct.transactionComplete { + mgtr.applier.CurrentCoordinatesMutex.Lock() + mgtr.applier.AppliedTransactionCoordinates = eventStruct.coords.Clone() + mgtr.applier.CurrentCoordinatesMutex.Unlock() + } if eventStruct.writeFunc != nil { if err := mgtr.retryOperation(*eventStruct.writeFunc); err != nil { return mgtr.migrationContext.Log.Errore(err) @@ -3033,16 +3061,19 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { dmlEvents = append(dmlEvents, eventStruct.dmlEvent) lastEventTimestamp := eventStruct.eventTimestamp var nonDmlStructToApply *applyEventStruct + var completedTransaction *applyEventStruct availableEvents := len(mgtr.applyEventsQueue) batchSize := int(atomic.LoadInt64(&mgtr.migrationContext.DMLBatchSize)) - if availableEvents > batchSize-1 { - // The "- 1" is because we already consumed one event: the original event that led to this function getting called. - // So, if DMLBatchSize==1 we wish to not process any further events - availableEvents = batchSize - 1 - } - for i := 0; i < availableEvents; i++ { + for i := 0; i < availableEvents && len(dmlEvents) < batchSize; i++ { additionalStruct := <-mgtr.applyEventsQueue + if additionalStruct.transactionComplete { + // Markers need no SQL and don't count toward DMLBatchSize. + // Acknowledge only after this entire batch succeeds, never while + // merely collecting rows from the queue. + completedTransaction = additionalStruct + continue + } if additionalStruct.dmlEvent == nil { // Not a DML. We don't group this, and we don't batch any further nonDmlStructToApply = additionalStruct @@ -3061,6 +3092,9 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { // update applier coordinates mgtr.applier.CurrentCoordinatesMutex.Lock() mgtr.applier.CurrentCoordinates = eventStruct.coords + if completedTransaction != nil { + mgtr.applier.AppliedTransactionCoordinates = completedTransaction.coords.Clone() + } mgtr.applier.CurrentCoordinatesMutex.Unlock() // In move-tables mode there is no changelog heartbeat; writer lag is derived @@ -3081,13 +3115,24 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { } // Checkpoint attempts to write a checkpoint of the Migrator's current state. -// It gets the binlog coordinates of the last received trx and waits until the -// applier reaches that trx. At that point it's safe to resume from these coordinates. +// It samples reader progress and waits for an applied transaction boundary +// covering that progress. Row/batch progress alone is not a safe resume point. func (mgtr *Migrator) Checkpoint(ctx context.Context) (*Checkpoint, error) { - coords := mgtr.eventsStreamer.GetCurrentBinlogCoordinates() + if err := mgtr.checkAbort(); err != nil { + return nil, err + } + // Observe both the caller's deadline and migration cancellation, including + // while executing the checkpoint INSERT rather than only while polling. + ctx, cancel := context.WithCancel(ctx) + //nolint:contextcheck // Merge migration cancellation with the caller-derived checkpoint context. + stopCancel := context.AfterFunc(mgtr.migrationContext.GetContext(), cancel) + defer stopCancel() + defer cancel() + if mgtr.migrationContext.IsMoveTablesMode() { - return mgtr.checkpointMoveTables(ctx, coords) + return mgtr.checkpointMoveTables(ctx) } + coords := mgtr.eventsStreamer.GetCurrentBinlogCoordinates() mgtr.applier.LastIterationRangeMutex.Lock() if mgtr.applier.LastIterationRangeMaxValues == nil || mgtr.applier.LastIterationRangeMinValues == nil { mgtr.applier.LastIterationRangeMutex.Unlock() @@ -3097,25 +3142,37 @@ func (mgtr *Migrator) Checkpoint(ctx context.Context) (*Checkpoint, error) { Iteration: mgtr.migrationContext.GetIteration(), IterationRangeMin: mgtr.applier.LastIterationRangeMinValues.Clone(), IterationRangeMax: mgtr.applier.LastIterationRangeMaxValues.Clone(), - LastTrxCoords: coords, RowsCopied: atomic.LoadInt64(&mgtr.migrationContext.TotalRowsCopied), DMLApplied: atomic.LoadInt64(&mgtr.migrationContext.TotalDMLEventsApplied), } mgtr.applier.LastIterationRangeMutex.Unlock() for { + if err := mgtr.checkAbort(); err != nil { + return nil, err + } if err := ctx.Err(); err != nil { return nil, err } mgtr.applier.CurrentCoordinatesMutex.Lock() - if coords.SmallerThanOrEquals(mgtr.applier.CurrentCoordinates) { - id, err := mgtr.applier.WriteCheckpoint(chk) + applied := mgtr.applier.AppliedTransactionCoordinates + if applied != nil && coords.SmallerThanOrEquals(applied) { + // Persist the acknowledged commit boundary, not the sampled reader + // position, which may be inside a row event or transaction. + chk.LastTrxCoords = applied.Clone() + id, err := mgtr.applier.WriteCheckpoint(ctx, chk) chk.Id = id mgtr.applier.CurrentCoordinatesMutex.Unlock() return chk, err } mgtr.applier.CurrentCoordinatesMutex.Unlock() - time.Sleep(500 * time.Millisecond) + sleepDuration := 500 * time.Millisecond + metrics.RecordSleep(mgtr.migrationContext.Metrics, "replica_wait", sleepDuration) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(sleepDuration): + } } } @@ -3157,33 +3214,28 @@ func (mgtr *Migrator) moveTablesCheckpointSummary(coords mysql.BinlogCoordinates } } -// checkpointMoveTables writes one checkpoint row per migrated table based on -// applier coordinates or idle queues. Neither proves a completed transaction; -// see #1781. -func (mgtr *Migrator) checkpointMoveTables(ctx context.Context, coords mysql.BinlogCoordinates) (*Checkpoint, error) { - for { - if err := ctx.Err(); err != nil { - return nil, err - } - mgtr.applier.CurrentCoordinatesMutex.Lock() - applied := coords.SmallerThanOrEquals(mgtr.applier.CurrentCoordinates) - idle := len(mgtr.applyEventsQueue) == 0 && (mgtr.eventsStreamer == nil || len(mgtr.eventsStreamer.eventsChannel) == 0) - if applied || idle { - if !applied { - mgtr.applier.CurrentCoordinates = coords.Clone() - } - mgtr.applier.CurrentCoordinatesMutex.Unlock() - rows := mgtr.buildMoveTableCheckpointRows(coords, false, false, nil) - if err := mgtr.applier.WriteMoveTableCheckpoints(rows); err != nil { - return nil, err - } - return mgtr.moveTablesCheckpointSummary(coords), nil - } - mgtr.applier.CurrentCoordinatesMutex.Unlock() - sleepDuration := 500 * time.Millisecond - metrics.RecordSleep(mgtr.migrationContext.Metrics, "replica_wait", sleepDuration) - time.Sleep(sleepDuration) +// checkpointMoveTables persists the last applied transaction boundary. Unlike +// standard mode, there is no source heartbeat to advance the reader on a quiet +// source, so a safe older boundary is preferable to waiting for reader progress. +func (mgtr *Migrator) checkpointMoveTables(ctx context.Context) (*Checkpoint, error) { + if err := mgtr.checkAbort(); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + mgtr.applier.CurrentCoordinatesMutex.Lock() + defer mgtr.applier.CurrentCoordinatesMutex.Unlock() + applied := mgtr.applier.AppliedTransactionCoordinates + if applied == nil { + return nil, errors.New("no applied transaction boundary, not checkpointing") } + coords := applied.Clone() + rows := mgtr.buildMoveTableCheckpointRows(coords, false, false, nil) + if err := mgtr.applier.writeMoveTableCheckpoints(ctx, rows); err != nil { + return nil, err + } + return mgtr.moveTablesCheckpointSummary(coords), nil } // CheckpointAfterCutOver writes a final checkpoint after the cutover completes successfully. @@ -3210,7 +3262,7 @@ func (mgtr *Migrator) CheckpointAfterCutOver() (*Checkpoint, error) { } mgtr.applier.LastIterationRangeMutex.Unlock() - id, err := mgtr.applier.WriteCheckpoint(chk) + id, err := mgtr.applier.WriteCheckpoint(mgtr.migrationContext.GetContext(), chk) chk.Id = id return chk, err } @@ -3222,7 +3274,15 @@ func (mgtr *Migrator) checkpointLoop() { } checkpointInterval := time.Duration(mgtr.migrationContext.CheckpointIntervalSeconds) * time.Second ticker := time.NewTicker(checkpointInterval) - for t := range ticker.C { + defer ticker.Stop() + migrationCtx := mgtr.migrationContext.GetContext() + for { + var t time.Time + select { + case <-migrationCtx.Done(): + return + case t = <-ticker.C: + } if atomic.LoadInt64(&mgtr.finishedMigrating) > 0 || atomic.LoadInt64(&mgtr.migrationContext.CutOverCompleteFlag) > 0 { return } @@ -3230,7 +3290,7 @@ func (mgtr *Migrator) checkpointLoop() { continue } mgtr.migrationContext.Log.Infof("starting checkpoint at %+v", t) - ctx, cancel := context.WithTimeout(context.Background(), checkpointTimeout) + ctx, cancel := context.WithTimeout(migrationCtx, checkpointTimeout) chk, err := mgtr.Checkpoint(ctx) if err != nil { if errors.Is(err, context.DeadlineExceeded) { diff --git a/go/logic/migrator_move_tables_cutover_test.go b/go/logic/migrator_move_tables_cutover_test.go index d45c0d708..3ee0dd4dd 100644 --- a/go/logic/migrator_move_tables_cutover_test.go +++ b/go/logic/migrator_move_tables_cutover_test.go @@ -352,6 +352,38 @@ func (s *MoveTablesCutOverSuite) setupCutOverCheckpoint(m *Migrator, mc *base.Mi } } +func (s *MoveTablesCutOverSuite) TestMoveTablesCheckpointUsesAppliedTransactionBoundary() { + start, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1") + s.Require().NoError(err) + end, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-2") + s.Require().NoError(err) + m, mc := s.buildMigrator(nil, end) + s.setupCutOverCheckpoint(m, mc, false) + m.applier.AppliedTransactionCoordinates = start.Clone() + + // Row progress may already be at end, but without its commit marker only + // the earlier boundary is safe to persist and replay from. + chk, err := m.Checkpoint(context.Background()) + s.Require().NoError(err) + s.Require().True(chk.LastTrxCoords.Equals(start)) + rows, err := m.applier.ReadMoveTableCheckpoints() + s.Require().NoError(err) + s.Require().True(rows[testMysqlTableName].LastTrxCoords.Equals(start)) + + s.Require().NoError(m.onApplyEventStruct(&applyEventStruct{coords: end, transactionComplete: true})) + chk, err = m.Checkpoint(context.Background()) + s.Require().NoError(err) + s.Require().True(chk.LastTrxCoords.Equals(end)) + rows, err = m.applier.ReadMoveTableCheckpoints() + s.Require().NoError(err) + s.Require().True(rows[testMysqlTableName].LastTrxCoords.Equals(end)) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = m.Checkpoint(ctx) + s.Require().ErrorIs(err, context.Canceled) +} + func (s *MoveTablesCutOverSuite) TestMoveTablesListenersBeforeStreaming() { ctx := context.Background() tableNames := []string{testMysqlTableName, "testing_other"} @@ -380,6 +412,7 @@ func (s *MoveTablesCutOverSuite) TestMoveTablesListenersBeforeStreaming() { mc.MoveTables.TableNames = tableNames mc.UseGTIDs = true m := NewMigrator(mc, "test") + m.applier = NewApplier(mc) s.Require().NoError(m.initiateStreaming()) s.T().Cleanup(func() { atomic.StoreInt64(&m.finishedMigrating, 1) @@ -401,11 +434,15 @@ func (s *MoveTablesCutOverSuite) TestMoveTablesListenersBeforeStreaming() { seen := make(map[string]int) deadline := time.NewTimer(10 * time.Second) defer deadline.Stop() - for range len(tableNames) * rowsPerTable { + for rows := 0; rows < len(tableNames)*rowsPerTable; { select { case event := <-m.applyEventsQueue: + if event.transactionComplete { + continue + } s.Require().NotNil(event.dmlEvent) seen[event.dmlEvent.TableName]++ + rows++ case <-deadline.C: s.FailNow("timed out waiting for pending DML") } diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 7d3d00120..355c882be 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -40,6 +40,7 @@ type EventsStreamer struct { migrationContext *base.MigrationContext initialBinlogCoordinates mysql.BinlogCoordinates listeners [](*BinlogEventListener) + transactionListeners []func(*binlog.BinlogEntry) error listenersMutex *sync.Mutex eventsChannel chan *binlog.BinlogEntry binlogReader *binlog.GoMySQLReader @@ -80,6 +81,14 @@ func (es *EventsStreamer) AddListener( return nil } +// AddTransactionCompleteListener registers a synchronous commit listener. +// Consumers must also use synchronous DML listeners to preserve queue order. +func (es *EventsStreamer) AddTransactionCompleteListener(listener func(*binlog.BinlogEntry) error) { + es.listenersMutex.Lock() + defer es.listenersMutex.Unlock() + es.transactionListeners = append(es.transactionListeners, listener) +} + // shouldDecodeRowsEvent returns true when at least one listener is registered for // the table on which the rows event operates. This is used by the binlog parser // after it decodes the row event header/table map, but before it decodes row data. @@ -99,11 +108,34 @@ func (es *EventsStreamer) shouldDecodeRowsEvent(databaseName, tableName string) return false } -// notifyListeners will notify relevant listeners with given DML event. Only -// listeners registered for changes on the table on which the DML operates are notified. -func (es *EventsStreamer) notifyListeners(binlogEntry *binlog.BinlogEntry) { +// notifyListeners dispatches table-filtered DML or a transaction completion +// marker. Synchronous listeners run on the same dispatcher to preserve order. +// Failed delivery is fatal: continuing could acknowledge a transaction whose +// preceding DML never reached the apply queue. +func (es *EventsStreamer) notifyListeners(binlogEntry *binlog.BinlogEntry) (err error) { es.listenersMutex.Lock() defer es.listenersMutex.Unlock() + defer func() { + if err != nil { + es.migrationContext.SetAbortError(err) + es.migrationContext.CancelContext() + } + }() + if err := es.migrationContext.GetAbortError(); err != nil { + return err + } + if err := es.migrationContext.GetContext().Err(); err != nil { + return err + } + + if binlogEntry.TransactionComplete { + for _, listener := range es.transactionListeners { + if err := listener(binlogEntry); err != nil { + return err + } + } + return nil + } for _, listener := range es.listeners { listener := listener @@ -117,10 +149,11 @@ func (es *EventsStreamer) notifyListeners(binlogEntry *binlog.BinlogEntry) { go func() { listener.onDmlEvent(binlogEntry) }() - } else { - listener.onDmlEvent(binlogEntry) + } else if err := listener.onDmlEvent(binlogEntry); err != nil { + return err } } + return nil } func (es *EventsStreamer) InitDBConnections() (err error) { @@ -216,19 +249,32 @@ func (es *EventsStreamer) readCurrentMariaDBGTIDCoordinates() error { // StreamEvents will begin streaming events. It will be blocking, so should be // executed by a goroutine func (es *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { + ctx := es.migrationContext.GetContext() go func() { - for binlogEntry := range es.eventsChannel { - if binlogEntry.DmlEvent != nil { - es.notifyListeners(binlogEntry) + for { + select { + case <-ctx.Done(): + return + case binlogEntry, ok := <-es.eventsChannel: + if !ok { + return + } + if binlogEntry.DmlEvent != nil || binlogEntry.TransactionComplete { + if err := es.notifyListeners(binlogEntry); err != nil { + return + } + } } } }() // The next should block and execute forever, unless there's a serious error. var successiveFailures int var reconnectCoords mysql.BinlogCoordinates - ctx := es.migrationContext.GetContext() for { - // Check for context cancellation each iteration + // Preserve a listener's failure rather than hiding it as cancellation. + if err := es.migrationContext.GetAbortError(); err != nil { + return err + } if err := ctx.Err(); err != nil { return err } @@ -239,6 +285,12 @@ func (es *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { // of the last trx that was read completely from the streamer. // Since row event application is idempotent, it's OK if we reapply some events. if err := es.binlogReader.StreamEvents(canStopStreaming, es.eventsChannel); err != nil { + if abortErr := es.migrationContext.GetAbortError(); abortErr != nil { + return abortErr + } + if ctx.Err() != nil { + return ctx.Err() + } if canStopStreaming() { return nil } diff --git a/go/logic/streamer_test.go b/go/logic/streamer_test.go index 5b28c47c2..4871e2365 100644 --- a/go/logic/streamer_test.go +++ b/go/logic/streamer_test.go @@ -3,10 +3,12 @@ package logic import ( "context" gosql "database/sql" + "errors" "fmt" "testing" "time" + "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/binlog" "github.com/stretchr/testify/suite" "github.com/testcontainers/testcontainers-go" @@ -140,6 +142,49 @@ func (suite *EventsStreamerTestSuite) TestStreamEvents() { suite.Require().Len(dmlEvents, 3) } +func (suite *EventsStreamerTestSuite) TestStreamEventsStopsOnListenerError() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestTableName())) + suite.Require().NoError(err) + config, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + migrationContext := newTestMigrationContext() + defer migrationContext.CancelContext() + migrationContext.ApplierConnectionConfig = config + migrationContext.InspectorConnectionConfig = config + suite.Require().NoError(migrationContext.SetConnectionConfig("innodb")) + streamer := NewEventsStreamer(migrationContext) + suite.Require().NoError(streamer.InitDBConnections()) + defer streamer.Close() + defer streamer.Teardown() + failure := errors.New("DML enqueue failed") + suite.Require().NoError(streamer.AddListener(false, testMysqlDatabase, testMysqlTableName, func(*binlog.BinlogEntry) error { + return failure + })) + streamer.AddTransactionCompleteListener(func(*binlog.BinlogEntry) error { + suite.T().Error("marker delivered after failed DML") + return nil + }) + done := make(chan error, 1) + go func() { + //nolint:contextcheck // StreamEvents uses the migration context, canceled by listener failure. + err := streamer.StreamEvents(func() bool { return false }) + _ = base.SendWithContext(ctx, done, err) + }() + // Multiple rows also exercise cancellation of reader sends after the + // dispatcher exits, rather than only cancellation of GetEvent. + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s VALUES (1),(2),(3),(4),(5)", getTestTableName())) + suite.Require().NoError(err) + select { + case err := <-done: + suite.Require().ErrorIs(err, failure) + case <-ctx.Done(): + suite.T().Fatal("streamer did not stop after listener failure") + } + suite.Require().ErrorIs(migrationContext.GetAbortError(), failure) +} + func (suite *EventsStreamerTestSuite) TestStreamEventsAutomaticallyReconnects() { ctx := context.Background() _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY, name VARCHAR(255))", getTestTableName())) diff --git a/go/logic/transaction_marker_test.go b/go/logic/transaction_marker_test.go new file mode 100644 index 000000000..4a0339ba0 --- /dev/null +++ b/go/logic/transaction_marker_test.go @@ -0,0 +1,184 @@ +package logic + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/github/gh-ost/go/binlog" + "github.com/github/gh-ost/go/mysql" + "github.com/github/gh-ost/go/sql" + "github.com/stretchr/testify/require" +) + +func TestTransactionMarkersFollowSynchronousDMLListeners(t *testing.T) { + ctx := newTestMigrationContext() + streamer := NewEventsStreamer(ctx) + migrator := NewMigrator(ctx, "test") + migrator.eventsStreamer = streamer + migrator.applier = NewApplier(ctx) + require.NoError(t, migrator.addDMLEventsListener()) + coords := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 1234) + for i := 0; i < 3; i++ { + require.NoError(t, streamer.notifyListeners(&binlog.BinlogEntry{ + Coordinates: coords, + DmlEvent: binlog.NewBinlogDMLEvent(ctx.DatabaseName, ctx.OriginalTableName, binlog.InsertDML), + })) + } + require.NoError(t, streamer.notifyListeners(&binlog.BinlogEntry{Coordinates: coords, TransactionComplete: true})) + require.Len(t, migrator.applyEventsQueue, 4) + require.Nil(t, migrator.applier.AppliedTransactionCoordinates) + for i := 0; i < 3; i++ { + event := <-migrator.applyEventsQueue + require.NotNil(t, event.dmlEvent) + require.False(t, event.transactionComplete) + } + marker := <-migrator.applyEventsQueue + require.True(t, marker.transactionComplete) + require.Nil(t, marker.dmlEvent) +} + +func TestMoveTablesTransactionMarkerFollowsAllTableDML(t *testing.T) { + ctx := newTestMigrationContext() + defer ctx.CancelContext() + ctx.MoveTables.TableNames = []string{"first", "second"} + migrator := NewMigrator(ctx, "test") + migrator.applier = NewApplier(ctx) + migrator.eventsStreamer = NewEventsStreamer(ctx) + require.NoError(t, migrator.addDMLEventsListener()) + coords := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 1234) + timestamp := time.Now() + for _, table := range ctx.MoveTables.TableNames { + require.NoError(t, migrator.eventsStreamer.notifyListeners(&binlog.BinlogEntry{ + Coordinates: coords, + Timestamp: timestamp, + DmlEvent: binlog.NewBinlogDMLEvent(ctx.DatabaseName, table, binlog.InsertDML), + })) + } + require.NoError(t, migrator.eventsStreamer.notifyListeners(&binlog.BinlogEntry{Coordinates: coords, TransactionComplete: true})) + for _, table := range ctx.MoveTables.TableNames { + event := <-migrator.applyEventsQueue + require.Equal(t, table, event.dmlEvent.TableName) + require.Equal(t, timestamp, event.eventTimestamp) + } + marker := <-migrator.applyEventsQueue + require.True(t, marker.transactionComplete) + require.NoError(t, migrator.onApplyEventStruct(marker)) + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(coords)) +} + +func TestTransactionMarkerRejectedAfterAbort(t *testing.T) { + for _, abortErr := range []error{context.Canceled, errors.New("migration aborted")} { + t.Run(abortErr.Error(), func(t *testing.T) { + ctx := newTestMigrationContext() + defer ctx.CancelContext() + migrator := NewMigrator(ctx, "test") + migrator.applier = NewApplier(ctx) + previous := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 100) + migrator.applier.AppliedTransactionCoordinates = previous.Clone() + marker := &applyEventStruct{coords: mysql.NewFileBinlogCoordinates("mysql-bin.000001", 200), transactionComplete: true} + if !errors.Is(abortErr, context.Canceled) { + ctx.SetAbortError(abortErr) + } + ctx.CancelContext() + require.ErrorIs(t, migrator.onApplyEventStruct(marker), abortErr) + require.True(t, previous.Equals(migrator.applier.AppliedTransactionCoordinates)) + }) + } +} + +func TestFailedSynchronousListenerStopsTransactionDelivery(t *testing.T) { + for _, failMarker := range []bool{false, true} { + t.Run(fmt.Sprintf("marker=%t", failMarker), func(t *testing.T) { + ctx := newTestMigrationContext() + defer ctx.CancelContext() + streamer := NewEventsStreamer(ctx) + failure := errors.New("listener rejected event") + called := 0 + fail := func(*binlog.BinlogEntry) error { called++; return failure } + unexpected := func(*binlog.BinlogEntry) error { t.Error("delivered after listener failure"); return nil } + entry := &binlog.BinlogEntry{TransactionComplete: failMarker} + if failMarker { + streamer.AddTransactionCompleteListener(fail) + } else { + entry.DmlEvent = binlog.NewBinlogDMLEvent(ctx.DatabaseName, ctx.OriginalTableName, binlog.DeleteDML) + require.NoError(t, streamer.AddListener(false, ctx.DatabaseName, ctx.OriginalTableName, fail)) + require.NoError(t, streamer.AddListener(false, ctx.DatabaseName, ctx.OriginalTableName, unexpected)) + } + streamer.AddTransactionCompleteListener(unexpected) + require.ErrorIs(t, streamer.notifyListeners(entry), failure) + require.ErrorIs(t, ctx.GetAbortError(), failure) + require.ErrorIs(t, ctx.GetContext().Err(), context.Canceled) + require.ErrorIs(t, streamer.notifyListeners(&binlog.BinlogEntry{TransactionComplete: true}), failure) + require.Equal(t, 1, called) + }) + } +} + +func TestCanceledDispatcherRejectsReadyQueue(t *testing.T) { + ctx := newTestMigrationContext() + migrator := NewMigrator(ctx, "test") + migrator.eventsStreamer = NewEventsStreamer(ctx) + require.NoError(t, migrator.addDMLEventsListener()) + ctx.CancelContext() + coords := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 100) + for _, entry := range []*binlog.BinlogEntry{ + {Coordinates: coords, DmlEvent: binlog.NewBinlogDMLEvent(ctx.DatabaseName, ctx.OriginalTableName, binlog.DeleteDML)}, + {Coordinates: coords, TransactionComplete: true}, + } { + require.ErrorIs(t, migrator.eventsStreamer.notifyListeners(entry), context.Canceled) + } + require.Empty(t, migrator.applyEventsQueue) +} + +func TestTransactionMarkerClonesGTIDCoordinates(t *testing.T) { + for _, test := range []struct { + flavor string + committed string + next string + }{ + {mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1", "00000000-0000-0000-0000-000000000001:2"}, + {mysql.MariaDBFlavor, "0-1-1", "0-1-2"}, + } { + t.Run(test.flavor, func(t *testing.T) { + ctx := newTestMigrationContext() + ctx.UseGTIDs = true + migrator := NewMigrator(ctx, "test") + migrator.applier = NewApplier(ctx) + coords, err := mysql.NewGTIDBinlogCoordinates(test.flavor, test.committed) + require.NoError(t, err) + require.NoError(t, migrator.onTransactionComplete(&binlog.BinlogEntry{Coordinates: coords, TransactionComplete: true})) + require.Nil(t, migrator.applier.AppliedTransactionCoordinates, "enqueueing is not applying") + marker := <-migrator.applyEventsQueue + require.NoError(t, coords.GTIDSet.Update(test.next)) + require.NoError(t, migrator.onApplyEventStruct(marker)) + require.Equal(t, test.committed, migrator.applier.AppliedTransactionCoordinates.String()) + // The acknowledged watermark also owns its own copy of the marker. + require.NoError(t, marker.coords.(*mysql.GTIDBinlogCoordinates).GTIDSet.Update(test.next)) + require.Equal(t, test.committed, migrator.applier.AppliedTransactionCoordinates.String()) + }) + } +} + +func TestHeartbeatDoesNotAcknowledgeItsTransaction(t *testing.T) { + ctx := newTestMigrationContext() + migrator := NewMigrator(ctx, "test") + migrator.applier = NewApplier(ctx) + coords := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 1234) + heartbeat := &binlog.BinlogEntry{ + Coordinates: coords, + DmlEvent: &binlog.BinlogDMLEvent{ + DML: binlog.InsertDML, + NewColumnValues: sql.ToColumnValues([]interface{}{1, 0, "heartbeat", time.Now().Format(time.RFC3339Nano)}), + }, + } + require.NoError(t, migrator.onChangelogHeartbeatEvent(heartbeat)) + require.NoError(t, migrator.onApplyEventStruct(<-migrator.applyEventsQueue)) + require.True(t, migrator.applier.CurrentCoordinates.Equals(coords)) + require.Nil(t, migrator.applier.AppliedTransactionCoordinates, "a heartbeat row may precede more rows in the same transaction") + require.NoError(t, migrator.onTransactionComplete(&binlog.BinlogEntry{Coordinates: coords, TransactionComplete: true})) + require.NoError(t, migrator.onApplyEventStruct(<-migrator.applyEventsQueue)) + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(coords)) +} From ef26a3a1d4da721771c52a73c8b0fc78b0a5f5a6 Mon Sep 17 00:00:00 2001 From: Bastian Bartmann Date: Fri, 20 Nov 2026 09:48:50 +0100 Subject: [PATCH 2/2] Fix checkpoint review findings Use applied transaction boundaries for cutover recovery, keep recovery writes independent of migration cancellation, and batch checkpoint rows. Advance checkpoints for filtered transactions, avoid blocking apply during checkpoint writes, and return listener failures during startup. --- go/binlog/cancellation_test.go | 30 +-- go/binlog/filtered_transaction_test.go | 121 +++++++++++ go/binlog/gomysql_reader.go | 17 +- go/logic/applier.go | 25 ++- go/logic/applier_checkpoint_test.go | 191 ++++++++++++++++++ go/logic/checkpoint_review_test.go | 178 ++++++++++++++++ go/logic/migrator.go | 75 ++++--- go/logic/migrator_move_tables_cutover_test.go | 6 +- 8 files changed, 593 insertions(+), 50 deletions(-) create mode 100644 go/binlog/filtered_transaction_test.go create mode 100644 go/logic/applier_checkpoint_test.go create mode 100644 go/logic/checkpoint_review_test.go diff --git a/go/binlog/cancellation_test.go b/go/binlog/cancellation_test.go index c4c83a4e4..1b0c90027 100644 --- a/go/binlog/cancellation_test.go +++ b/go/binlog/cancellation_test.go @@ -12,10 +12,15 @@ import ( ) func TestTransactionMarkerSendCanceled(t *testing.T) { - assertTransactionMarkerSendCanceled(t, nil) + t.Run("decoded rows", func(t *testing.T) { + assertTransactionMarkerSendCanceled(t, [][]interface{}{{1}}) + }) + t.Run("filtered rows", func(t *testing.T) { + assertTransactionMarkerSendCanceled(t, nil) + }) } -func assertTransactionMarkerSendCanceled(t *testing.T, wrapEvents func([]*replication.BinlogEvent) []*replication.BinlogEvent) { +func assertTransactionMarkerSendCanceled(t *testing.T, rows [][]interface{}) { t.Helper() reader := transactionTestReader(false) defer reader.migrationContext.CancelContext() @@ -25,18 +30,14 @@ func assertTransactionMarkerSendCanceled(t *testing.T, wrapEvents func([]*replic Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 200}, Event: &replication.RowsEvent{ Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")}, - Rows: [][]interface{}{{1}}, + Rows: rows, }, } commit := &replication.BinlogEvent{ Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: 220}, Event: &replication.XIDEvent{}, } - events := []*replication.BinlogEvent{row, commit} - if wrapEvents != nil { - events = wrapEvents(events) - } - for _, event := range events { + for _, event := range []*replication.BinlogEvent{row, commit} { require.NoError(t, reader.binlogStreamer.AddEventToStreamer(event)) } entries := make(chan *BinlogEntry) @@ -45,11 +46,13 @@ func assertTransactionMarkerSendCanceled(t *testing.T, wrapEvents func([]*replic err := reader.StreamEvents(func() bool { return false }, entries) _ = base.SendWithContext(t.Context(), done, err) }() - select { - case entry := <-entries: - require.NotNil(t, entry.DmlEvent) - case <-time.After(time.Second): - t.Fatal("row was not delivered") + if len(rows) > 0 { + select { + case entry := <-entries: + require.NotNil(t, entry.DmlEvent) + case <-time.After(time.Second): + t.Fatal("row was not delivered") + } } // Wait until the commit position has been read, then stop consuming. // The reader must propagate a canceled completion-marker send. @@ -63,7 +66,6 @@ func assertTransactionMarkerSendCanceled(t *testing.T, wrapEvents func([]*replic case <-time.After(time.Second): t.Fatal("completion marker send ignored cancellation") } - require.True(t, reader.transactionHasRows, "failed marker delivery must retain pending state") require.True(t, previous.Equals(reader.LastTrxCoords), "failed delivery must not advance reconnect coordinates") } diff --git a/go/binlog/filtered_transaction_test.go b/go/binlog/filtered_transaction_test.go new file mode 100644 index 000000000..f3702cf73 --- /dev/null +++ b/go/binlog/filtered_transaction_test.go @@ -0,0 +1,121 @@ +package binlog + +import ( + "testing" + "time" + + "github.com/github/gh-ost/go/mysql" + "github.com/go-mysql-org/go-mysql/replication" + "github.com/stretchr/testify/require" +) + +func filteredRowsEvent(t *testing.T, pos uint32) *replication.BinlogEvent { + t.Helper() + rows := newTestRowsEvent("testdb", "ignored_table") + decode := newRowsEventDecodeFunc(func(_, _ string) bool { return false }) + require.NoError(t, decode(rows, newRowsEventDataWithInvalidRowPayload())) + require.Empty(t, rows.Rows) + return &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv1, LogPos: pos}, + Event: rows, + } +} + +func TestFilteredTransactionCompletionMarker(t *testing.T) { + for _, test := range []struct { + name, flavor, committed, next string + xidGTIDSet bool + }{ + {name: "file"}, + {"mysql", mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1", "00000000-0000-0000-0000-000000000001:2", true}, + {"mysql fallback", mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1", "00000000-0000-0000-0000-000000000001:2", false}, + {"mariadb", mysql.MariaDBFlavor, "0-1-1", "0-1-2", true}, + {"mariadb fallback", mysql.MariaDBFlavor, "0-1-1", "0-1-2", false}, + } { + t.Run(test.name, func(t *testing.T) { + reader := transactionTestReader(test.flavor != "") + defer reader.migrationContext.CancelContext() + xid := &replication.XIDEvent{} + var committed mysql.BinlogCoordinates = mysql.NewFileBinlogCoordinates("mysql-bin.000001", 220) + if test.flavor != "" { + coords, err := mysql.NewGTIDBinlogCoordinates(test.flavor, test.committed) + require.NoError(t, err) + reader.currentCoordinates = coords + committed = coords.Clone() + if test.xidGTIDSet { + xid.GSet = coords.GTIDSet + } + } + + // Filtering the only row event must not acknowledge an open transaction. + require.Empty(t, transactionTestStream(t, reader, filteredRowsEvent(t, 200))) + require.Nil(t, reader.LastTrxCoords) + entries := transactionTestStream(t, reader, &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: 220}, + Event: xid, + }) + require.Len(t, entries, 1) + require.True(t, entries[0].TransactionComplete) + require.Nil(t, entries[0].DmlEvent) + require.True(t, committed.Equals(entries[0].Coordinates)) + require.True(t, committed.Equals(reader.LastTrxCoords)) + + // Later events (and the syncer's XID set) must not mutate the marker. + if test.flavor != "" { + require.NoError(t, reader.currentCoordinates.(*mysql.GTIDBinlogCoordinates).GTIDSet.Update(test.next)) + } + require.True(t, committed.Equals(entries[0].Coordinates)) + require.True(t, committed.Equals(reader.LastTrxCoords)) + }) + } +} + +func TestFilteredTransactionMarkersPreserveDMLOrder(t *testing.T) { + reader := transactionTestReader(false) + defer reader.migrationContext.CancelContext() + rows := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 100}, + Event: &replication.RowsEvent{ + Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")}, + Rows: [][]interface{}{{1}, {2}}, + }, + } + commit := func(pos uint32) *replication.BinlogEvent { + return &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: pos}, + Event: &replication.XIDEvent{}, + } + } + // Filtered transactions immediately after a marker are coalesced. A later + // relevant transaction still gets a marker behind its own DML. + nextRows := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 600}, + Event: rows.Event, + } + entries := transactionTestStream(t, reader, + rows, filteredRowsEvent(t, 200), commit(220), + filteredRowsEvent(t, 300), commit(320), + filteredRowsEvent(t, 400), commit(420), + commit(520), nextRows, commit(620), + ) + require.Len(t, entries, 6) + for _, index := range []int{0, 1, 3, 4} { + require.NotNil(t, entries[index].DmlEvent) + require.False(t, entries[index].TransactionComplete) + } + for index, pos := range map[int]int64{2: 220, 5: 620} { + entry := entries[index] + require.True(t, entry.TransactionComplete) + require.Nil(t, entry.DmlEvent) + require.Equal(t, pos, entry.Coordinates.(*mysql.FileBinlogCoordinates).LogPos) + } + require.Equal(t, int64(620), reader.LastTrxCoords.(*mysql.FileBinlogCoordinates).LogPos) + + // Under sustained unrelated traffic, another ordered marker eventually + // advances the applied watermark without producing one for every XID. + reader.lastTransactionMarkerTime = time.Now().Add(-filteredTransactionMarkerInterval) + entries = transactionTestStream(t, reader, filteredRowsEvent(t, 700), commit(720)) + require.Len(t, entries, 1) + require.True(t, entries[0].TransactionComplete) + require.Equal(t, int64(720), entries[0].Coordinates.(*mysql.FileBinlogCoordinates).LogPos) +} diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 5d5c6f000..e38a5963e 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -37,6 +37,9 @@ func newRowsEventDecodeFunc(rowsEventFilter RowsEventFilterFunc) func(*replicati } } +// Limit markers for unrelated transactions so busy sources cannot starve row copy. +const filteredTransactionMarkerInterval = time.Second + type GoMySQLReader struct { migrationContext *base.MigrationContext connectionConfig *mysql.ConnectionConfig @@ -49,7 +52,8 @@ type GoMySQLReader struct { // For file coordinates, this is the end position of the transaction's XID event. LastTrxCoords mysql.BinlogCoordinates - transactionHasRows bool + transactionHasRows bool + lastTransactionMarkerTime time.Time } func NewGoMySQLReader(migrationContext *base.MigrationContext, rowsEventFilters ...RowsEventFilterFunc) *GoMySQLReader { @@ -167,7 +171,10 @@ func (gmr *GoMySQLReader) completeTransaction(event *replication.XIDEvent, entri // already updated by the preceding GTID event. coords = gmr.GetCurrentBinlogCoordinates() } - if gmr.transactionHasRows { + // Relevant transactions always need a marker. Sample filtered-only commits + // periodically too, so their progress cannot leave checkpoints on old binlogs. + // Every emitted marker follows preceding DML through the same FIFO. + if gmr.transactionHasRows || time.Since(gmr.lastTransactionMarkerTime) >= filteredTransactionMarkerInterval { marker := &BinlogEntry{ Coordinates: coords.Clone(), TransactionComplete: true, @@ -175,9 +182,11 @@ func (gmr *GoMySQLReader) completeTransaction(event *replication.XIDEvent, entri if err := base.SendWithContext(gmr.migrationContext.GetContext(), entriesChannel, marker); err != nil { return err } - gmr.transactionHasRows = false + gmr.lastTransactionMarkerTime = time.Now() } - // Reconnect only after all rows and their completion marker were delivered. + gmr.transactionHasRows = false + // Filtered-only transactions contain no delivered rows, so reconnect may + // safely advance even when their checkpoint marker was coalesced. gmr.LastTrxCoords = coords return nil } diff --git a/go/logic/applier.go b/go/logic/applier.go index a513cce8f..500f5ac02 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -36,6 +36,8 @@ const ( GhostChangelogTableComment = "gh-ost changelog" atomicCutOverMagicHint = "ghost-cut-over-sentry" moveTablesSourceLockName = "gh-ost-move-tables" + + moveTablesCutOverCheckpointTimeout = 10 * time.Second ) // ErrNoCheckpointFound is returned when an empty checkpoint table is queried. @@ -1314,8 +1316,15 @@ var moveTablesCheckpointColumns = []string{ // markers, drain GTID); each row carries its own table name, iteration, // rows-copied, and serialized iteration range. The latest row therefore always // reflects the freshest run-wide state. +// +// This entry point is used for cutover recovery markers, which must survive +// migration cancellation (the source tables may already have been renamed). +// Bound the write independently; periodic checkpoints instead pass their caller's +// cancellable context to writeMoveTableCheckpoints. func (apl *Applier) WriteMoveTableCheckpoints(rows []*Checkpoint) error { - return apl.writeMoveTableCheckpoints(apl.migrationContext.GetContext(), rows) + ctx, cancel := context.WithTimeout(context.Background(), moveTablesCutOverCheckpointTimeout) + defer cancel() + return apl.writeMoveTableCheckpoints(ctx, rows) } func (apl *Applier) writeMoveTableCheckpoints(ctx context.Context, rows []*Checkpoint) error { @@ -1327,19 +1336,23 @@ func (apl *Applier) writeMoveTableCheckpoints(ctx context.Context, rows []*Check escaped[i] = sql.EscapeName(c) } placeholders := "(" + strings.TrimSuffix(strings.Repeat("?, ", len(moveTablesCheckpointColumns)), ", ") + ")" + // One statement avoids spending the periodic checkpoint deadline on a prefix + // of the tables on every attempt. With InnoDB it also persists an atomic + // snapshot rather than mixing rows from different calls. query := fmt.Sprintf("insert /* gh-ost */ into %s.%s (%s) values %s", sql.EscapeName(apl.checkpointDatabaseName()), sql.EscapeName(apl.migrationContext.GetCheckpointTableName()), strings.Join(escaped, ", "), - placeholders, + strings.TrimSuffix(strings.Repeat(placeholders+", ", len(rows)), ", "), ) + args := make([]interface{}, 0, len(rows)*len(moveTablesCheckpointColumns)) now := time.Now().Unix() for _, chk := range rows { coordStr := "" if chk.LastTrxCoords != nil { coordStr = chk.LastTrxCoords.String() } - args := sqlutils.Args( + args = append(args, now, chk.TableName, coordStr, @@ -1352,11 +1365,9 @@ func (apl *Applier) writeMoveTableCheckpoints(ctx context.Context, rows []*Check serializeRangeValues(chk.IterationRangeMin), serializeRangeValues(chk.IterationRangeMax), ) - if _, err := apl.checkpointDB().ExecContext(ctx, query, args...); err != nil { - return err - } } - return nil + _, err := apl.checkpointDB().ExecContext(ctx, query, args...) + return err } // ReadMoveTableCheckpoints returns the latest checkpoint row per migrated table, diff --git a/go/logic/applier_checkpoint_test.go b/go/logic/applier_checkpoint_test.go new file mode 100644 index 000000000..82f3ea454 --- /dev/null +++ b/go/logic/applier_checkpoint_test.go @@ -0,0 +1,191 @@ +/* + Copyright 2025 GitHub Inc. + See https://github.com/github/gh-ost/blob/master/LICENSE +*/ + +package logic + +import ( + "context" + gosql "database/sql" + "database/sql/driver" + "errors" + "strings" + "testing" + "time" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" + "github.com/github/gh-ost/go/sql" + "github.com/stretchr/testify/require" +) + +// checkpointExecConnector exercises database/sql context handling without a +// server. Each Exec is a server round trip, so tests can detect per-table writes. +type checkpointExecConnector struct { + exec func(context.Context, string, []driver.NamedValue) (driver.Result, error) +} + +func (c *checkpointExecConnector) Connect(context.Context) (driver.Conn, error) { + return &checkpointExecConn{connector: c}, nil +} +func (c *checkpointExecConnector) Driver() driver.Driver { return checkpointExecDriver{} } + +type checkpointExecDriver struct{} + +func (checkpointExecDriver) Open(string) (driver.Conn, error) { + return nil, errors.New("use checkpointExecConnector") +} + +type checkpointExecConn struct{ connector *checkpointExecConnector } + +func (*checkpointExecConn) Prepare(string) (driver.Stmt, error) { + return nil, errors.New("unexpected Prepare") +} +func (*checkpointExecConn) Close() error { return nil } +func (*checkpointExecConn) Begin() (driver.Tx, error) { + return nil, errors.New("unexpected Begin") +} +func (c *checkpointExecConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + return c.connector.exec(ctx, query, args) +} + +func newCheckpointExecApplier(t *testing.T, exec func(context.Context, string, []driver.NamedValue) (driver.Result, error)) *Applier { + t.Helper() + mc := base.NewMigrationContext() + mc.DatabaseName = "source" + mc.MoveTables.TableNames = []string{"first", "last"} + mc.MoveTables.TargetDatabase = "target" + apl := NewApplier(mc) + apl.moveTablesTargetDB = gosql.OpenDB(&checkpointExecConnector{exec: exec}) + t.Cleanup(func() { require.NoError(t, apl.moveTablesTargetDB.Close()) }) + return apl +} + +func TestApplierMoveTableCheckpointSnapshotSingleStatement(t *testing.T) { + coords, err := mysql.ParseFileBinlogCoordinates("mysql-bin.000007:123") + require.NoError(t, err) + drain, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-20") + require.NoError(t, err) + rows := []*Checkpoint{ + { + TableName: "first", LastTrxCoords: coords, Iteration: 3, RowsCopied: 100, DMLApplied: 20, + MoveTablesCutOverStarted: true, MoveTablesCutOverDrainGTID: drain, + IterationRangeMin: sql.ToColumnValues([]interface{}{1, "a"}), + IterationRangeMax: sql.ToColumnValues([]interface{}{9, "君"}), + }, + { + TableName: "last", LastTrxCoords: coords, Iteration: 5, RowsCopied: 200, DMLApplied: 20, + IsCutover: true, MoveTablesCutOverStarted: true, MoveTablesCutOverDrainGTID: drain, + }, + } + calls := 0 + var timestamp interface{} + var apl *Applier + apl = newCheckpointExecApplier(t, func(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + calls++ + require.NoError(t, ctx.Err()) + require.Contains(t, query, "into `target`."+sql.EscapeName(apl.migrationContext.GetCheckpointTableName())) + require.Equal(t, len(rows)*len(moveTablesCheckpointColumns), strings.Count(query, "?")) + require.Contains(t, query, "), (") + require.Len(t, args, len(rows)*len(moveTablesCheckpointColumns)) + timestamp = args[0].Value + for i, row := range rows { + offset := i * len(moveTablesCheckpointColumns) + got := make([]interface{}, len(moveTablesCheckpointColumns)) + for j := range got { + got[j] = args[offset+j].Value + } + require.Equal(t, []interface{}{ + timestamp, row.TableName, coords.String(), row.Iteration, row.RowsCopied, row.DMLApplied, + row.IsCutover, row.MoveTablesCutOverStarted, drain.String(), + serializeRangeValues(row.IterationRangeMin), serializeRangeValues(row.IterationRangeMax), + }, got) + } + return driver.RowsAffected(len(rows)), nil + }) + ctx, cancel := context.WithTimeout(context.Background(), checkpointTimeout) + defer cancel() + require.NoError(t, apl.writeMoveTableCheckpoints(ctx, rows)) + require.Equal(t, 1, calls, "all tables must share one round trip, not consume the deadline in table order") + require.InDelta(t, time.Now().Unix(), timestamp, 1) +} + +func TestApplierMoveTableCutOverCheckpointSurvivesCancellation(t *testing.T) { + for _, cancelBefore := range []bool{true, false} { + name := "during-write" + if cancelBefore { + name = "before-write" + } + t.Run(name, func(t *testing.T) { + calls := 0 + var apl *Applier + var writeCtx context.Context + apl = newCheckpointExecApplier(t, func(ctx context.Context, _ string, _ []driver.NamedValue) (driver.Result, error) { + calls++ + writeCtx = ctx + apl.migrationContext.CancelContext() + require.ErrorIs(t, apl.migrationContext.GetContext().Err(), context.Canceled) + require.NoError(t, ctx.Err(), "recovery marker must not inherit migration cancellation") + deadline, ok := ctx.Deadline() + require.True(t, ok, "recovery writes must still be bounded") + require.WithinDuration(t, time.Now().Add(moveTablesCutOverCheckpointTimeout), deadline, time.Second) + return driver.RowsAffected(2), nil + }) + if cancelBefore { + apl.migrationContext.CancelContext() + } + require.NoError(t, apl.WriteMoveTableCheckpoints([]*Checkpoint{ + {TableName: "first", MoveTablesCutOverStarted: true}, + {TableName: "last", IsCutover: true, MoveTablesCutOverStarted: true}, + })) + require.Equal(t, 1, calls) + require.ErrorIs(t, writeCtx.Err(), context.Canceled, "release the independent context after the write") + }) + } +} + +func TestApplierMoveTablePeriodicCheckpointCancellation(t *testing.T) { + t.Run("already canceled", func(t *testing.T) { + apl := newCheckpointExecApplier(t, func(context.Context, string, []driver.NamedValue) (driver.Result, error) { + t.Fatal("canceled periodic checkpoint must not reach the database") + return nil, nil + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, apl.writeMoveTableCheckpoints(ctx, []*Checkpoint{{TableName: "first"}}), context.Canceled) + }) + t.Run("in flight", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apl := newCheckpointExecApplier(t, func(writeCtx context.Context, _ string, _ []driver.NamedValue) (driver.Result, error) { + cancel() + <-writeCtx.Done() + return nil, writeCtx.Err() + }) + require.ErrorIs(t, apl.writeMoveTableCheckpoints(ctx, []*Checkpoint{{TableName: "first"}}), context.Canceled) + }) + t.Run("deadline", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + apl := newCheckpointExecApplier(t, func(writeCtx context.Context, _ string, _ []driver.NamedValue) (driver.Result, error) { + <-writeCtx.Done() + return nil, writeCtx.Err() + }) + require.ErrorIs(t, apl.writeMoveTableCheckpoints(ctx, []*Checkpoint{{TableName: "first"}}), context.DeadlineExceeded) + }) +} + +func TestApplierMoveTableCheckpointErrorAndEmptySnapshot(t *testing.T) { + writeErr := errors.New("checkpoint insert failed") + calls := 0 + apl := newCheckpointExecApplier(t, func(context.Context, string, []driver.NamedValue) (driver.Result, error) { + calls++ + return nil, writeErr + }) + require.NoError(t, apl.WriteMoveTableCheckpoints(nil)) + require.NoError(t, apl.writeMoveTableCheckpoints(context.Background(), []*Checkpoint{})) + require.Zero(t, calls) + require.ErrorIs(t, apl.WriteMoveTableCheckpoints([]*Checkpoint{{TableName: "first"}, {TableName: "last"}}), writeErr) + require.Equal(t, 1, calls) +} diff --git a/go/logic/checkpoint_review_test.go b/go/logic/checkpoint_review_test.go new file mode 100644 index 000000000..a51c853ef --- /dev/null +++ b/go/logic/checkpoint_review_test.go @@ -0,0 +1,178 @@ +package logic + +import ( + "context" + "database/sql/driver" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/github/gh-ost/go/binlog" + "github.com/github/gh-ost/go/mysql" + "github.com/stretchr/testify/require" +) + +func TestMoveTablesCutOverCheckpointUsesAppliedBoundary(t *testing.T) { + var written []driver.NamedValue + apl := newCheckpointExecApplier(t, func(_ context.Context, _ string, args []driver.NamedValue) (driver.Result, error) { + written = append([]driver.NamedValue(nil), args...) + return driver.RowsAffected(2), nil + }) + mc := apl.migrationContext + defer mc.CancelContext() + mc.InitMoveTableContainers() + m := NewMigrator(mc, "test") + m.applier = apl + start, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1") + require.NoError(t, err) + partial, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-2") + require.NoError(t, err) + apl.CurrentCoordinates = partial.Clone() + apl.AppliedTransactionCoordinates = start.Clone() + // Recovery must be recorded even after cancellation, and row-level GTID + // progress must never be used as the resume boundary. + mc.CancelContext() + require.NoError(t, m.persistMoveTablesCutOverCheckpoint(partial, false)) + require.Len(t, written, 2*len(moveTablesCheckpointColumns)) + for i := 0; i < 2; i++ { + require.Equal(t, start.String(), written[i*len(moveTablesCheckpointColumns)+2].Value) + require.Equal(t, true, written[i*len(moveTablesCheckpointColumns)+7].Value) + require.Equal(t, partial.String(), written[i*len(moveTablesCheckpointColumns)+8].Value) + } + apl.AppliedTransactionCoordinates = nil + require.ErrorContains(t, m.persistMoveTablesCutOverCheckpoint(partial, false), "no applied transaction boundary") +} + +func TestMoveTablesCheckpointDoesNotBlockApplyAndOrdersRecovery(t *testing.T) { + entered := make(chan struct{}) + recoveryEntered := make(chan struct{}) + release := make(chan struct{}) + writes := make(chan []driver.NamedValue, 2) + var calls int32 + apl := newCheckpointExecApplier(t, func(ctx context.Context, _ string, args []driver.NamedValue) (driver.Result, error) { + writes <- append([]driver.NamedValue(nil), args...) + if atomic.AddInt32(&calls, 1) == 1 { + close(entered) + select { + case <-release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } else { + close(recoveryEntered) + } + return driver.RowsAffected(2), nil + }) + mc := apl.migrationContext + defer mc.CancelContext() + mc.InitMoveTableContainers() + m := NewMigrator(mc, "test") + m.applier = apl + start := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 100) + end := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 200) + apl.AppliedTransactionCoordinates = start.Clone() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + periodic := make(chan error, 1) + go func() { _, err := m.Checkpoint(ctx); periodic <- err }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("periodic checkpoint did not enter write") + } + defer func() { + select { + case <-release: + default: + close(release) + } + }() + applied := make(chan error, 1) + go func() { applied <- m.onApplyEventStruct(&applyEventStruct{coords: end, transactionComplete: true}) }() + select { + case err := <-applied: + require.NoError(t, err) + case <-ctx.Done(): + t.Fatal("checkpoint SQL blocked transaction acknowledgement") + } + atomic.StoreInt64(&mc.InCutOverCriticalSectionFlag, 1) + recovery := make(chan error, 1) + recoveryStarted := make(chan struct{}) + go func() { + close(recoveryStarted) + recovery <- m.persistMoveTablesCutOverCheckpoint(end, false) + }() + <-recoveryStarted + // A canceled waiter must not be trapped behind the outstanding write. + waitCtx, waitCancel := context.WithCancel(ctx) + waitCancel() + _, err := m.Checkpoint(waitCtx) + require.ErrorIs(t, err, context.Canceled) + select { + case <-recoveryEntered: + t.Fatal("recovery write overtook the in-flight periodic checkpoint") + case <-time.After(50 * time.Millisecond): + } + close(release) + require.NoError(t, <-periodic) + require.NoError(t, <-recovery) + first, second := <-writes, <-writes + require.Equal(t, start.String(), first[2].Value) + require.Equal(t, false, first[7].Value) + require.Equal(t, end.String(), second[2].Value) + require.Equal(t, true, second[7].Value) + _, err = m.Checkpoint(ctx) + require.ErrorContains(t, err, "cutover started") +} + +func TestMoveTablesDrainUsesAppliedBoundaryWithBacklog(t *testing.T) { + mc := newTestMigrationContext() + defer mc.CancelContext() + mc.CutOverLockTimeoutSeconds = 1 + m := NewMigrator(mc, "test") + m.applier = NewApplier(mc) + drain, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-2") + require.NoError(t, err) + m.applier.AppliedTransactionCoordinates = drain.Clone() + m.applyEventsQueue <- &applyEventStruct{coords: drain, transactionComplete: true} + require.NoError(t, m.drainMoveTablesCutOver(drain)) + require.Len(t, m.applyEventsQueue, 1) +} + +func TestWaitForGhostTableMigratedReturnsListenerFailure(t *testing.T) { + mc := newTestMigrationContext() + defer mc.CancelContext() + m := NewMigrator(mc, "test") + streamer := NewEventsStreamer(mc) + failure := errors.New("changelog listener failed") + require.NoError(t, streamer.AddListener(false, mc.DatabaseName, mc.GetChangelogTableName(), func(*binlog.BinlogEntry) error { + return failure + })) + done := make(chan error, 1) + go func() { done <- m.waitForGhostTableMigrated() }() + err := streamer.notifyListeners(&binlog.BinlogEntry{DmlEvent: binlog.NewBinlogDMLEvent(mc.DatabaseName, mc.GetChangelogTableName(), binlog.InsertDML)}) + require.ErrorIs(t, err, failure) + select { + case err := <-done: + require.ErrorIs(t, err, failure) + case <-time.After(time.Second): + t.Fatal("startup did not stop after listener failure") + } +} + +func TestWaitForGhostTableMigratedSuccessAndCancellation(t *testing.T) { + t.Run("success", func(t *testing.T) { + mc := newTestMigrationContext() + defer mc.CancelContext() + m := NewMigrator(mc, "test") + go func() { m.ghostTableMigrated <- true }() + require.NoError(t, m.waitForGhostTableMigrated()) + }) + t.Run("canceled", func(t *testing.T) { + mc := newTestMigrationContext() + m := NewMigrator(mc, "test") + mc.CancelContext() + require.ErrorIs(t, m.waitForGhostTableMigrated(), context.Canceled) + }) +} diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 44152d3ed..b8b2f5a0d 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -115,7 +115,9 @@ type Migrator struct { applyEventsQueue chan *applyEventStruct applyEventsInFlight int64 - finishedMigrating int64 + // Serializes move-table checkpoint snapshots and writes without blocking apply. + moveTablesCheckpointLock chan struct{} + finishedMigrating int64 } func NewMigrator(context *base.MigrationContext, appVersion string) *Migrator { @@ -137,9 +139,10 @@ func NewMigrator(context *base.MigrationContext, appVersion string) *Migrator { // and OOM when MaxRetries() is extremely large. allEventsUpToLockProcessed: make(chan *lockProcessedStruct, 1), - copyRowsQueue: make(chan tableWriteFunc), - applyEventsQueue: make(chan *applyEventStruct, base.MaxEventsBatchSize), - finishedMigrating: 0, + copyRowsQueue: make(chan tableWriteFunc), + applyEventsQueue: make(chan *applyEventStruct, base.MaxEventsBatchSize), + moveTablesCheckpointLock: make(chan struct{}, 1), + finishedMigrating: 0, } return migrator } @@ -491,6 +494,14 @@ func (mgtr *Migrator) checkAbort() error { return nil } +func (mgtr *Migrator) waitForGhostTableMigrated() error { + select { + case <-mgtr.ghostTableMigrated: + case <-mgtr.migrationContext.GetContext().Done(): + } + return mgtr.checkAbort() +} + // Migrate executes the complete migration logic. This is *the* major gh-ost function. func (mgtr *Migrator) Migrate() (err error) { mgtr.migrationContext.Log.Infof("Migrating %s.%s", sql.EscapeName(mgtr.migrationContext.DatabaseName), sql.EscapeName(mgtr.migrationContext.OriginalTableName)) @@ -572,7 +583,9 @@ func (mgtr *Migrator) Migrate() (err error) { initialLag, _ := mgtr.inspector.getReplicationLag() if !mgtr.migrationContext.Resume { mgtr.migrationContext.Log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag) - <-mgtr.ghostTableMigrated + if err := mgtr.waitForGhostTableMigrated(); err != nil { + return err + } mgtr.migrationContext.Log.Debugf("ghost table migrated") } // Yay! We now know the Ghost and Changelog tables are good to examine! @@ -981,22 +994,15 @@ func (mgtr *Migrator) validateMoveTablesResumeSourceTables(renamed bool) error { } func (mgtr *Migrator) persistMoveTablesCutOverCheckpoint(drainGTID mysql.BinlogCoordinates, isCutover bool) error { - mgtr.applier.CurrentCoordinatesMutex.Lock() - safeCoords := mgtr.applier.CurrentCoordinates - mgtr.applier.CurrentCoordinatesMutex.Unlock() + // Serialize client writes with periodic snapshots. A canceled SQL statement + // may still commit on the server, so recovery reads select cutover rows. + mgtr.moveTablesCheckpointLock <- struct{}{} + defer func() { <-mgtr.moveTablesCheckpointLock }() + safeCoords := mgtr.appliedTransactionCoordinates() if safeCoords == nil || safeCoords.IsEmpty() { - // This quiet-source fallback does not prove a completed transaction boundary; - // it remains subject to the shared checkpoint issue #1781. - if mgtr.eventsStreamer != nil && len(mgtr.applyEventsQueue) == 0 && len(mgtr.eventsStreamer.eventsChannel) == 0 { - safeCoords = mgtr.eventsStreamer.GetCurrentBinlogCoordinates() - } - if safeCoords == nil || safeCoords.IsEmpty() { - return errors.New("current coordinates are empty, cannot checkpoint move-tables cutover") - } + return errors.New("no applied transaction boundary, cannot checkpoint move-tables cutover") } - safeCoords = safeCoords.Clone() - rows := mgtr.buildMoveTableCheckpointRows(safeCoords, isCutover, true, drainGTID) return mgtr.applier.WriteMoveTableCheckpoints(rows) } @@ -1048,7 +1054,13 @@ func (mgtr *Migrator) drainMoveTablesCutOver(drainGTID mysql.BinlogCoordinates) if err := mgtr.checkAbort(); err != nil { return err } - // Primary signal: applier's coordinate (advances when relevant apply work runs). + // A completed boundary proves that all prior DML committed, even if + // unrelated transactions continue to fill the queues after the rename. + if moveTablesDrainCoordinateReached(mgtr.appliedTransactionCoordinates(), drainGTID) { + mgtr.migrationContext.Log.Infof("T3: drain complete at applied transaction boundary") + return nil + } + // Legacy fallback for a non-DML tail without a completion marker. mgtr.applier.CurrentCoordinatesMutex.Lock() applierCoords := mgtr.applier.CurrentCoordinates mgtr.applier.CurrentCoordinatesMutex.Unlock() @@ -3214,23 +3226,38 @@ func (mgtr *Migrator) moveTablesCheckpointSummary(coords mysql.BinlogCoordinates } } +func (mgtr *Migrator) appliedTransactionCoordinates() mysql.BinlogCoordinates { + mgtr.applier.CurrentCoordinatesMutex.Lock() + defer mgtr.applier.CurrentCoordinatesMutex.Unlock() + if mgtr.applier.AppliedTransactionCoordinates == nil { + return nil + } + return mgtr.applier.AppliedTransactionCoordinates.Clone() +} + // checkpointMoveTables persists the last applied transaction boundary. Unlike // standard mode, there is no source heartbeat to advance the reader on a quiet // source, so a safe older boundary is preferable to waiting for reader progress. func (mgtr *Migrator) checkpointMoveTables(ctx context.Context) (*Checkpoint, error) { + select { + case mgtr.moveTablesCheckpointLock <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-mgtr.moveTablesCheckpointLock }() if err := mgtr.checkAbort(); err != nil { return nil, err } if err := ctx.Err(); err != nil { return nil, err } - mgtr.applier.CurrentCoordinatesMutex.Lock() - defer mgtr.applier.CurrentCoordinatesMutex.Unlock() - applied := mgtr.applier.AppliedTransactionCoordinates - if applied == nil { + if atomic.LoadInt64(&mgtr.migrationContext.InCutOverCriticalSectionFlag) > 0 || atomic.LoadInt64(&mgtr.migrationContext.MoveTablesSourceRenamedFlag) > 0 { + return nil, errors.New("move-tables cutover started, not checkpointing") + } + coords := mgtr.appliedTransactionCoordinates() + if coords == nil || coords.IsEmpty() { return nil, errors.New("no applied transaction boundary, not checkpointing") } - coords := applied.Clone() rows := mgtr.buildMoveTableCheckpointRows(coords, false, false, nil) if err := mgtr.applier.writeMoveTableCheckpoints(ctx, rows); err != nil { return nil, err diff --git a/go/logic/migrator_move_tables_cutover_test.go b/go/logic/migrator_move_tables_cutover_test.go index 3ee0dd4dd..734a774db 100644 --- a/go/logic/migrator_move_tables_cutover_test.go +++ b/go/logic/migrator_move_tables_cutover_test.go @@ -330,7 +330,8 @@ func (s *MoveTablesCutOverSuite) buildMigrator(fakeHooks base.Hooks, initialCoor m.applier = NewApplier(mc) if initialCoords != nil { m.applier.CurrentCoordinatesMutex.Lock() - m.applier.CurrentCoordinates = initialCoords + m.applier.CurrentCoordinates = initialCoords.Clone() + m.applier.AppliedTransactionCoordinates = initialCoords.Clone() m.applier.CurrentCoordinatesMutex.Unlock() } return m, mc @@ -799,6 +800,8 @@ func (s *MoveTablesCutOverSuite) TestDrainWaitsForQueuedDML() { fakeHooks := &recordingHooks{name: "fake", calls: &calls} m, mc := s.buildMigrator(fakeHooks, s.containingDrainGTID()) mc.CutOverLockTimeoutSeconds = 1 + // Row-level progress alone is not an applied transaction boundary. + m.applier.AppliedTransactionCoordinates = nil m.applyEventsQueue <- newApplyEventStructByDML(&binlog.BinlogEntry{ DmlEvent: &binlog.BinlogDMLEvent{ DatabaseName: testMysqlDatabase, @@ -866,6 +869,7 @@ func (s *MoveTablesCutOverSuite) TestDrainWaitsForInFlightApplyEvent() { mc.CutOverLockTimeoutSeconds = 1 m.applier.CurrentCoordinatesMutex.Lock() m.applier.CurrentCoordinates = drainGTID + m.applier.AppliedTransactionCoordinates = nil m.applier.CurrentCoordinatesMutex.Unlock() started := make(chan struct{})