diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index c1216054c..0255fde29 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -18,8 +18,9 @@ type BinlogEntry struct { // Timestamp is the wall-clock time recorded in the binlog event header of the // event that produced this entry. It is used in move-tables mode to measure // writer lag (now - last applied event timestamp). - Timestamp time.Time - DmlEvent *BinlogDMLEvent + Timestamp time.Time + DmlEvent *BinlogDMLEvent + TransactionComplete bool } // NewBinlogEntryAt creates an empty, ready to go BinlogEntry object diff --git a/go/binlog/cancellation_test.go b/go/binlog/cancellation_test.go new file mode 100644 index 000000000..1b0c90027 --- /dev/null +++ b/go/binlog/cancellation_test.go @@ -0,0 +1,117 @@ +package binlog + +import ( + "context" + "testing" + "time" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" + "github.com/go-mysql-org/go-mysql/replication" + "github.com/stretchr/testify/require" +) + +func TestTransactionMarkerSendCanceled(t *testing.T) { + t.Run("decoded rows", func(t *testing.T) { + assertTransactionMarkerSendCanceled(t, [][]interface{}{{1}}) + }) + t.Run("filtered rows", func(t *testing.T) { + assertTransactionMarkerSendCanceled(t, nil) + }) +} + +func assertTransactionMarkerSendCanceled(t *testing.T, rows [][]interface{}) { + t.Helper() + reader := transactionTestReader(false) + defer reader.migrationContext.CancelContext() + previous := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 4) + reader.LastTrxCoords = previous.Clone() + row := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 200}, + Event: &replication.RowsEvent{ + Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")}, + Rows: rows, + }, + } + commit := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: 220}, + Event: &replication.XIDEvent{}, + } + for _, event := range []*replication.BinlogEvent{row, commit} { + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(event)) + } + entries := make(chan *BinlogEntry) + done := make(chan error, 1) + go func() { + err := reader.StreamEvents(func() bool { return false }, entries) + _ = base.SendWithContext(t.Context(), done, err) + }() + if len(rows) > 0 { + select { + case entry := <-entries: + require.NotNil(t, entry.DmlEvent) + case <-time.After(time.Second): + t.Fatal("row was not delivered") + } + } + // Wait until the commit position has been read, then stop consuming. + // The reader must propagate a canceled completion-marker send. + require.Eventually(t, func() bool { + return reader.GetCurrentBinlogCoordinates().(*mysql.FileBinlogCoordinates).LogPos == 220 + }, time.Second, time.Millisecond) + reader.migrationContext.CancelContext() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("completion marker send ignored cancellation") + } + require.True(t, previous.Equals(reader.LastTrxCoords), "failed delivery must not advance reconnect coordinates") +} + +func TestRowsSendCanceled(t *testing.T) { + reader := transactionTestReader(false) + defer reader.migrationContext.CancelContext() + event := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2}, + Event: &replication.RowsEvent{ + Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")}, + Rows: [][]interface{}{{1}, {2}}, + }, + } + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(event)) + entries := make(chan *BinlogEntry) + done := make(chan error, 1) + go func() { + err := reader.StreamEvents(func() bool { return false }, entries) + _ = base.SendWithContext(t.Context(), done, err) + }() + select { + case <-entries: + case <-time.After(time.Second): + t.Fatal("first row was not delivered") + } + reader.migrationContext.CancelContext() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("blocked row send ignored cancellation") + } +} + +func TestIdleReaderCanceled(t *testing.T) { + reader := transactionTestReader(false) + reader.migrationContext.CancelContext() + done := make(chan error, 1) + go func() { + err := reader.StreamEvents(func() bool { return false }, make(chan *BinlogEntry)) + _ = base.SendWithContext(t.Context(), done, err) + }() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("idle reader ignored cancellation") + } +} diff --git a/go/binlog/filtered_transaction_test.go b/go/binlog/filtered_transaction_test.go new file mode 100644 index 000000000..f3702cf73 --- /dev/null +++ b/go/binlog/filtered_transaction_test.go @@ -0,0 +1,121 @@ +package binlog + +import ( + "testing" + "time" + + "github.com/github/gh-ost/go/mysql" + "github.com/go-mysql-org/go-mysql/replication" + "github.com/stretchr/testify/require" +) + +func filteredRowsEvent(t *testing.T, pos uint32) *replication.BinlogEvent { + t.Helper() + rows := newTestRowsEvent("testdb", "ignored_table") + decode := newRowsEventDecodeFunc(func(_, _ string) bool { return false }) + require.NoError(t, decode(rows, newRowsEventDataWithInvalidRowPayload())) + require.Empty(t, rows.Rows) + return &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv1, LogPos: pos}, + Event: rows, + } +} + +func TestFilteredTransactionCompletionMarker(t *testing.T) { + for _, test := range []struct { + name, flavor, committed, next string + xidGTIDSet bool + }{ + {name: "file"}, + {"mysql", mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1", "00000000-0000-0000-0000-000000000001:2", true}, + {"mysql fallback", mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1", "00000000-0000-0000-0000-000000000001:2", false}, + {"mariadb", mysql.MariaDBFlavor, "0-1-1", "0-1-2", true}, + {"mariadb fallback", mysql.MariaDBFlavor, "0-1-1", "0-1-2", false}, + } { + t.Run(test.name, func(t *testing.T) { + reader := transactionTestReader(test.flavor != "") + defer reader.migrationContext.CancelContext() + xid := &replication.XIDEvent{} + var committed mysql.BinlogCoordinates = mysql.NewFileBinlogCoordinates("mysql-bin.000001", 220) + if test.flavor != "" { + coords, err := mysql.NewGTIDBinlogCoordinates(test.flavor, test.committed) + require.NoError(t, err) + reader.currentCoordinates = coords + committed = coords.Clone() + if test.xidGTIDSet { + xid.GSet = coords.GTIDSet + } + } + + // Filtering the only row event must not acknowledge an open transaction. + require.Empty(t, transactionTestStream(t, reader, filteredRowsEvent(t, 200))) + require.Nil(t, reader.LastTrxCoords) + entries := transactionTestStream(t, reader, &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: 220}, + Event: xid, + }) + require.Len(t, entries, 1) + require.True(t, entries[0].TransactionComplete) + require.Nil(t, entries[0].DmlEvent) + require.True(t, committed.Equals(entries[0].Coordinates)) + require.True(t, committed.Equals(reader.LastTrxCoords)) + + // Later events (and the syncer's XID set) must not mutate the marker. + if test.flavor != "" { + require.NoError(t, reader.currentCoordinates.(*mysql.GTIDBinlogCoordinates).GTIDSet.Update(test.next)) + } + require.True(t, committed.Equals(entries[0].Coordinates)) + require.True(t, committed.Equals(reader.LastTrxCoords)) + }) + } +} + +func TestFilteredTransactionMarkersPreserveDMLOrder(t *testing.T) { + reader := transactionTestReader(false) + defer reader.migrationContext.CancelContext() + rows := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 100}, + Event: &replication.RowsEvent{ + Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")}, + Rows: [][]interface{}{{1}, {2}}, + }, + } + commit := func(pos uint32) *replication.BinlogEvent { + return &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: pos}, + Event: &replication.XIDEvent{}, + } + } + // Filtered transactions immediately after a marker are coalesced. A later + // relevant transaction still gets a marker behind its own DML. + nextRows := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 600}, + Event: rows.Event, + } + entries := transactionTestStream(t, reader, + rows, filteredRowsEvent(t, 200), commit(220), + filteredRowsEvent(t, 300), commit(320), + filteredRowsEvent(t, 400), commit(420), + commit(520), nextRows, commit(620), + ) + require.Len(t, entries, 6) + for _, index := range []int{0, 1, 3, 4} { + require.NotNil(t, entries[index].DmlEvent) + require.False(t, entries[index].TransactionComplete) + } + for index, pos := range map[int]int64{2: 220, 5: 620} { + entry := entries[index] + require.True(t, entry.TransactionComplete) + require.Nil(t, entry.DmlEvent) + require.Equal(t, pos, entry.Coordinates.(*mysql.FileBinlogCoordinates).LogPos) + } + require.Equal(t, int64(620), reader.LastTrxCoords.(*mysql.FileBinlogCoordinates).LogPos) + + // Under sustained unrelated traffic, another ordered marker eventually + // advances the applied watermark without producing one for every XID. + reader.lastTransactionMarkerTime = time.Now().Add(-filteredTransactionMarkerInterval) + entries = transactionTestStream(t, reader, filteredRowsEvent(t, 700), commit(720)) + require.Len(t, entries, 1) + require.True(t, entries[0].TransactionComplete) + require.Equal(t, int64(720), entries[0].Coordinates.(*mysql.FileBinlogCoordinates).LogPos) +} diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index ee50b0d1a..e38a5963e 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -15,8 +15,6 @@ import ( "time" - "context" - gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" ) @@ -39,6 +37,9 @@ func newRowsEventDecodeFunc(rowsEventFilter RowsEventFilterFunc) func(*replicati } } +// Limit markers for unrelated transactions so busy sources cannot starve row copy. +const filteredTransactionMarkerInterval = time.Second + type GoMySQLReader struct { migrationContext *base.MigrationContext connectionConfig *mysql.ConnectionConfig @@ -47,8 +48,12 @@ type GoMySQLReader struct { currentCoordinates mysql.BinlogCoordinates currentCoordinatesMutex *sync.Mutex // LastTrxCoords are the coordinates of the last transaction completely read. - // If using the file coordinates it is binlog position of the transaction's XID event. + // They are for streamer reconnects, not durable applier checkpoints. + // For file coordinates, this is the end position of the transaction's XID event. LastTrxCoords mysql.BinlogCoordinates + + transactionHasRows bool + lastTransactionMarkerTime time.Time } func NewGoMySQLReader(migrationContext *base.MigrationContext, rowsEventFilters ...RowsEventFilterFunc) *GoMySQLReader { @@ -148,15 +153,48 @@ func (gmr *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent // decides whether action is taken synchronously (meaning we wait before // next iteration) or asynchronously (we keep pushing more events) // In reality, reads will be synchronous - entriesChannel <- binlogEntry + if err := base.SendWithContext(gmr.migrationContext.GetContext(), entriesChannel, binlogEntry); err != nil { + return err + } + gmr.transactionHasRows = true + } + return nil +} + +func (gmr *GoMySQLReader) completeTransaction(event *replication.XIDEvent, entriesChannel chan<- *BinlogEntry) error { + var coords mysql.BinlogCoordinates + if gmr.migrationContext.UseGTIDs && event.GSet != nil { + coords = (&mysql.GTIDBinlogCoordinates{GTIDSet: event.GSet}).Clone() + } else { + // File positions use the current event's coordinates. For GTID + // events parsed without the syncer, fall back to the coordinates + // already updated by the preceding GTID event. + coords = gmr.GetCurrentBinlogCoordinates() + } + // Relevant transactions always need a marker. Sample filtered-only commits + // periodically too, so their progress cannot leave checkpoints on old binlogs. + // Every emitted marker follows preceding DML through the same FIFO. + if gmr.transactionHasRows || time.Since(gmr.lastTransactionMarkerTime) >= filteredTransactionMarkerInterval { + marker := &BinlogEntry{ + Coordinates: coords.Clone(), + TransactionComplete: true, + } + if err := base.SendWithContext(gmr.migrationContext.GetContext(), entriesChannel, marker); err != nil { + return err + } + gmr.lastTransactionMarkerTime = time.Now() } + gmr.transactionHasRows = false + // Filtered-only transactions contain no delivered rows, so reconnect may + // safely advance even when their checkpoint marker was coalesced. + gmr.LastTrxCoords = coords return nil } // StreamEvents func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChannel chan<- *BinlogEntry) error { for !canStopStreaming() { - ev, err := gmr.binlogStreamer.GetEvent(context.Background()) + ev, err := gmr.binlogStreamer.GetEvent(gmr.migrationContext.GetContext()) if err != nil { return err } @@ -215,14 +253,8 @@ func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChan gmr.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", coords.LogFile, int64(ev.Header.LogPos), event.NextLogName) gmr.currentCoordinatesMutex.Unlock() case *replication.XIDEvent: - if gmr.migrationContext.UseGTIDs { - // event.GSet is the full executed GTID set maintained by the - // syncer (MysqlGTIDSet or MariadbGTIDSet depending on flavor). - if event.GSet != nil { - gmr.LastTrxCoords = &mysql.GTIDBinlogCoordinates{GTIDSet: event.GSet} - } - } else { - gmr.LastTrxCoords = gmr.currentCoordinates.Clone() + if err := gmr.completeTransaction(event, entriesChannel); err != nil { + return err } case *replication.RowsEvent: if err := gmr.handleRowsEvent(ev, event, entriesChannel); err != nil { diff --git a/go/binlog/transaction_marker_test.go b/go/binlog/transaction_marker_test.go new file mode 100644 index 000000000..efc152a38 --- /dev/null +++ b/go/binlog/transaction_marker_test.go @@ -0,0 +1,74 @@ +package binlog + +import ( + "testing" + + "github.com/github/gh-ost/go/mysql" + "github.com/go-mysql-org/go-mysql/replication" + "github.com/stretchr/testify/require" +) + +func TestTransactionMarkerClonesGTIDSet(t *testing.T) { + for _, test := range []struct { + flavor, committed, next string + }{ + {mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1", "00000000-0000-0000-0000-000000000001:2"}, + {mysql.MariaDBFlavor, "0-1-1", "0-1-2"}, + } { + t.Run(test.flavor, func(t *testing.T) { + reader := transactionTestReader(true) + defer reader.migrationContext.CancelContext() + coords, err := mysql.NewGTIDBinlogCoordinates(test.flavor, test.committed) + require.NoError(t, err) + reader.currentCoordinates = coords.Clone() + rows := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2}, + Event: &replication.RowsEvent{ + Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")}, + Rows: [][]interface{}{{1}, {2}}, + }, + } + commit := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.XID_EVENT}, + Event: &replication.XIDEvent{GSet: coords.GTIDSet}, + } + entries := transactionTestStream(t, reader, rows, commit) + require.Len(t, entries, 3) + require.True(t, entries[2].TransactionComplete) + require.NoError(t, coords.GTIDSet.Update(test.next)) + require.Equal(t, test.committed, reader.LastTrxCoords.String()) + require.Equal(t, test.committed, entries[2].Coordinates.String()) + }) + } +} + +func TestMultiRowEventDoesNotCompleteTransaction(t *testing.T) { + reader := transactionTestReader(false) + rows := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 200, EventSize: 100}, + Event: &replication.RowsEvent{ + Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")}, + Rows: [][]interface{}{{1}, {2}, {3}}, + }, + } + // Stop before XID. All rows share the event end position, but none of + // them may independently acknowledge that position for checkpointing. + entries := transactionTestStream(t, reader, rows) + require.Len(t, entries, 3) + for _, entry := range entries { + require.Equal(t, int64(200), entry.Coordinates.(*mysql.FileBinlogCoordinates).LogPos) + require.False(t, entry.TransactionComplete) + } + require.Nil(t, reader.LastTrxCoords) + + commit := &replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: 220, EventSize: 20}, + Event: &replication.XIDEvent{}, + } + entries = transactionTestStream(t, reader, commit) + require.Len(t, entries, 1) + require.True(t, entries[0].TransactionComplete) + require.Nil(t, entries[0].DmlEvent) + require.Equal(t, int64(220), entries[0].Coordinates.(*mysql.FileBinlogCoordinates).LogPos) + require.Equal(t, reader.LastTrxCoords, entries[0].Coordinates) +} diff --git a/go/binlog/transaction_test_helpers_test.go b/go/binlog/transaction_test_helpers_test.go new file mode 100644 index 000000000..f916e26e9 --- /dev/null +++ b/go/binlog/transaction_test_helpers_test.go @@ -0,0 +1,47 @@ +package binlog + +import ( + "sync" + "testing" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" + "github.com/go-mysql-org/go-mysql/replication" + "github.com/stretchr/testify/require" +) + +func transactionTestReader(useGTIDs bool) *GoMySQLReader { + ctx := base.NewMigrationContext() + ctx.UseGTIDs = useGTIDs + ctx.DatabaseName = "testdb" + ctx.OriginalTableName = "testing" + var coords mysql.BinlogCoordinates = &mysql.FileBinlogCoordinates{LogFile: "mysql-bin.000001", LogPos: 4} + if useGTIDs { + coords = &mysql.GTIDBinlogCoordinates{} + } + return &GoMySQLReader{ + migrationContext: ctx, + binlogStreamer: replication.NewBinlogStreamer(), + currentCoordinates: coords, + currentCoordinatesMutex: &sync.Mutex{}, + } +} + +func transactionTestStream(t *testing.T, reader *GoMySQLReader, events ...*replication.BinlogEvent) []*BinlogEntry { + t.Helper() + for _, event := range events { + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(event)) + } + entries := make(chan *BinlogEntry, 32) + iterations := 0 + require.NoError(t, reader.StreamEvents(func() bool { + iterations++ + return iterations > len(events) + }, entries)) + close(entries) + var result []*BinlogEntry + for entry := range entries { + result = append(result, entry) + } + return result +} diff --git a/go/logic/applier.go b/go/logic/applier.go index e6d2682ba..500f5ac02 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -36,6 +36,8 @@ const ( GhostChangelogTableComment = "gh-ost changelog" atomicCutOverMagicHint = "ghost-cut-over-sentry" moveTablesSourceLockName = "gh-ost-move-tables" + + moveTablesCutOverCheckpointTimeout = 10 * time.Second ) // ErrNoCheckpointFound is returned when an empty checkpoint table is queried. @@ -76,8 +78,9 @@ type Applier struct { finishedMigrating int64 name string - CurrentCoordinatesMutex sync.Mutex - CurrentCoordinates mysql.BinlogCoordinates + CurrentCoordinatesMutex sync.Mutex + CurrentCoordinates mysql.BinlogCoordinates + AppliedTransactionCoordinates mysql.BinlogCoordinates LastIterationRangeMutex sync.Mutex LastIterationRangeMinValues *sql.ColumnValues @@ -1272,7 +1275,7 @@ func (apl *Applier) WriteChangelogState(value string) (string, error) { // WriteCheckpoint writes a standard-mode checkpoint row to the _ghk table. In // move-tables mode use WriteMoveTableCheckpoints instead; calling this there is a // programmer error (the checkpoint schema and query builder are standard-only). -func (apl *Applier) WriteCheckpoint(chk *Checkpoint) (int64, error) { +func (apl *Applier) WriteCheckpoint(ctx context.Context, chk *Checkpoint) (int64, error) { if apl.migrationContext.IsMoveTablesMode() { panic("WriteCheckpoint() must not be called in move-tables mode; use WriteMoveTableCheckpoints") } @@ -1285,7 +1288,7 @@ func (apl *Applier) WriteCheckpoint(chk *Checkpoint) (int64, error) { } args := sqlutils.Args(chk.LastTrxCoords.String(), chk.Iteration, chk.RowsCopied, chk.DMLApplied, chk.IsCutover) args = append(args, uniqueKeyArgs...) - res, err := apl.checkpointDB().Exec(query, args...) + res, err := apl.checkpointDB().ExecContext(ctx, query, args...) if err != nil { return insertId, err } @@ -1313,7 +1316,18 @@ var moveTablesCheckpointColumns = []string{ // markers, drain GTID); each row carries its own table name, iteration, // rows-copied, and serialized iteration range. The latest row therefore always // reflects the freshest run-wide state. +// +// This entry point is used for cutover recovery markers, which must survive +// migration cancellation (the source tables may already have been renamed). +// Bound the write independently; periodic checkpoints instead pass their caller's +// cancellable context to writeMoveTableCheckpoints. func (apl *Applier) WriteMoveTableCheckpoints(rows []*Checkpoint) error { + ctx, cancel := context.WithTimeout(context.Background(), moveTablesCutOverCheckpointTimeout) + defer cancel() + return apl.writeMoveTableCheckpoints(ctx, rows) +} + +func (apl *Applier) writeMoveTableCheckpoints(ctx context.Context, rows []*Checkpoint) error { if len(rows) == 0 { return nil } @@ -1322,19 +1336,23 @@ func (apl *Applier) WriteMoveTableCheckpoints(rows []*Checkpoint) error { escaped[i] = sql.EscapeName(c) } placeholders := "(" + strings.TrimSuffix(strings.Repeat("?, ", len(moveTablesCheckpointColumns)), ", ") + ")" + // One statement avoids spending the periodic checkpoint deadline on a prefix + // of the tables on every attempt. With InnoDB it also persists an atomic + // snapshot rather than mixing rows from different calls. query := fmt.Sprintf("insert /* gh-ost */ into %s.%s (%s) values %s", sql.EscapeName(apl.checkpointDatabaseName()), sql.EscapeName(apl.migrationContext.GetCheckpointTableName()), strings.Join(escaped, ", "), - placeholders, + strings.TrimSuffix(strings.Repeat(placeholders+", ", len(rows)), ", "), ) + args := make([]interface{}, 0, len(rows)*len(moveTablesCheckpointColumns)) now := time.Now().Unix() for _, chk := range rows { coordStr := "" if chk.LastTrxCoords != nil { coordStr = chk.LastTrxCoords.String() } - args := sqlutils.Args( + args = append(args, now, chk.TableName, coordStr, @@ -1347,11 +1365,9 @@ func (apl *Applier) WriteMoveTableCheckpoints(rows []*Checkpoint) error { serializeRangeValues(chk.IterationRangeMin), serializeRangeValues(chk.IterationRangeMax), ) - if _, err := apl.checkpointDB().Exec(query, args...); err != nil { - return err - } } - return nil + _, err := apl.checkpointDB().ExecContext(ctx, query, args...) + return err } // ReadMoveTableCheckpoints returns the latest checkpoint row per migrated table, diff --git a/go/logic/applier_checkpoint_test.go b/go/logic/applier_checkpoint_test.go new file mode 100644 index 000000000..82f3ea454 --- /dev/null +++ b/go/logic/applier_checkpoint_test.go @@ -0,0 +1,191 @@ +/* + Copyright 2025 GitHub Inc. + See https://github.com/github/gh-ost/blob/master/LICENSE +*/ + +package logic + +import ( + "context" + gosql "database/sql" + "database/sql/driver" + "errors" + "strings" + "testing" + "time" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" + "github.com/github/gh-ost/go/sql" + "github.com/stretchr/testify/require" +) + +// checkpointExecConnector exercises database/sql context handling without a +// server. Each Exec is a server round trip, so tests can detect per-table writes. +type checkpointExecConnector struct { + exec func(context.Context, string, []driver.NamedValue) (driver.Result, error) +} + +func (c *checkpointExecConnector) Connect(context.Context) (driver.Conn, error) { + return &checkpointExecConn{connector: c}, nil +} +func (c *checkpointExecConnector) Driver() driver.Driver { return checkpointExecDriver{} } + +type checkpointExecDriver struct{} + +func (checkpointExecDriver) Open(string) (driver.Conn, error) { + return nil, errors.New("use checkpointExecConnector") +} + +type checkpointExecConn struct{ connector *checkpointExecConnector } + +func (*checkpointExecConn) Prepare(string) (driver.Stmt, error) { + return nil, errors.New("unexpected Prepare") +} +func (*checkpointExecConn) Close() error { return nil } +func (*checkpointExecConn) Begin() (driver.Tx, error) { + return nil, errors.New("unexpected Begin") +} +func (c *checkpointExecConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + return c.connector.exec(ctx, query, args) +} + +func newCheckpointExecApplier(t *testing.T, exec func(context.Context, string, []driver.NamedValue) (driver.Result, error)) *Applier { + t.Helper() + mc := base.NewMigrationContext() + mc.DatabaseName = "source" + mc.MoveTables.TableNames = []string{"first", "last"} + mc.MoveTables.TargetDatabase = "target" + apl := NewApplier(mc) + apl.moveTablesTargetDB = gosql.OpenDB(&checkpointExecConnector{exec: exec}) + t.Cleanup(func() { require.NoError(t, apl.moveTablesTargetDB.Close()) }) + return apl +} + +func TestApplierMoveTableCheckpointSnapshotSingleStatement(t *testing.T) { + coords, err := mysql.ParseFileBinlogCoordinates("mysql-bin.000007:123") + require.NoError(t, err) + drain, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-20") + require.NoError(t, err) + rows := []*Checkpoint{ + { + TableName: "first", LastTrxCoords: coords, Iteration: 3, RowsCopied: 100, DMLApplied: 20, + MoveTablesCutOverStarted: true, MoveTablesCutOverDrainGTID: drain, + IterationRangeMin: sql.ToColumnValues([]interface{}{1, "a"}), + IterationRangeMax: sql.ToColumnValues([]interface{}{9, "君"}), + }, + { + TableName: "last", LastTrxCoords: coords, Iteration: 5, RowsCopied: 200, DMLApplied: 20, + IsCutover: true, MoveTablesCutOverStarted: true, MoveTablesCutOverDrainGTID: drain, + }, + } + calls := 0 + var timestamp interface{} + var apl *Applier + apl = newCheckpointExecApplier(t, func(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + calls++ + require.NoError(t, ctx.Err()) + require.Contains(t, query, "into `target`."+sql.EscapeName(apl.migrationContext.GetCheckpointTableName())) + require.Equal(t, len(rows)*len(moveTablesCheckpointColumns), strings.Count(query, "?")) + require.Contains(t, query, "), (") + require.Len(t, args, len(rows)*len(moveTablesCheckpointColumns)) + timestamp = args[0].Value + for i, row := range rows { + offset := i * len(moveTablesCheckpointColumns) + got := make([]interface{}, len(moveTablesCheckpointColumns)) + for j := range got { + got[j] = args[offset+j].Value + } + require.Equal(t, []interface{}{ + timestamp, row.TableName, coords.String(), row.Iteration, row.RowsCopied, row.DMLApplied, + row.IsCutover, row.MoveTablesCutOverStarted, drain.String(), + serializeRangeValues(row.IterationRangeMin), serializeRangeValues(row.IterationRangeMax), + }, got) + } + return driver.RowsAffected(len(rows)), nil + }) + ctx, cancel := context.WithTimeout(context.Background(), checkpointTimeout) + defer cancel() + require.NoError(t, apl.writeMoveTableCheckpoints(ctx, rows)) + require.Equal(t, 1, calls, "all tables must share one round trip, not consume the deadline in table order") + require.InDelta(t, time.Now().Unix(), timestamp, 1) +} + +func TestApplierMoveTableCutOverCheckpointSurvivesCancellation(t *testing.T) { + for _, cancelBefore := range []bool{true, false} { + name := "during-write" + if cancelBefore { + name = "before-write" + } + t.Run(name, func(t *testing.T) { + calls := 0 + var apl *Applier + var writeCtx context.Context + apl = newCheckpointExecApplier(t, func(ctx context.Context, _ string, _ []driver.NamedValue) (driver.Result, error) { + calls++ + writeCtx = ctx + apl.migrationContext.CancelContext() + require.ErrorIs(t, apl.migrationContext.GetContext().Err(), context.Canceled) + require.NoError(t, ctx.Err(), "recovery marker must not inherit migration cancellation") + deadline, ok := ctx.Deadline() + require.True(t, ok, "recovery writes must still be bounded") + require.WithinDuration(t, time.Now().Add(moveTablesCutOverCheckpointTimeout), deadline, time.Second) + return driver.RowsAffected(2), nil + }) + if cancelBefore { + apl.migrationContext.CancelContext() + } + require.NoError(t, apl.WriteMoveTableCheckpoints([]*Checkpoint{ + {TableName: "first", MoveTablesCutOverStarted: true}, + {TableName: "last", IsCutover: true, MoveTablesCutOverStarted: true}, + })) + require.Equal(t, 1, calls) + require.ErrorIs(t, writeCtx.Err(), context.Canceled, "release the independent context after the write") + }) + } +} + +func TestApplierMoveTablePeriodicCheckpointCancellation(t *testing.T) { + t.Run("already canceled", func(t *testing.T) { + apl := newCheckpointExecApplier(t, func(context.Context, string, []driver.NamedValue) (driver.Result, error) { + t.Fatal("canceled periodic checkpoint must not reach the database") + return nil, nil + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, apl.writeMoveTableCheckpoints(ctx, []*Checkpoint{{TableName: "first"}}), context.Canceled) + }) + t.Run("in flight", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apl := newCheckpointExecApplier(t, func(writeCtx context.Context, _ string, _ []driver.NamedValue) (driver.Result, error) { + cancel() + <-writeCtx.Done() + return nil, writeCtx.Err() + }) + require.ErrorIs(t, apl.writeMoveTableCheckpoints(ctx, []*Checkpoint{{TableName: "first"}}), context.Canceled) + }) + t.Run("deadline", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + apl := newCheckpointExecApplier(t, func(writeCtx context.Context, _ string, _ []driver.NamedValue) (driver.Result, error) { + <-writeCtx.Done() + return nil, writeCtx.Err() + }) + require.ErrorIs(t, apl.writeMoveTableCheckpoints(ctx, []*Checkpoint{{TableName: "first"}}), context.DeadlineExceeded) + }) +} + +func TestApplierMoveTableCheckpointErrorAndEmptySnapshot(t *testing.T) { + writeErr := errors.New("checkpoint insert failed") + calls := 0 + apl := newCheckpointExecApplier(t, func(context.Context, string, []driver.NamedValue) (driver.Result, error) { + calls++ + return nil, writeErr + }) + require.NoError(t, apl.WriteMoveTableCheckpoints(nil)) + require.NoError(t, apl.writeMoveTableCheckpoints(context.Background(), []*Checkpoint{})) + require.Zero(t, calls) + require.ErrorIs(t, apl.WriteMoveTableCheckpoints([]*Checkpoint{{TableName: "first"}, {TableName: "last"}}), writeErr) + require.Equal(t, 1, calls) +} diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index da4d9b8ec..465841cf5 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -1374,13 +1374,20 @@ func (suite *ApplierTestSuite) TestWriteCheckpoint() { DMLApplied: 200000, IsCutover: true, } - id, err := applier.WriteCheckpoint(chk) + id, err := applier.WriteCheckpoint(ctx, chk) suite.Require().NoError(err) suite.Require().Equal(int64(1), id) + // A canceled write must leave the last durable checkpoint untouched. + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + _, err = applier.WriteCheckpoint(canceledCtx, chk) + suite.Require().ErrorIs(err, context.Canceled) + gotChk, err := applier.ReadLastCheckpoint() suite.Require().NoError(err) + suite.Require().Equal(id, gotChk.Id) suite.Require().Equal(chk.Iteration, gotChk.Iteration) suite.Require().Equal(chk.LastTrxCoords.String(), gotChk.LastTrxCoords.String()) suite.Require().Equal(chk.IterationRangeMin.String(), gotChk.IterationRangeMin.String()) diff --git a/go/logic/binlog_checkpoint_test.go b/go/logic/binlog_checkpoint_test.go new file mode 100644 index 000000000..0d6030e50 --- /dev/null +++ b/go/logic/binlog_checkpoint_test.go @@ -0,0 +1,311 @@ +package logic + +import ( + "context" + gosql "database/sql" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/binlog" + "github.com/github/gh-ost/go/mysql" + "github.com/github/gh-ost/go/sql" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + testmysql "github.com/testcontainers/testcontainers-go/modules/mysql" +) + +// queueCheckpointTestEvents uses the production reader, dispatcher and listeners, +// but leaves consumption of the apply queue under the test's control. +// +//nolint:contextcheck // Streamer APIs use MigrationContext; the test deadline closes the reader. +func queueCheckpointTestEvents(t *testing.T, ctx context.Context, migrationContext *base.MigrationContext, migrator *Migrator, start, end mysql.BinlogCoordinates) { + t.Helper() + reader := binlog.NewGoMySQLReader(migrationContext, func(database, table string) bool { + return database == testMysqlDatabase && table == testMysqlTableName + }) + defer reader.Close() + require.NoError(t, reader.ConnectBinlogStreamer(start)) + streamer := NewEventsStreamer(migrationContext) + streamer.binlogReader = reader + migrator.eventsStreamer = streamer + require.NoError(t, migrator.addDMLEventsListener()) + + delivered := make(chan struct{}, 1) + streamer.AddTransactionCompleteListener(func(entry *binlog.BinlogEntry) error { + if entry.Coordinates.Equals(end) { + delivered <- struct{}{} + } + return nil + }) + done := make(chan error, 1) + go func() { + done <- streamer.StreamEvents(func() bool { + return reader.LastTrxCoords != nil && reader.LastTrxCoords.Equals(end) + }) + }() + select { + case err := <-done: + require.NoError(t, err) + case <-ctx.Done(): + reader.Close() + <-done + t.Fatal("timed out reading transaction") + } + // The reader has stopped sending. Let the dispatcher drain and exit. + close(streamer.eventsChannel) + select { + case <-delivered: + case <-ctx.Done(): + t.Fatal("timed out dispatching transaction completion") + } +} + +//nolint:contextcheck // Migrator APIs use MigrationContext's internal context; SQL/checkpoint calls use the test deadline. +func TestBinlogCheckpointWaitsForTransactionIntegration(t *testing.T) { + if testing.Short() { + t.Skip("requires MySQL 8.0") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + container, err := testmysql.Run(ctx, testMysqlContainerImage, + testmysql.WithDatabase(testMysqlDatabase), + testmysql.WithUsername(testMysqlUser), + testmysql.WithPassword(testMysqlPass), + testcontainers.WithCmdArgs("--server-id=1", "--gtid-mode=ON", "--enforce-gtid-consistency=ON"), + ) + require.NoError(t, err) + defer testcontainers.TerminateContainer(container) + dsn, err := container.ConnectionString(ctx) + require.NoError(t, err) + db, err := gosql.Open("mysql", dsn) + require.NoError(t, err) + defer db.Close() + conn, err := db.Conn(ctx) + require.NoError(t, err) + defer conn.Close() + for _, query := range []string{ + "SET SESSION time_zone='+00:00'", + "CREATE TABLE testing (id INT PRIMARY KEY, amount DECIMAL(5,2), created_at TIMESTAMP, padding TEXT)", + } { + _, err := conn.ExecContext(ctx, query) + require.NoError(t, err) + } + status := func() (*mysql.FileBinlogCoordinates, *mysql.GTIDBinlogCoordinates) { + var file, doDB, ignoreDB, gtid string + var pos int64 + require.NoError(t, conn.QueryRowContext(ctx, "SHOW MASTER STATUS").Scan(&file, &pos, &doDB, &ignoreDB, >id)) + coords, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, gtid) + require.NoError(t, err) + return mysql.NewFileBinlogCoordinates(file, pos), coords + } + type transaction struct { + name string + startFile, endFile *mysql.FileBinlogCoordinates + startGTID, endGTID *mysql.GTIDBinlogCoordinates + } + var transactions []transaction + for _, compression := range []string{"OFF"} { + _, err := conn.ExecContext(ctx, "SET SESSION binlog_transaction_compression="+compression) + require.NoError(t, err) + startFile, startGTID := status() + tx, err := conn.BeginTx(ctx, nil) + require.NoError(t, err) + for _, query := range []string{ + "INSERT INTO testing VALUES (1, 123.45, '2023-11-14 22:13:20', REPEAT('x', 10000))", + "UPDATE testing SET amount=456.78 WHERE id=1", + "DELETE FROM testing WHERE id=1", + } { + _, err := tx.ExecContext(ctx, query) + require.NoError(t, err) + } + require.NoError(t, tx.Commit()) + endFile, endGTID := status() + transactions = append(transactions, transaction{compression, startFile, endFile, startGTID, endGTID}) + } + config, err := getTestConnectionConfig(ctx, container) + require.NoError(t, err) + + for _, transaction := range transactions { + for _, mode := range []struct { + useGTIDs bool + cancelAt string + }{ + {false, "none"}, {true, "none"}, + {false, "apply"}, {true, "apply"}, + {false, "insert"}, {true, "insert"}, + } { + useGTIDs := mode.useGTIDs + t.Run(fmt.Sprintf("compression=%s/GTID=%t/cancel=%s", transaction.name, useGTIDs, mode.cancelAt), func(t *testing.T) { + migrationContext := newTestMigrationContext() + defer migrationContext.CancelContext() + migrationContext.Checkpoint = true + migrationContext.UseGTIDs = useGTIDs + // Replaying an already-applied INSERT uses INSERT IGNORE. Duplicate + // warnings are expected on resume; use the default production policy. + migrationContext.PanicOnWarnings = false + migrationContext.ApplierConnectionConfig = config + migrationContext.InspectorConnectionConfig = config + require.NoError(t, migrationContext.SetConnectionConfig("innodb")) + migrationContext.SetDMLBatchSize(1) + columns := sql.NewColumnList([]string{"id", "amount", "created_at", "padding"}) + migrationContext.OriginalTableColumns = columns + migrationContext.SharedColumns = columns + migrationContext.MappedSharedColumns = columns + uniqueKeyColumns := sql.NewColumnList([]string{"id"}) + uniqueKeyColumns.GetColumn("id").MySQLType = "int" + migrationContext.UniqueKey = &sql.UniqueKey{Name: "PRIMARY", NameInGhostTable: "PRIMARY", Columns: *uniqueKeyColumns} + _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s LIKE %s", getTestGhostTableName(), getTestTableName())) + require.NoError(t, err) + defer func() { + _, err := conn.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", getTestGhostTableName())) + require.NoError(t, err) + }() + + migrator := NewMigrator(migrationContext, "test") + migrator.applier = NewApplier(migrationContext) + require.NoError(t, migrator.applier.InitDBConnections()) + defer migrator.applier.Teardown() + migrator.applier.LastIterationRangeMinValues = sql.ToColumnValues([]interface{}{int64(1)}) + migrator.applier.LastIterationRangeMaxValues = sql.ToColumnValues([]interface{}{int64(1)}) + require.NoError(t, migrator.applier.prepareQueries()) + require.NoError(t, migrator.applier.CreateCheckpointTable()) + defer func() { require.NoError(t, migrator.applier.DropCheckpointTable()) }() + start, end := transaction.startFile.Clone(), transaction.endFile.Clone() + if useGTIDs { + start, end = transaction.startGTID.Clone(), transaction.endGTID.Clone() + } + seed := &Checkpoint{ + LastTrxCoords: start.Clone(), + IterationRangeMin: migrator.applier.LastIterationRangeMinValues.Clone(), + IterationRangeMax: migrator.applier.LastIterationRangeMaxValues.Clone(), + } + seedID, err := migrator.applier.WriteCheckpoint(ctx, seed) + require.NoError(t, err) + migrator.applier.AppliedTransactionCoordinates = start.Clone() + queueCheckpointTestEvents(t, ctx, migrationContext, migrator, start, end) + require.Len(t, migrator.applyEventsQueue, 4) + first := <-migrator.applyEventsQueue + require.NoError(t, migrator.onApplyEventStruct(first)) + require.Len(t, migrator.applyEventsQueue, 3, "two DMLs and the completion marker remain pending") + + assertCheckpointBlocked := func() { + checkpointCtx, cancelCheckpoint := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancelCheckpoint() + checkpoint, err := migrator.Checkpoint(checkpointCtx) + require.ErrorIs(t, err, context.DeadlineExceeded, "premature checkpoint: %+v", checkpoint) + stored, err := migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + require.Equal(t, seedID, stored.Id, "failed checkpoint must not replace the previous durable checkpoint") + require.True(t, stored.LastTrxCoords.Equals(start)) + } + assertCheckpointBlocked() + switch mode.cancelAt { + case "apply": + assertCanceledTransactionCheckpoint(t, ctx, migrationContext, migrator, seedID) + return + case "insert": + for len(migrator.applyEventsQueue) > 0 { + require.NoError(t, migrator.onApplyEventStruct(<-migrator.applyEventsQueue)) + } + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(end)) + assertCanceledCheckpointInsert(t, ctx, migrationContext, migrator, seedID) + return + } + + // Simulate a crash after the first applied row: discard volatile + // queue contents, then restart from the last persisted checkpoint. + for len(migrator.applyEventsQueue) > 0 { + <-migrator.applyEventsQueue + } + stored, err := migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + queueCheckpointTestEvents(t, ctx, migrationContext, migrator, stored.LastTrxCoords, end) + require.Len(t, migrator.applyEventsQueue, 4) + var transactionRows []*applyEventStruct + for len(migrator.applyEventsQueue) > 1 { + row := <-migrator.applyEventsQueue + transactionRows = append(transactionRows, row) + require.NoError(t, migrator.onApplyEventStruct(row)) + } + // Even the final row cannot independently acknowledge its commit. + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(start)) + if !useGTIDs && transaction.name == "OFF" { + // Sample a real row-event position P between the previous commit A + // and this transaction's XID B. A must block; B must be persisted + // rather than P once its marker is acknowledged. + sampled := first.coords.Clone() + require.True(t, start.SmallerThan(sampled)) + require.True(t, sampled.SmallerThan(end)) + reader := binlog.NewGoMySQLReader(migrationContext, nil) + defer reader.Close() + require.NoError(t, reader.ConnectBinlogStreamer(sampled)) + migrator.eventsStreamer.binlogReader = reader + } + assertCheckpointBlocked() + marker := <-migrator.applyEventsQueue + require.True(t, marker.transactionComplete) + require.NoError(t, migrator.onApplyEventStruct(marker)) + checkpoint, err := migrator.Checkpoint(ctx) + require.NoError(t, err) + require.True(t, checkpoint.LastTrxCoords.Equals(end)) + stored, err = migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + require.True(t, stored.LastTrxCoords.Equals(end)) + var remaining int + require.NoError(t, conn.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s", getTestGhostTableName())).Scan(&remaining)) + require.Zero(t, remaining, "resuming after the partial apply must not skip the DELETE") + + // Resume again from the completed checkpoint and apply later changes. + _, err = conn.ExecContext(ctx, "INSERT INTO testing VALUES (100, 12.34, '2023-11-14 22:13:20', 'after checkpoint') ON DUPLICATE KEY UPDATE amount=amount+1") + require.NoError(t, err) + resumeFile, resumeGTID := status() + resumeEnd := mysql.BinlogCoordinates(resumeFile) + if useGTIDs { + resumeEnd = resumeGTID + } + queueCheckpointTestEvents(t, ctx, migrationContext, migrator, stored.LastTrxCoords, resumeEnd) + for len(migrator.applyEventsQueue) > 0 { + require.NoError(t, migrator.onApplyEventStruct(<-migrator.applyEventsQueue)) + } + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(resumeEnd)) + var sourceRows, ghostRows string + query := "SELECT GROUP_CONCAT(CONCAT(id, ':', amount) ORDER BY id) FROM %s" + require.NoError(t, conn.QueryRowContext(ctx, fmt.Sprintf(query, getTestTableName())).Scan(&sourceRows)) + require.NoError(t, conn.QueryRowContext(ctx, fmt.Sprintf(query, getTestGhostTableName())).Scan(&ghostRows)) + require.Equal(t, sourceRows, ghostRows) + + // Batch across two source-transaction markers, ending inside the + // second transaction. Only the first may be acknowledged yet. + migrationContext.SetDMLBatchSize(4) + migrator.applier.AppliedTransactionCoordinates = start.Clone() + nextMarker := &applyEventStruct{coords: resumeEnd.Clone(), transactionComplete: true} + for _, event := range []*applyEventStruct{transactionRows[1], transactionRows[2], marker, transactionRows[0], transactionRows[1], transactionRows[2], nextMarker} { + migrator.applyEventsQueue <- event + } + appliedBefore := atomic.LoadInt64(&migrationContext.TotalDMLEventsApplied) + require.NoError(t, migrator.onApplyEventStruct(transactionRows[0])) + require.Equal(t, appliedBefore+4, atomic.LoadInt64(&migrationContext.TotalDMLEventsApplied)) + require.Len(t, migrator.applyEventsQueue, 3) + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(end)) + require.NoError(t, migrator.onApplyEventStruct(<-migrator.applyEventsQueue)) + require.Empty(t, migrator.applyEventsQueue) + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(resumeEnd)) + + // A marker collected while assembling a batch cannot acknowledge + // anything if the destination SQL transaction fails. + _, err = conn.ExecContext(ctx, fmt.Sprintf("DROP TABLE %s", getTestGhostTableName())) + require.NoError(t, err) + migrationContext.SetDefaultNumRetries(1) + migrationContext.PanicAbort = make(chan error, 1) + migrator.applier.AppliedTransactionCoordinates = end.Clone() + migrator.applyEventsQueue <- nextMarker + migrator.applyEventsQueue <- transactionRows[1] + require.Error(t, migrator.onApplyEventStruct(first)) + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(end)) + }) + } + } +} diff --git a/go/logic/checkpoint_cancellation_test.go b/go/logic/checkpoint_cancellation_test.go new file mode 100644 index 000000000..b5d719c8e --- /dev/null +++ b/go/logic/checkpoint_cancellation_test.go @@ -0,0 +1,191 @@ +package logic + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/binlog" + "github.com/github/gh-ost/go/metrics" + "github.com/github/gh-ost/go/sql" + drivermysql "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Signal when Checkpoint is waiting for its sampled position to be applied, +// without relying on sleeps to arrange the cancellation interleaving. +type checkpointWaitMetrics struct { + metrics.Emitter + waiting chan struct{} + once sync.Once +} + +func (m *checkpointWaitMetrics) Histogram(name string, _ float64, tags ...string) { + if name == "sleep.duration_milliseconds" && len(tags) == 1 && tags[0] == "stage:replica_wait" { + m.once.Do(func() { close(m.waiting) }) + } +} + +// The caller applied the INSERT and seeded an older durable checkpoint. The +// UPDATE, DELETE and completion marker are still pending. Replay their delivery +// under cancellation to exercise the failed-enqueue/marker interleaving. +// +//nolint:contextcheck // Migrator and dispatcher APIs use MigrationContext's internal context. +func assertCanceledTransactionCheckpoint(t *testing.T, ctx context.Context, migrationContext *base.MigrationContext, migrator *Migrator, seedID int64) { + t.Helper() + var pending []*applyEventStruct + for len(migrator.applyEventsQueue) > 0 { + pending = append(pending, <-migrator.applyEventsQueue) + } + require.Len(t, pending, 3) + marker := pending[2] + require.True(t, marker.transactionComplete) + previous := migrator.applier.AppliedTransactionCoordinates.Clone() + + wait := &checkpointWaitMetrics{Emitter: metrics.Noop, waiting: make(chan struct{})} + migrationContext.Metrics = wait + checkpointCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := migrator.Checkpoint(checkpointCtx) + _ = base.SendWithContext(ctx, done, err) + }() + select { + case <-wait.waiting: + case <-checkpointCtx.Done(): + t.Fatal("checkpoint never waited for the pending transaction") + } + + migrationContext.CancelContext() + for _, event := range pending[:2] { + err := migrator.eventsStreamer.notifyListeners(&binlog.BinlogEntry{Coordinates: event.coords, DmlEvent: event.dmlEvent}) + require.ErrorIs(t, err, context.Canceled) + } + require.ErrorIs(t, migrator.eventsStreamer.notifyListeners(&binlog.BinlogEntry{Coordinates: marker.coords, TransactionComplete: true}), context.Canceled) + require.Empty(t, migrator.applyEventsQueue) + // Also reject a marker that was already dequeued before cancellation, + // e.g. by a worker that passed its loop's abort check before throttling. + require.ErrorIs(t, migrator.onApplyEventStruct(marker), context.Canceled) + require.True(t, previous.Equals(migrator.applier.AppliedTransactionCoordinates)) + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-checkpointCtx.Done(): + t.Fatal("migration cancellation did not interrupt the pending checkpoint") + } + require.NoError(t, checkpointCtx.Err(), "checkpoint's caller has not canceled") + _, err := migrator.Checkpoint(checkpointCtx) + require.ErrorIs(t, err, context.Canceled, "new checkpoints must also reject an aborted migration") + stored, err := migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + require.Equal(t, seedID, stored.Id) + require.True(t, stored.LastTrxCoords.Equals(previous)) + var remaining int + require.NoError(t, migrator.applier.db.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s", getTestGhostTableName())).Scan(&remaining)) + require.Equal(t, 1, remaining, "the unapplied DELETE must still be replayed from the safe checkpoint") +} + +// Block the production checkpoint INSERT with a real MySQL table lock. Unlike +// cancellation during polling, this requires the migration cancellation bridge +// to reach an already-running ExecContext with an independent caller context. +func assertCanceledCheckpointInsert(t *testing.T, ctx context.Context, migrationContext *base.MigrationContext, migrator *Migrator, seedID int64) { + t.Helper() + previous, err := migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + require.Equal(t, seedID, previous.Id) + checkpointTable := fmt.Sprintf("%s.%s", sql.EscapeName(migrationContext.DatabaseName), sql.EscapeName(migrationContext.GetCheckpointTableName())) + blocker, err := migrator.applier.db.Conn(ctx) + require.NoError(t, err) + defer blocker.Close() + _, err = blocker.ExecContext(ctx, "LOCK TABLES "+checkpointTable+" READ") + require.NoError(t, err) + + checkpointCtx, cancelCheckpoint := context.WithTimeout(ctx, 30*time.Second) + var checkpointConnectionID int64 + done := make(chan error, 1) + workerDone := make(chan struct{}) + go func() { + defer close(workerDone) + _, err := migrator.Checkpoint(checkpointCtx) + _ = base.SendWithContext(ctx, done, err) + }() + defer func() { + // Release both waits on assertion failures, including mutation tests + // that remove cancellation propagation into the SQL operation. + cancelCheckpoint() + // Driver cancellation closes the client connection, but MySQL may + // retain a lock-blocked statement. Kill our test connection before + // unlocking so that cleanup cannot accidentally execute its INSERT. + if checkpointConnectionID != 0 { + _, err := migrator.applier.db.ExecContext(ctx, fmt.Sprintf("KILL CONNECTION %d", checkpointConnectionID)) + var mysqlErr *drivermysql.MySQLError + if !errors.As(err, &mysqlErr) || mysqlErr.Number != 1094 { // ER_NO_SUCH_THREAD: already disconnected + assert.NoError(t, err) + } + } + _, err := blocker.ExecContext(ctx, "UNLOCK TABLES") + assert.NoError(t, err) + select { + case <-workerDone: + case <-ctx.Done(): + t.Error("checkpoint worker did not exit during cleanup") + } + }() + + require.EventuallyWithT(t, func(collect *assert.CollectT) { + // Observing the server-side lock wait proves that Checkpoint passed + // its abort/polling checks and the INSERT is actually executing. + err := migrator.applier.db.QueryRowContext(ctx, ` + SELECT ID FROM information_schema.PROCESSLIST + WHERE DB = ? AND INFO LIKE '%insert /* gh-ost */%' + AND LOCATE(?, INFO) > 0 AND STATE = 'Waiting for table metadata lock'`, + migrationContext.DatabaseName, checkpointTable).Scan(&checkpointConnectionID) + assert.NoError(collect, err) + }, 5*time.Second, 10*time.Millisecond, "checkpoint INSERT never entered the table-lock wait") + select { + case err := <-done: + t.Fatalf("checkpoint returned before cancellation: %v", err) + default: + } + + migrationContext.CancelContext() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("migration cancellation did not interrupt the blocked checkpoint INSERT") + } + require.NoError(t, checkpointCtx.Err(), "checkpoint's caller must remain live") + + // READ permits this SELECT while still preventing the pending INSERT. + // Check before cleanup: neither the lock nor the caller has been canceled + // to make the checkpoint call return. + stored, err := migrator.applier.ReadLastCheckpoint() + require.NoError(t, err) + require.Equal(t, previous.Id, stored.Id) + require.True(t, previous.LastTrxCoords.Equals(stored.LastTrxCoords)) +} + +func TestCheckpointLoopStopsOnCancellation(t *testing.T) { + migrationContext := newTestMigrationContext() + defer migrationContext.CancelContext() + migrationContext.CheckpointIntervalSeconds = 3600 + migrator := NewMigrator(migrationContext, "test") + done := make(chan struct{}) + go func() { + defer close(done) + migrator.checkpointLoop() + }() + migrationContext.CancelContext() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("checkpoint loop waited for its ticker after cancellation") + } +} diff --git a/go/logic/checkpoint_review_test.go b/go/logic/checkpoint_review_test.go new file mode 100644 index 000000000..a51c853ef --- /dev/null +++ b/go/logic/checkpoint_review_test.go @@ -0,0 +1,178 @@ +package logic + +import ( + "context" + "database/sql/driver" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/github/gh-ost/go/binlog" + "github.com/github/gh-ost/go/mysql" + "github.com/stretchr/testify/require" +) + +func TestMoveTablesCutOverCheckpointUsesAppliedBoundary(t *testing.T) { + var written []driver.NamedValue + apl := newCheckpointExecApplier(t, func(_ context.Context, _ string, args []driver.NamedValue) (driver.Result, error) { + written = append([]driver.NamedValue(nil), args...) + return driver.RowsAffected(2), nil + }) + mc := apl.migrationContext + defer mc.CancelContext() + mc.InitMoveTableContainers() + m := NewMigrator(mc, "test") + m.applier = apl + start, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1") + require.NoError(t, err) + partial, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-2") + require.NoError(t, err) + apl.CurrentCoordinates = partial.Clone() + apl.AppliedTransactionCoordinates = start.Clone() + // Recovery must be recorded even after cancellation, and row-level GTID + // progress must never be used as the resume boundary. + mc.CancelContext() + require.NoError(t, m.persistMoveTablesCutOverCheckpoint(partial, false)) + require.Len(t, written, 2*len(moveTablesCheckpointColumns)) + for i := 0; i < 2; i++ { + require.Equal(t, start.String(), written[i*len(moveTablesCheckpointColumns)+2].Value) + require.Equal(t, true, written[i*len(moveTablesCheckpointColumns)+7].Value) + require.Equal(t, partial.String(), written[i*len(moveTablesCheckpointColumns)+8].Value) + } + apl.AppliedTransactionCoordinates = nil + require.ErrorContains(t, m.persistMoveTablesCutOverCheckpoint(partial, false), "no applied transaction boundary") +} + +func TestMoveTablesCheckpointDoesNotBlockApplyAndOrdersRecovery(t *testing.T) { + entered := make(chan struct{}) + recoveryEntered := make(chan struct{}) + release := make(chan struct{}) + writes := make(chan []driver.NamedValue, 2) + var calls int32 + apl := newCheckpointExecApplier(t, func(ctx context.Context, _ string, args []driver.NamedValue) (driver.Result, error) { + writes <- append([]driver.NamedValue(nil), args...) + if atomic.AddInt32(&calls, 1) == 1 { + close(entered) + select { + case <-release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } else { + close(recoveryEntered) + } + return driver.RowsAffected(2), nil + }) + mc := apl.migrationContext + defer mc.CancelContext() + mc.InitMoveTableContainers() + m := NewMigrator(mc, "test") + m.applier = apl + start := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 100) + end := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 200) + apl.AppliedTransactionCoordinates = start.Clone() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + periodic := make(chan error, 1) + go func() { _, err := m.Checkpoint(ctx); periodic <- err }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("periodic checkpoint did not enter write") + } + defer func() { + select { + case <-release: + default: + close(release) + } + }() + applied := make(chan error, 1) + go func() { applied <- m.onApplyEventStruct(&applyEventStruct{coords: end, transactionComplete: true}) }() + select { + case err := <-applied: + require.NoError(t, err) + case <-ctx.Done(): + t.Fatal("checkpoint SQL blocked transaction acknowledgement") + } + atomic.StoreInt64(&mc.InCutOverCriticalSectionFlag, 1) + recovery := make(chan error, 1) + recoveryStarted := make(chan struct{}) + go func() { + close(recoveryStarted) + recovery <- m.persistMoveTablesCutOverCheckpoint(end, false) + }() + <-recoveryStarted + // A canceled waiter must not be trapped behind the outstanding write. + waitCtx, waitCancel := context.WithCancel(ctx) + waitCancel() + _, err := m.Checkpoint(waitCtx) + require.ErrorIs(t, err, context.Canceled) + select { + case <-recoveryEntered: + t.Fatal("recovery write overtook the in-flight periodic checkpoint") + case <-time.After(50 * time.Millisecond): + } + close(release) + require.NoError(t, <-periodic) + require.NoError(t, <-recovery) + first, second := <-writes, <-writes + require.Equal(t, start.String(), first[2].Value) + require.Equal(t, false, first[7].Value) + require.Equal(t, end.String(), second[2].Value) + require.Equal(t, true, second[7].Value) + _, err = m.Checkpoint(ctx) + require.ErrorContains(t, err, "cutover started") +} + +func TestMoveTablesDrainUsesAppliedBoundaryWithBacklog(t *testing.T) { + mc := newTestMigrationContext() + defer mc.CancelContext() + mc.CutOverLockTimeoutSeconds = 1 + m := NewMigrator(mc, "test") + m.applier = NewApplier(mc) + drain, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-2") + require.NoError(t, err) + m.applier.AppliedTransactionCoordinates = drain.Clone() + m.applyEventsQueue <- &applyEventStruct{coords: drain, transactionComplete: true} + require.NoError(t, m.drainMoveTablesCutOver(drain)) + require.Len(t, m.applyEventsQueue, 1) +} + +func TestWaitForGhostTableMigratedReturnsListenerFailure(t *testing.T) { + mc := newTestMigrationContext() + defer mc.CancelContext() + m := NewMigrator(mc, "test") + streamer := NewEventsStreamer(mc) + failure := errors.New("changelog listener failed") + require.NoError(t, streamer.AddListener(false, mc.DatabaseName, mc.GetChangelogTableName(), func(*binlog.BinlogEntry) error { + return failure + })) + done := make(chan error, 1) + go func() { done <- m.waitForGhostTableMigrated() }() + err := streamer.notifyListeners(&binlog.BinlogEntry{DmlEvent: binlog.NewBinlogDMLEvent(mc.DatabaseName, mc.GetChangelogTableName(), binlog.InsertDML)}) + require.ErrorIs(t, err, failure) + select { + case err := <-done: + require.ErrorIs(t, err, failure) + case <-time.After(time.Second): + t.Fatal("startup did not stop after listener failure") + } +} + +func TestWaitForGhostTableMigratedSuccessAndCancellation(t *testing.T) { + t.Run("success", func(t *testing.T) { + mc := newTestMigrationContext() + defer mc.CancelContext() + m := NewMigrator(mc, "test") + go func() { m.ghostTableMigrated <- true }() + require.NoError(t, m.waitForGhostTableMigrated()) + }) + t.Run("canceled", func(t *testing.T) { + mc := newTestMigrationContext() + m := NewMigrator(mc, "test") + mc.CancelContext() + require.ErrorIs(t, m.waitForGhostTableMigrated(), context.Canceled) + }) +} diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 2e72149e6..b8b2f5a0d 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -56,10 +56,11 @@ type lockProcessedStruct struct { } type applyEventStruct struct { - writeFunc *tableWriteFunc - dmlEvent *binlog.BinlogDMLEvent - coords mysql.BinlogCoordinates - eventTimestamp time.Time + writeFunc *tableWriteFunc + dmlEvent *binlog.BinlogDMLEvent + coords mysql.BinlogCoordinates + eventTimestamp time.Time + transactionComplete bool } func newApplyEventStructByFunc(writeFunc *tableWriteFunc) *applyEventStruct { @@ -114,7 +115,9 @@ type Migrator struct { applyEventsQueue chan *applyEventStruct applyEventsInFlight int64 - finishedMigrating int64 + // Serializes move-table checkpoint snapshots and writes without blocking apply. + moveTablesCheckpointLock chan struct{} + finishedMigrating int64 } func NewMigrator(context *base.MigrationContext, appVersion string) *Migrator { @@ -136,9 +139,10 @@ func NewMigrator(context *base.MigrationContext, appVersion string) *Migrator { // and OOM when MaxRetries() is extremely large. allEventsUpToLockProcessed: make(chan *lockProcessedStruct, 1), - copyRowsQueue: make(chan tableWriteFunc), - applyEventsQueue: make(chan *applyEventStruct, base.MaxEventsBatchSize), - finishedMigrating: 0, + copyRowsQueue: make(chan tableWriteFunc), + applyEventsQueue: make(chan *applyEventStruct, base.MaxEventsBatchSize), + moveTablesCheckpointLock: make(chan struct{}, 1), + finishedMigrating: 0, } return migrator } @@ -490,6 +494,14 @@ func (mgtr *Migrator) checkAbort() error { return nil } +func (mgtr *Migrator) waitForGhostTableMigrated() error { + select { + case <-mgtr.ghostTableMigrated: + case <-mgtr.migrationContext.GetContext().Done(): + } + return mgtr.checkAbort() +} + // Migrate executes the complete migration logic. This is *the* major gh-ost function. func (mgtr *Migrator) Migrate() (err error) { mgtr.migrationContext.Log.Infof("Migrating %s.%s", sql.EscapeName(mgtr.migrationContext.DatabaseName), sql.EscapeName(mgtr.migrationContext.OriginalTableName)) @@ -571,7 +583,9 @@ func (mgtr *Migrator) Migrate() (err error) { initialLag, _ := mgtr.inspector.getReplicationLag() if !mgtr.migrationContext.Resume { mgtr.migrationContext.Log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag) - <-mgtr.ghostTableMigrated + if err := mgtr.waitForGhostTableMigrated(); err != nil { + return err + } mgtr.migrationContext.Log.Debugf("ghost table migrated") } // Yay! We now know the Ghost and Changelog tables are good to examine! @@ -980,22 +994,15 @@ func (mgtr *Migrator) validateMoveTablesResumeSourceTables(renamed bool) error { } func (mgtr *Migrator) persistMoveTablesCutOverCheckpoint(drainGTID mysql.BinlogCoordinates, isCutover bool) error { - mgtr.applier.CurrentCoordinatesMutex.Lock() - safeCoords := mgtr.applier.CurrentCoordinates - mgtr.applier.CurrentCoordinatesMutex.Unlock() + // Serialize client writes with periodic snapshots. A canceled SQL statement + // may still commit on the server, so recovery reads select cutover rows. + mgtr.moveTablesCheckpointLock <- struct{}{} + defer func() { <-mgtr.moveTablesCheckpointLock }() + safeCoords := mgtr.appliedTransactionCoordinates() if safeCoords == nil || safeCoords.IsEmpty() { - // This quiet-source fallback does not prove a completed transaction boundary; - // it remains subject to the shared checkpoint issue #1781. - if mgtr.eventsStreamer != nil && len(mgtr.applyEventsQueue) == 0 && len(mgtr.eventsStreamer.eventsChannel) == 0 { - safeCoords = mgtr.eventsStreamer.GetCurrentBinlogCoordinates() - } - if safeCoords == nil || safeCoords.IsEmpty() { - return errors.New("current coordinates are empty, cannot checkpoint move-tables cutover") - } + return errors.New("no applied transaction boundary, cannot checkpoint move-tables cutover") } - safeCoords = safeCoords.Clone() - rows := mgtr.buildMoveTableCheckpointRows(safeCoords, isCutover, true, drainGTID) return mgtr.applier.WriteMoveTableCheckpoints(rows) } @@ -1047,7 +1054,13 @@ func (mgtr *Migrator) drainMoveTablesCutOver(drainGTID mysql.BinlogCoordinates) if err := mgtr.checkAbort(); err != nil { return err } - // Primary signal: applier's coordinate (advances when relevant apply work runs). + // A completed boundary proves that all prior DML committed, even if + // unrelated transactions continue to fill the queues after the rename. + if moveTablesDrainCoordinateReached(mgtr.appliedTransactionCoordinates(), drainGTID) { + mgtr.migrationContext.Log.Infof("T3: drain complete at applied transaction boundary") + return nil + } + // Legacy fallback for a non-DML tail without a completion marker. mgtr.applier.CurrentCoordinatesMutex.Lock() applierCoords := mgtr.applier.CurrentCoordinates mgtr.applier.CurrentCoordinatesMutex.Unlock() @@ -2578,6 +2591,13 @@ func (mgtr *Migrator) initiateStreaming() error { if err := mgtr.eventsStreamer.InitDBConnections(); err != nil { return err } + // The starting position is a safe boundary before any streamed DML. In + // move-tables mode there is no heartbeat to advance it on a quiet source. + if mgtr.migrationContext.IsMoveTablesMode() { + mgtr.applier.CurrentCoordinatesMutex.Lock() + mgtr.applier.AppliedTransactionCoordinates = mgtr.eventsStreamer.initialBinlogCoordinates.Clone() + mgtr.applier.CurrentCoordinatesMutex.Unlock() + } if mgtr.migrationContext.IsMoveTablesMode() { mgtr.migrationContext.Log.Info("Skipping stream of the changelog table") @@ -2640,6 +2660,8 @@ func (mgtr *Migrator) initiateStreaming() error { // by table name. The streamer already dispatches per (database, table), and the // applier routes DML to the right per-table query builders by table name. func (mgtr *Migrator) addDMLEventsListener() error { + // Register before the DML listeners to preserve row/commit order. + mgtr.eventsStreamer.AddTransactionCompleteListener(mgtr.onTransactionComplete) enqueue := func(dmlEntry *binlog.BinlogEntry) error { // Record that the streamer just delivered an event for a moved table, so // the idle-bump rule can tell "falling behind" from "source is quiet". @@ -2673,6 +2695,14 @@ func (mgtr *Migrator) addDMLEventsListener() error { ) } +// onTransactionComplete enqueues an acknowledgement, not another SQL write. +// The applier may batch across these markers, but cannot acknowledge them +// until the batch containing their preceding DML has successfully committed. +func (mgtr *Migrator) onTransactionComplete(entry *binlog.BinlogEntry) error { + marker := &applyEventStruct{coords: entry.Coordinates.Clone(), transactionComplete: true} + return base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.applyEventsQueue, marker) +} + // initiateThrottler kicks in the throttling collection and the throttling checks. func (mgtr *Migrator) initiateThrottler() { mgtr.throttler = NewThrottler(mgtr.migrationContext, mgtr.applier, mgtr.inspector, mgtr.appVersion) @@ -3017,7 +3047,17 @@ func (mgtr *Migrator) iterateChunksMoveTables() error { func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { atomic.AddInt64(&mgtr.applyEventsInFlight, 1) defer atomic.AddInt64(&mgtr.applyEventsInFlight, -1) + // Cancellation may happen after the worker's loop check, while throttled. + // A standalone marker must not bypass the abort checks used by SQL tasks. + if err := mgtr.checkAbort(); err != nil { + return err + } handleNonDMLEventStruct := func(eventStruct *applyEventStruct) error { + if eventStruct.transactionComplete { + mgtr.applier.CurrentCoordinatesMutex.Lock() + mgtr.applier.AppliedTransactionCoordinates = eventStruct.coords.Clone() + mgtr.applier.CurrentCoordinatesMutex.Unlock() + } if eventStruct.writeFunc != nil { if err := mgtr.retryOperation(*eventStruct.writeFunc); err != nil { return mgtr.migrationContext.Log.Errore(err) @@ -3033,16 +3073,19 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { dmlEvents = append(dmlEvents, eventStruct.dmlEvent) lastEventTimestamp := eventStruct.eventTimestamp var nonDmlStructToApply *applyEventStruct + var completedTransaction *applyEventStruct availableEvents := len(mgtr.applyEventsQueue) batchSize := int(atomic.LoadInt64(&mgtr.migrationContext.DMLBatchSize)) - if availableEvents > batchSize-1 { - // The "- 1" is because we already consumed one event: the original event that led to this function getting called. - // So, if DMLBatchSize==1 we wish to not process any further events - availableEvents = batchSize - 1 - } - for i := 0; i < availableEvents; i++ { + for i := 0; i < availableEvents && len(dmlEvents) < batchSize; i++ { additionalStruct := <-mgtr.applyEventsQueue + if additionalStruct.transactionComplete { + // Markers need no SQL and don't count toward DMLBatchSize. + // Acknowledge only after this entire batch succeeds, never while + // merely collecting rows from the queue. + completedTransaction = additionalStruct + continue + } if additionalStruct.dmlEvent == nil { // Not a DML. We don't group this, and we don't batch any further nonDmlStructToApply = additionalStruct @@ -3061,6 +3104,9 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { // update applier coordinates mgtr.applier.CurrentCoordinatesMutex.Lock() mgtr.applier.CurrentCoordinates = eventStruct.coords + if completedTransaction != nil { + mgtr.applier.AppliedTransactionCoordinates = completedTransaction.coords.Clone() + } mgtr.applier.CurrentCoordinatesMutex.Unlock() // In move-tables mode there is no changelog heartbeat; writer lag is derived @@ -3081,13 +3127,24 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { } // Checkpoint attempts to write a checkpoint of the Migrator's current state. -// It gets the binlog coordinates of the last received trx and waits until the -// applier reaches that trx. At that point it's safe to resume from these coordinates. +// It samples reader progress and waits for an applied transaction boundary +// covering that progress. Row/batch progress alone is not a safe resume point. func (mgtr *Migrator) Checkpoint(ctx context.Context) (*Checkpoint, error) { - coords := mgtr.eventsStreamer.GetCurrentBinlogCoordinates() + if err := mgtr.checkAbort(); err != nil { + return nil, err + } + // Observe both the caller's deadline and migration cancellation, including + // while executing the checkpoint INSERT rather than only while polling. + ctx, cancel := context.WithCancel(ctx) + //nolint:contextcheck // Merge migration cancellation with the caller-derived checkpoint context. + stopCancel := context.AfterFunc(mgtr.migrationContext.GetContext(), cancel) + defer stopCancel() + defer cancel() + if mgtr.migrationContext.IsMoveTablesMode() { - return mgtr.checkpointMoveTables(ctx, coords) + return mgtr.checkpointMoveTables(ctx) } + coords := mgtr.eventsStreamer.GetCurrentBinlogCoordinates() mgtr.applier.LastIterationRangeMutex.Lock() if mgtr.applier.LastIterationRangeMaxValues == nil || mgtr.applier.LastIterationRangeMinValues == nil { mgtr.applier.LastIterationRangeMutex.Unlock() @@ -3097,25 +3154,37 @@ func (mgtr *Migrator) Checkpoint(ctx context.Context) (*Checkpoint, error) { Iteration: mgtr.migrationContext.GetIteration(), IterationRangeMin: mgtr.applier.LastIterationRangeMinValues.Clone(), IterationRangeMax: mgtr.applier.LastIterationRangeMaxValues.Clone(), - LastTrxCoords: coords, RowsCopied: atomic.LoadInt64(&mgtr.migrationContext.TotalRowsCopied), DMLApplied: atomic.LoadInt64(&mgtr.migrationContext.TotalDMLEventsApplied), } mgtr.applier.LastIterationRangeMutex.Unlock() for { + if err := mgtr.checkAbort(); err != nil { + return nil, err + } if err := ctx.Err(); err != nil { return nil, err } mgtr.applier.CurrentCoordinatesMutex.Lock() - if coords.SmallerThanOrEquals(mgtr.applier.CurrentCoordinates) { - id, err := mgtr.applier.WriteCheckpoint(chk) + applied := mgtr.applier.AppliedTransactionCoordinates + if applied != nil && coords.SmallerThanOrEquals(applied) { + // Persist the acknowledged commit boundary, not the sampled reader + // position, which may be inside a row event or transaction. + chk.LastTrxCoords = applied.Clone() + id, err := mgtr.applier.WriteCheckpoint(ctx, chk) chk.Id = id mgtr.applier.CurrentCoordinatesMutex.Unlock() return chk, err } mgtr.applier.CurrentCoordinatesMutex.Unlock() - time.Sleep(500 * time.Millisecond) + sleepDuration := 500 * time.Millisecond + metrics.RecordSleep(mgtr.migrationContext.Metrics, "replica_wait", sleepDuration) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(sleepDuration): + } } } @@ -3157,33 +3226,43 @@ func (mgtr *Migrator) moveTablesCheckpointSummary(coords mysql.BinlogCoordinates } } -// checkpointMoveTables writes one checkpoint row per migrated table based on -// applier coordinates or idle queues. Neither proves a completed transaction; -// see #1781. -func (mgtr *Migrator) checkpointMoveTables(ctx context.Context, coords mysql.BinlogCoordinates) (*Checkpoint, error) { - for { - if err := ctx.Err(); err != nil { - return nil, err - } - mgtr.applier.CurrentCoordinatesMutex.Lock() - applied := coords.SmallerThanOrEquals(mgtr.applier.CurrentCoordinates) - idle := len(mgtr.applyEventsQueue) == 0 && (mgtr.eventsStreamer == nil || len(mgtr.eventsStreamer.eventsChannel) == 0) - if applied || idle { - if !applied { - mgtr.applier.CurrentCoordinates = coords.Clone() - } - mgtr.applier.CurrentCoordinatesMutex.Unlock() - rows := mgtr.buildMoveTableCheckpointRows(coords, false, false, nil) - if err := mgtr.applier.WriteMoveTableCheckpoints(rows); err != nil { - return nil, err - } - return mgtr.moveTablesCheckpointSummary(coords), nil - } - mgtr.applier.CurrentCoordinatesMutex.Unlock() - sleepDuration := 500 * time.Millisecond - metrics.RecordSleep(mgtr.migrationContext.Metrics, "replica_wait", sleepDuration) - time.Sleep(sleepDuration) +func (mgtr *Migrator) appliedTransactionCoordinates() mysql.BinlogCoordinates { + mgtr.applier.CurrentCoordinatesMutex.Lock() + defer mgtr.applier.CurrentCoordinatesMutex.Unlock() + if mgtr.applier.AppliedTransactionCoordinates == nil { + return nil + } + return mgtr.applier.AppliedTransactionCoordinates.Clone() +} + +// checkpointMoveTables persists the last applied transaction boundary. Unlike +// standard mode, there is no source heartbeat to advance the reader on a quiet +// source, so a safe older boundary is preferable to waiting for reader progress. +func (mgtr *Migrator) checkpointMoveTables(ctx context.Context) (*Checkpoint, error) { + select { + case mgtr.moveTablesCheckpointLock <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-mgtr.moveTablesCheckpointLock }() + if err := mgtr.checkAbort(); err != nil { + return nil, err } + if err := ctx.Err(); err != nil { + return nil, err + } + if atomic.LoadInt64(&mgtr.migrationContext.InCutOverCriticalSectionFlag) > 0 || atomic.LoadInt64(&mgtr.migrationContext.MoveTablesSourceRenamedFlag) > 0 { + return nil, errors.New("move-tables cutover started, not checkpointing") + } + coords := mgtr.appliedTransactionCoordinates() + if coords == nil || coords.IsEmpty() { + return nil, errors.New("no applied transaction boundary, not checkpointing") + } + rows := mgtr.buildMoveTableCheckpointRows(coords, false, false, nil) + if err := mgtr.applier.writeMoveTableCheckpoints(ctx, rows); err != nil { + return nil, err + } + return mgtr.moveTablesCheckpointSummary(coords), nil } // CheckpointAfterCutOver writes a final checkpoint after the cutover completes successfully. @@ -3210,7 +3289,7 @@ func (mgtr *Migrator) CheckpointAfterCutOver() (*Checkpoint, error) { } mgtr.applier.LastIterationRangeMutex.Unlock() - id, err := mgtr.applier.WriteCheckpoint(chk) + id, err := mgtr.applier.WriteCheckpoint(mgtr.migrationContext.GetContext(), chk) chk.Id = id return chk, err } @@ -3222,7 +3301,15 @@ func (mgtr *Migrator) checkpointLoop() { } checkpointInterval := time.Duration(mgtr.migrationContext.CheckpointIntervalSeconds) * time.Second ticker := time.NewTicker(checkpointInterval) - for t := range ticker.C { + defer ticker.Stop() + migrationCtx := mgtr.migrationContext.GetContext() + for { + var t time.Time + select { + case <-migrationCtx.Done(): + return + case t = <-ticker.C: + } if atomic.LoadInt64(&mgtr.finishedMigrating) > 0 || atomic.LoadInt64(&mgtr.migrationContext.CutOverCompleteFlag) > 0 { return } @@ -3230,7 +3317,7 @@ func (mgtr *Migrator) checkpointLoop() { continue } mgtr.migrationContext.Log.Infof("starting checkpoint at %+v", t) - ctx, cancel := context.WithTimeout(context.Background(), checkpointTimeout) + ctx, cancel := context.WithTimeout(migrationCtx, checkpointTimeout) chk, err := mgtr.Checkpoint(ctx) if err != nil { if errors.Is(err, context.DeadlineExceeded) { diff --git a/go/logic/migrator_move_tables_cutover_test.go b/go/logic/migrator_move_tables_cutover_test.go index d45c0d708..734a774db 100644 --- a/go/logic/migrator_move_tables_cutover_test.go +++ b/go/logic/migrator_move_tables_cutover_test.go @@ -330,7 +330,8 @@ func (s *MoveTablesCutOverSuite) buildMigrator(fakeHooks base.Hooks, initialCoor m.applier = NewApplier(mc) if initialCoords != nil { m.applier.CurrentCoordinatesMutex.Lock() - m.applier.CurrentCoordinates = initialCoords + m.applier.CurrentCoordinates = initialCoords.Clone() + m.applier.AppliedTransactionCoordinates = initialCoords.Clone() m.applier.CurrentCoordinatesMutex.Unlock() } return m, mc @@ -352,6 +353,38 @@ func (s *MoveTablesCutOverSuite) setupCutOverCheckpoint(m *Migrator, mc *base.Mi } } +func (s *MoveTablesCutOverSuite) TestMoveTablesCheckpointUsesAppliedTransactionBoundary() { + start, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1") + s.Require().NoError(err) + end, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-2") + s.Require().NoError(err) + m, mc := s.buildMigrator(nil, end) + s.setupCutOverCheckpoint(m, mc, false) + m.applier.AppliedTransactionCoordinates = start.Clone() + + // Row progress may already be at end, but without its commit marker only + // the earlier boundary is safe to persist and replay from. + chk, err := m.Checkpoint(context.Background()) + s.Require().NoError(err) + s.Require().True(chk.LastTrxCoords.Equals(start)) + rows, err := m.applier.ReadMoveTableCheckpoints() + s.Require().NoError(err) + s.Require().True(rows[testMysqlTableName].LastTrxCoords.Equals(start)) + + s.Require().NoError(m.onApplyEventStruct(&applyEventStruct{coords: end, transactionComplete: true})) + chk, err = m.Checkpoint(context.Background()) + s.Require().NoError(err) + s.Require().True(chk.LastTrxCoords.Equals(end)) + rows, err = m.applier.ReadMoveTableCheckpoints() + s.Require().NoError(err) + s.Require().True(rows[testMysqlTableName].LastTrxCoords.Equals(end)) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = m.Checkpoint(ctx) + s.Require().ErrorIs(err, context.Canceled) +} + func (s *MoveTablesCutOverSuite) TestMoveTablesListenersBeforeStreaming() { ctx := context.Background() tableNames := []string{testMysqlTableName, "testing_other"} @@ -380,6 +413,7 @@ func (s *MoveTablesCutOverSuite) TestMoveTablesListenersBeforeStreaming() { mc.MoveTables.TableNames = tableNames mc.UseGTIDs = true m := NewMigrator(mc, "test") + m.applier = NewApplier(mc) s.Require().NoError(m.initiateStreaming()) s.T().Cleanup(func() { atomic.StoreInt64(&m.finishedMigrating, 1) @@ -401,11 +435,15 @@ func (s *MoveTablesCutOverSuite) TestMoveTablesListenersBeforeStreaming() { seen := make(map[string]int) deadline := time.NewTimer(10 * time.Second) defer deadline.Stop() - for range len(tableNames) * rowsPerTable { + for rows := 0; rows < len(tableNames)*rowsPerTable; { select { case event := <-m.applyEventsQueue: + if event.transactionComplete { + continue + } s.Require().NotNil(event.dmlEvent) seen[event.dmlEvent.TableName]++ + rows++ case <-deadline.C: s.FailNow("timed out waiting for pending DML") } @@ -762,6 +800,8 @@ func (s *MoveTablesCutOverSuite) TestDrainWaitsForQueuedDML() { fakeHooks := &recordingHooks{name: "fake", calls: &calls} m, mc := s.buildMigrator(fakeHooks, s.containingDrainGTID()) mc.CutOverLockTimeoutSeconds = 1 + // Row-level progress alone is not an applied transaction boundary. + m.applier.AppliedTransactionCoordinates = nil m.applyEventsQueue <- newApplyEventStructByDML(&binlog.BinlogEntry{ DmlEvent: &binlog.BinlogDMLEvent{ DatabaseName: testMysqlDatabase, @@ -829,6 +869,7 @@ func (s *MoveTablesCutOverSuite) TestDrainWaitsForInFlightApplyEvent() { mc.CutOverLockTimeoutSeconds = 1 m.applier.CurrentCoordinatesMutex.Lock() m.applier.CurrentCoordinates = drainGTID + m.applier.AppliedTransactionCoordinates = nil m.applier.CurrentCoordinatesMutex.Unlock() started := make(chan struct{}) diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 7d3d00120..355c882be 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -40,6 +40,7 @@ type EventsStreamer struct { migrationContext *base.MigrationContext initialBinlogCoordinates mysql.BinlogCoordinates listeners [](*BinlogEventListener) + transactionListeners []func(*binlog.BinlogEntry) error listenersMutex *sync.Mutex eventsChannel chan *binlog.BinlogEntry binlogReader *binlog.GoMySQLReader @@ -80,6 +81,14 @@ func (es *EventsStreamer) AddListener( return nil } +// AddTransactionCompleteListener registers a synchronous commit listener. +// Consumers must also use synchronous DML listeners to preserve queue order. +func (es *EventsStreamer) AddTransactionCompleteListener(listener func(*binlog.BinlogEntry) error) { + es.listenersMutex.Lock() + defer es.listenersMutex.Unlock() + es.transactionListeners = append(es.transactionListeners, listener) +} + // shouldDecodeRowsEvent returns true when at least one listener is registered for // the table on which the rows event operates. This is used by the binlog parser // after it decodes the row event header/table map, but before it decodes row data. @@ -99,11 +108,34 @@ func (es *EventsStreamer) shouldDecodeRowsEvent(databaseName, tableName string) return false } -// notifyListeners will notify relevant listeners with given DML event. Only -// listeners registered for changes on the table on which the DML operates are notified. -func (es *EventsStreamer) notifyListeners(binlogEntry *binlog.BinlogEntry) { +// notifyListeners dispatches table-filtered DML or a transaction completion +// marker. Synchronous listeners run on the same dispatcher to preserve order. +// Failed delivery is fatal: continuing could acknowledge a transaction whose +// preceding DML never reached the apply queue. +func (es *EventsStreamer) notifyListeners(binlogEntry *binlog.BinlogEntry) (err error) { es.listenersMutex.Lock() defer es.listenersMutex.Unlock() + defer func() { + if err != nil { + es.migrationContext.SetAbortError(err) + es.migrationContext.CancelContext() + } + }() + if err := es.migrationContext.GetAbortError(); err != nil { + return err + } + if err := es.migrationContext.GetContext().Err(); err != nil { + return err + } + + if binlogEntry.TransactionComplete { + for _, listener := range es.transactionListeners { + if err := listener(binlogEntry); err != nil { + return err + } + } + return nil + } for _, listener := range es.listeners { listener := listener @@ -117,10 +149,11 @@ func (es *EventsStreamer) notifyListeners(binlogEntry *binlog.BinlogEntry) { go func() { listener.onDmlEvent(binlogEntry) }() - } else { - listener.onDmlEvent(binlogEntry) + } else if err := listener.onDmlEvent(binlogEntry); err != nil { + return err } } + return nil } func (es *EventsStreamer) InitDBConnections() (err error) { @@ -216,19 +249,32 @@ func (es *EventsStreamer) readCurrentMariaDBGTIDCoordinates() error { // StreamEvents will begin streaming events. It will be blocking, so should be // executed by a goroutine func (es *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { + ctx := es.migrationContext.GetContext() go func() { - for binlogEntry := range es.eventsChannel { - if binlogEntry.DmlEvent != nil { - es.notifyListeners(binlogEntry) + for { + select { + case <-ctx.Done(): + return + case binlogEntry, ok := <-es.eventsChannel: + if !ok { + return + } + if binlogEntry.DmlEvent != nil || binlogEntry.TransactionComplete { + if err := es.notifyListeners(binlogEntry); err != nil { + return + } + } } } }() // The next should block and execute forever, unless there's a serious error. var successiveFailures int var reconnectCoords mysql.BinlogCoordinates - ctx := es.migrationContext.GetContext() for { - // Check for context cancellation each iteration + // Preserve a listener's failure rather than hiding it as cancellation. + if err := es.migrationContext.GetAbortError(); err != nil { + return err + } if err := ctx.Err(); err != nil { return err } @@ -239,6 +285,12 @@ func (es *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { // of the last trx that was read completely from the streamer. // Since row event application is idempotent, it's OK if we reapply some events. if err := es.binlogReader.StreamEvents(canStopStreaming, es.eventsChannel); err != nil { + if abortErr := es.migrationContext.GetAbortError(); abortErr != nil { + return abortErr + } + if ctx.Err() != nil { + return ctx.Err() + } if canStopStreaming() { return nil } diff --git a/go/logic/streamer_test.go b/go/logic/streamer_test.go index 5b28c47c2..4871e2365 100644 --- a/go/logic/streamer_test.go +++ b/go/logic/streamer_test.go @@ -3,10 +3,12 @@ package logic import ( "context" gosql "database/sql" + "errors" "fmt" "testing" "time" + "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/binlog" "github.com/stretchr/testify/suite" "github.com/testcontainers/testcontainers-go" @@ -140,6 +142,49 @@ func (suite *EventsStreamerTestSuite) TestStreamEvents() { suite.Require().Len(dmlEvents, 3) } +func (suite *EventsStreamerTestSuite) TestStreamEventsStopsOnListenerError() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestTableName())) + suite.Require().NoError(err) + config, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + migrationContext := newTestMigrationContext() + defer migrationContext.CancelContext() + migrationContext.ApplierConnectionConfig = config + migrationContext.InspectorConnectionConfig = config + suite.Require().NoError(migrationContext.SetConnectionConfig("innodb")) + streamer := NewEventsStreamer(migrationContext) + suite.Require().NoError(streamer.InitDBConnections()) + defer streamer.Close() + defer streamer.Teardown() + failure := errors.New("DML enqueue failed") + suite.Require().NoError(streamer.AddListener(false, testMysqlDatabase, testMysqlTableName, func(*binlog.BinlogEntry) error { + return failure + })) + streamer.AddTransactionCompleteListener(func(*binlog.BinlogEntry) error { + suite.T().Error("marker delivered after failed DML") + return nil + }) + done := make(chan error, 1) + go func() { + //nolint:contextcheck // StreamEvents uses the migration context, canceled by listener failure. + err := streamer.StreamEvents(func() bool { return false }) + _ = base.SendWithContext(ctx, done, err) + }() + // Multiple rows also exercise cancellation of reader sends after the + // dispatcher exits, rather than only cancellation of GetEvent. + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s VALUES (1),(2),(3),(4),(5)", getTestTableName())) + suite.Require().NoError(err) + select { + case err := <-done: + suite.Require().ErrorIs(err, failure) + case <-ctx.Done(): + suite.T().Fatal("streamer did not stop after listener failure") + } + suite.Require().ErrorIs(migrationContext.GetAbortError(), failure) +} + func (suite *EventsStreamerTestSuite) TestStreamEventsAutomaticallyReconnects() { ctx := context.Background() _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY, name VARCHAR(255))", getTestTableName())) diff --git a/go/logic/transaction_marker_test.go b/go/logic/transaction_marker_test.go new file mode 100644 index 000000000..4a0339ba0 --- /dev/null +++ b/go/logic/transaction_marker_test.go @@ -0,0 +1,184 @@ +package logic + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/github/gh-ost/go/binlog" + "github.com/github/gh-ost/go/mysql" + "github.com/github/gh-ost/go/sql" + "github.com/stretchr/testify/require" +) + +func TestTransactionMarkersFollowSynchronousDMLListeners(t *testing.T) { + ctx := newTestMigrationContext() + streamer := NewEventsStreamer(ctx) + migrator := NewMigrator(ctx, "test") + migrator.eventsStreamer = streamer + migrator.applier = NewApplier(ctx) + require.NoError(t, migrator.addDMLEventsListener()) + coords := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 1234) + for i := 0; i < 3; i++ { + require.NoError(t, streamer.notifyListeners(&binlog.BinlogEntry{ + Coordinates: coords, + DmlEvent: binlog.NewBinlogDMLEvent(ctx.DatabaseName, ctx.OriginalTableName, binlog.InsertDML), + })) + } + require.NoError(t, streamer.notifyListeners(&binlog.BinlogEntry{Coordinates: coords, TransactionComplete: true})) + require.Len(t, migrator.applyEventsQueue, 4) + require.Nil(t, migrator.applier.AppliedTransactionCoordinates) + for i := 0; i < 3; i++ { + event := <-migrator.applyEventsQueue + require.NotNil(t, event.dmlEvent) + require.False(t, event.transactionComplete) + } + marker := <-migrator.applyEventsQueue + require.True(t, marker.transactionComplete) + require.Nil(t, marker.dmlEvent) +} + +func TestMoveTablesTransactionMarkerFollowsAllTableDML(t *testing.T) { + ctx := newTestMigrationContext() + defer ctx.CancelContext() + ctx.MoveTables.TableNames = []string{"first", "second"} + migrator := NewMigrator(ctx, "test") + migrator.applier = NewApplier(ctx) + migrator.eventsStreamer = NewEventsStreamer(ctx) + require.NoError(t, migrator.addDMLEventsListener()) + coords := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 1234) + timestamp := time.Now() + for _, table := range ctx.MoveTables.TableNames { + require.NoError(t, migrator.eventsStreamer.notifyListeners(&binlog.BinlogEntry{ + Coordinates: coords, + Timestamp: timestamp, + DmlEvent: binlog.NewBinlogDMLEvent(ctx.DatabaseName, table, binlog.InsertDML), + })) + } + require.NoError(t, migrator.eventsStreamer.notifyListeners(&binlog.BinlogEntry{Coordinates: coords, TransactionComplete: true})) + for _, table := range ctx.MoveTables.TableNames { + event := <-migrator.applyEventsQueue + require.Equal(t, table, event.dmlEvent.TableName) + require.Equal(t, timestamp, event.eventTimestamp) + } + marker := <-migrator.applyEventsQueue + require.True(t, marker.transactionComplete) + require.NoError(t, migrator.onApplyEventStruct(marker)) + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(coords)) +} + +func TestTransactionMarkerRejectedAfterAbort(t *testing.T) { + for _, abortErr := range []error{context.Canceled, errors.New("migration aborted")} { + t.Run(abortErr.Error(), func(t *testing.T) { + ctx := newTestMigrationContext() + defer ctx.CancelContext() + migrator := NewMigrator(ctx, "test") + migrator.applier = NewApplier(ctx) + previous := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 100) + migrator.applier.AppliedTransactionCoordinates = previous.Clone() + marker := &applyEventStruct{coords: mysql.NewFileBinlogCoordinates("mysql-bin.000001", 200), transactionComplete: true} + if !errors.Is(abortErr, context.Canceled) { + ctx.SetAbortError(abortErr) + } + ctx.CancelContext() + require.ErrorIs(t, migrator.onApplyEventStruct(marker), abortErr) + require.True(t, previous.Equals(migrator.applier.AppliedTransactionCoordinates)) + }) + } +} + +func TestFailedSynchronousListenerStopsTransactionDelivery(t *testing.T) { + for _, failMarker := range []bool{false, true} { + t.Run(fmt.Sprintf("marker=%t", failMarker), func(t *testing.T) { + ctx := newTestMigrationContext() + defer ctx.CancelContext() + streamer := NewEventsStreamer(ctx) + failure := errors.New("listener rejected event") + called := 0 + fail := func(*binlog.BinlogEntry) error { called++; return failure } + unexpected := func(*binlog.BinlogEntry) error { t.Error("delivered after listener failure"); return nil } + entry := &binlog.BinlogEntry{TransactionComplete: failMarker} + if failMarker { + streamer.AddTransactionCompleteListener(fail) + } else { + entry.DmlEvent = binlog.NewBinlogDMLEvent(ctx.DatabaseName, ctx.OriginalTableName, binlog.DeleteDML) + require.NoError(t, streamer.AddListener(false, ctx.DatabaseName, ctx.OriginalTableName, fail)) + require.NoError(t, streamer.AddListener(false, ctx.DatabaseName, ctx.OriginalTableName, unexpected)) + } + streamer.AddTransactionCompleteListener(unexpected) + require.ErrorIs(t, streamer.notifyListeners(entry), failure) + require.ErrorIs(t, ctx.GetAbortError(), failure) + require.ErrorIs(t, ctx.GetContext().Err(), context.Canceled) + require.ErrorIs(t, streamer.notifyListeners(&binlog.BinlogEntry{TransactionComplete: true}), failure) + require.Equal(t, 1, called) + }) + } +} + +func TestCanceledDispatcherRejectsReadyQueue(t *testing.T) { + ctx := newTestMigrationContext() + migrator := NewMigrator(ctx, "test") + migrator.eventsStreamer = NewEventsStreamer(ctx) + require.NoError(t, migrator.addDMLEventsListener()) + ctx.CancelContext() + coords := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 100) + for _, entry := range []*binlog.BinlogEntry{ + {Coordinates: coords, DmlEvent: binlog.NewBinlogDMLEvent(ctx.DatabaseName, ctx.OriginalTableName, binlog.DeleteDML)}, + {Coordinates: coords, TransactionComplete: true}, + } { + require.ErrorIs(t, migrator.eventsStreamer.notifyListeners(entry), context.Canceled) + } + require.Empty(t, migrator.applyEventsQueue) +} + +func TestTransactionMarkerClonesGTIDCoordinates(t *testing.T) { + for _, test := range []struct { + flavor string + committed string + next string + }{ + {mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1", "00000000-0000-0000-0000-000000000001:2"}, + {mysql.MariaDBFlavor, "0-1-1", "0-1-2"}, + } { + t.Run(test.flavor, func(t *testing.T) { + ctx := newTestMigrationContext() + ctx.UseGTIDs = true + migrator := NewMigrator(ctx, "test") + migrator.applier = NewApplier(ctx) + coords, err := mysql.NewGTIDBinlogCoordinates(test.flavor, test.committed) + require.NoError(t, err) + require.NoError(t, migrator.onTransactionComplete(&binlog.BinlogEntry{Coordinates: coords, TransactionComplete: true})) + require.Nil(t, migrator.applier.AppliedTransactionCoordinates, "enqueueing is not applying") + marker := <-migrator.applyEventsQueue + require.NoError(t, coords.GTIDSet.Update(test.next)) + require.NoError(t, migrator.onApplyEventStruct(marker)) + require.Equal(t, test.committed, migrator.applier.AppliedTransactionCoordinates.String()) + // The acknowledged watermark also owns its own copy of the marker. + require.NoError(t, marker.coords.(*mysql.GTIDBinlogCoordinates).GTIDSet.Update(test.next)) + require.Equal(t, test.committed, migrator.applier.AppliedTransactionCoordinates.String()) + }) + } +} + +func TestHeartbeatDoesNotAcknowledgeItsTransaction(t *testing.T) { + ctx := newTestMigrationContext() + migrator := NewMigrator(ctx, "test") + migrator.applier = NewApplier(ctx) + coords := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 1234) + heartbeat := &binlog.BinlogEntry{ + Coordinates: coords, + DmlEvent: &binlog.BinlogDMLEvent{ + DML: binlog.InsertDML, + NewColumnValues: sql.ToColumnValues([]interface{}{1, 0, "heartbeat", time.Now().Format(time.RFC3339Nano)}), + }, + } + require.NoError(t, migrator.onChangelogHeartbeatEvent(heartbeat)) + require.NoError(t, migrator.onApplyEventStruct(<-migrator.applyEventsQueue)) + require.True(t, migrator.applier.CurrentCoordinates.Equals(coords)) + require.Nil(t, migrator.applier.AppliedTransactionCoordinates, "a heartbeat row may precede more rows in the same transaction") + require.NoError(t, migrator.onTransactionComplete(&binlog.BinlogEntry{Coordinates: coords, TransactionComplete: true})) + require.NoError(t, migrator.onApplyEventStruct(<-migrator.applyEventsQueue)) + require.True(t, migrator.applier.AppliedTransactionCoordinates.Equals(coords)) +}