diff --git a/doc/understanding-output.md b/doc/understanding-output.md index e83da7d7d..e3450e3ae 100644 --- a/doc/understanding-output.md +++ b/doc/understanding-output.md @@ -5,6 +5,16 @@ You can control output levels: - `--verbose`: common use. Useful output, not tons of it - `--debug`: everything. Tons of output. +In move-tables mode, `WriterLag` replaces `Lag` and `HeartbeatLag`. It measures +the age of the source-binlog timestamp of the last applied DML batch. To keep +lag low when the source is idle, gh-ost also probes the source's current binlog +position at `--heartbeat-interval-millis` intervals (100ms by default, capped +at 1000ms). Once the reader has processed that position, a timestamped marker +passes through the same queues as DML and updates lag only after preceding +writes finish. The marker uses the time before the source query, so query and +queueing delays count toward lag. Streamer silence alone never resets lag. +Until the first DML batch or catch-up marker is processed, `WriterLag` is zero. + Initial output lines may look like this: ``` 2016-05-19 17:57:04 INFO starting gh-ost 0.7.14 diff --git a/go/base/context.go b/go/base/context.go index fcf9e6a17..14777c787 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -339,12 +339,9 @@ type MigrationContext struct { pointOfInterestTimeMutex *sync.Mutex lastHeartbeatOnChangelogTime time.Time lastHeartbeatOnChangelogMutex *sync.Mutex - // lastAppliedBinlogEventTime is the binlog-header timestamp of the last event - // applied to the target in move-tables mode. lastBinlogEventStreamedTime is the - // wall-clock time the streamer last delivered an event for the moved table. - // Together they drive the move-tables writer-lag metric (see GetBinlogWriterLag). + // lastAppliedBinlogEventTime is the timestamp of the last applied DML event + // or ordered source-position probe in move-tables mode. lastAppliedBinlogEventTime time.Time - lastBinlogEventStreamedTime time.Time binlogWriterLagMutex *sync.Mutex CurrentLag int64 currentProgress uint64 @@ -908,9 +905,8 @@ func (mctx *MigrationContext) GetLastHeartbeatOnChangelogTime() time.Time { return mctx.lastHeartbeatOnChangelogTime } -// UpdateLastAppliedBinlogEventTime records the binlog-header timestamp of the -// last event successfully applied to the target. Used by move-tables mode to -// derive writer lag. +// UpdateLastAppliedBinlogEventTime records the timestamp of the last event or +// source-position probe successfully processed by the writer in move-tables mode. func (mctx *MigrationContext) UpdateLastAppliedBinlogEventTime(t time.Time) { mctx.binlogWriterLagMutex.Lock() defer mctx.binlogWriterLagMutex.Unlock() @@ -918,28 +914,6 @@ func (mctx *MigrationContext) UpdateLastAppliedBinlogEventTime(t time.Time) { mctx.lastAppliedBinlogEventTime = t } -// MarkBinlogEventStreamed records that the streamer just delivered an event for -// the moved table. Used to distinguish "falling behind" from "source is idle". -func (mctx *MigrationContext) MarkBinlogEventStreamed() { - mctx.binlogWriterLagMutex.Lock() - defer mctx.binlogWriterLagMutex.Unlock() - - mctx.lastBinlogEventStreamedTime = time.Now() -} - -// BumpBinlogWriterLagIfIdle treats prolonged streamer silence as "caught up": -// if no event has been streamed for the moved table within idleThreshold, the -// last-applied timestamp is advanced to now so writer lag does not climb forever -// while the source is quiet. -func (mctx *MigrationContext) BumpBinlogWriterLagIfIdle(idleThreshold time.Duration) { - mctx.binlogWriterLagMutex.Lock() - defer mctx.binlogWriterLagMutex.Unlock() - - if mctx.lastBinlogEventStreamedTime.IsZero() || time.Since(mctx.lastBinlogEventStreamedTime) >= idleThreshold { - mctx.lastAppliedBinlogEventTime = time.Now() - } -} - // GetBinlogWriterLag returns now - last applied event timestamp, the move-tables // writer lag. It returns 0 before any event has been applied. func (mctx *MigrationContext) GetBinlogWriterLag() time.Duration { diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index c1216054c..7a66db195 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -16,8 +16,8 @@ import ( type BinlogEntry struct { Coordinates mysql.BinlogCoordinates // Timestamp is the wall-clock time recorded in the binlog event header of the - // event that produced this entry. It is used in move-tables mode to measure - // writer lag (now - last applied event timestamp). + // event that produced this entry. For a source-position probe (DmlEvent == nil) + // it is the local time before querying the source, not the time of delivery. Timestamp time.Time DmlEvent *BinlogDMLEvent } diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index ee50b0d1a..30164beba 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -6,7 +6,9 @@ package binlog import ( + "errors" "fmt" + "strings" "sync" "github.com/github/gh-ost/go/base" @@ -49,6 +51,8 @@ type GoMySQLReader struct { // LastTrxCoords are the coordinates of the last transaction completely read. // If using the file coordinates it is binlog position of the transaction's XID event. LastTrxCoords mysql.BinlogCoordinates + // ReadSourceCoordinates enables ordered catch-up probes in move-tables mode. + ReadSourceCoordinates func() (mysql.BinlogCoordinates, error) } func NewGoMySQLReader(migrationContext *base.MigrationContext, rowsEventFilters ...RowsEventFilterFunc) *GoMySQLReader { @@ -155,11 +159,32 @@ func (gmr *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent // StreamEvents func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChannel chan<- *BinlogEntry) error { + processedCoordinates := gmr.GetCurrentBinlogCoordinates() + interval := time.Duration(gmr.migrationContext.HeartbeatIntervalMilliseconds) * time.Millisecond + lastProbe := time.Now() + inTransaction := false for !canStopStreaming() { - ev, err := gmr.binlogStreamer.GetEvent(context.Background()) - if err != nil { + ctx := context.Background() + cancel := func() {} + if gmr.ReadSourceCoordinates != nil { + ctx, cancel = context.WithTimeout(gmr.migrationContext.GetContext(), interval) + } + ev, err := gmr.binlogStreamer.GetEvent(ctx) + cancel() + if err != nil && !(gmr.ReadSourceCoordinates != nil && errors.Is(err, context.DeadlineExceeded) && gmr.migrationContext.GetContext().Err() == nil) { return err } + if gmr.ReadSourceCoordinates != nil && time.Since(lastProbe) >= interval { + // processedCoordinates excludes the event just fetched. The marker must + // follow every row it claims to cover, including a complete GTID transaction. + if err := gmr.sendStreamProgress(processedCoordinates, entriesChannel); err != nil { + return err + } + lastProbe = time.Now() + } + if err != nil { + continue + } // Update binlog coords if using file-based coords. // GTID coordinates are updated on receiving GTID events. @@ -214,27 +239,74 @@ func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChan coords.LogFile = string(event.NextLogName) gmr.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", coords.LogFile, int64(ev.Header.LogPos), event.NextLogName) gmr.currentCoordinatesMutex.Unlock() + processedCoordinates = &mysql.FileBinlogCoordinates{ + LogFile: string(event.NextLogName), + LogPos: int64(event.Position), + } case *replication.XIDEvent: + inTransaction = false if gmr.migrationContext.UseGTIDs { // event.GSet is the full executed GTID set maintained by the // syncer (MysqlGTIDSet or MariadbGTIDSet depending on flavor). if event.GSet != nil { gmr.LastTrxCoords = &mysql.GTIDBinlogCoordinates{GTIDSet: event.GSet} + if gmr.ReadSourceCoordinates != nil { + processedCoordinates = gmr.LastTrxCoords.Clone() + } } } else { gmr.LastTrxCoords = gmr.currentCoordinates.Clone() } + case *replication.QueryEvent: + if gmr.ReadSourceCoordinates == nil || !gmr.migrationContext.UseGTIDs { + break + } + switch strings.ToUpper(strings.TrimSpace(string(event.Query))) { + case "BEGIN": + inTransaction = true + case "COMMIT", "ROLLBACK": + inTransaction = false + } + // Standalone DDL has no XID event. BEGIN's GSet, however, already + // includes the transaction whose row events have not arrived yet. + if !inTransaction && event.GSet != nil { + processedCoordinates = (&mysql.GTIDBinlogCoordinates{GTIDSet: event.GSet}).Clone() + } case *replication.RowsEvent: if err := gmr.handleRowsEvent(ev, event, entriesChannel); err != nil { return err } } + if gmr.ReadSourceCoordinates == nil { + continue + } + // Startup's synthetic rotate/format events can have a zero log position. + if !gmr.migrationContext.UseGTIDs && ev.Header.LogPos > 0 && ev.Header.EventType != replication.ROTATE_EVENT { + processedCoordinates = gmr.GetCurrentBinlogCoordinates() + } } gmr.migrationContext.Log.Debugf("done streaming events") return nil } +func (gmr *GoMySQLReader) sendStreamProgress(processed mysql.BinlogCoordinates, entriesChannel chan<- *BinlogEntry) error { + probeTime := time.Now() + source, err := gmr.ReadSourceCoordinates() + if err != nil { + return err + } + // Equality works for file positions and both GTID flavors, without assuming + // that a partially received transaction or an empty apply queue means caught up. + if !processed.Equals(source) { + return nil + } + return base.SendWithContext(gmr.migrationContext.GetContext(), entriesChannel, &BinlogEntry{ + Coordinates: processed.Clone(), + Timestamp: probeTime, + }) +} + func (gmr *GoMySQLReader) Close() error { gmr.binlogSyncer.Close() return nil diff --git a/go/binlog/gomysql_reader_test.go b/go/binlog/gomysql_reader_test.go index 6040716ad..375666cac 100644 --- a/go/binlog/gomysql_reader_test.go +++ b/go/binlog/gomysql_reader_test.go @@ -1,10 +1,16 @@ package binlog import ( + "context" + "errors" "reflect" + "sync" "testing" + "time" "unsafe" + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" "github.com/stretchr/testify/require" @@ -82,3 +88,196 @@ func TestRowsEventDecodeFuncDecodesRowDataForMatchingTable(t *testing.T) { func TestRowsEventDecodeFuncIsNilWithoutFilter(t *testing.T) { require.Nil(t, newRowsEventDecodeFunc(nil)) } + +func TestWriterLagProgressRequiresSourcePosition(t *testing.T) { + for _, flavor := range []string{"file", mysql.MySQLFlavor, mysql.MariaDBFlavor} { + t.Run(flavor, func(t *testing.T) { + var processed, source mysql.BinlogCoordinates + switch flavor { + case "file": + processed = &mysql.FileBinlogCoordinates{LogFile: "mysql-bin.000001", LogPos: 100} + source = &mysql.FileBinlogCoordinates{LogFile: "mysql-bin.000001", LogPos: 200} + case mysql.MySQLFlavor: + processed, _ = mysql.NewGTIDBinlogCoordinates(flavor, "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa:1") + source, _ = mysql.NewGTIDBinlogCoordinates(flavor, "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa:1-2") + case mysql.MariaDBFlavor: + processed, _ = mysql.NewGTIDBinlogCoordinates(flavor, "0-1-1") + source, _ = mysql.NewGTIDBinlogCoordinates(flavor, "0-1-2") + } + mc := base.NewMigrationContext() + defer mc.CancelContext() + reader := &GoMySQLReader{ + migrationContext: mc, + ReadSourceCoordinates: func() (mysql.BinlogCoordinates, error) { + return source, nil + }, + } + entries := make(chan *BinlogEntry, 1) + require.NoError(t, reader.sendStreamProgress(processed, entries)) + require.Empty(t, entries, "streamer silence must not conceal unread source events") + + source = processed.Clone() + var queryStarted time.Time + reader.ReadSourceCoordinates = func() (mysql.BinlogCoordinates, error) { + queryStarted = time.Now() + return source, nil + } + require.NoError(t, reader.sendStreamProgress(processed, entries)) + progress := <-entries + require.Nil(t, progress.DmlEvent) + require.False(t, progress.Timestamp.After(queryStarted), "query/delivery delay must count as lag") + }) + } +} + +func TestWriterLagProgressErrors(t *testing.T) { + mc := base.NewMigrationContext() + defer mc.CancelContext() + coords := &mysql.FileBinlogCoordinates{LogFile: "mysql-bin.000001", LogPos: 100} + boom := errors.New("source probe failed") + reader := &GoMySQLReader{ + migrationContext: mc, + ReadSourceCoordinates: func() (mysql.BinlogCoordinates, error) { + return nil, boom + }, + } + entries := make(chan *BinlogEntry) + require.ErrorIs(t, reader.sendStreamProgress(coords, entries), boom) + + reader.ReadSourceCoordinates = func() (mysql.BinlogCoordinates, error) { return coords, nil } + mc.CancelContext() + require.ErrorIs(t, reader.sendStreamProgress(coords, entries), context.Canceled) +} + +func TestWriterLagProgressOnIdleStream(t *testing.T) { + mc := base.NewMigrationContext() + defer mc.CancelContext() + mc.HeartbeatIntervalMilliseconds = 1 + coords := &mysql.FileBinlogCoordinates{LogFile: "mysql-bin.000001", LogPos: 100} + reader := &GoMySQLReader{ + migrationContext: mc, + currentCoordinates: coords.Clone(), + currentCoordinatesMutex: &sync.Mutex{}, + binlogStreamer: replication.NewBinlogStreamer(), + ReadSourceCoordinates: func() (mysql.BinlogCoordinates, error) { + return coords.Clone(), nil + }, + } + // Startup's synthetic events must not move the processed position backwards. + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(&replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.ROTATE_EVENT}, + Event: &replication.RotateEvent{NextLogName: []byte(coords.LogFile), Position: uint64(coords.LogPos)}, + })) + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(&replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.FORMAT_DESCRIPTION_EVENT}, + Event: &replication.FormatDescriptionEvent{}, + })) + entries := make(chan *BinlogEntry) + done := make(chan error, 1) + go func() { done <- reader.StreamEvents(func() bool { return false }, entries) }() + select { + case progress := <-entries: + require.Nil(t, progress.DmlEvent) + require.True(t, coords.Equals(progress.Coordinates)) + case <-time.After(5 * time.Second): + t.Fatal("idle source did not produce a catch-up marker") + } + mc.CancelContext() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestWriterLagProgressWaitsForCompleteGTIDTransaction(t *testing.T) { + mc := base.NewMigrationContext() + defer mc.CancelContext() + mc.UseGTIDs = true + mc.HeartbeatIntervalMilliseconds = 1 + initial, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa:1") + require.NoError(t, err) + source, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa:1-2") + require.NoError(t, err) + probed := make(chan struct{}, 1) + reader := &GoMySQLReader{ + migrationContext: mc, + currentCoordinates: initial, + currentCoordinatesMutex: &sync.Mutex{}, + binlogStreamer: replication.NewBinlogStreamer(), + ReadSourceCoordinates: func() (mysql.BinlogCoordinates, error) { + select { + case probed <- struct{}{}: + default: + } + return source, nil + }, + } + sid := make([]byte, 16) + for i := range sid { + sid[i] = 0xaa + } + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(&replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.GTID_EVENT}, + Event: &replication.GTIDEvent{SID: sid, GNO: 2}, + })) + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(&replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.QUERY_EVENT}, + Event: &replication.QueryEvent{Query: []byte("BEGIN"), GSet: source.GTIDSet}, + })) + entries := make(chan *BinlogEntry, 10) + done := make(chan error, 1) + go func() { done <- reader.StreamEvents(func() bool { return false }, entries) }() + for i := 0; i < 3; i++ { + select { + case <-probed: + case <-time.After(5 * time.Second): + t.Fatal("source position was not probed") + } + } + require.Empty(t, entries, "reading GTID_NEXT is not the same as processing its rows") + + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(&replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.XID_EVENT}, + Event: &replication.XIDEvent{GSet: source.GTIDSet}, + })) + select { + case progress := <-entries: + require.True(t, source.Equals(progress.Coordinates)) + case <-time.After(5 * time.Second): + t.Fatal("completed transaction did not produce a catch-up marker") + } + mc.CancelContext() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestWriterLagProgressAfterStandaloneDDL(t *testing.T) { + mc := base.NewMigrationContext() + defer mc.CancelContext() + mc.UseGTIDs = true + mc.HeartbeatIntervalMilliseconds = 1 + initial, err := mysql.NewGTIDBinlogCoordinates(mysql.MariaDBFlavor, "0-1-1") + require.NoError(t, err) + source, err := mysql.NewGTIDBinlogCoordinates(mysql.MariaDBFlavor, "0-1-2") + require.NoError(t, err) + reader := &GoMySQLReader{ + migrationContext: mc, + currentCoordinates: initial, + currentCoordinatesMutex: &sync.Mutex{}, + binlogStreamer: replication.NewBinlogStreamer(), + ReadSourceCoordinates: func() (mysql.BinlogCoordinates, error) { + return source, nil + }, + } + require.NoError(t, reader.binlogStreamer.AddEventToStreamer(&replication.BinlogEvent{ + Header: &replication.EventHeader{EventType: replication.QUERY_EVENT}, + Event: &replication.QueryEvent{Query: []byte("CREATE TABLE unrelated (id INT)"), GSet: source.GTIDSet}, + })) + entries := make(chan *BinlogEntry, 1) + done := make(chan error, 1) + go func() { done <- reader.StreamEvents(func() bool { return false }, entries) }() + select { + case progress := <-entries: + require.True(t, source.Equals(progress.Coordinates)) + case <-time.After(5 * time.Second): + t.Fatal("standalone DDL must not prevent idle catch-up detection") + } + mc.CancelContext() + require.ErrorIs(t, <-done, context.Canceled) +} diff --git a/go/logic/migrator.go b/go/logic/migrator.go index a201bd66a..1d2fe8934 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -2572,23 +2572,6 @@ func (mgtr *Migrator) initiateStreaming() error { } }() - // In move-tables mode there is no changelog heartbeat. Writer lag is derived - // from binlog-header timestamps of applied events; when the streamer has been - // silent for the heartbeat interval, treat that silence as "caught up" and bump - // the last-applied timestamp to now so lag does not climb forever while idle. - if mgtr.migrationContext.IsMoveTablesMode() { - go func() { - interval := time.Duration(mgtr.migrationContext.HeartbeatIntervalMilliseconds) * time.Millisecond - ticker := time.NewTicker(interval) - defer ticker.Stop() - for range ticker.C { - if atomic.LoadInt64(&mgtr.finishedMigrating) > 0 { - return - } - mgtr.migrationContext.BumpBinlogWriterLagIfIdle(interval) - } - }() - } return nil } @@ -2600,17 +2583,21 @@ func (mgtr *Migrator) initiateStreaming() error { // applier routes DML to the right per-table query builders by table name. func (mgtr *Migrator) addDMLEventsListener() error { enqueue := func(dmlEntry *binlog.BinlogEntry) error { - // Record that the streamer just delivered an event for a moved table, so - // the idle-bump rule can tell "falling behind" from "source is quiet". - if mgtr.migrationContext.IsMoveTablesMode() { - mgtr.migrationContext.MarkBinlogEventStreamed() - } // Use helper to prevent deadlock if buffer fills and executeWriteFuncs exits. // This is critical because this callback blocks the event streamer. return base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.applyEventsQueue, newApplyEventStructByDML(dmlEntry)) } if mgtr.migrationContext.IsMoveTablesMode() { + mgtr.eventsStreamer.onStreamProgress = func(probeTime time.Time) error { + // This barrier shares the DML queue, so it cannot hide queued or + // in-flight writes, even when the streamer is blocked by backpressure. + var applyProgress tableWriteFunc = func() error { + mgtr.migrationContext.UpdateLastAppliedBinlogEventTime(probeTime) + return nil + } + return base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.applyEventsQueue, newApplyEventStructByFunc(&applyProgress)) + } for _, tableName := range mgtr.migrationContext.MoveTables.TableNames { if err := mgtr.eventsStreamer.AddListener( false, diff --git a/go/logic/migrator_move_tables_cutover_test.go b/go/logic/migrator_move_tables_cutover_test.go index 57a83816f..a9d375ddc 100644 --- a/go/logic/migrator_move_tables_cutover_test.go +++ b/go/logic/migrator_move_tables_cutover_test.go @@ -388,14 +388,21 @@ func (s *MoveTablesCutOverSuite) TestMoveTablesListenersBeforeStreaming() { s.Require().Eventually(func() bool { return len(m.applyEventsQueue) == cap(m.applyEventsQueue) }, 10*time.Second, 10*time.Millisecond, "pending DML should fill the apply queue") + mc.UpdateLastAppliedBinlogEventTime(time.Now().Add(-3300 * time.Second)) + time.Sleep(2 * time.Duration(mc.HeartbeatIntervalMilliseconds) * time.Millisecond) + s.Require().GreaterOrEqual(mc.GetBinlogWriterLag(), 3300*time.Second, "backpressure must not look like an idle source") seen := make(map[string]int) deadline := time.NewTimer(10 * time.Second) defer deadline.Stop() - for range len(tableNames) * rowsPerTable { + for received := 0; received < len(tableNames)*rowsPerTable; { select { case event := <-m.applyEventsQueue: - s.Require().NotNil(event.dmlEvent) + if event.dmlEvent == nil { + s.Require().NotNil(event.writeFunc, "non-DML entries should be catch-up markers") + continue + } seen[event.dmlEvent.TableName]++ + received++ case <-deadline.C: s.FailNow("timed out waiting for pending DML") } diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 7d3d00120..d293027e2 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -43,6 +43,7 @@ type EventsStreamer struct { listenersMutex *sync.Mutex eventsChannel chan *binlog.BinlogEntry binlogReader *binlog.GoMySQLReader + onStreamProgress func(time.Time) error name string } @@ -148,6 +149,9 @@ func (es *EventsStreamer) InitDBConnections() (err error) { // initBinlogReader creates and connects the reader: we hook up to a MySQL server as a replica func (es *EventsStreamer) initBinlogReader(binlogCoordinates mysql.BinlogCoordinates) error { goMySQLReader := binlog.NewGoMySQLReader(es.migrationContext, es.shouldDecodeRowsEvent) + if es.migrationContext.IsMoveTablesMode() { + goMySQLReader.ReadSourceCoordinates = es.readBinlogCoordinates + } if err := goMySQLReader.ConnectBinlogStreamer(binlogCoordinates); err != nil { return err } @@ -161,6 +165,16 @@ func (es *EventsStreamer) GetCurrentBinlogCoordinates() mysql.BinlogCoordinates // readCurrentBinlogCoordinates reads master status from hooked server func (es *EventsStreamer) readCurrentBinlogCoordinates() error { + coords, err := es.readBinlogCoordinates() + if err != nil { + return err + } + es.initialBinlogCoordinates = coords + es.migrationContext.Log.Debugf("Streamer binlog coordinates: %+v", coords) + return nil +} + +func (es *EventsStreamer) readBinlogCoordinates() (mysql.BinlogCoordinates, error) { // MariaDB exposes no GTID column in SHOW MASTER STATUS; its current binlog // GTID position lives in @@global.gtid_binlog_pos. if es.migrationContext.UseGTIDs && mysql.IsMariaDB(es.dbVersion) { @@ -168,17 +182,23 @@ func (es *EventsStreamer) readCurrentBinlogCoordinates() error { } binaryLogStatusTerm := mysql.ReplicaTermFor(es.dbVersion, "master status") query := fmt.Sprintf("show /* gh-ost readCurrentBinlogCoordinates */ %s", binaryLogStatusTerm) + rows, err := es.db.QueryContext(es.migrationContext.GetContext(), query) + if err != nil { + return nil, err + } + defer rows.Close() foundMasterStatus := false - err := sqlutils.QueryRowsMap(es.db, query, func(m sqlutils.RowMap) error { + var coordinates mysql.BinlogCoordinates + err = sqlutils.ScanRowsToMaps(rows, func(m sqlutils.RowMap) error { if es.migrationContext.UseGTIDs { execGtidSet := m.GetString("Executed_Gtid_Set") coords, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, execGtidSet) if err != nil { return err } - es.initialBinlogCoordinates = coords + coordinates = coords } else { - es.initialBinlogCoordinates = &mysql.FileBinlogCoordinates{ + coordinates = &mysql.FileBinlogCoordinates{ LogFile: m.GetString("File"), LogPos: m.GetInt64("Position"), } @@ -187,29 +207,35 @@ func (es *EventsStreamer) readCurrentBinlogCoordinates() error { return nil }) if err != nil { - return err + return nil, err } if !foundMasterStatus { - return fmt.Errorf("got no results from SHOW %s. Bailing out", strings.ToUpper(binaryLogStatusTerm)) + return nil, fmt.Errorf("got no results from SHOW %s. Bailing out", strings.ToUpper(binaryLogStatusTerm)) } - es.migrationContext.Log.Debugf("Streamer binlog coordinates: %+v", es.initialBinlogCoordinates) - return nil + return coordinates, nil } // readCurrentMariaDBGTIDCoordinates reads the current binlog GTID position from // a MariaDB server, which is exposed via @@global.gtid_binlog_pos rather than a // column in SHOW MASTER STATUS. -func (es *EventsStreamer) readCurrentMariaDBGTIDCoordinates() error { +func (es *EventsStreamer) readCurrentMariaDBGTIDCoordinates() (mysql.BinlogCoordinates, error) { var gtidBinlogPos string - if err := es.db.QueryRow(`select @@global.gtid_binlog_pos`).Scan(>idBinlogPos); err != nil { - return err + if err := es.db.QueryRowContext(es.migrationContext.GetContext(), `select @@global.gtid_binlog_pos`).Scan(>idBinlogPos); err != nil { + return nil, err } coords, err := mysql.NewGTIDBinlogCoordinates(mysql.MariaDBFlavor, gtidBinlogPos) if err != nil { - return err + return nil, err + } + return coords, nil +} + +func (es *EventsStreamer) dispatchEntry(entry *binlog.BinlogEntry) error { + if entry.DmlEvent != nil { + es.notifyListeners(entry) + } else if es.onStreamProgress != nil { + return es.onStreamProgress(entry.Timestamp) } - es.initialBinlogCoordinates = coords - es.migrationContext.Log.Debugf("Streamer binlog coordinates: %+v", es.initialBinlogCoordinates) return nil } @@ -218,8 +244,9 @@ func (es *EventsStreamer) readCurrentMariaDBGTIDCoordinates() error { func (es *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { go func() { for binlogEntry := range es.eventsChannel { - if binlogEntry.DmlEvent != nil { - es.notifyListeners(binlogEntry) + if err := es.dispatchEntry(binlogEntry); err != nil { + _ = base.SendWithContext(es.migrationContext.GetContext(), es.migrationContext.PanicAbort, err) + return } } }() diff --git a/go/logic/streamer_test.go b/go/logic/streamer_test.go index 5b28c47c2..177b3a411 100644 --- a/go/logic/streamer_test.go +++ b/go/logic/streamer_test.go @@ -7,7 +7,9 @@ import ( "testing" "time" + "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/binlog" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/mysql" @@ -15,6 +17,46 @@ import ( "golang.org/x/sync/errgroup" ) +func TestWriterLagProgressWaitsForWriter(t *testing.T) { + mc := base.NewMigrationContext() + defer mc.CancelContext() + mc.DatabaseName = "test" + mc.MoveTables.TableNames = []string{"t"} + mc.SetHeartbeatIntervalMilliseconds(100) + mc.UpdateLastAppliedBinlogEventTime(time.Now().Add(-3300 * time.Second)) + m := NewMigrator(mc, "test") + m.eventsStreamer = NewEventsStreamer(mc) + require.NoError(t, m.addDMLEventsListener()) + + dml := &binlog.BinlogEntry{DmlEvent: binlog.NewBinlogDMLEvent("test", "t", binlog.InsertDML)} + require.NoError(t, m.eventsStreamer.dispatchEntry(dml)) + require.Same(t, dml.DmlEvent, (<-m.applyEventsQueue).dmlEvent) + + started := make(chan struct{}) + release := make(chan struct{}) + defer close(release) + var slowWrite tableWriteFunc = func() error { + close(started) + <-release + return nil + } + done := make(chan error, 1) + go func() { done <- m.onApplyEventStruct(newApplyEventStructByFunc(&slowWrite)) }() + <-started + probeTime := time.Now().Add(-5 * time.Second) + require.NoError(t, m.eventsStreamer.dispatchEntry(&binlog.BinlogEntry{Timestamp: probeTime})) + require.GreaterOrEqual(t, mc.GetBinlogWriterLag(), 3300*time.Second, "enqueueing a marker cannot reset lag") + + // Leave the writer blocked longer than the old idle threshold. + time.Sleep(2 * time.Duration(mc.HeartbeatIntervalMilliseconds) * time.Millisecond) + require.GreaterOrEqual(t, mc.GetBinlogWriterLag(), 3300*time.Second) + release <- struct{}{} + require.NoError(t, <-done) + require.GreaterOrEqual(t, mc.GetBinlogWriterLag(), 3300*time.Second, "prior work alone cannot reset lag") + require.NoError(t, m.onApplyEventStruct(<-m.applyEventsQueue)) + require.InDelta(t, time.Since(probeTime).Seconds(), mc.GetBinlogWriterLag().Seconds(), 0.1) +} + type EventsStreamerTestSuite struct { suite.Suite