Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions doc/understanding-output.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,16 @@ You can control output levels:
- `--verbose`: common use. Useful output, not tons of it
- `--debug`: everything. Tons of output.

In move-tables mode, `WriterLag` replaces `Lag` and `HeartbeatLag`. It measures
the age of the source-binlog timestamp of the last applied DML batch. To keep
lag low when the source is idle, gh-ost also probes the source's current binlog
position at `--heartbeat-interval-millis` intervals (100ms by default, capped
at 1000ms). Once the reader has processed that position, a timestamped marker
passes through the same queues as DML and updates lag only after preceding
writes finish. The marker uses the time before the source query, so query and
queueing delays count toward lag. Streamer silence alone never resets lag.
Until the first DML batch or catch-up marker is processed, `WriterLag` is zero.

Initial output lines may look like this:
```
2016-05-19 17:57:04 INFO starting gh-ost 0.7.14
Expand Down
34 changes: 4 additions & 30 deletions go/base/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -339,12 +339,9 @@ type MigrationContext struct {
pointOfInterestTimeMutex *sync.Mutex
lastHeartbeatOnChangelogTime time.Time
lastHeartbeatOnChangelogMutex *sync.Mutex
// lastAppliedBinlogEventTime is the binlog-header timestamp of the last event
// applied to the target in move-tables mode. lastBinlogEventStreamedTime is the
// wall-clock time the streamer last delivered an event for the moved table.
// Together they drive the move-tables writer-lag metric (see GetBinlogWriterLag).
// lastAppliedBinlogEventTime is the timestamp of the last applied DML event
// or ordered source-position probe in move-tables mode.
lastAppliedBinlogEventTime time.Time
lastBinlogEventStreamedTime time.Time
binlogWriterLagMutex *sync.Mutex
CurrentLag int64
currentProgress uint64
Expand Down Expand Up @@ -908,38 +905,15 @@ func (mctx *MigrationContext) GetLastHeartbeatOnChangelogTime() time.Time {
return mctx.lastHeartbeatOnChangelogTime
}

// UpdateLastAppliedBinlogEventTime records the binlog-header timestamp of the
// last event successfully applied to the target. Used by move-tables mode to
// derive writer lag.
// UpdateLastAppliedBinlogEventTime records the timestamp of the last event or
// source-position probe successfully processed by the writer in move-tables mode.
func (mctx *MigrationContext) UpdateLastAppliedBinlogEventTime(t time.Time) {
mctx.binlogWriterLagMutex.Lock()
defer mctx.binlogWriterLagMutex.Unlock()

mctx.lastAppliedBinlogEventTime = t
}

// MarkBinlogEventStreamed records that the streamer just delivered an event for
// the moved table. Used to distinguish "falling behind" from "source is idle".
func (mctx *MigrationContext) MarkBinlogEventStreamed() {
mctx.binlogWriterLagMutex.Lock()
defer mctx.binlogWriterLagMutex.Unlock()

mctx.lastBinlogEventStreamedTime = time.Now()
}

// BumpBinlogWriterLagIfIdle treats prolonged streamer silence as "caught up":
// if no event has been streamed for the moved table within idleThreshold, the
// last-applied timestamp is advanced to now so writer lag does not climb forever
// while the source is quiet.
func (mctx *MigrationContext) BumpBinlogWriterLagIfIdle(idleThreshold time.Duration) {
mctx.binlogWriterLagMutex.Lock()
defer mctx.binlogWriterLagMutex.Unlock()

if mctx.lastBinlogEventStreamedTime.IsZero() || time.Since(mctx.lastBinlogEventStreamedTime) >= idleThreshold {
mctx.lastAppliedBinlogEventTime = time.Now()
}
}

// GetBinlogWriterLag returns now - last applied event timestamp, the move-tables
// writer lag. It returns 0 before any event has been applied.
func (mctx *MigrationContext) GetBinlogWriterLag() time.Duration {
Expand Down
4 changes: 2 additions & 2 deletions go/binlog/binlog_entry.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ import (
type BinlogEntry struct {
Coordinates mysql.BinlogCoordinates
// Timestamp is the wall-clock time recorded in the binlog event header of the
// event that produced this entry. It is used in move-tables mode to measure
// writer lag (now - last applied event timestamp).
// event that produced this entry. For a source-position probe (DmlEvent == nil)
// it is the local time before querying the source, not the time of delivery.
Timestamp time.Time
DmlEvent *BinlogDMLEvent
}
Expand Down
76 changes: 74 additions & 2 deletions go/binlog/gomysql_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
package binlog

import (
"errors"
"fmt"
"strings"
"sync"

"github.com/github/gh-ost/go/base"
Expand Down Expand Up @@ -49,6 +51,8 @@
// LastTrxCoords are the coordinates of the last transaction completely read.
// If using the file coordinates it is binlog position of the transaction's XID event.
LastTrxCoords mysql.BinlogCoordinates
// ReadSourceCoordinates enables ordered catch-up probes in move-tables mode.
ReadSourceCoordinates func() (mysql.BinlogCoordinates, error)
}

func NewGoMySQLReader(migrationContext *base.MigrationContext, rowsEventFilters ...RowsEventFilterFunc) *GoMySQLReader {
Expand Down Expand Up @@ -155,11 +159,32 @@

// StreamEvents
func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChannel chan<- *BinlogEntry) error {
processedCoordinates := gmr.GetCurrentBinlogCoordinates()
interval := time.Duration(gmr.migrationContext.HeartbeatIntervalMilliseconds) * time.Millisecond
lastProbe := time.Now()
inTransaction := false
for !canStopStreaming() {
ev, err := gmr.binlogStreamer.GetEvent(context.Background())
if err != nil {
ctx := context.Background()
cancel := func() {}
if gmr.ReadSourceCoordinates != nil {
ctx, cancel = context.WithTimeout(gmr.migrationContext.GetContext(), interval)
}
ev, err := gmr.binlogStreamer.GetEvent(ctx)
cancel()
if err != nil && !(gmr.ReadSourceCoordinates != nil && errors.Is(err, context.DeadlineExceeded) && gmr.migrationContext.GetContext().Err() == nil) {

Check failure on line 174 in go/binlog/gomysql_reader.go

View workflow job for this annotation

GitHub Actions / lint

QF1001: could apply De Morgan's law (staticcheck)
return err
}
if gmr.ReadSourceCoordinates != nil && time.Since(lastProbe) >= interval {
// processedCoordinates excludes the event just fetched. The marker must
// follow every row it claims to cover, including a complete GTID transaction.
if err := gmr.sendStreamProgress(processedCoordinates, entriesChannel); err != nil {
return err
}
lastProbe = time.Now()
}
if err != nil {
continue
}

// Update binlog coords if using file-based coords.
// GTID coordinates are updated on receiving GTID events.
Expand Down Expand Up @@ -214,27 +239,74 @@
coords.LogFile = string(event.NextLogName)
gmr.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", coords.LogFile, int64(ev.Header.LogPos), event.NextLogName)
gmr.currentCoordinatesMutex.Unlock()
processedCoordinates = &mysql.FileBinlogCoordinates{
LogFile: string(event.NextLogName),
LogPos: int64(event.Position),
}
case *replication.XIDEvent:
inTransaction = false
if gmr.migrationContext.UseGTIDs {
// event.GSet is the full executed GTID set maintained by the
// syncer (MysqlGTIDSet or MariadbGTIDSet depending on flavor).
if event.GSet != nil {
gmr.LastTrxCoords = &mysql.GTIDBinlogCoordinates{GTIDSet: event.GSet}
if gmr.ReadSourceCoordinates != nil {
processedCoordinates = gmr.LastTrxCoords.Clone()
}
}
} else {
gmr.LastTrxCoords = gmr.currentCoordinates.Clone()
}
case *replication.QueryEvent:
if gmr.ReadSourceCoordinates == nil || !gmr.migrationContext.UseGTIDs {
break
}
switch strings.ToUpper(strings.TrimSpace(string(event.Query))) {
case "BEGIN":
inTransaction = true
case "COMMIT", "ROLLBACK":
inTransaction = false
}
// Standalone DDL has no XID event. BEGIN's GSet, however, already
// includes the transaction whose row events have not arrived yet.
if !inTransaction && event.GSet != nil {
processedCoordinates = (&mysql.GTIDBinlogCoordinates{GTIDSet: event.GSet}).Clone()
}
case *replication.RowsEvent:
if err := gmr.handleRowsEvent(ev, event, entriesChannel); err != nil {
return err
}
}
if gmr.ReadSourceCoordinates == nil {
continue
}
// Startup's synthetic rotate/format events can have a zero log position.
if !gmr.migrationContext.UseGTIDs && ev.Header.LogPos > 0 && ev.Header.EventType != replication.ROTATE_EVENT {
processedCoordinates = gmr.GetCurrentBinlogCoordinates()
}
}
gmr.migrationContext.Log.Debugf("done streaming events")

return nil
}

func (gmr *GoMySQLReader) sendStreamProgress(processed mysql.BinlogCoordinates, entriesChannel chan<- *BinlogEntry) error {
probeTime := time.Now()
source, err := gmr.ReadSourceCoordinates()
if err != nil {
return err
}
// Equality works for file positions and both GTID flavors, without assuming
// that a partially received transaction or an empty apply queue means caught up.
if !processed.Equals(source) {
return nil
}
return base.SendWithContext(gmr.migrationContext.GetContext(), entriesChannel, &BinlogEntry{
Coordinates: processed.Clone(),
Timestamp: probeTime,
})
}

func (gmr *GoMySQLReader) Close() error {
gmr.binlogSyncer.Close()
return nil
Expand Down
Loading
Loading