Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions go/binlog/binlog_entry.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,9 @@ type BinlogEntry struct {
// Timestamp is the wall-clock time recorded in the binlog event header of the
// event that produced this entry. It is used in move-tables mode to measure
// writer lag (now - last applied event timestamp).
Timestamp time.Time
DmlEvent *BinlogDMLEvent
Timestamp time.Time
DmlEvent *BinlogDMLEvent
TransactionComplete bool
}

// NewBinlogEntryAt creates an empty, ready to go BinlogEntry object
Expand Down
117 changes: 117 additions & 0 deletions go/binlog/cancellation_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package binlog

import (
"context"
"testing"
"time"

"github.com/github/gh-ost/go/base"
"github.com/github/gh-ost/go/mysql"
"github.com/go-mysql-org/go-mysql/replication"
"github.com/stretchr/testify/require"
)

func TestTransactionMarkerSendCanceled(t *testing.T) {
t.Run("decoded rows", func(t *testing.T) {
assertTransactionMarkerSendCanceled(t, [][]interface{}{{1}})
})
t.Run("filtered rows", func(t *testing.T) {
assertTransactionMarkerSendCanceled(t, nil)
})
}

func assertTransactionMarkerSendCanceled(t *testing.T, rows [][]interface{}) {
t.Helper()
reader := transactionTestReader(false)
defer reader.migrationContext.CancelContext()
previous := mysql.NewFileBinlogCoordinates("mysql-bin.000001", 4)
reader.LastTrxCoords = previous.Clone()
row := &replication.BinlogEvent{
Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 200},
Event: &replication.RowsEvent{
Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")},
Rows: rows,
},
}
commit := &replication.BinlogEvent{
Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: 220},
Event: &replication.XIDEvent{},
}
for _, event := range []*replication.BinlogEvent{row, commit} {
require.NoError(t, reader.binlogStreamer.AddEventToStreamer(event))
}
entries := make(chan *BinlogEntry)
done := make(chan error, 1)
go func() {
err := reader.StreamEvents(func() bool { return false }, entries)
_ = base.SendWithContext(t.Context(), done, err)
}()
if len(rows) > 0 {
select {
case entry := <-entries:
require.NotNil(t, entry.DmlEvent)
case <-time.After(time.Second):
t.Fatal("row was not delivered")
}
}
// Wait until the commit position has been read, then stop consuming.
// The reader must propagate a canceled completion-marker send.
require.Eventually(t, func() bool {
return reader.GetCurrentBinlogCoordinates().(*mysql.FileBinlogCoordinates).LogPos == 220
}, time.Second, time.Millisecond)
reader.migrationContext.CancelContext()
select {
case err := <-done:
require.ErrorIs(t, err, context.Canceled)
case <-time.After(time.Second):
t.Fatal("completion marker send ignored cancellation")
}
require.True(t, previous.Equals(reader.LastTrxCoords), "failed delivery must not advance reconnect coordinates")
}

func TestRowsSendCanceled(t *testing.T) {
reader := transactionTestReader(false)
defer reader.migrationContext.CancelContext()
event := &replication.BinlogEvent{
Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2},
Event: &replication.RowsEvent{
Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")},
Rows: [][]interface{}{{1}, {2}},
},
}
require.NoError(t, reader.binlogStreamer.AddEventToStreamer(event))
entries := make(chan *BinlogEntry)
done := make(chan error, 1)
go func() {
err := reader.StreamEvents(func() bool { return false }, entries)
_ = base.SendWithContext(t.Context(), done, err)
}()
select {
case <-entries:
case <-time.After(time.Second):
t.Fatal("first row was not delivered")
}
reader.migrationContext.CancelContext()
select {
case err := <-done:
require.ErrorIs(t, err, context.Canceled)
case <-time.After(time.Second):
t.Fatal("blocked row send ignored cancellation")
}
}

func TestIdleReaderCanceled(t *testing.T) {
reader := transactionTestReader(false)
reader.migrationContext.CancelContext()
done := make(chan error, 1)
go func() {
err := reader.StreamEvents(func() bool { return false }, make(chan *BinlogEntry))
_ = base.SendWithContext(t.Context(), done, err)
}()
select {
case err := <-done:
require.ErrorIs(t, err, context.Canceled)
case <-time.After(time.Second):
t.Fatal("idle reader ignored cancellation")
}
}
121 changes: 121 additions & 0 deletions go/binlog/filtered_transaction_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package binlog

import (
"testing"
"time"

"github.com/github/gh-ost/go/mysql"
"github.com/go-mysql-org/go-mysql/replication"
"github.com/stretchr/testify/require"
)

func filteredRowsEvent(t *testing.T, pos uint32) *replication.BinlogEvent {
t.Helper()
rows := newTestRowsEvent("testdb", "ignored_table")
decode := newRowsEventDecodeFunc(func(_, _ string) bool { return false })
require.NoError(t, decode(rows, newRowsEventDataWithInvalidRowPayload()))
require.Empty(t, rows.Rows)
return &replication.BinlogEvent{
Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv1, LogPos: pos},
Event: rows,
}
}

func TestFilteredTransactionCompletionMarker(t *testing.T) {
for _, test := range []struct {
name, flavor, committed, next string
xidGTIDSet bool
}{
{name: "file"},
{"mysql", mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1", "00000000-0000-0000-0000-000000000001:2", true},
{"mysql fallback", mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1", "00000000-0000-0000-0000-000000000001:2", false},
{"mariadb", mysql.MariaDBFlavor, "0-1-1", "0-1-2", true},
{"mariadb fallback", mysql.MariaDBFlavor, "0-1-1", "0-1-2", false},
} {
t.Run(test.name, func(t *testing.T) {
reader := transactionTestReader(test.flavor != "")
defer reader.migrationContext.CancelContext()
xid := &replication.XIDEvent{}
var committed mysql.BinlogCoordinates = mysql.NewFileBinlogCoordinates("mysql-bin.000001", 220)
if test.flavor != "" {
coords, err := mysql.NewGTIDBinlogCoordinates(test.flavor, test.committed)
require.NoError(t, err)
reader.currentCoordinates = coords
committed = coords.Clone()
if test.xidGTIDSet {
xid.GSet = coords.GTIDSet
}
}

// Filtering the only row event must not acknowledge an open transaction.
require.Empty(t, transactionTestStream(t, reader, filteredRowsEvent(t, 200)))
require.Nil(t, reader.LastTrxCoords)
entries := transactionTestStream(t, reader, &replication.BinlogEvent{
Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: 220},
Event: xid,
})
require.Len(t, entries, 1)
require.True(t, entries[0].TransactionComplete)
require.Nil(t, entries[0].DmlEvent)
require.True(t, committed.Equals(entries[0].Coordinates))
require.True(t, committed.Equals(reader.LastTrxCoords))

// Later events (and the syncer's XID set) must not mutate the marker.
if test.flavor != "" {
require.NoError(t, reader.currentCoordinates.(*mysql.GTIDBinlogCoordinates).GTIDSet.Update(test.next))
}
require.True(t, committed.Equals(entries[0].Coordinates))
require.True(t, committed.Equals(reader.LastTrxCoords))
})
}
}

func TestFilteredTransactionMarkersPreserveDMLOrder(t *testing.T) {
reader := transactionTestReader(false)
defer reader.migrationContext.CancelContext()
rows := &replication.BinlogEvent{
Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 100},
Event: &replication.RowsEvent{
Table: &replication.TableMapEvent{Schema: []byte("testdb"), Table: []byte("testing")},
Rows: [][]interface{}{{1}, {2}},
},
}
commit := func(pos uint32) *replication.BinlogEvent {
return &replication.BinlogEvent{
Header: &replication.EventHeader{EventType: replication.XID_EVENT, LogPos: pos},
Event: &replication.XIDEvent{},
}
}
// Filtered transactions immediately after a marker are coalesced. A later
// relevant transaction still gets a marker behind its own DML.
nextRows := &replication.BinlogEvent{
Header: &replication.EventHeader{EventType: replication.WRITE_ROWS_EVENTv2, LogPos: 600},
Event: rows.Event,
}
entries := transactionTestStream(t, reader,
rows, filteredRowsEvent(t, 200), commit(220),
filteredRowsEvent(t, 300), commit(320),
filteredRowsEvent(t, 400), commit(420),
commit(520), nextRows, commit(620),
)
require.Len(t, entries, 6)
for _, index := range []int{0, 1, 3, 4} {
require.NotNil(t, entries[index].DmlEvent)
require.False(t, entries[index].TransactionComplete)
}
for index, pos := range map[int]int64{2: 220, 5: 620} {
entry := entries[index]
require.True(t, entry.TransactionComplete)
require.Nil(t, entry.DmlEvent)
require.Equal(t, pos, entry.Coordinates.(*mysql.FileBinlogCoordinates).LogPos)
}
require.Equal(t, int64(620), reader.LastTrxCoords.(*mysql.FileBinlogCoordinates).LogPos)

// Under sustained unrelated traffic, another ordered marker eventually
// advances the applied watermark without producing one for every XID.
reader.lastTransactionMarkerTime = time.Now().Add(-filteredTransactionMarkerInterval)
entries = transactionTestStream(t, reader, filteredRowsEvent(t, 700), commit(720))
require.Len(t, entries, 1)
require.True(t, entries[0].TransactionComplete)
require.Equal(t, int64(720), entries[0].Coordinates.(*mysql.FileBinlogCoordinates).LogPos)
}
58 changes: 45 additions & 13 deletions go/binlog/gomysql_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@ import (

"time"

"context"

gomysql "github.com/go-mysql-org/go-mysql/mysql"
"github.com/go-mysql-org/go-mysql/replication"
)
Expand All @@ -39,6 +37,9 @@ func newRowsEventDecodeFunc(rowsEventFilter RowsEventFilterFunc) func(*replicati
}
}

// Limit markers for unrelated transactions so busy sources cannot starve row copy.
const filteredTransactionMarkerInterval = time.Second

type GoMySQLReader struct {
migrationContext *base.MigrationContext
connectionConfig *mysql.ConnectionConfig
Expand All @@ -47,8 +48,12 @@ type GoMySQLReader struct {
currentCoordinates mysql.BinlogCoordinates
currentCoordinatesMutex *sync.Mutex
// LastTrxCoords are the coordinates of the last transaction completely read.
// If using the file coordinates it is binlog position of the transaction's XID event.
// They are for streamer reconnects, not durable applier checkpoints.
// For file coordinates, this is the end position of the transaction's XID event.
LastTrxCoords mysql.BinlogCoordinates

transactionHasRows bool
lastTransactionMarkerTime time.Time
}

func NewGoMySQLReader(migrationContext *base.MigrationContext, rowsEventFilters ...RowsEventFilterFunc) *GoMySQLReader {
Expand Down Expand Up @@ -148,15 +153,48 @@ func (gmr *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent
// decides whether action is taken synchronously (meaning we wait before
// next iteration) or asynchronously (we keep pushing more events)
// In reality, reads will be synchronous
entriesChannel <- binlogEntry
if err := base.SendWithContext(gmr.migrationContext.GetContext(), entriesChannel, binlogEntry); err != nil {
return err
}
gmr.transactionHasRows = true
}
return nil
}

func (gmr *GoMySQLReader) completeTransaction(event *replication.XIDEvent, entriesChannel chan<- *BinlogEntry) error {
var coords mysql.BinlogCoordinates
if gmr.migrationContext.UseGTIDs && event.GSet != nil {
coords = (&mysql.GTIDBinlogCoordinates{GTIDSet: event.GSet}).Clone()
} else {
// File positions use the current event's coordinates. For GTID
// events parsed without the syncer, fall back to the coordinates
// already updated by the preceding GTID event.
coords = gmr.GetCurrentBinlogCoordinates()
}
// Relevant transactions always need a marker. Sample filtered-only commits
// periodically too, so their progress cannot leave checkpoints on old binlogs.
// Every emitted marker follows preceding DML through the same FIFO.
if gmr.transactionHasRows || time.Since(gmr.lastTransactionMarkerTime) >= filteredTransactionMarkerInterval {
marker := &BinlogEntry{
Coordinates: coords.Clone(),
TransactionComplete: true,
}
if err := base.SendWithContext(gmr.migrationContext.GetContext(), entriesChannel, marker); err != nil {
return err
}
gmr.lastTransactionMarkerTime = time.Now()
}
gmr.transactionHasRows = false
// Filtered-only transactions contain no delivered rows, so reconnect may
// safely advance even when their checkpoint marker was coalesced.
gmr.LastTrxCoords = coords
return nil
}

// StreamEvents
func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChannel chan<- *BinlogEntry) error {
for !canStopStreaming() {
ev, err := gmr.binlogStreamer.GetEvent(context.Background())
ev, err := gmr.binlogStreamer.GetEvent(gmr.migrationContext.GetContext())
if err != nil {
return err
}
Expand Down Expand Up @@ -215,14 +253,8 @@ func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChan
gmr.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", coords.LogFile, int64(ev.Header.LogPos), event.NextLogName)
gmr.currentCoordinatesMutex.Unlock()
case *replication.XIDEvent:
if gmr.migrationContext.UseGTIDs {
// event.GSet is the full executed GTID set maintained by the
// syncer (MysqlGTIDSet or MariadbGTIDSet depending on flavor).
if event.GSet != nil {
gmr.LastTrxCoords = &mysql.GTIDBinlogCoordinates{GTIDSet: event.GSet}
}
} else {
gmr.LastTrxCoords = gmr.currentCoordinates.Clone()
if err := gmr.completeTransaction(event, entriesChannel); err != nil {
return err
}
case *replication.RowsEvent:
if err := gmr.handleRowsEvent(ev, event, entriesChannel); err != nil {
Expand Down
Loading