diff --git a/common/version/version.go b/common/version/version.go index 98b922e51e..1d478c6d1e 100644 --- a/common/version/version.go +++ b/common/version/version.go @@ -5,7 +5,7 @@ import ( "runtime/debug" ) -var tag = "v4.7.15" +var tag = "v4.7.16" var commit = func() string { if info, ok := debug.ReadBuildInfo(); ok { diff --git a/rollup/internal/controller/relayer/l2_relayer.go b/rollup/internal/controller/relayer/l2_relayer.go index e57b6ca73f..1f0423a51c 100644 --- a/rollup/internal/controller/relayer/l2_relayer.go +++ b/rollup/internal/controller/relayer/l2_relayer.go @@ -194,6 +194,7 @@ func NewLayer2Relayer(ctx context.Context, l2Client *ethclient.Client, db *gorm. return nil, fmt.Errorf("failed to initialize and commit genesis batch, err: %v", err) } layer2Relayer.metrics = initL2RelayerMetrics(reg) + layer2Relayer.initializeMetrics(ctx) switch serviceType { case ServiceTypeL2RollupRelayer: @@ -205,6 +206,28 @@ func NewLayer2Relayer(ctx context.Context, l2Client *ethclient.Client, db *gorm. return layer2Relayer, nil } +// initializeMetrics seeds the commit block height gauge from DB so that a restart +// does not leave it at 0, which would otherwise make the commit-lag alert fire +// until the next batch is committed. +func (r *Layer2Relayer) initializeMetrics(ctx context.Context) { + latestBatch, err := r.batchOrm.GetLatestCommittedBatch(ctx) + if err != nil { + log.Warn("failed to initialize commit block height metric", "err", err) + return + } + if latestBatch == nil { + return + } + endChunk, err := r.chunkOrm.GetChunkByIndex(ctx, latestBatch.EndChunkIndex) + if err != nil { + log.Warn("failed to initialize commit block height metric", "err", err) + return + } + if endChunk != nil { + r.metrics.rollupL2RelayerCommitBlockHeight.Set(float64(endChunk.EndBlockNumber)) + } +} + func (r *Layer2Relayer) initializeGenesis() error { if count, err := r.batchOrm.GetBatchCount(r.ctx); err != nil { return fmt.Errorf("failed to get batch count: %v", err) diff --git a/rollup/internal/controller/watcher/batch_proposer.go b/rollup/internal/controller/watcher/batch_proposer.go index 522ce07cb7..c5177b7b41 100644 --- a/rollup/internal/controller/watcher/batch_proposer.go +++ b/rollup/internal/controller/watcher/batch_proposer.go @@ -2,6 +2,7 @@ package watcher import ( "context" + "errors" "fmt" "time" @@ -119,9 +120,33 @@ func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, minC }), } + p.initializeMetrics(ctx) + return p } +// initializeMetrics seeds the propose block height gauge from DB so that a restart +// does not leave it at 0, which would otherwise make the propose-lag alert fire +// until the next batch is proposed. +func (p *BatchProposer) initializeMetrics(ctx context.Context) { + latestBatch, err := p.batchOrm.GetLatestBatch(ctx) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return // no batches proposed yet, nothing to seed + } + log.Warn("failed to initialize batch propose block height metric", "err", err) + return + } + endChunk, err := p.chunkOrm.GetChunkByIndex(ctx, latestBatch.EndChunkIndex) + if err != nil { + log.Warn("failed to initialize batch propose block height metric", "err", err) + return + } + if endChunk != nil { + p.batchProposeBlockHeight.Set(float64(endChunk.EndBlockNumber)) + } +} + // SetReplayDB sets the replay database for the BatchProposer. // This is used for the proposer tool only, to change the l2_block data source. // This function is not thread-safe and should be called after initializing the BatchProposer and before starting to propose chunks. diff --git a/rollup/internal/controller/watcher/chunk_proposer.go b/rollup/internal/controller/watcher/chunk_proposer.go index 843abaf542..342011e4fe 100644 --- a/rollup/internal/controller/watcher/chunk_proposer.go +++ b/rollup/internal/controller/watcher/chunk_proposer.go @@ -127,9 +127,25 @@ func NewChunkProposer(ctx context.Context, cfg *config.ChunkProposerConfig, minC }), } + p.initializeMetrics(ctx) + return p } +// initializeMetrics seeds the propose block height gauge from DB so that a restart +// does not leave it at 0, which would otherwise make the propose-lag alert fire +// until the next chunk is proposed. +func (p *ChunkProposer) initializeMetrics(ctx context.Context) { + latestChunk, err := p.chunkOrm.GetLatestChunk(ctx) + if err != nil { + log.Warn("failed to initialize chunk propose block height metric", "err", err) + return + } + if latestChunk != nil { + p.chunkProposeBlockHeight.Set(float64(latestChunk.EndBlockNumber)) + } +} + // SetReplayDB sets the replay database for the ChunkProposer. // This is used for the proposer tool only, to change the l2_block data source. // This function is not thread-safe and should be called after initializing the ChunkProposer and before starting to propose chunks. diff --git a/rollup/internal/controller/watcher/l1_watcher.go b/rollup/internal/controller/watcher/l1_watcher.go index 3a93589a73..7af4e2afa1 100644 --- a/rollup/internal/controller/watcher/l1_watcher.go +++ b/rollup/internal/controller/watcher/l1_watcher.go @@ -44,7 +44,7 @@ func NewL1WatcherClient(ctx context.Context, rpcClient *rpc.Client, startHeight savedL1BlockHeight = startHeight } - return &L1WatcherClient{ + w := &L1WatcherClient{ ctx: ctx, rpcClient: rpcClient, client: ethclient.NewClient(rpcClient), @@ -53,6 +53,11 @@ func NewL1WatcherClient(ctx context.Context, rpcClient *rpc.Client, startHeight processedBlockHeight: savedL1BlockHeight, metrics: initL1WatcherMetrics(reg), } + + // Seed the gauge from the resumed height instead of starting from 0. + w.metrics.l1WatcherFetchBlockHeaderProcessedBlockHeight.Set(float64(w.processedBlockHeight)) + + return w } // ProcessedBlockHeight get processedBlockHeight diff --git a/rollup/internal/controller/watcher/l2_watcher.go b/rollup/internal/controller/watcher/l2_watcher.go index f083a65e70..4a93d90697 100644 --- a/rollup/internal/controller/watcher/l2_watcher.go +++ b/rollup/internal/controller/watcher/l2_watcher.go @@ -43,7 +43,7 @@ type L2WatcherClient struct { // NewL2WatcherClient take a l2geth instance to generate a l2watcherclient instance func NewL2WatcherClient(ctx context.Context, client *rpc.Client, confirmations rpc.BlockNumber, messageQueueAddress common.Address, withdrawTrieRootSlot common.Hash, chainCfg *params.ChainConfig, db *gorm.DB, validiumMode bool, reg prometheus.Registerer) *L2WatcherClient { - return &L2WatcherClient{ + w := &L2WatcherClient{ ctx: ctx, Client: ethclient.NewClient(client), rpcCli: client, @@ -61,6 +61,15 @@ func NewL2WatcherClient(ctx context.Context, client *rpc.Client, confirmations r chainCfg: chainCfg, } + + // Seed the gauge from the latest stored height instead of starting from 0. + if latestHeight, err := w.l2BlockOrm.GetL2BlocksLatestHeight(ctx); err != nil { + log.Warn("failed to initialize l2 watcher fetched height metric", "err", err) + } else { + w.metrics.fetchRunningMissingBlocksHeight.Set(float64(latestHeight)) + } + + return w } const blocksFetchLimit = uint64(10) diff --git a/rollup/internal/orm/batch.go b/rollup/internal/orm/batch.go index 706fac90e6..1e732efbf4 100644 --- a/rollup/internal/orm/batch.go +++ b/rollup/internal/orm/batch.go @@ -155,6 +155,24 @@ func (o *Batch) GetLatestBatch(ctx context.Context) (*Batch, error) { return &latestBatch, nil } +// GetLatestCommittedBatch retrieves the highest-index batch that has a commit tx hash set. +// It returns nil if no such batch exists. +func (o *Batch) GetLatestCommittedBatch(ctx context.Context) (*Batch, error) { + db := o.db.WithContext(ctx) + db = db.Model(&Batch{}) + db = db.Where("commit_tx_hash IS NOT NULL AND commit_tx_hash != ''") + db = db.Order("index desc") + + var latestBatch Batch + if err := db.First(&latestBatch).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return nil, fmt.Errorf("Batch.GetLatestCommittedBatch error: %w", err) + } + return &latestBatch, nil +} + // GetFirstUnbatchedChunkIndex retrieves the first unbatched chunk index. func (o *Batch) GetFirstUnbatchedChunkIndex(ctx context.Context) (uint64, error) { // Get the latest batch