fix: let a node that is one sequence behind rejoin the cluster

internal/bft/view.go

@@ -851,7 +851,6 @@ func (vv *voteVerifier) verifyVote(vote *protos.Message) {
 
 func (v *View) decide(proposal *types.Proposal, signatures []types.Signature, requests []types.RequestInfo) {
 	v.Logger.Infof("Deciding on seq %d", v.ProposalSequence)
-	v.ViewSequences.Store(ViewSequence{ProposalSeq: v.ProposalSequence, ViewActive: true})
 	// first make preparations for the next sequence so that the view will be ready to continue right after delivery
 	v.startNextSeq()
 	signatures = append(signatures, *v.myProposalSig)
@@ -866,6 +865,14 @@ func (v *View) startNextSeq() {
 
 	nextSeq := v.ProposalSequence
 
+	// Publish the sequence this view is about to propose next. Without this the stored
+	// sequence stays at the one that was just decided, and every reader of it - the
+	// heartbeat a leader broadcasts, the state a node collects from its peers, and the
+	// follower that compares its own sequence against it - ends up comparing a decided
+	// sequence against a proposal sequence, which hides a follower that is behind by
+	// exactly the sequence in flight.
+	v.ViewSequences.Store(ViewSequence{ProposalSeq: nextSeq, ViewActive: true})
+
 	v.MetricsView.ProposalSequence.Set(float64(v.ProposalSequence))
 	v.MetricsView.DecisionsInView.Set(float64(v.DecisionsInView))
 

internal/bft/view_test.go

@@ -743,7 +743,9 @@ func TestNormalPath(t *testing.T) {
 	view.HandleMessage(2, commit2Next)
 	view.HandleMessage(3, commit3Next)
 	deciderWG.Wait()
-	assert.Equal(t, bft.ViewSequence{ViewActive: true, ProposalSeq: 1}, viewSeq.Load().(bft.ViewSequence))
+	// after deciding sequence 1 the view advertises the sequence it proposes next, which
+	// is the one followers compare their own sequence against
+	assert.Equal(t, bft.ViewSequence{ViewActive: true, ProposalSeq: 2}, viewSeq.Load().(bft.ViewSequence))
 	dProp = <-decidedProposal
 	secondProposal := proposal
 	secondProposal.Metadata = bft.MarshalOrPanic(&protos.ViewMetadata{