From e60feee392a1fc0c3984d187f49572bc9d742d52 Mon Sep 17 00:00:00 2001 From: Arul Ajmani Date: Tue, 9 Jul 2024 12:54:20 -0400 Subject: [PATCH] raft: persist Lead into the HardState Previously, leader information for a given term was stored as volatile state in raft. This change persists it durably by moving it to HardState. This is done in service of leader leases, where we want to ensure a follower doesn't violate its promise of support to the leader even after a restart. Fixes https://github.com/cockroachdb/cockroach/issues/124421 Release note: None --- pkg/kv/kvserver/below_raft_protos_test.go | 2 + pkg/kv/kvserver/logstore/logstore.go | 1 + pkg/kv/kvserver/logstore/stateloader.go | 9 ++- pkg/kv/kvserver/replica_raft.go | 52 +++++++-------- pkg/kv/kvserver/replica_test.go | 22 ++++--- pkg/kv/kvserver/stateloader/initial_test.go | 10 +-- .../TestBelowRaftProtosDontChange/HardState | 2 +- pkg/raft/node.go | 5 +- pkg/raft/node_test.go | 14 ++-- pkg/raft/raft.go | 42 ++++++++++-- pkg/raft/raftpb/raft.proto | 38 ++++++----- pkg/raft/raftpb/raft_test.go | 4 +- ...ction_env_handler_process_append_thread.go | 1 + pkg/raft/rafttest/node_test.go | 2 +- pkg/raft/rawnode.go | 4 +- pkg/raft/rawnode_test.go | 26 ++++++-- pkg/raft/status.go | 17 +++++ pkg/raft/testdata/async_storage_writes.txt | 57 ++++++++-------- .../async_storage_writes_append_aba_race.txt | 40 +++++------ pkg/raft/testdata/campaign.txt | 21 +++--- .../testdata/campaign_learner_must_vote.txt | 20 +++--- pkg/raft/testdata/checkquorum.txt | 26 ++++---- .../confchange_disable_validation.txt | 2 +- .../testdata/confchange_drop_if_unapplied.txt | 4 +- .../testdata/confchange_v1_add_single.txt | 14 ++-- .../testdata/confchange_v1_remove_leader.txt | 12 ++-- .../confchange_v1_remove_leader_stepdown.txt | 10 +-- .../confchange_v2_add_double_auto.txt | 35 +++++----- .../confchange_v2_add_double_implicit.txt | 18 ++--- .../confchange_v2_add_single_auto.txt | 14 ++-- .../confchange_v2_add_single_explicit.txt | 22 +++---- .../testdata/confchange_v2_replace_leader.txt | 50 +++++++------- .../confchange_v2_replace_leader_stepdown.txt | 10 +-- pkg/raft/testdata/forget_leader.txt | 16 ++--- .../forget_leader_prevote_checkquorum.txt | 32 +++++---- pkg/raft/testdata/lagging_commit.txt | 6 +- pkg/raft/testdata/prevote.txt | 42 ++++++------ pkg/raft/testdata/prevote_checkquorum.txt | 66 ++++++++++--------- pkg/raft/testdata/probe_and_replicate.txt | 58 ++++++++-------- pkg/raft/testdata/single_node.txt | 9 +-- .../snapshot_succeed_via_app_resp.txt | 5 +- .../snapshot_succeed_via_app_resp_behind.txt | 6 +- pkg/raft/util.go | 3 +- 43 files changed, 466 insertions(+), 383 deletions(-) diff --git a/pkg/kv/kvserver/below_raft_protos_test.go b/pkg/kv/kvserver/below_raft_protos_test.go index 0214d9d6c82b..de7e9734a32c 100644 --- a/pkg/kv/kvserver/below_raft_protos_test.go +++ b/pkg/kv/kvserver/below_raft_protos_test.go @@ -54,6 +54,7 @@ func TestBelowRaftProtosDontChange(t *testing.T) { Term uint64 Vote raftpb.PeerID Commit uint64 + Lead raftpb.PeerID } // Conversion fails if new fields are added to `HardState`, in which case this method // and the expected sums should be updated. @@ -64,6 +65,7 @@ func TestBelowRaftProtosDontChange(t *testing.T) { Term: n % 3, Vote: raftpb.PeerID(n % 7), Commit: n % 11, + Lead: raftpb.PeerID(n % 13), } }, func(r *rand.Rand) protoutil.Message { diff --git a/pkg/kv/kvserver/logstore/logstore.go b/pkg/kv/kvserver/logstore/logstore.go index 3ad62c8aeedc..4e289ca5cc5f 100644 --- a/pkg/kv/kvserver/logstore/logstore.go +++ b/pkg/kv/kvserver/logstore/logstore.go @@ -202,6 +202,7 @@ func (s *LogStore) storeEntriesAndCommitBatch( Term: m.Term, Vote: m.Vote, Commit: m.Commit, + Lead: m.Lead, } if !raft.IsEmptyHardState(hs) { // NB: Note that without additional safeguards, it's incorrect to write diff --git a/pkg/kv/kvserver/logstore/stateloader.go b/pkg/kv/kvserver/logstore/stateloader.go index 0e907cfe5ec1..e6658ab510ba 100644 --- a/pkg/kv/kvserver/logstore/stateloader.go +++ b/pkg/kv/kvserver/logstore/stateloader.go @@ -174,6 +174,11 @@ func (sl StateLoader) SynthesizeHardState( return errors.Newf("can't decrease HardState.Commit from %d to %d", redact.Safe(oldHS.Commit), redact.Safe(newHS.Commit)) } + + // TODO(arul): This function can be called with an empty OldHS. In all other + // cases, where a term is included, we should be able to assert that the term + // isn't regressing (i.e. oldHS.Term >= newHS.Term). + if oldHS.Term > newHS.Term { // The existing HardState is allowed to be ahead of us, which is // relevant in practice for the split trigger. We already checked above @@ -181,9 +186,11 @@ func (sl StateLoader) SynthesizeHardState( // updated votes yet. newHS.Term = oldHS.Term } - // If the existing HardState voted in this term, remember that. + // If the existing HardState voted in this term and knows who the leader is, + // remember that. if oldHS.Term == newHS.Term { newHS.Vote = oldHS.Vote + newHS.Lead = oldHS.Lead } err := sl.SetHardState(ctx, readWriter, newHS) return errors.Wrapf(err, "writing HardState %+v", &newHS) diff --git a/pkg/kv/kvserver/replica_raft.go b/pkg/kv/kvserver/replica_raft.go index 25f13d015a1c..67a21c87e2c8 100644 --- a/pkg/kv/kvserver/replica_raft.go +++ b/pkg/kv/kvserver/replica_raft.go @@ -801,7 +801,6 @@ func (r *Replica) handleRaftReadyRaftMuLocked( } var hasReady bool - var softState *raft.SoftState var outboundMsgs []raftpb.Message var msgStorageAppend, msgStorageApply raftpb.Message r.mu.Lock() @@ -836,7 +835,6 @@ func (r *Replica) handleRaftReadyRaftMuLocked( logRaftReady(ctx, syncRd) asyncRd := makeAsyncReady(syncRd) - softState = asyncRd.SoftState outboundMsgs, msgStorageAppend, msgStorageApply = splitLocalStorageMsgs(asyncRd.Messages) } // We unquiesce if we have a Ready (= there's work to do). We also have @@ -876,24 +874,6 @@ func (r *Replica) handleRaftReadyRaftMuLocked( return stats, nil } - refreshReason := noReason - if softState != nil && leaderID != roachpb.ReplicaID(softState.Lead) { - // Refresh pending commands if the Raft leader has changed. This is usually - // the first indication we have of a new leader on a restarted node. - // - // TODO(peter): Re-proposing commands when SoftState.Lead changes can lead - // to wasteful multiple-reproposals when we later see an empty Raft command - // indicating a newly elected leader or a conf change. Replay protection - // prevents any corruption, so the waste is only a performance issue. - if log.V(3) { - log.Infof(ctx, "raft leader changed: %d -> %d", leaderID, softState.Lead) - } - if !r.store.TestingKnobs().DisableRefreshReasonNewLeader { - refreshReason = reasonNewLeader - } - leaderID = roachpb.ReplicaID(softState.Lead) - } - r.traceMessageSends(outboundMsgs, "sending messages") r.sendRaftMessages(ctx, outboundMsgs, pausedFollowers, true /* willDeliverLocal */) @@ -941,7 +921,28 @@ func (r *Replica) handleRaftReadyRaftMuLocked( } } + refreshReason := noReason if hasMsg(msgStorageAppend) { + // Leadership changes, if any, are communicated through MsgStorageAppends. + // Check if that's the case here. + if msgStorageAppend.Lead != raft.None && leaderID != roachpb.ReplicaID(msgStorageAppend.Lead) { + // Refresh pending commands if the Raft leader has changed. This is + // usually the first indication we have of a new leader on a restarted + // node. + // + // TODO(peter): Re-proposing commands when SoftState.Lead changes can lead + // to wasteful multiple-reproposals when we later see an empty Raft command + // indicating a newly elected leader or a conf change. Replay protection + // prevents any corruption, so the waste is only a performance issue. + if log.V(3) { + log.Infof(ctx, "raft leader changed: %d -> %d", leaderID, msgStorageAppend.Lead) + } + if !r.store.TestingKnobs().DisableRefreshReasonNewLeader { + refreshReason = reasonNewLeader + } + leaderID = roachpb.ReplicaID(msgStorageAppend.Lead) + } + if msgStorageAppend.Snapshot != nil { if inSnap.Desc == nil { // If we didn't expect Raft to have a snapshot but it has one @@ -969,6 +970,7 @@ func (r *Replica) handleRaftReadyRaftMuLocked( Term: msgStorageAppend.Term, Vote: msgStorageAppend.Vote, Commit: msgStorageAppend.Commit, + Lead: msgStorageAppend.Lead, } if len(msgStorageAppend.Entries) != 0 { log.Fatalf(ctx, "found Entries in MsgStorageAppend with non-empty Snapshot") @@ -1204,11 +1206,6 @@ func (r *Replica) handleRaftReadyRaftMuLocked( // All fields in asyncReady are read-only. // TODO(nvanbenschoten): move this into go.etcd.io/raft. type asyncReady struct { - // The current volatile state of a Node. - // SoftState will be nil if there is no update. - // It is not required to consume or store SoftState. - *raft.SoftState - // Messages specifies outbound messages to other peers and to local storage // threads. These messages can be sent in any order. // @@ -1220,8 +1217,7 @@ type asyncReady struct { // makeAsyncReady constructs an asyncReady from the provided Ready. func makeAsyncReady(rd raft.Ready) asyncReady { return asyncReady{ - SoftState: rd.SoftState, - Messages: rd.Messages, + Messages: rd.Messages, } } @@ -2023,7 +2019,7 @@ func (r *Replica) hasOutstandingSnapshotInFlightToStore( // HasRaftLeader returns true if the raft group has a raft leader currently. func HasRaftLeader(raftStatus *raft.Status) bool { - return raftStatus != nil && raftStatus.SoftState.Lead != 0 + return raftStatus != nil && raftStatus.HardState.Lead != 0 } // pendingCmdSlice sorts by increasing MaxLeaseIndex. diff --git a/pkg/kv/kvserver/replica_test.go b/pkg/kv/kvserver/replica_test.go index 8466ef80c6f2..8dcabb958182 100644 --- a/pkg/kv/kvserver/replica_test.go +++ b/pkg/kv/kvserver/replica_test.go @@ -134,8 +134,8 @@ func upToDateRaftStatus(repls []roachpb.ReplicaDescriptor) *raft.Status { } return &raft.Status{ BasicStatus: raft.BasicStatus{ - HardState: raftpb.HardState{Commit: 100}, - SoftState: raft.SoftState{Lead: 1, RaftState: raft.StateLeader}, + HardState: raftpb.HardState{Commit: 100, Lead: 1}, + SoftState: raft.SoftState{RaftState: raft.StateLeader}, }, Progress: prs, } @@ -8918,7 +8918,7 @@ func TestReplicaMetrics(t *testing.T) { } else { status.SoftState.RaftState = raft.StateFollower } - status.SoftState.Lead = lead + status.HardState.Lead = lead return status } desc := func(ids ...int) roachpb.RangeDescriptor { @@ -10117,9 +10117,7 @@ func TestFollowerQuiesceOnNotify(t *testing.T) { HardState: raftpb.HardState{ Term: 5, Commit: 10, - }, - SoftState: raft.SoftState{ - Lead: 1, + Lead: 1, }, }, }, @@ -11481,7 +11479,9 @@ func TestReplicaShouldCampaignOnWake(t *testing.T) { raftStatus: raft.BasicStatus{ SoftState: raft.SoftState{ RaftState: raft.StateFollower, - Lead: 2, + }, + HardState: raftpb.HardState{ + Lead: 2, }, }, livenessMap: livenesspb.IsLiveMap{ @@ -11599,7 +11599,9 @@ func TestReplicaShouldCampaignOnLeaseRequestRedirect(t *testing.T) { raftStatus: raft.BasicStatus{ SoftState: raft.SoftState{ RaftState: raft.StateFollower, - Lead: 2, + }, + HardState: raftpb.HardState{ + Lead: 2, }, }, livenessMap: livenesspb.IsLiveMap{ @@ -11710,7 +11712,9 @@ func TestReplicaShouldForgetLeaderOnVoteRequest(t *testing.T) { raftStatus: raft.BasicStatus{ SoftState: raft.SoftState{ RaftState: raft.StateFollower, - Lead: 2, + }, + HardState: raftpb.HardState{ + Lead: 2, }, }, livenessMap: livenesspb.IsLiveMap{ diff --git a/pkg/kv/kvserver/stateloader/initial_test.go b/pkg/kv/kvserver/stateloader/initial_test.go index f3df866773da..ca6331f58b3e 100644 --- a/pkg/kv/kvserver/stateloader/initial_test.go +++ b/pkg/kv/kvserver/stateloader/initial_test.go @@ -32,7 +32,7 @@ func TestSynthesizeHardState(t *testing.T) { eng := storage.NewDefaultInMemForTesting() stopper.AddCloser(eng) - tHS := raftpb.HardState{Term: 2, Vote: 3, Commit: 4} + tHS := raftpb.HardState{Term: 2, Vote: 3, Commit: 4, Lead: 5} testCases := []struct { TruncTerm kvpb.RaftTerm @@ -45,10 +45,10 @@ func TestSynthesizeHardState(t *testing.T) { // Can't wind back the committed index of the new HardState. {OldHS: &tHS, RaftAppliedIndex: kvpb.RaftIndex(tHS.Commit - 1), Err: "can't decrease HardState.Commit"}, {OldHS: &tHS, RaftAppliedIndex: kvpb.RaftIndex(tHS.Commit), NewHS: tHS}, - {OldHS: &tHS, RaftAppliedIndex: kvpb.RaftIndex(tHS.Commit + 1), NewHS: raftpb.HardState{Term: tHS.Term, Vote: 3, Commit: tHS.Commit + 1}}, - // Higher Term is picked up, but vote isn't carried over when the term - // changes. - {OldHS: &tHS, RaftAppliedIndex: kvpb.RaftIndex(tHS.Commit), TruncTerm: 11, NewHS: raftpb.HardState{Term: 11, Vote: 0, Commit: tHS.Commit}}, + {OldHS: &tHS, RaftAppliedIndex: kvpb.RaftIndex(tHS.Commit + 1), NewHS: raftpb.HardState{Term: tHS.Term, Vote: 3, Commit: tHS.Commit + 1, Lead: 5}}, + // Higher Term is picked up, but Vote and Lead aren't carried over when the + // term changes. + {OldHS: &tHS, RaftAppliedIndex: kvpb.RaftIndex(tHS.Commit), TruncTerm: 11, NewHS: raftpb.HardState{Term: 11, Vote: 0, Commit: tHS.Commit, Lead: 0}}, } for i, test := range testCases { diff --git a/pkg/kv/kvserver/testdata/TestBelowRaftProtosDontChange/HardState b/pkg/kv/kvserver/testdata/TestBelowRaftProtosDontChange/HardState index a1ea2ff7ced1..f3188ac57117 100644 --- a/pkg/kv/kvserver/testdata/TestBelowRaftProtosDontChange/HardState +++ b/pkg/kv/kvserver/testdata/TestBelowRaftProtosDontChange/HardState @@ -1,3 +1,3 @@ echo ---- -5484201021249543052 +2027450491469602084 diff --git a/pkg/raft/node.go b/pkg/raft/node.go index ef06e5b9a654..fd7e820bfb9f 100644 --- a/pkg/raft/node.go +++ b/pkg/raft/node.go @@ -41,12 +41,11 @@ var ( // SoftState provides state that is useful for logging and debugging. // The state is volatile and does not need to be persisted to the WAL. type SoftState struct { - Lead pb.PeerID // must use atomic operations to access; keep 64-bit aligned. RaftState StateType } func (a *SoftState) equal(b *SoftState) bool { - return a.Lead == b.Lead && a.RaftState == b.RaftState + return a.RaftState == b.RaftState } // Ready encapsulates the entries and messages that are ready to read, @@ -112,7 +111,7 @@ type Ready struct { } func isHardStateEqual(a, b pb.HardState) bool { - return a.Term == b.Term && a.Vote == b.Vote && a.Commit == b.Commit + return a.Term == b.Term && a.Vote == b.Vote && a.Commit == b.Commit && a.Lead == b.Lead } // IsEmptyHardState returns true if the given HardState is empty. diff --git a/pkg/raft/node_test.go b/pkg/raft/node_test.go index 48c43455259f..97bc49e9d37e 100644 --- a/pkg/raft/node_test.go +++ b/pkg/raft/node_test.go @@ -150,7 +150,7 @@ func TestNodePropose(t *testing.T) { rd := <-n.Ready() s.Append(rd.Entries) // change the step function to appendStep until this raft becomes leader - if rd.SoftState.Lead == r.id { + if rd.HardState.Lead == r.id { r.step = appendStep n.Advance() break @@ -215,7 +215,7 @@ func TestNodeProposeConfig(t *testing.T) { rd := <-n.Ready() s.Append(rd.Entries) // change the step function to appendStep until this raft becomes leader - if rd.SoftState.Lead == r.id { + if rd.HardState.Lead == r.id { r.step = appendStep n.Advance() break @@ -366,7 +366,7 @@ func TestNodeProposeWaitDropped(t *testing.T) { rd := <-n.Ready() s.Append(rd.Entries) // change the step function to dropStep until this raft becomes leader - if rd.SoftState.Lead == r.id { + if rd.HardState.Lead == r.id { r.step = dropStep n.Advance() break @@ -442,7 +442,7 @@ func TestNodeStart(t *testing.T) { require.NoError(t, err) wants := []Ready{ { - HardState: raftpb.HardState{Term: 1, Commit: 1, Vote: 0}, + HardState: raftpb.HardState{Term: 1, Commit: 1, Vote: 0, Lead: 0}, Entries: []raftpb.Entry{ {Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata}, }, @@ -452,13 +452,13 @@ func TestNodeStart(t *testing.T) { MustSync: true, }, { - HardState: raftpb.HardState{Term: 2, Commit: 2, Vote: 1}, + HardState: raftpb.HardState{Term: 2, Commit: 2, Vote: 1, Lead: 1}, Entries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, CommittedEntries: []raftpb.Entry{{Term: 2, Index: 2, Data: nil}}, MustSync: true, }, { - HardState: raftpb.HardState{Term: 2, Commit: 3, Vote: 1}, + HardState: raftpb.HardState{Term: 2, Commit: 3, Vote: 1, Lead: 1}, Entries: nil, CommittedEntries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, MustSync: false, @@ -647,7 +647,6 @@ func TestSoftStateEqual(t *testing.T) { we bool }{ {&SoftState{}, true}, - {&SoftState{Lead: 1}, false}, {&SoftState{RaftState: StateLeader}, false}, } for i, tt := range tests { @@ -664,6 +663,7 @@ func TestIsHardStateEqual(t *testing.T) { {raftpb.HardState{Vote: 1}, false}, {raftpb.HardState{Commit: 1}, false}, {raftpb.HardState{Term: 1}, false}, + {raftpb.HardState{Lead: 1}, false}, } for i, tt := range tests { diff --git a/pkg/raft/raft.go b/pkg/raft/raft.go index 954fa6f65271..4928d16317d0 100644 --- a/pkg/raft/raft.go +++ b/pkg/raft/raft.go @@ -443,7 +443,7 @@ func newRaft(c *Config) *raft { if c.Applied > 0 { raftlog.appliedTo(c.Applied, 0 /* size */) } - r.becomeFollower(r.Term, None) + r.becomeFollower(r.Term, r.lead) var nodesStrs []string for _, n := range r.trk.VoterNodes() { @@ -458,13 +458,14 @@ func newRaft(c *Config) *raft { func (r *raft) hasLeader() bool { return r.lead != None } -func (r *raft) softState() SoftState { return SoftState{Lead: r.lead, RaftState: r.state} } +func (r *raft) softState() SoftState { return SoftState{RaftState: r.state} } func (r *raft) hardState() pb.HardState { return pb.HardState{ Term: r.Term, Vote: r.Vote, Commit: r.raftLog.committed, + Lead: r.lead, } } @@ -739,6 +740,8 @@ func (r *raft) reset(term uint64) { r.Term = term r.Vote = None } + + // TODO(arul): we should only reset this if the term has changed. r.lead = None r.electionElapsed = 0 @@ -846,6 +849,11 @@ func (r *raft) tickHeartbeat() { } } +// TODO(arul): Consider removing the lead argument from this function. Instead, +// for all the methods that want to set the leader explicitly (the ones that are +// passing in m.From for this field), we can instead have them use an assignLead +// function instead; in there, we can add safety checks to ensure we're not +// overwriting the leader. func (r *raft) becomeFollower(term uint64, lead pb.PeerID) { r.step = stepFollower r.reset(term) @@ -879,6 +887,10 @@ func (r *raft) becomePreCandidate() { r.step = stepCandidate r.trk.ResetVotes() r.tick = r.tickElection + // TODO(arul): We're forgetting the raft leader here. From the perspective of + // leader leases, this is fine, because we wouldn't be here unless we'd + // revoked StoreLiveness support for the leader's store to begin with. It's + // a bit weird from the perspective of raft though. See if we can avoid this. r.lead = None r.state = StatePreCandidate r.logger.Infof("%x became pre-candidate at term %d", r.id, r.Term) @@ -1226,7 +1238,13 @@ func stepLeader(r *raft, m pb.Message) error { case pb.MsgCheckQuorum: if !r.trk.QuorumActive() { r.logger.Warningf("%x stepped down to follower since quorum is not active", r.id) - r.becomeFollower(r.Term, None) + // NB: Stepping down because of CheckQuorum is a special, in that we know + // the QSE is in the past. This means that the leader can safely call a + // new election or vote for a different peer without regressing the QSE. + // We don't need to/want to give this any special treatment -- instead, we + // handle this like the general step down case by simply remembering the + // term/lead information from our stint as the leader. + r.becomeFollower(r.Term, r.id) } // Mark everyone (but ourselves) as inactive in preparation for the next // CheckQuorum. @@ -1606,7 +1624,7 @@ func stepCandidate(r *raft, m pb.Message) error { case quorum.VoteLost: // pb.MsgPreVoteResp contains future term of pre-candidate // m.Term > r.Term; reuse r.Term - r.becomeFollower(r.Term, None) + r.becomeFollower(r.Term, r.lead) } case pb.MsgTimeoutNow: r.logger.Debugf("%x [term %d state %v] ignored MsgTimeoutNow from %x", r.id, r.Term, r.state, m.From) @@ -1623,11 +1641,19 @@ func stepFollower(r *raft, m pb.Message) error { } else if r.disableProposalForwarding { r.logger.Infof("%x not forwarding to leader %x at term %d; dropping proposal", r.id, r.lead, r.Term) return ErrProposalDropped + } else if r.lead == r.id { + r.logger.Infof("%x not forwarding to itself at term %d; dropping proposal", r.id, r.Term) + return ErrProposalDropped } m.To = r.lead r.send(m) case pb.MsgApp: r.electionElapsed = 0 + // TODO(arul): Once r.lead != None, we shouldn't need to update r.lead + // anymore within the course of a single term (in the context of which this + // function is always called). Instead, if r.lead != None, we should be able + // to assert that the leader hasn't changed within a given term. Maybe at + // the caller itself. r.lead = m.From r.handleAppendEntries(m) case pb.MsgHeartbeat: @@ -1642,6 +1668,9 @@ func stepFollower(r *raft, m pb.Message) error { if r.lead == None { r.logger.Infof("%x no leader at term %d; dropping leader transfer msg", r.id, r.Term) return nil + } else if r.lead == r.id { + r.logger.Infof("%x is itself the leader at term %d; dropping leader transfer msg", r.id, r.Term) + return nil } m.To = r.lead r.send(m) @@ -1944,7 +1973,9 @@ func (r *raft) switchToConfig(cfg tracker.Config, trk tracker.ProgressMap) pb.Co // interruption). This might still drop some proposals but it's better than // nothing. if r.stepDownOnRemoval { - r.becomeFollower(r.Term, None) + // NB: Similar to the CheckQuorum step down case, we must remember our + // prior stint as leader, lest we regress the QSE. + r.becomeFollower(r.Term, r.lead) } return cs } @@ -1978,6 +2009,7 @@ func (r *raft) loadState(state pb.HardState) { r.raftLog.committed = state.Commit r.Term = state.Term r.Vote = state.Vote + r.lead = state.Lead } // pastElectionTimeout returns true if r.electionElapsed is greater diff --git a/pkg/raft/raftpb/raft.proto b/pkg/raft/raftpb/raft.proto index fcdc8115139f..3780a56f07f9 100644 --- a/pkg/raft/raftpb/raft.proto +++ b/pkg/raft/raftpb/raft.proto @@ -88,11 +88,12 @@ message Message { optional uint64 index = 6 [(gogoproto.nullable) = false]; repeated Entry entries = 7 [(gogoproto.nullable) = false]; optional uint64 commit = 8 [(gogoproto.nullable) = false]; + optional uint64 lead = 16 [(gogoproto.nullable) = false, (gogoproto.casttype) = "PeerID"]; // (type=MsgStorageAppend,vote=5,term=10) means the local node is voting for - // peer 5 in term 10. For MsgStorageAppends, the term, vote, and commit fields - // will either all be set (to facilitate the construction of a HardState) if - // any of the fields have changed or will all be unset if none of the fields - // have changed. + // peer 5 in term 10. For MsgStorageAppends, the term, vote, lead, and commit + // fields will either all be set (to facilitate the construction of a + // HardState) if any of the fields have changed or will all be unset if none + // of the fields have changed. optional uint64 vote = 13 [(gogoproto.nullable) = false, (gogoproto.casttype) = "PeerID"]; // snapshot is non-nil and non-empty for MsgSnap messages and nil for all other // message types. However, peer nodes running older binary versions may send a @@ -107,26 +108,27 @@ message Message { // is complete. Populated for MsgStorageAppend and MsgStorageApply messages. repeated Message responses = 14 [(gogoproto.nullable) = false]; - // match is the log index up to which the follower's log must persistently - // match the leader's. If the follower's persistent log is shorter, it means - // the follower has broken its promise and violated safety of Raft. Typically - // this means the environment (Storage) hasn't provided the required - // durability guarantees. - // - // If a follower sees a match index exceeding its log's last index, it must - // cease its membership (stop voting and acking appends) in the raft group, in - // order to limit the damage. Today it simply panics. - // - // match is only populated by the leader when sending messages to a voting - // follower. This can be 0 if the leader hasn't yet established the follower's - // match index, or for backward compatibility. - optional uint64 match = 15 [(gogoproto.nullable) = false]; + // match is the log index up to which the follower's log must persistently + // match the leader's. If the follower's persistent log is shorter, it means + // the follower has broken its promise and violated safety of Raft. Typically + // this means the environment (Storage) hasn't provided the required + // durability guarantees. + // + // If a follower sees a match index exceeding its log's last index, it must + // cease its membership (stop voting and acking appends) in the raft group, in + // order to limit the damage. Today it simply panics. + // + // match is only populated by the leader when sending messages to a voting + // follower. This can be 0 if the leader hasn't yet established the follower's + // match index, or for backward compatibility. + optional uint64 match = 15 [(gogoproto.nullable) = false]; } message HardState { optional uint64 term = 1 [(gogoproto.nullable) = false]; optional uint64 vote = 2 [(gogoproto.nullable) = false, (gogoproto.casttype) = "PeerID"]; optional uint64 commit = 3 [(gogoproto.nullable) = false]; + optional uint64 lead = 4 [(gogoproto.nullable) = false, (gogoproto.casttype) = "PeerID"]; } // ConfChangeTransition specifies the behavior of a configuration change with diff --git a/pkg/raft/raftpb/raft_test.go b/pkg/raft/raftpb/raft_test.go index 47e4b0bd3914..b550350a7758 100644 --- a/pkg/raft/raftpb/raft_test.go +++ b/pkg/raft/raftpb/raft_test.go @@ -48,10 +48,10 @@ func TestProtoMemorySizes(t *testing.T) { assert(unsafe.Sizeof(s), if64Bit(144, 80), "Snapshot") var m Message - assert(unsafe.Sizeof(m), if64Bit(168, 112), "Message") + assert(unsafe.Sizeof(m), if64Bit(176, 112), "Message") var hs HardState - assert(unsafe.Sizeof(hs), 24, "HardState") + assert(unsafe.Sizeof(hs), 32, "HardState") var cs ConfState assert(unsafe.Sizeof(cs), if64Bit(104, 52), "ConfState") diff --git a/pkg/raft/rafttest/interaction_env_handler_process_append_thread.go b/pkg/raft/rafttest/interaction_env_handler_process_append_thread.go index f0ff8d076c29..72f2fb32642d 100644 --- a/pkg/raft/rafttest/interaction_env_handler_process_append_thread.go +++ b/pkg/raft/rafttest/interaction_env_handler_process_append_thread.go @@ -63,6 +63,7 @@ func (env *InteractionEnv) ProcessAppendThread(idx int) error { Term: m.Term, Vote: m.Vote, Commit: m.Commit, + Lead: m.Lead, } var snap raftpb.Snapshot if m.Snapshot != nil { diff --git a/pkg/raft/rafttest/node_test.go b/pkg/raft/rafttest/node_test.go index 9145284950a4..4323b76005b9 100644 --- a/pkg/raft/rafttest/node_test.go +++ b/pkg/raft/rafttest/node_test.go @@ -143,7 +143,7 @@ func waitLeader(ns []*node) int { l = make(map[pb.PeerID]struct{}) for i, n := range ns { - lead := n.Status().SoftState.Lead + lead := n.Status().HardState.Lead if lead != 0 { l[lead] = struct{}{} if n.id == lead { diff --git a/pkg/raft/rawnode.go b/pkg/raft/rawnode.go index d8954a9f7c93..e873a90bb836 100644 --- a/pkg/raft/rawnode.go +++ b/pkg/raft/rawnode.go @@ -180,9 +180,10 @@ func MustSync(st, prevst pb.HardState, entsnum int) bool { // Persistent state on all servers: // (Updated on stable storage before responding to RPCs) // currentTerm + // currentLead // votedFor // log entries[] - return entsnum != 0 || st.Vote != prevst.Vote || st.Term != prevst.Term + return entsnum != 0 || st.Vote != prevst.Vote || st.Term != prevst.Term || st.Lead != prevst.Lead } func needStorageAppendMsg(r *raft, rd Ready) bool { @@ -226,6 +227,7 @@ func newStorageAppendMsg(r *raft, rd Ready) pb.Message { m.Term = rd.Term m.Vote = rd.Vote m.Commit = rd.Commit + m.Lead = rd.Lead } if !IsEmptySnap(rd.Snapshot) { snap := rd.Snapshot diff --git a/pkg/raft/rawnode_test.go b/pkg/raft/rawnode_test.go index 4f79f020da66..547186894582 100644 --- a/pkg/raft/rawnode_test.go +++ b/pkg/raft/rawnode_test.go @@ -251,7 +251,7 @@ func TestRawNodeProposeAndConfChange(t *testing.T) { } rawNode.Advance(rd) // Once we are the leader, propose a command and a ConfChange. - if !proposed && rd.SoftState.Lead == rawNode.raft.id { + if !proposed && rd.HardState.Lead == rawNode.raft.id { require.NoError(t, rawNode.Propose([]byte("somedata"))) if ccv1, ok := tc.cc.AsV1(); ok { ccdata, err = ccv1.Marshal() @@ -380,7 +380,7 @@ func TestRawNodeJointAutoLeave(t *testing.T) { } rawNode.Advance(rd) // Once we are the leader, propose a command and a ConfChange. - if !proposed && rd.SoftState.Lead == rawNode.raft.id { + if !proposed && rd.HardState.Lead == rawNode.raft.id { require.NoError(t, rawNode.Propose([]byte("somedata"))) ccdata, err = testCc.Marshal() require.NoError(t, err) @@ -456,7 +456,7 @@ func TestRawNodeProposeAddDuplicateNode(t *testing.T) { for { rd = rawNode.Ready() s.Append(rd.Entries) - if rd.SoftState.Lead == rawNode.raft.id { + if rd.HardState.Lead == rawNode.raft.id { rawNode.Advance(rd) break } @@ -520,8 +520,8 @@ func TestRawNodeStart(t *testing.T) { {Term: 1, Index: 3, Data: []byte("foo")}, // non-empty entry } want := Ready{ - SoftState: &SoftState{Lead: 1, RaftState: StateLeader}, - HardState: pb.HardState{Term: 1, Commit: 3, Vote: 1}, + SoftState: &SoftState{RaftState: StateLeader}, + HardState: pb.HardState{Term: 1, Commit: 3, Vote: 1, Lead: 1}, Entries: nil, // emitted & checked in intermediate Ready cycle CommittedEntries: entries, MustSync: false, // since we're only applying, not appending @@ -610,10 +610,10 @@ func TestRawNodeRestart(t *testing.T) { {Term: 1, Index: 1}, {Term: 1, Index: 2, Data: []byte("foo")}, } - st := pb.HardState{Term: 1, Commit: 1} + st := pb.HardState{Term: 1, Commit: 1, Lead: 1} want := Ready{ - HardState: emptyState, + HardState: emptyState, // no HardState is emitted because there was no change // commit up to commit index in st CommittedEntries: entries[:st.Commit], MustSync: false, @@ -628,6 +628,18 @@ func TestRawNodeRestart(t *testing.T) { assert.Equal(t, want, rd) rawNode.Advance(rd) assert.False(t, rawNode.HasReady()) + // Ensure that the HardState was correctly loaded post restart. + assert.Equal(t, uint64(1), rawNode.raft.Term) + assert.Equal(t, uint64(1), rawNode.raft.raftLog.committed) + assert.Equal(t, pb.PeerID(1), rawNode.raft.lead) + assert.True(t, rawNode.raft.state == StateFollower) + + // Ensure we campaign after the election timeout has elapsed. + for i := 0; i < rawNode.raft.randomizedElectionTimeout; i++ { + rawNode.raft.tick() + } + assert.Equal(t, StateCandidate, rawNode.raft.state) + assert.Equal(t, uint64(2), rawNode.raft.Term) // this should in-turn bump the term } func TestRawNodeRestartFromSnapshot(t *testing.T) { diff --git a/pkg/raft/status.go b/pkg/raft/status.go index 5fa4341bce23..41e7946e6bd1 100644 --- a/pkg/raft/status.go +++ b/pkg/raft/status.go @@ -99,6 +99,23 @@ func getBasicStatus(r *raft) BasicStatus { s.HardState = r.hardState() s.SoftState = r.softState() s.Applied = r.raftLog.applied + if s.RaftState == StateFollower && s.Lead == r.id { + // A raft leader's term ends when it is shut down. It'll rejoin its peers as + // a follower when it comes back up, but its Lead and Term field may still + // correspond to its pre-restart leadership term. We expect this to quickly + // be updated when it hears from the new leader, if one was elected in its + // absence, or when it campaigns. + // + // The layers above raft (in particular kvserver) do not handle the case + // where a raft node's state is StateFollower but its lead field points to + // itself. They expect the Lead field to correspond to the current leader, + // which we know we are not. For their benefit, we overwrite the Lead field + // to None. + // + // TODO(arul): the layers above should not conflate Lead with current + // leader. Fix that and get rid of this overwrite. + s.HardState.Lead = None + } return s } diff --git a/pkg/raft/testdata/async_storage_writes.txt b/pkg/raft/testdata/async_storage_writes.txt index 1a281301f6f6..48a0763d6e2e 100644 --- a/pkg/raft/testdata/async_storage_writes.txt +++ b/pkg/raft/testdata/async_storage_writes.txt @@ -24,8 +24,8 @@ stabilize ---- > 1 handling Ready Ready MustSync=true: - Lead:0 State:StateCandidate - HardState Term:1 Vote:1 Commit:10 + State:StateCandidate + HardState Term:1 Vote:1 Commit:10 Lead:0 Messages: 1->2 MsgVote Term:1 Log:1/10 1->3 MsgVote Term:1 Log:1/10 @@ -49,14 +49,14 @@ stabilize 1->1 MsgVoteResp Term:1 Log:0/0 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:10 + HardState Term:1 Vote:1 Commit:10 Lead:0 Messages: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Responses:[ 2->1 MsgVoteResp Term:1 Log:0/0 ] > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:10 + HardState Term:1 Vote:1 Commit:10 Lead:0 Messages: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Responses:[ 3->1 MsgVoteResp Term:1 Log:0/0 @@ -83,13 +83,14 @@ stabilize 3->1 MsgVoteResp Term:1 Log:0/0 > 1 handling Ready Ready MustSync=true: - Lead:1 State:StateLeader + State:StateLeader + HardState Term:1 Vote:1 Commit:10 Lead:1 Entries: 1/11 EntryNormal "" Messages: 1->2 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] 1->3 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] - 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Entries:[1/11 EntryNormal ""] Responses:[ 1->1 MsgAppResp Term:1 Log:0/11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 ] @@ -99,27 +100,27 @@ stabilize 1->3 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] > 1 processing append thread Processing: - 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Entries:[1/11 EntryNormal ""] Responses: 1->1 MsgAppResp Term:1 Log:0/11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 > 2 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower + HardState Term:1 Vote:1 Commit:10 Lead:1 Entries: 1/11 EntryNormal "" Messages: - 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Entries:[1/11 EntryNormal ""] Responses:[ 2->1 MsgAppResp Term:1 Log:0/11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11 ] > 3 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower + HardState Term:1 Vote:1 Commit:10 Lead:1 Entries: 1/11 EntryNormal "" Messages: - 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Entries:[1/11 EntryNormal ""] Responses:[ 3->1 MsgAppResp Term:1 Log:0/11 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11 ] @@ -128,13 +129,13 @@ stabilize AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 > 2 processing append thread Processing: - 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Entries:[1/11 EntryNormal ""] Responses: 2->1 MsgAppResp Term:1 Log:0/11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11 > 3 processing append thread Processing: - 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Entries:[1/11 EntryNormal ""] Responses: 3->1 MsgAppResp Term:1 Log:0/11 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11 @@ -147,7 +148,7 @@ stabilize AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 + HardState Term:1 Vote:1 Commit:11 Lead:1 CommittedEntries: 1/11 EntryNormal "" Messages: @@ -172,7 +173,7 @@ stabilize ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 + HardState Term:1 Vote:1 Commit:11 Lead:1 CommittedEntries: 1/11 EntryNormal "" Messages: @@ -184,7 +185,7 @@ stabilize ] > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 + HardState Term:1 Vote:1 Commit:11 Lead:1 CommittedEntries: 1/11 EntryNormal "" Messages: @@ -374,7 +375,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:12 + HardState Term:1 Vote:1 Commit:12 Lead:1 Entries: 1/14 EntryNormal "prop_3" CommittedEntries: @@ -409,7 +410,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:12 + HardState Term:1 Vote:1 Commit:12 Lead:1 Entries: 1/14 EntryNormal "prop_3" CommittedEntries: @@ -425,7 +426,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:12 + HardState Term:1 Vote:1 Commit:12 Lead:1 Entries: 1/14 EntryNormal "prop_3" CommittedEntries: @@ -478,7 +479,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:13 + HardState Term:1 Vote:1 Commit:13 Lead:1 Entries: 1/15 EntryNormal "prop_4" CommittedEntries: @@ -513,7 +514,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:13 + HardState Term:1 Vote:1 Commit:13 Lead:1 Entries: 1/15 EntryNormal "prop_4" CommittedEntries: @@ -529,7 +530,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:13 + HardState Term:1 Vote:1 Commit:13 Lead:1 Entries: 1/15 EntryNormal "prop_4" CommittedEntries: @@ -603,7 +604,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:14 + HardState Term:1 Vote:1 Commit:14 Lead:1 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: @@ -631,7 +632,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:14 + HardState Term:1 Vote:1 Commit:14 Lead:1 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: @@ -644,7 +645,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:14 + HardState Term:1 Vote:1 Commit:14 Lead:1 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: @@ -715,7 +716,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:15 + HardState Term:1 Vote:1 Commit:15 Lead:1 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: @@ -741,7 +742,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:15 + HardState Term:1 Vote:1 Commit:15 Lead:1 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: @@ -753,7 +754,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:15 + HardState Term:1 Vote:1 Commit:15 Lead:1 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: diff --git a/pkg/raft/testdata/async_storage_writes_append_aba_race.txt b/pkg/raft/testdata/async_storage_writes_append_aba_race.txt index 83964fe7d087..fbf371d0d6d4 100644 --- a/pkg/raft/testdata/async_storage_writes_append_aba_race.txt +++ b/pkg/raft/testdata/async_storage_writes_append_aba_race.txt @@ -85,8 +85,8 @@ INFO 3 [logterm: 1, index: 11] sent MsgVote request to 7 at term 2 process-ready 3 ---- Ready MustSync=true: -Lead:0 State:StateCandidate -HardState Term:2 Vote:3 Commit:11 +State:StateCandidate +HardState Term:2 Vote:3 Commit:11 Lead:0 Messages: 3->1 MsgVote Term:2 Log:1/11 3->2 MsgVote Term:2 Log:1/11 @@ -117,24 +117,21 @@ process-ready 4 5 6 ---- > 4 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:2 Vote:3 Commit:11 + HardState Term:2 Vote:3 Commit:11 Lead:0 Messages: 4->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ 4->3 MsgVoteResp Term:2 Log:0/0 ] > 5 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:2 Vote:3 Commit:11 + HardState Term:2 Vote:3 Commit:11 Lead:0 Messages: 5->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ 5->3 MsgVoteResp Term:2 Log:0/0 ] > 6 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:2 Vote:3 Commit:11 + HardState Term:2 Vote:3 Commit:11 Lead:0 Messages: 6->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ 6->3 MsgVoteResp Term:2 Log:0/0 @@ -187,7 +184,8 @@ INFO 3 became leader at term 2 process-ready 3 ---- Ready MustSync=true: -Lead:3 State:StateLeader +State:StateLeader +HardState Term:2 Vote:3 Commit:11 Lead:3 Entries: 2/12 EntryNormal "" Messages: @@ -197,7 +195,7 @@ Messages: 3->5 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] 3->6 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] 3->7 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] -3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[2/12 EntryNormal ""] Responses:[ +3->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Entries:[2/12 EntryNormal ""] Responses:[ 3->3 MsgAppResp Term:2 Log:0/12 AppendThread->3 MsgStorageAppendResp Term:2 Log:2/12 ] @@ -222,8 +220,7 @@ dropped: 3->7 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] process-ready 1 ---- Ready MustSync=true: -Lead:3 State:StateFollower -HardState Term:2 Commit:11 +HardState Term:2 Commit:11 Lead:3 Entries: 2/12 EntryNormal "" Messages: @@ -249,8 +246,8 @@ INFO 4 [logterm: 1, index: 11] sent MsgVote request to 7 at term 3 process-ready 4 ---- Ready MustSync=true: -Lead:0 State:StateCandidate -HardState Term:3 Vote:4 Commit:11 +State:StateCandidate +HardState Term:3 Vote:4 Commit:11 Lead:0 Messages: 4->1 MsgVote Term:3 Log:1/11 4->2 MsgVote Term:3 Log:1/11 @@ -281,22 +278,21 @@ process-ready 5 6 7 ---- > 5 handling Ready Ready MustSync=true: - HardState Term:3 Vote:4 Commit:11 + HardState Term:3 Vote:4 Commit:11 Lead:0 Messages: 5->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ 5->4 MsgVoteResp Term:3 Log:0/0 ] > 6 handling Ready Ready MustSync=true: - HardState Term:3 Vote:4 Commit:11 + HardState Term:3 Vote:4 Commit:11 Lead:0 Messages: 6->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ 6->4 MsgVoteResp Term:3 Log:0/0 ] > 7 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:3 Vote:4 Commit:11 + HardState Term:3 Vote:4 Commit:11 Lead:0 Messages: 7->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ 7->4 MsgVoteResp Term:3 Log:0/0 @@ -344,7 +340,8 @@ INFO 4 became leader at term 3 process-ready 4 ---- Ready MustSync=true: -Lead:4 State:StateLeader +State:StateLeader +HardState Term:3 Vote:4 Commit:11 Lead:4 Entries: 3/12 EntryNormal "" Messages: @@ -354,7 +351,7 @@ Messages: 4->5 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] 4->6 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] 4->7 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] -4->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[ +4->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Entries:[3/12 EntryNormal ""] Responses:[ 4->4 MsgAppResp Term:3 Log:0/12 AppendThread->4 MsgStorageAppendResp Term:3 Log:3/12 ] @@ -394,8 +391,7 @@ INFO 1 became follower at term 3 process-ready 1 ---- Ready MustSync=true: -Lead:4 State:StateFollower -HardState Term:3 Commit:11 +HardState Term:3 Commit:11 Lead:4 Messages: 1->4 MsgHeartbeatResp Term:3 Log:0/0 1->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Responses:[ diff --git a/pkg/raft/testdata/campaign.txt b/pkg/raft/testdata/campaign.txt index 4eefab36d6bc..1f4b10e65d68 100644 --- a/pkg/raft/testdata/campaign.txt +++ b/pkg/raft/testdata/campaign.txt @@ -25,8 +25,8 @@ stabilize ---- > 1 handling Ready Ready MustSync=true: - Lead:0 State:StateCandidate - HardState Term:1 Vote:1 Commit:2 + State:StateCandidate + HardState Term:1 Vote:1 Commit:2 Lead:0 Messages: 1->2 MsgVote Term:1 Log:1/2 1->3 MsgVote Term:1 Log:1/2 @@ -44,12 +44,12 @@ stabilize INFO 3 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:2 + HardState Term:1 Vote:1 Commit:2 Lead:0 Messages: 2->1 MsgVoteResp Term:1 Log:0/0 > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:2 + HardState Term:1 Vote:1 Commit:2 Lead:0 Messages: 3->1 MsgVoteResp Term:1 Log:0/0 > 1 receiving messages @@ -60,7 +60,8 @@ stabilize 3->1 MsgVoteResp Term:1 Log:0/0 > 1 handling Ready Ready MustSync=true: - Lead:1 State:StateLeader + State:StateLeader + HardState Term:1 Vote:1 Commit:2 Lead:1 Entries: 1/3 EntryNormal "" Messages: @@ -72,14 +73,14 @@ stabilize 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] > 2 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower + HardState Term:1 Vote:1 Commit:2 Lead:1 Entries: 1/3 EntryNormal "" Messages: 2->1 MsgAppResp Term:1 Log:0/3 > 3 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower + HardState Term:1 Vote:1 Commit:2 Lead:1 Entries: 1/3 EntryNormal "" Messages: @@ -89,7 +90,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/3 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:3 + HardState Term:1 Vote:1 Commit:3 Lead:1 CommittedEntries: 1/3 EntryNormal "" Messages: @@ -101,14 +102,14 @@ stabilize 1->3 MsgApp Term:1 Log:1/3 Commit:3 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:3 + HardState Term:1 Vote:1 Commit:3 Lead:1 CommittedEntries: 1/3 EntryNormal "" Messages: 2->1 MsgAppResp Term:1 Log:0/3 > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:3 + HardState Term:1 Vote:1 Commit:3 Lead:1 CommittedEntries: 1/3 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/campaign_learner_must_vote.txt b/pkg/raft/testdata/campaign_learner_must_vote.txt index d05de374f191..e0510edec63f 100644 --- a/pkg/raft/testdata/campaign_learner_must_vote.txt +++ b/pkg/raft/testdata/campaign_learner_must_vote.txt @@ -61,8 +61,8 @@ INFO 2 [logterm: 1, index: 4] sent MsgVote request to 3 at term 2 process-ready 2 ---- Ready MustSync=true: -Lead:0 State:StateCandidate -HardState Term:2 Vote:2 Commit:4 +State:StateCandidate +HardState Term:2 Vote:2 Commit:4 Lead:0 Messages: 2->1 MsgVote Term:2 Log:1/4 2->3 MsgVote Term:2 Log:1/4 @@ -82,8 +82,7 @@ stabilize 3 INFO 3 [logterm: 1, index: 3, vote: 0] cast MsgVote for 2 [logterm: 1, index: 4] at term 2 > 3 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:2 Vote:2 Commit:3 + HardState Term:2 Vote:2 Commit:3 Lead:0 Messages: 3->2 MsgVoteResp Term:2 Log:0/0 @@ -96,7 +95,8 @@ stabilize 2 3 INFO 2 became leader at term 2 > 2 handling Ready Ready MustSync=true: - Lead:2 State:StateLeader + State:StateLeader + HardState Term:2 Vote:2 Commit:4 Lead:2 Entries: 2/5 EntryNormal "" Messages: @@ -106,8 +106,8 @@ stabilize 2 3 2->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] DEBUG 3 [logterm: 0, index: 4] rejected MsgApp [logterm: 1, index: 4] from 2 > 3 handling Ready - Ready MustSync=false: - Lead:2 State:StateFollower + Ready MustSync=true: + HardState Term:2 Vote:2 Commit:3 Lead:2 Messages: 3->2 MsgAppResp Term:2 Log:1/4 Rejected (Hint: 3) > 2 receiving messages @@ -128,7 +128,7 @@ stabilize 2 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:4 + HardState Term:2 Vote:2 Commit:4 Lead:2 Entries: 1/4 EntryConfChangeV2 v3 2/5 EntryNormal "" @@ -141,7 +141,7 @@ stabilize 2 3 3->2 MsgAppResp Term:2 Log:0/5 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:5 + HardState Term:2 Vote:2 Commit:5 Lead:2 CommittedEntries: 2/5 EntryNormal "" Messages: @@ -150,7 +150,7 @@ stabilize 2 3 2->3 MsgApp Term:2 Log:2/5 Commit:5 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:5 + HardState Term:2 Vote:2 Commit:5 Lead:2 CommittedEntries: 2/5 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/checkquorum.txt b/pkg/raft/testdata/checkquorum.txt index b25c1e63daec..015e05943678 100644 --- a/pkg/raft/testdata/checkquorum.txt +++ b/pkg/raft/testdata/checkquorum.txt @@ -36,8 +36,8 @@ stabilize ---- > 2 handling Ready Ready MustSync=true: - Lead:0 State:StateCandidate - HardState Term:2 Vote:2 Commit:11 + State:StateCandidate + HardState Term:2 Vote:2 Commit:11 Lead:0 Messages: 2->1 MsgVote Term:2 Log:1/11 2->3 MsgVote Term:2 Log:1/11 @@ -68,7 +68,7 @@ stabilize ---- > 1 handling Ready Ready MustSync=false: - Lead:0 State:StateFollower + State:StateFollower Messages: 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 @@ -128,7 +128,7 @@ stabilize INFO 1 [term: 2] ignored a MsgHeartbeatResp message with lower term from 3 [term: 1] > 1 handling Ready Ready MustSync=true: - HardState Term:2 Commit:11 + HardState Term:2 Commit:11 Lead:0 # Other nodes can now successfully campaign. Note that we haven't ticked 3, so # it won't grant votes. @@ -142,7 +142,7 @@ INFO 2 [logterm: 1, index: 11] sent MsgVote request to 3 at term 3 process-ready 2 ---- Ready MustSync=true: -HardState Term:3 Vote:2 Commit:11 +HardState Term:3 Vote:2 Commit:11 Lead:0 Messages: 2->1 MsgVote Term:3 Log:1/11 2->3 MsgVote Term:3 Log:1/11 @@ -165,7 +165,7 @@ stabilize ---- > 1 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:11 + HardState Term:3 Vote:2 Commit:11 Lead:0 Messages: 1->2 MsgVoteResp Term:3 Log:0/0 > 2 receiving messages @@ -175,7 +175,8 @@ stabilize INFO 2 became leader at term 3 > 2 handling Ready Ready MustSync=true: - Lead:2 State:StateLeader + State:StateLeader + HardState Term:3 Vote:2 Commit:11 Lead:2 Entries: 3/12 EntryNormal "" Messages: @@ -189,15 +190,14 @@ stabilize INFO 3 became follower at term 3 > 1 handling Ready Ready MustSync=true: - Lead:2 State:StateFollower + HardState Term:3 Vote:2 Commit:11 Lead:2 Entries: 3/12 EntryNormal "" Messages: 1->2 MsgAppResp Term:3 Log:0/12 > 3 handling Ready Ready MustSync=true: - Lead:2 State:StateFollower - HardState Term:3 Commit:11 + HardState Term:3 Commit:11 Lead:2 Entries: 3/12 EntryNormal "" Messages: @@ -207,7 +207,7 @@ stabilize 3->2 MsgAppResp Term:3 Log:0/12 > 2 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:12 + HardState Term:3 Vote:2 Commit:12 Lead:2 CommittedEntries: 3/12 EntryNormal "" Messages: @@ -219,14 +219,14 @@ stabilize 2->3 MsgApp Term:3 Log:3/12 Commit:12 > 1 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:12 + HardState Term:3 Vote:2 Commit:12 Lead:2 CommittedEntries: 3/12 EntryNormal "" Messages: 1->2 MsgAppResp Term:3 Log:0/12 > 3 handling Ready Ready MustSync=false: - HardState Term:3 Commit:12 + HardState Term:3 Commit:12 Lead:2 CommittedEntries: 3/12 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/confchange_disable_validation.txt b/pkg/raft/testdata/confchange_disable_validation.txt index 66c1113505f5..d25a17607e06 100644 --- a/pkg/raft/testdata/confchange_disable_validation.txt +++ b/pkg/raft/testdata/confchange_disable_validation.txt @@ -42,7 +42,7 @@ stabilize 1 1/5 EntryConfChangeV2 l2 l3 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 CommittedEntries: 1/4 EntryNormal "foo" > 1 handling Ready diff --git a/pkg/raft/testdata/confchange_drop_if_unapplied.txt b/pkg/raft/testdata/confchange_drop_if_unapplied.txt index 83f4f97f9125..40988d75200b 100644 --- a/pkg/raft/testdata/confchange_drop_if_unapplied.txt +++ b/pkg/raft/testdata/confchange_drop_if_unapplied.txt @@ -43,7 +43,7 @@ stabilize 1 ---- > 1 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:4 + HardState Term:1 Vote:1 Commit:4 Lead:1 Entries: 1/5 EntryNormal "" CommittedEntries: @@ -51,7 +51,7 @@ stabilize 1 INFO 1 switched to configuration voters=(1)&&(1) learners=(2 3) > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 CommittedEntries: 1/5 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/confchange_v1_add_single.txt b/pkg/raft/testdata/confchange_v1_add_single.txt index f13d30777dc0..5be9a6997086 100644 --- a/pkg/raft/testdata/confchange_v1_add_single.txt +++ b/pkg/raft/testdata/confchange_v1_add_single.txt @@ -15,8 +15,8 @@ INFO 1 became candidate at term 1 process-ready 1 ---- Ready MustSync=true: -Lead:0 State:StateCandidate -HardState Term:1 Vote:1 Commit:2 +State:StateCandidate +HardState Term:1 Vote:1 Commit:2 Lead:0 INFO 1 received MsgVoteResp from 1 at term 1 INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections INFO 1 became leader at term 1 @@ -41,13 +41,14 @@ stabilize ---- > 1 handling Ready Ready MustSync=true: - Lead:1 State:StateLeader + State:StateLeader + HardState Term:1 Vote:1 Commit:2 Lead:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChange v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 + HardState Term:1 Vote:1 Commit:4 Lead:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChange v2 @@ -63,8 +64,7 @@ stabilize DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 > 2 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower - HardState Term:1 Commit:0 + HardState Term:1 Commit:0 Lead:1 Messages: 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) > 1 receiving messages @@ -87,7 +87,7 @@ stabilize INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] > 2 handling Ready Ready MustSync=false: - HardState Term:1 Commit:4 + HardState Term:1 Commit:4 Lead:1 Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 2->1 MsgAppResp Term:1 Log:0/4 diff --git a/pkg/raft/testdata/confchange_v1_remove_leader.txt b/pkg/raft/testdata/confchange_v1_remove_leader.txt index cc91508a4578..9a60e0e51cb0 100644 --- a/pkg/raft/testdata/confchange_v1_remove_leader.txt +++ b/pkg/raft/testdata/confchange_v1_remove_leader.txt @@ -96,7 +96,7 @@ stabilize 1 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 CommittedEntries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" @@ -122,7 +122,7 @@ stabilize 2 1->2 MsgApp Term:1 Log:1/6 Commit:5 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 Entries: 1/6 EntryNormal "bar" CommittedEntries: @@ -153,7 +153,7 @@ stabilize 1->3 MsgApp Term:1 Log:1/6 Commit:5 > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 Entries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" @@ -176,7 +176,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/6 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 + HardState Term:1 Vote:1 Commit:6 Lead:1 CommittedEntries: 1/6 EntryNormal "bar" Messages: @@ -188,14 +188,14 @@ stabilize 1->3 MsgApp Term:1 Log:1/6 Commit:6 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 + HardState Term:1 Vote:1 Commit:6 Lead:1 CommittedEntries: 1/6 EntryNormal "bar" Messages: 2->1 MsgAppResp Term:1 Log:0/6 > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 + HardState Term:1 Vote:1 Commit:6 Lead:1 CommittedEntries: 1/6 EntryNormal "bar" Messages: diff --git a/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt b/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt index fe397650931a..aa92875967a1 100644 --- a/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt +++ b/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt @@ -95,7 +95,7 @@ stabilize 1 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 CommittedEntries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" @@ -108,7 +108,7 @@ stabilize 1 INFO 1 became follower at term 1 > 1 handling Ready Ready MustSync=false: - Lead:0 State:StateFollower + State:StateFollower raft-state ---- @@ -125,7 +125,7 @@ stabilize 2 1->2 MsgApp Term:1 Log:1/6 Commit:5 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 Entries: 1/6 EntryNormal "bar" CommittedEntries: @@ -156,7 +156,7 @@ stabilize 1->3 MsgApp Term:1 Log:1/6 Commit:5 > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 Entries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" @@ -181,7 +181,7 @@ stabilize # n1 can no longer propose. propose 1 baz ---- -INFO 1 no leader at term 1; dropping proposal +INFO 1 not forwarding to itself at term 1; dropping proposal raft proposal dropped # Nor can it campaign to become leader. diff --git a/pkg/raft/testdata/confchange_v2_add_double_auto.txt b/pkg/raft/testdata/confchange_v2_add_double_auto.txt index f290c980e404..b26e73974016 100644 --- a/pkg/raft/testdata/confchange_v2_add_double_auto.txt +++ b/pkg/raft/testdata/confchange_v2_add_double_auto.txt @@ -17,8 +17,8 @@ INFO 1 became candidate at term 1 process-ready 1 ---- Ready MustSync=true: -Lead:0 State:StateCandidate -HardState Term:1 Vote:1 Commit:2 +State:StateCandidate +HardState Term:1 Vote:1 Commit:2 Lead:0 INFO 1 received MsgVoteResp from 1 at term 1 INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections INFO 1 became leader at term 1 @@ -42,7 +42,8 @@ INFO newRaft 3 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastter process-ready 1 ---- Ready MustSync=true: -Lead:1 State:StateLeader +State:StateLeader +HardState Term:1 Vote:1 Commit:2 Lead:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 v3 @@ -54,7 +55,7 @@ Entries: process-ready 1 ---- Ready MustSync=false: -HardState Term:1 Vote:1 Commit:4 +HardState Term:1 Vote:1 Commit:4 Lead:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 v3 @@ -86,8 +87,7 @@ stabilize 1 2 DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 > 2 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower - HardState Term:1 Commit:0 + HardState Term:1 Commit:0 Lead:1 Messages: 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) > 1 receiving messages @@ -110,7 +110,7 @@ stabilize 1 2 INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] > 2 handling Ready Ready MustSync=false: - HardState Term:1 Commit:4 + HardState Term:1 Commit:4 Lead:1 Snapshot Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true Messages: 2->1 MsgAppResp Term:1 Log:0/4 @@ -133,7 +133,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -143,7 +143,7 @@ stabilize 1 2 1->2 MsgApp Term:1 Log:1/5 Commit:5 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Commit:5 + HardState Term:1 Commit:5 Lead:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -162,8 +162,7 @@ stabilize 1 3 DEBUG 3 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 > 3 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower - HardState Term:1 Commit:0 + HardState Term:1 Commit:0 Lead:1 Messages: 3->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) > 1 receiving messages @@ -186,7 +185,7 @@ stabilize 1 3 INFO 3 [commit: 5] restored snapshot [index: 5, term: 1] > 3 handling Ready Ready MustSync=false: - HardState Term:1 Commit:5 + HardState Term:1 Commit:5 Lead:1 Snapshot Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 3->1 MsgAppResp Term:1 Log:0/5 @@ -268,7 +267,7 @@ stabilize 1 3->1 MsgAppResp Term:1 Log:0/6 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 + HardState Term:1 Vote:1 Commit:6 Lead:1 CommittedEntries: 1/6 EntryConfChangeV2 r2 r3 Messages: @@ -299,7 +298,7 @@ stabilize 2 3 1->3 MsgApp Term:1 Log:1/8 Commit:6 Entries:[1/9 EntryConfChangeV2] > 2 handling Ready Ready MustSync=true: - HardState Term:1 Commit:6 + HardState Term:1 Commit:6 Lead:1 Entries: 1/7 EntryNormal "foo" 1/8 EntryNormal "bar" @@ -314,7 +313,7 @@ stabilize 2 3 INFO 2 switched to configuration voters=(1)&&(1 2 3) autoleave > 3 handling Ready Ready MustSync=true: - HardState Term:1 Commit:6 + HardState Term:1 Commit:6 Lead:1 Entries: 1/7 EntryNormal "foo" 1/8 EntryNormal "bar" @@ -344,7 +343,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/9 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:9 + HardState Term:1 Vote:1 Commit:9 Lead:1 CommittedEntries: 1/7 EntryNormal "foo" 1/8 EntryNormal "bar" @@ -367,7 +366,7 @@ stabilize 1->3 MsgApp Term:1 Log:1/9 Commit:9 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Commit:9 + HardState Term:1 Commit:9 Lead:1 CommittedEntries: 1/7 EntryNormal "foo" 1/8 EntryNormal "bar" @@ -379,7 +378,7 @@ stabilize INFO 2 switched to configuration voters=(1) > 3 handling Ready Ready MustSync=false: - HardState Term:1 Commit:9 + HardState Term:1 Commit:9 Lead:1 CommittedEntries: 1/7 EntryNormal "foo" 1/8 EntryNormal "bar" diff --git a/pkg/raft/testdata/confchange_v2_add_double_implicit.txt b/pkg/raft/testdata/confchange_v2_add_double_implicit.txt index 2db3b7c14c78..a2aac25d92e3 100644 --- a/pkg/raft/testdata/confchange_v2_add_double_implicit.txt +++ b/pkg/raft/testdata/confchange_v2_add_double_implicit.txt @@ -19,8 +19,8 @@ INFO 1 became candidate at term 1 process-ready 1 ---- Ready MustSync=true: -Lead:0 State:StateCandidate -HardState Term:1 Vote:1 Commit:2 +State:StateCandidate +HardState Term:1 Vote:1 Commit:2 Lead:0 INFO 1 received MsgVoteResp from 1 at term 1 INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections INFO 1 became leader at term 1 @@ -44,13 +44,14 @@ stabilize 1 2 ---- > 1 handling Ready Ready MustSync=true: - Lead:1 State:StateLeader + State:StateLeader + HardState Term:1 Vote:1 Commit:2 Lead:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 + HardState Term:1 Vote:1 Commit:4 Lead:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 @@ -69,8 +70,7 @@ stabilize 1 2 DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 > 2 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower - HardState Term:1 Commit:0 + HardState Term:1 Commit:0 Lead:1 Messages: 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) > 1 receiving messages @@ -93,7 +93,7 @@ stabilize 1 2 INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] > 2 handling Ready Ready MustSync=false: - HardState Term:1 Commit:4 + HardState Term:1 Commit:4 Lead:1 Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true Messages: 2->1 MsgAppResp Term:1 Log:0/4 @@ -116,7 +116,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -126,7 +126,7 @@ stabilize 1 2 1->2 MsgApp Term:1 Log:1/5 Commit:5 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Commit:5 + HardState Term:1 Commit:5 Lead:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: diff --git a/pkg/raft/testdata/confchange_v2_add_single_auto.txt b/pkg/raft/testdata/confchange_v2_add_single_auto.txt index 3c2341590fd5..df1a5e23e6d3 100644 --- a/pkg/raft/testdata/confchange_v2_add_single_auto.txt +++ b/pkg/raft/testdata/confchange_v2_add_single_auto.txt @@ -17,8 +17,8 @@ INFO 1 became candidate at term 1 process-ready 1 ---- Ready MustSync=true: -Lead:0 State:StateCandidate -HardState Term:1 Vote:1 Commit:2 +State:StateCandidate +HardState Term:1 Vote:1 Commit:2 Lead:0 INFO 1 received MsgVoteResp from 1 at term 1 INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections INFO 1 became leader at term 1 @@ -42,13 +42,14 @@ stabilize ---- > 1 handling Ready Ready MustSync=true: - Lead:1 State:StateLeader + State:StateLeader + HardState Term:1 Vote:1 Commit:2 Lead:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 + HardState Term:1 Vote:1 Commit:4 Lead:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 @@ -64,8 +65,7 @@ stabilize DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 > 2 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower - HardState Term:1 Commit:0 + HardState Term:1 Commit:0 Lead:1 Messages: 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) > 1 receiving messages @@ -88,7 +88,7 @@ stabilize INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] > 2 handling Ready Ready MustSync=false: - HardState Term:1 Commit:4 + HardState Term:1 Commit:4 Lead:1 Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 2->1 MsgAppResp Term:1 Log:0/4 diff --git a/pkg/raft/testdata/confchange_v2_add_single_explicit.txt b/pkg/raft/testdata/confchange_v2_add_single_explicit.txt index 1d390e55554c..e0857f027adb 100644 --- a/pkg/raft/testdata/confchange_v2_add_single_explicit.txt +++ b/pkg/raft/testdata/confchange_v2_add_single_explicit.txt @@ -17,8 +17,8 @@ INFO 1 became candidate at term 1 process-ready 1 ---- Ready MustSync=true: -Lead:0 State:StateCandidate -HardState Term:1 Vote:1 Commit:2 +State:StateCandidate +HardState Term:1 Vote:1 Commit:2 Lead:0 INFO 1 received MsgVoteResp from 1 at term 1 INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections INFO 1 became leader at term 1 @@ -42,13 +42,14 @@ stabilize 1 2 ---- > 1 handling Ready Ready MustSync=true: - Lead:1 State:StateLeader + State:StateLeader + HardState Term:1 Vote:1 Commit:2 Lead:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 + HardState Term:1 Vote:1 Commit:4 Lead:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 @@ -64,8 +65,7 @@ stabilize 1 2 DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 > 2 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower - HardState Term:1 Commit:0 + HardState Term:1 Commit:0 Lead:1 Messages: 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) > 1 receiving messages @@ -88,7 +88,7 @@ stabilize 1 2 INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] > 2 handling Ready Ready MustSync=false: - HardState Term:1 Commit:4 + HardState Term:1 Commit:4 Lead:1 Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:false Messages: 2->1 MsgAppResp Term:1 Log:0/4 @@ -136,7 +136,7 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/6 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 + HardState Term:1 Vote:1 Commit:6 Lead:1 CommittedEntries: 1/5 EntryNormal "" 1/6 EntryConfChangeV2 @@ -149,7 +149,7 @@ stabilize 1->2 MsgApp Term:1 Log:1/6 Commit:6 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Commit:6 + HardState Term:1 Commit:6 Lead:1 CommittedEntries: 1/5 EntryNormal "" 1/6 EntryConfChangeV2 @@ -187,7 +187,7 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/7 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:7 + HardState Term:1 Vote:1 Commit:7 Lead:1 CommittedEntries: 1/7 EntryNormal "" Messages: @@ -196,7 +196,7 @@ stabilize 1->2 MsgApp Term:1 Log:1/7 Commit:7 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Commit:7 + HardState Term:1 Commit:7 Lead:1 CommittedEntries: 1/7 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/confchange_v2_replace_leader.txt b/pkg/raft/testdata/confchange_v2_replace_leader.txt index 628cc447e112..44c16fc83668 100644 --- a/pkg/raft/testdata/confchange_v2_replace_leader.txt +++ b/pkg/raft/testdata/confchange_v2_replace_leader.txt @@ -77,7 +77,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/4 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 + HardState Term:1 Vote:1 Commit:4 Lead:1 CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: @@ -94,7 +94,7 @@ stabilize 1->4 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 r1 v4] > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 + HardState Term:1 Vote:1 Commit:4 Lead:1 CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: @@ -102,7 +102,7 @@ stabilize INFO 2 switched to configuration voters=(2 3 4)&&(1 2 3) > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 + HardState Term:1 Vote:1 Commit:4 Lead:1 CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: @@ -117,8 +117,7 @@ stabilize INFO 4 became follower at term 1 > 4 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower - HardState Term:1 Commit:0 + HardState Term:1 Commit:0 Lead:1 Messages: 4->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) > 1 receiving messages @@ -137,7 +136,7 @@ stabilize INFO 4 [commit: 4] restored snapshot [index: 4, term: 1] > 4 handling Ready Ready MustSync=false: - HardState Term:1 Commit:4 + HardState Term:1 Commit:4 Lead:1 Snapshot Index:4 Term:1 ConfState:Voters:[2 3 4] VotersOutgoing:[1 2 3] Learners:[] LearnersNext:[] AutoLeave:false Messages: 4->1 MsgAppResp Term:1 Log:0/4 @@ -176,8 +175,8 @@ stabilize INFO 4 [logterm: 1, index: 4] sent MsgVote request to 3 at term 2 > 4 handling Ready Ready MustSync=true: - Lead:0 State:StateCandidate - HardState Term:2 Vote:4 Commit:4 + State:StateCandidate + HardState Term:2 Vote:4 Commit:4 Lead:0 Messages: 4->1 MsgVote Term:2 Log:1/4 4->2 MsgVote Term:2 Log:1/4 @@ -201,20 +200,18 @@ stabilize INFO 3 [logterm: 1, index: 4, vote: 0] cast MsgVote for 4 [logterm: 1, index: 4] at term 2 > 1 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:2 Vote:4 Commit:4 + State:StateFollower + HardState Term:2 Vote:4 Commit:4 Lead:0 Messages: 1->4 MsgVoteResp Term:2 Log:0/0 > 2 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:2 Vote:4 Commit:4 + HardState Term:2 Vote:4 Commit:4 Lead:0 Messages: 2->4 MsgVoteResp Term:2 Log:0/0 > 3 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:2 Vote:4 Commit:4 + HardState Term:2 Vote:4 Commit:4 Lead:0 Messages: 3->4 MsgVoteResp Term:2 Log:0/0 > 4 receiving messages @@ -228,7 +225,8 @@ stabilize 3->4 MsgVoteResp Term:2 Log:0/0 > 4 handling Ready Ready MustSync=true: - Lead:4 State:StateLeader + State:StateLeader + HardState Term:2 Vote:4 Commit:4 Lead:4 Entries: 2/5 EntryNormal "" Messages: @@ -243,21 +241,21 @@ stabilize 4->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - Lead:4 State:StateFollower + HardState Term:2 Vote:4 Commit:4 Lead:4 Entries: 2/5 EntryNormal "" Messages: 1->4 MsgAppResp Term:2 Log:0/5 > 2 handling Ready Ready MustSync=true: - Lead:4 State:StateFollower + HardState Term:2 Vote:4 Commit:4 Lead:4 Entries: 2/5 EntryNormal "" Messages: 2->4 MsgAppResp Term:2 Log:0/5 > 3 handling Ready Ready MustSync=true: - Lead:4 State:StateFollower + HardState Term:2 Vote:4 Commit:4 Lead:4 Entries: 2/5 EntryNormal "" Messages: @@ -268,7 +266,7 @@ stabilize 3->4 MsgAppResp Term:2 Log:0/5 > 4 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 + HardState Term:2 Vote:4 Commit:5 Lead:4 CommittedEntries: 2/5 EntryNormal "" Messages: @@ -283,21 +281,21 @@ stabilize 4->3 MsgApp Term:2 Log:2/5 Commit:5 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 + HardState Term:2 Vote:4 Commit:5 Lead:4 CommittedEntries: 2/5 EntryNormal "" Messages: 1->4 MsgAppResp Term:2 Log:0/5 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 + HardState Term:2 Vote:4 Commit:5 Lead:4 CommittedEntries: 2/5 EntryNormal "" Messages: 2->4 MsgAppResp Term:2 Log:0/5 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 + HardState Term:2 Vote:4 Commit:5 Lead:4 CommittedEntries: 2/5 EntryNormal "" Messages: @@ -361,7 +359,7 @@ stabilize 3->4 MsgAppResp Term:2 Log:0/6 > 4 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 + HardState Term:2 Vote:4 Commit:6 Lead:4 CommittedEntries: 2/6 EntryConfChangeV2 Messages: @@ -377,7 +375,7 @@ stabilize 4->3 MsgApp Term:2 Log:2/6 Commit:6 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 + HardState Term:2 Vote:4 Commit:6 Lead:4 CommittedEntries: 2/6 EntryConfChangeV2 Messages: @@ -385,7 +383,7 @@ stabilize INFO 1 switched to configuration voters=(2 3 4) > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 + HardState Term:2 Vote:4 Commit:6 Lead:4 CommittedEntries: 2/6 EntryConfChangeV2 Messages: @@ -393,7 +391,7 @@ stabilize INFO 2 switched to configuration voters=(2 3 4) > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 + HardState Term:2 Vote:4 Commit:6 Lead:4 CommittedEntries: 2/6 EntryConfChangeV2 Messages: diff --git a/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt b/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt index 62d01d23b412..29dd420a8f1d 100644 --- a/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt +++ b/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt @@ -112,7 +112,7 @@ stabilize 4->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -129,10 +129,10 @@ stabilize 1->4 MsgApp Term:1 Log:1/5 Commit:5 > 1 handling Ready Ready MustSync=false: - Lead:0 State:StateFollower + State:StateFollower > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -140,7 +140,7 @@ stabilize INFO 2 switched to configuration voters=(2 3 4) > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 + HardState Term:1 Vote:1 Commit:5 Lead:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -148,7 +148,7 @@ stabilize INFO 3 switched to configuration voters=(2 3 4) > 4 handling Ready Ready MustSync=false: - HardState Term:1 Commit:5 + HardState Term:1 Commit:5 Lead:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: diff --git a/pkg/raft/testdata/forget_leader.txt b/pkg/raft/testdata/forget_leader.txt index a674b346c606..5b6ae89a5c63 100644 --- a/pkg/raft/testdata/forget_leader.txt +++ b/pkg/raft/testdata/forget_leader.txt @@ -80,11 +80,11 @@ stabilize 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 1->4 MsgHeartbeat Term:1 Log:0/0 Commit:11 > 2 handling Ready - Ready MustSync=false: - Lead:0 State:StateFollower + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:11 Lead:0 > 4 handling Ready - Ready MustSync=false: - Lead:0 State:StateFollower + Ready MustSync=true: + HardState Term:1 Commit:11 Lead:0 > 2 receiving messages 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 > 3 receiving messages @@ -92,8 +92,8 @@ stabilize > 4 receiving messages 1->4 MsgHeartbeat Term:1 Log:0/0 Commit:11 > 2 handling Ready - Ready MustSync=false: - Lead:1 State:StateFollower + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:11 Lead:1 Messages: 2->1 MsgHeartbeatResp Term:1 Log:0/0 > 3 handling Ready @@ -101,8 +101,8 @@ stabilize Messages: 3->1 MsgHeartbeatResp Term:1 Log:0/0 > 4 handling Ready - Ready MustSync=false: - Lead:1 State:StateFollower + Ready MustSync=true: + HardState Term:1 Commit:11 Lead:1 Messages: 4->1 MsgHeartbeatResp Term:1 Log:0/0 > 1 receiving messages diff --git a/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt b/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt index 9b3b80ffae15..f4a4a3c316fc 100644 --- a/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt +++ b/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt @@ -34,8 +34,9 @@ INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 2 at term 1 stabilize 3 ---- > 3 handling Ready - Ready MustSync=false: - Lead:0 State:StatePreCandidate + Ready MustSync=true: + State:StatePreCandidate + HardState Term:1 Vote:1 Commit:11 Lead:0 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -71,8 +72,9 @@ stabilize Messages: 2->1 MsgHeartbeatResp Term:1 Log:0/0 > 3 handling Ready - Ready MustSync=false: - Lead:1 State:StateFollower + Ready MustSync=true: + State:StateFollower + HardState Term:1 Vote:1 Commit:11 Lead:1 Messages: 3->1 MsgHeartbeatResp Term:1 Log:0/0 > 1 receiving messages @@ -107,8 +109,9 @@ INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 2 at term 1 stabilize 3 ---- > 3 handling Ready - Ready MustSync=false: - Lead:0 State:StatePreCandidate + Ready MustSync=true: + State:StatePreCandidate + HardState Term:1 Vote:1 Commit:11 Lead:0 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -118,8 +121,8 @@ stabilize 3 stabilize 2 ---- > 2 handling Ready - Ready MustSync=false: - Lead:0 State:StateFollower + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:11 Lead:0 > 2 receiving messages 3->2 MsgPreVote Term:2 Log:1/11 INFO 2 [logterm: 1, index: 11, vote: 1] cast MsgPreVote for 3 [logterm: 1, index: 11] at term 1 @@ -139,8 +142,8 @@ stabilize 3 INFO 3 [logterm: 1, index: 11] sent MsgVote request to 2 at term 2 > 3 handling Ready Ready MustSync=true: - Lead:0 State:StateCandidate - HardState Term:2 Vote:3 Commit:11 + State:StateCandidate + HardState Term:2 Vote:3 Commit:11 Lead:0 Messages: 3->1 MsgVote Term:2 Log:1/11 3->2 MsgVote Term:2 Log:1/11 @@ -203,8 +206,9 @@ INFO 1 [logterm: 2, index: 12] sent MsgPreVote request to 3 at term 2 process-ready 1 ---- -Ready MustSync=false: -Lead:0 State:StatePreCandidate +Ready MustSync=true: +State:StatePreCandidate +HardState Term:2 Commit:12 Lead:0 Messages: 1->2 MsgPreVote Term:3 Log:2/12 1->3 MsgPreVote Term:3 Log:2/12 @@ -214,8 +218,8 @@ INFO 1 has received 1 MsgPreVoteResp votes and 0 vote rejections stabilize 2 ---- > 2 handling Ready - Ready MustSync=false: - Lead:0 State:StateFollower + Ready MustSync=true: + HardState Term:2 Vote:3 Commit:12 Lead:0 > 2 receiving messages 1->2 MsgPreVote Term:3 Log:2/12 INFO 2 [logterm: 2, index: 13, vote: 3] rejected MsgPreVote from 1 [logterm: 2, index: 12] at term 2 diff --git a/pkg/raft/testdata/lagging_commit.txt b/pkg/raft/testdata/lagging_commit.txt index 8f8ba33635b2..82ccd3a25460 100644 --- a/pkg/raft/testdata/lagging_commit.txt +++ b/pkg/raft/testdata/lagging_commit.txt @@ -78,7 +78,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/13 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:13 + HardState Term:1 Vote:1 Commit:13 Lead:1 CommittedEntries: 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" @@ -92,7 +92,7 @@ stabilize 1 2 1->2 MsgApp Term:1 Log:1/13 Commit:13 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:13 + HardState Term:1 Vote:1 Commit:13 Lead:1 CommittedEntries: 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" @@ -164,7 +164,7 @@ stabilize 1 3 1->3 MsgApp Term:1 Log:1/13 Commit:13 > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:13 + HardState Term:1 Vote:1 Commit:13 Lead:1 CommittedEntries: 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" diff --git a/pkg/raft/testdata/prevote.txt b/pkg/raft/testdata/prevote.txt index db763d3599c6..0c6ff0e2a56e 100644 --- a/pkg/raft/testdata/prevote.txt +++ b/pkg/raft/testdata/prevote.txt @@ -65,8 +65,9 @@ INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 2 at term 1 process-ready 3 ---- -Ready MustSync=false: -Lead:0 State:StatePreCandidate +Ready MustSync=true: +State:StatePreCandidate +HardState Term:1 Vote:1 Commit:11 Lead:0 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -86,7 +87,7 @@ stabilize ---- > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:12 + HardState Term:1 Vote:1 Commit:12 Lead:1 CommittedEntries: 1/12 EntryNormal "prop_1" Messages: @@ -107,15 +108,15 @@ stabilize 2->3 MsgPreVoteResp Term:1 Log:0/0 Rejected (Hint: 0) > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:12 + HardState Term:1 Vote:1 Commit:12 Lead:1 CommittedEntries: 1/12 EntryNormal "prop_1" Messages: 2->1 MsgAppResp Term:1 Log:0/12 > 3 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower - HardState Term:1 Vote:1 Commit:12 + State:StateFollower + HardState Term:1 Vote:1 Commit:12 Lead:1 Entries: 1/12 EntryNormal "prop_1" CommittedEntries: @@ -139,8 +140,9 @@ INFO 2 [logterm: 1, index: 12] sent MsgPreVote request to 3 at term 1 stabilize ---- > 2 handling Ready - Ready MustSync=false: - Lead:0 State:StatePreCandidate + Ready MustSync=true: + State:StatePreCandidate + HardState Term:1 Vote:1 Commit:12 Lead:0 Messages: 2->1 MsgPreVote Term:2 Log:1/12 2->3 MsgPreVote Term:2 Log:1/12 @@ -170,8 +172,8 @@ stabilize 3->2 MsgPreVoteResp Term:2 Log:0/0 > 2 handling Ready Ready MustSync=true: - Lead:0 State:StateCandidate - HardState Term:2 Vote:2 Commit:12 + State:StateCandidate + HardState Term:2 Vote:2 Commit:12 Lead:0 Messages: 2->1 MsgVote Term:2 Log:1/12 2->3 MsgVote Term:2 Log:1/12 @@ -189,14 +191,13 @@ stabilize INFO 3 [logterm: 1, index: 12, vote: 0] cast MsgVote for 2 [logterm: 1, index: 12] at term 2 > 1 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:2 Vote:2 Commit:12 + State:StateFollower + HardState Term:2 Vote:2 Commit:12 Lead:0 Messages: 1->2 MsgVoteResp Term:2 Log:0/0 > 3 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:2 Vote:2 Commit:12 + HardState Term:2 Vote:2 Commit:12 Lead:0 Messages: 3->2 MsgVoteResp Term:2 Log:0/0 > 2 receiving messages @@ -207,7 +208,8 @@ stabilize 3->2 MsgVoteResp Term:2 Log:0/0 > 2 handling Ready Ready MustSync=true: - Lead:2 State:StateLeader + State:StateLeader + HardState Term:2 Vote:2 Commit:12 Lead:2 Entries: 2/13 EntryNormal "" Messages: @@ -219,14 +221,14 @@ stabilize 2->3 MsgApp Term:2 Log:1/12 Commit:12 Entries:[2/13 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - Lead:2 State:StateFollower + HardState Term:2 Vote:2 Commit:12 Lead:2 Entries: 2/13 EntryNormal "" Messages: 1->2 MsgAppResp Term:2 Log:0/13 > 3 handling Ready Ready MustSync=true: - Lead:2 State:StateFollower + HardState Term:2 Vote:2 Commit:12 Lead:2 Entries: 2/13 EntryNormal "" Messages: @@ -236,7 +238,7 @@ stabilize 3->2 MsgAppResp Term:2 Log:0/13 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:13 + HardState Term:2 Vote:2 Commit:13 Lead:2 CommittedEntries: 2/13 EntryNormal "" Messages: @@ -248,14 +250,14 @@ stabilize 2->3 MsgApp Term:2 Log:2/13 Commit:13 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:13 + HardState Term:2 Vote:2 Commit:13 Lead:2 CommittedEntries: 2/13 EntryNormal "" Messages: 1->2 MsgAppResp Term:2 Log:0/13 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:13 + HardState Term:2 Vote:2 Commit:13 Lead:2 CommittedEntries: 2/13 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/prevote_checkquorum.txt b/pkg/raft/testdata/prevote_checkquorum.txt index 6db6662b0349..7a5a7c86a339 100644 --- a/pkg/raft/testdata/prevote_checkquorum.txt +++ b/pkg/raft/testdata/prevote_checkquorum.txt @@ -34,8 +34,9 @@ INFO 2 [logterm: 1, index: 11] sent MsgPreVote request to 3 at term 1 stabilize ---- > 2 handling Ready - Ready MustSync=false: - Lead:0 State:StatePreCandidate + Ready MustSync=true: + State:StatePreCandidate + HardState Term:1 Vote:1 Commit:11 Lead:0 Messages: 2->1 MsgPreVote Term:2 Log:1/11 2->3 MsgPreVote Term:2 Log:1/11 @@ -67,8 +68,9 @@ INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 2 at term 1 process-ready 3 ---- -Ready MustSync=false: -Lead:0 State:StatePreCandidate +Ready MustSync=true: +State:StatePreCandidate +HardState Term:1 Vote:1 Commit:11 Lead:0 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -100,8 +102,8 @@ stabilize INFO 3 [logterm: 1, index: 11] sent MsgVote request to 2 at term 2 > 3 handling Ready Ready MustSync=true: - Lead:0 State:StateCandidate - HardState Term:2 Vote:3 Commit:11 + State:StateCandidate + HardState Term:2 Vote:3 Commit:11 Lead:0 Messages: 3->1 MsgVote Term:2 Log:1/11 3->2 MsgVote Term:2 Log:1/11 @@ -117,8 +119,8 @@ stabilize INFO 2 [logterm: 1, index: 11, vote: 0] cast MsgVote for 3 [logterm: 1, index: 11] at term 2 > 2 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:2 Vote:3 Commit:11 + State:StateFollower + HardState Term:2 Vote:3 Commit:11 Lead:0 Messages: 2->3 MsgVoteResp Term:2 Log:0/0 > 3 receiving messages @@ -128,7 +130,8 @@ stabilize INFO 3 became leader at term 2 > 3 handling Ready Ready MustSync=true: - Lead:3 State:StateLeader + State:StateLeader + HardState Term:2 Vote:3 Commit:11 Lead:3 Entries: 2/12 EntryNormal "" Messages: @@ -142,15 +145,15 @@ stabilize 3->2 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - Lead:3 State:StateFollower - HardState Term:2 Commit:11 + State:StateFollower + HardState Term:2 Commit:11 Lead:3 Entries: 2/12 EntryNormal "" Messages: 1->3 MsgAppResp Term:2 Log:0/12 > 2 handling Ready Ready MustSync=true: - Lead:3 State:StateFollower + HardState Term:2 Vote:3 Commit:11 Lead:3 Entries: 2/12 EntryNormal "" Messages: @@ -160,7 +163,7 @@ stabilize 2->3 MsgAppResp Term:2 Log:0/12 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:3 Commit:12 + HardState Term:2 Vote:3 Commit:12 Lead:3 CommittedEntries: 2/12 EntryNormal "" Messages: @@ -172,14 +175,14 @@ stabilize 3->2 MsgApp Term:2 Log:2/12 Commit:12 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Commit:12 + HardState Term:2 Commit:12 Lead:3 CommittedEntries: 2/12 EntryNormal "" Messages: 1->3 MsgAppResp Term:2 Log:0/12 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:3 Commit:12 + HardState Term:2 Vote:3 Commit:12 Lead:3 CommittedEntries: 2/12 EntryNormal "" Messages: @@ -206,8 +209,9 @@ INFO 1 [logterm: 2, index: 12] sent MsgPreVote request to 3 at term 2 stabilize ---- > 1 handling Ready - Ready MustSync=false: - Lead:0 State:StatePreCandidate + Ready MustSync=true: + State:StatePreCandidate + HardState Term:2 Commit:12 Lead:0 Messages: 1->2 MsgPreVote Term:3 Log:2/12 1->3 MsgPreVote Term:3 Log:2/12 @@ -230,8 +234,9 @@ INFO 2 [logterm: 2, index: 12] sent MsgPreVote request to 3 at term 2 stabilize ---- > 2 handling Ready - Ready MustSync=false: - Lead:0 State:StatePreCandidate + Ready MustSync=true: + State:StatePreCandidate + HardState Term:2 Vote:3 Commit:12 Lead:0 Messages: 2->1 MsgPreVote Term:3 Log:2/12 2->3 MsgPreVote Term:3 Log:2/12 @@ -256,8 +261,8 @@ stabilize INFO 2 [logterm: 2, index: 12] sent MsgVote request to 3 at term 3 > 2 handling Ready Ready MustSync=true: - Lead:0 State:StateCandidate - HardState Term:3 Vote:2 Commit:12 + State:StateCandidate + HardState Term:3 Vote:2 Commit:12 Lead:0 Messages: 2->1 MsgVote Term:3 Log:2/12 2->3 MsgVote Term:3 Log:2/12 @@ -273,8 +278,8 @@ stabilize INFO 3 [logterm: 2, index: 12, vote: 3] ignored MsgVote from 2 [logterm: 2, index: 12] at term 2: lease is not expired (remaining ticks: 3) > 1 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:3 Vote:2 Commit:12 + State:StateFollower + HardState Term:3 Vote:2 Commit:12 Lead:0 Messages: 1->2 MsgVoteResp Term:3 Log:0/0 > 2 receiving messages @@ -284,7 +289,8 @@ stabilize INFO 2 became leader at term 3 > 2 handling Ready Ready MustSync=true: - Lead:2 State:StateLeader + State:StateLeader + HardState Term:3 Vote:2 Commit:12 Lead:2 Entries: 3/13 EntryNormal "" Messages: @@ -298,15 +304,15 @@ stabilize INFO 3 became follower at term 3 > 1 handling Ready Ready MustSync=true: - Lead:2 State:StateFollower + HardState Term:3 Vote:2 Commit:12 Lead:2 Entries: 3/13 EntryNormal "" Messages: 1->2 MsgAppResp Term:3 Log:0/13 > 3 handling Ready Ready MustSync=true: - Lead:2 State:StateFollower - HardState Term:3 Commit:12 + State:StateFollower + HardState Term:3 Commit:12 Lead:2 Entries: 3/13 EntryNormal "" Messages: @@ -316,7 +322,7 @@ stabilize 3->2 MsgAppResp Term:3 Log:0/13 > 2 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:13 + HardState Term:3 Vote:2 Commit:13 Lead:2 CommittedEntries: 3/13 EntryNormal "" Messages: @@ -328,14 +334,14 @@ stabilize 2->3 MsgApp Term:3 Log:3/13 Commit:13 > 1 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:13 + HardState Term:3 Vote:2 Commit:13 Lead:2 CommittedEntries: 3/13 EntryNormal "" Messages: 1->2 MsgAppResp Term:3 Log:0/13 > 3 handling Ready Ready MustSync=false: - HardState Term:3 Commit:13 + HardState Term:3 Commit:13 Lead:2 CommittedEntries: 3/13 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/probe_and_replicate.txt b/pkg/raft/testdata/probe_and_replicate.txt index d970183c677a..a7c3b2a8265e 100644 --- a/pkg/raft/testdata/probe_and_replicate.txt +++ b/pkg/raft/testdata/probe_and_replicate.txt @@ -369,8 +369,8 @@ stabilize 1 ---- > 1 handling Ready Ready MustSync=true: - Lead:0 State:StateCandidate - HardState Term:8 Vote:1 Commit:18 + State:StateCandidate + HardState Term:8 Vote:1 Commit:18 Lead:0 Messages: 1->2 MsgVote Term:8 Log:6/20 1->3 MsgVote Term:8 Log:6/20 @@ -415,35 +415,34 @@ stabilize 2 3 4 5 6 7 INFO 7 [logterm: 3, index: 21, vote: 0] cast MsgVote for 1 [logterm: 6, index: 20] at term 8 > 2 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:8 Vote:1 Commit:18 + HardState Term:8 Vote:1 Commit:18 Lead:0 Messages: 2->1 MsgVoteResp Term:8 Log:0/0 > 3 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:14 + HardState Term:8 Vote:1 Commit:14 Lead:0 Messages: 3->1 MsgVoteResp Term:8 Log:0/0 > 4 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:8 Commit:18 + State:StateFollower + HardState Term:8 Commit:18 Lead:0 Messages: 4->1 MsgVoteResp Term:8 Log:0/0 Rejected (Hint: 0) > 5 handling Ready Ready MustSync=true: - Lead:0 State:StateFollower - HardState Term:8 Commit:18 + State:StateFollower + HardState Term:8 Commit:18 Lead:0 Messages: 5->1 MsgVoteResp Term:8 Log:0/0 Rejected (Hint: 0) > 6 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:15 + HardState Term:8 Vote:1 Commit:15 Lead:0 Messages: 6->1 MsgVoteResp Term:8 Log:0/0 > 7 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:13 + HardState Term:8 Vote:1 Commit:13 Lead:0 Messages: 7->1 MsgVoteResp Term:8 Log:0/0 @@ -469,7 +468,8 @@ stabilize 1 7->1 MsgVoteResp Term:8 Log:0/0 > 1 handling Ready Ready MustSync=true: - Lead:1 State:StateLeader + State:StateLeader + HardState Term:8 Vote:1 Commit:18 Lead:1 Entries: 8/21 EntryNormal "" Messages: @@ -486,8 +486,8 @@ stabilize 1 2 > 2 receiving messages 1->2 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 2 handling Ready - Ready MustSync=false: - Lead:1 State:StateFollower + Ready MustSync=true: + HardState Term:8 Vote:1 Commit:18 Lead:1 Messages: 2->1 MsgAppResp Term:8 Log:6/20 Rejected (Hint: 19) > 1 receiving messages @@ -519,8 +519,8 @@ stabilize 1 3 > 3 receiving messages 1->3 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 3 handling Ready - Ready MustSync=false: - Lead:1 State:StateFollower + Ready MustSync=true: + HardState Term:8 Vote:1 Commit:14 Lead:1 Messages: 3->1 MsgAppResp Term:8 Log:4/20 Rejected (Hint: 14) > 1 receiving messages @@ -549,7 +549,7 @@ stabilize 1 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:18 + HardState Term:8 Vote:1 Commit:18 Lead:1 Entries: 4/15 EntryNormal "prop_4_15" 5/16 EntryNormal "" @@ -576,7 +576,7 @@ stabilize 1 4 INFO replace the unstable entries from index 21 > 4 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower + HardState Term:8 Commit:18 Lead:1 Entries: 8/21 EntryNormal "" Messages: @@ -585,7 +585,7 @@ stabilize 1 4 4->1 MsgAppResp Term:8 Log:0/21 > 1 handling Ready Ready MustSync=false: - HardState Term:8 Vote:1 Commit:21 + HardState Term:8 Vote:1 Commit:21 Lead:1 CommittedEntries: 6/19 EntryNormal "prop_6_19" 6/20 EntryNormal "prop_6_20" @@ -598,7 +598,7 @@ stabilize 1 4 1->4 MsgApp Term:8 Log:8/21 Commit:21 > 4 handling Ready Ready MustSync=false: - HardState Term:8 Commit:21 + HardState Term:8 Commit:21 Lead:1 CommittedEntries: 6/19 EntryNormal "prop_6_19" 6/20 EntryNormal "prop_6_20" @@ -613,8 +613,8 @@ stabilize 1 5 > 5 receiving messages 1->5 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 5 handling Ready - Ready MustSync=false: - Lead:1 State:StateFollower + Ready MustSync=true: + HardState Term:8 Commit:18 Lead:1 Messages: 5->1 MsgAppResp Term:8 Log:6/20 Rejected (Hint: 18) > 1 receiving messages @@ -637,7 +637,7 @@ stabilize 1 5 INFO replace the unstable entries from index 19 > 5 handling Ready Ready MustSync=true: - HardState Term:8 Commit:21 + HardState Term:8 Commit:21 Lead:1 Entries: 6/19 EntryNormal "prop_6_19" 6/20 EntryNormal "prop_6_20" @@ -656,8 +656,8 @@ stabilize 1 6 > 6 receiving messages 1->6 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 6 handling Ready - Ready MustSync=false: - Lead:1 State:StateFollower + Ready MustSync=true: + HardState Term:8 Vote:1 Commit:15 Lead:1 Messages: 6->1 MsgAppResp Term:8 Log:4/20 Rejected (Hint: 17) > 1 receiving messages @@ -686,7 +686,7 @@ stabilize 1 6 INFO replace the unstable entries from index 16 > 6 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:21 + HardState Term:8 Vote:1 Commit:21 Lead:1 Entries: 5/16 EntryNormal "" 5/17 EntryNormal "prop_5_17" @@ -711,8 +711,8 @@ stabilize 1 7 > 7 receiving messages 1->7 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 7 handling Ready - Ready MustSync=false: - Lead:1 State:StateFollower + Ready MustSync=true: + HardState Term:8 Vote:1 Commit:13 Lead:1 Messages: 7->1 MsgAppResp Term:8 Log:3/20 Rejected (Hint: 20) > 1 receiving messages @@ -745,7 +745,7 @@ stabilize 1 7 INFO replace the unstable entries from index 14 > 7 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:21 + HardState Term:8 Vote:1 Commit:21 Lead:1 Entries: 4/14 EntryNormal "" 4/15 EntryNormal "prop_4_15" diff --git a/pkg/raft/testdata/single_node.txt b/pkg/raft/testdata/single_node.txt index f6aceb712ff4..8e87dc18127b 100644 --- a/pkg/raft/testdata/single_node.txt +++ b/pkg/raft/testdata/single_node.txt @@ -17,18 +17,19 @@ stabilize ---- > 1 handling Ready Ready MustSync=true: - Lead:0 State:StateCandidate - HardState Term:1 Vote:1 Commit:3 + State:StateCandidate + HardState Term:1 Vote:1 Commit:3 Lead:0 INFO 1 received MsgVoteResp from 1 at term 1 INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections INFO 1 became leader at term 1 > 1 handling Ready Ready MustSync=true: - Lead:1 State:StateLeader + State:StateLeader + HardState Term:1 Vote:1 Commit:3 Lead:1 Entries: 1/4 EntryNormal "" > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 + HardState Term:1 Vote:1 Commit:4 Lead:1 CommittedEntries: 1/4 EntryNormal "" diff --git a/pkg/raft/testdata/snapshot_succeed_via_app_resp.txt b/pkg/raft/testdata/snapshot_succeed_via_app_resp.txt index b1fc4144e747..28a969b9226c 100644 --- a/pkg/raft/testdata/snapshot_succeed_via_app_resp.txt +++ b/pkg/raft/testdata/snapshot_succeed_via_app_resp.txt @@ -76,8 +76,7 @@ stabilize 3 INFO 3 became follower at term 1 > 3 handling Ready Ready MustSync=true: - Lead:1 State:StateFollower - HardState Term:1 Commit:0 + HardState Term:1 Commit:0 Lead:1 Messages: 3->1 MsgHeartbeatResp Term:1 Log:0/0 @@ -115,7 +114,7 @@ stabilize 3 INFO 3 [commit: 11] restored snapshot [index: 11, term: 1] > 3 handling Ready Ready MustSync=false: - HardState Term:1 Commit:11 + HardState Term:1 Commit:11 Lead:1 Snapshot Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 3->1 MsgAppResp Term:1 Log:0/11 diff --git a/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt b/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt index 3bab832d6009..38ca5791fd68 100644 --- a/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt +++ b/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt @@ -99,8 +99,8 @@ DEBUG 3 [logterm: 0, index: 10] rejected MsgApp [logterm: 1, index: 10] from 1 # PendingSnapshot=lastIndex=12. process-ready 3 ---- -Ready MustSync=false: -Lead:1 State:StateFollower +Ready MustSync=true: +HardState Term:1 Vote:1 Commit:5 Lead:1 Messages: 3->1 MsgAppResp Term:1 Log:1/10 Rejected (Hint: 5) @@ -143,7 +143,7 @@ stabilize 3 ---- > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 + HardState Term:1 Vote:1 Commit:11 Lead:1 Snapshot Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 3->1 MsgAppResp Term:1 Log:0/11 diff --git a/pkg/raft/util.go b/pkg/raft/util.go index c5929cf9067a..488eff012c03 100644 --- a/pkg/raft/util.go +++ b/pkg/raft/util.go @@ -87,11 +87,12 @@ func DescribeHardState(hs pb.HardState) string { fmt.Fprintf(&buf, " Vote:%d", hs.Vote) } fmt.Fprintf(&buf, " Commit:%d", hs.Commit) + fmt.Fprintf(&buf, " Lead:%d", hs.Lead) return buf.String() } func DescribeSoftState(ss SoftState) string { - return fmt.Sprintf("Lead:%d State:%s", ss.Lead, ss.RaftState) + return fmt.Sprintf("State:%s", ss.RaftState) } func DescribeConfState(state pb.ConfState) string {