diff --git a/raft/interaction_test.go b/raft/interaction_test.go index 375e636dc001..c8b8cb00963f 100644 --- a/raft/interaction_test.go +++ b/raft/interaction_test.go @@ -22,6 +22,9 @@ import ( ) func TestInteraction(t *testing.T) { + // NB: if this test fails, run `go test ./raft -rewrite` and inspect the + // diff. Only commit the changes if you understand what caused them and if + // they are desired. datadriven.Walk(t, "testdata", func(t *testing.T, path string) { env := rafttest.NewInteractionEnv(nil) datadriven.RunTest(t, path, func(d *datadriven.TestData) string { diff --git a/raft/raft.go b/raft/raft.go index 02f493a8d89a..54a8192cebe6 100644 --- a/raft/raft.go +++ b/raft/raft.go @@ -1036,10 +1036,36 @@ func stepLeader(r *raft, m pb.Message) error { for i := range m.Entries { e := &m.Entries[i] - if e.Type == pb.EntryConfChange || e.Type == pb.EntryConfChangeV2 { - if r.pendingConfIndex > r.raftLog.applied { - r.logger.Infof("%x propose conf %s ignored since pending unapplied configuration [index %d, applied %d]", - r.id, e, r.pendingConfIndex, r.raftLog.applied) + var cc pb.ConfChangeI + if e.Type == pb.EntryConfChange { + var ccc pb.ConfChange + if err := ccc.Unmarshal(e.Data); err != nil { + panic(err) + } + cc = ccc + } else if e.Type == pb.EntryConfChangeV2 { + var ccc pb.ConfChangeV2 + if err := ccc.Unmarshal(e.Data); err != nil { + panic(err) + } + cc = ccc + } + if cc != nil { + alreadyPending := r.pendingConfIndex > r.raftLog.applied + alreadyJoint := len(r.prs.Config.Voters[1]) > 0 + wantsLeaveJoint := len(cc.AsV2().Changes) == 0 + + var refused string + if alreadyPending { + refused = fmt.Sprintf("possible unapplied conf change at index %d (applied to %d)", r.pendingConfIndex, r.raftLog.applied) + } else if alreadyJoint && !wantsLeaveJoint { + refused = "must transition out of joint config first" + } else if !alreadyJoint && wantsLeaveJoint { + refused = "not in joint state; refusing empty conf change" + } + + if refused != "" { + r.logger.Infof("%x ignoring conf change %v at config %s: %s", r.id, cc, r.prs.Config, refused) m.Entries[i] = pb.Entry{Type: pb.EntryNormal} } else { r.pendingConfIndex = r.raftLog.lastIndex() + uint64(i) + 1 diff --git a/raft/rafttest/interaction_env_handler_process_ready.go b/raft/rafttest/interaction_env_handler_process_ready.go index ff5cf2bd9da6..1006433c493a 100644 --- a/raft/rafttest/interaction_env_handler_process_ready.go +++ b/raft/rafttest/interaction_env_handler_process_ready.go @@ -19,7 +19,6 @@ import ( "github.com/cockroachdb/datadriven" "go.etcd.io/etcd/raft" - "go.etcd.io/etcd/raft/quorum" "go.etcd.io/etcd/raft/raftpb" ) @@ -50,6 +49,7 @@ func (env *InteractionEnv) ProcessReady(idx int) error { } for _, ent := range rd.CommittedEntries { var update []byte + var cs *raftpb.ConfState switch ent.Type { case raftpb.EntryConfChange: var cc raftpb.ConfChange @@ -57,13 +57,13 @@ func (env *InteractionEnv) ProcessReady(idx int) error { return err } update = cc.Context - rn.ApplyConfChange(cc) + cs = rn.ApplyConfChange(cc) case raftpb.EntryConfChangeV2: var cc raftpb.ConfChangeV2 if err := cc.Unmarshal(ent.Data); err != nil { return err } - rn.ApplyConfChange(cc) + cs = rn.ApplyConfChange(cc) update = cc.Context default: update = ent.Data @@ -78,13 +78,11 @@ func (env *InteractionEnv) ProcessReady(idx int) error { snap.Data = append(snap.Data, update...) snap.Metadata.Index = ent.Index snap.Metadata.Term = ent.Term - cfg := rn.Status().Config - snap.Metadata.ConfState = raftpb.ConfState{ - Voters: cfg.Voters[0].Slice(), - VotersOutgoing: cfg.Voters[1].Slice(), - Learners: quorum.MajorityConfig(cfg.Learners).Slice(), - LearnersNext: quorum.MajorityConfig(cfg.LearnersNext).Slice(), + if cs == nil { + sl := env.Nodes[idx].History + cs = &sl[len(sl)-1].Metadata.ConfState } + snap.Metadata.ConfState = *cs env.Nodes[idx].History = append(env.Nodes[idx].History, snap) } for _, msg := range rd.Messages { diff --git a/raft/rafttest/interaction_env_handler_stabilize.go b/raft/rafttest/interaction_env_handler_stabilize.go index ce6e3b6e99c9..4f5fb64a4ee8 100644 --- a/raft/rafttest/interaction_env_handler_stabilize.go +++ b/raft/rafttest/interaction_env_handler_stabilize.go @@ -67,9 +67,10 @@ func (env *InteractionEnv) Stabilize(idxs ...int) error { } var msgs []raftpb.Message for _, rn := range nodes { - msgs, env.Messages = splitMsgs(env.Messages, rn.Status().ID) + id := rn.Status().ID + msgs, env.Messages = splitMsgs(env.Messages, id) if len(msgs) > 0 { - fmt.Fprintf(env.Output, "> delivering messages\n") + fmt.Fprintf(env.Output, "> %d receiving messages\n", id) withIndent(func() { env.DeliverMsgs(msgs) }) done = false } diff --git a/raft/testdata/campaign.txt b/raft/testdata/campaign.txt index 19bb06b9e45a..c5deb2dc1825 100644 --- a/raft/testdata/campaign.txt +++ b/raft/testdata/campaign.txt @@ -31,12 +31,12 @@ stabilize Messages: 1->2 MsgVote Term:1 Log:1/2 1->3 MsgVote Term:1 Log:1/2 -> delivering messages +> 2 receiving messages 1->2 MsgVote Term:1 Log:1/2 INFO 2 [term: 0] received a MsgVote message with higher term from 1 [term: 1] INFO 2 became follower at term 1 INFO 2 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 -> delivering messages +> 3 receiving messages 1->3 MsgVote Term:1 Log:1/2 INFO 3 [term: 0] received a MsgVote message with higher term from 1 [term: 1] INFO 3 became follower at term 1 @@ -51,7 +51,7 @@ stabilize HardState Term:1 Vote:1 Commit:2 Messages: 3->1 MsgVoteResp Term:1 Log:0/0 -> delivering messages +> 1 receiving messages 2->1 MsgVoteResp Term:1 Log:0/0 INFO 1 received MsgVoteResp from 2 at term 1 INFO 1 has received 2 MsgVoteResp votes and 0 vote rejections @@ -65,9 +65,9 @@ stabilize Messages: 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] -> delivering messages +> 2 receiving messages 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] -> delivering messages +> 3 receiving messages 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] > 2 handling Ready Ready MustSync=true: @@ -83,7 +83,7 @@ stabilize 1/3 EntryNormal "" Messages: 3->1 MsgAppResp Term:1 Log:0/3 -> delivering messages +> 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 3->1 MsgAppResp Term:1 Log:0/3 > 1 handling Ready @@ -94,9 +94,9 @@ stabilize Messages: 1->2 MsgApp Term:1 Log:1/3 Commit:3 1->3 MsgApp Term:1 Log:1/3 Commit:3 -> delivering messages +> 2 receiving messages 1->2 MsgApp Term:1 Log:1/3 Commit:3 -> delivering messages +> 3 receiving messages 1->3 MsgApp Term:1 Log:1/3 Commit:3 > 2 handling Ready Ready MustSync=false: @@ -112,6 +112,6 @@ stabilize 1/3 EntryNormal "" Messages: 3->1 MsgAppResp Term:1 Log:0/3 -> delivering messages +> 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 3->1 MsgAppResp Term:1 Log:0/3 diff --git a/raft/testdata/campaign_learner_must_vote.txt b/raft/testdata/campaign_learner_must_vote.txt new file mode 100644 index 000000000000..50aae3817991 --- /dev/null +++ b/raft/testdata/campaign_learner_must_vote.txt @@ -0,0 +1,123 @@ +# Regression test that verifies that learners can vote. This holds only in the +# sense that if a learner is asked to vote, a candidate believes that they are a +# voter based on its current config, which may be more recent than that of the +# learner. If learners which are actually voters but don't know it yet don't +# vote in that situation, the raft group may end up unavailable despite a quorum +# of voters (as of the latest config) being available. +# +# See: +# https://github.com/etcd-io/etcd/pull/10998 + +# Turn output off during boilerplate. +log-level none +---- +ok + +add-nodes 3 voters=(1,2) learners=(3) index=2 +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok (quiet) + +propose-conf-change 1 +v3 +---- +ok + +stabilize 1 2 +---- +ok (quiet) + +log-level debug +---- +ok + +campaign 2 +---- +INFO 2 is starting a new election at term 1 +INFO 2 became candidate at term 2 +INFO 2 received MsgVoteResp from 2 at term 2 +INFO 2 [logterm: 1, index: 4] sent MsgVote request to 1 at term 2 +INFO 2 [logterm: 1, index: 4] sent MsgVote request to 3 at term 2 + +# n2 is now campaigning while n1 is down (does not respond). The latest config +# has n1 as a voter, but n1 doesn't even have the corresponding conf change in +# its log. Still, it casts a vote for n2 which can in turn become leader and +# catches up n3. +stabilize 2 3 +---- +> 2 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:2 Vote:2 Commit:4 + Messages: + 2->1 MsgVote Term:2 Log:1/4 + 2->3 MsgVote Term:2 Log:1/4 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChangeV2 v3] + 1->3 MsgApp Term:1 Log:1/4 Commit:4 + 2->3 MsgVote Term:2 Log:1/4 + INFO 3 [term: 1] received a MsgVote message with higher term from 2 [term: 2] + INFO 3 became follower at term 2 + INFO 3 [logterm: 1, index: 4, vote: 0] cast MsgVote for 2 [logterm: 1, index: 4] at term 2 +> 3 handling Ready + INFO 3 switched to configuration voters=(1 2 3) + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:2 Vote:2 Commit:4 + Entries: + 1/4 EntryConfChangeV2 v3 + CommittedEntries: + 1/4 EntryConfChangeV2 v3 + Messages: + 3->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/4 + 3->2 MsgVoteResp Term:2 Log:0/0 +> 2 receiving messages + 3->2 MsgVoteResp Term:2 Log:0/0 + INFO 2 received MsgVoteResp from 3 at term 2 + INFO 2 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 2 became leader at term 2 +> 2 handling Ready + Ready MustSync=true: + Lead:2 State:StateLeader + Entries: + 2/5 EntryNormal "" + Messages: + 2->1 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] + 2->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] +> 3 receiving messages + 2->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] +> 3 handling Ready + Ready MustSync=true: + Lead:2 State:StateFollower + Entries: + 2/5 EntryNormal "" + Messages: + 3->2 MsgAppResp Term:2 Log:0/5 +> 2 receiving messages + 3->2 MsgAppResp Term:2 Log:0/5 +> 2 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:2 Commit:5 + CommittedEntries: + 2/5 EntryNormal "" + Messages: + 2->3 MsgApp Term:2 Log:2/5 Commit:5 +> 3 receiving messages + 2->3 MsgApp Term:2 Log:2/5 Commit:5 +> 3 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:2 Commit:5 + CommittedEntries: + 2/5 EntryNormal "" + Messages: + 3->2 MsgAppResp Term:2 Log:0/5 +> 2 receiving messages + 3->2 MsgAppResp Term:2 Log:0/5 diff --git a/raft/testdata/confchange_v1_add_single.txt b/raft/testdata/confchange_v1_add_single.txt new file mode 100644 index 000000000000..4c92ee53cbd0 --- /dev/null +++ b/raft/testdata/confchange_v1_add_single.txt @@ -0,0 +1,97 @@ +# Run a V1 membership change that adds a single voter. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 became leader at term 1 + +# Add v2 (with an auto transition). +propose-conf-change 1 v1=true +v2 +---- +ok + +# Pull n2 out of thin air. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, immediately transitions into +# the final config, and catches up n2. Note that it's using an EntryConfChange, not an +# EntryConfChangeV2, so this is compatible with nodes that don't know about V2 conf changes. +stabilize +---- +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2) + Ready MustSync=true: + Lead:1 State:StateLeader + HardState Term:1 Vote:1 Commit:4 + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChange v2 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChange v2 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChange v2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChange v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=1 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false +> 2 receiving messages + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2) + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 diff --git a/raft/testdata/confchange_v1_remove_leader.txt b/raft/testdata/confchange_v1_remove_leader.txt new file mode 100644 index 000000000000..004b971d1500 --- /dev/null +++ b/raft/testdata/confchange_v1_remove_leader.txt @@ -0,0 +1,102 @@ +# We'll turn this back on after the boilerplate. +log-level none +---- +ok + +# Run a V1 membership change that removes the leader. +# Bootstrap n1, n2, n3. +add-nodes 3 voters=(1,2,3) index=2 +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok (quiet) + +log-level debug +---- +ok + +# Remove n1. +propose-conf-change 1 v1=true +r1 +---- +ok + +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/4 EntryConfChange r1 + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] + 1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/4 EntryConfChange r1 + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 3 handling Ready + Ready MustSync=true: + Entries: + 1/4 EntryConfChange r1 + Messages: + 3->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/4 +> 1 handling Ready + INFO 1 switched to configuration voters=(2 3) + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/4 EntryConfChange r1 + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 + 1->3 MsgApp Term:1 Log:1/4 Commit:4 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/4 Commit:4 +> 2 handling Ready + INFO 2 switched to configuration voters=(2 3) + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/4 EntryConfChange r1 + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 3 handling Ready + INFO 3 switched to configuration voters=(2 3) + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/4 EntryConfChange r1 + Messages: + 3->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/4 + +status 1 +---- +2: StateReplicate match=4 next=5 +3: StateReplicate match=4 next=5 + +# TODO(tbg): the leader now drops any proposals, but if it has any other +# uncommitted proposals in its log already, it will likely try to distribute +# them which could be buggy. Test that. +propose-conf-change 1 v1=true +v1 +---- +raft proposal dropped diff --git a/raft/testdata/confchange_upreplicate_1_to_3_v2_auto.txt b/raft/testdata/confchange_v2_add_double_auto.txt similarity index 87% rename from raft/testdata/confchange_upreplicate_1_to_3_v2_auto.txt rename to raft/testdata/confchange_v2_add_double_auto.txt index 4714bfceec13..866053dbb9ce 100644 --- a/raft/testdata/confchange_upreplicate_1_to_3_v2_auto.txt +++ b/raft/testdata/confchange_v2_add_double_auto.txt @@ -15,7 +15,7 @@ INFO 1 became candidate at term 1 INFO 1 received MsgVoteResp from 1 at term 1 INFO 1 became leader at term 1 -propose-conf-change 1 +propose-conf-change 1 transition=auto v2 v3 ---- ok @@ -32,7 +32,9 @@ INFO newRaft 3 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastter # n1 immediately gets to commit & apply the conf change using only itself. We see that # it starts transitioning out of that joint configuration (though we will only see that -# proposal in the next ready handling loop, when it is emitted). +# proposal in the next ready handling loop, when it is emitted). We also see that this +# is using joint consensus, which it has to since we're carrying out two additions at +# once. process-ready 1 ---- INFO 1 switched to configuration voters=(1 2 3)&&(1) autoleave @@ -65,7 +67,7 @@ stabilize 1 # which transitions them out of their joint configuration into the final one (1 2 3). stabilize 1 2 ---- -> delivering messages +> 2 receiving messages 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2 v3] INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] INFO 2 became follower at term 1 @@ -76,7 +78,7 @@ stabilize 1 2 HardState Term:1 Commit:0 Messages: 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) -> delivering messages +> 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 2 for index 3 DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] @@ -85,27 +87,27 @@ stabilize 1 2 > 1 handling Ready Ready MustSync=false: Messages: - 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] -> delivering messages - 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true +> 2 receiving messages + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] - INFO 2 switched to configuration voters=(1 2 3)&&(1) + INFO 2 switched to configuration voters=(1 2 3)&&(1) autoleave INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] > 2 handling Ready Ready MustSync=false: HardState Term:1 Commit:4 - Snapshot Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] + Snapshot Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true Messages: 2->1 MsgAppResp Term:1 Log:0/4 -> delivering messages +> 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/4 DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] -> delivering messages +> 2 receiving messages 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] > 2 handling Ready Ready MustSync=true: @@ -113,7 +115,7 @@ stabilize 1 2 1/5 EntryConfChangeV2 Messages: 2->1 MsgAppResp Term:1 Log:0/5 -> delivering messages +> 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready INFO 1 switched to configuration voters=(1 2 3) @@ -123,7 +125,7 @@ stabilize 1 2 1/5 EntryConfChangeV2 Messages: 1->2 MsgApp Term:1 Log:1/5 Commit:5 -> delivering messages +> 2 receiving messages 1->2 MsgApp Term:1 Log:1/5 Commit:5 > 2 handling Ready INFO 2 switched to configuration voters=(1 2 3) @@ -133,13 +135,13 @@ stabilize 1 2 1/5 EntryConfChangeV2 Messages: 2->1 MsgAppResp Term:1 Log:0/5 -> delivering messages +> 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/5 # n3 immediately receives a snapshot in the final configuration. stabilize 1 3 ---- -> delivering messages +> 3 receiving messages 1->3 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2 v3] INFO 3 [term: 0] received a MsgApp message with higher term from 1 [term: 1] INFO 3 became follower at term 1 @@ -150,7 +152,7 @@ stabilize 1 3 HardState Term:1 Commit:0 Messages: 3->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) -> delivering messages +> 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 3 for index 3 DEBUG 1 decreased progress of 3 to [StateProbe match=0 next=1] @@ -159,9 +161,9 @@ stabilize 1 3 > 1 handling Ready Ready MustSync=false: Messages: - 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] -> delivering messages - 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] + 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false +> 3 receiving messages + 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 5, term: 1] INFO 3 switched to configuration voters=(1 2 3) INFO 3 [commit: 5, lastindex: 5, lastterm: 1] restored snapshot [index: 5, term: 1] @@ -169,23 +171,23 @@ stabilize 1 3 > 3 handling Ready Ready MustSync=false: HardState Term:1 Commit:5 - Snapshot Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] + Snapshot Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 3->1 MsgAppResp Term:1 Log:0/5 -> delivering messages +> 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/5 DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=5 next=6 paused pendingSnap=5] > 1 handling Ready Ready MustSync=false: Messages: 1->3 MsgApp Term:1 Log:1/5 Commit:5 -> delivering messages +> 3 receiving messages 1->3 MsgApp Term:1 Log:1/5 Commit:5 > 3 handling Ready Ready MustSync=false: Messages: 3->1 MsgAppResp Term:1 Log:0/5 -> delivering messages +> 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/5 # Nothing else happens. diff --git a/raft/testdata/confchange_v2_add_double_explicit.txt b/raft/testdata/confchange_v2_add_double_explicit.txt new file mode 100644 index 000000000000..60805fcfbfc3 --- /dev/null +++ b/raft/testdata/confchange_v2_add_double_explicit.txt @@ -0,0 +1,206 @@ +# Run a V2 membership change that adds a single voter but explicitly asks for the +# use of joint consensus, including wanting to transition out of the joint config +# manually. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 became leader at term 1 + +# Add v2 with an explicit transition. +propose-conf-change 1 transition=explicit +v2 +---- +ok + +# Pull n2 out of thin air. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, then starts catching up n2. +# Everyone remains in the joint config. Note that the snapshot below has AutoLeave unset. +stabilize 1 2 +---- +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2)&&(1) + Ready MustSync=true: + Lead:1 State:StateLeader + HardState Term:1 Vote:1 Commit:4 + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=1 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:false +> 2 receiving messages + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2)&&(1) + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + +# Check that we're not allowed to change membership again while in the joint state. +# This leads to an empty entry being proposed instead (index 5 in the stabilize block +# below). +propose-conf-change 1 +v3 v4 v5 +---- +INFO 1 ignoring conf change {ConfChangeTransitionAuto [{ConfChangeAddNode 3 []} {ConfChangeAddNode 4 []} {ConfChangeAddNode 5 []}] [] []} at config voters=(1 2)&&(1): must transition out of joint config first + +# Propose a transition out of the joint config. We'll see this at index 6 below. +propose-conf-change 1 +---- +ok + +# The group commits the command and everyone switches to the final config. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryNormal ""] + 1->2 MsgApp Term:1 Log:1/5 Commit:4 Entries:[1/6 EntryConfChangeV2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryNormal ""] + 1->2 MsgApp Term:1 Log:1/5 Commit:4 Entries:[1/6 EntryConfChangeV2] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/6 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/6 +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2) + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:6 + CommittedEntries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->2 MsgApp Term:1 Log:1/6 Commit:6 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->2 MsgApp Term:1 Log:1/6 Commit:6 +> 2 handling Ready + INFO 2 switched to configuration voters=(1 2) + Ready MustSync=false: + HardState Term:1 Commit:6 + CommittedEntries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + +# Check that trying to transition out again won't do anything. +propose-conf-change 1 +---- +INFO 1 ignoring conf change {ConfChangeTransitionAuto [] [] []} at config voters=(1 2): not in joint state; refusing empty conf change + +# Finishes work for the empty entry we just proposed. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/7 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:6 Entries:[1/7 EntryNormal ""] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/6 Commit:6 Entries:[1/7 EntryNormal ""] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/7 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/7 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/7 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:7 + CommittedEntries: + 1/7 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/7 Commit:7 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/7 Commit:7 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:7 + CommittedEntries: + 1/7 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/7 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/7 diff --git a/raft/testdata/confchange_v2_add_double_implicit.txt b/raft/testdata/confchange_v2_add_double_implicit.txt new file mode 100644 index 000000000000..d5061c0ca030 --- /dev/null +++ b/raft/testdata/confchange_v2_add_double_implicit.txt @@ -0,0 +1,125 @@ +# Run a V2 membership change that adds a single voter but explicitly asks for the +# use of joint consensus (with auto-leaving). + +# TODO(tbg): also verify that if the leader changes while in the joint state, the +# new leader will auto-transition out of the joint state just the same. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 became leader at term 1 + +propose-conf-change 1 transition=implicit +v2 +---- +ok + +# Add n2. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, then starts catching up n2. +# When that's done, it starts auto-transitioning out. Note that the snapshots propagating +# the joint config have the AutoLeave flag set in their config. +stabilize 1 2 +---- +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2)&&(1) autoleave + INFO initiating automatic transition out of joint configuration voters=(1 2)&&(1) autoleave + Ready MustSync=true: + Lead:1 State:StateLeader + HardState Term:1 Vote:1 Commit:4 + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=1 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true +> 2 receiving messages + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true + INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2)&&(1) autoleave + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/5 +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2) + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/5 Commit:5 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/5 Commit:5 +> 2 handling Ready + INFO 2 switched to configuration voters=(1 2) + Ready MustSync=false: + HardState Term:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/5 diff --git a/raft/testdata/confchange_v2_add_single_auto.txt b/raft/testdata/confchange_v2_add_single_auto.txt new file mode 100644 index 000000000000..2b00facbc2f0 --- /dev/null +++ b/raft/testdata/confchange_v2_add_single_auto.txt @@ -0,0 +1,198 @@ +# Run a V2 membership change that adds a single voter in auto mode, which means +# that joint consensus is not used but a direct transition into the new config +# takes place. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 became leader at term 1 + +# Add v2 (with an auto transition). +propose-conf-change 1 +v2 +---- +ok + +# Pull n2 out of thin air. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, immediately transitions into +# the final config, and catches up n2. +stabilize +---- +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2) + Ready MustSync=true: + Lead:1 State:StateLeader + HardState Term:1 Vote:1 Commit:4 + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=1 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false +> 2 receiving messages + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2) + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + +# Check that we're not allowed to change membership again while in the joint state. +# This leads to an empty entry being proposed instead (index 5 in the stabilize block +# below). +propose-conf-change 1 +v3 v4 v5 +---- +ok + +# Propose a transition out of the joint config. We'll see this at index 6 below. +propose-conf-change 1 +---- +INFO 1 ignoring conf change {ConfChangeTransitionAuto [] [] []} at config voters=(1 2): possible unapplied conf change at index 5 (applied to 4) + +# The group commits the command and everyone switches to the final config. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 v3 v4 v5 + 1/6 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2 v3 v4 v5] + 1->2 MsgApp Term:1 Log:1/5 Commit:4 Entries:[1/6 EntryNormal ""] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2 v3 v4 v5] + 1->2 MsgApp Term:1 Log:1/5 Commit:4 Entries:[1/6 EntryNormal ""] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 v3 v4 v5 + 1/6 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/6 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/6 +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2 3 4 5)&&(1 2) autoleave + INFO initiating automatic transition out of joint configuration voters=(1 2 3 4 5)&&(1 2) autoleave + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:6 + CommittedEntries: + 1/5 EntryConfChangeV2 v3 v4 v5 + 1/6 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->2 MsgApp Term:1 Log:1/6 Commit:6 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->2 MsgApp Term:1 Log:1/6 Commit:6 +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/7 EntryConfChangeV2 + Messages: + 1->3 MsgApp Term:1 Log:1/5 Commit:6 Entries:[1/6 EntryNormal ""] + 1->4 MsgApp Term:1 Log:1/5 Commit:6 Entries:[1/6 EntryNormal ""] + 1->5 MsgApp Term:1 Log:1/5 Commit:6 Entries:[1/6 EntryNormal ""] +> 2 handling Ready + INFO 2 switched to configuration voters=(1 2 3 4 5)&&(1 2) autoleave + Ready MustSync=false: + HardState Term:1 Commit:6 + CommittedEntries: + 1/5 EntryConfChangeV2 v3 v4 v5 + 1/6 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + +# Check that trying to transition out again won't do anything. +propose-conf-change 1 +---- +ok + +# Finishes work for the empty entry we just proposed. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/8 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:6 Entries:[1/7 EntryConfChangeV2, 1/8 EntryConfChangeV2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/6 Commit:6 Entries:[1/7 EntryConfChangeV2, 1/8 EntryConfChangeV2] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/7 EntryConfChangeV2 + 1/8 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/8 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/8 diff --git a/raft/testdata/snapshot_succeed_via_app_resp.txt b/raft/testdata/snapshot_succeed_via_app_resp.txt index c7ddfec41cde..09e0f588c5d2 100644 --- a/raft/testdata/snapshot_succeed_via_app_resp.txt +++ b/raft/testdata/snapshot_succeed_via_app_resp.txt @@ -70,7 +70,7 @@ Messages: # and responds. stabilize 3 ---- -> delivering messages +> 3 receiving messages 1->3 MsgHeartbeat Term:1 Log:0/0 INFO 3 [term: 0] received a MsgHeartbeat message with higher term from 1 [term: 1] INFO 3 became follower at term 1 @@ -84,14 +84,14 @@ stabilize 3 # The leader in turn will realize that n3 needs a snapshot, which it initiates. stabilize 1 ---- -> delivering messages +> 1 receiving messages 3->1 MsgHeartbeatResp Term:1 Log:0/0 DEBUG 1 [firstindex: 12, commit: 11] sent snapshot[index: 11, term: 1] to 3 [StateProbe match=0 next=11] DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=11 paused pendingSnap=11] > 1 handling Ready Ready MustSync=false: Messages: - 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] + 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false status 1 ---- @@ -105,8 +105,8 @@ status 1 # was now fully caught up. stabilize 3 ---- -> delivering messages - 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] +> 3 receiving messages + 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 11, term: 1] INFO 3 switched to configuration voters=(1 2 3) INFO 3 [commit: 11, lastindex: 11, lastterm: 1] restored snapshot [index: 11, term: 1] @@ -114,7 +114,7 @@ stabilize 3 > 3 handling Ready Ready MustSync=false: HardState Term:1 Commit:11 - Snapshot Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] + Snapshot Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 3->1 MsgAppResp Term:1 Log:0/11 @@ -122,7 +122,7 @@ stabilize 3 # Leader sends another MsgAppResp, to communicate the updated commit index. stabilize 1 ---- -> delivering messages +> 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/11 DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 next=12 paused pendingSnap=11] > 1 handling Ready @@ -139,9 +139,9 @@ status 1 # Let things settle. stabilize ---- -> delivering messages +> 2 receiving messages 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 -> delivering messages +> 3 receiving messages 1->3 MsgApp Term:1 Log:1/11 Commit:11 > 2 handling Ready Ready MustSync=false: @@ -151,6 +151,6 @@ stabilize Ready MustSync=false: Messages: 3->1 MsgAppResp Term:1 Log:0/11 -> delivering messages +> 1 receiving messages 2->1 MsgHeartbeatResp Term:1 Log:0/0 3->1 MsgAppResp Term:1 Log:0/11 diff --git a/raft/util.go b/raft/util.go index 881a6e14e241..785cf735d5db 100644 --- a/raft/util.go +++ b/raft/util.go @@ -77,8 +77,8 @@ func DescribeSoftState(ss SoftState) string { func DescribeConfState(state pb.ConfState) string { return fmt.Sprintf( - "Voters:%v VotersOutgoing:%v Learners:%v LearnersNext:%v", - state.Voters, state.VotersOutgoing, state.Learners, state.LearnersNext, + "Voters:%v VotersOutgoing:%v Learners:%v LearnersNext:%v AutoLeave:%v", + state.Voters, state.VotersOutgoing, state.Learners, state.LearnersNext, state.AutoLeave, ) }