diff --git a/server/etcdserver/raft.go b/server/etcdserver/raft.go index 6acacf8b4a7..bde23ad4ef9 100644 --- a/server/etcdserver/raft.go +++ b/server/etcdserver/raft.go @@ -552,12 +552,17 @@ func (b *bootstrappedRaft) newRaftNode(ss *snap.Snapshotter) *raftNode { // - ConfChangeAddNode, in which case the contained ID will be added into the set. // - ConfChangeRemoveNode, in which case the contained ID will be removed from the set. // - ConfChangeAddLearnerNode, in which the contained ID will be added into the set. -func getIDs(lg *zap.Logger, snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 { +func getIDs(lg *zap.Logger, snap *raftpb.Snapshot, ents []raftpb.Entry) ([]uint64, map[uint64]bool) { ids := make(map[uint64]bool) + isLearnerMap := make(map[uint64]bool) if snap != nil { for _, id := range snap.Metadata.ConfState.Voters { ids[id] = true } + for _, id := range snap.Metadata.ConfState.Learners { + ids[id] = true + isLearnerMap[id] = true + } } for _, e := range ents { if e.Type != raftpb.EntryConfChange { @@ -568,6 +573,7 @@ func getIDs(lg *zap.Logger, snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 switch cc.Type { case raftpb.ConfChangeAddLearnerNode: ids[cc.NodeID] = true + isLearnerMap[cc.NodeID] = true case raftpb.ConfChangeAddNode: ids[cc.NodeID] = true case raftpb.ConfChangeRemoveNode: @@ -583,7 +589,7 @@ func getIDs(lg *zap.Logger, snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 sids = append(sids, id) } sort.Sort(sids) - return []uint64(sids) + return []uint64(sids), isLearnerMap } // createConfigChangeEnts creates a series of Raft entries (i.e. @@ -591,7 +597,7 @@ func getIDs(lg *zap.Logger, snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 // `self` is _not_ removed, even if present in the set. // If `self` is not inside the given ids, it creates a Raft entry to add a // default member with the given `self`. -func createConfigChangeEnts(lg *zap.Logger, ids []uint64, self uint64, term, index uint64) []raftpb.Entry { +func createConfigChangeEnts(lg *zap.Logger, ids []uint64, isLearnerMap map[uint64]bool, self uint64, term, index uint64) []raftpb.Entry { found := false for _, id := range ids { if id == self { @@ -602,12 +608,10 @@ func createConfigChangeEnts(lg *zap.Logger, ids []uint64, self uint64, term, ind var ents []raftpb.Entry next := index + 1 - // NB: always add self first, then remove other nodes. Raft will panic if the - // set of voters ever becomes empty. - if !found { + addNodeFunc := func(id uint64, peerUrls []string) { m := membership.Member{ ID: types.ID(self), - RaftAttributes: membership.RaftAttributes{PeerURLs: []string{"http://localhost:2380"}}, + RaftAttributes: membership.RaftAttributes{PeerURLs: peerUrls}, } ctx, err := json.Marshal(m) if err != nil { @@ -615,7 +619,7 @@ func createConfigChangeEnts(lg *zap.Logger, ids []uint64, self uint64, term, ind } cc := &raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, - NodeID: self, + NodeID: id, Context: ctx, } e := raftpb.Entry{ @@ -628,10 +632,7 @@ func createConfigChangeEnts(lg *zap.Logger, ids []uint64, self uint64, term, ind next++ } - for _, id := range ids { - if id == self { - continue - } + delNodeFunc := func(id uint64) { cc := &raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: id, @@ -646,5 +647,50 @@ func createConfigChangeEnts(lg *zap.Logger, ids []uint64, self uint64, term, ind next++ } + promoteNodeFunc := func(id uint64) { + // build the context for the promote confChange. mark IsLearner to false and IsPromote to true. + promoteChangeContext := membership.ConfigChangeContext{ + Member: membership.Member{ + ID: types.ID(id), + }, + IsPromote: true, + } + + b, err := json.Marshal(promoteChangeContext) + if err != nil { + lg.Panic("failed to marshal member", zap.Error(err)) + } + + cc := &raftpb.ConfChange{ + Type: raftpb.ConfChangeAddNode, + NodeID: id, + Context: b, + } + + e := raftpb.Entry{ + Type: raftpb.EntryConfChange, + Data: pbutil.MustMarshal(cc), + Term: term, + Index: next, + } + ents = append(ents, e) + next++ + } + + // NB: always add self first, then remove other nodes. Raft will panic if the + // set of voters ever becomes empty. + if !found { + addNodeFunc(self, []string{"http://localhost:2380"}) + } else if isLearnerMap[self] { + promoteNodeFunc(self) + } + + for _, id := range ids { + if id == self { + continue + } + delNodeFunc(id) + } + return ents } diff --git a/server/etcdserver/raft_test.go b/server/etcdserver/raft_test.go index 3eb5345dc25..4224f3e9c42 100644 --- a/server/etcdserver/raft_test.go +++ b/server/etcdserver/raft_test.go @@ -66,7 +66,7 @@ func TestGetIDs(t *testing.T) { if tt.confState != nil { snap.Metadata.ConfState = *tt.confState } - idSet := getIDs(testLogger, &snap, tt.ents) + idSet, _ := getIDs(testLogger, &snap, tt.ents) if !reflect.DeepEqual(idSet, tt.widSet) { t.Errorf("#%d: idset = %#v, want %#v", i, idSet, tt.widSet) } @@ -146,7 +146,7 @@ func TestCreateConfigChangeEnts(t *testing.T) { } for i, tt := range tests { - gents := createConfigChangeEnts(testLogger, tt.ids, tt.self, tt.term, tt.index) + gents := createConfigChangeEnts(testLogger, tt.ids, make(map[uint64]bool), tt.self, tt.term, tt.index) if !reflect.DeepEqual(gents, tt.wents) { t.Errorf("#%d: ents = %v, want %v", i, gents, tt.wents) } diff --git a/server/etcdserver/storage.go b/server/etcdserver/storage.go index 080fe3f1f18..e07cc955e41 100644 --- a/server/etcdserver/storage.go +++ b/server/etcdserver/storage.go @@ -192,9 +192,11 @@ func (wal *bootstrappedWAL) CommitedEntries() []raftpb.Entry { } func (wal *bootstrappedWAL) ConfigChangeEntries() []raftpb.Entry { + ids, isLearnerMap := getIDs(wal.lg, wal.snapshot, wal.ents) return createConfigChangeEnts( wal.lg, - getIDs(wal.lg, wal.snapshot, wal.ents), + ids, + isLearnerMap, uint64(wal.id), wal.st.Term, wal.st.Commit,