Skip to content

Commit 7c09f49

Browse files
slackpadslackpad
authored
Merge pull request hashicorp#159 from hashicorp/f-snapshot-restore
Adds ability to access user snapshots and restore them.
2 parents c69c15d + fe8cdcd commit 7c09f49

8 files changed

+442
-104
lines changed

‎.travis.yml

-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
language: go
22

33
go:
4-
- 1.4
5-
- 1.5
64
- 1.6
75
- tip
86

‎Makefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
DEPS = $(go list -f '{{range .TestImports}}{{.}} {{end}}' ./...)
22

33
test:
4-
go test -timeout=45s ./...
4+
go test -timeout=60s ./...
55

66
integ: test
7-
INTEG_TESTS=yes go test -timeout=3s -run=Integ ./...
7+
INTEG_TESTS=yes go test -timeout=5s -run=Integ ./...
88

99
deps:
1010
go get -d -v ./...

‎api.go

+86-18
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package raft
33
import (
44
"errors"
55
"fmt"
6+
"io"
67
"log"
78
"os"
89
"strconv"
@@ -64,11 +65,14 @@ type Raft struct {
6465
// FSM is the client state machine to apply commands to
6566
fsm FSM
6667

67-
// fsmCommitCh is used to trigger async application of logs to the fsm
68-
fsmCommitCh chan commitTuple
69-
70-
// fsmRestoreCh is used to trigger a restore from snapshot
71-
fsmRestoreCh chan *restoreFuture
68+
// fsmMutateCh is used to send state-changing updates to the FSM. This
69+
// receives pointers to commitTuple structures when applying logs or
70+
// pointers to restoreFuture structures when restoring a snapshot. We
71+
// need control over the order of these operations when doing user
72+
// restores so that we finish applying any old log applies before we
73+
// take a user snapshot on the leader, otherwise we might restore the
74+
// snapshot and apply old logs to it that were in the pipe.
75+
fsmMutateCh chan interface{}
7276

7377
// fsmSnapshotCh is used to trigger a new snapshot being taken
7478
fsmSnapshotCh chan *reqSnapshotFuture
@@ -118,8 +122,12 @@ type Raft struct {
118122
// snapshots is used to store and retrieve snapshots
119123
snapshots SnapshotStore
120124

121-
// snapshotCh is used for user triggered snapshots
122-
snapshotCh chan *snapshotFuture
125+
// userSnapshotCh is used for user-triggered snapshots
126+
userSnapshotCh chan *userSnapshotFuture
127+
128+
// userRestoreCh is used for user-triggered restores of external
129+
// snapshots
130+
userRestoreCh chan *userRestoreFuture
123131

124132
// stable is a StableStore implementation for durable state
125133
// It provides stable storage for many fields in raftState
@@ -429,8 +437,7 @@ func NewRaft(conf *Config, fsm FSM, logs LogStore, stable StableStore, snaps Sna
429437
applyCh: make(chan *logFuture),
430438
conf: *conf,
431439
fsm: fsm,
432-
fsmCommitCh: make(chan commitTuple, 128),
433-
fsmRestoreCh: make(chan *restoreFuture),
440+
fsmMutateCh: make(chan interface{}, 128),
434441
fsmSnapshotCh: make(chan *reqSnapshotFuture),
435442
leaderCh: make(chan bool),
436443
localID: localID,
@@ -441,7 +448,8 @@ func NewRaft(conf *Config, fsm FSM, logs LogStore, stable StableStore, snaps Sna
441448
configurations: configurations{},
442449
rpcCh: trans.Consumer(),
443450
snapshots: snaps,
444-
snapshotCh: make(chan *snapshotFuture),
451+
userSnapshotCh: make(chan *userSnapshotFuture),
452+
userRestoreCh: make(chan *userRestoreFuture),
445453
shutdownCh: make(chan struct{}),
446454
stable: stable,
447455
trans: trans,
@@ -792,18 +800,78 @@ func (r *Raft) Shutdown() Future {
792800
return &shutdownFuture{nil}
793801
}
794802

795-
// Snapshot is used to manually force Raft to take a snapshot.
796-
// Returns a future that can be used to block until complete.
797-
func (r *Raft) Snapshot() Future {
798-
snapFuture := &snapshotFuture{}
799-
snapFuture.init()
803+
// Snapshot is used to manually force Raft to take a snapshot. Returns a future
804+
// that can be used to block until complete, and that contains a function that
805+
// can be used to open the snapshot.
806+
func (r *Raft) Snapshot() SnapshotFuture {
807+
future := &userSnapshotFuture{}
808+
future.init()
809+
select {
810+
case r.userSnapshotCh <- future:
811+
return future
812+
case <-r.shutdownCh:
813+
future.respond(ErrRaftShutdown)
814+
return future
815+
}
816+
}
817+
818+
// Restore is used to manually force Raft to consume an external snapshot, such
819+
// as if restoring from a backup. We will use the current Raft configuration,
820+
// not the one from the snapshot, so that we can restore into a new cluster. We
821+
// will also use the higher of the index of the snapshot, or the current index,
822+
// and then add 1 to that, so we force a new state with a hole in the Raft log,
823+
// so that the snapshot will be sent to followers and used for any new joiners.
824+
// This can only be run on the leader, and returns a future that can be used to
825+
// block until complete.
826+
//
827+
// WARNING! This operation has the leader take on the state of the snapshot and
828+
// then sets itself up so that it replicates that to its followers though the
829+
// install snapshot process. This involves a potentially dangerous period where
830+
// the leader commits ahead of its followers, so should only be used for disaster
831+
// recovery into a fresh cluster, and should not be used in normal operations.
832+
func (r *Raft) Restore(meta *SnapshotMeta, reader io.ReadCloser, timeout time.Duration) Future {
833+
metrics.IncrCounter([]string{"raft", "restore"}, 1)
834+
var timer <-chan time.Time
835+
if timeout > 0 {
836+
timer = time.After(timeout)
837+
}
838+
839+
// Perform the restore.
840+
restore := &userRestoreFuture{
841+
meta: meta,
842+
reader: reader,
843+
}
844+
restore.init()
800845
select {
801-
case r.snapshotCh <- snapFuture:
802-
return snapFuture
846+
case <-timer:
847+
return errorFuture{ErrEnqueueTimeout}
803848
case <-r.shutdownCh:
804849
return errorFuture{ErrRaftShutdown}
850+
case r.userRestoreCh <- restore:
851+
// If the restore is ingested then wait for it to complete.
852+
if err := restore.Error(); err != nil {
853+
return restore
854+
}
805855
}
806856

857+
// Apply a no-op log entry. Waiting for this allows us to wait until the
858+
// followers have gotten the restore and replicated at least this new
859+
// entry, which shows that we've also faulted and installed the
860+
// snapshot with the contents of the restore.
861+
noop := &logFuture{
862+
log: Log{
863+
Type: LogNoop,
864+
},
865+
}
866+
noop.init()
867+
select {
868+
case <-timer:
869+
return errorFuture{ErrEnqueueTimeout}
870+
case <-r.shutdownCh:
871+
return errorFuture{ErrRaftShutdown}
872+
case r.applyCh <- noop:
873+
return noop
874+
}
807875
}
808876

809877
// State is used to return the current raft state.
@@ -870,7 +938,7 @@ func (r *Raft) Stats() map[string]string {
870938
"last_log_term": toString(lastLogTerm),
871939
"commit_index": toString(r.getCommitIndex()),
872940
"applied_index": toString(r.getLastApplied()),
873-
"fsm_pending": toString(uint64(len(r.fsmCommitCh))),
941+
"fsm_pending": toString(uint64(len(r.fsmMutateCh))),
874942
"last_snapshot_index": toString(lastSnapIndex),
875943
"last_snapshot_term": toString(lastSnapTerm),
876944
"protocol_version": toString(uint64(r.protocolVersion)),

‎fsm.go

+72-52
Original file line numberDiff line numberDiff line change
@@ -48,67 +48,87 @@ type FSMSnapshot interface {
4848
// the FSM to block our internal operations.
4949
func (r *Raft) runFSM() {
5050
var lastIndex, lastTerm uint64
51-
for {
52-
select {
53-
case req := <-r.fsmRestoreCh:
54-
// Open the snapshot
55-
meta, source, err := r.snapshots.Open(req.ID)
56-
if err != nil {
57-
req.respond(fmt.Errorf("failed to open snapshot %v: %v", req.ID, err))
58-
continue
59-
}
6051

61-
// Attempt to restore
52+
commit := func(req *commitTuple) {
53+
// Apply the log if a command
54+
var resp interface{}
55+
if req.log.Type == LogCommand {
6256
start := time.Now()
63-
if err := r.fsm.Restore(source); err != nil {
64-
req.respond(fmt.Errorf("failed to restore snapshot %v: %v", req.ID, err))
65-
source.Close()
66-
continue
67-
}
57+
resp = r.fsm.Apply(req.log)
58+
metrics.MeasureSince([]string{"raft", "fsm", "apply"}, start)
59+
}
60+
61+
// Update the indexes
62+
lastIndex = req.log.Index
63+
lastTerm = req.log.Term
64+
65+
// Invoke the future if given
66+
if req.future != nil {
67+
req.future.response = resp
68+
req.future.respond(nil)
69+
}
70+
}
71+
72+
restore := func(req *restoreFuture) {
73+
// Open the snapshot
74+
meta, source, err := r.snapshots.Open(req.ID)
75+
if err != nil {
76+
req.respond(fmt.Errorf("failed to open snapshot %v: %v", req.ID, err))
77+
return
78+
}
79+
80+
// Attempt to restore
81+
start := time.Now()
82+
if err := r.fsm.Restore(source); err != nil {
83+
req.respond(fmt.Errorf("failed to restore snapshot %v: %v", req.ID, err))
6884
source.Close()
69-
metrics.MeasureSince([]string{"raft", "fsm", "restore"}, start)
85+
return
86+
}
87+
source.Close()
88+
metrics.MeasureSince([]string{"raft", "fsm", "restore"}, start)
7089

71-
// Update the last index and term
72-
lastIndex = meta.Index
73-
lastTerm = meta.Term
74-
req.respond(nil)
90+
// Update the last index and term
91+
lastIndex = meta.Index
92+
lastTerm = meta.Term
93+
req.respond(nil)
94+
}
7595

76-
case req := <-r.fsmSnapshotCh:
77-
// Is there something to snapshot?
78-
if lastIndex == 0 {
79-
req.respond(ErrNothingNewToSnapshot)
80-
continue
81-
}
96+
snapshot := func(req *reqSnapshotFuture) {
97+
// Is there something to snapshot?
98+
if lastIndex == 0 {
99+
req.respond(ErrNothingNewToSnapshot)
100+
return
101+
}
82102

83-
// Start a snapshot
84-
start := time.Now()
85-
snap, err := r.fsm.Snapshot()
86-
metrics.MeasureSince([]string{"raft", "fsm", "snapshot"}, start)
87-
88-
// Respond to the request
89-
req.index = lastIndex
90-
req.term = lastTerm
91-
req.snapshot = snap
92-
req.respond(err)
93-
94-
case commitEntry := <-r.fsmCommitCh:
95-
// Apply the log if a command
96-
var resp interface{}
97-
if commitEntry.log.Type == LogCommand {
98-
start := time.Now()
99-
resp = r.fsm.Apply(commitEntry.log)
100-
metrics.MeasureSince([]string{"raft", "fsm", "apply"}, start)
101-
}
103+
// Start a snapshot
104+
start := time.Now()
105+
snap, err := r.fsm.Snapshot()
106+
metrics.MeasureSince([]string{"raft", "fsm", "snapshot"}, start)
102107

103-
// Update the indexes
104-
lastIndex = commitEntry.log.Index
105-
lastTerm = commitEntry.log.Term
108+
// Respond to the request
109+
req.index = lastIndex
110+
req.term = lastTerm
111+
req.snapshot = snap
112+
req.respond(err)
113+
}
106114

107-
// Invoke the future if given
108-
if commitEntry.future != nil {
109-
commitEntry.future.response = resp
110-
commitEntry.future.respond(nil)
115+
for {
116+
select {
117+
case ptr := <-r.fsmMutateCh:
118+
switch req := ptr.(type) {
119+
case *commitTuple:
120+
commit(req)
121+
122+
case *restoreFuture:
123+
restore(req)
124+
125+
default:
126+
panic(fmt.Errorf("bad type passed to fsmMutateCh: %#v", ptr))
111127
}
128+
129+
case req := <-r.fsmSnapshotCh:
130+
snapshot(req)
131+
112132
case <-r.shutdownCh:
113133
return
114134
}

‎future.go

+46-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package raft
22

33
import (
4+
"fmt"
5+
"io"
46
"sync"
57
"time"
68
)
@@ -46,6 +48,16 @@ type ConfigurationFuture interface {
4648
Configuration() Configuration
4749
}
4850

51+
// SnapshotFuture is used for waiting on a user-triggered snapshot to complete.
52+
type SnapshotFuture interface {
53+
Future
54+
55+
// Open is a function you can call to access the underlying snapshot and
56+
// its metadata. This must not be called until after the Error method
57+
// has returned.
58+
Open() (*SnapshotMeta, io.ReadCloser, error)
59+
}
60+
4961
// errorFuture is used to return a static error.
5062
type errorFuture struct {
5163
err error
@@ -150,9 +162,41 @@ func (s *shutdownFuture) Error() error {
150162
return nil
151163
}
152164

153-
// snapshotFuture is used for waiting on a snapshot to complete.
154-
type snapshotFuture struct {
165+
// userSnapshotFuture is used for waiting on a user-triggered snapshot to
166+
// complete.
167+
type userSnapshotFuture struct {
168+
deferError
169+
170+
// opener is a function used to open the snapshot. This is filled in
171+
// once the future returns with no error.
172+
opener func() (*SnapshotMeta, io.ReadCloser, error)
173+
}
174+
175+
// Open is a function you can call to access the underlying snapshot and its
176+
// metadata.
177+
func (u *userSnapshotFuture) Open() (*SnapshotMeta, io.ReadCloser, error) {
178+
if u.opener == nil {
179+
return nil, nil, fmt.Errorf("no snapshot available")
180+
} else {
181+
// Invalidate the opener so it can't get called multiple times,
182+
// which isn't generally safe.
183+
defer func() {
184+
u.opener = nil
185+
}()
186+
return u.opener()
187+
}
188+
}
189+
190+
// userRestoreFuture is used for waiting on a user-triggered restore of an
191+
// external snapshot to complete.
192+
type userRestoreFuture struct {
155193
deferError
194+
195+
// meta is the metadata that belongs with the snapshot.
196+
meta *SnapshotMeta
197+
198+
// reader is the interface to read the snapshot contents from.
199+
reader io.ReadCloser
156200
}
157201

158202
// reqSnapshotFuture is used for requesting a snapshot start.

0 commit comments

Comments
 (0)