From 55134a5ea1cb184e196e09b2ba7814e82ffccd90 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sat, 25 Nov 2023 17:02:28 +0800 Subject: [PATCH] tests: introduce powerfailure case Signed-off-by: Wei Fu --- .github/workflows/failpoint_test.yaml | 4 +- go.mod | 3 + go.sum | 10 +- tests/failpoint/powerfailure_test.go | 180 ++++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 tests/failpoint/powerfailure_test.go diff --git a/.github/workflows/failpoint_test.yaml b/.github/workflows/failpoint_test.yaml index 46cafab6c..2c6830c75 100644 --- a/.github/workflows/failpoint_test.yaml +++ b/.github/workflows/failpoint_test.yaml @@ -16,4 +16,6 @@ jobs: go-version: ${{ steps.goversion.outputs.goversion }} - run: | make gofail-enable - make test-failpoint + # build bbolt with failpoint + go install ./cmd/bbolt + sudo -E PATH=$PATH make test-failpoint diff --git a/go.mod b/go.mod index 26f2d2f52..f0a682821 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module go.etcd.io/bbolt go 1.21 require ( + github.com/fuweid/go-dmflakey v0.1.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 @@ -14,6 +15,8 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 204bc990c..1b46c090f 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,15 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fuweid/go-dmflakey v0.1.0 h1:Cv5EH85FCBBGfzIjLrUqPguzipnJnbz8TawQ47ITh30= +github.com/fuweid/go-dmflakey v0.1.0/go.mod h1:LyUyUu5fLT6AhjTmoztxGdgvFOTyA+3hU2kM92drr9M= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -18,7 +25,8 @@ golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tests/failpoint/powerfailure_test.go b/tests/failpoint/powerfailure_test.go new file mode 100644 index 000000000..9d0cd8a20 --- /dev/null +++ b/tests/failpoint/powerfailure_test.go @@ -0,0 +1,180 @@ +//go:build linux + +package failpoint + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/fuweid/go-dmflakey" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// TestRestartFromPowerFailure is to test data after unexpected power failure. +func TestRestartFromPowerFailure(t *testing.T) { + flakey := initFlakeyDevice(t, "") + root := flakey.rootfs() + + dbPath := filepath.Join(root, "boltdb") + + args := []string{"bbolt", "bench", + "-work", // keep the database + "-path", dbPath, + "-count=1000000000", + "-batch-size=5", // separate total count into multiple truncation + } + + logPath := filepath.Join(t.TempDir(), fmt.Sprintf("%s.log", t.Name())) + logFd, err := os.Create(logPath) + require.NoError(t, err) + defer logFd.Close() + + // FIXME: gofail should support unix socket so that the test cases won't + // be conflicted. + fpURL := "127.0.0.1:12345" + + cmd := exec.Command(args[0], args[1:]...) + cmd.Stdout = logFd + cmd.Stderr = logFd + cmd.Env = append(cmd.Env, "GOFAIL_HTTP="+fpURL) + t.Logf("start %s", strings.Join(args, " ")) + require.NoError(t, cmd.Start(), "args: %v", args) + + errCh := make(chan error, 1) + go func() { + defer close(errCh) + errCh <- cmd.Wait() + }() + + defer func() { + if t.Failed() { + logData, err := os.ReadFile(logPath) + assert.NoError(t, err) + t.Logf("dump log:\n: %s", string(logData)) + } + }() + + time.Sleep(time.Duration(time.Now().UnixNano()%5+1) * time.Second) + t.Logf("simulate power failure") + + activeFailpoint(t, fpURL, "beforeSyncMetaPage", "panic") + + select { + case <-time.After(10 * time.Second): + t.Error("bbolt should stop with panic in seconds") + assert.NoError(t, cmd.Process.Kill()) + case err := <-errCh: + require.Error(t, err) + } + require.NoError(t, flakey.powerFailure("")) + + st, err := os.Stat(dbPath) + require.NoError(t, err) + t.Logf("db size: %d", st.Size()) + + t.Logf("verify data") + output, err := exec.Command("bbolt", "check", dbPath).CombinedOutput() + require.NoError(t, err, "bbolt check output: %s", string(output)) +} + +// activeFailpoint actives the failpoint by http. +func activeFailpoint(t *testing.T, targetUrl string, fpName, fpVal string) { + u, err := url.Parse("http://" + path.Join(targetUrl, fpName)) + require.NoError(t, err, "parse url %s", targetUrl) + + req, err := http.NewRequest("PUT", u.String(), bytes.NewBuffer([]byte(fpVal))) + require.NoError(t, err) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, 204, resp.StatusCode, "response body: %s", string(data)) +} + +// initFlakeyDevice inits flakey device with ext4 filesystem. +func initFlakeyDevice(t *testing.T, mntOpt string) *flakeyDevice { + imgDir := t.TempDir() + rootDir := t.TempDir() + fsType := dmflakey.FSTypeEXT4 + + flakey, err := dmflakey.InitFlakey("bbolt-failpoint", imgDir, fsType) + require.NoError(t, err, "init flakey device") + t.Cleanup(func() { + assert.NoError(t, flakey.Teardown()) + }) + + err = unix.Mount(flakey.DevicePath(), rootDir, string(fsType), 0, mntOpt) + require.NoError(t, err, "init rootfs with flakey device") + t.Cleanup(func() { + assert.NoError(t, unmount(rootDir)) + }) + + return &flakeyDevice{ + flakey: flakey, + + rootDir: rootDir, + } +} + +type flakeyDevice struct { + flakey dmflakey.Flakey + + rootDir string +} + +// rootfs returns the rootfs where flakey device mounts. +func (f *flakeyDevice) rootfs() string { + return f.rootDir +} + +// powerFailure drops all the pending writes and remount the rootfs. +func (f *flakeyDevice) powerFailure(mntOpt string) error { + if err := f.flakey.DropWrites(); err != nil { + return fmt.Errorf("failed to drop_writes: %w", err) + } + + if err := unmount(f.rootDir); err != nil { + return fmt.Errorf("failed to unmount rootfs %s: %w", f.rootDir, err) + } + + if err := f.flakey.AllowWrites(); err != nil { + return fmt.Errorf("failed to allow_writes: %w", err) + } + + if err := unix.Mount(f.flakey.DevicePath(), f.rootDir, string(f.flakey.Filesystem()), 0, mntOpt); err != nil { + return fmt.Errorf("failed to mount rootfs %s: %w", f.rootDir, err) + } + return nil +} + +func unmount(target string) error { + for i := 0; i < 50; i++ { + if err := unix.Unmount(target, 0); err != nil { + switch err { + case unix.EBUSY: + time.Sleep(500 * time.Millisecond) + continue + case unix.EINVAL: + default: + return fmt.Errorf("failed to umount %s: %w", target, err) + } + } + return nil + } + return unix.EBUSY +}