diff --git a/.gitignore b/.gitignore index 0e9d8df0b..f9af653df 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,7 @@ bin/ release/ coverage/ coverage.* -tests/*.yaml # Jetbrain IDE .idea -*.iml \ No newline at end of file +*.iml diff --git a/tests/connstr_test.go b/cmd/definitions/tests/connstr_test.go similarity index 100% rename from tests/connstr_test.go rename to cmd/definitions/tests/connstr_test.go diff --git a/tests/doc.go b/cmd/definitions/tests/doc.go similarity index 100% rename from tests/doc.go rename to cmd/definitions/tests/doc.go diff --git a/tests/generated.go b/cmd/definitions/tests/generated.go similarity index 100% rename from tests/generated.go rename to cmd/definitions/tests/generated.go diff --git a/tests/service.go b/cmd/definitions/tests/service.go similarity index 100% rename from tests/service.go rename to cmd/definitions/tests/service.go diff --git a/tests/service.toml b/cmd/definitions/tests/service.toml similarity index 100% rename from tests/service.toml rename to cmd/definitions/tests/service.toml diff --git a/tests/storage.go b/cmd/definitions/tests/storage.go similarity index 100% rename from tests/storage.go rename to cmd/definitions/tests/storage.go diff --git a/tests/storager_bench_test.go b/cmd/definitions/tests/storager_bench_test.go similarity index 100% rename from tests/storager_bench_test.go rename to cmd/definitions/tests/storager_bench_test.go diff --git a/tests/utils.go b/cmd/definitions/tests/utils.go similarity index 100% rename from tests/utils.go rename to cmd/definitions/tests/utils.go diff --git a/go.mod b/go.mod index 8c45f65be..11a2fd42b 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,11 @@ require ( github.com/Xuanwo/templateutils v0.1.0 github.com/dave/dst v0.26.2 github.com/golang/mock v1.6.0 + github.com/google/uuid v1.3.0 github.com/kevinburke/go-bindata v3.22.0+incompatible github.com/pelletier/go-toml v1.9.4 github.com/sirupsen/logrus v1.8.1 + github.com/smartystreets/goconvey v1.6.6 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 621844135..593ea7dde 100644 --- a/go.sum +++ b/go.sum @@ -19,7 +19,13 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts= github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -39,6 +45,10 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.6 h1:lH+Snxmzl92r1jww8/jYPqKkhs3C9AF4LunzU56ZZr4= +github.com/smartystreets/goconvey v1.6.6/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= @@ -53,6 +63,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -72,6 +83,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..112a2f2c2 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,3 @@ +# Storage Integration Test + +This package designed for integration test. diff --git a/tests/appender.go b/tests/appender.go new file mode 100644 index 000000000..540b2dcf3 --- /dev/null +++ b/tests/appender.go @@ -0,0 +1,179 @@ +package tests + +import ( + "bytes" + "crypto/sha256" + "io" + "io/ioutil" + "math/rand" + "testing" + + "github.com/google/uuid" + . "github.com/smartystreets/goconvey/convey" + + "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/randbytes" + "go.beyondstorage.io/v5/types" +) + +func TestAppender(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + ap, ok := store.(types.Appender) + So(ok, ShouldBeTrue) + + Convey("When CreateAppend", func() { + path := uuid.NewString() + o, err := ap.CreateAppend(path) + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The Object Mode should be appendable", func() { + // Append object's mode must be appendable. + So(o.Mode.IsAppend(), ShouldBeTrue) + }) + }) + + Convey("When CreateAppend with an existing object", func() { + path := uuid.NewString() + o, err := ap.CreateAppend(path) + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("The first returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), size) + + _, err = ap.WriteAppend(o, r, size) + if err != nil { + t.Fatal(err) + } + + err = ap.CommitAppend(o) + if err != nil { + t.Fatal(err) + } + + o, err = ap.CreateAppend(path) + + Convey("The second returned error also should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The Object Mode should be appendable", func() { + // Append object's mode must be appendable. + So(o.Mode.IsAppend(), ShouldBeTrue) + }) + + Convey("The object append offset should be 0", func() { + So(o.MustGetAppendOffset(), ShouldBeZeroValue) + }) + }) + + Convey("When Delete", func() { + path := uuid.NewString() + _, err := ap.CreateAppend(path) + if err != nil { + t.Error(err) + } + + err = store.Delete(path) + Convey("The first returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + err = store.Delete(path) + Convey("The second returned error also should be nil", func() { + So(err, ShouldBeNil) + }) + }) + + Convey("When WriteAppend", func() { + path := uuid.NewString() + o, err := ap.CreateAppend(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, _ := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + r := bytes.NewReader(content) + + n, err := ap.WriteAppend(o, r, size) + + Convey("WriteAppend error should be nil", func() { + So(err, ShouldBeNil) + }) + Convey("WriteAppend size should be equal to n", func() { + So(n, ShouldEqual, size) + }) + }) + + Convey("When CommitAppend", func() { + path := uuid.NewString() + o, err := ap.CreateAppend(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, _ := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + + _, err = ap.WriteAppend(o, bytes.NewReader(content), size) + if err != nil { + t.Error(err) + } + + _, err = ap.WriteAppend(o, bytes.NewReader(content), size) + if err != nil { + t.Error(err) + } + + err = ap.CommitAppend(o) + + Convey("CommitAppend error should be nil", func() { + So(err, ShouldBeNil) + }) + + var buf bytes.Buffer + _, err = store.Read(path, &buf, pairs.WithSize(size*2)) + + Convey("Read error should be nil", func() { + So(err, ShouldBeNil) + }) + Convey("The content should be match", func() { + So(sha256.Sum256(buf.Bytes()), ShouldResemble, sha256.Sum256(bytes.Repeat(content, 2))) + }) + }) + }) +} diff --git a/tests/copier.go b/tests/copier.go new file mode 100644 index 000000000..5b6224adb --- /dev/null +++ b/tests/copier.go @@ -0,0 +1,245 @@ +package tests + +import ( + "bytes" + "crypto/md5" + "errors" + "io" + "io/ioutil" + "math/rand" + "testing" + + "github.com/google/uuid" + . "github.com/smartystreets/goconvey/convey" + + "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/randbytes" + "go.beyondstorage.io/v5/services" + "go.beyondstorage.io/v5/types" +) + +func TestCopier(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + c, ok := store.(types.Copier) + So(ok, ShouldBeTrue) + + Convey("When Copy a file", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, _ := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + src := uuid.New().String() + + _, err := store.Write(src, bytes.NewReader(content), size) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(src) + if err != nil { + t.Error(err) + } + }() + + dst := uuid.New().String() + err = c.Copy(src, dst) + + defer func() { + err = store.Delete(dst) + if err != nil { + t.Error(err) + } + }() + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Read should get dst object data without error", func() { + var buf bytes.Buffer + n, err := store.Read(dst, &buf) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + So(n, ShouldEqual, size) + So(md5.Sum(buf.Bytes()), ShouldResemble, md5.Sum(content)) + }) + }) + }) + + Convey("When Copy to an existing file", func() { + srcSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, _ := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), srcSize)) + src := uuid.New().String() + + _, err := store.Write(src, bytes.NewReader(content), srcSize) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(src) + if err != nil { + t.Error(err) + } + }() + + dstSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), dstSize) + dst := uuid.New().String() + + _, err = store.Write(dst, r, dstSize) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(dst) + if err != nil { + t.Error(err) + } + }() + + err = c.Copy(src, dst) + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Read should get dst object data without error", func() { + var buf bytes.Buffer + n, err := store.Read(dst, &buf) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + So(n, ShouldEqual, srcSize) + So(md5.Sum(buf.Bytes()), ShouldResemble, md5.Sum(content)) + }) + }) + }) + }) +} + +func TestCopierWithDir(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + c, ok := store.(types.Copier) + So(ok, ShouldBeTrue) + + d := store.(types.Direr) + + Convey("When Copy to an existing dir", func() { + srcSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), srcSize) + src := uuid.New().String() + + _, err := store.Write(src, r, srcSize) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(src) + if err != nil { + t.Error(err) + } + }() + + dst := uuid.New().String() + _, err = d.CreateDir(dst) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(dst, pairs.WithObjectMode(types.ModeDir)) + if err != nil { + t.Error(err) + } + }() + + err = c.Copy(src, dst) + Convey("The error should be ErrObjectModeInvalid", func() { + So(errors.Is(err, services.ErrObjectModeInvalid), ShouldBeTrue) + }) + }) + }) +} + +func TestCopierWithVirtualDir(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + c, ok := store.(types.Copier) + So(ok, ShouldBeTrue) + + d := store.(types.Direr) + + Convey("When Copy to an existing dir", func() { + srcSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), srcSize) + src := uuid.New().String() + + _, err := store.Write(src, r, srcSize) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(src) + if err != nil { + t.Error(err) + } + }() + + dst := uuid.New().String() + _, err = d.CreateDir(dst) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(dst, pairs.WithObjectMode(types.ModeDir)) + if err != nil { + t.Error(err) + } + }() + + err = c.Copy(src, dst) + + defer func() { + err = store.Delete(dst) + if err != nil { + t.Error(err) + } + }() + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Stat should get dst object without error", func() { + o, err := store.Stat(dst) + + So(err, ShouldBeNil) + So(o, ShouldNotBeNil) + + Convey("The Object Mode should be read", func() { + So(o.Mode.IsRead(), ShouldBeTrue) + }) + + Convey("The path and size should be match", func() { + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, dst) + + osize, ok := o.GetContentLength() + So(ok, ShouldBeTrue) + So(osize, ShouldEqual, srcSize) + }) + }) + }) + }) +} diff --git a/tests/direr.go b/tests/direr.go new file mode 100644 index 000000000..8cf01f1a0 --- /dev/null +++ b/tests/direr.go @@ -0,0 +1,117 @@ +package tests + +import ( + "testing" + + "github.com/google/uuid" + . "github.com/smartystreets/goconvey/convey" + + "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/types" +) + +func TestDirer(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + d, ok := store.(types.Direr) + So(ok, ShouldBeTrue) + + Convey("When CreateDir", func() { + path := uuid.New().String() + o, err := d.CreateDir(path) + + defer func() { + err := store.Delete(path, pairs.WithObjectMode(types.ModeDir)) + if err != nil { + t.Error(err) + } + }() + + Convey("The first returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + o, err = d.CreateDir(path) + Convey("The second returned error also should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The Object Path should equal to the input path", func() { + So(o.Path, ShouldEqual, path) + }) + + Convey("The Object Mode should be dir", func() { + // Dir object's mode must be Dir. + So(o.Mode.IsDir(), ShouldBeTrue) + }) + }) + + Convey("When Create with ModeDir", func() { + path := uuid.New().String() + o := store.Create(path, pairs.WithObjectMode(types.ModeDir)) + + defer func() { + err := store.Delete(path, pairs.WithObjectMode(types.ModeDir)) + if err != nil { + t.Error(err) + } + }() + + Convey("The Object Path should equal to the input path", func() { + So(o.Path, ShouldEqual, path) + }) + + Convey("The Object Mode should be dir", func() { + // Dir object's mode must be Dir. + So(o.Mode.IsDir(), ShouldBeTrue) + }) + }) + + Convey("When Stat with ModeDir", func() { + path := uuid.New().String() + _, err := d.CreateDir(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path, pairs.WithObjectMode(types.ModeDir)) + if err != nil { + t.Error(err) + } + }() + + o, err := store.Stat(path, pairs.WithObjectMode(types.ModeDir)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The Object Path should equal to the input path", func() { + So(o.Path, ShouldEqual, path) + }) + + Convey("The Object Mode should be dir", func() { + // Dir object's mode must be Dir. + So(o.Mode.IsDir(), ShouldBeTrue) + }) + }) + + Convey("When Delete with ModeDir", func() { + path := uuid.New().String() + _, err := d.CreateDir(path) + if err != nil { + t.Error(err) + } + + err = store.Delete(path, pairs.WithObjectMode(types.ModeDir)) + Convey("The first returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + err = store.Delete(path, pairs.WithObjectMode(types.ModeDir)) + Convey("The second returned error also should be nil", func() { + So(err, ShouldBeNil) + }) + }) + }) +} diff --git a/tests/linker.go b/tests/linker.go new file mode 100644 index 000000000..c6ab94fc7 --- /dev/null +++ b/tests/linker.go @@ -0,0 +1,207 @@ +package tests + +import ( + "io" + "math/rand" + "path/filepath" + "testing" + + "github.com/google/uuid" + . "github.com/smartystreets/goconvey/convey" + + "go.beyondstorage.io/v5/pkg/randbytes" + "go.beyondstorage.io/v5/types" +) + +func TestLinker(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + l, ok := store.(types.Linker) + So(ok, ShouldBeTrue) + + workDir := store.Metadata().WorkDir + + Convey("When create a link object", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), size) + target := uuid.New().String() + + _, err := store.Write(target, r, size) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(target) + if err != nil { + t.Error(err) + } + }() + + path := uuid.New().String() + o, err := l.CreateLink(path, target) + + defer func() { + err = store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The object mode should be link", func() { + // Link object's mode must be link. + So(o.Mode.IsLink(), ShouldBeTrue) + }) + + Convey("The linkTarget of the object must be the same as the target", func() { + // The linkTarget must be the same as the target. + linkTarget, ok := o.GetLinkTarget() + + So(ok, ShouldBeTrue) + So(linkTarget, ShouldEqual, filepath.Join(workDir, target)) + }) + + Convey("Stat should get path object without error", func() { + obj, err := store.Stat(path) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The object mode should be link", func() { + // Link object's mode must be link. + So(obj.Mode.IsLink(), ShouldBeTrue) + }) + + Convey("The linkTarget of the object must be the same as the target", func() { + // The linkTarget must be the same as the target. + linkTarget, ok := obj.GetLinkTarget() + + So(ok, ShouldBeTrue) + So(linkTarget, ShouldEqual, filepath.Join(workDir, target)) + }) + }) + }) + + Convey("When create a link object from a not existing target", func() { + target := uuid.New().String() + + path := uuid.New().String() + o, err := l.CreateLink(path, target) + + defer func() { + err = store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The object mode should be link", func() { + // Link object's mode must be link. + So(o.Mode.IsLink(), ShouldBeTrue) + }) + + Convey("The linkTarget of the object must be the same as the target", func() { + linkTarget, ok := o.GetLinkTarget() + + So(ok, ShouldBeTrue) + So(linkTarget, ShouldEqual, filepath.Join(workDir, target)) + }) + + Convey("Stat should get path object without error", func() { + obj, err := store.Stat(path) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The object mode should be link", func() { + // Link object's mode must be link. + So(obj.Mode.IsLink(), ShouldBeTrue) + }) + + Convey("The linkTarget of the object must be the same as the target", func() { + // The linkTarget must be the same as the target. + linkTarget, ok := obj.GetLinkTarget() + + So(ok, ShouldBeTrue) + So(linkTarget, ShouldEqual, filepath.Join(workDir, target)) + }) + }) + }) + + Convey("When CreateLink to an existing path", func() { + firstSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + firstR := io.LimitReader(randbytes.NewRand(), firstSize) + firstTarget := uuid.New().String() + + _, err := store.Write(firstTarget, firstR, firstSize) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(firstTarget) + if err != nil { + t.Error(err) + } + }() + + path := uuid.New().String() + o, err := l.CreateLink(path, firstTarget) + + defer func() { + err = store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("The first returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + secondSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + secondR := io.LimitReader(randbytes.NewRand(), secondSize) + secondTarget := uuid.New().String() + + _, err = store.Write(secondTarget, secondR, secondSize) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(secondTarget) + if err != nil { + t.Error(err) + } + }() + + o, err = l.CreateLink(path, secondTarget) + + Convey("The second returned error should also be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The object mode should be link", func() { + // Link object's mode must be link. + So(o.Mode.IsLink(), ShouldBeTrue) + }) + + Convey("The linkTarget of the object must be the same as the secondTarget", func() { + // The linkTarget must be the same as the secondTarget. + linkTarget, ok := o.GetLinkTarget() + + So(ok, ShouldBeTrue) + So(linkTarget, ShouldEqual, filepath.Join(workDir, secondTarget)) + }) + }) + }) +} diff --git a/tests/mover.go b/tests/mover.go new file mode 100644 index 000000000..e40b09fbb --- /dev/null +++ b/tests/mover.go @@ -0,0 +1,263 @@ +package tests + +import ( + "bytes" + "crypto/md5" + "errors" + "io" + "io/ioutil" + "math/rand" + "testing" + + "github.com/google/uuid" + . "github.com/smartystreets/goconvey/convey" + + "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/randbytes" + "go.beyondstorage.io/v5/services" + "go.beyondstorage.io/v5/types" +) + +func TestMover(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + m, ok := store.(types.Mover) + So(ok, ShouldBeTrue) + + Convey("When Move a file", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, _ := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + src := uuid.New().String() + + _, err := store.Write(src, bytes.NewReader(content), size) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(src) + if err != nil { + t.Error(err) + } + }() + + dst := uuid.New().String() + err = m.Move(src, dst) + + defer func() { + err = store.Delete(dst) + if err != nil { + t.Error(err) + } + }() + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Stat should get src object not exist", func() { + _, err := store.Stat(src) + + Convey("The error should be ErrObjectNotExist", func() { + So(errors.Is(err, services.ErrObjectNotExist), ShouldBeTrue) + }) + }) + + Convey("Read should get dst object data without error", func() { + var buf bytes.Buffer + n, err := store.Read(dst, &buf) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + So(n, ShouldEqual, size) + So(md5.Sum(buf.Bytes()), ShouldResemble, md5.Sum(content)) + }) + }) + }) + + Convey("When Move to an existing file", func() { + srcSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, _ := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), srcSize)) + src := uuid.New().String() + + _, err := store.Write(src, bytes.NewReader(content), srcSize) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(src) + if err != nil { + t.Error(err) + } + }() + + dstSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), dstSize) + dst := uuid.New().String() + + _, err = store.Write(dst, r, dstSize) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(dst) + if err != nil { + t.Error(err) + } + }() + + err = m.Move(src, dst) + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Stat should get src object not exist", func() { + _, err := store.Stat(src) + + Convey("The error should be ErrObjectNotExist", func() { + So(errors.Is(err, services.ErrObjectNotExist), ShouldBeTrue) + }) + }) + + Convey("Read should get dst object data without error", func() { + var buf bytes.Buffer + n, err := store.Read(dst, &buf) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + So(n, ShouldEqual, srcSize) + So(md5.Sum(buf.Bytes()), ShouldResemble, md5.Sum(content)) + }) + }) + }) + }) +} + +func TestMoverWithDir(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + m, ok := store.(types.Mover) + So(ok, ShouldBeTrue) + + d := store.(types.Direr) + + Convey("When Move to an existing dir", func() { + + srcSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), srcSize) + src := uuid.New().String() + + _, err := store.Write(src, r, srcSize) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(src) + if err != nil { + t.Error(err) + } + }() + + dst := uuid.New().String() + _, err = d.CreateDir(dst) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(dst, pairs.WithObjectMode(types.ModeDir)) + if err != nil { + t.Error(err) + } + }() + + err = m.Move(src, dst) + Convey("The error should be ErrObjectModeInvalid", func() { + So(errors.Is(err, services.ErrObjectModeInvalid), ShouldBeTrue) + }) + }) + }) +} + +func TestMoverWithVirtualDir(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + m, ok := store.(types.Mover) + So(ok, ShouldBeTrue) + + d := store.(types.Direr) + + Convey("When Move to an existing dir", func() { + + srcSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), srcSize) + src := uuid.New().String() + + _, err := store.Write(src, r, srcSize) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(src) + if err != nil { + t.Error(err) + } + }() + + dst := uuid.New().String() + _, err = d.CreateDir(dst) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = store.Delete(dst, pairs.WithObjectMode(types.ModeDir)) + if err != nil { + t.Error(err) + } + }() + + err = m.Move(src, dst) + + defer func() { + err = store.Delete(dst) + if err != nil { + t.Error(err) + } + }() + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Stat should get dst object without error", func() { + o, err := store.Stat(dst) + + So(err, ShouldBeNil) + So(o, ShouldNotBeNil) + + Convey("The Object Mode should be read", func() { + So(o.Mode.IsRead(), ShouldBeTrue) + }) + + Convey("The path and size should be match", func() { + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, dst) + + osize, ok := o.GetContentLength() + So(ok, ShouldBeTrue) + So(osize, ShouldEqual, srcSize) + }) + }) + }) + }) +} diff --git a/tests/multipart_http_signer.go b/tests/multipart_http_signer.go new file mode 100644 index 000000000..e1771f84a --- /dev/null +++ b/tests/multipart_http_signer.go @@ -0,0 +1,211 @@ +package tests + +import ( + "bytes" + "io" + "io/ioutil" + "math/rand" + "net/http" + "testing" + "time" + + "github.com/google/uuid" + . "github.com/smartystreets/goconvey/convey" + + "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/randbytes" + "go.beyondstorage.io/v5/types" +) + +func TestMultipartHTTPSigner(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + signer, ok := store.(types.MultipartHTTPSigner) + So(ok, ShouldBeTrue) + + Convey("When CreateMultipart via QuerySignHTTPCreateMultipart", func() { + path := uuid.New().String() + req, err := signer.QuerySignHTTPCreateMultipart(path, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + client := http.Client{} + _, err = client.Do(req) + + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("List with ModePart should get the object without error", func() { + it, err := store.List(path, pairs.WithListMode(types.ListModePart)) + + So(err, ShouldBeNil) + + o, err := it.Next() + So(err, ShouldBeNil) + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, path) + }) + + defer func() { + it, err := store.List(path, pairs.WithListMode(types.ListModePart)) + if err != nil { + t.Error(err) + } + + o, err := it.Next() + if err != nil { + t.Error(err) + } + + err = store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + if err != nil { + t.Error(err) + } + }() + }) + + Convey("When WriteMultipart via QuerySignHTTPWriteMultipart", func() { + path := uuid.New().String() + o, err := store.(types.Multiparter).CreateMultipart(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) + content, err := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + if err != nil { + t.Error(err) + } + + req, err := signer.QuerySignHTTPWriteMultipart(o, size, 0, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + req.Body = ioutil.NopCloser(bytes.NewReader(content)) + + client := http.Client{} + resp, err := client.Do(req) + + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + }) + + Convey("The size should be match", func() { + So(resp.Request.ContentLength, ShouldEqual, size) + }) + }) + + Convey("When ListMultiPart via QuerySignHTTPListMultiPart", func() { + mu, ok := store.(types.Multiparter) + So(ok, ShouldBeTrue) + + path := uuid.New().String() + o, err := mu.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + partNumber := rand.Intn(1000) // Choose a random part number from [0, 1000) + r := io.LimitReader(randbytes.NewRand(), size) + + _, _, err = mu.WriteMultipart(o, r, size, partNumber) + if err != nil { + t.Error(err) + } + + req, err := signer.QuerySignHTTPListMultipart(o, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + client := http.Client{} + _, err = client.Do(req) + + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + }) + + Convey("When CompletePart via QuerySignHTTPCompletePart", func() { + mu, ok := store.(types.Multiparter) + So(ok, ShouldBeTrue) + + path := uuid.New().String() + o, err := mu.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + // Set 0 to `partNumber` here as the part numbers must be continuous for `CompleteMultipartUpload` in `cos` which is different with other storages. + partNumber := 0 + r := io.LimitReader(randbytes.NewRand(), size) + + _, part, err := mu.WriteMultipart(o, r, size, partNumber) + if err != nil { + t.Error(err) + } + + req, err := signer.QuerySignHTTPCompleteMultipart(o, []*types.Part{part}, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + client := http.Client{} + _, err = client.Do(req) + + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The object should be readable after complete", func() { + ro, err := store.Stat(path) + + So(err, ShouldBeNil) + So(ro.Mode.IsRead(), ShouldBeTrue) + So(ro.Mode.IsPart(), ShouldBeFalse) + }) + }) + }) +} diff --git a/tests/multiparter.go b/tests/multiparter.go new file mode 100644 index 000000000..f6076b72c --- /dev/null +++ b/tests/multiparter.go @@ -0,0 +1,311 @@ +package tests + +import ( + "io" + "math/rand" + "testing" + + "github.com/google/uuid" + . "github.com/smartystreets/goconvey/convey" + + "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/randbytes" + "go.beyondstorage.io/v5/types" +) + +func TestMultiparter(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + m, ok := store.(types.Multiparter) + So(ok, ShouldBeTrue) + + Convey("When CreateMultipart", func() { + path := uuid.New().String() + o, err := m.CreateMultipart(path) + + Convey("The first returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + defer func(multipartID string) { + err := store.Delete(path, pairs.WithMultipartID(multipartID)) + if err != nil { + t.Error(err) + } + }(o.MustGetMultipartID()) + + o, err = m.CreateMultipart(path) + + Convey("The second returned error also should be nil", func() { + So(err, ShouldBeNil) + }) + + defer func() { + err := store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + if err != nil { + t.Error(err) + } + }() + + Convey("The Object Mode should be part", func() { + // Multipart object's mode must be Part. + So(o.Mode.IsPart(), ShouldBeTrue) + // Multipart object's mode must not be Read. + So(o.Mode.IsRead(), ShouldBeFalse) + }) + + Convey("The Object must have multipart id", func() { + // Multipart object must have multipart id. + _, ok := o.GetMultipartID() + So(ok, ShouldBeTrue) + }) + }) + + Convey("When Delete with multipart id", func() { + path := uuid.New().String() + o, err := m.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + err = store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + Convey("The first returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + err = store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + Convey("The second returned error also should be nil", func() { + So(err, ShouldBeNil) + }) + }) + + Convey("When Stat with multipart id", func() { + path := uuid.New().String() + o, err := m.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + multipartId := o.MustGetMultipartID() + + defer func() { + err := store.Delete(path, pairs.WithMultipartID(multipartId)) + if err != nil { + t.Error(err) + } + }() + + mo, err := store.Stat(path, pairs.WithMultipartID(multipartId)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + So(mo, ShouldNotBeNil) + }) + + Convey("The Object Mode should be part", func() { + // Multipart object's mode must be Part. + So(mo.Mode.IsPart(), ShouldBeTrue) + // Multipart object's mode must not be Read. + So(mo.Mode.IsRead(), ShouldBeFalse) + }) + + Convey("The Object must have multipart id", func() { + // Multipart object must have multipart id. + mid, ok := mo.GetMultipartID() + So(ok, ShouldBeTrue) + So(mid, ShouldEqual, multipartId) + }) + }) + + Convey("When Create with multipart id", func() { + path := uuid.New().String() + o, err := m.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + multipartId := o.MustGetMultipartID() + + defer func() { + err := store.Delete(path, pairs.WithMultipartID(multipartId)) + if err != nil { + t.Error(err) + } + }() + + mo := store.Create(path, pairs.WithMultipartID(multipartId)) + + Convey("The Object Mode should be part", func() { + // Multipart object's mode must be Part. + So(mo.Mode.IsPart(), ShouldBeTrue) + // Multipart object's mode must not be Read. + So(mo.Mode.IsRead(), ShouldBeFalse) + }) + + Convey("The Object must have multipart id", func() { + // Multipart object must have multipart id. + mid, ok := mo.GetMultipartID() + So(ok, ShouldBeTrue) + So(mid, ShouldEqual, multipartId) + }) + }) + + Convey("When WriteMultipart", func() { + path := uuid.New().String() + o, err := m.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), size) + + n, part, err := m.WriteMultipart(o, r, size, 0) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The part should not be nil", func() { + So(part, ShouldNotBeNil) + }) + + Convey("The size should be match", func() { + So(n, ShouldEqual, size) + }) + }) + + Convey("When ListMultiPart", func() { + path := uuid.New().String() + o, err := m.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + partNumber := rand.Intn(1000) // Choose a random part number from [0, 1000) + r := io.LimitReader(randbytes.NewRand(), size) + + _, _, err = m.WriteMultipart(o, r, size, partNumber) + if err != nil { + t.Error(err) + } + + it, err := m.ListMultipart(o) + + Convey("ListMultipart error should be nil", func() { + So(err, ShouldBeNil) + So(it, ShouldNotBeNil) + }) + + p, err := it.Next() + Convey("Next error should be nil", func() { + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + }) + Convey("The part number and size should be match", func() { + So(p.Index, ShouldEqual, partNumber) + So(p.Size, ShouldEqual, size) + }) + }) + + Convey("When List with part type", func() { + path := uuid.New().String() + o, err := m.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + partNumber := rand.Intn(1000) // Choose a random part number from [0, 1000) + r := io.LimitReader(randbytes.NewRand(), size) + + _, _, err = m.WriteMultipart(o, r, size, partNumber) + if err != nil { + t.Error(err) + } + + it, err := store.List("", pairs.WithListMode(types.ListModePart)) + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + Convey("The iterator should not be nil", func() { + So(it, ShouldNotBeNil) + }) + + mo, err := it.Next() + Convey("Next error should be nil", func() { + So(err, ShouldBeNil) + So(mo, ShouldNotBeNil) + }) + Convey("The path and multipart id should be match", func() { + So(mo.Path, ShouldEqual, path) + So(mo.Mode.IsPart(), ShouldBeTrue) + + // Multipart object must have multipart id. + mid, ok := mo.GetMultipartID() + So(ok, ShouldBeTrue) + So(mid, ShouldEqual, o.MustGetMultipartID()) + }) + }) + + Convey("When CompletePart", func() { + path := uuid.New().String() + o, err := m.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + // Set 0 to `partNumber` here as the part numbers must be continuous for `CompleteMultipartUpload` in `cos` which is different with other storages. + partNumber := 0 + r := io.LimitReader(randbytes.NewRand(), size) + + _, part, err := m.WriteMultipart(o, r, size, partNumber) + if err != nil { + t.Error(err) + } + + err = m.CompleteMultipart(o, []*types.Part{part}) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The object should be readable after complete", func() { + ro, err := store.Stat(path) + + So(err, ShouldBeNil) + So(ro.Mode.IsRead(), ShouldBeTrue) + So(ro.Mode.IsPart(), ShouldBeFalse) + }) + }) + }) +} diff --git a/tests/storage_http_signer.go b/tests/storage_http_signer.go new file mode 100644 index 000000000..452026850 --- /dev/null +++ b/tests/storage_http_signer.go @@ -0,0 +1,201 @@ +package tests + +import ( + "bytes" + "crypto/sha256" + "errors" + "io" + "io/ioutil" + "math/rand" + "net/http" + "testing" + "time" + + "github.com/google/uuid" + . "github.com/smartystreets/goconvey/convey" + + "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/randbytes" + "go.beyondstorage.io/v5/services" + "go.beyondstorage.io/v5/types" +) + +func TestStorageHTTPSignerRead(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + signer, ok := store.(types.StorageHTTPSigner) + So(ok, ShouldBeTrue) + + Convey("When Read via QuerySignHTTPRead", func() { + size := rand.Int63n(4 * 1024 * 1024) + content, err := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + if err != nil { + t.Error(err) + } + + path := uuid.New().String() + _, err = store.Write(path, bytes.NewReader(content), size) + if err != nil { + t.Error(err) + } + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + req, err := signer.QuerySignHTTPRead(path, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + client := http.Client{} + resp, err := client.Do(req) + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + }) + + defer resp.Body.Close() + + buf, err := ioutil.ReadAll(resp.Body) + Convey("The content should be match", func() { + So(err, ShouldBeNil) + So(buf, ShouldNotBeNil) + + So(resp.ContentLength, ShouldEqual, size) + So(sha256.Sum256(buf), ShouldResemble, sha256.Sum256(content)) + }) + }) + }) +} + +func TestStorageHTTPSignerWrite(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + signer, ok := store.(types.StorageHTTPSigner) + So(ok, ShouldBeTrue) + + Convey("When Write via QuerySignHTTPWrite", func() { + size := rand.Int63n(4 * 1024 * 1024) + content, err := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + if err != nil { + t.Error(err) + } + + path := uuid.New().String() + req, err := signer.QuerySignHTTPWrite(path, size, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + req.Body = ioutil.NopCloser(bytes.NewReader(content)) + + client := http.Client{} + _, err = client.Do(req) + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("Read should get object data without error", func() { + var buf bytes.Buffer + n, err := store.Read(path, &buf) + + Convey("The content should be match", func() { + So(err, ShouldBeNil) + So(buf, ShouldNotBeNil) + + So(n, ShouldEqual, size) + So(sha256.Sum256(buf.Bytes()), ShouldResemble, sha256.Sum256(content)) + }) + }) + }) + }) +} + +func TestStorageHTTPSignerDelete(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + signer, ok := store.(types.StorageHTTPSigner) + So(ok, ShouldBeTrue) + + Convey("When Delete via QuerySignHTTPDelete", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), size) + + path := uuid.New().String() + _, err := store.Write(path, r, size) + if err != nil { + t.Error(err) + } + + req, err := signer.QuerySignHTTPDelete(path, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + client := http.Client{} + _, err = client.Do(req) + + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Stat should get nil Object and ObjectNotFound error", func() { + o, err := store.Stat(path) + + So(errors.Is(err, services.ErrObjectNotExist), ShouldBeTrue) + So(o, ShouldBeNil) + }) + }) + + Convey("When Delete with multipart id via QuerySignHTTPDelete", func() { + mu, ok := store.(types.Multiparter) + So(ok, ShouldBeTrue) + + path := uuid.New().String() + o, err := mu.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + req, err := signer.QuerySignHTTPDelete(path, time.Duration(time.Hour), pairs.WithMultipartID(o.MustGetMultipartID())) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + client := http.Client{} + _, err = client.Do(req) + + Convey("The first request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + _, err = client.Do(req) + + Convey("The second request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + }) + }) +} diff --git a/tests/storager.go b/tests/storager.go new file mode 100644 index 000000000..7a7ac7ef9 --- /dev/null +++ b/tests/storager.go @@ -0,0 +1,663 @@ +package tests + +import ( + "bytes" + "crypto/sha256" + "errors" + "io" + "io/ioutil" + "math/rand" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + . "github.com/smartystreets/goconvey/convey" + + ps "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/randbytes" + "go.beyondstorage.io/v5/services" + "go.beyondstorage.io/v5/types" +) + +func TestStorager(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + So(store, ShouldNotBeNil) + + Convey("When String called", func() { + s := store.String() + + Convey("The string should not be empty", func() { + So(s, ShouldNotBeEmpty) + }) + }) + + Convey("When Metadata called", func() { + m := store.Metadata() + + Convey("The metadata should not be empty", func() { + So(m, ShouldNotBeEmpty) + }) + }) + + workDir := store.Metadata().WorkDir + + Convey("When Read a file", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, err := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + if err != nil { + t.Error(err) + } + + path := uuid.New().String() + _, err = store.Write(path, bytes.NewReader(content), size) + if err != nil { + t.Error(err) + } + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + var buf bytes.Buffer + + n, err := store.Read(path, &buf) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + + So(n, ShouldEqual, size) + So(sha256.Sum256(buf.Bytes()), ShouldResemble, sha256.Sum256(content)) + }) + }) + + Convey("When Read a file with offset or size", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, err := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + if err != nil { + t.Error(err) + } + + path := uuid.New().String() + _, err = store.Write(path, bytes.NewReader(content), size) + if err != nil { + t.Error(err) + } + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("When Read with offset", func() { + offset := rand.Int63n(size) + + var buf bytes.Buffer + n, err := store.Read(path, &buf, ps.WithOffset(offset)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + + So(n, ShouldEqual, size-offset) + So(sha256.Sum256(buf.Bytes()), ShouldResemble, sha256.Sum256(content[offset:])) + }) + }) + + Convey("When Read with size", func() { + len := rand.Int63n(size) + + var buf bytes.Buffer + n, err := store.Read(path, &buf, ps.WithSize(len)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + + So(n, ShouldEqual, len) + So(sha256.Sum256(buf.Bytes()), ShouldResemble, sha256.Sum256(content[:len])) + }) + }) + + Convey("When Read with offset and size", func() { + offset := rand.Int63n(size) + len := rand.Int63n(size - offset) + + var buf bytes.Buffer + n, err := store.Read(path, &buf, ps.WithOffset(offset), ps.WithSize(len)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + + So(n, ShouldEqual, len) + So(sha256.Sum256(buf.Bytes()), ShouldResemble, sha256.Sum256(content[offset:offset+len])) + }) + }) + }) + + Convey("When Write a file", func() { + firstSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), firstSize) + path := uuid.New().String() + + _, err := store.Write(path, r, firstSize) + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("The first returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + secondSize := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, _ := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), secondSize)) + + _, err = store.Write(path, bytes.NewReader(content), secondSize) + + Convey("The second returned error also should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Stat should get Object without error", func() { + o, err := store.Stat(path) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The name and size should be match", func() { + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, path) + + osize, ok := o.GetContentLength() + So(ok, ShouldBeTrue) + So(osize, ShouldEqual, secondSize) + }) + }) + + Convey("Read should get Object data without error", func() { + var buf bytes.Buffer + n, err := store.Read(path, &buf) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + + So(n, ShouldEqual, secondSize) + So(sha256.Sum256(buf.Bytes()), ShouldResemble, sha256.Sum256(content)) + }) + }) + }) + + Convey("When Write and Read a file with IoCallback", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, err := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + if err != nil { + t.Error(err) + } + + path := uuid.New().String() + + curWrite := int64(0) + writeFn := func(bs []byte) { + curWrite += int64(len(bs)) + } + _, err = store.Write(path, bytes.NewReader(content), size, ps.WithIoCallback(writeFn)) + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("The error returned by Write should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The write size should be match", func() { + So(curWrite, ShouldEqual, size) + }) + + curRead := int64(0) + readFn := func(bs []byte) { + curRead += int64(len(bs)) + } + var buf bytes.Buffer + n, err := store.Read(path, &buf, ps.WithIoCallback(readFn)) + + Convey("The error returned be Read should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The read size should be match", func() { + So(curRead, ShouldEqual, n) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + + So(n, ShouldEqual, size) + So(sha256.Sum256(buf.Bytes()), ShouldResemble, sha256.Sum256(content)) + }) + }) + + Convey("When write a file with a nil io.Reader and 0 size", func() { + path := uuid.New().String() + var size int64 = 0 + + _, err := store.Write(path, nil, size) + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Stat should get Object without error", func() { + o, err := store.Stat(path) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The name and size should be match", func() { + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, path) + + osize, ok := o.GetContentLength() + So(ok, ShouldBeTrue) + So(osize, ShouldEqual, size) + }) + }) + }) + + Convey("When write a file with a nil io.Reader and valid size", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + path := uuid.New().String() + + _, err := store.Write(path, nil, size) + + Convey("The error should not be nil", func() { + So(err, ShouldNotBeNil) + }) + + Convey("Stat should get nil Object and ObjectNotFound error", func() { + o, err := store.Stat(path) + + So(errors.Is(err, services.ErrObjectNotExist), ShouldBeTrue) + So(o, ShouldBeNil) + }) + }) + + Convey("When write a file with a valid io.Reader and 0 size", func() { + var size int64 = 0 + n := rand.Int63n(4 * 1024 * 1024) + r := io.LimitReader(randbytes.NewRand(), n) + path := uuid.New().String() + + _, err := store.Write(path, r, size) + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Stat should get Object without error", func() { + o, err := store.Stat(path) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The name and size should be match", func() { + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, path) + + osize, ok := o.GetContentLength() + So(ok, ShouldBeTrue) + So(osize, ShouldEqual, size) + }) + }) + }) + + Convey("When write a file with a valid io.Reader and length greater than size", func() { + n := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + size := rand.Int63n(n) + r, _ := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), n)) + path := uuid.New().String() + + _, err := store.Write(path, bytes.NewReader(r), size) + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Stat should get Object without error", func() { + o, err := store.Stat(path) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The name and size should be match", func() { + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, path) + + osize, ok := o.GetContentLength() + So(ok, ShouldBeTrue) + So(osize, ShouldEqual, size) + }) + }) + + Convey("Read should get Object without error", func() { + content, _ := ioutil.ReadAll(io.LimitReader(bytes.NewReader(r), size)) + var buf bytes.Buffer + n, err := store.Read(path, &buf) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should match the size limit of the content", func() { + So(buf, ShouldNotBeNil) + + So(n, ShouldEqual, size) + So(sha256.Sum256(buf.Bytes()), ShouldResemble, sha256.Sum256(content)) + }) + }) + }) + + Convey("When Stat a file", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, err := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + if err != nil { + t.Error(err) + } + + path := uuid.New().String() + _, err = store.Write(path, bytes.NewReader(content), size) + if err != nil { + t.Error(err) + } + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + o, err := store.Stat(path) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The Object name and size should be match", func() { + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, path) + + osize, ok := o.GetContentLength() + So(ok, ShouldBeTrue) + So(osize, ShouldEqual, size) + }) + }) + + Convey("When Delete a file", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, err := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + if err != nil { + t.Error(err) + } + + path := uuid.New().String() + _, err = store.Write(path, bytes.NewReader(content), size) + if err != nil { + t.Error(err) + } + + err = store.Delete(path) + + Convey("The first returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + err = store.Delete(path) + + Convey("The second returned error also should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Stat should get nil Object and ObjectNotFound error", func() { + o, err := store.Stat(path) + + So(errors.Is(err, services.ErrObjectNotExist), ShouldBeTrue) + So(o, ShouldBeNil) + }) + }) + + Convey("When List an empty dir", func() { + it, err := store.List("", ps.WithListMode(types.ListModeDir)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + Convey("The iterator should not be nil", func() { + So(it, ShouldNotBeNil) + }) + + o, err := it.Next() + + Convey("The next should be done", func() { + So(err, ShouldBeError, types.IterateDone) + }) + Convey("The object should be nil", func() { + So(o, ShouldBeNil) + }) + }) + + Convey("When List a dir within files", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), size) + path := uuid.New().String() + _, err := store.Write(path, r, size) + if err != nil { + t.Error(err) + } + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + it, err := store.List("", ps.WithListMode(types.ListModeDir)) + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + Convey("The iterator should not be nil", func() { + So(it, ShouldNotBeNil) + }) + + o, err := it.Next() + Convey("The name and size should be match", func() { + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, path) + + osize, ok := o.GetContentLength() + So(ok, ShouldBeTrue) + So(osize, ShouldEqual, size) + }) + }) + + Convey("When List without ListMode", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), size) + path := uuid.New().String() + _, err := store.Write(path, r, size) + if err != nil { + t.Error(err) + } + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + it, err := store.List("") + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + Convey("The iterator should not be nil", func() { + So(it, ShouldNotBeNil) + }) + + o, err := it.Next() + Convey("The name and size should be match", func() { + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, path) + + osize, ok := o.GetContentLength() + So(ok, ShouldBeTrue) + So(osize, ShouldEqual, size) + }) + }) + + Convey("When testing GSP-749 unify path behavior", func() { + Convey("When using absolute path", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, err := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + if err != nil { + t.Error(err) + } + + path := uuid.New().String() + absPath := filepath.Join(workDir, path) + _, err = store.Write(absPath, bytes.NewReader(content), size) + if err != nil { + t.Error(err) + } + defer func() { + err := store.Delete(absPath) + if err != nil { + t.Error(err) + } + }() + + Convey("Stat should get Object without error", func() { + o, err := store.Stat(absPath) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, strings.ReplaceAll(absPath, "\\", "/")) + }) + }) + + Convey("Read should get Object content without error", func() { + var buf bytes.Buffer + n, err := store.Read(absPath, &buf) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + + So(n, ShouldEqual, size) + So(sha256.Sum256(buf.Bytes()), ShouldResemble, sha256.Sum256(content)) + }) + }) + }) + + Convey("When using backslash in path", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + content, err := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + if err != nil { + t.Error(err) + } + + path := uuid.New().String() + "\\" + uuid.New().String() + _, err = store.Write(path, bytes.NewReader(content), size) + if err != nil { + t.Error(err) + } + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + Convey("Stat should get Object without error", func() { + o, err := store.Stat(path) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, strings.ReplaceAll(path, "\\", "/")) + }) + }) + + Convey("Read should get Object content without error", func() { + var buf bytes.Buffer + n, err := store.Read(path, &buf) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The content should be match", func() { + So(buf, ShouldNotBeNil) + + So(n, ShouldEqual, size) + So(sha256.Sum256(buf.Bytes()), ShouldResemble, sha256.Sum256(content)) + }) + }) + }) + }) + }) +}