diff --git a/go.mod b/go.mod index dbb9983..60d1617 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/beyondstorage/go-integration-test/v4 go 1.15 require ( - github.com/beyondstorage/go-storage/v4 v4.7.0 + github.com/beyondstorage/go-storage/v4 v4.8.0 github.com/google/uuid v1.3.0 github.com/smartystreets/goconvey v1.6.4 ) diff --git a/go.sum b/go.sum index 31a2faa..d905417 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,10 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Xuanwo/gg v0.2.0/go.mod h1:0fLiiSxR87u2UA0ZNZiKZXuz3jnJdbDHWtU2xpdcH3s= +github.com/Xuanwo/go-bufferpool v0.2.0/go.mod h1:Mle++9GGouhOwGj52i9PJLNAPmW2nb8PWBP7JJzNCzk= github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= -github.com/beyondstorage/go-storage/v4 v4.7.0 h1:7hpRFNoPY0vWRSH/p2biD2dUQdZMrs1TuIkkqDefmCE= -github.com/beyondstorage/go-storage/v4 v4.7.0/go.mod h1:mc9VzBImjXDg1/1sLfta2MJH79elfM6m47ZZvZ+q/Uw= +github.com/beyondstorage/go-storage/v4 v4.8.0 h1:f0vC20h0deCrsid/nIApKGFw37Fv4VslpQBFtbNRJNU= +github.com/beyondstorage/go-storage/v4 v4.8.0/go.mod h1:zknx9z1WOSs0ow8eg8iuTgvMCqNnNJWxB2S9cAvhitM= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -24,10 +28,12 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -37,6 +43,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= @@ -75,5 +82,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/multipart_http_signer.go b/multipart_http_signer.go new file mode 100644 index 0000000..795d07e --- /dev/null +++ b/multipart_http_signer.go @@ -0,0 +1,211 @@ +package tests + +import ( + "bytes" + "io" + "io/ioutil" + "math/rand" + "net/http" + "testing" + "time" + + "github.com/google/uuid" + . "github.com/smartystreets/goconvey/convey" + + "github.com/beyondstorage/go-storage/v4/pairs" + "github.com/beyondstorage/go-storage/v4/pkg/randbytes" + "github.com/beyondstorage/go-storage/v4/types" +) + +func TestMultipartHTTPSigner(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + signer, ok := store.(types.MultipartHTTPSigner) + So(ok, ShouldBeTrue) + + Convey("When CreateMultipart via QuerySignHTTPCreateMultipart", func() { + path := uuid.New().String() + req, err := signer.QuerySignHTTPCreateMultipart(path, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + client := http.Client{} + _, err = client.Do(req) + + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("List with ModePart should get the object without error", func() { + it, err := store.List(path, pairs.WithListMode(types.ListModePart)) + + So(err, ShouldBeNil) + + o, err := it.Next() + So(err, ShouldBeNil) + So(o, ShouldNotBeNil) + So(o.Path, ShouldEqual, path) + }) + + defer func() { + it, err := store.List(path, pairs.WithListMode(types.ListModePart)) + if err != nil { + t.Error(err) + } + + o, err := it.Next() + if err != nil { + t.Error(err) + } + + err = store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + if err != nil { + t.Error(err) + } + }() + }) + + Convey("When WriteMultipart via QuerySignHTTPWriteMultipart", func() { + path := uuid.New().String() + o, err := store.(types.Multiparter).CreateMultipart(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) + content, err := ioutil.ReadAll(io.LimitReader(randbytes.NewRand(), size)) + if err != nil { + t.Error(err) + } + + req, err := signer.QuerySignHTTPWriteMultipart(o, size, 0, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + req.Body = ioutil.NopCloser(bytes.NewReader(content)) + + client := http.Client{} + resp, err := client.Do(req) + + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + }) + + Convey("The size should be match", func() { + So(resp.Request.ContentLength, ShouldEqual, size) + }) + }) + + Convey("When ListMultiPart via QuerySignHTTPListMultiPart", func() { + mu, ok := store.(types.Multiparter) + So(ok, ShouldBeTrue) + + path := uuid.New().String() + o, err := mu.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path, pairs.WithMultipartID(o.MustGetMultipartID())) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + partNumber := rand.Intn(1000) // Choose a random part number from [0, 1000) + r := io.LimitReader(randbytes.NewRand(), size) + + _, _, err = mu.WriteMultipart(o, r, size, partNumber) + if err != nil { + t.Error(err) + } + + req, err := signer.QuerySignHTTPListMultipart(o, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + client := http.Client{} + _, err = client.Do(req) + + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + }) + + Convey("When CompletePart via QuerySignHTTPCompletePart", func() { + mu, ok := store.(types.Multiparter) + So(ok, ShouldBeTrue) + + path := uuid.New().String() + o, err := mu.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + defer func() { + err := store.Delete(path) + if err != nil { + t.Error(err) + } + }() + + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + // Set 0 to `partNumber` here as the part numbers must be continuous for `CompleteMultipartUpload` in `cos` which is different with other storages. + partNumber := 0 + r := io.LimitReader(randbytes.NewRand(), size) + + _, part, err := mu.WriteMultipart(o, r, size, partNumber) + if err != nil { + t.Error(err) + } + + req, err := signer.QuerySignHTTPCompleteMultipart(o, []*types.Part{part}, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + client := http.Client{} + _, err = client.Do(req) + + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("The object should be readable after complete", func() { + ro, err := store.Stat(path) + + So(err, ShouldBeNil) + So(ro.Mode.IsRead(), ShouldBeTrue) + So(ro.Mode.IsPart(), ShouldBeFalse) + }) + }) + }) +} diff --git a/storage_http_signer.go b/storage_http_signer.go index 2fe0b42..9ef9474 100644 --- a/storage_http_signer.go +++ b/storage_http_signer.go @@ -3,6 +3,7 @@ package tests import ( "bytes" "crypto/sha256" + "errors" "io" "io/ioutil" "math/rand" @@ -13,7 +14,9 @@ import ( "github.com/google/uuid" . "github.com/smartystreets/goconvey/convey" + "github.com/beyondstorage/go-storage/v4/pairs" "github.com/beyondstorage/go-storage/v4/pkg/randbytes" + "github.com/beyondstorage/go-storage/v4/services" "github.com/beyondstorage/go-storage/v4/types" ) @@ -122,3 +125,77 @@ func TestStorageHTTPSignerWrite(t *testing.T, store types.Storager) { }) }) } + +func TestStorageHTTPSignerDelete(t *testing.T, store types.Storager) { + Convey("Given a basic Storager", t, func() { + signer, ok := store.(types.StorageHTTPSigner) + So(ok, ShouldBeTrue) + + Convey("When Delete via QuerySignHTTPDelete", func() { + size := rand.Int63n(4 * 1024 * 1024) // Max file size is 4MB + r := io.LimitReader(randbytes.NewRand(), size) + + path := uuid.New().String() + _, err := store.Write(path, r, size) + if err != nil { + t.Error(err) + } + + req, err := signer.QuerySignHTTPDelete(path, time.Duration(time.Hour)) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + client := http.Client{} + _, err = client.Do(req) + + Convey("The request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + Convey("Stat should get nil Object and ObjectNotFound error", func() { + o, err := store.Stat(path) + + So(errors.Is(err, services.ErrObjectNotExist), ShouldBeTrue) + So(o, ShouldBeNil) + }) + }) + + Convey("When Delete with multipart id via QuerySignHTTPDelete", func() { + mu, ok := store.(types.Multiparter) + So(ok, ShouldBeTrue) + + path := uuid.New().String() + o, err := mu.CreateMultipart(path) + if err != nil { + t.Error(err) + } + + req, err := signer.QuerySignHTTPDelete(path, time.Duration(time.Hour), pairs.WithMultipartID(o.MustGetMultipartID())) + + Convey("The error should be nil", func() { + So(err, ShouldBeNil) + + So(req, ShouldNotBeNil) + So(req.URL, ShouldNotBeNil) + }) + + client := http.Client{} + _, err = client.Do(req) + + Convey("The first request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + + _, err = client.Do(req) + + Convey("The second request returned error should be nil", func() { + So(err, ShouldBeNil) + }) + }) + }) +}