From 5e412d66d24c7eeaff0e6619ad29ff796c522fc5 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Fri, 23 Apr 2021 14:26:38 +0800 Subject: [PATCH 1/4] Implement appender support --- generated.go | 130 +++++++++++++++++++++++++++++++++++++++--- go.mod | 4 +- go.sum | 20 +++---- service.toml | 3 + storage.go | 46 +++++++++++++++ tests/storage_test.go | 4 ++ utils.go | 1 + 7 files changed, 188 insertions(+), 20 deletions(-) diff --git a/generated.go b/generated.go index 3f21b50..d79036f 100644 --- a/generated.go +++ b/generated.go @@ -416,13 +416,15 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { // DefaultStoragePairs is default pairs for specific action type DefaultStoragePairs struct { - Create []Pair - Delete []Pair - List []Pair - Metadata []Pair - Read []Pair - Stat []Pair - Write []Pair + Create []Pair + CreateAppend []Pair + Delete []Pair + List []Pair + Metadata []Pair + Read []Pair + Stat []Pair + Write []Pair + WriteAppend []Pair } // pairStorageCreate is the parsed struct @@ -457,6 +459,38 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) return result, nil } +// pairStorageCreateAppend is the parsed struct +type pairStorageCreateAppend struct { + pairs []Pair + + // Required pairs + // Optional pairs + // Generated pairs +} + +// parsePairStorageCreateAppend will parse Pair slice into *pairStorageCreateAppend +func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAppend, error) { + result := pairStorageCreateAppend{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + // Required pairs + // Optional pairs + // Generated pairs + default: + + if s.pairPolicy.All || s.pairPolicy.CreateAppend { + return pairStorageCreateAppend{}, services.NewPairUnsupportedError(v) + } + + } + } + + return result, nil +} + // pairStorageDelete is the parsed struct type pairStorageDelete struct { pairs []Pair @@ -689,6 +723,38 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { return result, nil } +// pairStorageWriteAppend is the parsed struct +type pairStorageWriteAppend struct { + pairs []Pair + + // Required pairs + // Optional pairs + // Generated pairs +} + +// parsePairStorageWriteAppend will parse Pair slice into *pairStorageWriteAppend +func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppend, error) { + result := pairStorageWriteAppend{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + // Required pairs + // Optional pairs + // Generated pairs + default: + + if s.pairPolicy.All || s.pairPolicy.WriteAppend { + return pairStorageWriteAppend{}, services.NewPairUnsupportedError(v) + } + + } + } + + return result, nil +} + // Create will create a new object without any api call. // // This function will create a context by default. @@ -702,6 +768,31 @@ func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { return s.create(path, opt) } +// CreateAppend will create an append object. +// +// This function will create a context by default. +func (s *Storage) CreateAppend(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateAppendWithContext(ctx, path, pairs...) +} + +// CreateAppendWithContext will create an append object. +func (s *Storage) CreateAppendWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + pairs = append(pairs, s.defaultPairs.CreateAppend...) + var opt pairStorageCreateAppend + + defer func() { + err = s.formatError("create_append", err, path) + }() + + opt, err = s.parsePairStorageCreateAppend(pairs) + if err != nil { + return + } + + return s.createAppend(ctx, path, opt) +} + // Delete will delete an Object from service. // // This function will create a context by default. @@ -851,3 +942,28 @@ func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader return s.write(ctx, path, r, size, opt) } + +// WriteAppend will append content to an append object. +// +// This function will create a context by default. +func (s *Storage) WriteAppend(o *Object, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.WriteAppendWithContext(ctx, o, r, size, pairs...) +} + +// WriteAppendWithContext will append content to an append object. +func (s *Storage) WriteAppendWithContext(ctx context.Context, o *Object, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + pairs = append(pairs, s.defaultPairs.WriteAppend...) + var opt pairStorageWriteAppend + + defer func() { + err = s.formatError("write_append", err) + }() + + opt, err = s.parsePairStorageWriteAppend(pairs) + if err != nil { + return + } + + return s.writeAppend(ctx, o, r, size, opt) +} diff --git a/go.mod b/go.mod index 1efc0f4..cfea870 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.14 require ( github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible - github.com/aos-dev/go-integration-test/v3 v3.0.0-20210205075620-0b779f4b3afc - github.com/aos-dev/go-storage/v3 v3.4.3-0.20210417162535-67db0dd18784 + github.com/aos-dev/go-integration-test/v3 v3.0.0 + github.com/aos-dev/go-storage/v3 v3.5.1-0.20210422060150-bc0fae4c3fa7 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/google/uuid v1.2.0 github.com/satori/go.uuid v1.2.0 // indirect diff --git a/go.sum b/go.sum index 0d13e15..dc699eb 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,15 @@ -github.com/Xuanwo/templateutils v0.0.0-20201019101431-c73b871118f2/go.mod h1:x0qS7gfgEm24b8V9U+0zBEBAu/VpxZhO/+EWtpgEgDw= github.com/Xuanwo/templateutils v0.0.0-20201216100309-46f73cd4e4b1 h1:SQ8kQujQ0yl39YBMU0xZ+yyfxx/qSWfGWlCPNzHp5kw= github.com/Xuanwo/templateutils v0.0.0-20201216100309-46f73cd4e4b1/go.mod h1:x0qS7gfgEm24b8V9U+0zBEBAu/VpxZhO/+EWtpgEgDw= github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible h1:hLUNPbx10wawWW7DeNExvTrlb90db3UnnNTFKHZEFhE= github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aos-dev/go-integration-test/v3 v3.0.0-20210205075620-0b779f4b3afc h1:s8BQqJwWe2yoTGJFOsIxnkwwMsnDqMgkO39yzEP4smw= -github.com/aos-dev/go-integration-test/v3 v3.0.0-20210205075620-0b779f4b3afc/go.mod h1:2VqxZhJVvI1jKTTERWHfCBMZQaJt10xoEJ+CxbfYZlo= -github.com/aos-dev/go-storage/v3 v3.0.1-0.20210205074802-e8a5b22166c2/go.mod h1:JznhvyhPDnETfZ3RyWm3mT/S5ic+uuYFWagSIkUZujc= -github.com/aos-dev/go-storage/v3 v3.4.3-0.20210417162535-67db0dd18784 h1:eeKDDn+8RPtYuk7lrQC48OtTNeSRF/l9P5YHa/Q3omk= -github.com/aos-dev/go-storage/v3 v3.4.3-0.20210417162535-67db0dd18784/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY= -github.com/aos-dev/specs/go v0.0.0-20210205073047-af8ef94af73d/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA= -github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b h1:qIehSnBbr31ATAckM9u9h6gSz+9PkGqu79vTngx6wPw= +github.com/aos-dev/go-integration-test/v3 v3.0.0 h1:rxIc7YBfiw9JS5JiVU4BZAqQqoKJemIftFaeJpYCb5M= +github.com/aos-dev/go-integration-test/v3 v3.0.0/go.mod h1:woC3E9Ld1G/Cpo2tSEQ+iwIr4MMdgf+L/8UuyJOE11Q= +github.com/aos-dev/go-storage/v3 v3.5.0/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY= +github.com/aos-dev/go-storage/v3 v3.5.1-0.20210422060150-bc0fae4c3fa7 h1:YirUP3+06blEpQ/o6VT0RXRf5ystxNB91iaYgO0SoWM= +github.com/aos-dev/go-storage/v3 v3.5.1-0.20210422060150-bc0fae4c3fa7/go.mod h1:JFshvl851ZDDXtFGWDFKqkg34QEPH0xuhLJ2LjikZYc= github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA= +github.com/aos-dev/specs/go v0.0.0-20210420062803-1a60efa2eae3 h1:LiW0Ki0Gw6opu11JwMhxWw5M+V6I9JypJ5eAIp+Rqt4= +github.com/aos-dev/specs/go v0.0.0-20210420062803-1a60efa2eae3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= @@ -22,7 +21,6 @@ github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -40,8 +38,9 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.0 h1:NOd0BRdOKpPf0SxkL3HxSQOG7rNh+4kl6PHcBPFs7Q0= +github.com/pelletier/go-toml v1.9.0/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= @@ -74,7 +73,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5 h1:MeC2gMlMdkd67dn17MEby3rGXRxZtWeiRXOnISfTQ74= golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/service.toml b/service.toml index 77d08ba..dc7fc5c 100644 --- a/service.toml +++ b/service.toml @@ -6,6 +6,9 @@ name = "oss" required = ["credential"] optional = ["default_service_pairs", "endpoint", "http_client_options"] +[namespace.storage] +implement = ["appender"] + [namespace.storage.new] required = ["name"] optional = ["default_storage_pairs", "pair_policy", "work_dir"] diff --git a/storage.go b/storage.go index 220c5ff..453a101 100644 --- a/storage.go +++ b/storage.go @@ -22,6 +22,35 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { return o } +func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { + rp := s.getAbsPath(path) + + var nextPos int64 = 0 + isExist, err := s.bucket.IsObjectExist(rp) + if err != nil { + return + } + + if isExist { + props, errGetMeta := s.bucket.GetObjectDetailedMeta(rp) + if errGetMeta != nil { + err = errGetMeta + return + } + nextPos, err = strconv.ParseInt(props.Get(oss.HTTPHeaderOssNextAppendPosition), 10, 64) + if err != nil { + return + } + } + + o = s.newObject(true) + o.Mode = ModeRead | ModeAppend + o.ID = rp + o.Path = path + o.SetAppendOffset(nextPos) + return o, nil +} + func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { rp := s.getAbsPath(path) @@ -216,3 +245,20 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 } return size, nil } + +func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) { + rp := o.GetID() + nextPos := o.MustGetAppendOffset() + + options := make([]oss.Option, 0) + options = append(options, oss.ContentLength(size)) + + nextPos, err = s.bucket.AppendObject(rp, r, nextPos, options...) + if err != nil { + return + } + + o.SetAppendOffset(nextPos) + + return nextPos, err +} diff --git a/tests/storage_test.go b/tests/storage_test.go index 2eb0faf..5404378 100644 --- a/tests/storage_test.go +++ b/tests/storage_test.go @@ -11,3 +11,7 @@ import ( func TestStorage(t *testing.T) { tests.TestStorager(t, setupTest(t)) } + +func TestAppend(t *testing.T) { + tests.TestAppender(t, setupTest(t)) +} diff --git a/utils.go b/utils.go index 6d4c8f0..9a9a79a 100644 --- a/utils.go +++ b/utils.go @@ -39,6 +39,7 @@ type Storage struct { pairPolicy typ.PairPolicy typ.UnimplementedStorager + typ.UnimplementedAppender } // String implements Storager.String From a95e40122452011d47d1550a19d62f9453452dce Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Fri, 23 Apr 2021 16:43:30 +0800 Subject: [PATCH 2/4] modify the behavior of createAppend and the test function --- Makefile | 2 +- storage.go | 34 +++++++++++++++------------------- tests/storage_test.go | 3 +++ 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index f14abf4..3e6da2b 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ test: go tool cover -html="coverage.txt" -o "coverage.html" integration_test: - go test -count=1 -race -covermode=atomic -v ./tests + STORAGE_OSS_INTEGRATION_TEST=on go test -count=1 -race -covermode=atomic -v ./tests tidy: go mod tidy diff --git a/storage.go b/storage.go index 5764574..a1e33c0 100644 --- a/storage.go +++ b/storage.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "strconv" + "strings" "time" "github.com/aliyun/aliyun-oss-go-sdk/oss" @@ -25,29 +26,19 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { rp := s.getAbsPath(path) - var nextPos int64 = 0 - isExist, err := s.bucket.IsObjectExist(rp) + options := make([]oss.Option, 0) + options = append(options, oss.ContentLength(0)) + + offset, err := s.bucket.AppendObject(rp, strings.NewReader(""), 0, options...) if err != nil { return } - if isExist { - props, errGetMeta := s.bucket.GetObjectDetailedMeta(rp) - if errGetMeta != nil { - err = errGetMeta - return - } - nextPos, err = strconv.ParseInt(props.Get(oss.HTTPHeaderOssNextAppendPosition), 10, 64) - if err != nil { - return - } - } - o = s.newObject(true) o.Mode = ModeRead | ModeAppend o.ID = rp o.Path = path - o.SetAppendOffset(nextPos) + o.SetAppendOffset(offset) return o, nil } @@ -263,17 +254,22 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) { rp := o.GetID() - nextPos := o.MustGetAppendOffset() + + offset, ok := o.GetAppendOffset() + if !ok { + err = fmt.Errorf("append offset is not set") + return + } options := make([]oss.Option, 0) options = append(options, oss.ContentLength(size)) - nextPos, err = s.bucket.AppendObject(rp, r, nextPos, options...) + offset, err = s.bucket.AppendObject(rp, r, offset, options...) if err != nil { return } - o.SetAppendOffset(nextPos) + o.SetAppendOffset(offset) - return nextPos, err + return offset, err } diff --git a/tests/storage_test.go b/tests/storage_test.go index e9414f1..16f4b00 100644 --- a/tests/storage_test.go +++ b/tests/storage_test.go @@ -15,5 +15,8 @@ func TestStorage(t *testing.T) { } func TestAppend(t *testing.T) { + if os.Getenv("STORAGE_OSS_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_OSS_INTEGRATION_TEST is not 'on', skipped") + } tests.TestAppender(t, setupTest(t)) } From 967ac3c097ab2bd794244a53bc9fce512bb891dc Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Fri, 23 Apr 2021 17:44:53 +0800 Subject: [PATCH 3/4] remove the AppendObject request in createAppend --- Makefile | 2 +- storage.go | 17 +++-------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 3e6da2b..f14abf4 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ test: go tool cover -html="coverage.txt" -o "coverage.html" integration_test: - STORAGE_OSS_INTEGRATION_TEST=on go test -count=1 -race -covermode=atomic -v ./tests + go test -count=1 -race -covermode=atomic -v ./tests tidy: go mod tidy diff --git a/storage.go b/storage.go index a1e33c0..43cc831 100644 --- a/storage.go +++ b/storage.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "strconv" - "strings" "time" "github.com/aliyun/aliyun-oss-go-sdk/oss" @@ -24,21 +23,11 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { } func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { - rp := s.getAbsPath(path) - - options := make([]oss.Option, 0) - options = append(options, oss.ContentLength(0)) - - offset, err := s.bucket.AppendObject(rp, strings.NewReader(""), 0, options...) - if err != nil { - return - } - - o = s.newObject(true) + o = s.newObject(false) o.Mode = ModeRead | ModeAppend - o.ID = rp + o.ID = s.getAbsPath(path) o.Path = path - o.SetAppendOffset(offset) + o.SetAppendOffset(0) return o, nil } From 3671f21ecc6dcc7b7e7535a3c445a761a11c7bbe Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Sun, 25 Apr 2021 10:45:16 +0800 Subject: [PATCH 4/4] modify newObject stat parameter and add SSE support for append --- generated.go | 47 +++++++++++++++++++++++++++++++++++++++++------ go.mod | 2 +- go.sum | 11 ++++++----- service.toml | 3 +++ storage.go | 19 ++++++++++++++----- utils.go | 4 ++-- 6 files changed, 67 insertions(+), 19 deletions(-) diff --git a/generated.go b/generated.go index 24ab805..1252765 100644 --- a/generated.go +++ b/generated.go @@ -45,14 +45,34 @@ const ( pairStorageClass = "oss_storage_class" ) -// Service available metadata. -const ( - MetadataServerSideEncryption = "oss-server_side_encryption" +// ObjectMetadata stores service metadata for object. +type ObjectMetadata struct { + // ServerSideEncryption + ServerSideEncryption string + // ServerSideEncryptionKeyID + ServerSideEncryptionKeyID string + // StorageClass + StorageClass string +} - MetadataServerSideEncryptionKeyID = "oss-server_side_encryption_key_id" +// GetObjectMetadata will get ObjectMetadata from Object. +// +// - This function should not be called by service implementer. +// - The returning ObjectMetadata is read only and should not be modified. +func GetObjectMetadata(o *Object) ObjectMetadata { + om, ok := o.GetServiceMetadata() + if ok { + return om.(ObjectMetadata) + } + return ObjectMetadata{} +} - MetadataStorageClass = "oss-storage-class" -) +// setObjectMetadata will set ObjectMetadata into Object. +// +// - This function should only be called once, please make sure all data has been written before set. +func setObjectMetadata(o *Object, om ObjectMetadata) { + o.SetServiceMetadata(om) +} // WithDefaultServicePairs will apply default_service_pairs value to Options // DefaultServicePairs set default pairs for service actions @@ -797,6 +817,12 @@ type pairStorageWriteAppend struct { // Required pairs // Optional pairs + HasServerSideDataEncryption bool + ServerSideDataEncryption string + HasServerSideEncryption bool + ServerSideEncryption string + HasServerSideEncryptionKeyID bool + ServerSideEncryptionKeyID string // Generated pairs } @@ -810,6 +836,15 @@ func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppe switch v.Key { // Required pairs // Optional pairs + case pairServerSideDataEncryption: + result.HasServerSideDataEncryption = true + result.ServerSideDataEncryption = v.Value.(string) + case pairServerSideEncryption: + result.HasServerSideEncryption = true + result.ServerSideEncryption = v.Value.(string) + case pairServerSideEncryptionKeyID: + result.HasServerSideEncryptionKeyID = true + result.ServerSideEncryptionKeyID = v.Value.(string) // Generated pairs default: diff --git a/go.mod b/go.mod index cfea870..5e3e176 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible github.com/aos-dev/go-integration-test/v3 v3.0.0 - github.com/aos-dev/go-storage/v3 v3.5.1-0.20210422060150-bc0fae4c3fa7 + github.com/aos-dev/go-storage/v3 v3.6.0 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/google/uuid v1.2.0 github.com/satori/go.uuid v1.2.0 // indirect diff --git a/go.sum b/go.sum index 7aa76e2..cca0cb7 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,16 @@ -github.com/Xuanwo/templateutils v0.0.0-20201216100309-46f73cd4e4b1 h1:SQ8kQujQ0yl39YBMU0xZ+yyfxx/qSWfGWlCPNzHp5kw= github.com/Xuanwo/templateutils v0.0.0-20201216100309-46f73cd4e4b1/go.mod h1:x0qS7gfgEm24b8V9U+0zBEBAu/VpxZhO/+EWtpgEgDw= +github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= +github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible h1:hLUNPbx10wawWW7DeNExvTrlb90db3UnnNTFKHZEFhE= github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aos-dev/go-integration-test/v3 v3.0.0 h1:rxIc7YBfiw9JS5JiVU4BZAqQqoKJemIftFaeJpYCb5M= github.com/aos-dev/go-integration-test/v3 v3.0.0/go.mod h1:woC3E9Ld1G/Cpo2tSEQ+iwIr4MMdgf+L/8UuyJOE11Q= github.com/aos-dev/go-storage/v3 v3.5.0/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY= -github.com/aos-dev/go-storage/v3 v3.5.1-0.20210422060150-bc0fae4c3fa7 h1:YirUP3+06blEpQ/o6VT0RXRf5ystxNB91iaYgO0SoWM= -github.com/aos-dev/go-storage/v3 v3.5.1-0.20210422060150-bc0fae4c3fa7/go.mod h1:JFshvl851ZDDXtFGWDFKqkg34QEPH0xuhLJ2LjikZYc= +github.com/aos-dev/go-storage/v3 v3.6.0 h1:ywjMvh320+esJH81MqB9nyuMNLW97Krujz2UiprC2ZM= +github.com/aos-dev/go-storage/v3 v3.6.0/go.mod h1:ZQwybmoCcTWUOWg+G15gT/NQJoI8G8KH1pF41TuJqYk= github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA= -github.com/aos-dev/specs/go v0.0.0-20210420062803-1a60efa2eae3 h1:LiW0Ki0Gw6opu11JwMhxWw5M+V6I9JypJ5eAIp+Rqt4= -github.com/aos-dev/specs/go v0.0.0-20210420062803-1a60efa2eae3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= +github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3 h1:e65ozDhdfHfhnDpZF9SLcY5mwtAg/sAvNIUAkFd+4D0= +github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= diff --git a/service.toml b/service.toml index 83a05e5..3390b60 100644 --- a/service.toml +++ b/service.toml @@ -22,6 +22,9 @@ optional = ["offset", "io_callback", "size"] [namespace.storage.op.write] optional = ["content_md5", "content_type", "io_callback", "storage_class", "server_side_encryption", "server_side_data_encryption", "server_side_encryption_key_id"] +[namespace.storage.op.write_append] +optional = ["server_side_encryption", "server_side_data_encryption", "server_side_encryption_key_id"] + [pairs.default_service_pairs] type = "DefaultServicePairs" description = "set default pairs for service actions" diff --git a/storage.go b/storage.go index 43cc831..e9991fe 100644 --- a/storage.go +++ b/storage.go @@ -23,7 +23,7 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { } func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { - o = s.newObject(false) + o = s.newObject(true) o.Mode = ModeRead | ModeAppend o.ID = s.getAbsPath(path) o.Path = path @@ -194,15 +194,15 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o o.SetContentType(v) } - sm := make(map[string]string) + var sm ObjectMetadata if v := output.Get(storageClassHeader); v != "" { - sm[MetadataStorageClass] = v + sm.StorageClass = v } if v := output.Get(serverSideEncryptionHeader); v != "" { - sm[MetadataServerSideEncryption] = v + sm.ServerSideEncryption = v } if v := output.Get(serverSideEncryptionKeyIdHeader); v != "" { - sm[MetadataServerSideEncryptionKeyID] = v + sm.ServerSideEncryptionKeyID = v } o.SetServiceMetadata(sm) @@ -252,6 +252,15 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size options := make([]oss.Option, 0) options = append(options, oss.ContentLength(size)) + if opt.HasServerSideEncryption { + options = append(options, oss.ServerSideEncryption(opt.ServerSideEncryption)) + } + if opt.HasServerSideDataEncryption { + options = append(options, oss.ServerSideDataEncryption(opt.ServerSideDataEncryption)) + } + if opt.HasServerSideEncryptionKeyID { + options = append(options, oss.ServerSideEncryptionKeyID(opt.ServerSideEncryptionKeyID)) + } offset, err = s.bucket.AppendObject(rp, r, offset, options...) if err != nil { diff --git a/utils.go b/utils.go index dfd2e2c..4c8fca9 100644 --- a/utils.go +++ b/utils.go @@ -255,9 +255,9 @@ func (s *Storage) formatFileObject(v oss.ObjectProperties) (o *typ.Object, err e o.SetEtag(v.ETag) } - sm := make(map[string]string) + var sm ObjectMetadata if value := v.Type; value != "" { - sm[MetadataStorageClass] = value + sm.StorageClass = value } o.SetServiceMetadata(sm)