From a537bd1f7e49f8c3b29d651a645aeef2f4afcbdb Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Fri, 23 Apr 2021 18:20:08 +0800 Subject: [PATCH 1/6] Add appender support --- generated.go | 116 ++++++++++++++++++++++++++++++++++++++++++ go.mod | 4 +- go.sum | 12 ++--- service.toml | 2 +- storage.go | 34 +++++++++++++ tests/storage_test.go | 7 +++ utils.go | 1 + 7 files changed, 166 insertions(+), 10 deletions(-) diff --git a/generated.go b/generated.go index fb2ef2b..ac89af0 100644 --- a/generated.go +++ b/generated.go @@ -523,6 +523,7 @@ type DefaultStoragePairs struct { CompleteMultipart []Pair Copy []Pair Create []Pair + CreateAppend []Pair CreateMultipart []Pair Delete []Pair Fetch []Pair @@ -534,6 +535,7 @@ type DefaultStoragePairs struct { Read []Pair Stat []Pair Write []Pair + WriteAppend []Pair WriteMultipart []Pair } @@ -658,6 +660,38 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) return result, nil } +// pairStorageCreateAppend is the parsed struct +type pairStorageCreateAppend struct { + pairs []Pair + + // Required pairs + // Optional pairs + // Generated pairs +} + +// parsePairStorageCreateAppend will parse Pair slice into *pairStorageCreateAppend +func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAppend, error) { + result := pairStorageCreateAppend{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + // Required pairs + // Optional pairs + // Generated pairs + default: + + if s.pairPolicy.All || s.pairPolicy.CreateAppend { + return pairStorageCreateAppend{}, services.NewPairUnsupportedError(v) + } + + } + } + + return result, nil +} + // pairStorageCreateMultipart is the parsed struct type pairStorageCreateMultipart struct { pairs []Pair @@ -1098,6 +1132,38 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { return result, nil } +// pairStorageWriteAppend is the parsed struct +type pairStorageWriteAppend struct { + pairs []Pair + + // Required pairs + // Optional pairs + // Generated pairs +} + +// parsePairStorageWriteAppend will parse Pair slice into *pairStorageWriteAppend +func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppend, error) { + result := pairStorageWriteAppend{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + // Required pairs + // Optional pairs + // Generated pairs + default: + + if s.pairPolicy.All || s.pairPolicy.WriteAppend { + return pairStorageWriteAppend{}, services.NewPairUnsupportedError(v) + } + + } + } + + return result, nil +} + // pairStorageWriteMultipart is the parsed struct type pairStorageWriteMultipart struct { pairs []Pair @@ -1203,6 +1269,31 @@ func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { return s.create(path, opt) } +// CreateAppend will create an append object. +// +// This function will create a context by default. +func (s *Storage) CreateAppend(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateAppendWithContext(ctx, path, pairs...) +} + +// CreateAppendWithContext will create an append object. +func (s *Storage) CreateAppendWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + pairs = append(pairs, s.defaultPairs.CreateAppend...) + var opt pairStorageCreateAppend + + defer func() { + err = s.formatError("create_append", err, path) + }() + + opt, err = s.parsePairStorageCreateAppend(pairs) + if err != nil { + return + } + + return s.createAppend(ctx, path, opt) +} + // CreateMultipart will create a new multipart. // // This function will create a context by default. @@ -1478,6 +1569,31 @@ func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader return s.write(ctx, path, r, size, opt) } +// WriteAppend will append content to an append object. +// +// This function will create a context by default. +func (s *Storage) WriteAppend(o *Object, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.WriteAppendWithContext(ctx, o, r, size, pairs...) +} + +// WriteAppendWithContext will append content to an append object. +func (s *Storage) WriteAppendWithContext(ctx context.Context, o *Object, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + pairs = append(pairs, s.defaultPairs.WriteAppend...) + var opt pairStorageWriteAppend + + defer func() { + err = s.formatError("write_append", err) + }() + + opt, err = s.parsePairStorageWriteAppend(pairs) + if err != nil { + return + } + + return s.writeAppend(ctx, o, r, size, opt) +} + // WriteMultipart will write content to a multipart. // // This function will create a context by default. diff --git a/go.mod b/go.mod index e0d7b27..5003e74 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.14 require ( bou.ke/monkey v1.0.2 - github.com/aos-dev/go-integration-test/v3 v3.0.0-20210420062414-e972ad8801cc - github.com/aos-dev/go-storage/v3 v3.5.1-0.20210421092104-4de248aed1f1 + github.com/aos-dev/go-integration-test/v3 v3.0.0 + github.com/aos-dev/go-storage/v3 v3.5.1-0.20210423013030-4d27effb22ca github.com/golang/mock v1.5.0 github.com/google/uuid v1.2.0 github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 diff --git a/go.sum b/go.sum index 7fdd9e9..d9c63e0 100644 --- a/go.sum +++ b/go.sum @@ -4,12 +4,11 @@ github.com/Xuanwo/go-bufferpool v0.0.0-20200622083641-bc954721ce54 h1:wA7f87ODtF github.com/Xuanwo/go-bufferpool v0.0.0-20200622083641-bc954721ce54/go.mod h1:Mle++9GGouhOwGj52i9PJLNAPmW2nb8PWBP7JJzNCzk= github.com/Xuanwo/templateutils v0.0.0-20201216100309-46f73cd4e4b1 h1:SQ8kQujQ0yl39YBMU0xZ+yyfxx/qSWfGWlCPNzHp5kw= github.com/Xuanwo/templateutils v0.0.0-20201216100309-46f73cd4e4b1/go.mod h1:x0qS7gfgEm24b8V9U+0zBEBAu/VpxZhO/+EWtpgEgDw= -github.com/aos-dev/go-integration-test/v3 v3.0.0-20210420062414-e972ad8801cc h1:ha0GpuSyD/R19if+4D6QCKF6j7UvajiL2LF17xpqKTs= -github.com/aos-dev/go-integration-test/v3 v3.0.0-20210420062414-e972ad8801cc/go.mod h1:2i6Itxp4ScMeZbaJetPPo+b283GYExF9i9Dlcr8E9Y0= -github.com/aos-dev/go-storage/v3 v3.4.2/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY= -github.com/aos-dev/go-storage/v3 v3.5.1-0.20210421092104-4de248aed1f1 h1:Kkb/cVJRwc77lXGcnV2ThUC4QUqybiEVrFwq8qnM4V4= -github.com/aos-dev/go-storage/v3 v3.5.1-0.20210421092104-4de248aed1f1/go.mod h1:JFshvl851ZDDXtFGWDFKqkg34QEPH0xuhLJ2LjikZYc= -github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b h1:qIehSnBbr31ATAckM9u9h6gSz+9PkGqu79vTngx6wPw= +github.com/aos-dev/go-integration-test/v3 v3.0.0 h1:rxIc7YBfiw9JS5JiVU4BZAqQqoKJemIftFaeJpYCb5M= +github.com/aos-dev/go-integration-test/v3 v3.0.0/go.mod h1:woC3E9Ld1G/Cpo2tSEQ+iwIr4MMdgf+L/8UuyJOE11Q= +github.com/aos-dev/go-storage/v3 v3.5.0/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY= +github.com/aos-dev/go-storage/v3 v3.5.1-0.20210423013030-4d27effb22ca h1:H7wkWwIe/7Kz1+uAFBhxDYh8GX1kcyxsUIBxPE12ksk= +github.com/aos-dev/go-storage/v3 v3.5.1-0.20210423013030-4d27effb22ca/go.mod h1:JFshvl851ZDDXtFGWDFKqkg34QEPH0xuhLJ2LjikZYc= github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA= github.com/aos-dev/specs/go v0.0.0-20210420062803-1a60efa2eae3 h1:LiW0Ki0Gw6opu11JwMhxWw5M+V6I9JypJ5eAIp+Rqt4= github.com/aos-dev/specs/go v0.0.0-20210420062803-1a60efa2eae3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= @@ -39,7 +38,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.0 h1:NOd0BRdOKpPf0SxkL3HxSQOG7rNh+4kl6PHcBPFs7Q0= github.com/pelletier/go-toml v1.9.0/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= diff --git a/service.toml b/service.toml index 915f33e..2a16c66 100644 --- a/service.toml +++ b/service.toml @@ -17,7 +17,7 @@ optional = ["location"] optional = ["location"] [namespace.storage] -implement = ["copier", "fetcher", "mover", "multiparter", "reacher"] +implement = ["copier", "fetcher", "mover", "multiparter", "reacher", "appender"] [namespace.storage.new] required = ["name"] diff --git a/storage.go b/storage.go index dd5920d..1621021 100644 --- a/storage.go +++ b/storage.go @@ -79,6 +79,15 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { return o } +func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { + o = s.newObject(false) + o.Mode = ModeRead | ModeAppend + o.ID = s.getAbsPath(path) + o.Path = path + o.SetAppendOffset(0) + return o, nil +} + func (s *Storage) createMultipart(ctx context.Context, path string, opt pairStorageCreateMultipart) (o *Object, err error) { input := &service.InitiateMultipartUploadInput{} if opt.HasEncryptionCustomerAlgorithm { @@ -484,6 +493,31 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 return size, nil } +func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) { + rp := o.GetID() + + offset, ok := o.GetAppendOffset() + if !ok { + err = fmt.Errorf("append offset is not set") + return + } + + output, err := s.bucket.AppendObjectWithContext(ctx, rp, &service.AppendObjectInput{ + Position: &offset, + ContentLength: &size, + Body: io.LimitReader(r, size), + }) + if err != nil { + return + } + + offset = *output.XQSNextAppendPosition + o.SetAppendOffset(offset) + + return offset, nil + +} + func (s *Storage) writeMultipart(ctx context.Context, o *Object, r io.Reader, size int64, index int, opt pairStorageWriteMultipart) (n int64, err error) { if o.Mode&ModePart == 0 { return 0, fmt.Errorf("object is not a part object") diff --git a/tests/storage_test.go b/tests/storage_test.go index 95aa4da..168d118 100644 --- a/tests/storage_test.go +++ b/tests/storage_test.go @@ -20,3 +20,10 @@ func TestMultiparter(t *testing.T) { } tests.TestMultiparter(t, setupTest(t)) } + +func TestAppend(t *testing.T) { + if os.Getenv("STORAGE_QINGSTOR_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_QINGSTOR_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestAppender(t, setupTest(t)) +} diff --git a/utils.go b/utils.go index d2ad6bb..bb32c6f 100644 --- a/utils.go +++ b/utils.go @@ -64,6 +64,7 @@ type Storage struct { typ.UnimplementedMover typ.UnimplementedMultiparter typ.UnimplementedReacher + typ.UnimplementedAppender } // String implements Storager.String From 1b20f447712a507d2f3b3cc452a358576748ed1e Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Sun, 25 Apr 2021 11:41:44 +0800 Subject: [PATCH 2/6] modify newObject stat parameter and add optional parameters for append --- generated.go | 45 ++++++++++++++++++++++++++++++++++++++++----- go.mod | 2 +- go.sum | 11 ++++++----- service.toml | 5 ++++- storage.go | 23 +++++++++++++++++------ utils.go | 4 ++-- 6 files changed, 70 insertions(+), 20 deletions(-) diff --git a/generated.go b/generated.go index ac89af0..31bc765 100644 --- a/generated.go +++ b/generated.go @@ -41,12 +41,32 @@ const ( pairStorageClass = "qingstor_storage_class" ) -// Service available metadata. -const ( - MetadataEncryptionCustomerAlgorithm = "qingstor-encryption_customer_algorithm" +// ObjectMetadata stores service metadata for object. +type ObjectMetadata struct { + // EncryptionCustomerAlgorithm + EncryptionCustomerAlgorithm string + // StorageClass + StorageClass string +} - MetadataStorageClass = "qingstor-storage-class" -) +// GetObjectMetadata will get ObjectMetadata from Object. +// +// - This function should not be called by service implementer. +// - The returning ObjectMetadata is read only and should not be modified. +func GetObjectMetadata(o *Object) ObjectMetadata { + om, ok := o.GetServiceMetadata() + if ok { + return om.(ObjectMetadata) + } + return ObjectMetadata{} +} + +// setObjectMetadata will set ObjectMetadata into Object. +// +// - This function should only be called once, please make sure all data has been written before set. +func setObjectMetadata(o *Object, om ObjectMetadata) { + o.SetServiceMetadata(om) +} // WithCopySourceEncryptionCustomerAlgorithm will apply copy_source_encryption_customer_algorithm value to Options // CopySourceEncryptionCustomerAlgorithm is the encryption algorithm for the source object. Only AES256 is supported now. @@ -1138,6 +1158,12 @@ type pairStorageWriteAppend struct { // Required pairs // Optional pairs + HasContentMd5 bool + ContentMd5 string + HasContentType bool + ContentType string + HasStorageClass bool + StorageClass string // Generated pairs } @@ -1151,6 +1177,15 @@ func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppe switch v.Key { // Required pairs // Optional pairs + case "content_md5": + result.HasContentMd5 = true + result.ContentMd5 = v.Value.(string) + case "content_type": + result.HasContentType = true + result.ContentType = v.Value.(string) + case pairStorageClass: + result.HasStorageClass = true + result.StorageClass = v.Value.(string) // Generated pairs default: diff --git a/go.mod b/go.mod index 5003e74..b220793 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( bou.ke/monkey v1.0.2 github.com/aos-dev/go-integration-test/v3 v3.0.0 - github.com/aos-dev/go-storage/v3 v3.5.1-0.20210423013030-4d27effb22ca + github.com/aos-dev/go-storage/v3 v3.6.0 github.com/golang/mock v1.5.0 github.com/google/uuid v1.2.0 github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 diff --git a/go.sum b/go.sum index d9c63e0..f0bc8fc 100644 --- a/go.sum +++ b/go.sum @@ -2,16 +2,17 @@ bou.ke/monkey v1.0.2 h1:kWcnsrCNUatbxncxR/ThdYqbytgOIArtYWqcQLQzKLI= bou.ke/monkey v1.0.2/go.mod h1:OqickVX3tNx6t33n1xvtTtu85YN5s6cKwVug+oHMaIA= github.com/Xuanwo/go-bufferpool v0.0.0-20200622083641-bc954721ce54 h1:wA7f87ODtF2oA7FCmeGHrPvyG+vVDxSP9ZRsltXB2Gc= github.com/Xuanwo/go-bufferpool v0.0.0-20200622083641-bc954721ce54/go.mod h1:Mle++9GGouhOwGj52i9PJLNAPmW2nb8PWBP7JJzNCzk= -github.com/Xuanwo/templateutils v0.0.0-20201216100309-46f73cd4e4b1 h1:SQ8kQujQ0yl39YBMU0xZ+yyfxx/qSWfGWlCPNzHp5kw= github.com/Xuanwo/templateutils v0.0.0-20201216100309-46f73cd4e4b1/go.mod h1:x0qS7gfgEm24b8V9U+0zBEBAu/VpxZhO/+EWtpgEgDw= +github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= +github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= github.com/aos-dev/go-integration-test/v3 v3.0.0 h1:rxIc7YBfiw9JS5JiVU4BZAqQqoKJemIftFaeJpYCb5M= github.com/aos-dev/go-integration-test/v3 v3.0.0/go.mod h1:woC3E9Ld1G/Cpo2tSEQ+iwIr4MMdgf+L/8UuyJOE11Q= github.com/aos-dev/go-storage/v3 v3.5.0/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY= -github.com/aos-dev/go-storage/v3 v3.5.1-0.20210423013030-4d27effb22ca h1:H7wkWwIe/7Kz1+uAFBhxDYh8GX1kcyxsUIBxPE12ksk= -github.com/aos-dev/go-storage/v3 v3.5.1-0.20210423013030-4d27effb22ca/go.mod h1:JFshvl851ZDDXtFGWDFKqkg34QEPH0xuhLJ2LjikZYc= +github.com/aos-dev/go-storage/v3 v3.6.0 h1:ywjMvh320+esJH81MqB9nyuMNLW97Krujz2UiprC2ZM= +github.com/aos-dev/go-storage/v3 v3.6.0/go.mod h1:ZQwybmoCcTWUOWg+G15gT/NQJoI8G8KH1pF41TuJqYk= github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA= -github.com/aos-dev/specs/go v0.0.0-20210420062803-1a60efa2eae3 h1:LiW0Ki0Gw6opu11JwMhxWw5M+V6I9JypJ5eAIp+Rqt4= -github.com/aos-dev/specs/go v0.0.0-20210420062803-1a60efa2eae3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= +github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3 h1:e65ozDhdfHfhnDpZF9SLcY5mwtAg/sAvNIUAkFd+4D0= +github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= diff --git a/service.toml b/service.toml index 2a16c66..c96275c 100644 --- a/service.toml +++ b/service.toml @@ -17,7 +17,7 @@ optional = ["location"] optional = ["location"] [namespace.storage] -implement = ["copier", "fetcher", "mover", "multiparter", "reacher", "appender"] +implement = ["appender", "copier", "fetcher", "mover", "multiparter", "reacher"] [namespace.storage.new] required = ["name"] @@ -44,6 +44,9 @@ optional = ["offset", "io_callback", "size", "encryption_customer_algorithm", "e [namespace.storage.op.write] optional = ["content_md5", "content_type", "io_callback", "storage_class", "encryption_customer_algorithm", "encryption_customer_key"] +[namespace.storage.op.write_append] +optional = ["content_md5", "content_type", "storage_class"] + [namespace.storage.op.copy] optional = ["encryption_customer_algorithm", "encryption_customer_key", "copy_source_encryption_customer_algorithm", "copy_source_encryption_customer_key"] diff --git a/storage.go b/storage.go index 1621021..e2f2825 100644 --- a/storage.go +++ b/storage.go @@ -80,7 +80,7 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { } func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { - o = s.newObject(false) + o = s.newObject(true) o.Mode = ModeRead | ModeAppend o.ID = s.getAbsPath(path) o.Path = path @@ -450,12 +450,12 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o o.SetEtag(service.StringValue(output.ETag)) } - sm := make(map[string]string) + var sm ObjectMetadata if v := service.StringValue(output.XQSStorageClass); v != "" { - sm[MetadataStorageClass] = v + sm.StorageClass = v } if v := service.StringValue(output.XQSEncryptionCustomerAlgorithm); v != "" { - sm[MetadataEncryptionCustomerAlgorithm] = v + sm.EncryptionCustomerAlgorithm = v } o.SetServiceMetadata(sm) @@ -502,11 +502,22 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size return } - output, err := s.bucket.AppendObjectWithContext(ctx, rp, &service.AppendObjectInput{ + input := &service.AppendObjectInput{ Position: &offset, ContentLength: &size, Body: io.LimitReader(r, size), - }) + } + if opt.HasContentMd5 { + input.ContentMD5 = &opt.ContentMd5 + } + if opt.HasContentType { + input.ContentType = &opt.ContentType + } + if opt.HasStorageClass { + input.XQSStorageClass = &opt.StorageClass + } + + output, err := s.bucket.AppendObjectWithContext(ctx, rp, input) if err != nil { return } diff --git a/utils.go b/utils.go index bb32c6f..36971ed 100644 --- a/utils.go +++ b/utils.go @@ -373,9 +373,9 @@ func (s *Storage) formatFileObject(v *service.KeyType) (o *typ.Object, err error o.SetEtag(service.StringValue(v.Etag)) } - sm := make(map[string]string) + var sm ObjectMetadata if value := service.StringValue(v.StorageClass); value != "" { - sm[MetadataStorageClass] = value + sm.StorageClass = value } o.SetServiceMetadata(sm) From 39c09bdcdc837e8a54dcb510b164a50a249d35cd Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Sun, 25 Apr 2021 14:04:00 +0800 Subject: [PATCH 3/6] add the XQSNextAppendPosition pointer nil check --- storage.go | 10 +++++++--- storager_test.go | 5 +++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/storage.go b/storage.go index e2f2825..dbeeb32 100644 --- a/storage.go +++ b/storage.go @@ -522,11 +522,15 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size return } - offset = *output.XQSNextAppendPosition - o.SetAppendOffset(offset) + if output.XQSNextAppendPosition != nil { + offset = *output.XQSNextAppendPosition + o.SetAppendOffset(offset) + } else { + err = fmt.Errorf("next append position is empty") + return + } return offset, nil - } func (s *Storage) writeMultipart(ctx context.Context, o *Object, r io.Reader, size int64, index int, opt pairStorageWriteMultipart) (n int64, err error) { diff --git a/storager_test.go b/storager_test.go index 95ab457..88ef082 100644 --- a/storager_test.go +++ b/storager_test.go @@ -315,9 +315,10 @@ func TestStorage_Stat(t *testing.T) { checkSum, ok := o.GetEtag() assert.True(t, ok) assert.Equal(t, "test_etag", checkSum) - storageClass, ok := o.MustGetServiceMetadata()[MetadataStorageClass] + sm := o.MustGetServiceMetadata() + serviceMetadata, ok := sm.(ObjectMetadata) assert.True(t, ok) - assert.Equal(t, StorageClassStandard, storageClass) + assert.Equal(t, StorageClassStandard, serviceMetadata.StorageClass) } } } From bad705ead264890f13bd38028f66e13b3f276a37 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Sun, 25 Apr 2021 14:32:51 +0800 Subject: [PATCH 4/6] change the way to get storageClass --- storager_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/storager_test.go b/storager_test.go index 88ef082..0823867 100644 --- a/storager_test.go +++ b/storager_test.go @@ -315,10 +315,8 @@ func TestStorage_Stat(t *testing.T) { checkSum, ok := o.GetEtag() assert.True(t, ok) assert.Equal(t, "test_etag", checkSum) - sm := o.MustGetServiceMetadata() - serviceMetadata, ok := sm.(ObjectMetadata) - assert.True(t, ok) - assert.Equal(t, StorageClassStandard, serviceMetadata.StorageClass) + storageClass := GetObjectMetadata(o).StorageClass + assert.Equal(t, StorageClassStandard, storageClass) } } } From a6662784e8445fc2ffe314dab008ed066094a73c Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Wed, 28 Apr 2021 15:52:28 +0800 Subject: [PATCH 5/6] modify append object metadata setting and add isAppend check --- generated.go | 24 ++++++++++++------------ service.toml | 5 ++++- storage.go | 45 +++++++++++++++++++++++++++++++++------------ 3 files changed, 49 insertions(+), 25 deletions(-) diff --git a/generated.go b/generated.go index d813dad..227e275 100644 --- a/generated.go +++ b/generated.go @@ -727,6 +727,10 @@ type pairStorageCreateAppend struct { // Required pairs // Optional pairs + HasContentType bool + ContentType string + HasStorageClass bool + StorageClass string // Generated pairs } @@ -740,6 +744,12 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp switch v.Key { // Required pairs // Optional pairs + case "content_type": + result.HasContentType = true + result.ContentType = v.Value.(string) + case pairStorageClass: + result.HasStorageClass = true + result.StorageClass = v.Value.(string) // Generated pairs default: @@ -1199,12 +1209,8 @@ type pairStorageWriteAppend struct { // Required pairs // Optional pairs - HasContentMd5 bool - ContentMd5 string - HasContentType bool - ContentType string - HasStorageClass bool - StorageClass string + HasContentMd5 bool + ContentMd5 string // Generated pairs } @@ -1221,12 +1227,6 @@ func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppe case "content_md5": result.HasContentMd5 = true result.ContentMd5 = v.Value.(string) - case "content_type": - result.HasContentType = true - result.ContentType = v.Value.(string) - case pairStorageClass: - result.HasStorageClass = true - result.StorageClass = v.Value.(string) // Generated pairs default: diff --git a/service.toml b/service.toml index c96275c..e040227 100644 --- a/service.toml +++ b/service.toml @@ -44,8 +44,11 @@ optional = ["offset", "io_callback", "size", "encryption_customer_algorithm", "e [namespace.storage.op.write] optional = ["content_md5", "content_type", "io_callback", "storage_class", "encryption_customer_algorithm", "encryption_customer_key"] +[namespace.storage.op.create_append] +optional = ["content_type", "storage_class"] + [namespace.storage.op.write_append] -optional = ["content_md5", "content_type", "storage_class"] +optional = ["content_md5"] [namespace.storage.op.copy] optional = ["encryption_customer_algorithm", "encryption_customer_key", "copy_source_encryption_customer_algorithm", "copy_source_encryption_customer_key"] diff --git a/storage.go b/storage.go index f5b604f..395db3f 100644 --- a/storage.go +++ b/storage.go @@ -84,11 +84,34 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { } func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { + rp := s.getAbsPath(path) + + input := &service.AppendObjectInput{} + if opt.HasContentType { + input.ContentType = &opt.ContentType + } + if opt.HasStorageClass { + input.XQSStorageClass = &opt.StorageClass + } + + output, err := s.bucket.AppendObjectWithContext(ctx, rp, input) + if err != nil { + return + } + + var offset int64 = 0 + if output == nil || output.XQSNextAppendPosition == nil { + err = fmt.Errorf("next append position is empty") + return + } else { + offset = *output.XQSNextAppendPosition + } + o = s.newObject(true) o.Mode = ModeRead | ModeAppend - o.ID = s.getAbsPath(path) + o.ID = rp o.Path = path - o.SetAppendOffset(0) + o.SetAppendOffset(offset) return o, nil } @@ -498,6 +521,11 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 } func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) { + if !o.Mode.IsAppend() { + err = fmt.Errorf("object not appendable") + return + } + rp := o.GetID() offset, ok := o.GetAppendOffset() @@ -514,24 +542,17 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size if opt.HasContentMd5 { input.ContentMD5 = &opt.ContentMd5 } - if opt.HasContentType { - input.ContentType = &opt.ContentType - } - if opt.HasStorageClass { - input.XQSStorageClass = &opt.StorageClass - } output, err := s.bucket.AppendObjectWithContext(ctx, rp, input) if err != nil { return } - if output.XQSNextAppendPosition != nil { - offset = *output.XQSNextAppendPosition - o.SetAppendOffset(offset) - } else { + if output == nil || output.XQSNextAppendPosition == nil { err = fmt.Errorf("next append position is empty") return + } else { + offset = *output.XQSNextAppendPosition } return offset, nil From b2fa99507eb730da8ebeee2ea685f794300f9d0b Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Wed, 28 Apr 2021 18:58:16 +0800 Subject: [PATCH 6/6] set append position --- storage.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/storage.go b/storage.go index 395db3f..4651773 100644 --- a/storage.go +++ b/storage.go @@ -86,7 +86,10 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { rp := s.getAbsPath(path) - input := &service.AppendObjectInput{} + var offset int64 = 0 + input := &service.AppendObjectInput{ + Position: &offset, + } if opt.HasContentType { input.ContentType = &opt.ContentType } @@ -99,7 +102,6 @@ func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorage return } - var offset int64 = 0 if output == nil || output.XQSNextAppendPosition == nil { err = fmt.Errorf("next append position is empty") return